Import Cobalt 25.master.0.1033734
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4817f40..24f5adc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS
@@ -11,10 +11,10 @@ /tools/metrics/ @joeltine # Temporary monitoring for merges to Chromium upstreams -/base/ @andrewsavage1 -/build/ @andrewsavage1 -/crypto/ @andrewsavage1 -/net/ @andrewsavage1 -/third_party/abseil-cpp/ @andrewsavage1 -/third_party/modp_b64/ @andrewsavage1 -/url/ @andrewsavage1 +/base/ @andrewsavage1 @youtube/cobalt-3p-repository-owners +/build/ @andrewsavage1 @youtube/cobalt-3p-repository-owners +/crypto/ @andrewsavage1 @youtube/cobalt-3p-repository-owners +/net/ @andrewsavage1 @youtube/cobalt-3p-repository-owners +/third_party/abseil-cpp/ @andrewsavage1 @youtube/cobalt-3p-repository-owners +/third_party/modp_b64/ @andrewsavage1 @youtube/cobalt-3p-repository-owners +/url/ @andrewsavage1 @youtube/cobalt-3p-repository-owners
diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index 3b53e8c..3f2fbda 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml
@@ -43,12 +43,22 @@ # We need to set docker tag properly for pull requests. In those scenarios where no docker related files # were changed we need to use an existing image (e.g. main). In cases where docker image is rebuilt we have # to use tag generated by the image build. + - name: Retrieve Docker metadata for PR + id: pr-meta + uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v4.3.0 + with: + images: ${{env.REGISTRY}}/${{github.repository}}/${{inputs.docker_image}} + tags: type=raw,value=${{ github.base_ref }} - name: Set Docker Tag id: set-docker-tag-presubmit-non-fork env: REPO: ${{ github.repository }} if: ${{ (steps.changed-files.outputs.any_changed == 'false') && (github.event_name == 'pull_request') }} - run: echo "DOCKER_TAG=ghcr.io/${REPO}/${{inputs.docker_image}}:${GITHUB_BASE_REF%.1+}" >> $GITHUB_ENV + run: | + set -x + docker_tag="${{ steps.pr-meta.outputs.tags }}" + docker_tag="${docker_tag%.1[+,-]}" + echo "DOCKER_TAG=${docker_tag}" >> $GITHUB_ENV shell: bash - name: Set up Cloud SDK if: ${{ (steps.changed-files.outputs.any_changed == 'true') && (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork) }}
diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index d07f984..858804c 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml
@@ -43,7 +43,7 @@ echo "USE_SHARDING=1" >> $GITHUB_ENV fi shell: bash - - name: trigger ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform + - name: run ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform env: GITHUB_SHA: ${{ github.sha }} GITHUB_TOKEN: ${{ github.token }} @@ -59,44 +59,30 @@ GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} run: | set -uxe - SESSION_ID=$( - python3 tools/on_device_tests_gateway_client.py \ - --token ${GITHUB_TOKEN} \ - --change_id "${GITHUB_PR_HEAD_SHA:-$GITHUB_SHA}" \ - trigger \ - --test_type ${{ env.TEST_TYPE }} \ - --platform ${{ matrix.target_platform }} \ - --config ${{ matrix.config }} \ - --tag cobalt_github_${GITHUB_EVENT_NAME} \ - --builder_name github_${{ matrix.platform }}_tests \ - --build_number ${GITHUB_RUN_NUMBER} \ - ${LOADER_PLATFORM:+"--loader_config" "$LOADER_CONFIG"} \ - ${LOADER_PLATFORM:+"--loader_platform" "$LOADER_PLATFORM"} \ - ${DIMENSION:+"--dimension" "$DIMENSION"} \ - ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ - ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ - --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \ - --label github \ - --label ${GITHUB_EVENT_NAME} \ - --label ${WORKFLOW} \ - --label actor-${GITHUB_ACTOR} \ - --label actor_id-${GITHUB_ACTOR_ID} \ - --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ - --label sha-${GITHUB_SHA} \ - --label repository-${GITHUB_REPO} \ - --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ - --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} - ) - echo "SESSION_ID=$SESSION_ID" >> $GITHUB_ENV - shell: bash - - name: watch ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_SHA: ${{ github.sha }} - run: | - set -uxe - python3 tools/on_device_tests_gateway_client.py \ - --token "${GITHUB_TOKEN}" \ - --change_id "${GITHUB_SHA}" \ - watch ${{ env.SESSION_ID }} + python3 -u tools/on_device_tests_gateway_client.py \ + --token ${GITHUB_TOKEN} \ + --change_id "${GITHUB_PR_HEAD_SHA:-$GITHUB_SHA}" \ + trigger \ + --test_type ${{ env.TEST_TYPE }} \ + --platform ${{ matrix.target_platform }} \ + --config ${{ matrix.config }} \ + --tag cobalt_github_${GITHUB_EVENT_NAME} \ + --builder_name github_${{ matrix.platform }}_tests \ + --build_number ${GITHUB_RUN_NUMBER} \ + ${LOADER_PLATFORM:+"--loader_config" "$LOADER_CONFIG"} \ + ${LOADER_PLATFORM:+"--loader_platform" "$LOADER_PLATFORM"} \ + ${DIMENSION:+"--dimension" "$DIMENSION"} \ + ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ + ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ + --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \ + --label github \ + --label ${GITHUB_EVENT_NAME} \ + --label ${WORKFLOW} \ + --label actor-${GITHUB_ACTOR} \ + --label actor_id-${GITHUB_ACTOR_ID} \ + --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ + --label sha-${GITHUB_SHA} \ + --label repository-${GITHUB_REPO} \ + --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ + --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} shell: bash
diff --git a/.github/config/win32.json b/.github/config/win32.json index 9de9f00..4ffbc7c 100644 --- a/.github/config/win32.json +++ b/.github/config/win32.json
@@ -1,6 +1,7 @@ { "docker_service": "build-win-win32", "docker_runner_service": "runner-win-win32", + "runner_tag": "win32", "platforms": [ "win32" ],
diff --git a/.github/config/xb1.json b/.github/config/xb1.json new file mode 100644 index 0000000..39d6247 --- /dev/null +++ b/.github/config/xb1.json
@@ -0,0 +1,19 @@ +{ + "docker_service": "build-xb1", + "__comment" : "TODO: Deploy a runner and change this", + "docker_runner_service": "runner-xb1", + "runner_tag": "xb1", + "platforms": [ + "xb1" + ], + "includes": [ + { + "name":"xb1", + "platform":"xb1", + "target_platform":"xb1", + "extra_gn_arguments": "is_clang=false", + "target_cpu": "target_cpu=\\\"x64\\\"", + "target_os": "target_os=\\\"winuwp\\\"" + } + ] +}
diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..d8d6482 --- /dev/null +++ b/.github/release.yml
@@ -0,0 +1,4 @@ +changelog: + exclude: + labels: + - ignore-for-release
diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index 4622201..3623fdf 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml
@@ -12,7 +12,7 @@ - feature/* schedule: # GMT timezone. - - cron: '0 4 * * *' + - cron: '0 9 * * *' workflow_dispatch: inputs: nightly:
diff --git a/.github/workflows/evergreen.yaml b/.github/workflows/evergreen.yaml index bc9a056..e38d392 100644 --- a/.github/workflows/evergreen.yaml +++ b/.github/workflows/evergreen.yaml
@@ -12,7 +12,7 @@ - feature/* schedule: # GMT timezone. - - cron: '0 5 * * *' + - cron: '0 9 * * *' workflow_dispatch: inputs: nightly:
diff --git a/.github/workflows/gradle.yaml b/.github/workflows/gradle.yaml index 7dec25a..3d4da7a 100644 --- a/.github/workflows/gradle.yaml +++ b/.github/workflows/gradle.yaml
@@ -20,11 +20,11 @@ steps: - uses: kaidokert/checkout@v3.5.999 timeout-minutes: 30 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: 11 + java-version: 17 - name: Validate Gradle wrapper uses: gradle/wrapper-validation-action@ccb4328a959376b642e027874838f60f8e596de3 #v1.0.6 - name: Build with Gradle
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 76f456b..59e8db0 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml
@@ -63,6 +63,6 @@ uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee with: accessToken: ${{ secrets.GITHUB_TOKEN }} - pattern: 'b\/\d+' + pattern: '(b\/\d+|^(Bug|Fixed|Issue): \d+$)' flags: 'gm' error: 'PR title or description should include at least one bug ID.'
diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 43db38a..f144ef7 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml
@@ -12,7 +12,7 @@ - feature/* schedule: # GMT timezone. - - cron: '0 4 * * *' + - cron: '0 9 * * *' workflow_dispatch: inputs: nightly:
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6f9e38a..e7f6f09 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml
@@ -47,7 +47,7 @@ STARBOARD_TOOLCHAINS_DIR: /root/starboard-toolchains concurrency: - group: '${{ github.workflow }}-${{ github.event_name }}-${{ inputs.platform }} @ ${{ github.event.label.name || github.event.pull_request.number || github.sha }}' + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.platform }} @ ${{ github.event.label.name || github.event.pull_request.number || github.sha }} @ ${{ github.event.label.name && github.event.pull_request.number || github.event.action }} cancel-in-progress: true # A workflow run is made up of one or more jobs that can run sequentially or in parallel @@ -63,6 +63,7 @@ GITHUB_EVENT_NUMBER: ${{ github.event.number }} if: | github.event.action != 'labeled' || + github.event.pull_request.merged == false && ( github.event.action == 'labeled' && github.event.label.name == 'runtest' || @@ -139,7 +140,7 @@ # Builds, tags, and pushes Cobalt docker build images to ghr. docker-build-image: needs: [initialize] - runs-on: [self-hosted, linux, X64] + runs-on: [self-hosted, linux-runner] permissions: packages: write steps: @@ -177,7 +178,7 @@ needs: [initialize] permissions: packages: write - runs-on: [self-hosted, linux, X64] + runs-on: [self-hosted, linux-runner] steps: - name: Checkout files uses: kaidokert/checkout@v3.5.999 @@ -211,7 +212,7 @@ build: needs: [initialize, docker-build-image] permissions: {} - runs-on: [self-hosted, linux, X64] + runs-on: [self-hosted, linux-runner] name: ${{matrix.name}}_${{matrix.config}} strategy: fail-fast: false @@ -298,9 +299,8 @@ inputs.nightly == 'true' || github.event_name == 'schedule') && vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) - runs-on: [self-hosted, linux, X64] + runs-on: [self-hosted, odt-runner] name: ${{ matrix.name }}_on_device_${{ matrix.shard }} - container: ${{ needs.docker-unittest-image.outputs.docker_unittest_tag }} permissions: {} strategy: fail-fast: false @@ -328,7 +328,7 @@ needs: [initialize, docker-unittest-image, build] permissions: {} if: needs.initialize.outputs.on_host_test == 'true' - runs-on: [self-hosted, linux, X64] + runs-on: [self-hosted, linux-runner] name: ${{matrix.name}}_${{matrix.shard}}_test strategy: fail-fast: false
diff --git a/.github/workflows/main_win.yaml b/.github/workflows/main_win.yaml index 0b9f14b..146424b 100644 --- a/.github/workflows/main_win.yaml +++ b/.github/workflows/main_win.yaml
@@ -38,7 +38,7 @@ STARBOARD_TOOLCHAINS_DIR: /root/starboard-toolchains concurrency: - group: '${{ github.workflow }}-${{ github.event_name }}-${{ inputs.platform }} @ ${{ github.event.label.name || github.event.pull_request.number || github.sha }}' + group: ${{ github.workflow }}-${{ github.event_name }}-${{ inputs.platform }} @ ${{ github.event.label.name || github.event.pull_request.number || github.sha }} @ ${{ github.event.label.name && github.event.pull_request.number || github.event.action }} cancel-in-progress: true # A workflow run is made up of one or more jobs that can run sequentially or in parallel @@ -55,6 +55,7 @@ # All triggers except draft PRs, unless PR is labeled with runtest if: | github.event.action != 'labeled' || + github.event.pull_request.merged == false && ( github.event.action == 'labeled' && github.event.label.name == 'runtest' || @@ -112,6 +113,11 @@ run: | docker_runner_service=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -rc '.docker_runner_service') echo "docker_runner_service=${docker_runner_service}" >> $GITHUB_ENV + - id: set-runner-tag + shell: bash + run: | + runner_tag=$(cat ${GITHUB_WORKSPACE}/.github/config/${{ inputs.platform }}.json | jq -rc '.runner_tag') + echo "runner_tag=${runner_tag}" >> $GITHUB_ENV outputs: platforms: ${{ env.platforms }} includes: ${{ env.includes }} @@ -120,6 +126,7 @@ on_host_test_shards: ${{ env.on_host_test_shards }} docker_service: ${{ env.docker_service }} docker_runner_service: ${{ env.docker_runner_service }} + runner_tag: ${{ env.runner_tag }} # Build windows docker images. build-docker-image: needs: [initialize] @@ -152,7 +159,7 @@ build: needs: [initialize] permissions: {} - runs-on: [self-hosted, win32] + runs-on: [self-hosted, "${{ needs.initialize.outputs.runner_tag }}"] name: ${{matrix.name}}_${{matrix.config}} strategy: fail-fast: false @@ -186,7 +193,7 @@ needs: [initialize, build] permissions: {} if: needs.initialize.outputs.on_host_test == 'true' - runs-on: [self-hosted, win32] + runs-on: [self-hosted, "${{ needs.initialize.outputs.runner_tag }}"] name: ${{matrix.name}}_${{matrix.shard}}_test strategy: fail-fast: false
diff --git a/.github/workflows/nightly_trigger.yaml b/.github/workflows/nightly_trigger.yaml index 2cb01b6..22f1556 100644 --- a/.github/workflows/nightly_trigger.yaml +++ b/.github/workflows/nightly_trigger.yaml
@@ -3,7 +3,7 @@ on: schedule: # GMT timezone. - - cron: '30 4 * * *' + - cron: '0 9 * * *' workflow_dispatch: jobs:
diff --git a/.github/workflows/nightly_trigger_24.lts.1+.yaml b/.github/workflows/nightly_trigger_24.lts.1+.yaml index f72b625..45ca399 100644 --- a/.github/workflows/nightly_trigger_24.lts.1+.yaml +++ b/.github/workflows/nightly_trigger_24.lts.1+.yaml
@@ -3,7 +3,7 @@ on: schedule: # GMT timezone. - - cron: '30 5 * * *' + - cron: '0 10 * * *' workflow_dispatch: jobs:
diff --git a/.github/workflows/raspi-2.yaml b/.github/workflows/raspi-2.yaml index ac42ac8..8613cfa 100644 --- a/.github/workflows/raspi-2.yaml +++ b/.github/workflows/raspi-2.yaml
@@ -12,7 +12,7 @@ - feature/* schedule: # GMT timezone. - - cron: '0 4 * * *' + - cron: '0 9 * * *' workflow_dispatch: inputs: nightly:
diff --git a/.github/workflows/stub.yaml b/.github/workflows/stub.yaml index b5953b9..4747ba5 100644 --- a/.github/workflows/stub.yaml +++ b/.github/workflows/stub.yaml
@@ -28,4 +28,4 @@ platform: stub nightly: ${{ github.event.inputs.nightly }} run_api_leak_detector: true - leak_manifest_filename: "gn_built_docker_debian10_manifest" + leak_manifest_filename: "gn_built_docker_debian11_manifest"
diff --git a/.github/workflows/win32.yaml b/.github/workflows/win32.yaml index b522992..988cbb2 100644 --- a/.github/workflows/win32.yaml +++ b/.github/workflows/win32.yaml
@@ -12,7 +12,7 @@ - feature/* schedule: # GTM timezone. - - cron: '0 4 * * *' + - cron: '0 9 * * *' workflow_dispatch: inputs: nightly:
diff --git a/.github/workflows/xb1.yaml b/.github/workflows/xb1.yaml new file mode 100644 index 0000000..d819721 --- /dev/null +++ b/.github/workflows/xb1.yaml
@@ -0,0 +1,32 @@ +name: xb1 + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + branches: + - main + - feature/* + push: + branches: + - main + - feature/* + schedule: + # GTM timezone. + - cron: '0 9 * * *' + workflow_dispatch: + inputs: + nightly: + description: 'Nightly workflow.' + required: true + type: boolean + default: false + +jobs: + xb1: + uses: ./.github/workflows/main_win.yaml + permissions: + packages: write + pull-requests: write + with: + platform: xb1 + nightly: ${{ github.event.inputs.nightly }}
diff --git a/.gitignore b/.gitignore index 9a5a5ca..31136b8 100644 --- a/.gitignore +++ b/.gitignore
@@ -9,3 +9,4 @@ /venv/* _certs/ .coverage +compile_commands.json
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2173391..2c16019 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml
@@ -37,6 +37,7 @@ # Ignore everything under tools/metrics _except_ Cobalt files. We # need those validated to keep the telemetry/metrics pipeline working. tools/metrics/((?!cobalt\/).)*$| + .*_pb2\.py$ | .*\.pb\.cc$ | .*\.pb\.h$ | .*\.patch$ | @@ -203,3 +204,11 @@ entry: gn format language: system files: '.*\.gni?$' + - id: meta-validate + name: Validate METADATA files + entry: python -m tools.metadata.validate + language: python + additional_dependencies: + - "protobuf" + always_run: true + pass_filenames: false
diff --git a/BUILD_STATUS.md b/BUILD_STATUS.md index 28ea016..f55fc68 100644 --- a/BUILD_STATUS.md +++ b/BUILD_STATUS.md
@@ -9,6 +9,7 @@ | Raspi-2 | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2.yaml?query=event%3Apush+branch%3Amain) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_24.lts.1+.yaml?query=event%3Apush+branch%3A24.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_23.lts.1+.yaml?query=event%3Apush+branch%3A23.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_22.lts.1+.yaml?query=branch%3A22.lts.1%2B+event%3Apush) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_21.lts.1+.yaml?query=branch%3A21.lts.1%2B+event%3Apush) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_20.lts.1+.yaml?query=branch%3A20.lts.1%2B+event%3Apush) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_19.lts.1+.yaml?query=branch%3A19.lts.1%2B+event%3Apush) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_rc_11.yaml?query=event%3Apush+branch%3Arc_11) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_COBALT_9.yaml?query=event%3Apush+branch%3ACOBALT_9) | | Stub | [](https://github.com/youtube/cobalt/actions/workflows/stub.yaml?query=event%3Apush+branch%3Amain) | [](https://github.com/youtube/cobalt/actions/workflows/stub_24.lts.1+.yaml?query=event%3Apush+branch%3A24.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/stub_23.lts.1+.yaml?query=event%3Apush+branch%3A23.lts.1%2B) | | | | | | | | Win32 | [](https://github.com/youtube/cobalt/actions/workflows/win32.yaml?query=event%3Apush+branch%3Amain) | [](https://github.com/youtube/cobalt/actions/workflows/win32_24.lts.1+.yaml?query=event%3Apush+branch%3A24.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/win32_23.lts.1+.yaml?query=event%3Apush+branch%3A23.lts.1%2B) | | | | | | | +| Xbox One | [](https://github.com/youtube/cobalt/actions/workflows/xb1.yaml?query=event%3Apush+branch%3Amain) | | | | | | | | | Python | [](https://github.com/youtube/cobalt/actions/workflows/pytest.yaml?query=event%3Apush+branch%3Amain) | | | | | | | | | Java | [](https://github.com/youtube/cobalt/actions/workflows/gradle.yaml?query=event%3Apush+branch%3Amain) | | | | | | | | @@ -20,3 +21,4 @@ | Linux | [](https://github.com/youtube/cobalt/actions/workflows/linux.yaml?query=event%3Aschedule+branch%3Amain) | [](https://github.com/youtube/cobalt/actions/workflows/linux_24.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A24.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/linux_23.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A23.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/linux_22.lts.1+.yaml?query=branch%3A22.lts.1%2B+event%3Aworkflow_dispatch) | [](https://github.com/youtube/cobalt/actions/workflows/linux_21.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A21.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/linux_20.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A20.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/linux_19.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A19.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/linux_rc_11.yaml?query=event%3Aworkflow_dispatch+branch%3Arc_11) | [](https://github.com/youtube/cobalt/actions/workflows/linux_COBALT_9.yaml?query=event%3Aworkflow_dispatch+branch%3ACOBALT_9) | | Raspi-2 | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2.yaml?query=event%3Aschedule+branch%3Amain) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_24.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A24.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_23.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A23.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_22.lts.1+.yaml?query=branch%3A22.lts.1%2B+event%3Aworkflow_dispatch) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_21.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A21.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_20.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A20.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_19.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A19.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_rc_11.yaml?query=event%3Aworkflow_dispatch+branch%3Arc_11) | [](https://github.com/youtube/cobalt/actions/workflows/raspi-2_COBALT_9.yaml?query=event%3Aworkflow_dispatch+branch%3ACOBALT_9) | | Win32 | [](https://github.com/youtube/cobalt/actions/workflows/win32.yaml?query=event%3Aschedule+branch%3Amain) | [](https://github.com/youtube/cobalt/actions/workflows/win32_24.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A24.lts.1%2B) | [](https://github.com/youtube/cobalt/actions/workflows/win32_23.lts.1+.yaml?query=event%3Aworkflow_dispatch+branch%3A23.lts.1%2B) | | | | | | | +| Xbox One | [](https://github.com/youtube/cobalt/actions/workflows/xb1.yaml?query=event%3Aschedule+branch%3Amain) | | | | | | | | |
diff --git a/README.md b/README.md index 32d3816..34d2b03 100644 --- a/README.md +++ b/README.md
@@ -9,6 +9,7 @@ [](https://github.com/youtube/cobalt/actions/workflows/linux.yaml?query=event%3Apush+branch%3Amain) [](https://github.com/youtube/cobalt/actions/workflows/raspi-2.yaml?query=event%3Apush+branch%3Amain) [](https://github.com/youtube/cobalt/actions/workflows/win32.yaml?query=event%3Apush+branch%3Amain) +[](https://github.com/youtube/cobalt/actions/workflows/xb1.yaml?query=event%3Apush+branch%3Amain) ## Overview
diff --git a/base/files/file.cc b/base/files/file.cc index 36ae965..db43fad 100644 --- a/base/files/file.cc +++ b/base/files/file.cc
@@ -64,7 +64,13 @@ tracing_path_(other.tracing_path_), error_details_(other.error_details()), created_(other.created()), - async_(other.async_) {} + async_(other.async_) +#if defined(STARBOARD) + , + append_(other.append_) +#endif +{ +} File::~File() { // Go through the AssertIOAllowed logic. @@ -78,6 +84,9 @@ error_details_ = other.error_details(); created_ = other.created(); async_ = other.async_; +#if defined(STARBOARD) + append_ = other.append_; +#endif return *this; }
diff --git a/base/files/file.h b/base/files/file.h index 336fe37..443abf8 100644 --- a/base/files/file.h +++ b/base/files/file.h
@@ -380,7 +380,7 @@ bool async_; #if defined(STARBOARD) - bool append_; + bool append_ = false; #endif DISALLOW_COPY_AND_ASSIGN(File);
diff --git a/base/files/file_enumerator_starboard.cc b/base/files/file_enumerator_starboard.cc index 978e940..ed6fe9d 100644 --- a/base/files/file_enumerator_starboard.cc +++ b/base/files/file_enumerator_starboard.cc
@@ -43,7 +43,8 @@ } base::Time FileEnumerator::FileInfo::GetLastModifiedTime() const { - return base::Time::FromSbTime(sb_info_.last_modified); + return base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(sb_info_.last_modified)); } // FileEnumerator --------------------------------------------------------------
diff --git a/base/files/file_starboard.cc b/base/files/file_starboard.cc index 40e63d2..b3f7362 100644 --- a/base/files/file_starboard.cc +++ b/base/files/file_starboard.cc
@@ -256,9 +256,12 @@ info->is_directory = file_info.is_directory; info->is_symbolic_link = file_info.is_symbolic_link; info->size = file_info.size; - info->last_modified = base::Time::FromSbTime(file_info.last_modified); - info->last_accessed = base::Time::FromSbTime(file_info.last_accessed); - info->creation_time = base::Time::FromSbTime(file_info.creation_time); + info->last_modified = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(file_info.last_modified)); + info->last_accessed = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(file_info.last_accessed)); + info->creation_time = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(file_info.creation_time)); return true; }
diff --git a/base/files/file_util_starboard.cc b/base/files/file_util_starboard.cc index be80ab1..83194f4 100644 --- a/base/files/file_util_starboard.cc +++ b/base/files/file_util_starboard.cc
@@ -374,9 +374,12 @@ results->is_directory = info.is_directory; results->size = info.size; - results->last_modified = base::Time::FromSbTime(info.last_modified); - results->last_accessed = base::Time::FromSbTime(info.last_accessed); - results->creation_time = base::Time::FromSbTime(info.creation_time); + results->last_modified = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(info.last_modified)); + results->last_accessed = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(info.last_accessed)); + results->creation_time = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(info.creation_time)); return true; }
diff --git a/base/i18n/icu_util.cc b/base/i18n/icu_util.cc index 5a812c7..dccc1ac 100644 --- a/base/i18n/icu_util.cc +++ b/base/i18n/icu_util.cc
@@ -52,7 +52,7 @@ #if defined(STARBOARD) bool InitializeICU() { - SbIcuInit(); + IcuInit(); return true; }
diff --git a/base/logging.cc b/base/logging.cc index 5a159ff..df473b4 100644 --- a/base/logging.cc +++ b/base/logging.cc
@@ -15,11 +15,11 @@ #include "starboard/client_porting/eztime/eztime.h" #include "starboard/common/log.h" #include "starboard/common/mutex.h" +#include "starboard/common/time.h" #include "starboard/configuration.h" #include "starboard/configuration_constants.h" #include "starboard/file.h" #include "starboard/system.h" -#include "starboard/time.h" typedef SbFile FileHandle; typedef SbMutex MutexHandle; #else @@ -210,7 +210,7 @@ uint64_t TickCount() { #if defined(STARBOARD) - return static_cast<uint64_t>(SbTimeGetMonotonicNow()); + return starboard::CurrentMonotonicTime(); #else #if defined(OS_WIN) return GetTickCount();
diff --git a/base/message_loop/message_pump_io_starboard.cc b/base/message_loop/message_pump_io_starboard.cc index 0194e54..5072fc1 100644 --- a/base/message_loop/message_pump_io_starboard.cc +++ b/base/message_loop/message_pump_io_starboard.cc
@@ -215,7 +215,7 @@ } else { TimeDelta delay = delayed_work_time_ - TimeTicks::Now(); if (delay > TimeDelta()) { - SbSocketWaiterWaitTimed(waiter_, delay.ToSbTime()); + SbSocketWaiterWaitTimed(waiter_, delay.InMicroseconds()); } else { // It looks like delayed_work_time_ indicates a time in the past, so we // need to call DoDelayedWork now.
diff --git a/base/message_loop/message_pump_ui_starboard.cc b/base/message_loop/message_pump_ui_starboard.cc index ca6ce46..32b9e09 100644 --- a/base/message_loop/message_pump_ui_starboard.cc +++ b/base/message_loop/message_pump_ui_starboard.cc
@@ -120,7 +120,7 @@ CancelDelayedLocked(); outstanding_delayed_events_.insert( - SbEventSchedule(&CallMessagePumpDelayed, this, delay.ToSbTime())); + SbEventSchedule(&CallMessagePumpDelayed, this, delay.InMicroseconds())); } }
diff --git a/base/strings/safe_sprintf_unittest.cc b/base/strings/safe_sprintf_unittest.cc index 912db4c..af83f87 100644 --- a/base/strings/safe_sprintf_unittest.cc +++ b/base/strings/safe_sprintf_unittest.cc
@@ -16,7 +16,6 @@ #include "testing/gtest/include/gtest/gtest.h" #if defined(STARBOARD) -#include "starboard/client_porting/poem/stdio_poem.h" #include "starboard/common/string.h" #include "starboard/memory.h" #include "starboard/types.h"
diff --git a/base/strings/string_util_starboard.h b/base/strings/string_util_starboard.h index c162395..a4c10c8 100644 --- a/base/strings/string_util_starboard.h +++ b/base/strings/string_util_starboard.h
@@ -16,6 +16,9 @@ #define BASE_STRING_UTIL_STARBOARD_H_ #include <stdarg.h> +#if SB_API_VERSION >= 16 +#include <stdio.h> +#endif #include "base/logging.h" #include "base/strings/string_util.h" @@ -25,13 +28,9 @@ namespace base { -#if defined(vsnprintf) -#undef vsnprintf -#endif - inline int vsnprintf(char* buffer, size_t size, const char* format, va_list arguments) { - return SbStringFormat(buffer, size, format, arguments); + return ::vsnprintf(buffer, size, format, arguments); } inline int strncmp16(const char16* s1, const char16* s2, size_t count) { @@ -42,12 +41,6 @@ #endif } -inline int vswprintf(wchar_t* buffer, size_t size, - const wchar_t* format, va_list arguments) { - DCHECK(base::IsWprintfFormatPortable(format)); - return SbStringFormatWide(buffer, size, format, arguments); -} - } // namespace base #endif // BASE_STRING_UTIL_STARBOARD_H_
diff --git a/base/strings/string_util_unittest.cc b/base/strings/string_util_unittest.cc index 474358a..0162a07 100644 --- a/base/strings/string_util_unittest.cc +++ b/base/strings/string_util_unittest.cc
@@ -12,7 +12,6 @@ #include "base/macros.h" #include "base/strings/string16.h" #include "base/strings/utf_string_conversions.h" -#include "starboard/client_porting/poem/string_poem.h" #include "starboard/common/string.h" #include "starboard/memory.h" #include "starboard/types.h"
diff --git a/base/strings/stringprintf_unittest.cc b/base/strings/stringprintf_unittest.cc index ad617ed..d31916a 100644 --- a/base/strings/stringprintf_unittest.cc +++ b/base/strings/stringprintf_unittest.cc
@@ -120,7 +120,7 @@ const int kRefSize = 320000; char* ref = new char[kRefSize]; #if defined(STARBOARD) - SbStringFormatF(ref, kRefSize, fmt, src, src, src, src, src, src, src); + snprintf(ref, kRefSize, fmt, src, src, src, src, src, src, src); #else #if defined(OS_WIN) sprintf_s(ref, kRefSize, fmt, src, src, src, src, src, src, src);
diff --git a/base/synchronization/condition_variable_starboard.cc b/base/synchronization/condition_variable_starboard.cc index 67747f0..b5a782d 100644 --- a/base/synchronization/condition_variable_starboard.cc +++ b/base/synchronization/condition_variable_starboard.cc
@@ -56,7 +56,7 @@ void ConditionVariable::TimedWait(const TimeDelta& max_time) { internal::ScopedBlockingCallWithBaseSyncPrimitives scoped_blocking_call( BlockingType::MAY_BLOCK); - SbTime duration = max_time.ToSbTime(); + int64_t duration = max_time.InMicroseconds(); #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON) user_lock_->CheckHeldAndUnmark();
diff --git a/base/test/BUILD.gn b/base/test/BUILD.gn index a4e3b91..dbfb550 100644 --- a/base/test/BUILD.gn +++ b/base/test/BUILD.gn
@@ -378,7 +378,9 @@ } } -static_library("run_all_unittests") { +# TODO: b/315170518 - Revert to static library after fixing +# symbol visibility issues for windows based modular platform builds. +source_set("run_all_unittests") { testonly = true sources = [ "run_all_unittests.cc",
diff --git a/base/threading/platform_thread_starboard.cc b/base/threading/platform_thread_starboard.cc index 0139ec2..4f26ede 100644 --- a/base/threading/platform_thread_starboard.cc +++ b/base/threading/platform_thread_starboard.cc
@@ -101,7 +101,7 @@ // static void PlatformThread::Sleep(TimeDelta duration) { - SbThreadSleep(duration.ToSbTime()); + SbThreadSleep(duration.InMicroseconds()); } // static
diff --git a/base/time/time.cc b/base/time/time.cc index 5b180b1..8e6c82b 100644 --- a/base/time/time.cc +++ b/base/time/time.cc
@@ -320,9 +320,7 @@ // static ThreadTicks ThreadTicks::Now() { - if (SbTimeIsTimeThreadNowSupported()) - return internal::g_thread_ticks_now_function(); - return ThreadTicks(); + return internal::g_thread_ticks_now_function(); } std::ostream& operator<<(std::ostream& os, ThreadTicks thread_ticks) {
diff --git a/base/time/time.h b/base/time/time.h index 58305d8..10dd488 100644 --- a/base/time/time.h +++ b/base/time/time.h
@@ -64,7 +64,7 @@ #include "build/build_config.h" #if defined(STARBOARD) -#include "starboard/time.h" +#include "starboard/common/time.h" #endif #if defined(OS_FUCHSIA) @@ -182,10 +182,6 @@ return delta_ == std::numeric_limits<int64_t>::min(); } -#if defined(STARBOARD) - SbTime ToSbTime() const; -#endif - #if defined(OS_POSIX) || defined(OS_FUCHSIA) struct timespec ToTimeSpec() const; #endif @@ -578,11 +574,6 @@ static Time FromJsTime(double ms_since_epoch); double ToJsTime() const; -#if defined(STARBOARD) - static Time FromSbTime(SbTime t); - SbTime ToSbTime() const; -#endif - // Converts to/from Java convention for times, a number of milliseconds since // the epoch. Because the Java format has less resolution, converting to Java // time is a lossy operation. @@ -1038,7 +1029,7 @@ // Returns true if ThreadTicks::Now() is supported on this system. static bool IsSupported() WARN_UNUSED_RESULT { #if defined(STARBOARD) - return SbTimeIsTimeThreadNowSupported(); + return starboard::CurrentMonotonicThreadTime() != 0; #else #if (defined(_POSIX_THREAD_CPUTIME) && (_POSIX_THREAD_CPUTIME >= 0)) || \ (defined(OS_MACOSX) && !defined(OS_IOS)) || defined(OS_ANDROID) || \
diff --git a/base/time/time_now_starboard.cc b/base/time/time_now_starboard.cc index 8892846..9e40e42 100644 --- a/base/time/time_now_starboard.cc +++ b/base/time/time_now_starboard.cc
@@ -19,9 +19,8 @@ #include "base/time/time_override.h" #include "build/build_config.h" -#include "starboard/client_porting/poem/eztime_poem.h" #include "starboard/common/log.h" -#include "starboard/time.h" +#include "starboard/common/time.h" #include "starboard/types.h" namespace base { @@ -30,7 +29,8 @@ namespace subtle { Time TimeNowIgnoringOverride() { - return Time() + TimeDelta::FromMicroseconds(SbTimeGetNow()); + return Time() + TimeDelta::FromMicroseconds( + starboard::PosixTimeToWindowsTime(starboard::CurrentPosixTime())); } Time TimeNowFromSystemTimeIgnoringOverride() { @@ -43,7 +43,8 @@ namespace subtle { TimeTicks TimeTicksNowIgnoringOverride() { - return TimeTicks() + TimeDelta::FromMicroseconds(SbTimeGetMonotonicNow()); + return TimeTicks() + TimeDelta::FromMicroseconds( + starboard::CurrentMonotonicTime()); } } // namespace subtle @@ -62,10 +63,8 @@ namespace subtle { ThreadTicks ThreadTicksNowIgnoringOverride() { - if (SbTimeIsTimeThreadNowSupported()) - return ThreadTicks() + - TimeDelta::FromMicroseconds(SbTimeGetMonotonicThreadNow()); - return ThreadTicks(); + return ThreadTicks() + TimeDelta::FromMicroseconds( + starboard::CurrentMonotonicThreadTime()); } } // namespace subtle
diff --git a/base/time/time_starboard.cc b/base/time/time_starboard.cc index 83c2302..62b2f4e 100644 --- a/base/time/time_starboard.cc +++ b/base/time/time_starboard.cc
@@ -16,7 +16,6 @@ #include "base/logging.h" #include "starboard/client_porting/eztime/eztime.h" -#include "starboard/time.h" namespace base { @@ -26,12 +25,8 @@ } } // namespace -SbTime TimeDelta::ToSbTime() const { - return InMicroseconds(); -} - void Time::Explode(bool is_local, Exploded *exploded) const { - EzTimeValue value = EzTimeValueFromSbTime(ToSbTime()); + EzTimeValue value = EzTimeValueFromSbTime(us_); EzTimeExploded ez_exploded; int millisecond; bool result = EzTimeValueExplode(&value, GetTz(is_local), &ez_exploded, @@ -60,7 +55,12 @@ ez_exploded.tm_isdst = -1; EzTimeValue value = EzTimeValueImplode(&ez_exploded, exploded.millisecond, GetTz(is_local)); - base::Time converted_time(Time::FromSbTime(EzTimeValueToSbTime(&value))); + int64_t posix_microseconds = (value.tv_sec * Time::kMicrosecondsPerSecond) + + value.tv_usec; + int64_t windows_microseconds = posix_microseconds + + Time::kTimeTToMicrosecondsOffset; + base::Time converted_time = base::Time::FromDeltaSinceWindowsEpoch( + base::TimeDelta::FromMicroseconds(windows_microseconds)); // If |exploded.day_of_month| is set to 31 on a 28-30 day month, it will // return the first day of the next month. Thus round-trip the time and @@ -82,15 +82,6 @@ } // static -Time Time::FromSbTime(SbTime t) { - return Time(t); -} - -SbTime Time::ToSbTime() const { - return us_; -} - -// static bool TimeTicks::IsHighResolution() { return true; }
diff --git a/base/time/time_unittest.cc b/base/time/time_unittest.cc index f36e5e8..39476e9 100644 --- a/base/time/time_unittest.cc +++ b/base/time/time_unittest.cc
@@ -11,22 +11,21 @@ #include <limits> #include <string> -#include "starboard/types.h" - #include "base/build_time.h" #include "base/compiler_specific.h" #include "base/logging.h" #include "base/macros.h" #include "base/strings/stringprintf.h" -#if defined(STARBOARD) -#include "base/test/time_helpers.h" -#endif // defined(STARBOARD) #include "base/threading/platform_thread.h" #include "base/time/time_override.h" #include "build/build_config.h" #include "testing/gtest/include/gtest/gtest.h" -#if defined(OS_ANDROID) +#if defined(STARBOARD) +#include "starboard/common/time.h" +#include "starboard/types.h" +#include "base/test/time_helpers.h" +#elif defined(OS_ANDROID) #include "base/android/jni_android.h" #elif defined(OS_IOS) #include "base/ios/ios_util.h" @@ -1006,7 +1005,7 @@ #define MAYBE_NowOverride NowOverride #endif TEST(ThreadTicks, MAYBE_NowOverride) { - if (!SbTimeIsTimeThreadNowSupported()) { + if (starboard::CurrentMonotonicThreadTime() == 0) { SB_LOG(INFO) << "Time thread now not supported. Test skipped."; return; }
diff --git a/base/trace_event/heap_profiler_allocation_context_tracker.cc b/base/trace_event/heap_profiler_allocation_context_tracker.cc index a2725dc..c8bed1b 100644 --- a/base/trace_event/heap_profiler_allocation_context_tracker.cc +++ b/base/trace_event/heap_profiler_allocation_context_tracker.cc
@@ -71,7 +71,7 @@ #endif // defined(OS_LINUX) || defined(OS_ANDROID) // Use tid if we don't have a thread name. - SbStringFormatF(name, sizeof(name), "%lu", + snprintf(name, sizeof(name), "%lu", static_cast<unsigned long>(PlatformThread::CurrentId())); return strdup(name); }
diff --git a/build/.gitignore b/build/.gitignore index 2e96339..2204698 100644 --- a/build/.gitignore +++ b/build/.gitignore
@@ -5,6 +5,7 @@ /android/bin /android/binary_size/apks/**/*.apk /args/chromeos/*.gni +/args/chromeos/rewrapper* /config/gclient_args.gni /cros_cache/ /Debug
diff --git a/build/BUILD.gn b/build/BUILD.gn index 51ef9b0..6634132 100644 --- a/build/BUILD.gn +++ b/build/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,6 +7,9 @@ import("//build/config/chromecast_build.gni") import("//build/config/chromeos/args.gni") import("//build/config/chromeos/ui_mode.gni") +import("//build/config/features.gni") +import("//build/util/process_version.gni") +import("//build_overrides/build.gni") source_set("buildflag_header_h") { sources = [ "buildflag.h" ] @@ -28,10 +31,20 @@ } } +buildflag_header("blink_buildflags") { + header = "blink_buildflags.h" + flags = [ "USE_BLINK=$use_blink" ] +} + buildflag_header("chromecast_buildflags") { header = "chromecast_buildflags.h" - flags = [ "IS_CHROMECAST=$is_chromecast" ] + flags = [ + "IS_CHROMECAST=$is_chromecast", + "IS_CASTOS=$is_castos", + "IS_CAST_ANDROID=$is_cast_android", + "ENABLE_CAST_RECEIVER=$enable_cast_receiver", + ] } buildflag_header("chromeos_buildflags") { @@ -42,5 +55,27 @@ "IS_CHROMEOS_LACROS=$is_chromeos_lacros", "IS_CHROMEOS_ASH=$is_chromeos_ash", + "IS_CHROMEOS_WITH_HW_DETAILS=$is_chromeos_with_hw_details", + "IS_REVEN=$is_reven", ] } + +if (build_with_chromium) { + group("gold_common_pytype") { + testonly = true + + data = [ "//build/skia_gold_common/" ] + + data_deps = [ "//testing:pytype_dependencies" ] + } +} + +if (is_chromeos) { + process_version("version_metadata") { + sources = [ "//chrome/VERSION" ] + + template_file = "metadata.json.in" + output = "$root_out_dir/metadata.json" + process_only = true + } +}
diff --git a/build/METADATA b/build/METADATA deleted file mode 100644 index 282cadc..0000000 --- a/build/METADATA +++ /dev/null
@@ -1,20 +0,0 @@ -name: "build" -description: - "Subtree at build." - -third_party { - url { - type: LOCAL_SOURCE - value: "/build_mirror" - } - url { - type: GIT - value: "https://chromium.googlesource.com/chromium/src/build" - } - version: "cfaca2819f0d31f6bb086250f240ae9fc4a07074" - last_upgrade_date { - year: 2021 - month: 5 - day: 4 - } -}
diff --git a/build/OWNERS b/build/OWNERS index 405ccdc..dce9d55 100644 --- a/build/OWNERS +++ b/build/OWNERS
@@ -1,26 +1,34 @@ set noparent # NOTE: keep this in sync with lsc-owners-override@chromium.org owners +# by emailing lsc-policy@chromium.org when this list changes. agrieve@chromium.org brucedawson@chromium.org dpranke@google.com jochen@chromium.org +sdefresne@chromium.org thakis@chromium.org thomasanderson@chromium.org tikuta@chromium.org # Clang build config changes: -hans@chromium.org +file://tools/clang/scripts/OWNERS # For java build changes: +smaier@chromium.org wnwen@chromium.org # NOTE: keep this in sync with lsc-owners-override@chromium.org owners +# by emailing lsc-policy@chromium.org when this list changes. + +# Mac build changes: +per-file mac_toolchain.py=erikchen@chromium.org +per-file mac_toolchain.py=justincohen@chromium.org +per-file mac_toolchain.py=file://build/mac/OWNERS +per-file xcode_binaries.yaml=file://build/mac/OWNERS per-file .gitignore=* per-file check_gn_headers_whitelist.txt=* -per-file mac_toolchain.py=erikchen@chromium.org -per-file mac_toolchain.py=justincohen@chromium.org per-file whitespace_file.txt=* per-file OWNERS.status=* per-file OWNERS.setnoparent=set noparent -per-file OWNERS.setnoparent=file://ENG_REVIEW_OWNERS +per-file OWNERS.setnoparent=file://ATL_OWNERS
diff --git a/build/OWNERS.setnoparent b/build/OWNERS.setnoparent index 5797d4d..52755b5 100644 --- a/build/OWNERS.setnoparent +++ b/build/OWNERS.setnoparent
@@ -2,15 +2,15 @@ # docs/code_reviews.md#owners-file-details for more details. # Overall project governance. -file://ENG_REVIEW_OWNERS +file://ATL_OWNERS # Third-party dependency review, see //docs/adding_to_third_party.md file://third_party/OWNERS # Security reviews +file://build/fuchsia/SECURITY_OWNERS file://chromeos/SECURITY_OWNERS -file://content/browser/SITE_ISOLATION_OWNERS -file://fuchsia/SECURITY_OWNERS +file://content/browser/CHILD_PROCESS_SECURITY_POLICY_OWNERS file://ipc/SECURITY_OWNERS file://net/base/SECURITY_OWNERS file://sandbox/linux/OWNERS @@ -28,6 +28,9 @@ # expose to the open web. file://third_party/blink/API_OWNERS +# third_party/blink/web_tests/VirtualTestSuites need special care. +file://third_party/blink/web_tests/VIRTUAL_OWNERS + # Extension related files. file://chrome/browser/extensions/component_extensions_allowlist/EXTENSION_ALLOWLIST_OWNERS file://extensions/common/api/API_OWNERS @@ -42,7 +45,7 @@ # Chrome and Chrome OS). # The rules are documented at: # https://sites.google.com/a/chromium.org/dev/developers/how-tos/enterprise/adding-new-policies -file://components/policy/resources/ENTERPRISE_POLICY_OWNERS +file://components/policy/ENTERPRISE_POLICY_OWNERS # This restriction is in place due to the complicated compliance regulations # around this code. @@ -51,7 +54,7 @@ # Notification channels appear in system UI and are persisted forever by # Android, so should not be added or removed lightly, and the proper # deprecation and versioning steps must be taken when doing so. -file://chrome/android/java/src/org/chromium/chrome/browser/notifications/channels/NOTIFICATION_CHANNEL_OWNERS +file://chrome/browser/notifications/android/java/src/org/chromium/chrome/browser/notifications/channels/NOTIFICATION_CHANNEL_OWNERS # The Weblayer API is supposed to be stable and will be used outside of the # chromium repository. @@ -60,3 +63,23 @@ # New features for lock/login UI on Chrome OS need to work stably in all corner # cases. file://ash/login/LOGIN_LOCK_OWNERS + +# Changes to the CQ/CI configuration can have a significant impact on infra cost +# and performance. Approval should be limited to a small subset of the users +# that can make infra changes. +file://infra/config/groups/cq-usage/CQ_USAGE_OWNERS +file://infra/config/groups/sheriff-rotations/CHROMIUM_OWNERS + +# Origin Trials owners are responsible for determining trials that need to be +# completed manually. +file://third_party/blink/common/origin_trials/OT_OWNERS + +# New notifiers added to //ash/constants/notifier_catalogs.h and +# //ash/constants/quick_settings_catalogs.h should be reviewed +# by //ash/system owners to ensure that the correct notifier is being used. +file://ash/system/OWNERS + +# WebUI surfaces are user visible and frequently are kept around indefinitely. +# New WebUI additions should be reviewed by WebUI PLATFORM_OWNERS to ensure +# they follow the guidance at https://www.chromium.org/developers/webui +file://ui/webui/PLATFORM_OWNERS
diff --git a/build/PRESUBMIT.py b/build/PRESUBMIT.py new file mode 100644 index 0000000..fba4d32 --- /dev/null +++ b/build/PRESUBMIT.py
@@ -0,0 +1,57 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +PRESUBMIT_VERSION = '2.0.0' + +# This line is 'magic' in that git-cl looks for it to decide whether to +# use Python3 instead of Python2 when running the code in this file. +USE_PYTHON3 = True + +import textwrap + + +def CheckNoBadDeps(input_api, output_api): + """Prevent additions of bad dependencies from the //build prefix.""" + build_file_patterns = [ + r'(.+/)?BUILD\.gn', + r'.+\.gni', + ] + blocklist_pattern = input_api.re.compile(r'^[^#]*"//(?!build).+?/.*"') + allowlist_pattern = input_api.re.compile(r'^[^#]*"//third_party/junit') + + warning_message = textwrap.dedent(""" + The //build directory is meant to be as hermetic as possible so that + other projects (webrtc, v8, angle) can make use of it. If you are adding + a new dep from //build onto another directory, you should consider: + 1) Can that dep live within //build? + 2) Can the dep be guarded by "build_with_chromium"? + 3) Have you made this new dep easy to pull in for other projects (ideally + a matter of adding a DEPS entry).:""") + + def FilterFile(affected_file): + return input_api.FilterSourceFile(affected_file, + files_to_check=build_file_patterns) + + problems = [] + for f in input_api.AffectedSourceFiles(FilterFile): + local_path = f.LocalPath() + for line_number, line in f.ChangedContents(): + if blocklist_pattern.search(line) and not allowlist_pattern.search(line): + problems.append('%s:%d\n %s' % + (local_path, line_number, line.strip())) + if problems: + return [output_api.PresubmitPromptOrNotify(warning_message, problems)] + else: + return [] + + +def CheckPythonTests(input_api, output_api): + return input_api.RunTests( + input_api.canned_checks.GetUnitTestsInDirectory( + input_api, + output_api, + input_api.PresubmitLocalPath(), + files_to_check=[r'.+_(?:unit)?test\.py$'], + run_on_python2=False, + run_on_python3=True))
diff --git a/build/PRESUBMIT_test.py b/build/PRESUBMIT_test.py new file mode 100755 index 0000000..c5065f4 --- /dev/null +++ b/build/PRESUBMIT_test.py
@@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys +import unittest + +import PRESUBMIT + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +from PRESUBMIT_test_mocks import MockAffectedFile +from PRESUBMIT_test_mocks import MockInputApi, MockOutputApi + +USE_PYTHON3 = True + + +def _fails_deps_check(line, filename='BUILD.gn'): + mock_input_api = MockInputApi() + mock_input_api.files = [MockAffectedFile(filename, [line])] + errors = PRESUBMIT.CheckNoBadDeps(mock_input_api, MockOutputApi()) + return bool(errors) + + +class CheckNoBadDepsTest(unittest.TestCase): + def testComments(self): + self.assertFalse(_fails_deps_check('no # import("//third_party/foo")')) + + def testFiles(self): + self.assertFalse( + _fails_deps_check('import("//third_party/foo")', filename='foo.txt')) + self.assertTrue( + _fails_deps_check('import("//third_party/foo")', filename='foo.gni')) + + def testPaths(self): + self.assertFalse(_fails_deps_check('import("//build/things.gni")')) + self.assertTrue(_fails_deps_check('import("//chrome/things.gni")')) + + +if __name__ == '__main__': + unittest.main()
diff --git a/build/README.md b/build/README.md index f9dde97..2667125 100644 --- a/build/README.md +++ b/build/README.md
@@ -11,7 +11,7 @@ Changes to `//build` should be landed in the Chromium repo. They will then be replicated to the stand-alone [build repo](https://chromium.googlesource.com/chromium/src/build) -by the [gsubtreed tool.](https://chromium.googlesource.com/infra/infra/+/master/infra/services/gsubtreed) +by the [gsubtreed tool.](https://chromium.googlesource.com/infra/infra/+/main/infra/services/gsubtreed) Note: You can find all directories already available through gsubtreed in the [list of all chromium repos](https://chromium.googlesource.com/).
diff --git a/build/action_helpers.py b/build/action_helpers.py new file mode 100644 index 0000000..046a292 --- /dev/null +++ b/build/action_helpers.py
@@ -0,0 +1,126 @@ +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Helper functions useful when writing scripts used by action() targets.""" + +import contextlib +import filecmp +import os +import pathlib +import posixpath +import shutil +import tempfile + +import gn_helpers + + +@contextlib.contextmanager +def atomic_output(path, mode='w+b', only_if_changed=True): + """Prevent half-written files and dirty mtimes for unchanged files. + + Args: + path: Path to the final output file, which will be written atomically. + mode: The mode to open the file in (str). + only_if_changed: Whether to maintain the mtime if the file has not changed. + Returns: + A Context Manager that yields a NamedTemporaryFile instance. On exit, the + manager will check if the file contents is different from the destination + and if so, move it into place. + + Example: + with action_helpers.atomic_output(output_path) as tmp_file: + subprocess.check_call(['prog', '--output', tmp_file.name]) + """ + # Create in same directory to ensure same filesystem when moving. + dirname = os.path.dirname(path) or '.' + os.makedirs(dirname, exist_ok=True) + with tempfile.NamedTemporaryFile(mode, + suffix=os.path.basename(path), + dir=dirname, + delete=False) as f: + try: + yield f + + # File should be closed before comparison/move. + f.close() + if not (only_if_changed and os.path.exists(path) + and filecmp.cmp(f.name, path)): + shutil.move(f.name, path) + finally: + f.close() + if os.path.exists(f.name): + os.unlink(f.name) + + +def add_depfile_arg(parser): + if hasattr(parser, 'add_option'): + func = parser.add_option + else: + func = parser.add_argument + func('--depfile', help='Path to depfile (refer to "gn help depfile")') + + +def write_depfile(depfile_path, first_gn_output, inputs=None): + """Writes a ninja depfile. + + See notes about how to use depfiles in //build/docs/writing_gn_templates.md. + + Args: + depfile_path: Path to file to write. + first_gn_output: Path of first entry in action's outputs. + inputs: List of inputs to add to depfile. + """ + assert depfile_path != first_gn_output # http://crbug.com/646165 + assert not isinstance(inputs, str) # Easy mistake to make + + def _process_path(path): + assert not os.path.isabs(path), f'Found abs path in depfile: {path}' + if os.path.sep != posixpath.sep: + path = str(pathlib.Path(path).as_posix()) + assert '\\' not in path, f'Found \\ in depfile: {path}' + return path.replace(' ', '\\ ') + + sb = [] + sb.append(_process_path(first_gn_output)) + if inputs: + # Sort and uniquify to ensure file is hermetic. + # One path per line to keep it human readable. + sb.append(': \\\n ') + sb.append(' \\\n '.join(sorted(_process_path(p) for p in set(inputs)))) + else: + sb.append(': ') + sb.append('\n') + + path = pathlib.Path(depfile_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(''.join(sb)) + + +def parse_gn_list(value): + """Converts a "GN-list" command-line parameter into a list. + + Conversions handled: + * None -> [] + * '' -> [] + * 'asdf' -> ['asdf'] + * '["a", "b"]' -> ['a', 'b'] + * ['["a", "b"]', 'c'] -> ['a', 'b', 'c'] (action='append') + + This allows passing args like: + gn_list = [ "one", "two", "three" ] + args = [ "--items=$gn_list" ] + """ + # Convert None to []. + if not value: + return [] + # Convert a list of GN lists to a flattened list. + if isinstance(value, list): + ret = [] + for arg in value: + ret.extend(parse_gn_list(arg)) + return ret + # Convert normal GN list. + if value.startswith('['): + return gn_helpers.GNValueParser(value).ParseList() + # Convert a single string value to a list. + return [value]
diff --git a/build/action_helpers_unittest.py b/build/action_helpers_unittest.py new file mode 100755 index 0000000..6a9f908 --- /dev/null +++ b/build/action_helpers_unittest.py
@@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import pathlib +import shutil +import sys +import tempfile +import time +import unittest + +import action_helpers + + +class ActionHelpersTest(unittest.TestCase): + def test_atomic_output(self): + tmp_file = pathlib.Path(tempfile.mktemp()) + tmp_file.write_text('test') + try: + # Test that same contents does not change mtime. + orig_mtime = os.path.getmtime(tmp_file) + with action_helpers.atomic_output(str(tmp_file), 'wt') as af: + time.sleep(.01) + af.write('test') + + self.assertEqual(os.path.getmtime(tmp_file), orig_mtime) + + # Test that contents is written. + with action_helpers.atomic_output(str(tmp_file), 'wt') as af: + af.write('test2') + self.assertEqual(tmp_file.read_text(), 'test2') + self.assertNotEqual(os.path.getmtime(tmp_file), orig_mtime) + finally: + tmp_file.unlink() + + def test_parse_gn_list(self): + def test(value, expected): + self.assertEqual(action_helpers.parse_gn_list(value), expected) + + test(None, []) + test('', []) + test('asdf', ['asdf']) + test('["one"]', ['one']) + test(['["one"]', '["two"]'], ['one', 'two']) + test(['["one", "two"]', '["three"]'], ['one', 'two', 'three']) + + def test_write_depfile(self): + tmp_file = pathlib.Path(tempfile.mktemp()) + try: + + def capture_output(inputs): + action_helpers.write_depfile(str(tmp_file), 'output', inputs) + return tmp_file.read_text() + + self.assertEqual(capture_output(None), 'output: \n') + self.assertEqual(capture_output([]), 'output: \n') + self.assertEqual(capture_output(['a']), 'output: \\\n a\n') + # Check sorted. + self.assertEqual(capture_output(['b', 'a']), 'output: \\\n a \\\n b\n') + # Check converts to forward slashes. + self.assertEqual(capture_output(['a', os.path.join('b', 'c')]), + 'output: \\\n a \\\n b/c\n') + + # Arg should be a list. + with self.assertRaises(AssertionError): + capture_output('a') + + # Do not use depfile itself as an output. + with self.assertRaises(AssertionError): + capture_output([str(tmp_file)]) + + # Do not use absolute paths. + with self.assertRaises(AssertionError): + capture_output([os.path.sep + 'foo']) + + # Do not use absolute paths (output path). + with self.assertRaises(AssertionError): + action_helpers.write_depfile(str(tmp_file), '/output', []) + + finally: + tmp_file.unlink() + + +if __name__ == '__main__': + unittest.main()
diff --git a/build/add_rts_filters.py b/build/add_rts_filters.py index 4186c39..94297c5 100755 --- a/build/add_rts_filters.py +++ b/build/add_rts_filters.py
@@ -1,11 +1,12 @@ -#!/usr/bin/env python -# Copyright (c) 2021 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Creates a dummy RTS filter file if a real one doesn't exist yes. - Real filter files are generated by the RTS binary for suites with any - skippable tests. The rest of the suites need to have dummy files because gn - will expect the file to be present. +"""Creates a dummy RTS filter file and a dummy inverse filter file if a + real ones do not exist yet. Real filter files (and their inverse) are + generated by the RTS binary for suites with any skippable tests. The + rest of the suites need to have dummy files because gn will expect the + file to be present. Implementation uses try / except because the filter files are written relatively close to when this code creates the dummy files. @@ -22,6 +23,15 @@ def main(): filter_file = sys.argv[1] + # '*' is a dummy that means run everything + write_filter_file(filter_file, '*') + + inverted_filter_file = sys.argv[2] + # '-*' is a dummy that means run nothing + write_filter_file(inverted_filter_file, '-*') + + +def write_filter_file(filter_file, filter_string): directory = os.path.dirname(filter_file) try: os.makedirs(directory) @@ -30,7 +40,6 @@ pass else: raise - try: fp = os.open(filter_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY) except OSError as err: @@ -40,7 +49,7 @@ raise else: with os.fdopen(fp, 'w') as file_obj: - file_obj.write('*') # '*' is a dummy that means run everything + file_obj.write(filter_string) if __name__ == '__main__':
diff --git a/build/android/AndroidManifest.xml b/build/android/AndroidManifest.xml index 3c4ed29..821108f 100644 --- a/build/android/AndroidManifest.xml +++ b/build/android/AndroidManifest.xml
@@ -5,14 +5,8 @@ LICENSE file. --> -<!-- - This is a dummy manifest which is required by: - 1. aapt when generating R.java in java.gypi: - Nothing in the manifest is used, but it is still required by aapt. ---> <manifest xmlns:android="http://schemas.android.com/apk/res/android" - package="org.dummy" + package="no.manifest.configured" android:versionCode="1" android:versionName="1.0"> - </manifest>
diff --git a/build/android/BUILD.gn b/build/android/BUILD.gn index 1be9f47..4d035b8 100644 --- a/build/android/BUILD.gn +++ b/build/android/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,34 +16,88 @@ create_cache = true } - if (enable_jdk_library_desugaring) { - dex_jdk_libs("all_jdk_libs") { - output = "$target_out_dir/$target_name.l8.dex" - min_sdk_version = default_min_sdk_version - } - } - generate_build_config_srcjar("build_config_gen") { use_final_fields = false } - java_library("build_config_java") { - supports_android = true - srcjar_deps = [ ":build_config_gen" ] - jar_excluded_patterns = [ "*/build/BuildConfig.class" ] + generate_build_config_srcjar("build_config_for_testing_gen") { + use_final_fields = false + testonly = true } write_native_libraries_java("native_libraries_gen") { use_final_fields = false } - android_library("native_libraries_java") { - srcjar_deps = [ ":native_libraries_gen" ] + java_library("build_java") { + supports_android = true + srcjar_deps = [ + ":build_config_gen", + ":native_libraries_gen", + ] + sources = [ + "java/src/org/chromium/build/annotations/AlwaysInline.java", + "java/src/org/chromium/build/annotations/CheckDiscard.java", + "java/src/org/chromium/build/annotations/DoNotClassMerge.java", + "java/src/org/chromium/build/annotations/DoNotInline.java", + "java/src/org/chromium/build/annotations/DoNotStripLogs.java", + "java/src/org/chromium/build/annotations/IdentifierNameString.java", + "java/src/org/chromium/build/annotations/MainDex.java", + "java/src/org/chromium/build/annotations/MockedInTests.java", + "java/src/org/chromium/build/annotations/UsedByReflection.java", + ] + + jar_excluded_patterns = [ "*/build/BuildConfig.class" ] # New version of NativeLibraries.java (with the actual correct values) will # be created when creating an apk. - jar_excluded_patterns = [ "*/NativeLibraries.class" ] + jar_excluded_patterns += [ "*/NativeLibraries.class" ] + + proguard_configs = [ "chromium_annotations.flags" ] } + + # Not all //build embedders pull in junit_binary deps that live in //third_party. + if (build_with_chromium) { + android_assets("junit_test_assets") { + testonly = true + + # We just need any file here, so use the test itself. + sources = [ "junit/src/org/chromium/build/AndroidAssetsTest.java" ] + } + android_resources("junit_test_resources") { + testonly = true + sources = [ "junit/res/values/strings.xml" ] + mergeable_android_manifests = [ "junit/AndroidManifest_mergetest.xml" ] + } + robolectric_binary("build_junit_tests") { + # Test has no JNI, so skip JNI Generator step. + generate_final_jni = false + resources_package = "org.chromium.build" + sources = [ + "junit/src/org/chromium/build/AndroidAssetsTest.java", + "junit/src/org/chromium/build/IncrementalJavacTest.java", + ] + deps = [ + ":junit_test_assets", + ":junit_test_resources", + "//build/android/test/incremental_javac_gn:no_signature_change_prebuilt_java", + "//third_party/junit", + ] + } + } +} + +# TODO(go/turn-down-test-results): Remove once we turn down +# test-results.appspot.com +python_library("test_result_presentations_py") { + pydeps_file = "pylib/results/presentation/test_results_presentation.pydeps" + data = [ + "//build/android/pylib/results/presentation/template", + "//build/android/pylib/results/presentation/javascript/main_html.js", + "//third_party/catapult/third_party/gsutil/", + "//third_party/jinja2/debug.py", + "//third_party/six", + ] } python_library("devil_chromium_py") { @@ -68,9 +122,7 @@ "//build/android/pylib/device/commands", "//tools/android/md5sum", ] - data = [ - "//third_party/android_build_tools/bundletool/bundletool-all-1.4.0.jar", - ] + data = [ "//third_party/android_build_tools/bundletool/bundletool.jar" ] } } @@ -79,24 +131,41 @@ deps = [ ":apk_installer_data" ] } -python_library("test_runner_py") { +group("test_runner_py") { + testonly = true + deps = [ + ":test_runner_core_py", + ":test_runner_device_support", + ] +} + +python_library("test_runner_core_py") { testonly = true pydeps_file = "test_runner.pydeps" data = [ "pylib/gtest/filter/", "pylib/instrumentation/render_test.html.jinja", "test_wrapper/logdog_wrapper.py", - "${android_sdk_build_tools}/aapt", - "${android_sdk_build_tools}/dexdump", - "${android_sdk_build_tools}/lib64/libc++.so", - "${android_sdk_build_tools}/split-select", - "${android_sdk_root}/platform-tools/adb", "//third_party/requests/", ] + data_deps = [ ":logdog_wrapper_py" ] +} + +group("test_runner_device_support") { + testonly = true + + # We hardcode using these tools from the public sdk in devil_chromium.json and + # in pylib's constants. + data = [ + "${public_android_sdk_build_tools}/aapt", + "${public_android_sdk_build_tools}/dexdump", + "${public_android_sdk_build_tools}/lib64/libc++.so", + "${public_android_sdk_build_tools}/split-select", + "${public_android_sdk_root}/platform-tools/adb", + ] data_deps = [ ":apk_installer_data", ":devil_chromium_py", - ":logdog_wrapper_py", ":stack_tools", ] @@ -104,6 +173,9 @@ if (build_with_chromium) { data_deps += [ "//tools/android/forwarder2" ] data += [ "//tools/android/avd/proto/" ] + if (enable_chrome_android_internal) { + data += [ "//clank/tools/android/avd/proto/" ] + } if (is_asan) { data_deps += [ "//tools/android/asan/third_party:asan_device_setup" ] } @@ -111,7 +183,7 @@ # Proguard is needed only when using apks (rather than native executables). if (enable_java_templates) { - deps = [ "//build/android/stacktrace:java_deobfuscate" ] + data_deps += [ "//build/android/stacktrace:java_deobfuscate" ] } } @@ -125,9 +197,11 @@ ":devil_chromium_py", "//third_party/catapult/tracing:convert_chart_json", ] + data = [ build_vars_file, android_readelf, + rebase_path("$android_ndk_library_path/libc++.so.1", root_build_dir), ] }
diff --git a/build/android/COMMON_METADATA b/build/android/COMMON_METADATA new file mode 100644 index 0000000..7a2580a --- /dev/null +++ b/build/android/COMMON_METADATA
@@ -0,0 +1 @@ +os: ANDROID
diff --git a/build/android/DIR_METADATA b/build/android/DIR_METADATA index 7a2580a..cdc2d6f 100644 --- a/build/android/DIR_METADATA +++ b/build/android/DIR_METADATA
@@ -1 +1 @@ -os: ANDROID +mixins: "//build/android/COMMON_METADATA"
diff --git a/build/android/OWNERS b/build/android/OWNERS index 0b64bda..94fa768 100644 --- a/build/android/OWNERS +++ b/build/android/OWNERS
@@ -1,7 +1,6 @@ +agrieve@chromium.org bjoyce@chromium.org -jbudorick@chromium.org mheikal@chromium.org pasko@chromium.org -skyostil@chromium.org -tiborg@chromium.org +smaier@chromium.org wnwen@chromium.org
diff --git a/build/android/PRESUBMIT.py b/build/android/PRESUBMIT.py index 2cf0602..8348558 100644 --- a/build/android/PRESUBMIT.py +++ b/build/android/PRESUBMIT.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,8 +8,16 @@ details on the presubmit API built into depot_tools. """ +USE_PYTHON3 = True + def CommonChecks(input_api, output_api): + # These tools don't run on Windows so these tests don't work and give many + # verbose and cryptic failure messages. Linting the code is also skipped on + # Windows because it will fail due to os differences. + if input_api.sys.platform == 'win32': + return [] + build_android_dir = input_api.PresubmitLocalPath() def J(*dirs): @@ -29,9 +37,7 @@ output_api, pylintrc='pylintrc', files_to_skip=[ - r'.*_pb2\.py', - r'.*list_java_targets\.py', # crbug.com/1100664 - r'.*fast_local_dev_server\.py', # crbug.com/1100664 + r'.*_pb2\.py' ] + build_pys, extra_paths_list=[ J(), @@ -46,7 +52,8 @@ J('..', '..', 'third_party', 'depot_tools'), J('..', '..', 'third_party', 'colorama', 'src'), J('..', '..', 'build'), - ])) + ], + version='2.7')) tests.extend( input_api.canned_checks.GetPylint( input_api, @@ -55,13 +62,27 @@ files_to_skip=[ r'.*_pb2\.py', r'.*_pb2\.py', + r'.*create_unwind_table\.py', + r'.*create_unwind_table_tests\.py', ], - extra_paths_list=[J('gyp'), J('gn')])) + extra_paths_list=[J('gyp'), J('gn')], + version='2.7')) + + tests.extend( + input_api.canned_checks.GetPylint( + input_api, + output_api, + files_to_check=[ + r'.*create_unwind_table\.py', + r'.*create_unwind_table_tests\.py', + ], + extra_paths_list=[J('gyp'), J('gn')], + version='2.7')) # yapf: enable # Disabled due to http://crbug.com/410936 #output.extend(input_api.canned_checks.RunUnitTestsInDirectory( - #input_api, output_api, J('buildbot', 'tests'))) + #input_api, output_api, J('buildbot', 'tests', skip_shebang_check=True))) pylib_test_env = dict(input_api.environ) pylib_test_env.update({ @@ -73,12 +94,7 @@ input_api, output_api, unit_tests=[ - J('.', 'emma_coverage_stats_test.py'), J('.', 'list_class_verification_failures_test.py'), - J('gyp', 'util', 'build_utils_test.py'), - J('gyp', 'util', 'manifest_utils_test.py'), - J('gyp', 'util', 'md5_check_test.py'), - J('gyp', 'util', 'resource_utils_test.py'), J('pylib', 'constants', 'host_paths_unittest.py'), J('pylib', 'gtest', 'gtest_test_instance_test.py'), J('pylib', 'instrumentation', @@ -93,20 +109,22 @@ J('pylib', 'output', 'noop_output_manager_test.py'), J('pylib', 'output', 'remote_output_manager_test.py'), J('pylib', 'results', 'json_results_test.py'), - J('pylib', 'symbols', 'apk_native_libs_unittest.py'), - J('pylib', 'symbols', 'elf_symbolizer_unittest.py'), - J('pylib', 'symbols', 'symbol_utils_unittest.py'), J('pylib', 'utils', 'chrome_proxy_utils_test.py'), J('pylib', 'utils', 'decorators_test.py'), J('pylib', 'utils', 'device_dependencies_test.py'), J('pylib', 'utils', 'dexdump_test.py'), J('pylib', 'utils', 'gold_utils_test.py'), - J('pylib', 'utils', 'proguard_test.py'), J('pylib', 'utils', 'test_filter_test.py'), - J('.', 'convert_dex_profile_tests.py'), + J('gyp', 'dex_test.py'), + J('gyp', 'util', 'build_utils_test.py'), + J('gyp', 'util', 'manifest_utils_test.py'), + J('gyp', 'util', 'md5_check_test.py'), + J('gyp', 'util', 'resource_utils_test.py'), ], env=pylib_test_env, - run_on_python2=False)) + run_on_python2=False, + run_on_python3=True, + skip_shebang_check=True)) return input_api.RunTests(tests)
diff --git a/build/android/adb_chrome_public_command_line b/build/android/adb_chrome_public_command_line index 86ece8c..0684934 100755 --- a/build/android/adb_chrome_public_command_line +++ b/build/android/adb_chrome_public_command_line
@@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/adb_command_line.py b/build/android/adb_command_line.py index c3ec8d4..8557085 100755 --- a/build/android/adb_command_line.py +++ b/build/android/adb_command_line.py
@@ -1,11 +1,10 @@ -#!/usr/bin/env vpython -# Copyright 2015 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Utility for reading / writing command-line flag files on device(s).""" -from __future__ import print_function import argparse import logging @@ -27,7 +26,7 @@ raise device_errors.CommandFailedError( 'WebView only respects flags on a userdebug or eng device, yours ' 'is a user build.', device) - elif device.IsUserBuild(): + if device.IsUserBuild(): logging.warning( 'Your device (%s) is a user build; Chrome may or may not pick up ' 'your commandline flags. Check your '
diff --git a/build/android/adb_gdb b/build/android/adb_gdb index 6de4273..885d597 100755 --- a/build/android/adb_gdb +++ b/build/android/adb_gdb
@@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # @@ -355,8 +355,8 @@ fi if [ -z "$NDK_DIR" ]; then - ANDROID_NDK_ROOT=$(PYTHONPATH=$CHROMIUM_SRC/build/android python -c \ -'from pylib.constants import ANDROID_NDK_ROOT; print ANDROID_NDK_ROOT,') + ANDROID_NDK_ROOT=$(PYTHONPATH=$CHROMIUM_SRC/build/android python3 -c \ + 'from pylib.constants import ANDROID_NDK_ROOT; print(ANDROID_NDK_ROOT,)') else if [ ! -d "$NDK_DIR" ]; then panic "Invalid directory: $NDK_DIR" @@ -573,42 +573,6 @@ echo "$FILE" } -# Find the path to an NDK's toolchain full prefix for a given architecture -# $1: NDK install path -# $2: NDK target architecture name -# Out: install path + binary prefix (e.g. -# ".../path/to/bin/arm-linux-androideabi-") -get_ndk_toolchain_fullprefix () { - local NDK_DIR="$1" - local ARCH="$2" - local TARGET NAME HOST_OS HOST_ARCH LD CONFIG - - # NOTE: This will need to be updated if the NDK changes the names or moves - # the location of its prebuilt toolchains. - # - LD= - HOST_OS=$(get_ndk_host_system) - HOST_ARCH=$(get_ndk_host_arch) - CONFIG=$(get_arch_gnu_config $ARCH) - LD=$(get_ndk_toolchain_prebuilt \ - "$NDK_DIR" "$ARCH" "$HOST_OS-$HOST_ARCH/bin/$CONFIG-ld") - if [ -z "$LD" -a "$HOST_ARCH" = "x86_64" ]; then - LD=$(get_ndk_toolchain_prebuilt \ - "$NDK_DIR" "$ARCH" "$HOST_OS-x86/bin/$CONFIG-ld") - fi - if [ ! -f "$LD" -a "$ARCH" = "x86" ]; then - # Special case, the x86 toolchain used to be incorrectly - # named i686-android-linux-gcc! - LD=$(get_ndk_toolchain_prebuilt \ - "$NDK_DIR" "$ARCH" "$HOST_OS-x86/bin/i686-android-linux-ld") - fi - if [ -z "$LD" ]; then - panic "Cannot find Android NDK toolchain for '$ARCH' architecture. \ -Please verify your NDK installation!" - fi - echo "${LD%%ld}" -} - # $1: NDK install path get_ndk_host_gdb_client() { local NDK_DIR="$1" @@ -634,28 +598,6 @@ echo "$BINARY" } -# Check/probe the path to the Android toolchain installation. Always -# use the NDK versions of gdb and gdbserver. They must match to avoid -# issues when both binaries do not speak the same wire protocol. -# -if [ -z "$TOOLCHAIN" ]; then - ANDROID_TOOLCHAIN=$(get_ndk_toolchain_fullprefix \ - "$ANDROID_NDK_ROOT" "$TARGET_ARCH") - ANDROID_TOOLCHAIN=$(dirname "$ANDROID_TOOLCHAIN") - log "Auto-config: --toolchain=$ANDROID_TOOLCHAIN" -else - # Be flexible, allow one to specify either the install path or the bin - # sub-directory in --toolchain: - # - if [ -d "$TOOLCHAIN/bin" ]; then - TOOLCHAIN=$TOOLCHAIN/bin - fi - ANDROID_TOOLCHAIN=$TOOLCHAIN -fi - -# Cosmetic: Remove trailing directory separator. -ANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN%/} - # Find host GDB client binary if [ -z "$GDB" ]; then GDB=$(get_ndk_host_gdb_client "$ANDROID_NDK_ROOT") @@ -986,7 +928,7 @@ echo "GDB wrapper script: $SYM_GDB" echo "App executable: $SYM_EXE" echo "gdbinit: $SYM_INIT" - echo "Connect with vscode: https://chromium.googlesource.com/chromium/src/+/master/docs/vscode.md#Launch-Commands" + echo "Connect with vscode: https://chromium.googlesource.com/chromium/src/+/main/docs/vscode.md#Launch-Commands" echo "Showing gdbserver logs. Press Ctrl-C to disconnect." tail -f "$GDBSERVER_LOG" else
diff --git a/build/android/adb_install_apk.py b/build/android/adb_install_apk.py index 6ec98e2..7cc6eb0 100755 --- a/build/android/adb_install_apk.py +++ b/build/android/adb_install_apk.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -59,8 +59,11 @@ parser.add_argument('--adb-path', type=os.path.abspath, help='Absolute path to the adb binary to use.') parser.add_argument('--denylist-file', help='Device denylist JSON file.') - parser.add_argument('-v', '--verbose', action='count', - help='Enable verbose logging.') + parser.add_argument('-v', + '--verbose', + action='count', + help='Enable verbose logging.', + default=0) parser.add_argument('--downgrade', action='store_true', help='If set, allows downgrading of apk.') parser.add_argument('--timeout', type=int,
diff --git a/build/android/adb_logcat_monitor.py b/build/android/adb_logcat_monitor.py index a919722..0b52997 100755 --- a/build/android/adb_logcat_monitor.py +++ b/build/android/adb_logcat_monitor.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,7 +16,6 @@ early enough to not miss anything. """ -from __future__ import print_function import logging import os @@ -33,12 +32,10 @@ class TimeoutException(Exception): """Exception used to signal a timeout.""" - pass class SigtermError(Exception): """Exception used to catch a sigterm.""" - pass def StartLogcatIfNecessary(device_id, adb_cmd, base_dir): @@ -48,12 +45,11 @@ if process.poll() is None: # Logcat process is still happily running return - else: - logging.info('Logcat for device %s has died', device_id) - error_filter = re.compile('- waiting for device -') - for line in process.stderr: - if not error_filter.match(line): - logging.error(device_id + ': ' + line) + logging.info('Logcat for device %s has died', device_id) + error_filter = re.compile('- waiting for device -') + for line in process.stderr: + if not error_filter.match(line): + logging.error(device_id + ': ' + line) logging.info('Starting logcat %d for device %s', logcat_num, device_id) @@ -85,7 +81,7 @@ stderr=subprocess.PIPE).communicate() if err: logging.warning('adb device error %s', err.strip()) - return re.findall('^(\\S+)\tdevice$', out, re.MULTILINE) + return re.findall('^(\\S+)\tdevice$', out.decode('latin1'), re.MULTILINE) except TimeoutException: logging.warning('"adb devices" command timed out') return [] @@ -141,7 +137,7 @@ except: # pylint: disable=bare-except logging.exception('Unexpected exception in main.') finally: - for process, _ in devices.itervalues(): + for process, _ in devices.values(): if process: try: process.terminate() @@ -151,8 +147,11 @@ if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) if 2 <= len(sys.argv) <= 3: print('adb_logcat_monitor: Initializing') - sys.exit(main(*sys.argv[1:3])) + if len(sys.argv) == 2: + sys.exit(main(sys.argv[1])) + sys.exit(main(sys.argv[1], sys.argv[2])) print('Usage: %s <base_dir> [<adb_binary_path>]' % sys.argv[0])
diff --git a/build/android/adb_logcat_printer.py b/build/android/adb_logcat_printer.py index a715170..7f3c52a 100755 --- a/build/android/adb_logcat_printer.py +++ b/build/android/adb_logcat_printer.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,9 +19,9 @@ """ # pylint: disable=W0702 -import cStringIO +import argparse +import io import logging -import optparse import os import re import signal @@ -108,7 +108,7 @@ """ device_logs = [] - for device, device_files in log_filenames.iteritems(): + for device, device_files in log_filenames.items(): logger.debug('%s: %s', device, str(device_files)) device_file_lines = [] for cur_file in device_files: @@ -152,15 +152,15 @@ def main(argv): - parser = optparse.OptionParser(usage='Usage: %prog [options] <log dir>') - parser.add_option('--output-path', - help='Output file path (if unspecified, prints to stdout)') - options, args = parser.parse_args(argv) - if len(args) != 1: - parser.error('Wrong number of unparsed args') - base_dir = args[0] + parser = argparse.ArgumentParser() + parser.add_argument( + '--output-path', + help='Output file path (if unspecified, prints to stdout)') + parser.add_argument('log_dir') + args = parser.parse_args(argv) + base_dir = args.log_dir - log_stringio = cStringIO.StringIO() + log_stringio = io.StringIO() logger = logging.getLogger('LogcatPrinter') logger.setLevel(LOG_LEVEL) sh = logging.StreamHandler(log_stringio) @@ -168,16 +168,16 @@ ' %(message)s')) logger.addHandler(sh) - if options.output_path: - if not os.path.exists(os.path.dirname(options.output_path)): + if args.output_path: + if not os.path.exists(os.path.dirname(args.output_path)): logger.warning('Output dir %s doesn\'t exist. Creating it.', - os.path.dirname(options.output_path)) - os.makedirs(os.path.dirname(options.output_path)) - output_file = open(options.output_path, 'w') - logger.info('Dumping logcat to local file %s. If running in a build, ' - 'this file will likely will be uploaded to google storage ' - 'in a later step. It can be downloaded from there.', - options.output_path) + os.path.dirname(args.output_path)) + os.makedirs(os.path.dirname(args.output_path)) + output_file = open(args.output_path, 'w') + logger.info( + 'Dumping logcat to local file %s. If running in a build, ' + 'this file will likely will be uploaded to google storage ' + 'in a later step. It can be downloaded from there.', args.output_path) else: output_file = sys.stdout
diff --git a/build/android/adb_profile_chrome b/build/android/adb_profile_chrome index d3244ff..27ecb6d 100755 --- a/build/android/adb_profile_chrome +++ b/build/android/adb_profile_chrome
@@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. #
diff --git a/build/android/adb_profile_chrome_startup b/build/android/adb_profile_chrome_startup index d5836cd..bb639b9 100755 --- a/build/android/adb_profile_chrome_startup +++ b/build/android/adb_profile_chrome_startup
@@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. #
diff --git a/build/android/adb_reverse_forwarder.py b/build/android/adb_reverse_forwarder.py index 90d3139..c78f44d 100755 --- a/build/android/adb_reverse_forwarder.py +++ b/build/android/adb_reverse_forwarder.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -62,7 +62,7 @@ if len(args.ports) < 2 or len(args.ports) % 2: parser.error('Need even number of port pairs') - port_pairs = zip(args.ports[::2], args.ports[1::2]) + port_pairs = list(zip(args.ports[::2], args.ports[1::2])) if args.build_type: constants.SetBuildType(args.build_type)
diff --git a/build/android/adb_system_webengine_command_line b/build/android/adb_system_webengine_command_line new file mode 100755 index 0000000..2dce6d2 --- /dev/null +++ b/build/android/adb_system_webengine_command_line
@@ -0,0 +1,16 @@ +#!/bin/bash +# +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# If no flags are given, prints the current content shell flags. +# +# Otherwise, the given flags are used to REPLACE (not modify) the content shell +# flags. For example: +# adb_system_webengine_command_line --enable-webgl +# +# To remove all content shell flags, pass an empty string for the flags: +# adb_system_webengine_command_line "" + +exec $(dirname $0)/adb_command_line.py --name weblayer-command-line "$@"
diff --git a/build/android/adb_system_webview_command_line b/build/android/adb_system_webview_command_line index a0d2705..6b9fb4e 100755 --- a/build/android/adb_system_webview_command_line +++ b/build/android/adb_system_webview_command_line
@@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/android_only_explicit_jni_exports.lst b/build/android/android_only_explicit_jni_exports.lst index f989691..eb7b1f2 100644 --- a/build/android/android_only_explicit_jni_exports.lst +++ b/build/android/android_only_explicit_jni_exports.lst
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/android_only_jni_exports.lst b/build/android/android_only_jni_exports.lst index 1336fee..c44cb9b 100644 --- a/build/android/android_only_jni_exports.lst +++ b/build/android/android_only_jni_exports.lst
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/apk_operations.py b/build/android/apk_operations.py index d6cd583..2838240 100755 --- a/build/android/apk_operations.py +++ b/build/android/apk_operations.py
@@ -1,12 +1,11 @@ -#!/usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # Using colorama.Fore/Back/Style members # pylint: disable=no-member -from __future__ import print_function import argparse import collections @@ -125,42 +124,15 @@ optimize_for=optimize_for) -def _InstallBundle(devices, apk_helper_instance, package_name, - command_line_flags_file, modules, fake_modules): - # Path Chrome creates after validating fake modules. This needs to be cleared - # for pushed fake modules to be picked up. - SPLITCOMPAT_PATH = '/data/data/' + package_name + '/files/splitcompat' - # Chrome command line flag needed for fake modules to work. - FAKE_FEATURE_MODULE_INSTALL = '--fake-feature-module-install' - - def ShouldWarnFakeFeatureModuleInstallFlag(device): - if command_line_flags_file: - changer = flag_changer.FlagChanger(device, command_line_flags_file) - return FAKE_FEATURE_MODULE_INSTALL not in changer.GetCurrentFlags() - return False - - def ClearFakeModules(device): - if device.PathExists(SPLITCOMPAT_PATH, as_root=True): - device.RemovePath( - SPLITCOMPAT_PATH, force=True, recursive=True, as_root=True) - logging.info('Removed %s', SPLITCOMPAT_PATH) - else: - logging.info('Skipped removing nonexistent %s', SPLITCOMPAT_PATH) +def _InstallBundle(devices, apk_helper_instance, modules, fake_modules): def Install(device): - ClearFakeModules(device) - if fake_modules and ShouldWarnFakeFeatureModuleInstallFlag(device): - # Print warning if command line is not set up for fake modules. - msg = ('Command line has no %s: Fake modules will be ignored.' % - FAKE_FEATURE_MODULE_INSTALL) - print(_Colorize(msg, colorama.Fore.YELLOW + colorama.Style.BRIGHT)) - - device.Install( - apk_helper_instance, - permissions=[], - modules=modules, - fake_modules=fake_modules, - allow_downgrade=True) + device.Install(apk_helper_instance, + permissions=[], + modules=modules, + fake_modules=fake_modules, + allow_downgrade=True, + reinstall=True) # Basic checks for |modules| and |fake_modules|. # * |fake_modules| cannot include 'base'. @@ -215,24 +187,95 @@ return debug_process_name -def _LaunchUrl(devices, package_name, argv=None, command_line_flags_file=None, - url=None, apk=None, wait_for_java_debugger=False, - debug_process_name=None, nokill=None): +def _ResolveActivity(device, package_name, category, action): + # E.g.: + # Activity Resolver Table: + # Schemes: + # http: + # 67e97c0 org.chromium.pkg/.MainActivityfilter c91d43e + # Action: "android.intent.action.VIEW" + # Category: "android.intent.category.DEFAULT" + # Category: "android.intent.category.BROWSABLE" + # Scheme: "http" + # Scheme: "https" + # + # Non-Data Actions: + # android.intent.action.MAIN: + # 67e97c0 org.chromium.pkg/.MainActivity filter 4a34cf9 + # Action: "android.intent.action.MAIN" + # Category: "android.intent.category.LAUNCHER" + lines = device.RunShellCommand(['dumpsys', 'package', package_name], + check_return=True) + + # Extract the Activity Resolver Table: section. + start_idx = next((i for i, l in enumerate(lines) + if l.startswith('Activity Resolver Table:')), None) + if start_idx is None: + if not device.IsApplicationInstalled(package_name): + raise Exception('Package not installed: ' + package_name) + raise Exception('No Activity Resolver Table in:\n' + '\n'.join(lines)) + line_count = next(i for i, l in enumerate(lines[start_idx + 1:]) + if l and not l[0].isspace()) + data = '\n'.join(lines[start_idx:start_idx + line_count]) + + # Split on each Activity entry. + entries = re.split(r'^ [0-9a-f]+ ', data, flags=re.MULTILINE) + + def activity_name_from_entry(entry): + assert entry.startswith(package_name), 'Got: ' + entry + activity_name = entry[len(package_name) + 1:].split(' ', 1)[0] + if activity_name[0] == '.': + activity_name = package_name + activity_name + return activity_name + + # Find the one with the text we want. + category_text = f'Category: "{category}"' + action_text = f'Action: "{action}"' + matched_entries = [ + e for e in entries[1:] if category_text in e and action_text in e + ] + + if not matched_entries: + raise Exception(f'Did not find {category_text}, {action_text} in\n{data}') + if len(matched_entries) > 1: + # When there are multiple matches, look for the one marked as default. + # Necessary for Monochrome, which also has MonochromeLauncherActivity. + default_entries = [ + e for e in matched_entries if 'android.intent.category.DEFAULT' in e + ] + matched_entries = default_entries or matched_entries + + # See if all matches point to the same activity. + activity_names = {activity_name_from_entry(e) for e in matched_entries} + + if len(activity_names) > 1: + raise Exception('Found multiple launcher activities:\n * ' + + '\n * '.join(sorted(activity_names))) + return next(iter(activity_names)) + + +def _LaunchUrl(devices, + package_name, + argv=None, + command_line_flags_file=None, + url=None, + wait_for_java_debugger=False, + debug_process_name=None, + nokill=None): if argv and command_line_flags_file is None: raise Exception('This apk does not support any flags.') - if url: - # TODO(agrieve): Launch could be changed to require only package name by - # parsing "dumpsys package" rather than relying on the apk. - if not apk: - raise Exception('Launching with URL is not supported when using ' - '--package-name. Use --apk-path instead.') - view_activity = apk.GetViewActivityName() - if not view_activity: - raise Exception('APK does not support launching with URLs.') debug_process_name = _NormalizeProcessName(debug_process_name, package_name) + if url is None: + category = 'android.intent.category.LAUNCHER' + action = 'android.intent.action.MAIN' + else: + category = 'android.intent.category.BROWSABLE' + action = 'android.intent.action.VIEW' + def launch(device): + activity = _ResolveActivity(device, package_name, category, action) # --persistent is required to have Settings.Global.DEBUG_APP be set, which # we currently use to allow reading of flags. https://crbug.com/784947 if not nokill: @@ -255,18 +298,13 @@ except device_errors.AdbShellCommandFailedError: logging.exception('Failed to set flags') - if url is None: - # Simulate app icon click if no url is present. - cmd = [ - 'am', 'start', '-p', package_name, '-c', - 'android.intent.category.LAUNCHER', '-a', 'android.intent.action.MAIN' - ] - device.RunShellCommand(cmd, check_return=True) - else: - launch_intent = intent.Intent(action='android.intent.action.VIEW', - activity=view_activity, data=url, - package=package_name) - device.StartActivity(launch_intent) + launch_intent = intent.Intent(action=action, + activity=activity, + data=url, + package=package_name) + logging.info('Sending launch intent for %s', activity) + device.StartActivity(launch_intent) + device_utils.DeviceUtils.parallel(devices).pMap(launch) if wait_for_java_debugger: print('Waiting for debugger to attach to process: ' + @@ -472,8 +510,8 @@ code_path = re.search(r'codePath=(.*)', package_output).group(1) lib_path = re.search(r'(?:legacyN|n)ativeLibrary(?:Dir|Path)=(.*)', package_output).group(1) - except AttributeError: - raise Exception('Error parsing dumpsys output: ' + package_output) + except AttributeError as e: + raise Exception('Error parsing dumpsys output: ' + package_output) from e if code_path.startswith('/system'): logging.warning('Measurement of system image apks can be innacurate') @@ -535,8 +573,8 @@ compilation_filter) def print_sizes(desc, sizes): - print('%s: %d KiB' % (desc, sum(sizes.itervalues()))) - for path, size in sorted(sizes.iteritems()): + print('%s: %d KiB' % (desc, sum(sizes.values()))) + for path, size in sorted(sizes.items()): print(' %s: %s KiB' % (path, size)) parallel_devices = device_utils.DeviceUtils.parallel(devices) @@ -548,7 +586,7 @@ (data_dir_sizes, code_cache_sizes, apk_sizes, lib_sizes, odex_sizes, compilation_filter) = result - total = sum(sum(sizes.itervalues()) for sizes in result[:-1]) + total = sum(sum(sizes.values()) for sizes in result[:-1]) print_sizes('Apk', apk_sizes) print_sizes('App Data (non-code cache)', data_dir_sizes) @@ -563,12 +601,12 @@ print('Total: %s KiB (%.1f MiB)' % (total, total / 1024.0)) -class _LogcatProcessor(object): +class _LogcatProcessor: ParsedLine = collections.namedtuple( 'ParsedLine', ['date', 'invokation_time', 'pid', 'tid', 'priority', 'tag', 'message']) - class NativeStackSymbolizer(object): + class NativeStackSymbolizer: """Buffers lines from native stacks and symbolizes them when done.""" # E.g.: #06 pc 0x0000d519 /apex/com.android.runtime/lib/libart.so # E.g.: #01 pc 00180c8d /data/data/.../lib/libbase.cr.so @@ -582,9 +620,12 @@ def _FlushLines(self): """Prints queued lines after sending them through stack.py.""" + if self._crash_lines_buffer is None: + return + crash_lines = self._crash_lines_buffer self._crash_lines_buffer = None - with tempfile.NamedTemporaryFile() as f: + with tempfile.NamedTemporaryFile(mode='w') as f: f.writelines(x[0].message + '\n' for x in crash_lines) f.flush() proc = self._stack_script_context.Popen( @@ -612,8 +653,7 @@ self._crash_lines_buffer.append((parsed_line, dim)) return - if self._crash_lines_buffer is not None: - self._FlushLines() + self._FlushLines() self._print_func(parsed_line, dim) @@ -624,6 +664,7 @@ 'ActivityManager', # Shows activity lifecycle messages. 'ActivityTaskManager', # More activity lifecycle messages. 'AndroidRuntime', # Java crash dumps + 'AppZygoteInit', # Android's native application zygote support. 'DEBUG', # Native crash dump. } @@ -647,11 +688,19 @@ package_name, stack_script_context, deobfuscate=None, - verbose=False): + verbose=False, + exit_on_match=None, + extra_package_names=None): self._device = device self._package_name = package_name + self._extra_package_names = extra_package_names or [] self._verbose = verbose self._deobfuscator = deobfuscate + if exit_on_match is not None: + self._exit_on_match = re.compile(exit_on_match) + else: + self._exit_on_match = None + self._found_exit_match = False self._native_stack_symbolizer = _LogcatProcessor.NativeStackSymbolizer( stack_script_context, self._PrintParsedLine) # Process ID for the app's main process (with no :name suffix). @@ -665,7 +714,7 @@ # START u0 {act=android.intent.action.MAIN \ # cat=[android.intent.category.LAUNCHER] \ # flg=0x10000000 pkg=com.google.chromeremotedesktop} from uid 2000 - self._start_pattern = re.compile(r'START .*pkg=' + package_name) + self._start_pattern = re.compile(r'START .*(?:cmp|pkg)=' + package_name) self.nonce = 'Chromium apk_operations.py nonce={}'.format(random.random()) # Holds lines buffered on start-up, before we find our nonce message. @@ -674,7 +723,7 @@ # Give preference to PID reported by "ps" over those found from # _start_pattern. There can be multiple "Start proc" messages from prior # runs of the app. - self._found_initial_pid = self._primary_pid != None + self._found_initial_pid = self._primary_pid is not None # Retrieve any additional patterns that are relevant for the User. self._user_defined_highlight = None user_regex = os.environ.get('CHROMIUM_LOGCAT_HIGHLIGHT') @@ -690,20 +739,21 @@ # ProcessLine method below also includes lines from processes which may # have already exited. self._primary_pid = None - for process in _GetPackageProcesses(self._device, self._package_name): - # We take only the first "main" process found in order to account for - # possibly forked() processes. - if ':' not in process.name and self._primary_pid is None: - self._primary_pid = process.pid - self._my_pids.add(process.pid) + for package_name in [self._package_name] + self._extra_package_names: + for process in _GetPackageProcesses(self._device, package_name): + # We take only the first "main" process found in order to account for + # possibly forked() processes. + if ':' not in process.name and self._primary_pid is None: + self._primary_pid = process.pid + self._my_pids.add(process.pid) def _GetPidStyle(self, pid, dim=False): if pid == self._primary_pid: return colorama.Fore.WHITE - elif pid in self._my_pids: + if pid in self._my_pids: # TODO(wnwen): Use one separate persistent color per process, pop LRU return colorama.Fore.YELLOW - elif dim: + if dim: return colorama.Style.DIM return '' @@ -712,7 +762,7 @@ if dim: return '' style = colorama.Fore.BLACK - if priority == 'E' or priority == 'F': + if priority in ('E', 'F'): style += colorama.Back.RED elif priority == 'W': style += colorama.Back.YELLOW @@ -758,6 +808,9 @@ date, invokation_time, pid, tid, priority, tag, original_message) def _PrintParsedLine(self, parsed_line, dim=False): + if self._exit_on_match and self._exit_on_match.search(parsed_line.message): + self._found_exit_match = True + tid_style = colorama.Style.NORMAL user_match = self._user_defined_highlight and ( re.search(self._user_defined_highlight, parsed_line.tag) @@ -794,6 +847,9 @@ self._initial_buffered_lines = None self.nonce = None + def FoundExitMatch(self): + return self._found_exit_match + def ProcessLine(self, line): if not line or line.startswith('------'): return @@ -842,14 +898,26 @@ self._initial_buffered_lines.append((log, not owned_pid)) -def _RunLogcat(device, package_name, stack_script_context, deobfuscate, - verbose): - logcat_processor = _LogcatProcessor( - device, package_name, stack_script_context, deobfuscate, verbose) +def _RunLogcat(device, + package_name, + stack_script_context, + deobfuscate, + verbose, + exit_on_match=None, + extra_package_names=None): + logcat_processor = _LogcatProcessor(device, + package_name, + stack_script_context, + deobfuscate, + verbose, + exit_on_match=exit_on_match, + extra_package_names=extra_package_names) device.RunShellCommand(['log', logcat_processor.nonce]) for line in device.adb.Logcat(logcat_format='threadtime'): try: logcat_processor.ProcessLine(line) + if logcat_processor.FoundExitMatch(): + return except: sys.stderr.write('Failed to process line: ' + line + '\n') # Skip stack trace for the common case of the adb server being @@ -860,9 +928,11 @@ def _GetPackageProcesses(device, package_name): + my_names = (package_name, package_name + '_zygote') return [ p for p in device.ListProcesses(package_name) - if p.name == package_name or p.name.startswith(package_name + ':')] + if p.name in my_names or p.name.startswith(package_name + ':') + ] def _RunPs(devices, package_name): @@ -912,7 +982,7 @@ def _RunProfile(device, package_name, host_build_directory, pprof_out_path, - process_specifier, thread_specifier, extra_args): + process_specifier, thread_specifier, events, extra_args): simpleperf.PrepareDevice(device) device_simpleperf_path = simpleperf.InstallSimpleperf(device, package_name) with tempfile.NamedTemporaryFile() as fh: @@ -920,11 +990,10 @@ with simpleperf.RunSimpleperf(device, device_simpleperf_path, package_name, process_specifier, thread_specifier, - extra_args, host_simpleperf_out_path): - sys.stdout.write('Profiler is running; press Enter to stop...') + events, extra_args, host_simpleperf_out_path): + sys.stdout.write('Profiler is running; press Enter to stop...\n') sys.stdin.read(1) - sys.stdout.write('Post-processing data...') - sys.stdout.flush() + sys.stdout.write('Post-processing data...\n') simpleperf.ConvertSimpleperfToPprof(host_simpleperf_out_path, host_build_directory, pprof_out_path) @@ -941,7 +1010,7 @@ """ % {'s': pprof_out_path})) -class _StackScriptContext(object): +class _StackScriptContext: """Maintains temporary files needed by stack.py.""" def __init__(self, @@ -1000,7 +1069,7 @@ if input_file: cmd.append(input_file) logging.info('Running stack.py') - return subprocess.Popen(cmd, **kwargs) + return subprocess.Popen(cmd, universal_newlines=True, **kwargs) def _GenerateAvailableDevicesMessage(devices): @@ -1063,7 +1132,7 @@ logging.info('Wrote device cache: %s', cache_path) -class _Command(object): +class _Command: name = None description = None long_description = None @@ -1078,7 +1147,7 @@ calls_exec = False supports_multiple_devices = True - def __init__(self, from_wrapper_script, is_bundle): + def __init__(self, from_wrapper_script, is_bundle, is_test_apk): self._parser = None self._from_wrapper_script = from_wrapper_script self.args = None @@ -1087,6 +1156,7 @@ self.install_dict = None self.devices = None self.is_bundle = is_bundle + self.is_test_apk = is_test_apk self.bundle_generation_info = None # Only support incremental install from APK wrapper scripts. if is_bundle or not from_wrapper_script: @@ -1095,7 +1165,7 @@ def RegisterBundleGenerationInfo(self, bundle_generation_info): self.bundle_generation_info = bundle_generation_info - def _RegisterExtraArgs(self, subp): + def _RegisterExtraArgs(self, group): pass def RegisterArgs(self, parser): @@ -1341,8 +1411,7 @@ modules = list( set(self.args.module) - set(self.args.no_module) - set(self.args.fake)) - _InstallBundle(self.devices, self.apk_helper, self.args.package_name, - self.args.command_line_flags_file, modules, self.args.fake) + _InstallBundle(self.devices, self.apk_helper, modules, self.args.fake) else: _InstallApk(self.devices, self.apk_helper, self.install_dict) @@ -1393,13 +1462,13 @@ group.add_argument('url', nargs='?', help='A URL to launch with.') def Run(self): - if self.args.url and self.is_bundle: - # TODO(digit): Support this, maybe by using 'dumpsys' as described - # in the _LaunchUrl() comment. - raise Exception('Launching with URL not supported for bundles yet!') - _LaunchUrl(self.devices, self.args.package_name, argv=self.args.args, + if self.is_test_apk: + raise Exception('Use the bin/run_* scripts to run test apks.') + _LaunchUrl(self.devices, + self.args.package_name, + argv=self.args.args, command_line_flags_file=self.args.command_line_flags_file, - url=self.args.url, apk=self.apk_helper, + url=self.args.url, wait_for_java_debugger=self.args.wait_for_java_debugger, debug_process_name=self.args.debug_process_name, nokill=self.args.nokill) @@ -1511,9 +1580,20 @@ self.args.apk_path, self.bundle_generation_info, quiet=True) + + extra_package_names = [] + if self.is_test_apk and self.additional_apk_helpers: + for additional_apk_helper in self.additional_apk_helpers: + extra_package_names.append(additional_apk_helper.GetPackageName()) + try: - _RunLogcat(self.devices[0], self.args.package_name, stack_script_context, - deobfuscate, bool(self.args.verbose_count)) + _RunLogcat(self.devices[0], + self.args.package_name, + stack_script_context, + deobfuscate, + bool(self.args.verbose_count), + self.args.exit_on_match, + extra_package_names=extra_package_names) except KeyboardInterrupt: pass # Don't show stack trace upon Ctrl-C finally: @@ -1529,6 +1609,8 @@ group.set_defaults(no_deobfuscate=False) group.add_argument('--proguard-mapping-path', help='Path to ProGuard map (enables deobfuscation)') + group.add_argument('--exit-on-match', + help='Exits logcat when a message matches this regex.') class _PsCommand(_Command): @@ -1628,6 +1710,8 @@ def Run(self): keytool = os.path.join(_JAVA_HOME, 'bin', 'keytool') + pem_certificate_pattern = re.compile( + r'-+BEGIN CERTIFICATE-+([\r\n0-9A-Za-z+/=]+)-+END CERTIFICATE-+[\r\n]*') if self.is_bundle: # Bundles are not signed until converted to .apks. The wrapper scripts # record which key will be used to sign though. @@ -1647,36 +1731,72 @@ if self.args.full_cert: # Redirect stderr to hide a keytool warning about using non-standard # keystore format. - full_output = subprocess.check_output( - cmd + ['-rfc'], stderr=subprocess.STDOUT) + pem_encoded_certificate = subprocess.check_output( + cmd + ['-rfc'], stderr=subprocess.STDOUT).decode() else: - cmd = [ - build_tools.GetPath('apksigner'), 'verify', '--print-certs', - '--verbose', self.apk_helper.path - ] - logging.warning('Running: %s', ' '.join(cmd)) - env = os.environ.copy() - env['PATH'] = os.path.pathsep.join( - [os.path.join(_JAVA_HOME, 'bin'), - env.get('PATH')]) - stdout = subprocess.check_output(cmd, env=env) - print(stdout) - if self.args.full_cert: - if 'v1 scheme (JAR signing): true' not in stdout: - raise Exception( - 'Cannot print full certificate because apk is not V1 signed.') - cmd = [keytool, '-printcert', '-jarfile', self.apk_helper.path, '-rfc'] - # Redirect stderr to hide a keytool warning about using non-standard - # keystore format. - full_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + def run_apksigner(min_sdk_version): + cmd = [ + build_tools.GetPath('apksigner'), 'verify', '--min-sdk-version', + str(min_sdk_version), '--print-certs-pem', '--verbose', + self.apk_helper.path + ] + logging.warning('Running: %s', ' '.join(cmd)) + env = os.environ.copy() + env['PATH'] = os.path.pathsep.join( + [os.path.join(_JAVA_HOME, 'bin'), + env.get('PATH')]) + # Redirect stderr to hide verification failures (see explanation below). + return subprocess.check_output(cmd, + env=env, + universal_newlines=True, + stderr=subprocess.STDOUT) + + # apksigner's default behavior is nonintuitive: it will print "Verified + # using <scheme number>...: false" for any scheme which is obsolete for + # the APK's minSdkVersion even if it actually was signed with that scheme + # (ex. it prints "Verified using v1 scheme: false" for Monochrome because + # v1 was obsolete by N). To workaround this, we force apksigner to use the + # lowest possible minSdkVersion. We need to fallback to higher + # minSdkVersions in case the APK fails to verify for that minSdkVersion + # (which means the APK is genuinely not signed with that scheme). These + # SDK values are the highest SDK version before the next scheme is + # available: + versions = [ + version_codes.MARSHMALLOW, # before v2 launched in N + version_codes.OREO_MR1, # before v3 launched in P + version_codes.Q, # before v4 launched in R + version_codes.R, + ] + stdout = None + for min_sdk_version in versions: + try: + stdout = run_apksigner(min_sdk_version) + break + except subprocess.CalledProcessError: + # Doesn't verify with this min-sdk-version, so try again with a higher + # one + continue + if not stdout: + raise RuntimeError('apksigner was not able to verify APK') + + # Separate what the '--print-certs' flag would output vs. the additional + # signature output included by '--print-certs-pem'. The additional PEM + # output is only printed when self.args.full_cert is specified. + verification_hash_info = pem_certificate_pattern.sub('', stdout) + print(verification_hash_info) + if self.args.full_cert: + m = pem_certificate_pattern.search(stdout) + if not m: + raise Exception('apksigner did not print a certificate') + pem_encoded_certificate = m.group(0) + if self.args.full_cert: - m = re.search( - r'-+BEGIN CERTIFICATE-+([\r\n0-9A-Za-z+/=]+)-+END CERTIFICATE-+', - full_output, re.MULTILINE) + m = pem_certificate_pattern.search(pem_encoded_certificate) if not m: - raise Exception('Unable to parse certificate:\n{}'.format(full_output)) + raise Exception( + 'Unable to parse certificate:\n{}'.format(pem_encoded_certificate)) signature = re.sub(r'[\r\n]+', '', m.group(1)) print() print('Full Signature:') @@ -1712,13 +1832,18 @@ 'use --profile-thread=main).')) group.add_argument('--profile-output', default='profile.pb', help='Output file for profiling data') + group.add_argument('--profile-events', default='cpu-cycles', + help=('A comma separated list of perf events to capture ' + '(e.g. \'cpu-cycles,branch-misses\'). Run ' + '`simpleperf list` on your device to see available ' + 'events.')) def Run(self): extra_args = shlex.split(self.args.args or '') _RunProfile(self.devices[0], self.args.package_name, self.args.output_directory, self.args.profile_output, self.args.profile_process, self.args.profile_thread, - extra_args) + self.args.profile_events, extra_args) class _RunCommand(_InstallCommand, _LaunchCommand, _LogcatCommand): @@ -1735,6 +1860,8 @@ help='Install and launch, but do not enter logcat.') def Run(self): + if self.is_test_apk: + raise Exception('Use the bin/run_* scripts to run test apks.') logging.warning('Installing...') _InstallCommand.Run(self) logging.warning('Sending launch intent...') @@ -1785,13 +1912,23 @@ class _ManifestCommand(_Command): name = 'dump-manifest' - description = 'Dump the android manifest from this bundle, as XML, to stdout.' + description = 'Dump the android manifest as XML, to stdout.' need_device_args = False + needs_apk_helper = True def Run(self): - bundletool.RunBundleTool([ - 'dump', 'manifest', '--bundle', self.bundle_generation_info.bundle_path - ]) + if self.is_bundle: + sys.stdout.write( + bundletool.RunBundleTool([ + 'dump', 'manifest', '--bundle', + self.bundle_generation_info.bundle_path + ])) + else: + apkanalyzer = os.path.join(_DIR_SOURCE_ROOT, 'third_party', 'android_sdk', + 'public', 'cmdline-tools', 'latest', 'bin', + 'apkanalyzer') + subprocess.check_call( + [apkanalyzer, 'manifest', 'print', self.apk_helper.path]) class _StackCommand(_Command): @@ -1839,19 +1976,22 @@ _ProfileCommand, _RunCommand, _StackCommand, + _ManifestCommand, ] # Commands specific to app bundles. _BUNDLE_COMMANDS = [ _BuildBundleApks, - _ManifestCommand, ] -def _ParseArgs(parser, from_wrapper_script, is_bundle): +def _ParseArgs(parser, from_wrapper_script, is_bundle, is_test_apk): subparsers = parser.add_subparsers() command_list = _COMMANDS + (_BUNDLE_COMMANDS if is_bundle else []) - commands = [clazz(from_wrapper_script, is_bundle) for clazz in command_list] + commands = [ + clazz(from_wrapper_script, is_bundle, is_test_apk) + for clazz in command_list + ] for command in commands: if from_wrapper_script or not command.needs_output_directory: @@ -1868,13 +2008,17 @@ def _RunInternal(parser, output_directory=None, additional_apk_paths=None, - bundle_generation_info=None): + bundle_generation_info=None, + is_test_apk=False): colorama.init() parser.set_defaults( additional_apk_paths=additional_apk_paths, output_directory=output_directory) from_wrapper_script = bool(output_directory) - args = _ParseArgs(parser, from_wrapper_script, bool(bundle_generation_info)) + args = _ParseArgs(parser, + from_wrapper_script, + is_bundle=bool(bundle_generation_info), + is_test_apk=is_test_apk) run_tests_helper.SetLogLevel(args.verbose_count) if bundle_generation_info: args.command.RegisterBundleGenerationInfo(bundle_generation_info) @@ -1961,6 +2105,39 @@ bundle_generation_info=bundle_generation_info) +def RunForTestApk(*, output_directory, package_name, test_apk_path, + test_apk_json, proguard_mapping_path, additional_apk_paths): + """Entry point for generated test apk wrapper scripts. + + This is intended to make commands like logcat (with proguard deobfuscation) + available. The run_* scripts should be used to actually run tests. + + Args: + output_dir: Chromium output directory path. + package_name: The package name for the test apk. + test_apk_path: The test apk to install. + test_apk_json: The incremental json dict for the test apk. + proguard_mapping_path: Input path to the Proguard mapping file, used to + deobfuscate Java stack traces. + additional_apk_paths: Additional APKs to install. + """ + constants.SetOutputDirectory(output_directory) + devil_chromium.Initialize(output_directory=output_directory) + + parser = argparse.ArgumentParser() + exists_or_none = lambda p: p if p and os.path.exists(p) else None + + parser.set_defaults(apk_path=exists_or_none(test_apk_path), + incremental_json=exists_or_none(test_apk_json), + package_name=package_name, + proguard_mapping_path=proguard_mapping_path) + + _RunInternal(parser, + output_directory=output_directory, + additional_apk_paths=additional_apk_paths, + is_test_apk=True) + + def main(): devil_chromium.Initialize() _RunInternal(argparse.ArgumentParser())
diff --git a/build/android/apk_operations.pydeps b/build/android/apk_operations.pydeps index 60b1289..d20bcf2 100644 --- a/build/android/apk_operations.pydeps +++ b/build/android/apk_operations.pydeps
@@ -64,7 +64,8 @@ ../../third_party/catapult/devil/devil/utils/zip_utils.py ../../third_party/catapult/third_party/six/six.py ../../third_party/jinja2/__init__.py -../../third_party/jinja2/_compat.py +../../third_party/jinja2/_identifier.py +../../third_party/jinja2/async_utils.py ../../third_party/jinja2/bccache.py ../../third_party/jinja2/compiler.py ../../third_party/jinja2/defaults.py @@ -84,11 +85,12 @@ ../../third_party/markupsafe/__init__.py ../../third_party/markupsafe/_compat.py ../../third_party/markupsafe/_native.py +../action_helpers.py ../gn_helpers.py ../print_python_deps.py +../zip_helpers.py adb_command_line.py apk_operations.py -convert_dex_profile.py devil_chromium.py gyp/bundletool.py gyp/dex.py @@ -96,7 +98,6 @@ gyp/util/build_utils.py gyp/util/md5_check.py gyp/util/resource_utils.py -gyp/util/zipalign.py incremental_install/__init__.py incremental_install/installer.py pylib/__init__.py @@ -104,6 +105,7 @@ pylib/constants/host_paths.py pylib/symbols/__init__.py pylib/symbols/deobfuscator.py +pylib/symbols/expensive_line_transformer.py pylib/utils/__init__.py pylib/utils/app_bundle_utils.py pylib/utils/simpleperf.py
diff --git a/build/android/apply_shared_preference_file.py b/build/android/apply_shared_preference_file.py index 187bf18..a4aa499 100755 --- a/build/android/apply_shared_preference_file.py +++ b/build/android/apply_shared_preference_file.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/asan_symbolize.py b/build/android/asan_symbolize.py index 6585089..3274b95 100755 --- a/build/android/asan_symbolize.py +++ b/build/android/asan_symbolize.py
@@ -1,13 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function +import argparse import collections -import optparse import os import re import sys @@ -52,7 +51,7 @@ return AsanParsedLine(prefix=m.group('prefix'), library=m.group('lib'), pos=m.group('pos'), - rel_address='%08x' % int(m.group('addr'), 16)) + rel_address=int(m.group('addr'), 16)) def _FindASanLibraries(): @@ -98,16 +97,16 @@ # Maps library -> { address -> [(symbol, location, obj_sym_with_offset)...] } all_symbols = collections.defaultdict(dict) - for library, items in libraries.iteritems(): + for library, items in libraries.items(): libname = _TranslateLibPath(library, asan_libs) - lib_relative_addrs = set([i.rel_address for i in items]) + lib_relative_addrs = set(i.rel_address for i in items) # pylint: disable=no-member - info_dict = symbol.SymbolInformationForSet(libname, - lib_relative_addrs, - True, - cpu_arch=arch) - if info_dict: - all_symbols[library] = info_dict + symbols_by_library = symbol.SymbolInformationForSet(libname, + lib_relative_addrs, + True, + cpu_arch=arch) + if symbols_by_library: + all_symbols[library] = symbols_by_library for log_line in asan_log_lines: m = log_line.parsed @@ -118,33 +117,36 @@ # that usually one wants to display the last list item, not the first. # The code below takes the first, is this the best choice here? s = all_symbols[m.library][m.rel_address][0] - print('%s%s %s %s' % (m.prefix, m.pos, s[0], s[1])) + symbol_name = s[0] + symbol_location = s[1] + print('%s%s %s %s @ \'%s\'' % + (m.prefix, m.pos, hex(m.rel_address), symbol_name, symbol_location)) else: print(log_line.raw) def main(): - parser = optparse.OptionParser() - parser.add_option('-l', '--logcat', - help='File containing adb logcat output with ASan stacks. ' - 'Use stdin if not specified.') - parser.add_option('--output-directory', - help='Path to the root build directory.') - parser.add_option('--arch', default='arm', - help='CPU architecture name') - options, _ = parser.parse_args() + parser = argparse.ArgumentParser() + parser.add_argument('-l', + '--logcat', + help='File containing adb logcat output with ASan ' + 'stacks. Use stdin if not specified.') + parser.add_argument('--output-directory', + help='Path to the root build directory.') + parser.add_argument('--arch', default='arm', help='CPU architecture name') + args = parser.parse_args() - if options.output_directory: - constants.SetOutputDirectory(options.output_directory) + if args.output_directory: + constants.SetOutputDirectory(args.output_directory) # Do an up-front test that the output directory is known. constants.CheckOutputDirectory() - if options.logcat: - asan_input = file(options.logcat, 'r') + if args.logcat: + asan_input = open(args.logcat, 'r') else: asan_input = sys.stdin - _PrintSymbolized(asan_input.readlines(), options.arch) + _PrintSymbolized(asan_input.readlines(), args.arch) if __name__ == "__main__":
diff --git a/build/android/bytecode/BUILD.gn b/build/android/bytecode/BUILD.gn index 36b5432..9478807 100644 --- a/build/android/bytecode/BUILD.gn +++ b/build/android/bytecode/BUILD.gn
@@ -1,21 +1,25 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/android/rules.gni") java_binary("bytecode_processor") { + main_class = "org.chromium.bytecode.ByteCodeProcessor" + wrapper_script_name = "helper/bytecode_processor" + deps = [ ":bytecode_processor_java" ] +} + +java_library("bytecode_processor_java") { sources = [ "java/org/chromium/bytecode/ByteCodeProcessor.java", "java/org/chromium/bytecode/ClassPathValidator.java", "java/org/chromium/bytecode/TypeUtils.java", ] - main_class = "org.chromium.bytecode.ByteCodeProcessor" deps = [ "//third_party/android_deps:org_ow2_asm_asm_java", "//third_party/android_deps:org_ow2_asm_asm_util_java", ] - wrapper_script_name = "helper/bytecode_processor" enable_bytecode_checks = false } @@ -54,3 +58,29 @@ "//third_party/android_deps:org_ow2_asm_asm_util_java", ] } + +java_binary("trace_event_adder") { + main_class = "org.chromium.bytecode.TraceEventAdder" + deps = [ ":trace_event_adder_java" ] + wrapper_script_name = "helper/trace_event_adder" +} + +java_library("trace_event_adder_java") { + visibility = [ ":*" ] + sources = [ + "java/org/chromium/bytecode/ByteCodeRewriter.java", + "java/org/chromium/bytecode/EmptyOverrideGeneratorClassAdapter.java", + "java/org/chromium/bytecode/MethodCheckerClassAdapter.java", + "java/org/chromium/bytecode/MethodDescription.java", + "java/org/chromium/bytecode/ParentMethodCheckerClassAdapter.java", + "java/org/chromium/bytecode/TraceEventAdder.java", + "java/org/chromium/bytecode/TraceEventAdderClassAdapter.java", + "java/org/chromium/bytecode/TraceEventAdderMethodAdapter.java", + ] + deps = [ + ":bytecode_processor_java", + "//third_party/android_deps:org_ow2_asm_asm_commons_java", + "//third_party/android_deps:org_ow2_asm_asm_java", + "//third_party/android_deps:org_ow2_asm_asm_util_java", + ] +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/ByteCodeProcessor.java b/build/android/bytecode/java/org/chromium/bytecode/ByteCodeProcessor.java index b767f4f..4862491 100644 --- a/build/android/bytecode/java/org/chromium/bytecode/ByteCodeProcessor.java +++ b/build/android/bytecode/java/org/chromium/bytecode/ByteCodeProcessor.java
@@ -1,4 +1,4 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. +// Copyright 2017 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/bytecode/java/org/chromium/bytecode/ByteCodeRewriter.java b/build/android/bytecode/java/org/chromium/bytecode/ByteCodeRewriter.java index 3d0d9cd..b97f87d 100644 --- a/build/android/bytecode/java/org/chromium/bytecode/ByteCodeRewriter.java +++ b/build/android/bytecode/java/org/chromium/bytecode/ByteCodeRewriter.java
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -9,6 +9,7 @@ import org.objectweb.asm.ClassWriter; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -31,10 +32,10 @@ if (!inputJar.exists()) { throw new FileNotFoundException("Input jar not found: " + inputJar.getPath()); } - try (InputStream inputStream = new BufferedInputStream(new FileInputStream(inputJar))) { - try (OutputStream outputStream = new FileOutputStream(outputJar)) { - processZip(inputStream, outputStream); - } + + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(inputJar)); + OutputStream outputStream = new FileOutputStream(outputJar)) { + processZip(inputStream, outputStream); } } @@ -42,6 +43,13 @@ protected abstract boolean shouldRewriteClass(String classPath); /** + * Returns true if the class at the given {@link ClassReader} should be rewritten. + */ + protected boolean shouldRewriteClass(ClassReader classReader) { + return true; + } + + /** * Returns the ClassVisitor that should be used to modify the bytecode of class at the given * path in the archive. */ @@ -49,21 +57,35 @@ String classPath, ClassVisitor delegate); private void processZip(InputStream inputStream, OutputStream outputStream) { - try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) { - ZipInputStream zipInputStream = new ZipInputStream(inputStream); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream); + ZipInputStream zipInputStream = new ZipInputStream(inputStream)) { ZipEntry entry; while ((entry = zipInputStream.getNextEntry()) != null) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - boolean handled = processClassEntry(entry, zipInputStream, buffer); + // Get the uncompressed contents of the current zip entry and wrap in an input + // stream. This is done because ZipInputStreams can't be reset so they can only be + // read once, and classes that don't need rewriting need to be read twice, first to + // parse and then to copy. + byte[] currentEntryBytes = zipInputStream.readAllBytes(); + ByteArrayInputStream currentEntryInputStream = + new ByteArrayInputStream(currentEntryBytes); + ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream(); + boolean handled = processClassEntry(entry, currentEntryInputStream, outputBuffer); + + ZipEntry newEntry = new ZipEntry(entry.getName()); + newEntry.setTime(entry.getTime()); + zipOutputStream.putNextEntry(newEntry); if (handled) { - ZipEntry newEntry = new ZipEntry(entry.getName()); - zipOutputStream.putNextEntry(newEntry); - zipOutputStream.write(buffer.toByteArray(), 0, buffer.size()); + zipOutputStream.write(outputBuffer.toByteArray(), 0, outputBuffer.size()); } else { - zipOutputStream.putNextEntry(entry); - zipInputStream.transferTo(zipOutputStream); + // processClassEntry may have advanced currentEntryInputStream, so reset it to + // copy zip entry contents unmodified. + currentEntryInputStream.reset(); + currentEntryInputStream.transferTo(zipOutputStream); } + zipOutputStream.closeEntry(); } + + zipOutputStream.finish(); } catch (IOException e) { throw new RuntimeException(e); } @@ -76,6 +98,9 @@ } try { ClassReader reader = new ClassReader(inputStream); + if (!shouldRewriteClass(reader)) { + return false; + } ClassWriter writer = new ClassWriter(reader, ClassWriter.COMPUTE_FRAMES); ClassVisitor classVisitor = getClassVisitorForClass(entry.getName(), writer); reader.accept(classVisitor, ClassReader.EXPAND_FRAMES);
diff --git a/build/android/bytecode/java/org/chromium/bytecode/ClassPathValidator.java b/build/android/bytecode/java/org/chromium/bytecode/ClassPathValidator.java index 9f45df5..a997bf0 100644 --- a/build/android/bytecode/java/org/chromium/bytecode/ClassPathValidator.java +++ b/build/android/bytecode/java/org/chromium/bytecode/ClassPathValidator.java
@@ -1,4 +1,4 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. +// Copyright 2018 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -71,6 +71,11 @@ // API. return; } + if (className.matches("^android\\b.*")) { + // OS APIs sometime pop up in prebuilts. Rather than force prebuilt targets to set a + // proper alternative_android_sdk_dep, just ignore android.* + return; + } try { classLoader.loadClass(className.replace('/', '.')); } catch (ClassNotFoundException e) {
diff --git a/build/android/bytecode/java/org/chromium/bytecode/EmptyOverrideGeneratorClassAdapter.java b/build/android/bytecode/java/org/chromium/bytecode/EmptyOverrideGeneratorClassAdapter.java new file mode 100644 index 0000000..3cf3a83 --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/EmptyOverrideGeneratorClassAdapter.java
@@ -0,0 +1,104 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +import static org.objectweb.asm.Opcodes.ACC_ABSTRACT; +import static org.objectweb.asm.Opcodes.ACC_INTERFACE; +import static org.objectweb.asm.Opcodes.ALOAD; +import static org.objectweb.asm.Opcodes.ASM7; +import static org.objectweb.asm.Opcodes.ILOAD; +import static org.objectweb.asm.Opcodes.INVOKESPECIAL; +import static org.objectweb.asm.Opcodes.IRETURN; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import java.util.ArrayList; + +class EmptyOverrideGeneratorClassAdapter extends ClassVisitor { + private final ArrayList<MethodDescription> mMethodsToGenerate; + private String mSuperClassName; + private boolean mIsAbstract; + private boolean mIsInterface; + + public EmptyOverrideGeneratorClassAdapter( + ClassVisitor cv, ArrayList<MethodDescription> methodsToGenerate) { + super(ASM7, cv); + mMethodsToGenerate = methodsToGenerate; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, + String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + + mSuperClassName = superName; + mIsAbstract = (access & ACC_ABSTRACT) == ACC_ABSTRACT; + mIsInterface = (access & ACC_INTERFACE) == ACC_INTERFACE; + } + + @Override + public void visitEnd() { + if (mIsAbstract || mIsInterface || mMethodsToGenerate.isEmpty()) { + super.visitEnd(); + return; + } + + for (MethodDescription method : mMethodsToGenerate) { + if (!method.shouldCreateOverride) { + continue; + } + + MethodVisitor mv = super.visitMethod( + method.access, method.methodName, method.description, null, null); + writeOverrideCode(mv, method.access, method.methodName, method.description); + } + + super.visitEnd(); + } + + /** + * Writes code to a method to call that method's parent implementation. + * <pre> + * {@code + * // Calling writeOverrideCode(mv, ACC_PUBLIC, "doFoo", "(Ljava/lang/String;)I") writes the + * following method body: public int doFoo(String arg){ return super.doFoo(arg); + * } + * } + * </pre> + * + * This will be rewritten later by TraceEventAdderClassAdapter to wrap the body in a trace + * event. + */ + private void writeOverrideCode( + MethodVisitor mv, final int access, final String name, final String descriptor) { + assert access != 0; + Type[] argTypes = Type.getArgumentTypes(descriptor); + Type returnType = Type.getReturnType(descriptor); + + mv.visitCode(); + + // Variable 0 contains `this`, load it into the operand stack. + mv.visitVarInsn(ALOAD, 0); + + // Variables 1..n contain all arguments, load them all into the operand stack. + int i = 1; + for (Type arg : argTypes) { + // getOpcode(ILOAD) returns the ILOAD equivalent to the current argument's type. + mv.visitVarInsn(arg.getOpcode(ILOAD), i); + i += arg.getSize(); + } + + // Call the parent class method with the same arguments. + mv.visitMethodInsn(INVOKESPECIAL, mSuperClassName, name, descriptor, false); + + // Return the result. + mv.visitInsn(returnType.getOpcode(IRETURN)); + + mv.visitMaxs(0, 0); + mv.visitEnd(); + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java b/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java index a40f39c..0966be0 100644 --- a/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java +++ b/build/android/bytecode/java/org/chromium/bytecode/FragmentActivityReplacer.java
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -13,6 +13,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Method; /** * Java application that modifies Fragment.getActivity() to return an Activity instead of a @@ -75,11 +76,29 @@ * the replaced method. */ private static class InvocationReplacer extends ClassVisitor { + /** + * A ClassLoader that will resolve R classes to Object. + * + * R won't be in our classpath, and we don't access any information about them, so resolving + * it to a dummy value is fine. + */ + private static class ResourceStubbingClassLoader extends ClassLoader { + @Override + protected Class<?> findClass(String name) throws ClassNotFoundException { + if (name.matches(".*\\.R(\\$.+)?")) { + return Object.class; + } + return super.findClass(name); + } + } + private final boolean mSingleAndroidX; + private final ClassLoader mClassLoader; private InvocationReplacer(ClassVisitor baseVisitor, boolean singleAndroidX) { super(Opcodes.ASM7, baseVisitor); mSingleAndroidX = singleAndroidX; + mClassLoader = new ResourceStubbingClassLoader(); } @Override @@ -90,6 +109,28 @@ @Override public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + // Change the return type of getActivity and replaceActivity. + if (isActivityGetterInvocation(opcode, owner, name, descriptor)) { + super.visitMethodInsn( + opcode, owner, name, NEW_METHOD_DESCRIPTOR, isInterface); + if (mSingleAndroidX) { + super.visitTypeInsn( + Opcodes.CHECKCAST, "androidx/fragment/app/FragmentActivity"); + } + } else if (isDowncastableFragmentActivityMethodInvocation( + opcode, owner, name, descriptor)) { + // Replace FragmentActivity.foo() with Activity.foo() to fix cases where the + // above code changed the getActivity return type. See the + // isDowncastableFragmentActivityMethodInvocation documentation for details. + super.visitMethodInsn( + opcode, "android/app/Activity", name, descriptor, isInterface); + } else { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + } + + private boolean isActivityGetterInvocation( + int opcode, String owner, String name, String descriptor) { boolean isFragmentGetActivity = name.equals(GET_ACTIVITY_METHOD_NAME) && descriptor.equals(OLD_METHOD_DESCRIPTOR) && isFragmentSubclass(owner); @@ -100,39 +141,63 @@ name.equals(GET_LIFECYCLE_ACTIVITY_METHOD_NAME) && descriptor.equals(OLD_METHOD_DESCRIPTOR) && owner.equals(SUPPORT_LIFECYCLE_FRAGMENT_IMPL_BINARY_NAME); - if ((opcode == Opcodes.INVOKEVIRTUAL || opcode == Opcodes.INVOKESPECIAL) + return (opcode == Opcodes.INVOKEVIRTUAL || opcode == Opcodes.INVOKESPECIAL) && (isFragmentGetActivity || isFragmentRequireActivity - || isSupportLifecycleFragmentImplGetLifecycleActivity)) { - super.visitMethodInsn( - opcode, owner, name, NEW_METHOD_DESCRIPTOR, isInterface); - if (mSingleAndroidX) { - super.visitTypeInsn( - Opcodes.CHECKCAST, "androidx/fragment/app/FragmentActivity"); + || isSupportLifecycleFragmentImplGetLifecycleActivity); + } + + /** + * Returns true if the given method belongs to FragmentActivity, and also exists on + * Activity. + * + * The Java code `requireActivity().getClassLoader()` will compile to the following + * bytecode: + * aload_0 + * // Method requireActivity:()Landroid/app/Activity; + * invokevirtual #n + * // Method androidx/fragment/app/FragmentActivity.getClassLoader:()LClassLoader; + * invokevirtual #m + * + * The second invokevirtual instruction doesn't typecheck because the + * requireActivity() return type was changed from FragmentActivity to Activity. Note + * that this is only an issue when validating the bytecode on the JVM, not in + * Dalvik, so while the above code works on device, it fails in robolectric tests. + * + * To fix the example above, we'd replace the second invokevirtual call with a call + * to android/app/Activity.getClassLoader:()Ljava/lang/ClassLoader. In general, any + * call to FragmentActivity.foo, where foo also exists on Activity, will be replaced + * with a call to Activity.foo. Activity.foo will still resolve to + * FragmentActivity.foo at runtime, while typechecking in robolectric tests. + */ + private boolean isDowncastableFragmentActivityMethodInvocation( + int opcode, String owner, String name, String descriptor) { + // Return if this isn't an invoke instruction on a FragmentActivity. + if (!(opcode == Opcodes.INVOKEVIRTUAL || opcode == Opcodes.INVOKESPECIAL) + || !owner.equals("androidx/fragment/app/FragmentActivity")) { + return false; + } + try { + // Check if the method exists in Activity. + Class<?> activity = mClassLoader.loadClass("android.app.Activity"); + for (Method activityMethod : activity.getMethods()) { + if (activityMethod.getName().equals(name) + && Type.getMethodDescriptor(activityMethod) + .equals(descriptor)) { + return true; + } } - } else { - super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + return false; + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); } } private boolean isFragmentSubclass(String internalType) { - // Look up classes with a ClassLoader that will resolve any R classes to Object. - // This is fine in this case as resource classes shouldn't be in the class - // hierarchy of any Fragments. - ClassLoader resourceStubbingClassLoader = new ClassLoader() { - @Override - protected Class<?> findClass(String name) throws ClassNotFoundException { - if (name.matches(".*\\.R(\\$.+)?")) { - return Object.class; - } - return super.findClass(name); - } - }; - // This doesn't use Class#isAssignableFrom to avoid us needing to load // AndroidX's Fragment class, which may not be on the classpath. try { String binaryName = Type.getObjectType(internalType).getClassName(); - Class<?> clazz = resourceStubbingClassLoader.loadClass(binaryName); + Class<?> clazz = mClassLoader.loadClass(binaryName); while (clazz != null) { if (clazz.getName().equals("androidx.fragment.app.Fragment")) { return true;
diff --git a/build/android/bytecode/java/org/chromium/bytecode/MethodCheckerClassAdapter.java b/build/android/bytecode/java/org/chromium/bytecode/MethodCheckerClassAdapter.java new file mode 100644 index 0000000..6794a77 --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/MethodCheckerClassAdapter.java
@@ -0,0 +1,144 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +import static org.objectweb.asm.ClassReader.EXPAND_FRAMES; +import static org.objectweb.asm.Opcodes.ACC_ABSTRACT; +import static org.objectweb.asm.Opcodes.ACC_INTERFACE; +import static org.objectweb.asm.Opcodes.ASM7; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; + +/** + * This ClassVisitor verifies that a class and its methods are suitable for rewriting. + * Given a class and a list of methods it performs the following checks: + * 1. Class is subclass of a class that we want to trace. + * 2. Class is not abstract or an interface. + * + * For each method provided in {@code methodsToCheck}: + * If the class overrides the method then we can rewrite it directly. + * If the class doesn't override the method then we can generate an override with {@link + * EmptyOverrideGeneratorClassAdapter}, but first we must check if the parent method is private or + * final using {@link ParentMethodCheckerClassAdapter}. + * + * This adapter modifies the provided method list to indicate which methods should be overridden or + * skipped. + */ +class MethodCheckerClassAdapter extends ClassVisitor { + private static final String VIEW_CLASS_DESCRIPTOR = "android/view/View"; + private static final String ANIMATOR_UPDATE_LISTENER_CLASS_DESCRIPTOR = + "android/animation/ValueAnimator$AnimatorUpdateListener"; + private static final String ANIMATOR_LISTENER_CLASS_DESCRIPTOR = + "android/animation/Animator$AnimatorListener"; + + private final ArrayList<MethodDescription> mMethodsToCheck; + private final ClassLoader mJarClassLoader; + private String mSuperName; + + public MethodCheckerClassAdapter( + ArrayList<MethodDescription> methodsToCheck, ClassLoader jarClassLoader) { + super(ASM7); + mMethodsToCheck = methodsToCheck; + mJarClassLoader = jarClassLoader; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, + String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + + mSuperName = superName; + + boolean isAbstract = (access & ACC_ABSTRACT) == ACC_ABSTRACT; + boolean isInterface = (access & ACC_INTERFACE) == ACC_INTERFACE; + + if (isAbstract || isInterface || !shouldTraceClass(name)) { + mMethodsToCheck.clear(); + return; + } + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (mMethodsToCheck.isEmpty()) { + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + + for (MethodDescription method : mMethodsToCheck) { + if (method.methodName.equals(name) && method.description.equals(descriptor)) { + method.shouldCreateOverride = false; + } + } + + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + + @Override + public void visitEnd() { + if (mMethodsToCheck.isEmpty()) { + super.visitEnd(); + return; + } + + boolean areAnyUncheckedMethods = false; + + for (MethodDescription method : mMethodsToCheck) { + if (method.shouldCreateOverride == null) { + areAnyUncheckedMethods = true; + break; + } + } + + if (areAnyUncheckedMethods) { + checkParentClass(mSuperName, mMethodsToCheck, mJarClassLoader); + } + + super.visitEnd(); + } + + private boolean shouldTraceClass(String desc) { + Class clazz = getClass(desc); + return isClassDerivedFrom(clazz, VIEW_CLASS_DESCRIPTOR) + || isClassDerivedFrom(clazz, ANIMATOR_UPDATE_LISTENER_CLASS_DESCRIPTOR) + || isClassDerivedFrom(clazz, ANIMATOR_LISTENER_CLASS_DESCRIPTOR); + } + + private boolean isClassDerivedFrom(Class clazz, String classDescriptor) { + Class superClass = getClass(classDescriptor); + if (clazz == null || superClass == null) return false; + return superClass.isAssignableFrom(clazz); + } + + private Class getClass(String desc) { + try { + return mJarClassLoader.loadClass(desc.replace('/', '.')); + } catch (ClassNotFoundException | NoClassDefFoundError | IllegalAccessError e) { + return null; + } + } + + static void checkParentClass(String superClassName, ArrayList<MethodDescription> methodsToCheck, + ClassLoader jarClassLoader) { + try { + ClassReader cr = new ClassReader(getClassAsStream(jarClassLoader, superClassName)); + ParentMethodCheckerClassAdapter parentChecker = + new ParentMethodCheckerClassAdapter(methodsToCheck, jarClassLoader); + cr.accept(parentChecker, EXPAND_FRAMES); + } catch (IOException ex) { + // Ignore errors in case class can't be loaded. + } + } + + private static InputStream getClassAsStream(ClassLoader jarClassLoader, String desc) { + return jarClassLoader.getResourceAsStream(desc.replace('.', '/') + ".class"); + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/MethodDescription.java b/build/android/bytecode/java/org/chromium/bytecode/MethodDescription.java new file mode 100644 index 0000000..26717c0 --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/MethodDescription.java
@@ -0,0 +1,20 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +class MethodDescription { + public final String methodName; + public final String description; + public final int access; + public Boolean shouldCreateOverride; + + public MethodDescription(String methodName, String description, int access) { + this.methodName = methodName; + this.description = description; + this.access = access; + // A null value means we haven't checked the method. + this.shouldCreateOverride = null; + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/ParentMethodCheckerClassAdapter.java b/build/android/bytecode/java/org/chromium/bytecode/ParentMethodCheckerClassAdapter.java new file mode 100644 index 0000000..4656c34 --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/ParentMethodCheckerClassAdapter.java
@@ -0,0 +1,109 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +import static org.objectweb.asm.Opcodes.ACC_FINAL; +import static org.objectweb.asm.Opcodes.ACC_PRIVATE; +import static org.objectweb.asm.Opcodes.ACC_PROTECTED; +import static org.objectweb.asm.Opcodes.ACC_PUBLIC; +import static org.objectweb.asm.Opcodes.ASM7; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; + +import java.util.ArrayList; + +/** + * This ClassVisitor checks if the given class overrides methods on {@code methodsToCheck}, and if + * so it determines whether they can be overridden by a child class. If at the end any unchecked + * methods remain then we recurse on the class's superclass. + */ +class ParentMethodCheckerClassAdapter extends ClassVisitor { + private static final String OBJECT_CLASS_DESCRIPTOR = "java/lang/Object"; + + private final ArrayList<MethodDescription> mMethodsToCheck; + private final ClassLoader mJarClassLoader; + private String mSuperName; + private boolean mIsCheckingObjectClass; + + public ParentMethodCheckerClassAdapter( + ArrayList<MethodDescription> methodsToCheck, ClassLoader jarClassLoader) { + super(ASM7); + mMethodsToCheck = methodsToCheck; + mJarClassLoader = jarClassLoader; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, + String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + + if (name.equals(OBJECT_CLASS_DESCRIPTOR)) { + mIsCheckingObjectClass = true; + return; + } + + mSuperName = superName; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (mIsCheckingObjectClass) { + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + + for (MethodDescription methodToCheck : mMethodsToCheck) { + if (methodToCheck.shouldCreateOverride != null || !methodToCheck.methodName.equals(name) + || !methodToCheck.description.equals(descriptor)) { + continue; + } + + // This class contains methodToCheck. + boolean isMethodPrivate = (access & ACC_PRIVATE) == ACC_PRIVATE; + boolean isMethodFinal = (access & ACC_FINAL) == ACC_FINAL; + boolean isMethodPackagePrivate = + (access & (ACC_PUBLIC | ACC_PROTECTED | ACC_PRIVATE)) == 0; + + // If the method is private or final then don't create an override. + methodToCheck.shouldCreateOverride = + !isMethodPrivate && !isMethodFinal && !isMethodPackagePrivate; + } + + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + + @Override + public void visitEnd() { + if (mIsCheckingObjectClass) { + // We support tracing methods that are defined in classes that are derived from View, + // but are not defined in View itself. If we've reached the Object class in the + // hierarchy, it means the method doesn't exist in this hierarchy, so don't override it, + // and stop looking for it. + for (MethodDescription method : mMethodsToCheck) { + if (method.shouldCreateOverride == null) { + method.shouldCreateOverride = false; + } + } + return; + } + + boolean areAnyUncheckedMethods = false; + + for (MethodDescription method : mMethodsToCheck) { + if (method.shouldCreateOverride == null) { + areAnyUncheckedMethods = true; + break; + } + } + + if (areAnyUncheckedMethods) { + MethodCheckerClassAdapter.checkParentClass( + mSuperName, mMethodsToCheck, mJarClassLoader); + } + + super.visitEnd(); + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdder.java b/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdder.java new file mode 100644 index 0000000..4a85159 --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdder.java
@@ -0,0 +1,109 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Opcodes; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Java application that modifies all implementations of "draw", "onMeasure" and "onLayout" on all + * {@link android.view.View} subclasses to wrap them in trace events. + */ +public class TraceEventAdder extends ByteCodeRewriter { + private final ClassLoader mClassPathJarsClassLoader; + private ArrayList<MethodDescription> mMethodsToTrace; + + public static void main(String[] args) throws IOException { + // Invoke this script using //build/android/gyp/trace_event_bytecode_rewriter.py + + if (args.length < 2) { + System.err.println("Expected arguments: <':' separated list with N input jar paths> " + + "<':' separated list with N output jar paths>"); + System.exit(1); + } + + String[] inputJars = args[0].split(":"); + String[] outputJars = args[1].split(":"); + + assert inputJars.length + == outputJars.length : "Input and output lists are not the same length. Inputs: " + + inputJars.length + " Outputs: " + outputJars.length; + + // outputJars[n] must be the same as inputJars[n] but with a suffix, validate this. + for (int i = 0; i < inputJars.length; i++) { + File inputJarPath = new File(inputJars[i]); + String inputJarFilename = inputJarPath.getName(); + File outputJarPath = new File(outputJars[i]); + + String inputFilenameNoExtension = + inputJarFilename.substring(0, inputJarFilename.lastIndexOf(".jar")); + + assert outputJarPath.getName().startsWith(inputFilenameNoExtension); + } + + ArrayList<String> classPathJarsPaths = new ArrayList<>(); + classPathJarsPaths.addAll(Arrays.asList(inputJars)); + ClassLoader classPathJarsClassLoader = ByteCodeProcessor.loadJars(classPathJarsPaths); + + TraceEventAdder adder = new TraceEventAdder(classPathJarsClassLoader); + for (int i = 0; i < inputJars.length; i++) { + adder.rewrite(new File(inputJars[i]), new File(outputJars[i])); + } + } + + public TraceEventAdder(ClassLoader classPathJarsClassLoader) { + mClassPathJarsClassLoader = classPathJarsClassLoader; + } + + @Override + protected boolean shouldRewriteClass(String classPath) { + return true; + } + + @Override + protected boolean shouldRewriteClass(ClassReader classReader) { + mMethodsToTrace = new ArrayList<>(Arrays.asList( + // Methods on View.java + new MethodDescription( + "dispatchTouchEvent", "(Landroid/view/MotionEvent;)Z", Opcodes.ACC_PUBLIC), + new MethodDescription("draw", "(Landroid/graphics/Canvas;)V", Opcodes.ACC_PUBLIC), + new MethodDescription("onMeasure", "(II)V", Opcodes.ACC_PROTECTED), + new MethodDescription("onLayout", "(ZIIII)V", Opcodes.ACC_PROTECTED), + // Methods on RecyclerView.java in AndroidX + new MethodDescription("scrollStep", "(II[I)V", 0), + // Methods on Animator.AnimatorListener + new MethodDescription( + "onAnimationStart", "(Landroid/animation/Animator;)V", Opcodes.ACC_PUBLIC), + new MethodDescription( + "onAnimationEnd", "(Landroid/animation/Animator;)V", Opcodes.ACC_PUBLIC), + // Methods on ValueAnimator.AnimatorUpdateListener + new MethodDescription("onAnimationUpdate", "(Landroid/animation/ValueAnimator;)V", + Opcodes.ACC_PUBLIC))); + + // This adapter will modify mMethodsToTrace to indicate which methods already exist in the + // class and which ones need to be overridden. In case the class is not an Android view + // we'll clear the list and skip rewriting. + MethodCheckerClassAdapter methodChecker = + new MethodCheckerClassAdapter(mMethodsToTrace, mClassPathJarsClassLoader); + + classReader.accept(methodChecker, ClassReader.EXPAND_FRAMES); + + return !mMethodsToTrace.isEmpty(); + } + + @Override + protected ClassVisitor getClassVisitorForClass(String classPath, ClassVisitor delegate) { + ClassVisitor chain = new TraceEventAdderClassAdapter(delegate, mMethodsToTrace); + chain = new EmptyOverrideGeneratorClassAdapter(chain, mMethodsToTrace); + + return chain; + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdderClassAdapter.java b/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdderClassAdapter.java new file mode 100644 index 0000000..f2d03fb --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdderClassAdapter.java
@@ -0,0 +1,47 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +import static org.objectweb.asm.Opcodes.ASM7; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; + +import java.util.ArrayList; + +/** + * A ClassVisitor for adding TraceEvent.begin and TraceEvent.end methods to any methods specified in + * a list. + */ +class TraceEventAdderClassAdapter extends ClassVisitor { + private final ArrayList<MethodDescription> mMethodsToTrace; + private String mShortClassName; + + TraceEventAdderClassAdapter(ClassVisitor visitor, ArrayList<MethodDescription> methodsToTrace) { + super(ASM7, visitor); + mMethodsToTrace = methodsToTrace; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, + String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + mShortClassName = name.substring(name.lastIndexOf('/') + 1); + } + + @Override + public MethodVisitor visitMethod(final int access, final String name, String desc, + String signature, String[] exceptions) { + MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); + + for (MethodDescription method : mMethodsToTrace) { + if (method.methodName.equals(name) && method.description.equals(desc)) { + return new TraceEventAdderMethodAdapter(mv, mShortClassName, name); + } + } + + return mv; + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdderMethodAdapter.java b/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdderMethodAdapter.java new file mode 100644 index 0000000..11f2a27 --- /dev/null +++ b/build/android/bytecode/java/org/chromium/bytecode/TraceEventAdderMethodAdapter.java
@@ -0,0 +1,83 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.bytecode; + +import static org.objectweb.asm.Opcodes.ASM7; +import static org.objectweb.asm.Opcodes.ATHROW; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; +import static org.objectweb.asm.Opcodes.IRETURN; +import static org.objectweb.asm.Opcodes.RETURN; + +import static org.chromium.bytecode.TypeUtils.STRING; +import static org.chromium.bytecode.TypeUtils.VOID; + +import org.objectweb.asm.MethodVisitor; + +/** + * MethodVisitor that wraps all code in TraceEvent.begin and TraceEvent.end calls. TraceEvent.end + * calls are added on all returns and thrown exceptions. + * + * Example: + * <pre> + * {@code + * int methodToTrace(String foo){ + * + * //Line added by rewriter: + * TraceEvent.begin("ClassName.methodToTrace"); + * + * if(foo == null){ + * //Line added by rewriter: + * TraceEvent.end("ClassName.methodToTrace"); + * + * throw new Exception(); + * } + * else if(foo.equals("Two")){ + * //Line added by rewriter: + * TraceEvent.end("ClassName.methodToTrace"); + * + * return 2; + * } + * + * //Line added by rewriter: + * TraceEvent.end("ClassName.methodToTrace"); + * + * return 0; + * } + * } + * </pre> + * + */ +class TraceEventAdderMethodAdapter extends MethodVisitor { + private static final String TRACE_EVENT_DESCRIPTOR = "org/chromium/base/TraceEvent"; + private static final String TRACE_EVENT_SIGNATURE = TypeUtils.getMethodDescriptor(VOID, STRING); + private final String mEventName; + + public TraceEventAdderMethodAdapter( + MethodVisitor methodVisitor, String shortClassName, String methodName) { + super(ASM7, methodVisitor); + + mEventName = shortClassName + "." + methodName; + } + + @Override + public void visitCode() { + super.visitCode(); + + mv.visitLdcInsn(mEventName); + mv.visitMethodInsn( + INVOKESTATIC, TRACE_EVENT_DESCRIPTOR, "begin", TRACE_EVENT_SIGNATURE, false); + } + + @Override + public void visitInsn(int opcode) { + if ((opcode >= IRETURN && opcode <= RETURN) || opcode == ATHROW) { + mv.visitLdcInsn(mEventName); + mv.visitMethodInsn( + INVOKESTATIC, TRACE_EVENT_DESCRIPTOR, "end", TRACE_EVENT_SIGNATURE, false); + } + + mv.visitInsn(opcode); + } +}
diff --git a/build/android/bytecode/java/org/chromium/bytecode/TypeUtils.java b/build/android/bytecode/java/org/chromium/bytecode/TypeUtils.java index ed2dc2d..e62a912 100644 --- a/build/android/bytecode/java/org/chromium/bytecode/TypeUtils.java +++ b/build/android/bytecode/java/org/chromium/bytecode/TypeUtils.java
@@ -1,4 +1,4 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. +// Copyright 2017 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/chromium_annotations.flags b/build/android/chromium_annotations.flags new file mode 100644 index 0000000..e3f7afa --- /dev/null +++ b/build/android/chromium_annotations.flags
@@ -0,0 +1,79 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Contains flags related to annotations in //build/android that can be safely +# shared with Cronet, and thus would be appropriate for third-party apps to +# include. + +# Keep all annotation related attributes that can affect runtime +-keepattributes RuntimeVisible*Annotations +-keepattributes AnnotationDefault + +# Keep the annotations, because if we don't, the ProGuard rules that use them +# will not be respected. These classes then show up in our final dex, which we +# do not want - see crbug.com/628226. +-keep @interface org.chromium.base.annotations.AccessedByNative +-keep @interface org.chromium.base.annotations.CalledByNative +-keep @interface org.chromium.base.annotations.CalledByNativeUnchecked +-keep @interface org.chromium.build.annotations.DoNotInline +-keep @interface org.chromium.build.annotations.UsedByReflection +-keep @interface org.chromium.build.annotations.IdentifierNameString + +# Keeps for class level annotations. +-keep,allowaccessmodification @org.chromium.build.annotations.UsedByReflection class ** {} + +# Keeps for method level annotations. +-keepclasseswithmembers,allowaccessmodification class ** { + @org.chromium.base.annotations.AccessedByNative <fields>; +} +-keepclasseswithmembers,includedescriptorclasses,allowaccessmodification class ** { + @org.chromium.base.annotations.CalledByNative <methods>; +} +-keepclasseswithmembers,includedescriptorclasses,allowaccessmodification class ** { + @org.chromium.base.annotations.CalledByNativeUnchecked <methods>; +} +-keepclasseswithmembers,allowaccessmodification class ** { + @org.chromium.build.annotations.UsedByReflection <methods>; +} +-keepclasseswithmembers,allowaccessmodification class ** { + @org.chromium.build.annotations.UsedByReflection <fields>; +} + +# Never inline classes, methods, or fields with this annotation, but allow +# shrinking and obfuscation. +# Relevant to fields when they are needed to store strong references to objects +# that are held as weak references by native code. +-if @org.chromium.build.annotations.DoNotInline class * { + *** *(...); +} +-keep,allowobfuscation,allowaccessmodification class <1> { + *** <2>(...); +} +-keepclassmembers,allowobfuscation,allowaccessmodification class * { + @org.chromium.build.annotations.DoNotInline <methods>; +} +-keepclassmembers,allowobfuscation,allowaccessmodification class * { + @org.chromium.build.annotations.DoNotInline <fields>; +} + +-alwaysinline class * { + @org.chromium.build.annotations.AlwaysInline *; +} + +# Keep all logs (Log.VERBOSE = 2). R8 does not allow setting to 0. +-maximumremovedandroidloglevel 1 class ** { + @org.chromium.build.annotations.DoNotStripLogs <methods>; +} +-maximumremovedandroidloglevel 1 @org.chromium.build.annotations.DoNotStripLogs class ** { + <methods>; +} + +# Never merge classes horizontally or vertically with this annotation. +# Relevant to classes being used as a key in maps or sets. +-keep,allowaccessmodification,allowobfuscation,allowshrinking @org.chromium.build.annotations.DoNotClassMerge class * + +# Mark members annotated with IdentifierNameString as identifier name strings +-identifiernamestring class * { + @org.chromium.build.annotations.IdentifierNameString *; +}
diff --git a/build/android/convert_dex_profile.py b/build/android/convert_dex_profile.py index f9fdeb6..13a48ed 100755 --- a/build/android/convert_dex_profile.py +++ b/build/android/convert_dex_profile.py
@@ -1,11 +1,12 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse import collections +import functools import logging import re import subprocess @@ -66,7 +67,9 @@ 'double': 'D' } -class Method(object): + +@functools.total_ordering +class Method: def __init__(self, name, class_name, param_types=None, return_type=None): self.name = name self.class_name = class_name @@ -81,16 +84,23 @@ return 'Method<{}->{}({}){}>'.format(self.class_name, self.name, self.param_types or '', self.return_type or '') - def __cmp__(self, other): - return cmp((self.class_name, self.name, self.param_types, self.return_type), - (other.class_name, other.name, other.param_types, other.return_type)) + @staticmethod + def serialize(method): + return (method.class_name, method.name, method.param_types, + method.return_type) + + def __eq__(self, other): + return self.serialize(self) == self.serialize(other) + + def __lt__(self, other): + return self.serialize(self) < self.serialize(other) def __hash__(self): # only hash name and class_name since other fields may not be set yet. return hash((self.name, self.class_name)) -class Class(object): +class Class: def __init__(self, name): self.name = name self._methods = [] @@ -149,13 +159,13 @@ logging.warning('ambigous methods in dex %s at lines %s in class "%s"', found_methods, hint_lines, self.name) return found_methods - else: - logging.warning('No method named "%s" in class "%s" is ' - 'mapped to lines %s', method_name, self.name, hint_lines) - return None + logging.warning( + 'No method named "%s" in class "%s" is ' + 'mapped to lines %s', method_name, self.name, hint_lines) + return None -class Profile(object): +class Profile: def __init__(self): # {Method: set(char)} self._methods = collections.defaultdict(set) @@ -178,7 +188,7 @@ output_profile.write(line) -class ProguardMapping(object): +class ProguardMapping: def __init__(self): # {Method: set(Method)} self._method_mapping = collections.defaultdict(set) @@ -214,7 +224,8 @@ class MalformedLineException(Exception): def __init__(self, message, line_number): - super(MalformedLineException, self).__init__(message) + super().__init__(message) + self.message = message self.line_number = line_number def __str__(self): @@ -230,7 +241,8 @@ def _RunDexDump(dexdump_path, dex_file_path): - return subprocess.check_output([dexdump_path, dex_file_path]).splitlines() + return subprocess.check_output([dexdump_path, + dex_file_path]).decode('utf-8').splitlines() def _ReadFile(file_path):
diff --git a/build/android/convert_dex_profile_tests.py b/build/android/convert_dex_profile_tests.py old mode 100644 new mode 100755 index 0ddc5ce..915d263 --- a/build/android/convert_dex_profile_tests.py +++ b/build/android/convert_dex_profile_tests.py
@@ -1,4 +1,5 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -41,36 +42,36 @@ positions : 0x0001 line=310 0x0057 line=313 - locals : + locals : #1 : (in La;) name : '<init>' type : '()V' positions : - locals : + locals : Virtual methods - #0 : (in La;) name : 'a' type : '(Ljava/lang/String;)I' - positions : + positions : 0x0000 line=2 0x0003 line=3 0x001b line=8 - locals : - 0x0000 - 0x0021 reg=3 this La; + locals : + 0x0000 - 0x0021 reg=3 this La; #1 : (in La;) name : 'a' type : '(Ljava/lang/Object;)I' - positions : + positions : 0x0000 line=8 0x0003 line=9 - locals : - 0x0000 - 0x0021 reg=3 this La; + locals : + 0x0000 - 0x0021 reg=3 this La; #2 : (in La;) name : 'b' type : '()La;' - positions : + positions : 0x0000 line=1 - locals : + locals : """ # pylint: disable=line-too-long @@ -109,36 +110,36 @@ positions : 0x0001 line=310 0x0057 line=313 - locals : + locals : #1 : (in La;) name : '<init>' type : '()V' positions : - locals : + locals : Virtual methods - #0 : (in La;) name : 'a' type : '(Ljava/lang/String;)I' - positions : + positions : 0x0000 line=2 0x0003 line=3 0x001b line=8 - locals : - 0x0000 - 0x0021 reg=3 this La; + locals : + 0x0000 - 0x0021 reg=3 this La; #1 : (in La;) name : 'c' type : '(Ljava/lang/Object;)I' - positions : + positions : 0x0000 line=8 0x0003 line=9 - locals : - 0x0000 - 0x0021 reg=3 this La; + locals : + 0x0000 - 0x0021 reg=3 this La; #2 : (in La;) name : 'b' type : '()La;' - positions : + positions : 0x0000 line=1 - locals : + locals : """ # pylint: disable=line-too-long @@ -167,14 +168,14 @@ dex = cp.ProcessDex(DEX_DUMP.splitlines()) self.assertIsNotNone(dex['a']) - self.assertEquals(len(dex['a'].FindMethodsAtLine('<clinit>', 311, 313)), 1) - self.assertEquals(len(dex['a'].FindMethodsAtLine('<clinit>', 309, 315)), 1) + self.assertEqual(len(dex['a'].FindMethodsAtLine('<clinit>', 311, 313)), 1) + self.assertEqual(len(dex['a'].FindMethodsAtLine('<clinit>', 309, 315)), 1) clinit = dex['a'].FindMethodsAtLine('<clinit>', 311, 313)[0] - self.assertEquals(clinit.name, '<clinit>') - self.assertEquals(clinit.return_type, 'V') - self.assertEquals(clinit.param_types, 'Ljava/lang/String;') + self.assertEqual(clinit.name, '<clinit>') + self.assertEqual(clinit.return_type, 'V') + self.assertEqual(clinit.param_types, 'Ljava/lang/String;') - self.assertEquals(len(dex['a'].FindMethodsAtLine('a', 8, None)), 2) + self.assertEqual(len(dex['a'].FindMethodsAtLine('a', 8, None)), 2) self.assertIsNone(dex['a'].FindMethodsAtLine('a', 100, None)) # pylint: disable=protected-access @@ -183,7 +184,7 @@ mapping, reverse = cp.ProcessProguardMapping( PROGUARD_MAPPING.splitlines(), dex) - self.assertEquals('La;', reverse.GetClassMapping('Lorg/chromium/Original;')) + self.assertEqual('La;', reverse.GetClassMapping('Lorg/chromium/Original;')) getInstance = cp.Method( 'getInstance', 'Lorg/chromium/Original;', '', 'Lorg/chromium/Original;') @@ -196,7 +197,7 @@ mapped = mapping.GetMethodMapping( cp.Method('a', 'La;', 'Ljava/lang/String;', 'I')) - self.assertEquals(len(mapped), 2) + self.assertEqual(len(mapped), 2) self.assertIn(getInstance, mapped) self.assertNotIn(subclassInit, mapped) self.assertNotIn( @@ -205,18 +206,18 @@ mapped = mapping.GetMethodMapping( cp.Method('a', 'La;', 'Ljava/lang/Object;', 'I')) - self.assertEquals(len(mapped), 1) + self.assertEqual(len(mapped), 1) self.assertIn(getInstance, mapped) mapped = mapping.GetMethodMapping(cp.Method('b', 'La;', '', 'La;')) - self.assertEquals(len(mapped), 1) + self.assertEqual(len(mapped), 1) self.assertIn(another, mapped) - for from_method, to_methods in mapping._method_mapping.iteritems(): + for from_method, to_methods in mapping._method_mapping.items(): for to_method in to_methods: self.assertIn(from_method, reverse.GetMethodMapping(to_method)) - for from_class, to_class in mapping._class_mapping.iteritems(): - self.assertEquals(from_class, reverse.GetClassMapping(to_class)) + for from_class, to_class in mapping._class_mapping.items(): + self.assertEqual(from_class, reverse.GetClassMapping(to_class)) def testProcessProfile(self): dex = cp.ProcessDex(DEX_DUMP.splitlines()) @@ -234,9 +235,9 @@ self.assertIn(initialize, profile._methods) self.assertIn(another, profile._methods) - self.assertEquals(profile._methods[getInstance], set(['H', 'S', 'P'])) - self.assertEquals(profile._methods[initialize], set(['H', 'P'])) - self.assertEquals(profile._methods[another], set(['P'])) + self.assertEqual(profile._methods[getInstance], set(['H', 'S', 'P'])) + self.assertEqual(profile._methods[initialize], set(['H', 'P'])) + self.assertEqual(profile._methods[another], set(['P'])) def testEndToEnd(self): dex = cp.ProcessDex(DEX_DUMP.splitlines()) @@ -247,7 +248,7 @@ profile.WriteToFile(temp.name) with open(temp.name, 'r') as f: for a, b in zip(sorted(f), sorted(UNOBFUSCATED_PROFILE.splitlines())): - self.assertEquals(a.strip(), b.strip()) + self.assertEqual(a.strip(), b.strip()) def testObfuscateProfile(self): with build_utils.TempDir() as temp_dir: @@ -269,7 +270,7 @@ obfuscated_profile = sorted(obfuscated_file.readlines()) for a, b in zip( sorted(OBFUSCATED_PROFILE_2.splitlines()), obfuscated_profile): - self.assertEquals(a.strip(), b.strip()) + self.assertEqual(a.strip(), b.strip()) if __name__ == '__main__':
diff --git a/build/android/dcheck_is_off.flags b/build/android/dcheck_is_off.flags index 78b9cc2..5718c27 100644 --- a/build/android/dcheck_is_off.flags +++ b/build/android/dcheck_is_off.flags
@@ -1,17 +1,12 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # Contains flags that are applied only when ENABLE_DCHECK=false. --checkdiscard @org.chromium.base.annotations.CheckDiscard class ** { +-checkdiscard @org.chromium.build.annotations.CheckDiscard class ** { *; } -checkdiscard class ** { - @org.chromium.base.annotations.CheckDiscard *; -} - -# Ensure @RemovableInRelease actually works. --checkdiscard class ** { - @org.chromium.base.annotations.RemovableInRelease *; + @org.chromium.build.annotations.CheckDiscard *; }
diff --git a/build/android/devil_chromium.json b/build/android/devil_chromium.json index 0bfcfd8..784406d 100644 --- a/build/android/devil_chromium.json +++ b/build/android/devil_chromium.json
@@ -1,15 +1,6 @@ { "config_type": "BaseConfig", "dependencies": { - "aapt": { - "file_info": { - "linux2_x86_64": { - "local_paths": [ - "../../third_party/android_sdk/public/build-tools/27.0.3/aapt" - ] - } - } - }, "adb": { "file_info": { "linux2_x86_64": { @@ -19,15 +10,6 @@ } } }, - "android_build_tools_libc++": { - "file_info": { - "linux2_x86_64": { - "local_paths": [ - "../../third_party/android_sdk/public/build-tools/27.0.3/lib64/libc++.so" - ] - } - } - }, "android_sdk": { "file_info": { "linux2_x86_64": { @@ -37,24 +19,6 @@ } } }, - "dexdump": { - "file_info": { - "linux2_x86_64": { - "local_paths": [ - "../../third_party/android_sdk/public/build-tools/27.0.3/dexdump" - ] - } - } - }, - "split-select": { - "file_info": { - "linux2_x86_64": { - "local_paths": [ - "../../third_party/android_sdk/public/build-tools/27.0.3/split-select" - ] - } - } - }, "simpleperf": { "file_info": { "android_armeabi-v7a": { @@ -111,7 +75,7 @@ "file_info": { "default": { "local_paths": [ - "../../third_party/android_build_tools/bundletool/bundletool-all-1.4.0.jar" + "../../third_party/android_build_tools/bundletool/bundletool.jar" ] } }
diff --git a/build/android/devil_chromium.py b/build/android/devil_chromium.py index 20ae1e3..fbc5389 100644 --- a/build/android/devil_chromium.py +++ b/build/android/devil_chromium.py
@@ -1,4 +1,4 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -128,7 +128,7 @@ for dep_config in dep_configs } } - for dep_name, dep_configs in _DEVIL_BUILD_PRODUCT_DEPS.iteritems() + for dep_name, dep_configs in _DEVIL_BUILD_PRODUCT_DEPS.items() }
diff --git a/build/android/diff_resource_sizes.py b/build/android/diff_resource_sizes.py index eefb6cd..ff21d81 100755 --- a/build/android/diff_resource_sizes.py +++ b/build/android/diff_resource_sizes.py
@@ -1,11 +1,10 @@ -#!/usr/bin/env python -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Runs resource_sizes.py on two apks and outputs the diff.""" -from __future__ import print_function import argparse import json @@ -49,8 +48,8 @@ base_results: The chartjson-formatted size results of the base APK. diff_results: The chartjson-formatted size results of the diff APK. """ - for graph_title, graph in base_results['charts'].iteritems(): - for trace_title, trace in graph.iteritems(): + for graph_title, graph in base_results['charts'].items(): + for trace_title, trace in graph.items(): perf_tests_results_helper.ReportPerfResult( chartjson, graph_title, trace_title, diff_results['charts'][graph_title][trace_title]['value'] @@ -67,8 +66,8 @@ base_results: The chartjson-formatted size results of the base APK. diff_results: The chartjson-formatted size results of the diff APK. """ - for graph_title, graph in base_results['charts'].iteritems(): - for trace_title, trace in graph.iteritems(): + for graph_title, graph in base_results['charts'].items(): + for trace_title, trace in graph.items(): perf_tests_results_helper.ReportPerfResult( chartjson, graph_title + '_base_apk', trace_title, trace['value'], trace['units'], trace['improvement_direction'], @@ -76,8 +75,8 @@ # Both base_results and diff_results should have the same charts/traces, but # loop over them separately in case they don't - for graph_title, graph in diff_results['charts'].iteritems(): - for trace_title, trace in graph.iteritems(): + for graph_title, graph in diff_results['charts'].items(): + for trace_title, trace in graph.items(): perf_tests_results_helper.ReportPerfResult( chartjson, graph_title + '_diff_apk', trace_title, trace['value'], trace['units'], trace['improvement_direction'], @@ -194,6 +193,7 @@ logging.critical('Dumping diff histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout) + return 0 if __name__ == '__main__':
diff --git a/build/android/docs/README.md b/build/android/docs/README.md index 6392f7d..5ee0ca6 100644 --- a/build/android/docs/README.md +++ b/build/android/docs/README.md
@@ -1,6 +1,7 @@ # Android Build Docs -* [android_app_bundles.md](android_app_bundles.md) +* [//docs/android_build_instructions.md](/docs/android_build_instructions.md) +* [//docs/android_dynamic_feature_modules.md](/docs/android_dynamic_feature_modules.md) * [build_config.md](build_config.md) * [coverage.md](coverage.md) * [java_toolchain.md](java_toolchain.md) @@ -8,6 +9,8 @@ * [lint.md](lint.md) * [life_of_a_resource.md](life_of_a_resource.md) * [../incremental_install/README.md](../incremental_install/README.md) +* [//docs/ui/android/bytecode_rewriting.md](/docs/ui/android/bytecode_rewriting.md) +* [go/doubledown](https://goto.google.com/doubledown) (Googlers only) See also: * [//build/README.md](../../README.md)
diff --git a/build/android/docs/android_app_bundles.md b/build/android/docs/android_app_bundles.md deleted file mode 100644 index e71fe27..0000000 --- a/build/android/docs/android_app_bundles.md +++ /dev/null
@@ -1,205 +0,0 @@ -# Introduction - -This document describes how the Chromium build system supports Android app -bundles. - -[TOC] - -# Overview of app bundles - -An Android app bundle is an alternative application distribution format for -Android applications on the Google Play Store, that allows reducing the size -of binaries sent for installation to individual devices that run on Android L -and beyond. For more information about them, see the official Android -[documentation](https://developer.android.com/guide/app-bundle/). - -For the context of this document, the most important points are: - - - Unlike a regular APK (e.g. `foo.apk`), the bundle (e.g. `foo.aab`) cannot - be installed directly on a device. - - - Instead, it must be processed into a set of installable split APKs, which - are stored inside a special zip archive (e.g. `foo.apks`). - - - The splitting can be based on various criteria: e.g. language or screen - density for resources, or cpu ABI for native code. - - - The bundle also uses the notion of dynamic features modules (DFMs) to - separate several application features. Each module has its own code, assets - and resources, and can be installed separately from the rest of the - application if needed. - - - The main application itself is stored in the '`base`' module (this name - cannot be changed). - - -# Declaring app bundles with GN templates - -Here's an example that shows how to declare a simple bundle that contains a -single base module, which enables language-based splits: - -```gn - - # First declare the first bundle module. The base module is the one - # that contains the main application's code, resources and assets. - android_app_bundle_module("foo_base_module") { - # Declaration are similar to android_apk here. - ... - } - - # Second, declare the bundle itself. - android_app_bundle("foo_bundle") { - # Indicate the base module to use for this bundle - base_module_target = ":foo_base_module" - - # The name of our bundle file (without any suffix). Default would - # be 'foo_bundle' otherwise. - bundle_name = "FooBundle" - - # Enable language-based splits for this bundle. Which means that - # resources and assets specific to a given language will be placed - # into their own split APK in the final .apks archive. - enable_language_splits = true - - # Proguard settings must be passed at the bundle, not module, target. - proguard_enabled = !is_java_debug - } -``` - -When generating the `foo_bundle` target with Ninja, you will end up with -the following: - - - The bundle file under `out/Release/apks/FooBundle.aab` - - - A helper script called `out/Release/bin/foo_bundle`, which can be used - to install / launch / uninstall the bundle on local devices. - - This works like an APK wrapper script (e.g. `foo_apk`). Use `--help` - to see all possible commands supported by the script. - - -# Declaring dynamic feature modules with GN templates - -Please see -[Dynamic Feature Modules](../../../docs/android_dynamic_feature_modules.md) for -more details. In short, if you need more modules besides the base one, you -will need to list all the extra ones using the extra_modules variable which -takes a list of GN scopes, as in: - -```gn - - android_app_bundle_module("foo_base_module") { - ... - } - - android_app_bundle_module("foo_extra_module") { - ... - } - - android_app_bundle("foo_bundle") { - base_module_target = ":foo_base_module" - - extra_modules = [ - { # NOTE: Scopes require one field per line, and no comma separators. - name = "my_module" - module_target = ":foo_extra_module" - } - ] - - ... - } -``` - -Note that each extra module is identified by a unique name, which cannot -be '`base`'. - - -# Bundle signature issues - -Signing an app bundle is not necessary, unless you want to upload it to the -Play Store. Since this process is very slow (it uses `jarsigner` instead of -the much faster `apkbuilder`), you can control it with the `sign_bundle` -variable, as described in the example above. - -The `.apks` archive however always contains signed split APKs. The keystore -path/password/alias being used are the default ones, unless you use custom -values when declaring the bundle itself, as in: - -```gn - android_app_bundle("foo_bundle") { - ... - keystore_path = "//path/to/keystore" - keystore_password = "K3y$t0Re-Pa$$w0rd" - keystore_name = "my-signing-key-name" - } -``` - -These values are not stored in the bundle itself, but in the wrapper script, -which will use them to generate the `.apks` archive for you. This allows you -to properly install updates on top of existing applications on any device. - - -# Proguard and bundles - -When using an app bundle that is made of several modules, it is crucial to -ensure that proguard, if enabled: - -- Keeps the obfuscated class names used by each module consistent. -- Does not remove classes that are not used in one module, but referenced - by others. - -To achieve this, a special scheme called *synchronized proguarding* is -performed, which consists of the following steps: - -- The list of unoptimized .jar files from all modules are sent to a single - proguard command. This generates a new temporary optimized *group* .jar file. - -- Each module extracts the optimized class files from the optimized *group* - .jar file, to generate its own, module-specific, optimized .jar. - -- Each module-specific optimized .jar is then sent to dex generation. - -This synchronized proguarding step is added by the `android_app_bundle()` GN -template. In practice this means the following: - - - `proguard_enabled` must be passed to `android_app_bundle` targets, but not - to `android_app_bundle_module` ones. - - - `proguard_configs` can be still passed to individual modules, just - like regular APKs. All proguard configs will be merged during the - synchronized proguard step. - - -# Manual generation and installation of .apks archives - -Note that the `foo_bundle` script knows how to generate the .apks archive -from the bundle file, and install it to local devices for you. For example, -to install and launch a bundle, use: - -```sh - out/Release/bin/foo_bundle run -``` - -If you want to manually look or use the `.apks` archive, use the following -command to generate it: - -```sh - out/Release/bin/foo_bundle build-bundle-apks \ - --output-apks=/tmp/BundleFoo.apks -``` - -All split APKs within the archive will be properly signed. And you will be -able to look at its content (with `unzip -l`), or install it manually with: - -```sh - build/android/gyp/bundletool.py install-apks \ - --apks=/tmp/BundleFoo.apks \ - --adb=$(which adb) -``` - -The task of examining the manifest is simplified by running the following, -which dumps the application manifest as XML to stdout: - -```sh - build/android/gyp/bundletool.py dump-manifest -```
diff --git a/build/android/docs/build_config.md b/build/android/docs/build_config.md index 8a301c8..8f752a6 100644 --- a/build/android/docs/build_config.md +++ b/build/android/docs/build_config.md
@@ -1,19 +1,19 @@ # Introduction -This document describes the `.build_config` files that are used by the +This document describes the `.build_config.json` files that are used by the Chromium build system for Android-specific targets like APK, resources, and more. [TOC] -# I. Overview of .build_config files: +# I. Overview of .build_config.json files: The Android build requires performing computations about dependencies in various targets, which are not possible with the GN build language. To address -this, `.build_config` files are written during the build to store the needed +this, `.build_config.json` files are written during the build to store the needed per-target information as JSON files. -They are always written to `$target_gen_dir/${target_name}.build_config`. +They are always written to `$target_gen_dir/${target_name}.build_config.json`. Many scripts under [`build/android/gyp/`](build/android_gyp/), which are used during the build, can also accept parameter arguments using @@ -25,7 +25,7 @@ return the value at `[key1][key2]...[keyN]` for the `--some-param` option. Apart from that, the scripts do not need to know anything about the structure -of `.build_config` files (but the GN rules that invoke them do and select +of `.build_config.json` files (but the GN rules that invoke them do and select which `@FileArg()` references to use). For a concrete example, consider the following GN fragment: @@ -42,17 +42,17 @@ ``` This will end up generating the following JSON file under -`$CHROMIUM_OUTPUT_DIR/gen/ui/android/ui_java_resources.build_config`: +`$CHROMIUM_OUTPUT_DIR/gen/ui/android/ui_java_resources.build_config.json`: ```json { "deps_info": { "deps_configs": [ - "gen/ui/android/ui_strings_grd.build_config" + "gen/ui/android/ui_strings_grd.build_config.json" ], - "name": "ui_java_resources.build_config", + "name": "ui_java_resources.build_config.json", "package_name": "org.chromium.ui", - "path": "gen/ui/android/ui_java_resources.build_config", + "path": "gen/ui/android/ui_java_resources.build_config.json", "r_text": "gen/ui/android/ui_java_resources_R.txt", "resources_dirs": [ "../../ui/android/java/res" @@ -71,10 +71,10 @@ } ``` -NOTE: All path values in `.build_config` files are relative to your +NOTE: All path values in `.build_config.json` files are relative to your `$CHROMIUM_OUTPUT_DIR`. -# II. Generation of .build_config files: +# II. Generation of .build_config.json files: They are generated by the GN [`write_build_config()`](gn_write_build_config) internal template, which ends up invoking @@ -85,8 +85,8 @@ python ../../build/android/gyp/write_build_config.py \ --type=android_resources \ --depfile gen/ui/android/ui_java_resources__build_config_crbug_908819.d \ - --deps-configs=\[\"gen/ui/android/ui_strings_grd.build_config\"\] \ - --build-config gen/ui/android/ui_java_resources.build_config \ + --deps-configs=\[\"gen/ui/android/ui_strings_grd.build_config.json\"\] \ + --build-config gen/ui/android/ui_java_resources.build_config.json \ --resources-zip resource_zips/ui/android/ui_java_resources.resources.zip \ --package-name org.chromium.ui \ --r-text gen/ui/android/ui_java_resources_R.txt \ @@ -99,10 +99,10 @@ In particular, the `resources['dependency_zips']` entry was computed by inspecting the content of all dependencies (here, only -`ui_string_grd.build_config`), and collecting their +`ui_string_grd.build_config.json`), and collecting their `deps_configs['resources_zip']` values. -Because a target's `.build_config` file will always be generated after +Because a target's `.build_config.json` file will always be generated after that of all of its dependencies, [`write_build_config.py`](write_build_config_py) can traverse the whole (transitive) set of direct *and* indirect dependencies for a given target @@ -112,10 +112,10 @@ and is very powerful for Android builds. -# III. Usage of .build_config files: +# III. Usage of .build_config.json files: In addition to being parsed by `write_build_config.py`, when they are listed -in the `--deps-configs` of a given target, the `.build_config` files are used +in the `--deps-configs` of a given target, the `.build_config.json` files are used by other scripts under [build/android/gyp/] to build stuff. For example, the GN `android_resources` template uses it to invoke the @@ -127,8 +127,8 @@ --depfile gen/ui/android/ui_java_resources_1.d \ --android-sdk-jar ../../third_party/android_sdk/public/platforms/android-29/android.jar \ --aapt-path ../../third_party/android_sdk/public/build-tools/29.0.2/aapt \ - --dependencies-res-zips=@FileArg\(gen/ui/android/ui_java_resources.build_config:resources:dependency_zips\) \ - --extra-res-packages=@FileArg\(gen/ui/android/ui_java_resources.build_config:resources:extra_package_names\) \ + --dependencies-res-zips=@FileArg\(gen/ui/android/ui_java_resources.build_config.json:resources:dependency_zips\) \ + --extra-res-packages=@FileArg\(gen/ui/android/ui_java_resources.build_config.json:resources:extra_package_names\) \ --resource-dirs=\[\"../../ui/android/java/res\"\] \ --debuggable \ --resource-zip-out resource_zips/ui/android/ui_java_resources.resources.zip \ @@ -143,11 +143,11 @@ the information it needs. -# IV. Format of .build_config files: +# IV. Format of .build_config.json files: Thanks to `@FileArg()` references, Python build scripts under [`build/android/gyp/`](build/android/gyp/) do not need to know anything -about the internal format of `.build_config` files. +about the internal format of `.build_config.json` files. This format is decided between internal GN build rules and [`write_build_config.py`][write_build_config_py]. Since these changes rather @@ -155,7 +155,7 @@ can be extracted as a Markdown file and visualized with the following commands: ```sh -# Extract .build_config format documentation +# Extract .build_config.json format documentation build/android/gyp/write_build_config.py \ --generate-markdown-format-doc > /tmp/format.md @@ -163,6 +163,6 @@ python tools/md_browser/md_browser.py -d /tmp /tmp/format.md ``` -[build/android/gyp/]: https://chromium.googlesource.com/chromium/src/build/+/master/android/gyp/ +[build/android/gyp/]: https://chromium.googlesource.com/chromium/src/build/+/main/android/gyp/ [gn_write_build_config]: https://cs.chromium.org/chromium/src/build/config/android/internal_rules.gni?q=write_build_config&sq=package:chromium -[write_build_config_py]: https://chromium.googlesource.com/chromium/src/build/+/master/android/gyp/write_build_config.py +[write_build_config_py]: https://chromium.googlesource.com/chromium/src/build/+/main/android/gyp/write_build_config.py
diff --git a/build/android/docs/class_verification_failures.md b/build/android/docs/class_verification_failures.md index e3e4745..ab9a241 100644 --- a/build/android/docs/class_verification_failures.md +++ b/build/android/docs/class_verification_failures.md
@@ -2,6 +2,13 @@ [TOC] +## This document is obsolete + +While class verification failures still exist, our Java optimizer, R8, has +solved this problem for us. Developers should not have to worry about this +problem unless there is a bug in R8. See [this bug](http://b/138781768) for where +they implemented this solution for us. + ## What's this all about? This document aims to explain class verification on Android, how this can affect @@ -82,6 +89,9 @@ ## Chromium's solution +**Note:** This section is no longer relevant as R8 has fixed this for us. We intend +to remove these ApiHelperFor classes - see [this bug](https://crbug.com/1302156). + In Chromium, we try to avoid doing class verification at runtime by manually out-of-lining all Android API usage like so: @@ -127,8 +137,7 @@ * These need to exist in a separate class so that Android framework can successfully verify * classes without encountering the new APIs. */ -@VerifiesOnOMR1 -@TargetApi(Build.VERSION_CODES.O_MR1) +@RequiresApi(Build.VERSION_CODES.O_MR1) public class ApiHelperForOMR1 { private ApiHelperForOMR1() {} @@ -136,15 +145,14 @@ } ``` -* `@VerifiesOnO_MR1`: this is a chromium-defined annotation to tell proguard - (and similar tools) not to inline this class or its methods (since that would - defeat the point of out-of-lining!) -* `@TargetApi(Build.VERSION_CODES.O_MR1)`: this tells Android Lint it's OK to +* `@RequiresApi(Build.VERSION_CODES.O_MR1)`: this tells Android Lint it's OK to use OMR1 APIs since this class is only used on OMR1 and above. Substitute `O_MR1` for the [appropriate constant][4], depending when the APIs were introduced. * Don't put any `SDK_INT` checks inside this class, because it must only be called on >= OMR1. +* R8 is smart enough not to inline methods where doing so would introduce + verification failures (b/138781768) ### Out-of-lining if your method has a new type in its signature @@ -174,7 +182,7 @@ } @VerifiesOnP -@TargetApi(Build.VERSION_CODES.P) +@RequiresApi(Build.VERSION_CODES.P) public class ApiHelperForP { public static NewTypeInAndroidP getNewTypeInAndroidP() { return new NewTypeInAndroidP();
diff --git a/build/android/docs/coverage.md b/build/android/docs/coverage.md index 17c83c6..c7f3c1f 100644 --- a/build/android/docs/coverage.md +++ b/build/android/docs/coverage.md
@@ -1,7 +1,7 @@ # Android code coverage instructions These are instructions for collecting code coverage data for android -instrumentation and JUnit tests. +instrumentation and JUnit tests. For Clang(C++) code coverage refer to [clang coverage]. [TOC] @@ -9,7 +9,7 @@ In order to use JaCoCo code coverage, we need to create build time pre-instrumented class files and runtime **.exec** files. Then we need to process them using the -**build/android/generate_jacoco_report.py** script. +[build/android/generate_jacoco_report.py](https://source.chromium.org/chromium/chromium/src/+/main:build/android/generate_jacoco_report.py) script. ## How to collect coverage data @@ -71,3 +71,15 @@ --coverage-dir /tmp/coverage/ \ --sources-json-dir out/Debug/ \ ``` +3. If generating coverage and there are duplicate class files, as can happen + when generating coverage for downstream targets, use the + `--include-substr-filter` option to choose jars in the desired directory. Eg. + for generating coverage report for Clank internal repo + ```shell + build/android/generate_jacoco_report.py --format html \ + --output-dir /tmp/coverage_report/ --coverage-dir /tmp/coverage/ \ + --sources-json-dir out/java_coverage/ \ + --include-substr-filter obj/clank + ``` + +[clang coverage]: https://chromium.googlesource.com/chromium/src/+/HEAD/docs/testing/code_coverage.md \ No newline at end of file
diff --git a/build/android/docs/java_asserts.md b/build/android/docs/java_asserts.md new file mode 100644 index 0000000..37d94c1 --- /dev/null +++ b/build/android/docs/java_asserts.md
@@ -0,0 +1,80 @@ +# Java Asserts in Chromium +This doc exists to explain how asserts in Java are enabled and disabled by +Chromium's build system. + +## javac Assertion Bytecode +Whenever javac compiles a Java class, assertions are transformed into the +following bytecode: + +``` + Code: + 0: getstatic #2 // Static field $assertionsDisabled + 3: ifne 20 // Conditional jump past assertion throw + 12: new #3 // Class java/lang/AssertionError + 19: athrow // Throwing AssertionError + 20: return + +// NOTE: this static block was made just to check the desiredAssertionStatus. +// There was no static block on the class before javac created one. + static {}; + Code: + 2: invokevirtual #6 // Method java/lang/Class.desiredAssertionStatus() + 5: ifne 12 + 8: iconst_1 + 9: goto 13 + 12: iconst_0 + 13: putstatic #2 // Static field $assertionsDisabled + 16: return +``` + +TL;DR - every single assertion is gated behind a `assertionDisabled` flag check, +which is a static field that can be set by the JRE's +`setDefaultAssertionStatus`, `setPackageAssertionStatus`, and +`setClassAssertionStatus` methods. + +## Assertion Enabling/Disabling +Our tools which consume javac output, namely R8 and D8, each have flags which +the build system uses to enable or disable asserts. We control this with the +`enable_java_asserts` gn arg. It does this by deleting the gating check on +`assertionsDisabled` when enabling, and by eliminating any reference to the +assert when disabling. + +```java +// Example equivalents of: +a = foo(); +assert a != 0; +return a; + +// Traditional, unoptimized javac output. +a = foo(); +if (!assertionsDisabled && a == 0) { + throw new AssertionError(); +} +return a; + +// Optimized with assertions enabled. +a = foo(); +if (a == 0) { + throw new AssertionError(); +} +return a; + +// Optimized with assertions disabled. +a = foo(); +return a; +``` + +## Assertion Enabling on Canary +Recently we [enabled +asserts](https://chromium-review.googlesource.com/c/chromium/src/+/3307087) on +Canary. It spiked our crash rate, and it was decided to not do this again, as +it's bad user experience to crash the app incessantly for non-fatal issues. + +So, we asked the R8 team for a feature which would rewrite the bytecode of these +assertions, which they implemented for us. Now, instead of just turning it on +and throwing an `AssertionError`, [R8 would call a provided assertion +handler](https://r8.googlesource.com/r8/+/aefe7bc18a7ce19f3e9c6dac0bedf6d182bbe142/src/main/java/com/android/tools/r8/ParseFlagInfoImpl.java#124) +with the `AssertionError`. We then wrote a [silent assertion +reporter](https://chromium-review.googlesource.com/c/chromium/src/+/3746261) +and this reports Java `AssertionErrors` to our crash server without crashing +the browser.
diff --git a/build/android/docs/java_optimization.md b/build/android/docs/java_optimization.md index 0ba0d50..da10222 100644 --- a/build/android/docs/java_optimization.md +++ b/build/android/docs/java_optimization.md
@@ -84,7 +84,7 @@ zero-overhead abstractions. Annotating a class with [@CheckDiscard][checkdiscard] will add a `-checkdiscard` rule automatically. -[checkdiscard]: /base/android/java/src/org/chromium/base/annotations/CheckDiscard.java +[checkdiscard]: /build/android/java/src/org/chromium/build/annotations/CheckDiscard.java ``` Item void org.chromium.base.library_loader.LibraryPrefetcherJni.<init>() was not discarded.
diff --git a/build/android/docs/java_toolchain.md b/build/android/docs/java_toolchain.md index ef11548..a9d229d 100644 --- a/build/android/docs/java_toolchain.md +++ b/build/android/docs/java_toolchain.md
@@ -30,20 +30,23 @@ ### Step 1: Create interface .jar with turbine or ijar -For prebuilt `.jar` files, use [//third_party/ijar] to create interface `.jar` -from prebuilt `.jar`. - -For non-prebuilt targets, use [//third_party/turbine] to create interface `.jar` -from `.java` source files. Turbine is much faster than javac, and so enables -full compilation to happen more concurrently. - What are interface jars?: -* The contain `.class` files with all non-public symbols and function bodies +* They contain `.class` files with all private symbols and all method bodies removed. * Dependant targets use interface `.jar` files to skip having to be rebuilt when only private implementation details change. +For prebuilt `.jar` files: we use [//third_party/ijar] to create interface +`.jar` files from the prebuilt ones. + +For non-prebuilt `.jar` files`: we use [//third_party/turbine] to create +interface `.jar` files directly from `.java` source files. Turbine is faster +than javac because it does not compile method bodies. Although Turbine causes +us to compile files twice, it speeds up builds by allowing `javac` compilation +of targets to happen concurrently with their dependencies. We also use Turbine +to run our annotation processors. + [//third_party/ijar]: /third_party/ijar/README.chromium [//third_party/turbine]: /third_party/turbine/README.chromium @@ -223,7 +226,7 @@ * Runs as part of normal compilation. Controlled by GN arg: `use_errorprone_java_compiler`. * Most useful check: * Enforcement of `@GuardedBy` annotations. -* List of enabled / disabled checks exists [within javac.py](https://cs.chromium.org/chromium/src/build/android/gyp/javac.py?l=30) +* List of enabled / disabled checks exists [within compile_java.py](https://cs.chromium.org/chromium/src/build/android/gyp/compile_java.py?l=30) * Many checks are currently disabled because there is work involved in fixing violations they introduce. Please help! * Custom checks for Chrome: @@ -253,6 +256,8 @@ * In other words: Enforces that targets do not rely on indirect dependencies to populate their classpath. * Checks run on the entire codebase, not only on changed lines. +* This is the only static analysis that runs on prebuilt .jar files. +* The same tool is also used for [bytecode rewriting](/docs/ui/android/bytecode_rewriting.md). ### [PRESUBMIT.py](/PRESUBMIT.py): * Checks for banned patterns via `_BANNED_JAVA_FUNCTIONS`.
diff --git a/build/android/docs/life_of_a_resource.md b/build/android/docs/life_of_a_resource.md index 3aacd5e..5e46ef6 100644 --- a/build/android/docs/life_of_a_resource.md +++ b/build/android/docs/life_of_a_resource.md
@@ -12,21 +12,21 @@ [native resources]: https://www.chromium.org/developers/tools-we-use-in-chromium/grit/grit-users-guide The steps consume the following files as inputs: -* AndroidManifest.xml - * Including AndroidManifest.xml files from libraries, which get merged +* `AndroidManifest.xml` + * Including `AndroidManifest.xml` files from libraries, which get merged together * res/ directories The steps produce the following intermediate files: -* R.srcjar (contains R.java files) -* R.txt -* .resources.zip +* `R.srcjar` (contains `R.java` files) +* `R.txt` +* `.resources.zip` -The steps produce the following files within an .apk: -* AndroidManifest.xml (a binary xml file) -* resources.arsc (contains all values and configuration metadata) -* res/** (drawables and layouts) -* classes.dex (just a small portion of classes from generated R.java files) +The steps produce the following files within an `.apk`: +* `AndroidManifest.xml` (a binary xml file) +* `resources.arsc` (contains all values and configuration metadata) +* `res/**` (drawables and layouts) +* `classes.dex` (just a small portion of classes from generated `R.java` files) ## The Build Steps @@ -38,56 +38,78 @@ Inputs: * GN target metadata -* Other .build_config files +* Other `.build_config.json` files Outputs: -* Target-specific .build_config file +* Target-specific `.build_config.json` file -write_build_config.py is run to record target metadata needed by future steps. +`write_build_config.py` is run to record target metadata needed by future steps. For more details, see [build_config.md](build_config.md). ### 2. Prepares resources: Inputs: -* Target-specific build\_config file -* Target-specific Resource dirs (res/ directories) -* resources.zip files from dependencies (used to generate the R.txt/java files) +* Target-specific `.build_config.json` file +* Files listed as `sources` Outputs: -* Target-specific resources.zip (containing only resources in the - target-specific resource dirs, no dependant resources here). -* Target-specific R.txt - * Contains a list of resources and their ids (including of dependencies). -* Target-specific R.java .srcjar - * See [What are R.java files and how are they generated]( - #how-r_java-files-are-generated) +* Target-specific `resources.zip` (contains all resources listed in `sources`). +* Target-specific `R.txt` (list of all resources, including dependencies). -prepare\_resources.py zips up the target-specific resource dirs and generates -R.txt and R.java .srcjars. No optimizations, crunching, etc are done on the -resources. +`prepare_resources.py` zips up the target-specific resource files and generates +`R.txt`. No optimizations, crunching, etc are done on the resources. -**The following steps apply only to apk targets (not library targets).** +**The following steps apply only to apk & bundle targets (not to library +targets).** -### 3. Finalizes apk resources: +### 3. Create target-specific R.java files Inputs: -* Target-specific build\_config file -* Dependencies' resources.zip files +* `R.txt` from dependencies. + +Outputs: +* Target-specific (placeholder) `R.java` file. + +A target-specific `R.java` is generated for each `android_library()` target that +sets `resources_package`. Resource IDs are not known at this phase, so all +values are set as placeholders. This copy of `R` classes are discarded and +replaced with new copies at step 4. + +Example placeholder R.java file: +```java +package org.chromium.mypackage; + +public final class R { + public static class anim { + public static int abc_fade_in = 0; + public static int abc_fade_out = 0; + ... + } + ... +} +``` + +### 4. Finalizes apk resources: + +Inputs: +* Target-specific `.build_config.json` file +* Dependencies' `R.txt` files +* Dependencies' `resources.zip` files Output: -* Packaged resources zip (named foo.ap_) containing: - * AndroidManifest.xml (as binary xml) - * resources.arsc - * res/** -* Final R.txt +* Packaged `resources zip` (named `foo.ap_`) containing: + * `AndroidManifest.xml` (as binary xml) + * `resources.arsc` + * `res/**` +* Final `R.txt` * Contains a list of resources and their ids (including of dependencies). -* Final R.java .srcjar - * See [What are R.java files and how are they generated]( +* Final `R.java` files + * See [What are `R.java` files and how are they generated]( #how-r_java-files-are-generated) -#### 3(a). Compiles resources: +#### 4(a). Compiles resources: For each library / resources target your apk depends on, the following happens: * Use a regex (defined in the apk target) to remove select resources (optional). @@ -102,27 +124,34 @@ dependency). -#### 3(b). Links resources: +#### 4(b). Links resources: -After each dependency is compiled into an intermediate .zip, all those zips are -linked by the aapt2 link command which does the following: +After each dependency is compiled into an intermediate `.zip`, all those zips +are linked by the `aapt2 link` command which does the following: * Use the order of dependencies supplied so that some resources clober each other. -* Compile the AndroidManifest.xml to binary xml (references to resources are now - using ids rather than the string names) -* Create a resources.arsc file that has the name and values of string +* Compile the `AndroidManifest.xml` to binary xml (references to resources are + now using ids rather than the string names) +* Create a `resources.arsc` file that has the name and values of string resources as well as the name and path of non-string resources (ie. layouts and drawables). * Combine the compiled resources into one packaged resources apk (a zip file - with an .ap\_ extension) that has all the resources related files. + with an `.ap_` extension) that has all the resources related files. -#### 3(c). Optimizes resources: +#### 4(c). Optimizes resources: -This step obfuscates / strips resources names from the resources.arsc so that -they can be looked up only by their numeric ids (assigned in the compile -resources step). Access to resources via `Resources.getIdentifier()` no longer -work unless resources are [allowlisted](#adding-resources-to-the-allowlist). +Targets can opt into the following optimizations: +1) Resource name collapsing: Maps all resources to the same name. Access to + resources via `Resources.getIdentifier()` no longer work unless resources are + [allowlisted](#adding-resources-to-the-allowlist). +2) Resource filename obfuscation: Renames resource file paths from e.g.: + `res/drawable/something.png` to `res/a`. Rename mapping is stored alongside + APKs / bundles in a `.pathmap` file. Renames are based on hashes, and so are + stable between builds (unless a new hash collision occurs). +3) Unused resource removal: Referenced resources are extracted from the + optimized `.dex` and `AndroidManifest.xml`. Resources that are directly or + indirectly used by these files are removed. ## App Bundles and Modules: @@ -184,9 +213,9 @@ is `0x7f`. However, Webview is a shared library which gets loaded into other apks. The package id for webview resources is assigned dynamically at runtime. When webview is loaded it calls this [R file's][Base Module R.java File] -onResourcesLoaded function to have the correct package id. When deobfuscating -webview resource ids, disregard the first two bytes in the id when looking it up -in the `R.txt` file. +`onResourcesLoaded()` function to have the correct package id. When +deobfuscating webview resource ids, disregard the first two bytes in the id when +looking it up in the `R.txt` file. Monochrome, when loaded as webview, rewrites the package ids of resources used by the webview portion to the correct value at runtime, otherwise, its resources @@ -196,15 +225,20 @@ ## How R.java files are generated -R.java is a list of static classes, each with multiple static fields containing -ids. These ids are used in java code to reference resources in the apk. +`R.java` contain a set of nested static classes, each with static fields +containing ids. These ids are used in java code to reference resources in +the apk. -There are three types of R.java files in Chrome. -1. Base Module Root R.java Files -2. DFM Root R.java Files -3. Source R.java Files +There are three types of `R.java` files in Chrome. +1. Root / Base Module `R.java` Files +2. DFM `R.java` Files +3. Per-Library `R.java` Files -Example Base Module Root R.java File +### Root / Base Module `R.java` Files +Contain base android resources. All `R.java` files can access base module +resources through inheritance. + +Example Root / Base Module `R.java` File: ```java package gen.base_module; @@ -219,10 +253,12 @@ } } ``` -Base module root R.java files contain base android resources. All R.java files -can access base module resources through inheritance. -Example DFM Root R.java File +### DFM `R.java` Files +Extend base module root `R.java` files. This allows DFMs to access their own +resources as well as the base module's resources. + +Example DFM Root `R.java` File ```java package gen.vr_module; @@ -234,27 +270,20 @@ } } ``` -DFM root R.java files extend base module root R.java files. This allows DFMs to -access their own resources as well as the base module's resources. -Example Source R.java File +### Per-Library `R.java` Files +Generated for each `android_library()` target that sets `resources_package`. +First a placeholder copy is generated in the `android_library()` step, and then +a final copy is created during finalization. + +Example final per-library `R.java`: ```java package org.chromium.chrome.vr; public final class R { public static final class anim extends - gen.base_module.R.anim {} + gen.vr_module.R.anim {} public static final class animator extends - gen.base_module.R.animator {} + gen.vr_module.R.animator {} } ``` -Source R.java files extend root R.java files and have no resources of their own. -Developers can import these R.java files to access resources in the apk. - -The R.java file generated via the prepare resources step above has temporary ids -which are not marked `final`. That R.java file is only used so that javac can -compile the java code that references R.*. - -The R.java generated during the finalize apk resources step has -permanent ids. These ids are marked as `final` (except webview resources that -need to be [rewritten at runtime](#webview-resource-ids)).
diff --git a/build/android/docs/lint.md b/build/android/docs/lint.md index 4ba13d7..e97fd76 100644 --- a/build/android/docs/lint.md +++ b/build/android/docs/lint.md
@@ -83,7 +83,7 @@ that are too hard (or not possible) to suppress locally, and permanently ignoring warnings only for this target. To permanently ignore a warning for all targets, add the warning to the `_DISABLED_ALWAYS` list in -[build/android/gyp/lint.py](https://source.chromium.org/chromium/chromium/src/+/master:build/android/gyp/lint.py). +[build/android/gyp/lint.py](https://source.chromium.org/chromium/chromium/src/+/main:build/android/gyp/lint.py). Disabling globally makes lint a bit faster. The exception to the above rule is for warnings that affect multiple languages. @@ -115,26 +115,18 @@ One of the approaches above should be used instead. Eventually all the errors in baseline files should be either fixed or ignored permanently. -The following are some common scenarios where you may need to update baseline -files. - -### I updated `cmdline-tools` and now there are tons of new errors! - -This happens every time lint is updated, since lint is provided by -`cmdline-tools`. +Most devs do not need to update baseline files and should not need the script +below. Occasionally when making large build configuration changes it may be +necessary to update baseline files (e.g. increasing the min_sdk_version). Baseline files are defined via the `lint_baseline_file` gn variable. It is -usually defined near a target's `enable_lint` gn variable. To regenerate the -baseline file, delete it and re-run the lint target. The command will fail, but -the baseline file will have been generated. +usually defined near a target's `enable_lint` gn variable. To regenerate all +baseline files, run: -This may need to be repeated for all targets that have set `enable_lint = true`, -including downstream targets. Downstream baseline files should be updated and -first to avoid build breakages. Each target has its own `lint_baseline_file` -defined and so all these files can be removed and regenerated as needed. +``` +$ third_party/android_build_tools/lint/rebuild_baselines.py +``` -### I updated `library X` and now there are tons of new errors! - -This is usually because `library X`'s aar contains custom lint checks and/or -custom annotation definition. Follow the same procedure as updates to -`cmdline-tools`. +This script will also update baseline files in downstream //clank if needed. +Since downstream and upstream use separate lint binaries, it is usually safe +to simply land the update CLs in any order. \ No newline at end of file
diff --git a/build/android/download_doclava.py b/build/android/download_doclava.py index 1982fdb..04db084 100755 --- a/build/android/download_doclava.py +++ b/build/android/download_doclava.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2016 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/dump_apk_resource_strings.py b/build/android/dump_apk_resource_strings.py index 8417e29..962103e 100755 --- a/build/android/dump_apk_resource_strings.py +++ b/build/android/dump_apk_resource_strings.py
@@ -1,12 +1,11 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # encoding: utf-8 -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A script to parse and dump localized strings in resource.arsc files.""" -from __future__ import print_function import argparse import collections @@ -92,7 +91,7 @@ # pylint: disable=line-too-long # NOTE: aapt dump will quote the following characters only: \n, \ and " -# see https://android.googlesource.com/platform/frameworks/base/+/master/libs/androidfw/ResourceTypes.cpp#7270 +# see https://cs.android.com/search?q=f:ResourceTypes.cpp # pylint: enable=line-too-long @@ -122,7 +121,7 @@ while pos + count < size and s[pos + count] == '\\': count += 1 - result += '\\' * (count / 2) + result += '\\' * (count // 2) start = pos + count if count & 1: if start < size: @@ -185,7 +184,7 @@ return result -class ResourceStringValues(object): +class ResourceStringValues: """Models all possible values for a named string.""" def __init__(self): @@ -219,8 +218,8 @@ def ToStringList(self, res_id): """Convert entry to string list for human-friendly output.""" - values = sorted( - [(str(config), value) for config, value in self.res_values.iteritems()]) + values = sorted([(str(config), value) + for config, value in self.res_values.items()]) if res_id is None: # res_id will be None when the resource ID should not be part # of the output. @@ -236,7 +235,7 @@ return result -class ResourceStringMap(object): +class ResourceStringMap: """Convenience class to hold the set of all localized strings in a table. Usage is the following: @@ -256,7 +255,7 @@ def RemapResourceNames(self, id_name_map): """Rename all entries according to a given {res_id -> res_name} map.""" - for res_id, res_name in id_name_map.iteritems(): + for res_id, res_name in id_name_map.items(): if res_id in self._res_map: self._res_map[res_id].res_name = res_name @@ -278,15 +277,10 @@ result = ['Resource strings (count=%d) {' % len(self._res_map)] res_map = self._res_map - # A small function to compare two (res_id, values) tuples - # by resource name first, then resource ID. - def cmp_id_name(a, b): - result = cmp(a[1].res_name, b[1].res_name) - if result == 0: - result = cmp(a[0], b[0]) - return result - - for res_id, _ in sorted(res_map.iteritems(), cmp=cmp_id_name): + # Compare two (res_id, values) tuples by resource name first, then resource + # ID. + for res_id, _ in sorted(res_map.items(), + key=lambda x: (x[1].res_name, x[0])): result += res_map[res_id].ToStringList(None if omit_ids else res_id) result.append('} # Resource strings') return result @@ -386,7 +380,7 @@ _RE_BUNDLE_STRING_LOCALIZED_VALUE = re.compile( r'^\s+locale: "([0-9a-zA-Z-]+)" - \[STR\] "(.*)"$') assert _RE_BUNDLE_STRING_LOCALIZED_VALUE.match( - u' locale: "ar" - [STR] "گزینه\u200cهای بیشتر"'.encode('utf-8')) + ' locale: "ar" - [STR] "گزینه\u200cهای بیشتر"') def ParseBundleResources(bundle_tool_jar_path, bundle_path): @@ -537,11 +531,15 @@ res_map = ResourceStringMap() current_locale = None - current_resource_id = None + current_resource_id = -1 # represents undefined. current_resource_name = None need_value = False while True: - line = p.stdout.readline().rstrip() + try: + line = p.stdout.readline().rstrip().decode('utf8') + except UnicodeDecodeError: + continue + if not line: break m = _RE_AAPT_CONFIG.match(line)
diff --git a/build/android/emma_coverage_stats.py b/build/android/emma_coverage_stats.py deleted file mode 100755 index f45f4d4..0000000 --- a/build/android/emma_coverage_stats.py +++ /dev/null
@@ -1,479 +0,0 @@ -#!/usr/bin/env vpython -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Generates incremental code coverage reports for Java code in Chromium. - -Usage: - - build/android/emma_coverage_stats.py -v --out <output file path> --emma-dir - <EMMA file directory> --lines-for-coverage-file - <path to file containing lines for coverage> - - Creates a JSON representation of the overall and file coverage stats and saves - this information to the specified output file. -""" - -import argparse -import collections -import json -import logging -import os -import re -import sys -from xml.etree import ElementTree - -import devil_chromium -from devil.utils import run_tests_helper - -NOT_EXECUTABLE = -1 -NOT_COVERED = 0 -COVERED = 1 -PARTIALLY_COVERED = 2 - -# Coverage information about a single line of code. -LineCoverage = collections.namedtuple( - 'LineCoverage', - ['lineno', 'source', 'covered_status', 'fractional_line_coverage']) - - -class _EmmaHtmlParser(object): - """Encapsulates HTML file parsing operations. - - This class contains all operations related to parsing HTML files that were - produced using the EMMA code coverage tool. - - Example HTML: - - Package links: - <a href="_files/1.html">org.chromium.chrome</a> - This is returned by the selector |XPATH_SELECT_PACKAGE_ELEMENTS|. - - Class links: - <a href="1e.html">DoActivity.java</a> - This is returned by the selector |XPATH_SELECT_CLASS_ELEMENTS|. - - Line coverage data: - <tr class="p"> - <td class="l" title="78% line coverage (7 out of 9)">108</td> - <td title="78% line coverage (7 out of 9 instructions)"> - if (index < 0 || index = mSelectors.size()) index = 0;</td> - </tr> - <tr> - <td class="l">109</td> - <td> </td> - </tr> - <tr class="c"> - <td class="l">110</td> - <td> if (mSelectors.get(index) != null) {</td> - </tr> - <tr class="z"> - <td class="l">111</td> - <td> for (int i = 0; i < mSelectors.size(); i++) {</td> - </tr> - Each <tr> element is returned by the selector |XPATH_SELECT_LOC|. - - We can parse this to get: - 1. Line number - 2. Line of source code - 3. Coverage status (c, z, or p) - 4. Fractional coverage value (% out of 100 if PARTIALLY_COVERED) - """ - # Selector to match all <a> elements within the rows that are in the table - # that displays all of the different packages. - _XPATH_SELECT_PACKAGE_ELEMENTS = './/BODY/TABLE[4]/TR/TD/A' - - # Selector to match all <a> elements within the rows that are in the table - # that displays all of the different classes within a package. - _XPATH_SELECT_CLASS_ELEMENTS = './/BODY/TABLE[3]/TR/TD/A' - - # Selector to match all <tr> elements within the table containing Java source - # code in an EMMA HTML file. - _XPATH_SELECT_LOC = './/BODY/TABLE[4]/TR' - - # Children of HTML elements are represented as a list in ElementTree. These - # constants represent list indices corresponding to relevant child elements. - - # Child 1 contains percentage covered for a line. - _ELEMENT_PERCENT_COVERED = 1 - - # Child 1 contains the original line of source code. - _ELEMENT_CONTAINING_SOURCE_CODE = 1 - - # Child 0 contains the line number. - _ELEMENT_CONTAINING_LINENO = 0 - - # Maps CSS class names to corresponding coverage constants. - _CSS_TO_STATUS = {'c': COVERED, 'p': PARTIALLY_COVERED, 'z': NOT_COVERED} - - # UTF-8 no break space. - _NO_BREAK_SPACE = '\xc2\xa0' - - def __init__(self, emma_file_base_dir): - """Initializes _EmmaHtmlParser. - - Args: - emma_file_base_dir: Path to the location where EMMA report files are - stored. Should be where index.html is stored. - """ - self._base_dir = emma_file_base_dir - self._emma_files_path = os.path.join(self._base_dir, '_files') - self._index_path = os.path.join(self._base_dir, 'index.html') - - def GetLineCoverage(self, emma_file_path): - """Returns a list of LineCoverage objects for the given EMMA HTML file. - - Args: - emma_file_path: String representing the path to the EMMA HTML file. - - Returns: - A list of LineCoverage objects. - """ - line_tr_elements = self._FindElements( - emma_file_path, self._XPATH_SELECT_LOC) - line_coverage = [] - for tr in line_tr_elements: - # Get the coverage status. - coverage_status = self._CSS_TO_STATUS.get(tr.get('CLASS'), NOT_EXECUTABLE) - # Get the fractional coverage value. - if coverage_status == PARTIALLY_COVERED: - title_attribute = (tr[self._ELEMENT_PERCENT_COVERED].get('TITLE')) - # Parse string that contains percent covered: "83% line coverage ...". - percent_covered = title_attribute.split('%')[0] - fractional_coverage = int(percent_covered) / 100.0 - else: - fractional_coverage = 1.0 - - # Get the line number. - lineno_element = tr[self._ELEMENT_CONTAINING_LINENO] - # Handles oddly formatted HTML (where there is an extra <a> tag). - lineno = int(lineno_element.text or - lineno_element[self._ELEMENT_CONTAINING_LINENO].text) - # Get the original line of Java source code. - raw_source = tr[self._ELEMENT_CONTAINING_SOURCE_CODE].text - utf8_source = raw_source.encode('UTF-8') - source = utf8_source.replace(self._NO_BREAK_SPACE, ' ') - - line = LineCoverage(lineno, source, coverage_status, fractional_coverage) - line_coverage.append(line) - - return line_coverage - - def GetPackageNameToEmmaFileDict(self): - """Returns a dict mapping Java packages to EMMA HTML coverage files. - - Parses the EMMA index.html file to get a list of packages, then parses each - package HTML file to get a list of classes for that package, and creates - a dict with this info. - - Returns: - A dict mapping string representation of Java packages (with class - names appended) to the corresponding file paths of EMMA HTML files. - """ - # These <a> elements contain each package name and the path of the file - # where all classes within said package are listed. - package_link_elements = self._FindElements( - self._index_path, self._XPATH_SELECT_PACKAGE_ELEMENTS) - # Maps file path of package directory (EMMA generated) to package name. - # Example: emma_dir/f.html: org.chromium.chrome. - package_links = { - os.path.join(self._base_dir, link.attrib['HREF']): link.text - for link in package_link_elements if 'HREF' in link.attrib - } - - package_to_emma = {} - for package_emma_file_path, package_name in package_links.iteritems(): - # These <a> elements contain each class name in the current package and - # the path of the file where the coverage info is stored for each class. - coverage_file_link_elements = self._FindElements( - package_emma_file_path, self._XPATH_SELECT_CLASS_ELEMENTS) - - for class_name_element in coverage_file_link_elements: - emma_coverage_file_path = os.path.join( - self._emma_files_path, class_name_element.attrib['HREF']) - full_package_name = '%s.%s' % (package_name, class_name_element.text) - package_to_emma[full_package_name] = emma_coverage_file_path - - return package_to_emma - - # pylint: disable=no-self-use - def _FindElements(self, file_path, xpath_selector): - """Reads a HTML file and performs an XPath match. - - Args: - file_path: String representing the path to the HTML file. - xpath_selector: String representing xpath search pattern. - - Returns: - A list of ElementTree.Elements matching the given XPath selector. - Returns an empty list if there is no match. - """ - with open(file_path) as f: - file_contents = f.read().decode('ISO-8859-1').encode('UTF-8') - root = ElementTree.fromstring(file_contents) - return root.findall(xpath_selector) - - -class _EmmaCoverageStats(object): - """Computes code coverage stats for Java code using the coverage tool EMMA. - - This class provides an API that allows users to capture absolute code coverage - and code coverage on a subset of lines for each Java source file. Coverage - reports are generated in JSON format. - """ - # Regular expression to get package name from Java package statement. - RE_PACKAGE_MATCH_GROUP = 'package' - RE_PACKAGE = re.compile(r'package (?P<%s>[\w.]*);' % RE_PACKAGE_MATCH_GROUP) - - def __init__(self, emma_file_base_dir, files_for_coverage): - """Initialize _EmmaCoverageStats. - - Args: - emma_file_base_dir: String representing the path to the base directory - where EMMA HTML coverage files are stored, i.e. parent of index.html. - files_for_coverage: A list of Java source code file paths to get EMMA - coverage for. - """ - self._emma_parser = _EmmaHtmlParser(emma_file_base_dir) - self._source_to_emma = self._GetSourceFileToEmmaFileDict(files_for_coverage) - - def GetCoverageDict(self, lines_for_coverage): - """Returns a dict containing detailed coverage information. - - Gets detailed coverage stats for each file specified in the - |lines_for_coverage| dict and the total incremental number of lines covered - and executable for all files in |lines_for_coverage|. - - Args: - lines_for_coverage: A dict mapping Java source file paths to lists of line - numbers. - - Returns: - A dict containing coverage stats for the given dict of files and lines. - Contains absolute coverage stats for each file, coverage stats for each - file's lines specified in |lines_for_coverage|, line by line coverage - for each file, and overall coverage stats for the lines specified in - |lines_for_coverage|. - """ - file_coverage = {} - for file_path, line_numbers in lines_for_coverage.iteritems(): - file_coverage_dict = self.GetCoverageDictForFile(file_path, line_numbers) - if file_coverage_dict: - file_coverage[file_path] = file_coverage_dict - else: - logging.warning( - 'No code coverage data for %s, skipping.', file_path) - - covered_statuses = [s['incremental'] for s in file_coverage.itervalues()] - num_covered_lines = sum(s['covered'] for s in covered_statuses) - num_total_lines = sum(s['total'] for s in covered_statuses) - return { - 'files': file_coverage, - 'patch': { - 'incremental': { - 'covered': num_covered_lines, - 'total': num_total_lines - } - } - } - - def GetCoverageDictForFile(self, file_path, line_numbers): - """Returns a dict containing detailed coverage info for the given file. - - Args: - file_path: The path to the Java source file that we want to create the - coverage dict for. - line_numbers: A list of integer line numbers to retrieve additional stats - for. - - Returns: - A dict containing absolute, incremental, and line by line coverage for - a file. - """ - if file_path not in self._source_to_emma: - return None - emma_file = self._source_to_emma[file_path] - total_line_coverage = self._emma_parser.GetLineCoverage(emma_file) - incremental_line_coverage = [line for line in total_line_coverage - if line.lineno in line_numbers] - line_by_line_coverage = [ - { - 'line': line.source, - 'coverage': line.covered_status, - 'changed': line.lineno in line_numbers, - 'fractional_coverage': line.fractional_line_coverage, - } - for line in total_line_coverage - ] - total_covered_lines, total_lines = ( - self.GetSummaryStatsForLines(total_line_coverage)) - incremental_covered_lines, incremental_total_lines = ( - self.GetSummaryStatsForLines(incremental_line_coverage)) - - file_coverage_stats = { - 'absolute': { - 'covered': total_covered_lines, - 'total': total_lines - }, - 'incremental': { - 'covered': incremental_covered_lines, - 'total': incremental_total_lines - }, - 'source': line_by_line_coverage, - } - return file_coverage_stats - - # pylint: disable=no-self-use - def GetSummaryStatsForLines(self, line_coverage): - """Gets summary stats for a given list of LineCoverage objects. - - Args: - line_coverage: A list of LineCoverage objects. - - Returns: - A tuple containing the number of lines that are covered and the total - number of lines that are executable, respectively - """ - partially_covered_sum = 0 - covered_status_totals = {COVERED: 0, NOT_COVERED: 0, PARTIALLY_COVERED: 0} - for line in line_coverage: - status = line.covered_status - if status == NOT_EXECUTABLE: - continue - covered_status_totals[status] += 1 - if status == PARTIALLY_COVERED: - partially_covered_sum += line.fractional_line_coverage - - total_covered = covered_status_totals[COVERED] + partially_covered_sum - total_lines = sum(covered_status_totals.values()) - return total_covered, total_lines - - def _GetSourceFileToEmmaFileDict(self, files): - """Gets a dict used to correlate Java source files with EMMA HTML files. - - This method gathers the information needed to correlate EMMA HTML - files with Java source files. EMMA XML and plain text reports do not provide - line by line coverage data, so HTML reports must be used instead. - Unfortunately, the HTML files that are created are given garbage names - (i.e 1.html) so we need to manually correlate EMMA HTML files - with the original Java source files. - - Args: - files: A list of file names for which coverage information is desired. - - Returns: - A dict mapping Java source file paths to EMMA HTML file paths. - """ - # Maps Java source file paths to package names. - # Example: /usr/code/file.java -> org.chromium.file.java. - source_to_package = {} - for file_path in files: - package = self.GetPackageNameFromFile(file_path) - if package: - source_to_package[file_path] = package - else: - logging.warning("Skipping %s because it doesn\'t have a package " - "statement.", file_path) - - # Maps package names to EMMA report HTML files. - # Example: org.chromium.file.java -> out/coverage/1a.html. - package_to_emma = self._emma_parser.GetPackageNameToEmmaFileDict() - # Finally, we have a dict mapping Java file paths to EMMA report files. - # Example: /usr/code/file.java -> out/coverage/1a.html. - source_to_emma = {source: package_to_emma[package] - for source, package in source_to_package.iteritems() - if package in package_to_emma} - return source_to_emma - - @staticmethod - def NeedsCoverage(file_path): - """Checks to see if the file needs to be analyzed for code coverage. - - Args: - file_path: A string representing path to the file. - - Returns: - True for Java files that exist, False for all others. - """ - if os.path.splitext(file_path)[1] == '.java' and os.path.exists(file_path): - return True - else: - logging.info('Skipping file %s, cannot compute code coverage.', file_path) - return False - - @staticmethod - def GetPackageNameFromFile(file_path): - """Gets the full package name including the file name for a given file path. - - Args: - file_path: String representing the path to the Java source file. - - Returns: - A string representing the full package name with file name appended or - None if there is no package statement in the file. - """ - with open(file_path) as f: - file_content = f.read() - package_match = re.search(_EmmaCoverageStats.RE_PACKAGE, file_content) - if package_match: - package = package_match.group(_EmmaCoverageStats.RE_PACKAGE_MATCH_GROUP) - file_name = os.path.basename(file_path) - return '%s.%s' % (package, file_name) - else: - return None - - -def GenerateCoverageReport(line_coverage_file, out_file_path, coverage_dir): - """Generates a coverage report for a given set of lines. - - Writes the results of the coverage analysis to the file specified by - |out_file_path|. - - Args: - line_coverage_file: The path to a file which contains a dict mapping file - names to lists of line numbers. Example: {file1: [1, 2, 3], ...} means - that we should compute coverage information on lines 1 - 3 for file1. - out_file_path: A string representing the location to write the JSON report. - coverage_dir: A string representing the file path where the EMMA - HTML coverage files are located (i.e. folder where index.html is located). - """ - with open(line_coverage_file) as f: - potential_files_for_coverage = json.load(f) - - files_for_coverage = {f: lines - for f, lines in potential_files_for_coverage.iteritems() - if _EmmaCoverageStats.NeedsCoverage(f)} - - coverage_results = {} - if files_for_coverage: - code_coverage = _EmmaCoverageStats(coverage_dir, files_for_coverage.keys()) - coverage_results = code_coverage.GetCoverageDict(files_for_coverage) - else: - logging.info('No Java files requiring coverage were included in %s.', - line_coverage_file) - - with open(out_file_path, 'w+') as out_status_file: - json.dump(coverage_results, out_status_file) - - -def main(): - argparser = argparse.ArgumentParser() - argparser.add_argument('--out', required=True, type=str, - help='Report output file path.') - argparser.add_argument('--emma-dir', required=True, type=str, - help='EMMA HTML report directory.') - argparser.add_argument('--lines-for-coverage-file', required=True, type=str, - help='File containing a JSON object. Should contain a ' - 'dict mapping file names to lists of line numbers of ' - 'code for which coverage information is desired.') - argparser.add_argument('-v', '--verbose', action='count', - help='Print verbose log information.') - args = argparser.parse_args() - run_tests_helper.SetLogLevel(args.verbose) - devil_chromium.Initialize() - GenerateCoverageReport(args.lines_for_coverage_file, args.out, args.emma_dir) - - -if __name__ == '__main__': - sys.exit(main())
diff --git a/build/android/emma_coverage_stats_test.py b/build/android/emma_coverage_stats_test.py deleted file mode 100755 index d53292c..0000000 --- a/build/android/emma_coverage_stats_test.py +++ /dev/null
@@ -1,561 +0,0 @@ -#!/usr/bin/env vpython -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# pylint: disable=protected-access - -import unittest -from xml.etree import ElementTree - -import emma_coverage_stats - -import mock # pylint: disable=import-error - -EMPTY_COVERAGE_STATS_DICT = { - 'files': {}, - 'patch': { - 'incremental': { - 'covered': 0, 'total': 0 - } - } -} - - -class _EmmaHtmlParserTest(unittest.TestCase): - """Tests for _EmmaHtmlParser. - - Uses modified EMMA report HTML that contains only the subset of tags needed - for test verification. - """ - - def setUp(self): - self.emma_dir = 'fake/dir/' - self.parser = emma_coverage_stats._EmmaHtmlParser(self.emma_dir) - self.simple_html = '<TR><TD CLASS="p">Test HTML</TD></TR>' - self.index_html = ( - '<HTML>' - '<BODY>' - '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '<TABLE CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '<TABLE CLASS="it" CELLSPACING="0">' - '</TABLE>' - '<TABLE CELLSPACING="0" WIDTH="100%">' - '<TR>' - '<TH CLASS="f">name</TH>' - '<TH>class, %</TH>' - '<TH>method, %</TH>' - '<TH>block, %</TH>' - '<TH>line, %</TH>' - '</TR>' - '<TR CLASS="o">' - '<TD><A HREF="_files/0.html"' - '>org.chromium.chrome.browser</A></TD>' - '<TD CLASS="h">0% (0/3)</TD>' - '</TR>' - '<TR>' - '<TD><A HREF="_files/1.html"' - '>org.chromium.chrome.browser.tabmodel</A></TD>' - '<TD CLASS="h">0% (0/8)</TD>' - '</TR>' - '</TABLE>' - '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '</BODY>' - '</HTML>' - ) - self.package_1_class_list_html = ( - '<HTML>' - '<BODY>' - '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '<TABLE CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '<TABLE CELLSPACING="0" WIDTH="100%">' - '<TR>' - '<TH CLASS="f">name</TH>' - '<TH>class, %</TH>' - '<TH>method, %</TH>' - '<TH>block, %</TH>' - '<TH>line, %</TH>' - '</TR>' - '<TR CLASS="o">' - '<TD><A HREF="1e.html">IntentHelper.java</A></TD>' - '<TD CLASS="h">0% (0/3)</TD>' - '<TD CLASS="h">0% (0/9)</TD>' - '<TD CLASS="h">0% (0/97)</TD>' - '<TD CLASS="h">0% (0/26)</TD>' - '</TR>' - '</TABLE>' - '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '</BODY>' - '</HTML>' - ) - self.package_2_class_list_html = ( - '<HTML>' - '<BODY>' - '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '<TABLE CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '<TABLE CELLSPACING="0" WIDTH="100%">' - '<TR>' - '<TH CLASS="f">name</TH>' - '<TH>class, %</TH>' - '<TH>method, %</TH>' - '<TH>block, %</TH>' - '<TH>line, %</TH>' - '</TR>' - '<TR CLASS="o">' - '<TD><A HREF="1f.html">ContentSetting.java</A></TD>' - '<TD CLASS="h">0% (0/1)</TD>' - '</TR>' - '<TR>' - '<TD><A HREF="20.html">DevToolsServer.java</A></TD>' - '</TR>' - '<TR CLASS="o">' - '<TD><A HREF="21.html">FileProviderHelper.java</A></TD>' - '</TR>' - '<TR>' - '<TD><A HREF="22.html">ContextualMenuBar.java</A></TD>' - '</TR>' - '<TR CLASS="o">' - '<TD><A HREF="23.html">AccessibilityUtil.java</A></TD>' - '</TR>' - '<TR>' - '<TD><A HREF="24.html">NavigationPopup.java</A></TD>' - '</TR>' - '</TABLE>' - '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">' - '</TABLE>' - '</BODY>' - '</HTML>' - ) - self.partially_covered_tr_html = ( - '<TR CLASS="p">' - '<TD CLASS="l" TITLE="78% line coverage (7 out of 9)">108</TD>' - '<TD TITLE="78% line coverage (7 out of 9 instructions)">' - 'if (index < 0 || index = mSelectors.size()) index = 0;</TD>' - '</TR>' - ) - self.covered_tr_html = ( - '<TR CLASS="c">' - '<TD CLASS="l">110</TD>' - '<TD> if (mSelectors.get(index) != null) {</TD>' - '</TR>' - ) - self.not_executable_tr_html = ( - '<TR>' - '<TD CLASS="l">109</TD>' - '<TD> </TD>' - '</TR>' - ) - self.tr_with_extra_a_tag = ( - '<TR CLASS="z">' - '<TD CLASS="l">' - '<A name="1f">54</A>' - '</TD>' - '<TD> }</TD>' - '</TR>' - ) - - def testInit(self): - emma_dir = self.emma_dir - parser = emma_coverage_stats._EmmaHtmlParser(emma_dir) - self.assertEqual(parser._base_dir, emma_dir) - self.assertEqual(parser._emma_files_path, 'fake/dir/_files') - self.assertEqual(parser._index_path, 'fake/dir/index.html') - - def testFindElements_basic(self): - read_values = [self.simple_html] - found, _ = MockOpenForFunction(self.parser._FindElements, read_values, - file_path='fake', xpath_selector='.//TD') - self.assertIs(type(found), list) - self.assertIs(type(found[0]), ElementTree.Element) - self.assertEqual(found[0].text, 'Test HTML') - - def testFindElements_multipleElements(self): - multiple_trs = self.not_executable_tr_html + self.covered_tr_html - read_values = ['<div>' + multiple_trs + '</div>'] - found, _ = MockOpenForFunction(self.parser._FindElements, read_values, - file_path='fake', xpath_selector='.//TR') - self.assertEquals(2, len(found)) - - def testFindElements_noMatch(self): - read_values = [self.simple_html] - found, _ = MockOpenForFunction(self.parser._FindElements, read_values, - file_path='fake', xpath_selector='.//TR') - self.assertEqual(found, []) - - def testFindElements_badFilePath(self): - with self.assertRaises(IOError): - with mock.patch('os.path.exists', return_value=False): - self.parser._FindElements('fake', xpath_selector='//tr') - - def testGetPackageNameToEmmaFileDict_basic(self): - expected_dict = { - 'org.chromium.chrome.browser.AccessibilityUtil.java': - 'fake/dir/_files/23.html', - 'org.chromium.chrome.browser.ContextualMenuBar.java': - 'fake/dir/_files/22.html', - 'org.chromium.chrome.browser.tabmodel.IntentHelper.java': - 'fake/dir/_files/1e.html', - 'org.chromium.chrome.browser.ContentSetting.java': - 'fake/dir/_files/1f.html', - 'org.chromium.chrome.browser.DevToolsServer.java': - 'fake/dir/_files/20.html', - 'org.chromium.chrome.browser.NavigationPopup.java': - 'fake/dir/_files/24.html', - 'org.chromium.chrome.browser.FileProviderHelper.java': - 'fake/dir/_files/21.html'} - - read_values = [self.index_html, self.package_1_class_list_html, - self.package_2_class_list_html] - return_dict, mock_open = MockOpenForFunction( - self.parser.GetPackageNameToEmmaFileDict, read_values) - - self.assertDictEqual(return_dict, expected_dict) - self.assertEqual(mock_open.call_count, 3) - calls = [mock.call('fake/dir/index.html'), - mock.call('fake/dir/_files/1.html'), - mock.call('fake/dir/_files/0.html')] - mock_open.assert_has_calls(calls) - - def testGetPackageNameToEmmaFileDict_noPackageElements(self): - self.parser._FindElements = mock.Mock(return_value=[]) - return_dict = self.parser.GetPackageNameToEmmaFileDict() - self.assertDictEqual({}, return_dict) - - def testGetLineCoverage_status_basic(self): - line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html]) - self.assertEqual(line_coverage[0].covered_status, - emma_coverage_stats.COVERED) - - def testGetLineCoverage_status_statusMissing(self): - line_coverage = self.GetLineCoverageWithFakeElements( - [self.not_executable_tr_html]) - self.assertEqual(line_coverage[0].covered_status, - emma_coverage_stats.NOT_EXECUTABLE) - - def testGetLineCoverage_fractionalCoverage_basic(self): - line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html]) - self.assertEqual(line_coverage[0].fractional_line_coverage, 1.0) - - def testGetLineCoverage_fractionalCoverage_partial(self): - line_coverage = self.GetLineCoverageWithFakeElements( - [self.partially_covered_tr_html]) - self.assertEqual(line_coverage[0].fractional_line_coverage, 0.78) - - def testGetLineCoverage_lineno_basic(self): - line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html]) - self.assertEqual(line_coverage[0].lineno, 110) - - def testGetLineCoverage_lineno_withAlternativeHtml(self): - line_coverage = self.GetLineCoverageWithFakeElements( - [self.tr_with_extra_a_tag]) - self.assertEqual(line_coverage[0].lineno, 54) - - def testGetLineCoverage_source(self): - self.parser._FindElements = mock.Mock( - return_value=[ElementTree.fromstring(self.covered_tr_html)]) - line_coverage = self.parser.GetLineCoverage('fake_path') - self.assertEqual(line_coverage[0].source, - ' if (mSelectors.get(index) != null) {') - - def testGetLineCoverage_multipleElements(self): - line_coverage = self.GetLineCoverageWithFakeElements( - [self.covered_tr_html, self.partially_covered_tr_html, - self.tr_with_extra_a_tag]) - self.assertEqual(len(line_coverage), 3) - - def GetLineCoverageWithFakeElements(self, html_elements): - """Wraps GetLineCoverage so mock HTML can easily be used. - - Args: - html_elements: List of strings each representing an HTML element. - - Returns: - A list of LineCoverage objects. - """ - elements = [ElementTree.fromstring(string) for string in html_elements] - with mock.patch('emma_coverage_stats._EmmaHtmlParser._FindElements', - return_value=elements): - return self.parser.GetLineCoverage('fake_path') - - -class _EmmaCoverageStatsTest(unittest.TestCase): - """Tests for _EmmaCoverageStats.""" - - def setUp(self): - self.good_source_to_emma = { - '/path/to/1/File1.java': '/emma/1.html', - '/path/2/File2.java': '/emma/2.html', - '/path/2/File3.java': '/emma/3.html' - } - self.line_coverage = [ - emma_coverage_stats.LineCoverage( - 1, '', emma_coverage_stats.COVERED, 1.0), - emma_coverage_stats.LineCoverage( - 2, '', emma_coverage_stats.COVERED, 1.0), - emma_coverage_stats.LineCoverage( - 3, '', emma_coverage_stats.NOT_EXECUTABLE, 1.0), - emma_coverage_stats.LineCoverage( - 4, '', emma_coverage_stats.NOT_COVERED, 1.0), - emma_coverage_stats.LineCoverage( - 5, '', emma_coverage_stats.PARTIALLY_COVERED, 0.85), - emma_coverage_stats.LineCoverage( - 6, '', emma_coverage_stats.PARTIALLY_COVERED, 0.20) - ] - self.lines_for_coverage = [1, 3, 5, 6] - with mock.patch('emma_coverage_stats._EmmaHtmlParser._FindElements', - return_value=[]): - self.simple_coverage = emma_coverage_stats._EmmaCoverageStats( - 'fake_dir', {}) - - def testInit(self): - coverage_stats = self.simple_coverage - self.assertIsInstance(coverage_stats._emma_parser, - emma_coverage_stats._EmmaHtmlParser) - self.assertIsInstance(coverage_stats._source_to_emma, dict) - - def testNeedsCoverage_withExistingJavaFile(self): - test_file = '/path/to/file/File.java' - with mock.patch('os.path.exists', return_value=True): - self.assertTrue( - emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file)) - - def testNeedsCoverage_withNonJavaFile(self): - test_file = '/path/to/file/File.c' - with mock.patch('os.path.exists', return_value=True): - self.assertFalse( - emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file)) - - def testNeedsCoverage_fileDoesNotExist(self): - test_file = '/path/to/file/File.java' - with mock.patch('os.path.exists', return_value=False): - self.assertFalse( - emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file)) - - def testGetPackageNameFromFile_basic(self): - test_file_text = """// Test Copyright - package org.chromium.chrome.browser; - import android.graphics.RectF;""" - result_package, _ = MockOpenForFunction( - emma_coverage_stats._EmmaCoverageStats.GetPackageNameFromFile, - [test_file_text], file_path='/path/to/file/File.java') - self.assertEqual(result_package, 'org.chromium.chrome.browser.File.java') - - def testGetPackageNameFromFile_noPackageStatement(self): - result_package, _ = MockOpenForFunction( - emma_coverage_stats._EmmaCoverageStats.GetPackageNameFromFile, - ['not a package statement'], file_path='/path/to/file/File.java') - self.assertIsNone(result_package) - - def testGetSummaryStatsForLines_basic(self): - covered, total = self.simple_coverage.GetSummaryStatsForLines( - self.line_coverage) - self.assertEqual(covered, 3.05) - self.assertEqual(total, 5) - - def testGetSourceFileToEmmaFileDict(self): - package_names = { - '/path/to/1/File1.java': 'org.fake.one.File1.java', - '/path/2/File2.java': 'org.fake.File2.java', - '/path/2/File3.java': 'org.fake.File3.java' - } - package_to_emma = { - 'org.fake.one.File1.java': '/emma/1.html', - 'org.fake.File2.java': '/emma/2.html', - 'org.fake.File3.java': '/emma/3.html' - } - with mock.patch('os.path.exists', return_value=True): - coverage_stats = self.simple_coverage - coverage_stats._emma_parser.GetPackageNameToEmmaFileDict = mock.MagicMock( - return_value=package_to_emma) - coverage_stats.GetPackageNameFromFile = lambda x: package_names[x] - result_dict = coverage_stats._GetSourceFileToEmmaFileDict( - package_names.keys()) - self.assertDictEqual(result_dict, self.good_source_to_emma) - - def testGetCoverageDictForFile(self): - line_coverage = self.line_coverage - self.simple_coverage._emma_parser.GetLineCoverage = lambda x: line_coverage - self.simple_coverage._source_to_emma = {'/fake/src': 'fake/emma'} - lines = self.lines_for_coverage - expected_dict = { - 'absolute': { - 'covered': 3.05, - 'total': 5 - }, - 'incremental': { - 'covered': 2.05, - 'total': 3 - }, - 'source': [ - { - 'line': line_coverage[0].source, - 'coverage': line_coverage[0].covered_status, - 'changed': True, - 'fractional_coverage': line_coverage[0].fractional_line_coverage, - }, - { - 'line': line_coverage[1].source, - 'coverage': line_coverage[1].covered_status, - 'changed': False, - 'fractional_coverage': line_coverage[1].fractional_line_coverage, - }, - { - 'line': line_coverage[2].source, - 'coverage': line_coverage[2].covered_status, - 'changed': True, - 'fractional_coverage': line_coverage[2].fractional_line_coverage, - }, - { - 'line': line_coverage[3].source, - 'coverage': line_coverage[3].covered_status, - 'changed': False, - 'fractional_coverage': line_coverage[3].fractional_line_coverage, - }, - { - 'line': line_coverage[4].source, - 'coverage': line_coverage[4].covered_status, - 'changed': True, - 'fractional_coverage': line_coverage[4].fractional_line_coverage, - }, - { - 'line': line_coverage[5].source, - 'coverage': line_coverage[5].covered_status, - 'changed': True, - 'fractional_coverage': line_coverage[5].fractional_line_coverage, - } - ] - } - result_dict = self.simple_coverage.GetCoverageDictForFile( - '/fake/src', lines) - self.assertDictEqual(result_dict, expected_dict) - - def testGetCoverageDictForFile_emptyCoverage(self): - expected_dict = { - 'absolute': {'covered': 0, 'total': 0}, - 'incremental': {'covered': 0, 'total': 0}, - 'source': [] - } - self.simple_coverage._emma_parser.GetLineCoverage = lambda x: [] - self.simple_coverage._source_to_emma = {'fake_dir': 'fake/emma'} - result_dict = self.simple_coverage.GetCoverageDictForFile('fake_dir', {}) - self.assertDictEqual(result_dict, expected_dict) - - def testGetCoverageDictForFile_missingCoverage(self): - self.simple_coverage._source_to_emma = {} - result_dict = self.simple_coverage.GetCoverageDictForFile('fake_file', {}) - self.assertIsNone(result_dict) - - def testGetCoverageDict_basic(self): - files_for_coverage = { - '/path/to/1/File1.java': [1, 3, 4], - '/path/2/File2.java': [1, 2] - } - self.simple_coverage._source_to_emma = { - '/path/to/1/File1.java': 'emma_1', - '/path/2/File2.java': 'emma_2' - } - coverage_info = { - 'emma_1': [ - emma_coverage_stats.LineCoverage( - 1, '', emma_coverage_stats.COVERED, 1.0), - emma_coverage_stats.LineCoverage( - 2, '', emma_coverage_stats.PARTIALLY_COVERED, 0.5), - emma_coverage_stats.LineCoverage( - 3, '', emma_coverage_stats.NOT_EXECUTABLE, 1.0), - emma_coverage_stats.LineCoverage( - 4, '', emma_coverage_stats.COVERED, 1.0) - ], - 'emma_2': [ - emma_coverage_stats.LineCoverage( - 1, '', emma_coverage_stats.NOT_COVERED, 1.0), - emma_coverage_stats.LineCoverage( - 2, '', emma_coverage_stats.COVERED, 1.0) - ] - } - expected_dict = { - 'files': { - '/path/2/File2.java': { - 'absolute': {'covered': 1, 'total': 2}, - 'incremental': {'covered': 1, 'total': 2}, - 'source': [{'changed': True, 'coverage': 0, - 'line': '', 'fractional_coverage': 1.0}, - {'changed': True, 'coverage': 1, - 'line': '', 'fractional_coverage': 1.0}] - }, - '/path/to/1/File1.java': { - 'absolute': {'covered': 2.5, 'total': 3}, - 'incremental': {'covered': 2, 'total': 2}, - 'source': [{'changed': True, 'coverage': 1, - 'line': '', 'fractional_coverage': 1.0}, - {'changed': False, 'coverage': 2, - 'line': '', 'fractional_coverage': 0.5}, - {'changed': True, 'coverage': -1, - 'line': '', 'fractional_coverage': 1.0}, - {'changed': True, 'coverage': 1, - 'line': '', 'fractional_coverage': 1.0}] - } - }, - 'patch': {'incremental': {'covered': 3, 'total': 4}} - } - # Return the relevant coverage info for each file. - self.simple_coverage._emma_parser.GetLineCoverage = ( - lambda x: coverage_info[x]) - result_dict = self.simple_coverage.GetCoverageDict(files_for_coverage) - self.assertDictEqual(result_dict, expected_dict) - - def testGetCoverageDict_noCoverage(self): - result_dict = self.simple_coverage.GetCoverageDict({}) - self.assertDictEqual(result_dict, EMPTY_COVERAGE_STATS_DICT) - - -class EmmaCoverageStatsGenerateCoverageReport(unittest.TestCase): - """Tests for GenerateCoverageReport.""" - - def testGenerateCoverageReport_missingJsonFile(self): - with self.assertRaises(IOError): - with mock.patch('os.path.exists', return_value=False): - emma_coverage_stats.GenerateCoverageReport('', '', '') - - def testGenerateCoverageReport_invalidJsonFile(self): - with self.assertRaises(ValueError): - with mock.patch('os.path.exists', return_value=True): - MockOpenForFunction(emma_coverage_stats.GenerateCoverageReport, [''], - line_coverage_file='', out_file_path='', - coverage_dir='') - - -def MockOpenForFunction(func, side_effects, **kwargs): - """Allows easy mock open and read for callables that open multiple files. - - Will mock the python open function in a way such that each time read() is - called on an open file, the next element in |side_effects| is returned. This - makes it easier to test functions that call open() multiple times. - - Args: - func: The callable to invoke once mock files are setup. - side_effects: A list of return values for each file to return once read. - Length of list should be equal to the number calls to open in |func|. - **kwargs: Keyword arguments to be passed to |func|. - - Returns: - A tuple containing the return value of |func| and the MagicMock object used - to mock all calls to open respectively. - """ - mock_open = mock.mock_open() - mock_open.side_effect = [mock.mock_open(read_data=side_effect).return_value - for side_effect in side_effects] - with mock.patch('__builtin__.open', mock_open): - return func(**kwargs), mock_open - - -if __name__ == '__main__': - # Suppress logging messages. - unittest.main(buffer=True)
diff --git a/build/android/envsetup.sh b/build/android/envsetup.sh index 7f549d9..315db29 100755 --- a/build/android/envsetup.sh +++ b/build/android/envsetup.sh
@@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/fast_local_dev_server.py b/build/android/fast_local_dev_server.py index a35c500..282dcf5 100755 --- a/build/android/fast_local_dev_server.py +++ b/build/android/fast_local_dev_server.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Creates an server to offload non-critical-path GN targets.""" @@ -110,6 +110,7 @@ if line.startswith('procs_running'): return int(line.rstrip().split()[1]) assert False, 'Could not read /proc/stat' + return 0 def _maybe_start_tasks(self): if self._deactivated: @@ -174,6 +175,8 @@ # TODO(wnwen): Use ionice to reduce resource consumption. TaskStats.add_process() log(f'STARTING {self.name}') + # This use of preexec_fn is sufficiently simple, just one os.nice call. + # pylint: disable=subprocess-popen-preexec-fn self._proc = subprocess.Popen( self.cmd, stdout=subprocess.PIPE, @@ -281,6 +284,8 @@ tasks: Dict[Tuple[str, str], Task] = {} task_manager = TaskManager() try: + log('READY... Remember to set android_static_analysis="build_server" in ' + 'args.gn files') for data in _listen_for_request_data(sock): task = Task(name=data['name'], cwd=data['cwd'], @@ -303,11 +308,28 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.parse_args() + parser.add_argument( + '--fail-if-not-running', + action='store_true', + help='Used by GN to fail fast if the build server is not running.') + args = parser.parse_args() + if args.fail_if_not_running: + with socket.socket(socket.AF_UNIX) as sock: + try: + sock.connect(server_utils.SOCKET_ADDRESS) + except socket.error: + print('Build server is not running and ' + 'android_static_analysis="build_server" is set.\nPlease run ' + 'this command in a separate terminal:\n\n' + '$ build/android/fast_local_dev_server.py\n') + return 1 + else: + return 0 with socket.socket(socket.AF_UNIX) as sock: sock.bind(server_utils.SOCKET_ADDRESS) sock.listen() _process_requests(sock) + return 0 if __name__ == '__main__':
diff --git a/build/android/generate_jacoco_report.py b/build/android/generate_jacoco_report.py index d0a9987..44e82ac 100755 --- a/build/android/generate_jacoco_report.py +++ b/build/android/generate_jacoco_report.py
@@ -1,12 +1,11 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Aggregates Jacoco coverage files to produce output.""" -from __future__ import print_function import argparse import fnmatch @@ -35,40 +34,39 @@ _SOURCES_JSON_FILES_SUFFIX = '__jacoco_sources.json' -# These should match the jar class files generated in internal_rules.gni -_DEVICE_CLASS_EXCLUDE_SUFFIX = 'host_filter.jar' -_HOST_CLASS_EXCLUDE_SUFFIX = 'device_filter.jar' - -def _CreateClassfileArgs(class_files, exclude_suffix=None): - """Returns a list of files that don't have a given suffix. +def _CreateClassfileArgs(class_files, report_type, include_substr=None): + """Returns a filtered list of files with classfile option. Args: class_files: A list of class files. - exclude_suffix: Suffix to look for to exclude. + report_type: A string indicating if device or host files are desired. + include_substr: A substring that must be present to include the file. Returns: A list of files that don't use the suffix. """ + # These should match the jar class files generated in internal_rules.gni + search_jar_suffix = '%s.filter.jar' % report_type result_class_files = [] for f in class_files: - if exclude_suffix: - if not f.endswith(exclude_suffix): - result_class_files += ['--classfiles', f] - else: + include_file = False + if f.endswith(search_jar_suffix): + include_file = True + + # If include_substr is specified, remove files that don't have the + # required substring. + if include_file and include_substr and include_substr not in f: + include_file = False + if include_file: result_class_files += ['--classfiles', f] return result_class_files def _GenerateReportOutputArgs(args, class_files, report_type): - class_jar_exclude = None - if report_type == 'device': - class_jar_exclude = _DEVICE_CLASS_EXCLUDE_SUFFIX - elif report_type == 'host': - class_jar_exclude = _HOST_CLASS_EXCLUDE_SUFFIX - - cmd = _CreateClassfileArgs(class_files, class_jar_exclude) + cmd = _CreateClassfileArgs(class_files, report_type, + args.include_substr_filter) if args.format == 'html': report_dir = os.path.join(args.output_dir, report_type) if not os.path.exists(report_dir): @@ -141,6 +139,10 @@ 'host classpath files. Host would typically be used for junit tests ' ' and device for tests that run on the device. Only used for xml and csv' ' reports.') + parser.add_argument('--include-substr-filter', + help='Substring that must be included in classjars.', + type=str, + default='') parser.add_argument('--output-dir', help='html report output directory.') parser.add_argument('--output-file', help='xml file to write device coverage results.') @@ -241,6 +243,7 @@ # report and we wouldn't know which one a developer needed. device_cmd = cmd + _GenerateReportOutputArgs(args, class_files, 'device') host_cmd = cmd + _GenerateReportOutputArgs(args, class_files, 'host') + device_exit_code = cmd_helper.RunCmd(device_cmd) host_exit_code = cmd_helper.RunCmd(host_cmd) exit_code = device_exit_code or host_exit_code
diff --git a/build/android/gradle/AndroidManifest.xml b/build/android/gradle/AndroidManifest.xml index f3e50e0..dfbb9bd 100644 --- a/build/android/gradle/AndroidManifest.xml +++ b/build/android/gradle/AndroidManifest.xml
@@ -1,6 +1,6 @@ <?xml version="1.0" encoding="utf-8"?> <!-- - Copyright 2018 The Chromium Authors. All rights reserved. + Copyright 2018 The Chromium Authors Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. -->
diff --git a/build/android/gradle/android.jinja b/build/android/gradle/android.jinja index 7d566dd..3b66b97 100644 --- a/build/android/gradle/android.jinja +++ b/build/android/gradle/android.jinja
@@ -51,13 +51,13 @@ defaultConfig { vectorDrawables.useSupportLibrary = true - minSdkVersion 24 + minSdkVersion {{ min_sdk_version }} targetSdkVersion {{ target_sdk_version }} } compileOptions { - sourceCompatibility JavaVersion.VERSION_1_8 - targetCompatibility JavaVersion.VERSION_1_8 + sourceCompatibility JavaVersion.VERSION_11 + targetCompatibility JavaVersion.VERSION_11 } {% if native is defined %}
diff --git a/build/android/gradle/generate_gradle.py b/build/android/gradle/generate_gradle.py index 80d0b0a..bc05baf 100755 --- a/build/android/gradle/generate_gradle.py +++ b/build/android/gradle/generate_gradle.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2016 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,6 +12,7 @@ import json import logging import os +import pathlib import re import shutil import subprocess @@ -32,6 +33,12 @@ sys.path.append(os.path.dirname(_BUILD_ANDROID)) import gn_helpers +# Typically these should track the versions that works on the slowest release +# channel, i.e. Android Studio stable. +_DEFAULT_ANDROID_GRADLE_PLUGIN_VERSION = '7.3.1' +_DEFAULT_KOTLIN_GRADLE_PLUGIN_VERSION = '1.8.0' +_DEFAULT_GRADLE_WRAPPER_VERSION = '7.4' + _DEPOT_TOOLS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party', 'depot_tools') _DEFAULT_ANDROID_MANIFEST_PATH = os.path.join( @@ -45,8 +52,6 @@ _CMAKE_FILE = 'CMakeLists.txt' # This needs to come first alphabetically among all modules. _MODULE_ALL = '_all' -_SRC_INTERNAL = os.path.join( - os.path.dirname(host_paths.DIR_SOURCE_ROOT), 'src-internal') _INSTRUMENTATION_TARGET_SUFFIX = '_test_apk__test_apk__apk' _DEFAULT_TARGETS = [ @@ -57,6 +62,7 @@ '//chrome/android:chrome_junit_tests', '//chrome/android:chrome_public_apk', '//chrome/android:chrome_public_test_apk', + '//chrome/android:chrome_public_unit_test_apk', '//content/public/android:content_junit_tests', '//content/shell/android:content_shell_apk', # Below must be included even with --all since they are libraries. @@ -64,12 +70,6 @@ '//tools/android/errorprone_plugin:errorprone_plugin_java', ] -_EXCLUDED_PREBUILT_JARS = [ - # Android Studio already provides Desugar runtime. - # Including it would cause linking error because of a duplicate class. - 'lib.java/third_party/bazel/desugar/Desugar-runtime.jar' -] - def _TemplatePath(name): return os.path.join(_FILE_DIR, '{}.jinja'.format(name)) @@ -83,7 +83,7 @@ """ if path_or_list is None: return [] - if not isinstance(path_or_list, basestring): + if not isinstance(path_or_list, str): return [_RebasePath(p, new_cwd, old_cwd) for p in path_or_list] if old_cwd is None: old_cwd = constants.GetOutDirectory() @@ -94,11 +94,6 @@ return os.path.abspath(os.path.join(old_cwd, path_or_list)) -def _IsSubpathOf(child, parent): - """Returns whether |child| is a subpath of |parent|.""" - return not os.path.relpath(child, parent).startswith(os.pardir) - - def _WriteFile(path, data): """Writes |data| to |path|, constucting parent directories if necessary.""" logging.info('Writing %s', path) @@ -132,10 +127,10 @@ '--nested', '--build', '--output-directory', output_dir ] logging.info('Running: %r', cmd) - return subprocess.check_output(cmd).splitlines() + return subprocess.check_output(cmd, encoding='UTF-8').splitlines() -class _ProjectEntry(object): +class _ProjectEntry: """Helper class for project entries.""" _cached_entries = {} @@ -160,7 +155,7 @@ @classmethod def FromBuildConfigPath(cls, path): prefix = 'gen/' - suffix = '.build_config' + suffix = '.build_config.json' assert path.startswith(prefix) and path.endswith(suffix), path subdir = path[len(prefix):-len(suffix)] gn_target = '//%s:%s' % (os.path.split(subdir)) @@ -178,9 +173,6 @@ def NinjaTarget(self): return self._gn_target[2:] - def GnBuildConfigTarget(self): - return '%s__build_config_crbug_908819' % self._gn_target - def GradleSubdir(self): """Returns the output subdirectory.""" ninja_target = self.NinjaTarget() @@ -198,9 +190,9 @@ return self.GradleSubdir().replace(os.path.sep, '.') def BuildConfig(self): - """Reads and returns the project's .build_config JSON.""" + """Reads and returns the project's .build_config.json JSON.""" if not self._build_config: - path = os.path.join('gen', self.GradleSubdir() + '.build_config') + path = os.path.join('gen', self.GradleSubdir() + '.build_config.json') with open(_RebasePath(path)) as jsonfile: self._build_config = json.load(jsonfile) return self._build_config @@ -225,7 +217,7 @@ 'java_library', "java_annotation_processor", 'java_binary', - 'junit_binary', + 'robolectric_binary', ) def ResSources(self): @@ -233,17 +225,16 @@ def JavaFiles(self): if self._java_files is None: - java_sources_file = self.DepsInfo().get('java_sources_file') + target_sources_file = self.DepsInfo().get('target_sources_file') java_files = [] - if java_sources_file: - java_sources_file = _RebasePath(java_sources_file) - java_files = build_utils.ReadSourcesList(java_sources_file) + if target_sources_file: + target_sources_file = _RebasePath(target_sources_file) + java_files = build_utils.ReadSourcesList(target_sources_file) self._java_files = java_files return self._java_files def PrebuiltJars(self): - all_jars = self.Gradle().get('dependent_prebuilt_jars', []) - return [i for i in all_jars if i not in _EXCLUDED_PREBUILT_JARS] + return self.Gradle().get('dependent_prebuilt_jars', []) def AllEntries(self): """Returns a list of all entries that the current entry depends on. @@ -263,16 +254,15 @@ return self._all_entries -class _ProjectContextGenerator(object): +class _ProjectContextGenerator: """Helper class to generate gradle build files""" def __init__(self, project_dir, build_vars, use_gradle_process_resources, - jinja_processor, split_projects, channel): + jinja_processor, split_projects): self.project_dir = project_dir self.build_vars = build_vars self.use_gradle_process_resources = use_gradle_process_resources self.jinja_processor = jinja_processor self.split_projects = split_projects - self.channel = channel self.processed_java_dirs = set() self.processed_prebuilts = set() self.processed_res_dirs = set() @@ -303,12 +293,14 @@ for library targets.""" resource_packages = entry.Javac().get('resource_packages') if not resource_packages: - logging.debug('Target ' + entry.GnTarget() + ' includes resources from ' - 'unknown package. Unable to process with gradle.') + logging.debug( + 'Target %s includes resources from unknown package. ' + 'Unable to process with gradle.', entry.GnTarget()) return _DEFAULT_ANDROID_MANIFEST_PATH - elif len(resource_packages) > 1: - logging.debug('Target ' + entry.GnTarget() + ' includes resources from ' - 'multiple packages. Unable to process with gradle.') + if len(resource_packages) > 1: + logging.debug( + 'Target %s includes resources from multiple packages. ' + 'Unable to process with gradle.', entry.GnTarget()) return _DEFAULT_ANDROID_MANIFEST_PATH variables = {'package': resource_packages[0]} @@ -423,37 +415,42 @@ return excludes -def _ComputeJavaSourceDirsAndExcludes(output_dir, java_files): +def _ComputeJavaSourceDirsAndExcludes(output_dir, source_files): """Computes the list of java source directories and exclude patterns. - 1. Computes the root java source directories from the list of files. + This includes both Java and Kotlin files since both are listed in the same + "java" section for gradle. + + 1. Computes the root source directories from the list of files. 2. Compute exclude patterns that exclude all extra files only. - 3. Returns the list of java source directories and exclude patterns. + 3. Returns the list of source directories and exclude patterns. """ java_dirs = [] excludes = [] - if java_files: - java_files = _RebasePath(java_files) - computed_dirs = _ComputeJavaSourceDirs(java_files) - java_dirs = computed_dirs.keys() - all_found_java_files = set() + if source_files: + source_files = _RebasePath(source_files) + computed_dirs = _ComputeJavaSourceDirs(source_files) + java_dirs = list(computed_dirs.keys()) + all_found_source_files = set() - for directory, files in computed_dirs.iteritems(): - found_java_files = build_utils.FindInDirectory(directory, '*.java') - all_found_java_files.update(found_java_files) - unwanted_java_files = set(found_java_files) - set(files) - if unwanted_java_files: + for directory, files in computed_dirs.items(): + found_source_files = (build_utils.FindInDirectory(directory, '*.java') + + build_utils.FindInDirectory(directory, '*.kt')) + all_found_source_files.update(found_source_files) + unwanted_source_files = set(found_source_files) - set(files) + if unwanted_source_files: logging.debug('Directory requires excludes: %s', directory) excludes.extend( - _ComputeExcludeFilters(files, unwanted_java_files, directory)) + _ComputeExcludeFilters(files, unwanted_source_files, directory)) - missing_java_files = set(java_files) - all_found_java_files + missing_source_files = set(source_files) - all_found_source_files # Warn only about non-generated files that are missing. - missing_java_files = [p for p in missing_java_files - if not p.startswith(output_dir)] - if missing_java_files: - logging.warning( - 'Some java files were not found: %s', missing_java_files) + missing_source_files = [ + p for p in missing_source_files if not p.startswith(output_dir) + ] + if missing_source_files: + logging.warning('Some source files were not found: %s', + missing_source_files) return java_dirs, excludes @@ -486,6 +483,19 @@ return [] +def _ParseVersionFromFile(file_path, version_regex_string, default_version): + if os.path.exists(file_path): + content = pathlib.Path(file_path).read_text() + match = re.search(version_regex_string, content) + if match: + version = match.group(1) + logging.info('Using existing version %s in %s.', version, file_path) + return version + logging.warning('Unable to find %s in %s:\n%s', version_regex_string, + file_path, content) + return default_version + + def _GenerateLocalProperties(sdk_dir): """Returns the data for local.properties as a string.""" return '\n'.join([ @@ -495,14 +505,17 @@ ]) -def _GenerateGradleWrapperPropertiesCanary(): +def _GenerateGradleWrapperProperties(file_path): """Returns the data for gradle-wrapper.properties as a string.""" - # Before May 2020, this wasn't necessary. Might not be necessary at some point - # in the future? + + version = _ParseVersionFromFile(file_path, + r'/distributions/gradle-([\d.]+)-all.zip', + _DEFAULT_GRADLE_WRAPPER_VERSION) + return '\n'.join([ '# Generated by //build/android/gradle/generate_gradle.py', - ('distributionUrl=https\\://services.gradle.org/distributions/' - 'gradle-6.5-rc-1-all.zip\n'), + ('distributionUrl=https\\://services.gradle.org' + f'/distributions/gradle-{version}-all.zip'), '', ]) @@ -520,15 +533,16 @@ def _GenerateBaseVars(generator, build_vars): variables = {} - variables['compile_sdk_version'] = ( - 'android-%s' % build_vars['compile_sdk_version']) - target_sdk_version = build_vars['android_sdk_version'] - if target_sdk_version.isalpha(): + # Avoid pre-release SDKs since Studio might not know how to download them. + variables['compile_sdk_version'] = ('android-%s' % + build_vars['public_android_sdk_version']) + target_sdk_version = build_vars['public_android_sdk_version'] + if str(target_sdk_version).isalpha(): target_sdk_version = '"{}"'.format(target_sdk_version) variables['target_sdk_version'] = target_sdk_version + variables['min_sdk_version'] = build_vars['default_min_sdk_version'] variables['use_gradle_process_resources'] = ( generator.use_gradle_process_resources) - variables['channel'] = generator.channel return variables @@ -545,14 +559,14 @@ gradle_treat_as_prebuilt = deps_info.get('gradle_treat_as_prebuilt', False) if is_prebuilt or gradle_treat_as_prebuilt: return None - elif deps_info['requires_android']: + if deps_info['requires_android']: target_type = 'android_library' else: target_type = 'java_library' elif deps_info['type'] == 'java_binary': target_type = 'java_binary' variables['main_class'] = deps_info.get('main_class') - elif deps_info['type'] == 'junit_binary': + elif deps_info['type'] == 'robolectric_binary': target_type = 'android_junit' sourceSetName = 'test' else: @@ -570,7 +584,7 @@ test_entry = generator.Generate(e) test_entry['android_manifest'] = generator.GenerateManifest(e) variables['android_test'].append(test_entry) - for key, value in test_entry.iteritems(): + for key, value in test_entry.items(): if isinstance(value, list): test_entry[key] = sorted(set(value) - set(variables['main'][key])) @@ -640,12 +654,12 @@ 'android_manifest': Relativize(_DEFAULT_ANDROID_MANIFEST_PATH), 'java_dirs': Relativize(main_java_dirs), 'prebuilts': Relativize(prebuilts), - 'java_excludes': ['**/*.java'], + 'java_excludes': ['**/*.java', '**/*.kt'], 'res_dirs': Relativize(res_dirs), } variables['android_test'] = [{ 'java_dirs': Relativize(junit_test_java_dirs), - 'java_excludes': ['**/*.java'], + 'java_excludes': ['**/*.java', '**/*.kt'], }] if native_targets: variables['native'] = _GetNative( @@ -660,9 +674,20 @@ os.path.join(gradle_output_dir, _MODULE_ALL, _CMAKE_FILE), cmake_data) -def _GenerateRootGradle(jinja_processor, channel): +def _GenerateRootGradle(jinja_processor, file_path): """Returns the data for the root project's build.gradle.""" - return jinja_processor.Render(_TemplatePath('root'), {'channel': channel}) + android_gradle_plugin_version = _ParseVersionFromFile( + file_path, r'com.android.tools.build:gradle:([\d.]+)', + _DEFAULT_ANDROID_GRADLE_PLUGIN_VERSION) + kotlin_gradle_plugin_version = _ParseVersionFromFile( + file_path, r'org.jetbrains.kotlin:kotlin-gradle-plugin:([\d.]+)', + _DEFAULT_KOTLIN_GRADLE_PLUGIN_VERSION) + + return jinja_processor.Render( + _TemplatePath('root'), { + 'android_gradle_plugin_version': android_gradle_plugin_version, + 'kotlin_gradle_plugin_version': kotlin_gradle_plugin_version, + }) def _GenerateSettingsGradle(project_entries): @@ -766,26 +791,11 @@ action='append', help='GN native targets to generate for. May be ' 'repeated.') - parser.add_argument('--compile-sdk-version', - type=int, - default=0, - help='Override compileSdkVersion for android sdk docs. ' - 'Useful when sources for android_sdk_version is ' - 'not available in Android Studio.') parser.add_argument( '--sdk-path', default=os.path.expanduser('~/Android/Sdk'), help='The path to use as the SDK root, overrides the ' 'default at ~/Android/Sdk.') - version_group = parser.add_mutually_exclusive_group() - version_group.add_argument('--beta', - action='store_true', - help='Generate a project that is compatible with ' - 'Android Studio Beta.') - version_group.add_argument('--canary', - action='store_true', - help='Generate a project that is compatible with ' - 'Android Studio Canary.') args = parser.parse_args() if args.output_directory: constants.SetOutputDirectory(args.output_directory) @@ -835,19 +845,9 @@ build_vars = gn_helpers.ReadBuildVars(output_dir) jinja_processor = jinja_template.JinjaProcessor(_FILE_DIR) - if args.beta: - channel = 'beta' - elif args.canary: - channel = 'canary' - else: - channel = 'stable' - if args.compile_sdk_version: - build_vars['compile_sdk_version'] = args.compile_sdk_version - else: - build_vars['compile_sdk_version'] = build_vars['android_sdk_version'] generator = _ProjectContextGenerator(_gradle_output_dir, build_vars, - args.use_gradle_process_resources, jinja_processor, args.split_projects, - channel) + args.use_gradle_process_resources, + jinja_processor, args.split_projects) main_entries = [_ProjectEntry.FromGnTarget(t) for t in targets] @@ -856,7 +856,7 @@ # used by apks/bundles/binaries/tests or that are explicitly mentioned in # --targets. BASE_TYPES = ('android_apk', 'android_app_bundle_module', 'java_binary', - 'junit_binary') + 'robolectric_binary') main_entries = [ e for e in main_entries if (e.GetType() in BASE_TYPES or e.GnTarget() in targets_from_args @@ -887,8 +887,9 @@ _GenerateModuleAll(_gradle_output_dir, generator, build_vars, jinja_processor, args.native_targets) - _WriteFile(os.path.join(generator.project_dir, _GRADLE_BUILD_FILE), - _GenerateRootGradle(jinja_processor, channel)) + root_gradle_path = os.path.join(generator.project_dir, _GRADLE_BUILD_FILE) + _WriteFile(root_gradle_path, + _GenerateRootGradle(jinja_processor, root_gradle_path)) _WriteFile(os.path.join(generator.project_dir, 'settings.gradle'), _GenerateSettingsGradle(project_entries)) @@ -906,10 +907,8 @@ wrapper_properties = os.path.join(generator.project_dir, 'gradle', 'wrapper', 'gradle-wrapper.properties') - if os.path.exists(wrapper_properties): - os.unlink(wrapper_properties) - if args.canary: - _WriteFile(wrapper_properties, _GenerateGradleWrapperPropertiesCanary()) + _WriteFile(wrapper_properties, + _GenerateGradleWrapperProperties(wrapper_properties)) generated_inputs = set() for entry in entries: @@ -919,13 +918,19 @@ # Build all paths references by .gradle that exist within output_dir. generated_inputs.update(generator.GeneratedInputs(entry_to_gen)) if generated_inputs: - targets = _RebasePath(generated_inputs, output_dir) + # Skip targets outside the output_dir since those are not generated. + targets = [ + p for p in _RebasePath(generated_inputs, output_dir) + if not p.startswith(os.pardir) + ] _RunNinja(output_dir, targets) - logging.warning('Generated files will only appear once you\'ve built them.') - logging.warning('Generated projects for Android Studio %s', channel) - logging.warning('For more tips: https://chromium.googlesource.com/chromium' - '/src.git/+/master/docs/android_studio.md') + print('Generated projects for Android Studio.') + print('** Building using Android Studio / Gradle does not work.') + print('** This project is only for IDE editing & tools.') + print('Note: Generated files will appear only if they have been built') + print('For more tips: https://chromium.googlesource.com/chromium/src.git/' + '+/main/docs/android_studio.md') if __name__ == '__main__':
diff --git a/build/android/gradle/gn_to_cmake.py b/build/android/gradle/gn_to_cmake.py deleted file mode 100755 index d3e80ae..0000000 --- a/build/android/gradle/gn_to_cmake.py +++ /dev/null
@@ -1,689 +0,0 @@ -#!/usr/bin/env python -# Copyright 2016 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" -Usage: gn_to_cmake.py <json_file_name> - -gn gen out/config --ide=json --json-ide-script=../../gn/gn_to_cmake.py - -or - -gn gen out/config --ide=json -python gn/gn_to_cmake.py out/config/project.json - -The first is recommended, as it will auto-update. -""" - -from __future__ import print_function - -import functools -import json -import posixpath -import string -import sys - - -def CMakeStringEscape(a): - """Escapes the string 'a' for use inside a CMake string. - - This means escaping - '\' otherwise it may be seen as modifying the next character - '"' otherwise it will end the string - ';' otherwise the string becomes a list - - The following do not need to be escaped - '#' when the lexer is in string state, this does not start a comment - """ - return a.replace('\\', '\\\\').replace(';', '\\;').replace('"', '\\"') - - -def CMakeTargetEscape(a): - """Escapes the string 'a' for use as a CMake target name. - - CMP0037 in CMake 3.0 restricts target names to "^[A-Za-z0-9_.:+-]+$" - The ':' is only allowed for imported targets. - """ - def Escape(c): - if c in string.ascii_letters or c in string.digits or c in '_.+-': - return c - else: - return '__' - return ''.join([Escape(c) for c in a]) - - -def SetVariable(out, variable_name, value): - """Sets a CMake variable.""" - out.write('set("') - out.write(CMakeStringEscape(variable_name)) - out.write('" "') - out.write(CMakeStringEscape(value)) - out.write('")\n') - - -def SetVariableList(out, variable_name, values): - """Sets a CMake variable to a list.""" - if not values: - return SetVariable(out, variable_name, "") - if len(values) == 1: - return SetVariable(out, variable_name, values[0]) - out.write('list(APPEND "') - out.write(CMakeStringEscape(variable_name)) - out.write('"\n "') - out.write('"\n "'.join([CMakeStringEscape(value) for value in values])) - out.write('")\n') - - -def SetFilesProperty(output, variable, property_name, values, sep): - """Given a set of source files, sets the given property on them.""" - output.write('set_source_files_properties(') - WriteVariable(output, variable) - output.write(' PROPERTIES ') - output.write(property_name) - output.write(' "') - for value in values: - output.write(CMakeStringEscape(value)) - output.write(sep) - output.write('")\n') - - -def SetCurrentTargetProperty(out, property_name, values, sep=''): - """Given a target, sets the given property.""" - out.write('set_target_properties("${target}" PROPERTIES ') - out.write(property_name) - out.write(' "') - for value in values: - out.write(CMakeStringEscape(value)) - out.write(sep) - out.write('")\n') - - -def WriteVariable(output, variable_name, prepend=None): - if prepend: - output.write(prepend) - output.write('${') - output.write(variable_name) - output.write('}') - - -# See GetSourceFileType in gn -source_file_types = { - '.cc': 'cxx', - '.cpp': 'cxx', - '.cxx': 'cxx', - '.c': 'c', - '.s': 'asm', - '.S': 'asm', - '.asm': 'asm', - '.o': 'obj', - '.obj': 'obj', -} - - -class CMakeTargetType(object): - def __init__(self, command, modifier, property_modifier, is_linkable): - self.command = command - self.modifier = modifier - self.property_modifier = property_modifier - self.is_linkable = is_linkable -CMakeTargetType.custom = CMakeTargetType('add_custom_target', 'SOURCES', - None, False) - -# See GetStringForOutputType in gn -cmake_target_types = { - 'unknown': CMakeTargetType.custom, - 'group': CMakeTargetType.custom, - 'executable': CMakeTargetType('add_executable', None, 'RUNTIME', True), - 'loadable_module': CMakeTargetType('add_library', 'MODULE', 'LIBRARY', True), - 'shared_library': CMakeTargetType('add_library', 'SHARED', 'LIBRARY', True), - 'static_library': CMakeTargetType('add_library', 'STATIC', 'ARCHIVE', False), - 'source_set': CMakeTargetType('add_library', 'OBJECT', None, False), - 'copy': CMakeTargetType.custom, - 'action': CMakeTargetType.custom, - 'action_foreach': CMakeTargetType.custom, - 'bundle_data': CMakeTargetType.custom, - 'create_bundle': CMakeTargetType.custom, -} - - -def FindFirstOf(s, a): - return min(s.find(i) for i in a if i in s) - - -def GetCMakeTargetName(gn_target_name): - # See <chromium>/src/tools/gn/label.cc#Resolve - # //base/test:test_support(//build/toolchain/win:msvc) - path_separator = FindFirstOf(gn_target_name, (':', '(')) - location = None - name = None - toolchain = None - if not path_separator: - location = gn_target_name[2:] - else: - location = gn_target_name[2:path_separator] - toolchain_separator = gn_target_name.find('(', path_separator) - if toolchain_separator == -1: - name = gn_target_name[path_separator + 1:] - else: - if toolchain_separator > path_separator: - name = gn_target_name[path_separator + 1:toolchain_separator] - assert gn_target_name.endswith(')') - toolchain = gn_target_name[toolchain_separator + 1:-1] - assert location or name - - cmake_target_name = None - if location.endswith('/' + name): - cmake_target_name = location - elif location: - cmake_target_name = location + '_' + name - else: - cmake_target_name = name - if toolchain: - cmake_target_name += '--' + toolchain - return CMakeTargetEscape(cmake_target_name) - - -class Project(object): - def __init__(self, project_json): - self.targets = project_json['targets'] - build_settings = project_json['build_settings'] - self.root_path = build_settings['root_path'] - self.build_path = posixpath.join(self.root_path, - build_settings['build_dir'][2:]) - self.object_source_deps = {} - - def GetAbsolutePath(self, path): - if path.startswith("//"): - return self.root_path + "/" + path[2:] - else: - return path - - def GetObjectSourceDependencies(self, gn_target_name, object_dependencies): - """All OBJECT libraries whose sources have not been absorbed.""" - if gn_target_name in self.object_source_deps: - object_dependencies.update(self.object_source_deps[gn_target_name]) - return - target_deps = set() - dependencies = self.targets[gn_target_name].get('deps', []) - for dependency in dependencies: - dependency_type = self.targets[dependency].get('type', None) - if dependency_type == 'source_set': - target_deps.add(dependency) - if dependency_type not in gn_target_types_that_absorb_objects: - self.GetObjectSourceDependencies(dependency, target_deps) - self.object_source_deps[gn_target_name] = target_deps - object_dependencies.update(target_deps) - - def GetObjectLibraryDependencies(self, gn_target_name, object_dependencies): - """All OBJECT libraries whose libraries have not been absorbed.""" - dependencies = self.targets[gn_target_name].get('deps', []) - for dependency in dependencies: - dependency_type = self.targets[dependency].get('type', None) - if dependency_type == 'source_set': - object_dependencies.add(dependency) - self.GetObjectLibraryDependencies(dependency, object_dependencies) - - -class Target(object): - def __init__(self, gn_target_name, project): - self.gn_name = gn_target_name - self.properties = project.targets[self.gn_name] - self.cmake_name = GetCMakeTargetName(self.gn_name) - self.gn_type = self.properties.get('type', None) - self.cmake_type = cmake_target_types.get(self.gn_type, None) - - -def WriteAction(out, target, project, sources, synthetic_dependencies): - outputs = [] - output_directories = set() - for output in target.properties.get('outputs', []): - output_abs_path = project.GetAbsolutePath(output) - outputs.append(output_abs_path) - output_directory = posixpath.dirname(output_abs_path) - if output_directory: - output_directories.add(output_directory) - outputs_name = '${target}__output' - SetVariableList(out, outputs_name, outputs) - - out.write('add_custom_command(OUTPUT ') - WriteVariable(out, outputs_name) - out.write('\n') - - if output_directories: - out.write(' COMMAND ${CMAKE_COMMAND} -E make_directory "') - out.write('" "'.join([CMakeStringEscape(d) for d in output_directories])) - out.write('"\n') - - script = target.properties['script'] - arguments = target.properties['args'] - out.write(' COMMAND python "') - out.write(CMakeStringEscape(project.GetAbsolutePath(script))) - out.write('"') - if arguments: - out.write('\n "') - out.write('"\n "'.join([CMakeStringEscape(a) for a in arguments])) - out.write('"') - out.write('\n') - - out.write(' DEPENDS ') - for sources_type_name in sources.values(): - WriteVariable(out, sources_type_name, ' ') - out.write('\n') - - #TODO: CMake 3.7 is introducing DEPFILE - - out.write(' WORKING_DIRECTORY "') - out.write(CMakeStringEscape(project.build_path)) - out.write('"\n') - - out.write(' COMMENT "Action: ${target}"\n') - - out.write(' VERBATIM)\n') - - synthetic_dependencies.add(outputs_name) - - -def ExpandPlaceholders(source, a): - source_dir, source_file_part = posixpath.split(source) - source_name_part, _ = posixpath.splitext(source_file_part) - #TODO: {{source_gen_dir}}, {{source_out_dir}}, {{response_file_name}} - return a.replace('{{source}}', source) \ - .replace('{{source_file_part}}', source_file_part) \ - .replace('{{source_name_part}}', source_name_part) \ - .replace('{{source_dir}}', source_dir) \ - .replace('{{source_root_relative_dir}}', source_dir) - - -def WriteActionForEach(out, target, project, sources, synthetic_dependencies): - all_outputs = target.properties.get('outputs', []) - inputs = target.properties.get('sources', []) - # TODO: consider expanding 'output_patterns' instead. - outputs_per_input = len(all_outputs) / len(inputs) - for count, source in enumerate(inputs): - source_abs_path = project.GetAbsolutePath(source) - - outputs = [] - output_directories = set() - for output in all_outputs[outputs_per_input * count: - outputs_per_input * (count+1)]: - output_abs_path = project.GetAbsolutePath(output) - outputs.append(output_abs_path) - output_directory = posixpath.dirname(output_abs_path) - if output_directory: - output_directories.add(output_directory) - outputs_name = '${target}__output_' + str(count) - SetVariableList(out, outputs_name, outputs) - - out.write('add_custom_command(OUTPUT ') - WriteVariable(out, outputs_name) - out.write('\n') - - if output_directories: - out.write(' COMMAND ${CMAKE_COMMAND} -E make_directory "') - out.write('" "'.join([CMakeStringEscape(d) for d in output_directories])) - out.write('"\n') - - script = target.properties['script'] - # TODO: need to expand {{xxx}} in arguments - arguments = target.properties['args'] - out.write(' COMMAND python "') - out.write(CMakeStringEscape(project.GetAbsolutePath(script))) - out.write('"') - if arguments: - out.write('\n "') - expand = functools.partial(ExpandPlaceholders, source_abs_path) - out.write('"\n "'.join( - [CMakeStringEscape(expand(a)) for a in arguments])) - out.write('"') - out.write('\n') - - out.write(' DEPENDS') - if 'input' in sources: - WriteVariable(out, sources['input'], ' ') - out.write(' "') - out.write(CMakeStringEscape(source_abs_path)) - out.write('"\n') - - #TODO: CMake 3.7 is introducing DEPFILE - - out.write(' WORKING_DIRECTORY "') - out.write(CMakeStringEscape(project.build_path)) - out.write('"\n') - - out.write(' COMMENT "Action ${target} on ') - out.write(CMakeStringEscape(source_abs_path)) - out.write('"\n') - - out.write(' VERBATIM)\n') - - synthetic_dependencies.add(outputs_name) - - -def WriteCopy(out, target, project, sources, synthetic_dependencies): - inputs = target.properties.get('sources', []) - raw_outputs = target.properties.get('outputs', []) - - # TODO: consider expanding 'output_patterns' instead. - outputs = [] - for output in raw_outputs: - output_abs_path = project.GetAbsolutePath(output) - outputs.append(output_abs_path) - outputs_name = '${target}__output' - SetVariableList(out, outputs_name, outputs) - - out.write('add_custom_command(OUTPUT ') - WriteVariable(out, outputs_name) - out.write('\n') - - for src, dst in zip(inputs, outputs): - out.write(' COMMAND ${CMAKE_COMMAND} -E copy "') - out.write(CMakeStringEscape(project.GetAbsolutePath(src))) - out.write('" "') - out.write(CMakeStringEscape(dst)) - out.write('"\n') - - out.write(' DEPENDS ') - for sources_type_name in sources.values(): - WriteVariable(out, sources_type_name, ' ') - out.write('\n') - - out.write(' WORKING_DIRECTORY "') - out.write(CMakeStringEscape(project.build_path)) - out.write('"\n') - - out.write(' COMMENT "Copy ${target}"\n') - - out.write(' VERBATIM)\n') - - synthetic_dependencies.add(outputs_name) - - -def WriteCompilerFlags(out, target, project, sources): - # Hack, set linker language to c if no c or cxx files present. - if not 'c' in sources and not 'cxx' in sources: - SetCurrentTargetProperty(out, 'LINKER_LANGUAGE', ['C']) - - # Mark uncompiled sources as uncompiled. - if 'input' in sources: - SetFilesProperty(out, sources['input'], 'HEADER_FILE_ONLY', ('True',), '') - if 'other' in sources: - SetFilesProperty(out, sources['other'], 'HEADER_FILE_ONLY', ('True',), '') - - # Mark object sources as linkable. - if 'obj' in sources: - SetFilesProperty(out, sources['obj'], 'EXTERNAL_OBJECT', ('True',), '') - - # TODO: 'output_name', 'output_dir', 'output_extension' - # This includes using 'source_outputs' to direct compiler output. - - # Includes - includes = target.properties.get('include_dirs', []) - if includes: - out.write('set_property(TARGET "${target}" ') - out.write('APPEND PROPERTY INCLUDE_DIRECTORIES') - for include_dir in includes: - out.write('\n "') - out.write(project.GetAbsolutePath(include_dir)) - out.write('"') - out.write(')\n') - - # Defines - defines = target.properties.get('defines', []) - if defines: - SetCurrentTargetProperty(out, 'COMPILE_DEFINITIONS', defines, ';') - - # Compile flags - # "arflags", "asmflags", "cflags", - # "cflags_c", "clfags_cc", "cflags_objc", "clfags_objcc" - # CMake does not have per target lang compile flags. - # TODO: $<$<COMPILE_LANGUAGE:CXX>:cflags_cc style generator expression. - # http://public.kitware.com/Bug/view.php?id=14857 - flags = [] - flags.extend(target.properties.get('cflags', [])) - cflags_asm = target.properties.get('asmflags', []) - cflags_c = target.properties.get('cflags_c', []) - cflags_cxx = target.properties.get('cflags_cc', []) - if 'c' in sources and not any(k in sources for k in ('asm', 'cxx')): - flags.extend(cflags_c) - elif 'cxx' in sources and not any(k in sources for k in ('asm', 'c')): - flags.extend(cflags_cxx) - else: - # TODO: This is broken, one cannot generally set properties on files, - # as other targets may require different properties on the same files. - if 'asm' in sources and cflags_asm: - SetFilesProperty(out, sources['asm'], 'COMPILE_FLAGS', cflags_asm, ' ') - if 'c' in sources and cflags_c: - SetFilesProperty(out, sources['c'], 'COMPILE_FLAGS', cflags_c, ' ') - if 'cxx' in sources and cflags_cxx: - SetFilesProperty(out, sources['cxx'], 'COMPILE_FLAGS', cflags_cxx, ' ') - if flags: - SetCurrentTargetProperty(out, 'COMPILE_FLAGS', flags, ' ') - - # Linker flags - ldflags = target.properties.get('ldflags', []) - if ldflags: - SetCurrentTargetProperty(out, 'LINK_FLAGS', ldflags, ' ') - - -gn_target_types_that_absorb_objects = ( - 'executable', - 'loadable_module', - 'shared_library', - 'static_library' -) - - -def WriteSourceVariables(out, target, project): - # gn separates the sheep from the goats based on file extensions. - # A full separation is done here because of flag handing (see Compile flags). - source_types = {'cxx':[], 'c':[], 'asm':[], - 'obj':[], 'obj_target':[], 'input':[], 'other':[]} - - # TODO .def files on Windows - for source in target.properties.get('sources', []): - _, ext = posixpath.splitext(source) - source_abs_path = project.GetAbsolutePath(source) - source_types[source_file_types.get(ext, 'other')].append(source_abs_path) - - for input_path in target.properties.get('inputs', []): - input_abs_path = project.GetAbsolutePath(input_path) - source_types['input'].append(input_abs_path) - - # OBJECT library dependencies need to be listed as sources. - # Only executables and non-OBJECT libraries may reference an OBJECT library. - # https://gitlab.kitware.com/cmake/cmake/issues/14778 - if target.gn_type in gn_target_types_that_absorb_objects: - object_dependencies = set() - project.GetObjectSourceDependencies(target.gn_name, object_dependencies) - for dependency in object_dependencies: - cmake_dependency_name = GetCMakeTargetName(dependency) - obj_target_sources = '$<TARGET_OBJECTS:' + cmake_dependency_name + '>' - source_types['obj_target'].append(obj_target_sources) - - sources = {} - for source_type, sources_of_type in source_types.items(): - if sources_of_type: - sources[source_type] = '${target}__' + source_type + '_srcs' - SetVariableList(out, sources[source_type], sources_of_type) - return sources - - -def WriteTarget(out, target, project): - out.write('\n#') - out.write(target.gn_name) - out.write('\n') - - if target.cmake_type is None: - print('Target {} has unknown target type {}, skipping.'.format( - target.gn_name, target.gn_type)) - return - - SetVariable(out, 'target', target.cmake_name) - - sources = WriteSourceVariables(out, target, project) - - synthetic_dependencies = set() - if target.gn_type == 'action': - WriteAction(out, target, project, sources, synthetic_dependencies) - if target.gn_type == 'action_foreach': - WriteActionForEach(out, target, project, sources, synthetic_dependencies) - if target.gn_type == 'copy': - WriteCopy(out, target, project, sources, synthetic_dependencies) - - out.write(target.cmake_type.command) - out.write('("${target}"') - if target.cmake_type.modifier is not None: - out.write(' ') - out.write(target.cmake_type.modifier) - for sources_type_name in sources.values(): - WriteVariable(out, sources_type_name, ' ') - if synthetic_dependencies: - out.write(' DEPENDS') - for synthetic_dependencie in synthetic_dependencies: - WriteVariable(out, synthetic_dependencie, ' ') - out.write(')\n') - - if target.cmake_type.command != 'add_custom_target': - WriteCompilerFlags(out, target, project, sources) - - libraries = set() - nonlibraries = set() - - dependencies = set(target.properties.get('deps', [])) - # Transitive OBJECT libraries are in sources. - # Those sources are dependent on the OBJECT library dependencies. - # Those sources cannot bring in library dependencies. - object_dependencies = set() - if target.gn_type != 'source_set': - project.GetObjectLibraryDependencies(target.gn_name, object_dependencies) - for object_dependency in object_dependencies: - dependencies.update(project.targets.get(object_dependency).get('deps', [])) - - for dependency in dependencies: - gn_dependency_type = project.targets.get(dependency, {}).get('type', None) - cmake_dependency_type = cmake_target_types.get(gn_dependency_type, None) - cmake_dependency_name = GetCMakeTargetName(dependency) - if cmake_dependency_type.command != 'add_library': - nonlibraries.add(cmake_dependency_name) - elif cmake_dependency_type.modifier != 'OBJECT': - if target.cmake_type.is_linkable: - libraries.add(cmake_dependency_name) - else: - nonlibraries.add(cmake_dependency_name) - - # Non-library dependencies. - if nonlibraries: - out.write('add_dependencies("${target}"') - for nonlibrary in nonlibraries: - out.write('\n "') - out.write(nonlibrary) - out.write('"') - out.write(')\n') - - # Non-OBJECT library dependencies. - external_libraries = target.properties.get('libs', []) - if target.cmake_type.is_linkable and (external_libraries or libraries): - library_dirs = target.properties.get('lib_dirs', []) - if library_dirs: - SetVariableList(out, '${target}__library_directories', library_dirs) - - system_libraries = [] - for external_library in external_libraries: - if '/' in external_library: - libraries.add(project.GetAbsolutePath(external_library)) - else: - if external_library.endswith('.framework'): - external_library = external_library[:-len('.framework')] - system_library = 'library__' + external_library - if library_dirs: - system_library = system_library + '__for_${target}' - out.write('find_library("') - out.write(CMakeStringEscape(system_library)) - out.write('" "') - out.write(CMakeStringEscape(external_library)) - out.write('"') - if library_dirs: - out.write(' PATHS "') - WriteVariable(out, '${target}__library_directories') - out.write('"') - out.write(')\n') - system_libraries.append(system_library) - out.write('target_link_libraries("${target}"') - for library in libraries: - out.write('\n "') - out.write(CMakeStringEscape(library)) - out.write('"') - for system_library in system_libraries: - WriteVariable(out, system_library, '\n "') - out.write('"') - out.write(')\n') - - -def WriteProject(project): - out = open(posixpath.join(project.build_path, 'CMakeLists.txt'), 'w+') - out.write('# Generated by gn_to_cmake.py.\n') - out.write('cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR)\n') - out.write('cmake_policy(VERSION 2.8.8)\n\n') - - # Update the gn generated ninja build. - # If a build file has changed, this will update CMakeLists.ext if - # gn gen out/config --ide=json --json-ide-script=../../gn/gn_to_cmake.py - # style was used to create this config. - out.write('execute_process(COMMAND ninja -C "') - out.write(CMakeStringEscape(project.build_path)) - out.write('" build.ninja)\n') - - out.write('include(CMakeLists.ext)\n') - out.close() - - out = open(posixpath.join(project.build_path, 'CMakeLists.ext'), 'w+') - out.write('# Generated by gn_to_cmake.py.\n') - out.write('cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR)\n') - out.write('cmake_policy(VERSION 2.8.8)\n') - - # The following appears to be as-yet undocumented. - # http://public.kitware.com/Bug/view.php?id=8392 - out.write('enable_language(ASM)\n\n') - # ASM-ATT does not support .S files. - # output.write('enable_language(ASM-ATT)\n') - - # Current issues with automatic re-generation: - # The gn generated build.ninja target uses build.ninja.d - # but build.ninja.d does not contain the ide or gn. - # Currently the ide is not run if the project.json file is not changed - # but the ide needs to be run anyway if it has itself changed. - # This can be worked around by deleting the project.json file. - out.write('file(READ "') - gn_deps_file = posixpath.join(project.build_path, 'build.ninja.d') - out.write(CMakeStringEscape(gn_deps_file)) - out.write('" "gn_deps_string" OFFSET ') - out.write(str(len('build.ninja: '))) - out.write(')\n') - # One would think this would need to worry about escaped spaces - # but gn doesn't escape spaces here (it generates invalid .d files). - out.write('string(REPLACE " " ";" "gn_deps" ${gn_deps_string})\n') - out.write('foreach("gn_dep" ${gn_deps})\n') - out.write(' configure_file(${gn_dep} "CMakeLists.devnull" COPYONLY)\n') - out.write('endforeach("gn_dep")\n') - - for target_name in project.targets.keys(): - out.write('\n') - WriteTarget(out, Target(target_name, project), project) - - -def main(): - if len(sys.argv) != 2: - print('Usage: ' + sys.argv[0] + ' <json_file_name>') - exit(1) - - json_path = sys.argv[1] - project = None - with open(json_path, 'r') as json_file: - project = json.loads(json_file.read()) - - WriteProject(Project(project)) - - -if __name__ == "__main__": - main()
diff --git a/build/android/gradle/java.jinja b/build/android/gradle/java.jinja index 7626f61..61886e9 100644 --- a/build/android/gradle/java.jinja +++ b/build/android/gradle/java.jinja
@@ -25,8 +25,8 @@ } } -sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_11 +targetCompatibility = JavaVersion.VERSION_11 {% if template_type == 'java_binary' %} applicationName = "{{ target_name }}"
diff --git a/build/android/gradle/root.jinja b/build/android/gradle/root.jinja index 15b5e10..8009ebe 100644 --- a/build/android/gradle/root.jinja +++ b/build/android/gradle/root.jinja
@@ -3,24 +3,22 @@ {# found in the LICENSE file. #} // Generated by //build/android/generate_gradle.py +// This section is used to find the plugins. buildscript { repositories { google() - jcenter() -{% if channel == 'canary' %} - // Workaround for http://b/144885480. - //maven() { - // url "http://dl.bintray.com/kotlin/kotlin-eap" - //} -{% endif %} + mavenCentral() } dependencies { -{% if channel == 'canary' %} - classpath "com.android.tools.build:gradle:4.1.0-beta01" -{% elif channel == 'beta' %} - classpath "com.android.tools.build:gradle:4.0.0-rc01" -{% else %} - classpath "com.android.tools.build:gradle:4.0.1" -{% endif %} + classpath "com.android.tools.build:gradle:{{ android_gradle_plugin_version }}" + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:{{ kotlin_gradle_plugin_version }}" } } + +// This is used by individual modules to find/fetch dependencies. +allprojects { + repositories { + google() + mavenCentral() + } +} \ No newline at end of file
diff --git a/build/android/gtest_apk/BUILD.gn b/build/android/gtest_apk/BUILD.gn index 2a72bc4..69b0889 100644 --- a/build/android/gtest_apk/BUILD.gn +++ b/build/android/gtest_apk/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java index 652333b..7f5c4a8 100644 --- a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java +++ b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestInstrumentationTestRunner.java
@@ -1,4 +1,4 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java index a875e97..2020784 100644 --- a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java +++ b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/NativeTestIntent.java
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java index 520b748..98ebf44 100644 --- a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java +++ b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusIntent.java
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java index e539009..71c56a6 100644 --- a/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java +++ b/build/android/gtest_apk/java/src/org/chromium/build/gtest_apk/TestStatusReceiver.java
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gyp/OWNERS b/build/android/gyp/OWNERS index 25557e1..df0fa64 100644 --- a/build/android/gyp/OWNERS +++ b/build/android/gyp/OWNERS
@@ -2,3 +2,5 @@ digit@chromium.org smaier@chromium.org wnwen@chromium.org + +per-file create_unwind_table*.py=file://base/profiler/OWNERS \ No newline at end of file
diff --git a/build/android/gyp/aar.py b/build/android/gyp/aar.py index b157cd8..512d5db 100755 --- a/build/android/gyp/aar.py +++ b/build/android/gyp/aar.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,9 +16,7 @@ import zipfile from util import build_utils - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.pardir, os.pardir))) +import action_helpers # build_utils adds //build to sys.path. import gn_helpers @@ -58,11 +56,12 @@ return True -def _CreateInfo(aar_file): +def _CreateInfo(aar_file, resource_exclusion_globs): """Extracts and return .info data from an .aar file. Args: aar_file: Path to an input .aar file. + resource_exclusion_globs: List of globs that exclude res/ files. Returns: A dict containing .info data. @@ -90,7 +89,8 @@ if name.startswith('aidl/'): data['aidl'].append(name) elif name.startswith('res/'): - data['resources'].append(name) + if not build_utils.MatchesGlob(name, resource_exclusion_globs): + data['resources'].append(name) elif name.startswith('libs/') and name.endswith('.jar'): label = posixpath.basename(name)[:-4] label = re.sub(r'[^a-zA-Z0-9._]', '_', label) @@ -133,6 +133,11 @@ def _AddCommonArgs(parser): parser.add_argument( 'aar_file', help='Path to the AAR file.', type=os.path.normpath) + parser.add_argument('--ignore-resources', + action='store_true', + help='Whether to skip extraction of res/') + parser.add_argument('--resource-exclusion-globs', + help='GN list of globs for res/ files to ignore') def main(): @@ -155,14 +160,15 @@ help='Path to .info file. Asserts that it matches what ' '"list" would output.', type=argparse.FileType('r')) - subp.add_argument( - '--ignore-resources', - action='store_true', - help='Whether to skip extraction of res/') args = parser.parse_args() - aar_info = _CreateInfo(args.aar_file) + args.resource_exclusion_globs = action_helpers.parse_gn_list( + args.resource_exclusion_globs) + if args.ignore_resources: + args.resource_exclusion_globs.append('res/*') + + aar_info = _CreateInfo(args.aar_file, args.resource_exclusion_globs) formatted_info = """\ # Generated by //build/android/gyp/aar.py # To regenerate, use "update_android_aar_prebuilts = true" and run "gn gen". @@ -177,18 +183,18 @@ 'out-of-date. Run gn gen with ' 'update_android_aar_prebuilts=true to update it.') + # Extract all files except for filtered res/ files. with zipfile.ZipFile(args.aar_file) as zf: - names = zf.namelist() - if args.ignore_resources: - names = [n for n in names if not n.startswith('res')] + names = {n for n in zf.namelist() if not n.startswith('res/')} + names.update(aar_info['resources']) - _PerformExtract(args.aar_file, args.output_dir, set(names)) + _PerformExtract(args.aar_file, args.output_dir, names) elif args.command == 'list': aar_output_present = args.output != '-' and os.path.isfile(args.output) if aar_output_present: # Some .info files are read-only, for examples the cipd-controlled ones - # under third_party/android_deps/repositoty. To deal with these, first + # under third_party/android_deps/repository. To deal with these, first # that its content is correct, and if it is, exit without touching # the file system. file_info = open(args.output, 'r').read() @@ -203,8 +209,8 @@ except IOError as e: if not aar_output_present: raise e - raise Exception('Could not update output file: %s\n%s\n' % - (args.output, e)) + raise Exception('Could not update output file: %s\n' % args.output) from e + if __name__ == '__main__': sys.exit(main())
diff --git a/build/android/gyp/aar.pydeps b/build/android/gyp/aar.pydeps index 7e2924b..56f860e 100644 --- a/build/android/gyp/aar.pydeps +++ b/build/android/gyp/aar.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/aar.pydeps build/android/gyp/aar.py +../../action_helpers.py ../../gn_helpers.py aar.py util/__init__.py
diff --git a/build/android/gyp/aidl.py b/build/android/gyp/aidl.py index b8099aa..8eab45d 100755 --- a/build/android/gyp/aidl.py +++ b/build/android/gyp/aidl.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -14,6 +14,8 @@ import zipfile from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers def main(argv): @@ -23,10 +25,10 @@ option_parser.add_option('--includes', help='Directories to add as import search paths.') option_parser.add_option('--srcjar', help='Path for srcjar output.') - build_utils.AddDepfileOption(option_parser) + action_helpers.add_depfile_arg(option_parser) options, args = option_parser.parse_args(argv[1:]) - options.includes = build_utils.ParseGnList(options.includes) + options.includes = action_helpers.parse_gn_list(options.includes) with build_utils.TempDir() as temp_dir: for f in args: @@ -34,7 +36,7 @@ output = os.path.join(temp_dir, classname + '.java') aidl_cmd = [options.aidl_path] aidl_cmd += [ - '-p' + s for s in build_utils.ParseGnList(options.imports) + '-p' + s for s in action_helpers.parse_gn_list(options.imports) ] aidl_cmd += ['-I' + s for s in options.includes] aidl_cmd += [ @@ -43,7 +45,7 @@ ] build_utils.CheckOutput(aidl_cmd) - with build_utils.AtomicOutput(options.srcjar) as f: + with action_helpers.atomic_output(options.srcjar) as f: with zipfile.ZipFile(f, 'w') as srcjar: for path in build_utils.FindInDirectory(temp_dir, '*.java'): with open(path) as fileobj: @@ -51,13 +53,13 @@ pkg_name = re.search(r'^\s*package\s+(.*?)\s*;', data, re.M).group(1) arcname = '%s/%s' % ( pkg_name.replace('.', '/'), os.path.basename(path)) - build_utils.AddToZipHermetic(srcjar, arcname, data=data) + zip_helpers.add_to_zip_hermetic(srcjar, arcname, data=data) if options.depfile: include_files = [] for include_dir in options.includes: include_files += build_utils.FindInDirectory(include_dir, '*.java') - build_utils.WriteDepfile(options.depfile, options.srcjar, include_files) + action_helpers.write_depfile(options.depfile, options.srcjar, include_files) if __name__ == '__main__':
diff --git a/build/android/gyp/aidl.pydeps b/build/android/gyp/aidl.pydeps index 11c55ed..d841c94 100644 --- a/build/android/gyp/aidl.pydeps +++ b/build/android/gyp/aidl.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/aidl.pydeps build/android/gyp/aidl.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py aidl.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/allot_native_libraries.py b/build/android/gyp/allot_native_libraries.py index 978b173..61daac2 100755 --- a/build/android/gyp/allot_native_libraries.py +++ b/build/android/gyp/allot_native_libraries.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -46,6 +46,7 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. def _ModuleLibrariesPair(arg): @@ -145,7 +146,7 @@ help='A pair of parent module name and child module name ' '(format: "<parent>:<child>"). Can be specified multiple times.') options = parser.parse_args(build_utils.ExpandFileArgs(args)) - options.libraries = [(m, build_utils.ParseGnList(l)) + options.libraries = [(m, action_helpers.parse_gn_list(l)) for m, l in options.libraries] # Parse input creating libraries and dependency tree.
diff --git a/build/android/gyp/allot_native_libraries.pydeps b/build/android/gyp/allot_native_libraries.pydeps index d8b10cd..aacaaff 100644 --- a/build/android/gyp/allot_native_libraries.pydeps +++ b/build/android/gyp/allot_native_libraries.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/allot_native_libraries.pydeps build/android/gyp/allot_native_libraries.py +../../action_helpers.py ../../gn_helpers.py allot_native_libraries.py util/__init__.py
diff --git a/build/android/gyp/apkbuilder.py b/build/android/gyp/apkbuilder.py index f1e6563..fa5701b 100755 --- a/build/android/gyp/apkbuilder.py +++ b/build/android/gyp/apkbuilder.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,6 +9,7 @@ import argparse import logging import os +import posixpath import shutil import sys import tempfile @@ -19,10 +20,8 @@ from util import build_utils from util import diff_utils -from util import zipalign - -# Input dex.jar files are zipaligned. -zipalign.ApplyZipFileZipAlignFix() +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers # Taken from aapt's Package.cpp: @@ -35,11 +34,11 @@ def _ParseArgs(args): parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) - parser.add_argument( - '--assets', - help='GYP-list of files to add as assets in the form ' - '"srcPath:zipPath", where ":zipPath" is optional.') + action_helpers.add_depfile_arg(parser) + parser.add_argument('--assets', + action='append', + help='GYP-list of files to add as assets in the form ' + '"srcPath:zipPath", where ":zipPath" is optional.') parser.add_argument( '--java-resources', help='GYP-list of java_resources JARs to include.') parser.add_argument('--write-asset-list', @@ -58,9 +57,6 @@ default='apk', help='Specify output format.') parser.add_argument('--dex-file', help='Path to the classes.dex to use') - parser.add_argument( - '--jdk-libs-dex-file', - help='Path to classes.dex created by dex_jdk_libs.py') parser.add_argument('--uncompress-dex', action='store_true', help='Store .dex files uncompressed in the APK') parser.add_argument('--native-libs', @@ -115,43 +111,24 @@ '--library-always-compress', action='append', help='The list of library files that we always compress.') - parser.add_argument( - '--library-renames', - action='append', - help='The list of library files that we prepend crazy. to their names.') parser.add_argument('--warnings-as-errors', action='store_true', help='Treat all warnings as errors.') diff_utils.AddCommandLineFlags(parser) options = parser.parse_args(args) - options.assets = build_utils.ParseGnList(options.assets) - options.uncompressed_assets = build_utils.ParseGnList( + options.assets = action_helpers.parse_gn_list(options.assets) + options.uncompressed_assets = action_helpers.parse_gn_list( options.uncompressed_assets) - options.native_lib_placeholders = build_utils.ParseGnList( + options.native_lib_placeholders = action_helpers.parse_gn_list( options.native_lib_placeholders) - options.secondary_native_lib_placeholders = build_utils.ParseGnList( + options.secondary_native_lib_placeholders = action_helpers.parse_gn_list( options.secondary_native_lib_placeholders) - options.java_resources = build_utils.ParseGnList(options.java_resources) - options.native_libs = build_utils.ParseGnList(options.native_libs) - options.secondary_native_libs = build_utils.ParseGnList( + options.java_resources = action_helpers.parse_gn_list(options.java_resources) + options.native_libs = action_helpers.parse_gn_list(options.native_libs) + options.secondary_native_libs = action_helpers.parse_gn_list( options.secondary_native_libs) - options.library_always_compress = build_utils.ParseGnList( + options.library_always_compress = action_helpers.parse_gn_list( options.library_always_compress) - options.library_renames = build_utils.ParseGnList(options.library_renames) - - # --apksigner-jar, --zipalign-path, --key-xxx arguments are - # required when building an APK, but not a bundle module. - if options.format == 'apk': - required_args = [ - 'apksigner_jar', 'zipalign_path', 'key_path', 'key_passwd', 'key_name' - ] - for required in required_args: - if not vars(options)[required]: - raise Exception('Argument --%s is required for APKs.' % ( - required.replace('_', '-'))) - - options.uncompress_shared_libraries = \ - options.uncompress_shared_libraries in [ 'true', 'True' ] if not options.android_abi and (options.native_libs or options.native_lib_placeholders): @@ -203,7 +180,8 @@ def _GetAssetsToAdd(path_tuples, fast_align, disable_compression=False, - allow_reads=True): + allow_reads=True, + apk_root_dir=''): """Returns the list of file_detail tuples for assets in the apk. Args: @@ -227,12 +205,16 @@ os.path.splitext(src_path)[1] not in _NO_COMPRESS_EXTENSIONS) if target_compress == compress: - # AddToZipHermetic() uses this logic to avoid growing small files. + # add_to_zip_hermetic() uses this logic to avoid growing small files. # We need it here in order to set alignment correctly. if allow_reads and compress and os.path.getsize(src_path) < 16: compress = False - apk_path = 'assets/' + dest_path + if dest_path.startswith('../'): + # posixpath.join('', 'foo') == 'foo' + apk_path = posixpath.join(apk_root_dir, dest_path[3:]) + else: + apk_path = 'assets/' + dest_path alignment = 0 if compress and not fast_align else 4 assets_to_add.append((apk_path, src_path, compress, alignment)) return assets_to_add @@ -255,16 +237,15 @@ raise Exception( 'Multiple targets specified the asset path: %s' % apk_path) except KeyError: - zipalign.AddToZipHermetic( - apk, - apk_path, - src_path=src_path, - compress=compress, - alignment=alignment) + zip_helpers.add_to_zip_hermetic(apk, + apk_path, + src_path=src_path, + compress=compress, + alignment=alignment) -def _GetNativeLibrariesToAdd(native_libs, android_abi, uncompress, fast_align, - lib_always_compress, lib_renames): +def _GetNativeLibrariesToAdd(native_libs, android_abi, fast_align, + lib_always_compress): """Returns the list of file_detail tuples for native libraries in the apk. Returns: A list of (src_path, apk_path, compress, alignment) tuple @@ -275,12 +256,7 @@ for path in native_libs: basename = os.path.basename(path) - compress = not uncompress or any(lib_name in basename - for lib_name in lib_always_compress) - rename = any(lib_name in basename for lib_name in lib_renames) - if rename: - basename = 'crazy.' + basename - + compress = any(lib_name in basename for lib_name in lib_always_compress) lib_android_abi = android_abi if path.startswith('android_clang_arm64_hwasan/'): lib_android_abi = 'arm64-v8a-hwasan' @@ -318,10 +294,11 @@ # Compresses about twice as fast as the default. zlib.Z_DEFAULT_COMPRESSION = 1 - # Manually align only when alignment is necessary. # Python's zip implementation duplicates file comments in the central # directory, whereas zipalign does not, so use zipalign for official builds. - fast_align = options.format == 'apk' and not options.best_compression + requires_alignment = options.format == 'apk' + run_zipalign = requires_alignment and options.best_compression + fast_align = bool(requires_alignment and not run_zipalign) native_libs = sorted(options.native_libs) @@ -341,15 +318,16 @@ depfile_deps += secondary_native_libs if options.java_resources: - # Included via .build_config, so need to write it to depfile. + # Included via .build_config.json, so need to write it to depfile. depfile_deps.extend(options.java_resources) assets = _ExpandPaths(options.assets) uncompressed_assets = _ExpandPaths(options.uncompressed_assets) - # Included via .build_config, so need to write it to depfile. + # Included via .build_config.json, so need to write it to depfile. depfile_deps.extend(x[0] for x in assets) depfile_deps.extend(x[0] for x in uncompressed_assets) + depfile_deps.append(options.resource_apk) # Bundle modules have a structure similar to APKs, except that resources # are compiled in protobuf format (instead of binary xml), and that some @@ -375,23 +353,24 @@ ret = _GetAssetsToAdd(assets, fast_align, disable_compression=False, - allow_reads=allow_reads) + allow_reads=allow_reads, + apk_root_dir=apk_root_dir) ret.extend( _GetAssetsToAdd(uncompressed_assets, fast_align, disable_compression=True, - allow_reads=allow_reads)) + allow_reads=allow_reads, + apk_root_dir=apk_root_dir)) return ret - libs_to_add = _GetNativeLibrariesToAdd( - native_libs, options.android_abi, options.uncompress_shared_libraries, - fast_align, options.library_always_compress, options.library_renames) + libs_to_add = _GetNativeLibrariesToAdd(native_libs, options.android_abi, + fast_align, + options.library_always_compress) if options.secondary_android_abi: libs_to_add.extend( - _GetNativeLibrariesToAdd( - secondary_native_libs, options.secondary_android_abi, - options.uncompress_shared_libraries, fast_align, - options.library_always_compress, options.library_renames)) + _GetNativeLibrariesToAdd(secondary_native_libs, + options.secondary_android_abi, + fast_align, options.library_always_compress)) if options.expected_file: # We compute expectations without reading the files. This allows us to check @@ -408,9 +387,9 @@ if options.only_verify_expectations: if options.depfile: - build_utils.WriteDepfile(options.depfile, - options.actual_file, - inputs=depfile_deps) + action_helpers.write_depfile(options.depfile, + options.actual_file, + inputs=depfile_deps) return # If we are past this point, we are going to actually create the final apk so @@ -420,12 +399,13 @@ assets, uncompressed_assets, fast_align, allow_reads=True) # Targets generally do not depend on apks, so no need for only_if_changed. - with build_utils.AtomicOutput(options.output_apk, only_if_changed=False) as f: + with action_helpers.atomic_output(options.output_apk, + only_if_changed=False) as f: with zipfile.ZipFile(options.resource_apk) as resource_apk, \ zipfile.ZipFile(f, 'w') as out_apk: def add_to_zip(zip_path, data, compress=True, alignment=4): - zipalign.AddToZipHermetic( + zip_helpers.add_to_zip_hermetic( out_apk, zip_path, data=data, @@ -472,13 +452,6 @@ dex_zip.read(dex), compress=not options.uncompress_dex) - if options.jdk_libs_dex_file: - with open(options.jdk_libs_dex_file, 'rb') as dex_file_obj: - add_to_zip( - apk_dex_dir + 'classes{}.dex'.format(max_dex_number + 1), - dex_file_obj.read(), - compress=not options.uncompress_dex) - # 4. Native libraries. logging.debug('Adding lib/') _AddFiles(out_apk, libs_to_add) @@ -537,7 +510,7 @@ add_to_zip(apk_root_dir + apk_path, java_resource_jar.read(apk_path)) - if options.format == 'apk': + if options.format == 'apk' and options.key_path: zipalign_path = None if fast_align else options.zipalign_path finalize_apk.FinalizeApk(options.apksigner_jar, zipalign_path, @@ -551,9 +524,9 @@ logging.debug('Moving file into place') if options.depfile: - build_utils.WriteDepfile(options.depfile, - options.output_apk, - inputs=depfile_deps) + action_helpers.write_depfile(options.depfile, + options.output_apk, + inputs=depfile_deps) if __name__ == '__main__':
diff --git a/build/android/gyp/apkbuilder.pydeps b/build/android/gyp/apkbuilder.pydeps index e6122ed..28dfdb0 100644 --- a/build/android/gyp/apkbuilder.pydeps +++ b/build/android/gyp/apkbuilder.pydeps
@@ -1,9 +1,10 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/apkbuilder.pydeps build/android/gyp/apkbuilder.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py apkbuilder.py finalize_apk.py util/__init__.py util/build_utils.py util/diff_utils.py -util/zipalign.py
diff --git a/build/android/gyp/assert_static_initializers.py b/build/android/gyp/assert_static_initializers.py index 31f2a77..fd0bb02 100755 --- a/build/android/gyp/assert_static_initializers.py +++ b/build/android/gyp/assert_static_initializers.py
@@ -1,11 +1,10 @@ #!/usr/bin/env python3 -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Checks the number of static initializers in an APK's library.""" -from __future__ import print_function import argparse import os @@ -23,8 +22,9 @@ def _RunReadelf(so_path, options, tool_prefix=''): - return subprocess.check_output([tool_prefix + 'readelf'] + options + - [so_path]).decode('utf8') + return subprocess.check_output( + [tool_prefix + 'readobj', '--elf-output-style=GNU'] + options + + [so_path]).decode('utf8') def _ParseLibBuildId(so_path, tool_prefix): @@ -40,26 +40,16 @@ 'Your output directory is likely stale.') -def _GetStaticInitializers(so_path, tool_prefix): - output = subprocess.check_output( - [_DUMP_STATIC_INITIALIZERS_PATH, '-d', so_path, '-t', tool_prefix]) - summary = re.search(r'Found \d+ static initializers in (\d+) files.', output) - return output.splitlines()[:-1], int(summary.group(1)) - - -def _PrintDumpSIsCount(apk_so_name, unzipped_so, out_dir, tool_prefix): - lib_name = os.path.basename(apk_so_name).replace('crazy.', '') - so_with_symbols_path = os.path.join(out_dir, 'lib.unstripped', lib_name) +def _DumpStaticInitializers(apk_so_name, unzipped_so, out_dir, tool_prefix): + so_with_symbols_path = os.path.join(out_dir, 'lib.unstripped', + os.path.basename(apk_so_name)) if not os.path.exists(so_with_symbols_path): - raise Exception('Unstripped .so not found. Looked here: %s', + raise Exception('Unstripped .so not found. Looked here: %s' % so_with_symbols_path) _VerifyLibBuildIdsMatch(tool_prefix, unzipped_so, so_with_symbols_path) - sis, _ = _GetStaticInitializers(so_with_symbols_path, tool_prefix) - for si in sis: - print(si) + subprocess.check_call([_DUMP_STATIC_INITIALIZERS_PATH, so_with_symbols_path]) -# Mostly copied from //infra/scripts/legacy/scripts/slave/chromium/sizes.py. def _ReadInitArray(so_path, tool_prefix, expect_no_initializers): stdout = _RunReadelf(so_path, ['-SW'], tool_prefix) # Matches: .init_array INIT_ARRAY 000000000516add0 5169dd0 000010 00 WA 0 0 8 @@ -68,9 +58,8 @@ if match: raise Exception( 'Expected no initializers for %s, yet some were found' % so_path) - else: - return 0 - elif not match: + return 0 + if not match: raise Exception('Did not find section: .init_array in {}:\n{}'.format( so_path, stdout)) size_str = re.split(r'\W+', match.group(0))[5] @@ -92,13 +81,12 @@ # NOTE: this is very implementation-specific and makes assumptions # about how compiler and linker implement global static initializers. init_array_size = _ReadInitArray(so_path, tool_prefix, expect_no_initializers) - return init_array_size / word_size + assert init_array_size % word_size == 0 + return init_array_size // word_size def _AnalyzeStaticInitializers(apk_or_aab, tool_prefix, dump_sis, out_dir, ignored_libs, no_initializers_libs): - # Static initializer counting mostly copies logic in - # infra/scripts/legacy/scripts/slave/chromium/sizes.py. with zipfile.ZipFile(apk_or_aab) as z: so_files = [ f for f in z.infolist() if f.filename.endswith('.so') @@ -127,10 +115,7 @@ si_count += _CountStaticInitializers(temp.name, tool_prefix, expect_no_initializers) if dump_sis: - # Print count and list of SIs reported by dump-static-initializers.py. - # Doesn't work well on all archs (particularly arm), which is why - # the readelf method is used for tracking SI counts. - _PrintDumpSIsCount(f.filename, temp.name, out_dir, tool_prefix) + _DumpStaticInitializers(f.filename, temp.name, out_dir, tool_prefix) return si_count @@ -164,18 +149,16 @@ print('You have removed one or more static initializers. Thanks!') print('To fix the build, update the expectation in:') print(' //chrome/android/static_initializers.gni') - else: - print('Dumping static initializers via dump-static-initializers.py:') - sys.stdout.flush() - _AnalyzeStaticInitializers(args.apk_or_aab, args.tool_prefix, True, '.', - ignored_libs, no_initializers_libs) print() - print('If the above list is not useful, consider listing them with:') - print(' //tools/binary_size/diagnose_bloat.py') - print() - print('For more information:') - print(' https://chromium.googlesource.com/chromium/src/+/master/docs/' - 'static_initializers.md') + + print('Dumping static initializers via dump-static-initializers.py:') + sys.stdout.flush() + _AnalyzeStaticInitializers(args.apk_or_aab, args.tool_prefix, True, '.', + ignored_libs, no_initializers_libs) + print() + print('For more information:') + print(' https://chromium.googlesource.com/chromium/src/+/main/docs/' + 'static_initializers.md') sys.exit(1) if args.touch:
diff --git a/build/android/gyp/binary_baseline_profile.py b/build/android/gyp/binary_baseline_profile.py new file mode 100755 index 0000000..4049805 --- /dev/null +++ b/build/android/gyp/binary_baseline_profile.py
@@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Creates a binary profile from an HRF + dex + mapping.""" + +import argparse +import sys + +from util import build_utils +import action_helpers + + +def main(args): + parser = argparse.ArgumentParser(description=__doc__) + action_helpers.add_depfile_arg(parser) + parser.add_argument('--output-profile', + required=True, + help='Path to output binary profile.') + parser.add_argument('--output-metadata', + required=True, + help='Path to output binary profile metadata.') + parser.add_argument('--profgen', + required=True, + help='Path to profgen binary.') + parser.add_argument('--dex', + required=True, + help='Path to a zip containing release dex files.') + parser.add_argument('--proguard-mapping', + required=True, + help='Path to proguard mapping for release dex.') + parser.add_argument('--input-profile-path', + required=True, + help='Path to HRF baseline profile to apply.') + options = parser.parse_args(build_utils.ExpandFileArgs(args)) + + cmd = [ + options.profgen, + 'bin', + options.input_profile_path, + '-o', + options.output_profile, + '-om', + options.output_metadata, + '-a', + options.dex, + '-m', + options.proguard_mapping, + ] + build_utils.CheckOutput(cmd, env={'JAVA_HOME': build_utils.JAVA_HOME}) + action_helpers.write_depfile(options.depfile, + options.output_profile, + inputs=[options.dex]) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))
diff --git a/build/android/gyp/binary_baseline_profile.pydeps b/build/android/gyp/binary_baseline_profile.pydeps new file mode 100644 index 0000000..944f6ab --- /dev/null +++ b/build/android/gyp/binary_baseline_profile.pydeps
@@ -0,0 +1,7 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/binary_baseline_profile.pydeps build/android/gyp/binary_baseline_profile.py +../../action_helpers.py +../../gn_helpers.py +binary_baseline_profile.py +util/__init__.py +util/build_utils.py
diff --git a/build/android/gyp/bundletool.py b/build/android/gyp/bundletool.py index dc9b86a..7915133 100755 --- a/build/android/gyp/bundletool.py +++ b/build/android/gyp/bundletool.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,6 +8,8 @@ Bundletool is distributed as a versioned jar file. This script abstracts the location and version of this jar file, as well as the JVM invokation.""" +# Warning: Check if still being run as python2: https://crbug.com/1322618 + import logging import os import sys @@ -19,18 +21,13 @@ __file__, '..', '..', '..', '..', 'third_party', 'android_build_tools', 'bundletool')) -BUNDLETOOL_VERSION = '1.4.0' - -BUNDLETOOL_JAR_PATH = os.path.join( - BUNDLETOOL_DIR, 'bundletool-all-%s.jar' % BUNDLETOOL_VERSION) +BUNDLETOOL_JAR_PATH = os.path.join(BUNDLETOOL_DIR, 'bundletool.jar') -def RunBundleTool(args, warnings_as_errors=(), print_stdout=False): - # Use () instead of None because command-line flags are None by default. - verify = warnings_as_errors == () or warnings_as_errors +def RunBundleTool(args, print_stdout=False): # ASAN builds failed with the default of 1GB (crbug.com/1120202). # Bug for bundletool: https://issuetracker.google.com/issues/165911616 - cmd = build_utils.JavaCmd(verify, xmx='4G') + cmd = build_utils.JavaCmd(xmx='4G') cmd += ['-jar', BUNDLETOOL_JAR_PATH] cmd += args logging.debug(' '.join(cmd))
diff --git a/build/android/gyp/bytecode_processor.py b/build/android/gyp/bytecode_processor.py index d77f159..f6065db 100755 --- a/build/android/gyp/bytecode_processor.py +++ b/build/android/gyp/bytecode_processor.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,8 +8,10 @@ import argparse import sys +import javac_output_processor from util import build_utils from util import server_utils +import action_helpers # build_utils adds //build to sys.path. def _AddSwitch(parser, val): @@ -21,6 +23,9 @@ argv = build_utils.ExpandFileArgs(argv[1:]) parser = argparse.ArgumentParser() parser.add_argument('--target-name', help='Fully qualified GN target name.') + parser.add_argument('--use-build-server', + action='store_true', + help='Always use the build server.') parser.add_argument('--script', required=True, help='Path to the java binary wrapper script.') parser.add_argument('--gn-target', required=True) @@ -40,16 +45,19 @@ if server_utils.MaybeRunCommand(name=args.target_name, argv=sys.argv, - stamp_file=args.stamp): + stamp_file=args.stamp, + force=args.use_build_server): return - args.sdk_classpath_jars = build_utils.ParseGnList(args.sdk_classpath_jars) - args.direct_classpath_jars = build_utils.ParseGnList( + args.sdk_classpath_jars = action_helpers.parse_gn_list( + args.sdk_classpath_jars) + args.direct_classpath_jars = action_helpers.parse_gn_list( args.direct_classpath_jars) - args.full_classpath_jars = build_utils.ParseGnList(args.full_classpath_jars) - args.full_classpath_gn_targets = build_utils.ParseGnList( + args.full_classpath_jars = action_helpers.parse_gn_list( + args.full_classpath_jars) + args.full_classpath_gn_targets = action_helpers.parse_gn_list( args.full_classpath_gn_targets) - args.missing_classes_allowlist = build_utils.ParseGnList( + args.missing_classes_allowlist = action_helpers.parse_gn_list( args.missing_classes_allowlist) verbose = '--verbose' if args.verbose else '--not-verbose' @@ -64,11 +72,20 @@ cmd += [str(len(args.full_classpath_jars))] cmd += args.full_classpath_jars cmd += [str(len(args.full_classpath_gn_targets))] - cmd += args.full_classpath_gn_targets - build_utils.CheckOutput(cmd, - print_stdout=True, - fail_func=None, - fail_on_output=args.warnings_as_errors) + cmd += [ + javac_output_processor.ReplaceGmsPackageIfNeeded(t) + for t in args.full_classpath_gn_targets + ] + try: + build_utils.CheckOutput(cmd, + print_stdout=True, + fail_func=None, + fail_on_output=args.warnings_as_errors) + except build_utils.CalledProcessError as e: + # Do not output command line because it is massive and makes the actual + # error message hard to find. + sys.stderr.write(e.output) + sys.exit(1) if args.stamp: build_utils.Touch(args.stamp)
diff --git a/build/android/gyp/bytecode_processor.pydeps b/build/android/gyp/bytecode_processor.pydeps index 6105d93..e7f1d98 100644 --- a/build/android/gyp/bytecode_processor.pydeps +++ b/build/android/gyp/bytecode_processor.pydeps
@@ -1,7 +1,28 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/bytecode_processor.pydeps build/android/gyp/bytecode_processor.py +../../../third_party/catapult/devil/devil/__init__.py +../../../third_party/catapult/devil/devil/android/__init__.py +../../../third_party/catapult/devil/devil/android/constants/__init__.py +../../../third_party/catapult/devil/devil/android/constants/chrome.py +../../../third_party/catapult/devil/devil/android/sdk/__init__.py +../../../third_party/catapult/devil/devil/android/sdk/keyevent.py +../../../third_party/catapult/devil/devil/android/sdk/version_codes.py +../../../third_party/catapult/devil/devil/constants/__init__.py +../../../third_party/catapult/devil/devil/constants/exit_codes.py +../../../third_party/colorama/src/colorama/__init__.py +../../../third_party/colorama/src/colorama/ansi.py +../../../third_party/colorama/src/colorama/ansitowin32.py +../../../third_party/colorama/src/colorama/initialise.py +../../../third_party/colorama/src/colorama/win32.py +../../../third_party/colorama/src/colorama/winterm.py +../../../tools/android/modularization/convenience/lookup_dep.py +../../action_helpers.py ../../gn_helpers.py +../list_java_targets.py +../pylib/__init__.py +../pylib/constants/__init__.py bytecode_processor.py +javac_output_processor.py util/__init__.py util/build_utils.py util/server_utils.py
diff --git a/build/android/gyp/bytecode_rewriter.py b/build/android/gyp/bytecode_rewriter.py index ad232df..d16fee5 100755 --- a/build/android/gyp/bytecode_rewriter.py +++ b/build/android/gyp/bytecode_rewriter.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Wrapper script around ByteCodeRewriter subclass scripts.""" @@ -8,12 +8,13 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. def main(argv): argv = build_utils.ExpandFileArgs(argv[1:]) parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--script', required=True, help='Path to the java binary wrapper script.') @@ -22,8 +23,8 @@ parser.add_argument('--output-jar', required=True) args = parser.parse_args(argv) - classpath = build_utils.ParseGnList(args.classpath) - build_utils.WriteDepfile(args.depfile, args.output_jar, inputs=classpath) + classpath = action_helpers.parse_gn_list(args.classpath) + action_helpers.write_depfile(args.depfile, args.output_jar, inputs=classpath) classpath.append(args.input_jar) cmd = [
diff --git a/build/android/gyp/bytecode_rewriter.pydeps b/build/android/gyp/bytecode_rewriter.pydeps index b8f304a..b0a6560 100644 --- a/build/android/gyp/bytecode_rewriter.pydeps +++ b/build/android/gyp/bytecode_rewriter.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/bytecode_rewriter.pydeps build/android/gyp/bytecode_rewriter.py +../../action_helpers.py ../../gn_helpers.py bytecode_rewriter.py util/__init__.py
diff --git a/build/android/gyp/check_flag_expectations.py b/build/android/gyp/check_flag_expectations.py index 22da211..97be53d 100755 --- a/build/android/gyp/check_flag_expectations.py +++ b/build/android/gyp/check_flag_expectations.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gyp/check_flag_expectations.pydeps b/build/android/gyp/check_flag_expectations.pydeps index d8c394a..6bade94 100644 --- a/build/android/gyp/check_flag_expectations.pydeps +++ b/build/android/gyp/check_flag_expectations.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/check_flag_expectations.pydeps build/android/gyp/check_flag_expectations.py +../../action_helpers.py ../../gn_helpers.py check_flag_expectations.py util/__init__.py
diff --git a/build/android/gyp/compile_java.py b/build/android/gyp/compile_java.py index 2a92842..5fee0d7 100755 --- a/build/android/gyp/compile_java.py +++ b/build/android/gyp/compile_java.py
@@ -1,9 +1,10 @@ #!/usr/bin/env python3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import functools import logging import multiprocessing import optparse @@ -13,16 +14,15 @@ import sys import time import zipfile +import pathlib +import javac_output_processor from util import build_utils from util import md5_check from util import jar_info_utils from util import server_utils - -sys.path.insert( - 0, - os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'colorama', 'src')) -import colorama +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers _JAVAC_EXTRACTOR = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'android_prebuilts', 'build_tools', 'common', @@ -34,6 +34,9 @@ # Full list of checks: https://errorprone.info/bugpatterns ERRORPRONE_WARNINGS_TO_DISABLE = [ + # Temporarily disabling to roll doubledown. + # TODO(wnwen): Re-enable this upstream. + 'InlineMeInliner', # The following are super useful, but existing issues need to be fixed first # before they can start failing the build on new errors. 'InvalidParam', @@ -47,6 +50,21 @@ 'UnescapedEntity', 'NonCanonicalType', 'AlmostJavadoc', + 'ReturnValueIgnored', + # The following are added for errorprone update: https://crbug.com/1216032 + 'InlineMeSuggester', + 'DoNotClaimAnnotations', + 'JavaUtilDate', + 'IdentityHashMapUsage', + 'UnnecessaryMethodReference', + 'LongFloatConversion', + 'CharacterGetNumericValue', + 'ErroneousThreadPoolConstructorChecker', + 'StaticMockMember', + 'MissingSuperCall', + 'ToStringReturnsNull', + # If possible, this should be automatically fixed if turned on: + 'MalformedInlineTag', # TODO(crbug.com/834807): Follow steps in bug 'DoubleBraceInitialization', # TODO(crbug.com/834790): Follow steps in bug. @@ -62,6 +80,8 @@ # Android platform default is always UTF-8. # https://developer.android.com/reference/java/nio/charset/Charset.html#defaultCharset() 'DefaultCharset', + # Low priority since there are lots of tags that don't fit this check. + 'UnrecognisedJavadocTag', # Low priority since the alternatives still work. 'JdkObsolete', # We don't use that many lambdas. @@ -167,6 +187,11 @@ 'RemoveUnusedImports', # We do not care about unnecessary parenthesis enough to check for them. 'UnnecessaryParentheses', + # The only time we trigger this is when it is better to be explicit in a + # list of unicode characters, e.g. FindAddress.java + 'UnicodeEscape', + # Nice to have. + 'AlreadyChecked', ] # Full list of checks: https://errorprone.info/bugpatterns @@ -179,7 +204,6 @@ 'InvalidThrows', 'LongLiteralLowerCaseSuffix', 'MultiVariableDeclaration', - 'ParameterNotNullable', 'RedundantOverride', 'StaticQualifiedUsingExpression', 'StringEquality', @@ -190,14 +214,7 @@ ] -def ProcessJavacOutput(output): - fileline_prefix = r'(?P<fileline>(?P<file>[-.\w/\\]+.java):(?P<line>[0-9]+):)' - warning_re = re.compile(fileline_prefix + - r'(?P<full_message> warning: (?P<message>.*))$') - error_re = re.compile(fileline_prefix + - r'(?P<full_message> (?P<message>.*))$') - marker_re = re.compile(r'\s*(?P<marker>\^)\s*$') - +def ProcessJavacOutput(output, target_name): # These warnings cannot be suppressed even for third party code. Deprecation # warnings especially do not help since we must support older android version. deprecated_re = re.compile( @@ -208,17 +225,6 @@ activity_re = re.compile(r'^(?P<prefix>\s*location: )class Activity$') - warning_color = ['full_message', colorama.Fore.YELLOW + colorama.Style.DIM] - error_color = ['full_message', colorama.Fore.MAGENTA + colorama.Style.BRIGHT] - marker_color = ['marker', colorama.Fore.BLUE + colorama.Style.BRIGHT] - - def Colorize(line, regex, color): - match = regex.match(line) - start = match.start(color[0]) - end = match.end(color[0]) - return (line[:start] + color[1] + line[start:end] + colorama.Fore.RESET + - colorama.Style.RESET_ALL + line[end:]) - def ApplyFilters(line): return not (deprecated_re.match(line) or unchecked_re.match(line) or recompile_re.match(line)) @@ -230,31 +236,73 @@ line, prefix, 'docs/ui/android/bytecode_rewriting.md') return line - def ApplyColors(line): - if warning_re.match(line): - line = Colorize(line, warning_re, warning_color) - elif error_re.match(line): - line = Colorize(line, error_re, error_color) - elif marker_re.match(line): - line = Colorize(line, marker_re, marker_color) - return line + output = build_utils.FilterReflectiveAccessJavaWarnings(output) + + # Warning currently cannot be silenced via javac flag. + if 'Unsafe is internal proprietary API' in output: + # Example: + # HiddenApiBypass.java:69: warning: Unsafe is internal proprietary API and + # may be removed in a future release + # import sun.misc.Unsafe; + # ^ + output = re.sub(r'.*?Unsafe is internal proprietary API[\s\S]*?\^\n', '', + output) + output = re.sub(r'\d+ warnings\n', '', output) lines = (l for l in output.split('\n') if ApplyFilters(l)) - lines = (ApplyColors(Elaborate(l)) for l in lines) + lines = (Elaborate(l) for l in lines) + + output_processor = javac_output_processor.JavacOutputProcessor(target_name) + lines = output_processor.Process(lines) + return '\n'.join(lines) -def _ParsePackageAndClassNames(java_file): +def CreateJarFile(jar_path, + classes_dir, + service_provider_configuration_dir=None, + additional_jar_files=None, + extra_classes_jar=None): + """Zips files from compilation into a single jar.""" + logging.info('Start creating jar file: %s', jar_path) + with action_helpers.atomic_output(jar_path) as f: + with zipfile.ZipFile(f.name, 'w') as z: + zip_helpers.zip_directory(z, classes_dir) + if service_provider_configuration_dir: + config_files = build_utils.FindInDirectory( + service_provider_configuration_dir) + for config_file in config_files: + zip_path = os.path.relpath(config_file, + service_provider_configuration_dir) + zip_helpers.add_to_zip_hermetic(z, zip_path, src_path=config_file) + + if additional_jar_files: + for src_path, zip_path in additional_jar_files: + zip_helpers.add_to_zip_hermetic(z, zip_path, src_path=src_path) + if extra_classes_jar: + path_transform = lambda p: p if p.endswith('.class') else None + zip_helpers.merge_zips(z, [extra_classes_jar], + path_transform=path_transform) + logging.info('Completed jar file: %s', jar_path) + + +def _ParsePackageAndClassNames(source_file): + """This should support both Java and Kotlin files.""" package_name = '' class_names = [] - with open(java_file) as f: + with open(source_file) as f: for l in f: # Strip unindented comments. # Considers a leading * as a continuation of a multi-line comment (our # linter doesn't enforce a space before it like there should be). l = re.sub(r'^(?://.*|/?\*.*?(?:\*/\s*|$))', '', l) + # Stripping things between double quotes (strings), so if the word "class" + # shows up in a string this doesn't trigger. This isn't strictly correct + # (with escaped quotes) but covers a very large percentage of cases. + l = re.sub('(?:".*?")', '', l) - m = re.match(r'package\s+(.*?);', l) + # Java lines end in semicolon, whereas Kotlin lines do not. + m = re.match(r'package\s+(.*?)(;|\s*$)', l) if m and not package_name: package_name = m.group(1) @@ -266,12 +314,12 @@ return package_name, class_names -def _ProcessJavaFileForInfo(java_file): - package_name, class_names = _ParsePackageAndClassNames(java_file) - return java_file, package_name, class_names +def _ProcessSourceFileForInfo(source_file): + package_name, class_names = _ParsePackageAndClassNames(source_file) + return source_file, package_name, class_names -class _InfoFileContext(object): +class _InfoFileContext: """Manages the creation of the class->source file .info file.""" def __init__(self, chromium_code, excluded_globs): @@ -291,23 +339,29 @@ self._srcjar_files[path] = '{}/{}'.format( srcjar_path, os.path.relpath(path, parent_dir)) - def SubmitFiles(self, java_files): + def SubmitFiles(self, source_files): + if not source_files: + return if self._pool is None: # Restrict to just one process to not slow down compiling. Compiling # is always slower. self._pool = multiprocessing.Pool(1) - logging.info('Submitting %d files for info', len(java_files)) + logging.info('Submitting %d files for info', len(source_files)) self._results.append( - self._pool.imap_unordered( - _ProcessJavaFileForInfo, java_files, chunksize=1000)) + self._pool.imap_unordered(_ProcessSourceFileForInfo, + source_files, + chunksize=1000)) - def _CheckPathMatchesClassName(self, java_file, package_name, class_name): - parts = package_name.split('.') + [class_name + '.java'] - expected_path_suffix = os.path.sep.join(parts) - if not java_file.endswith(expected_path_suffix): - raise Exception(('Java package+class name do not match its path.\n' + def _CheckPathMatchesClassName(self, source_file, package_name, class_name): + if source_file.endswith('.java'): + parts = package_name.split('.') + [class_name + '.java'] + else: + parts = package_name.split('.') + [class_name + '.kt'] + expected_suffix = os.path.sep.join(parts) + if not source_file.endswith(expected_suffix): + raise Exception(('Source package+class name do not match its path.\n' 'Actual path: %s\nExpected path: %s') % - (java_file, expected_path_suffix)) + (source_file, expected_suffix)) def _ProcessInfo(self, java_file, package_name, class_names, source): for class_name in class_names: @@ -336,10 +390,9 @@ class_names, source): if self._ShouldIncludeInJarInfo(fully_qualified_name): ret[fully_qualified_name] = java_file - self._pool.terminate() return ret - def __del__(self): + def Close(self): # Work around for Python 2.x bug with multiprocessing and daemon threads: # https://bugs.python.org/issue4106 if self._pool is not None: @@ -358,32 +411,12 @@ entries = self._Collect() logging.info('Writing info file: %s', output_path) - with build_utils.AtomicOutput(output_path, mode='wb') as f: + with action_helpers.atomic_output(output_path, mode='wb') as f: jar_info_utils.WriteJarInfoFile(f, entries, self._srcjar_files) logging.info('Completed info file: %s', output_path) -def _CreateJarFile(jar_path, service_provider_configuration_dir, - additional_jar_files, classes_dir): - logging.info('Start creating jar file: %s', jar_path) - with build_utils.AtomicOutput(jar_path) as f: - with zipfile.ZipFile(f.name, 'w') as z: - build_utils.ZipDir(z, classes_dir) - if service_provider_configuration_dir: - config_files = build_utils.FindInDirectory( - service_provider_configuration_dir) - for config_file in config_files: - zip_path = os.path.relpath(config_file, - service_provider_configuration_dir) - build_utils.AddToZipHermetic(z, zip_path, src_path=config_file) - - if additional_jar_files: - for src_path, zip_path in additional_jar_files: - build_utils.AddToZipHermetic(z, zip_path, src_path=src_path) - logging.info('Completed jar file: %s', jar_path) - - -def _OnStaleMd5(options, javac_cmd, javac_args, java_files): +def _OnStaleMd5(changes, options, javac_cmd, javac_args, java_files, kt_files): logging.info('Starting _OnStaleMd5') if options.enable_kythe_annotations: # Kythe requires those env variables to be set and compile_java.py does the @@ -394,68 +427,146 @@ 'KYTHE_ROOT_DIRECTORY and KYTHE_OUTPUT_DIRECTORY ' 'environment variables to be set.') javac_extractor_cmd = build_utils.JavaCmd() + [ + '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', + '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', '-jar', _JAVAC_EXTRACTOR, ] try: - _RunCompiler(options, javac_extractor_cmd + javac_args, java_files, - options.classpath, options.jar_path + '.javac_extractor', - save_outputs=False), + # _RunCompiler()'s partial javac implementation does not support + # generating outputs in $KYTHE_OUTPUT_DIRECTORY. + _RunCompiler(changes, + options, + javac_extractor_cmd + javac_args, + java_files, + options.jar_path + '.javac_extractor', + enable_partial_javac=False) except build_utils.CalledProcessError as e: # Having no index for particular target is better than failing entire # codesearch. Log and error and move on. logging.error('Could not generate kzip: %s', e) + intermediates_out_dir = None + jar_info_path = None + if not options.enable_errorprone: + # Delete any stale files in the generated directory. The purpose of + # options.generated_dir is for codesearch. + shutil.rmtree(options.generated_dir, True) + intermediates_out_dir = options.generated_dir + + jar_info_path = options.jar_path + '.info' + # Compiles with Error Prone take twice as long to run as pure javac. Thus GN # rules run both in parallel, with Error Prone only used for checks. - _RunCompiler(options, - javac_cmd + javac_args, - java_files, - options.classpath, - options.jar_path, - save_outputs=not options.enable_errorprone) + try: + _RunCompiler(changes, + options, + javac_cmd + javac_args, + java_files, + options.jar_path, + kt_files=kt_files, + jar_info_path=jar_info_path, + intermediates_out_dir=intermediates_out_dir, + enable_partial_javac=True) + except build_utils.CalledProcessError as e: + # Do not output stacktrace as it takes up space on gerrit UI, forcing + # you to click though to find the actual compilation error. It's never + # interesting to see the Python stacktrace for a Java compilation error. + sys.stderr.write(e.output) + sys.exit(1) + logging.info('Completed all steps in _OnStaleMd5') -def _RunCompiler(options, javac_cmd, java_files, classpath, jar_path, - save_outputs=True): +def _RunCompiler(changes, + options, + javac_cmd, + java_files, + jar_path, + kt_files=None, + jar_info_path=None, + intermediates_out_dir=None, + enable_partial_javac=False): + """Runs java compiler. + + Args: + changes: md5_check.Changes object. + options: Object with command line flags. + javac_cmd: Command to execute. + java_files: List of java files passed from command line. + jar_path: Path of output jar file. + kt_files: List of Kotlin files passed from command line if any. + jar_info_path: Path of the .info file to generate. + If None, .info file will not be generated. + intermediates_out_dir: Directory for saving intermediate outputs. + If None a temporary directory is used. + enable_partial_javac: Enables compiling only Java files which have changed + in the special case that no method signatures have changed. This is + useful for large GN targets. + Not supported if compiling generates outputs other than |jar_path| and + |jar_info_path|. + """ logging.info('Starting _RunCompiler') - # Compiles with Error Prone take twice as long to run as pure javac. Thus GN - # rules run both in parallel, with Error Prone only used for checks. - save_outputs = not options.enable_errorprone + java_files = java_files.copy() + java_srcjars = options.java_srcjars + save_info_file = jar_info_path is not None # Use jar_path's directory to ensure paths are relative (needed for goma). temp_dir = jar_path + '.staging' - shutil.rmtree(temp_dir, True) + build_utils.DeleteDirectory(temp_dir) os.makedirs(temp_dir) + info_file_context = None try: classes_dir = os.path.join(temp_dir, 'classes') service_provider_configuration = os.path.join( temp_dir, 'service_provider_configuration') - if save_outputs: - input_srcjars_dir = os.path.join(options.generated_dir, 'input_srcjars') - annotation_processor_outputs_dir = os.path.join( - options.generated_dir, 'annotation_processor_outputs') - # Delete any stale files in the generated directory. The purpose of - # options.generated_dir is for codesearch. - shutil.rmtree(options.generated_dir, True) + if java_files: + os.makedirs(classes_dir) + + if enable_partial_javac: + all_changed_paths_are_java = all( + p.endswith(".java") for p in changes.IterChangedPaths()) + if (all_changed_paths_are_java and not changes.HasStringChanges() + and os.path.exists(jar_path) + and (jar_info_path is None or os.path.exists(jar_info_path))): + # Log message is used by tests to determine whether partial javac + # optimization was used. + logging.info('Using partial javac optimization for %s compile' % + (jar_path)) + + # Header jar corresponding to |java_files| did not change. + # As a build speed optimization (crbug.com/1170778), re-compile only + # java files which have changed. Re-use old jar .info file. + java_files = list(changes.IterChangedPaths()) + java_srcjars = None + + # Reuse old .info file. + save_info_file = False + + build_utils.ExtractAll(jar_path, classes_dir, pattern='*.class') + + if save_info_file: info_file_context = _InfoFileContext(options.chromium_code, options.jar_info_exclude_globs) - else: - input_srcjars_dir = os.path.join(temp_dir, 'input_srcjars') - annotation_processor_outputs_dir = os.path.join( - temp_dir, 'annotation_processor_outputs') - if options.java_srcjars: + if intermediates_out_dir is None: + intermediates_out_dir = temp_dir + + input_srcjars_dir = os.path.join(intermediates_out_dir, 'input_srcjars') + + if java_srcjars: logging.info('Extracting srcjars to %s', input_srcjars_dir) build_utils.MakeDirectory(input_srcjars_dir) for srcjar in options.java_srcjars: extracted_files = build_utils.ExtractAll( srcjar, no_clobber=True, path=input_srcjars_dir, pattern='*.java') java_files.extend(extracted_files) - if save_outputs: + if save_info_file: info_file_context.AddSrcJarSources(srcjar, extracted_files, input_srcjars_dir) logging.info('Done extracting srcjars') @@ -470,22 +581,18 @@ pattern='META-INF/services/*') logging.info('Done extracting service provider configs') - if save_outputs and java_files: + if save_info_file and java_files: info_file_context.SubmitFiles(java_files) + info_file_context.SubmitFiles(kt_files) if java_files: # Don't include the output directory in the initial set of args since it # being in a temp dir makes it unstable (breaks md5 stamping). cmd = list(javac_cmd) - os.makedirs(classes_dir) cmd += ['-d', classes_dir] - if options.processors: - os.makedirs(annotation_processor_outputs_dir) - cmd += ['-s', annotation_processor_outputs_dir] - - if classpath: - cmd += ['-classpath', ':'.join(classpath)] + if options.classpath: + cmd += ['-classpath', ':'.join(options.classpath)] # Pass source paths as response files to avoid extremely long command # lines that are tedius to debug. @@ -494,43 +601,43 @@ f.write(' '.join(java_files)) cmd += ['@' + java_files_rsp_path] + process_javac_output_partial = functools.partial( + ProcessJavacOutput, target_name=options.target_name) + logging.debug('Build command %s', cmd) start = time.time() build_utils.CheckOutput(cmd, print_stdout=options.chromium_code, - stdout_filter=ProcessJavacOutput, - stderr_filter=ProcessJavacOutput, + stdout_filter=process_javac_output_partial, + stderr_filter=process_javac_output_partial, fail_on_output=options.warnings_as_errors) end = time.time() - start logging.info('Java compilation took %ss', end) - if save_outputs: - if options.processors: - annotation_processor_java_files = build_utils.FindInDirectory( - annotation_processor_outputs_dir) - if annotation_processor_java_files: - info_file_context.SubmitFiles(annotation_processor_java_files) + CreateJarFile(jar_path, classes_dir, service_provider_configuration, + options.additional_jar_files, options.kotlin_jar_path) - _CreateJarFile(jar_path, service_provider_configuration, - options.additional_jar_files, classes_dir) - - info_file_context.Commit(jar_path + '.info') - else: - build_utils.Touch(jar_path) + if save_info_file: + info_file_context.Commit(jar_info_path) logging.info('Completed all steps in _RunCompiler') finally: + if info_file_context: + info_file_context.Close() shutil.rmtree(temp_dir) def _ParseOptions(argv): parser = optparse.OptionParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_option('--target-name', help='Fully qualified GN target name.') parser.add_option('--skip-build-server', action='store_true', help='Avoid using the build server.') + parser.add_option('--use-build-server', + action='store_true', + help='Always use the build server.') parser.add_option( '--java-srcjars', action='append', @@ -540,21 +647,8 @@ '--generated-dir', help='Subdirectory within target_gen_dir to place extracted srcjars and ' 'annotation processor output for codesearch to find.') - parser.add_option( - '--bootclasspath', - action='append', - default=[], - help='Boot classpath for javac. If this is specified multiple times, ' - 'they will all be appended to construct the classpath.') - parser.add_option( - '--java-version', - help='Java language version to use in -source and -target args to javac.') parser.add_option('--classpath', action='append', help='Classpath to use.') parser.add_option( - '--processors', - action='append', - help='GN list of annotation processor main classes.') - parser.add_option( '--processorpath', action='append', help='GN list of jars that comprise the classpath used for Annotation ' @@ -606,16 +700,18 @@ '--header-jar', help='This is the header jar for the current target that contains ' 'META-INF/services/* files to be included in the output jar.') + parser.add_option( + '--kotlin-jar-path', + help='Kotlin jar to be merged into the output jar. This contains the ' + ".class files from this target's .kt files.") options, args = parser.parse_args(argv) build_utils.CheckOptions(options, parser, required=('jar_path', )) - options.bootclasspath = build_utils.ParseGnList(options.bootclasspath) - options.classpath = build_utils.ParseGnList(options.classpath) - options.processorpath = build_utils.ParseGnList(options.processorpath) - options.processors = build_utils.ParseGnList(options.processors) - options.java_srcjars = build_utils.ParseGnList(options.java_srcjars) - options.jar_info_exclude_globs = build_utils.ParseGnList( + options.classpath = action_helpers.parse_gn_list(options.classpath) + options.processorpath = action_helpers.parse_gn_list(options.processorpath) + options.java_srcjars = action_helpers.parse_gn_list(options.java_srcjars) + options.jar_info_exclude_globs = action_helpers.parse_gn_list( options.jar_info_exclude_globs) additional_jar_files = [] @@ -624,30 +720,38 @@ additional_jar_files.append((filepath, jar_filepath)) options.additional_jar_files = additional_jar_files - java_files = [] + files = [] for arg in args: # Interpret a path prefixed with @ as a file containing a list of sources. if arg.startswith('@'): - java_files.extend(build_utils.ReadSourcesList(arg[1:])) + files.extend(build_utils.ReadSourcesList(arg[1:])) else: - java_files.append(arg) + files.append(arg) - return options, java_files + # The target's .sources file contains both Java and Kotlin files. We use + # compile_kt.py to compile the Kotlin files to .class and header jars. Javac + # is run only on .java files. + java_files = [f for f in files if f.endswith('.java')] + # Kotlin files are needed to populate the info file and attribute size in + # supersize back to the appropriate Kotlin file. + kt_files = [f for f in files if f.endswith('.kt')] + + return options, java_files, kt_files def main(argv): build_utils.InitLogging('JAVAC_DEBUG') argv = build_utils.ExpandFileArgs(argv) - options, java_files = _ParseOptions(argv) + options, java_files, kt_files = _ParseOptions(argv) # Only use the build server for errorprone runs. if (options.enable_errorprone and not options.skip_build_server and server_utils.MaybeRunCommand(name=options.target_name, argv=sys.argv, - stamp_file=options.jar_path)): + stamp_file=options.jar_path, + force=options.use_build_server)): return - colorama.init() javac_cmd = [] if options.gomacc_path: javac_cmd.append(options.gomacc_path) @@ -655,6 +759,10 @@ javac_args = [ '-g', + # We currently target JDK 11 everywhere, since Mockito is broken by JDK17. + # See crbug.com/1409661 for more details. + '--release', + '11', # Chromium only allows UTF8 source files. Being explicit avoids # javac pulling a default encoding from the user's environment. '-encoding', @@ -663,6 +771,9 @@ # See: http://blog.ltgt.net/most-build-tools-misuse-javac/ '-sourcepath', ':', + # protobuf-generated files fail this check (javadoc has @deprecated, + # but method missing @Deprecated annotation). + '-Xlint:-dep-ann', ] if options.enable_errorprone: @@ -684,6 +795,22 @@ '-XepPatchChecks:,' + ','.join(ERRORPRONE_CHECKS_TO_APPLY) ] + # These are required to use JDK 16, and are taken directly from + # https://errorprone.info/docs/installation + javac_args += [ + '-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=' + 'ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', + '-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', + ] + javac_args += ['-XDcompilePolicy=simple', ' '.join(errorprone_flags)] # This flag quits errorprone after checks and before code generation, since @@ -692,28 +819,11 @@ if not ERRORPRONE_CHECKS_TO_APPLY: javac_args += ['-XDshould-stop.ifNoError=FLOW'] - if options.java_version: - javac_args.extend([ - '-source', - options.java_version, - '-target', - options.java_version, - ]) - if options.java_version == '1.8': - # Android's boot jar doesn't contain all java 8 classes. - options.bootclasspath.append(build_utils.RT_JAR_PATH) - - if options.processors: - javac_args.extend(['-processor', ','.join(options.processors)]) - else: - # This effectively disables all annotation processors, even including - # annotation processors in service provider configuration files named - # META-INF/. See the following link for reference: - # https://docs.oracle.com/en/java/javase/11/tools/javac.html - javac_args.extend(['-proc:none']) - - if options.bootclasspath: - javac_args.extend(['-bootclasspath', ':'.join(options.bootclasspath)]) + # This effectively disables all annotation processors, even including + # annotation processors in service provider configuration files named + # META-INF/. See the following link for reference: + # https://docs.oracle.com/en/java/javase/11/tools/javac.html + javac_args.extend(['-proc:none']) if options.processorpath: javac_args.extend(['-processorpath', ':'.join(options.processorpath)]) @@ -723,12 +833,11 @@ javac_args.extend(options.javac_arg) - classpath_inputs = ( - options.bootclasspath + options.classpath + options.processorpath) + classpath_inputs = options.classpath + options.processorpath depfile_deps = classpath_inputs # Files that are already inputs in GN should go in input_paths. - input_paths = depfile_deps + options.java_srcjars + java_files + input_paths = depfile_deps + options.java_srcjars + java_files + kt_files if options.header_jar: input_paths.append(options.header_jar) input_paths += [x[0] for x in options.additional_jar_files] @@ -737,19 +846,19 @@ if not options.enable_errorprone: output_paths += [options.jar_path + '.info'] - input_strings = javac_cmd + javac_args + options.classpath + java_files + [ - options.warnings_as_errors, options.jar_info_exclude_globs - ] + input_strings = (javac_cmd + javac_args + options.classpath + java_files + + kt_files + + [options.warnings_as_errors, options.jar_info_exclude_globs]) - # Keep md5_check since we plan to use its changes feature to implement a build - # speed improvement for non-signature compiles: https://crbug.com/1170778 - md5_check.CallAndWriteDepfileIfStale( - lambda: _OnStaleMd5(options, javac_cmd, javac_args, java_files), - options, - depfile_deps=depfile_deps, - input_paths=input_paths, - input_strings=input_strings, - output_paths=output_paths) + # Use md5_check for |pass_changes| feature. + md5_check.CallAndWriteDepfileIfStale(lambda changes: _OnStaleMd5( + changes, options, javac_cmd, javac_args, java_files, kt_files), + options, + depfile_deps=depfile_deps, + input_paths=input_paths, + input_strings=input_strings, + output_paths=output_paths, + pass_changes=True) if __name__ == '__main__':
diff --git a/build/android/gyp/compile_java.pydeps b/build/android/gyp/compile_java.pydeps index f14fd0b..45617b1 100644 --- a/build/android/gyp/compile_java.pydeps +++ b/build/android/gyp/compile_java.pydeps
@@ -1,14 +1,30 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/compile_java.pydeps build/android/gyp/compile_java.py +../../../third_party/catapult/devil/devil/__init__.py +../../../third_party/catapult/devil/devil/android/__init__.py +../../../third_party/catapult/devil/devil/android/constants/__init__.py +../../../third_party/catapult/devil/devil/android/constants/chrome.py +../../../third_party/catapult/devil/devil/android/sdk/__init__.py +../../../third_party/catapult/devil/devil/android/sdk/keyevent.py +../../../third_party/catapult/devil/devil/android/sdk/version_codes.py +../../../third_party/catapult/devil/devil/constants/__init__.py +../../../third_party/catapult/devil/devil/constants/exit_codes.py ../../../third_party/colorama/src/colorama/__init__.py ../../../third_party/colorama/src/colorama/ansi.py ../../../third_party/colorama/src/colorama/ansitowin32.py ../../../third_party/colorama/src/colorama/initialise.py ../../../third_party/colorama/src/colorama/win32.py ../../../third_party/colorama/src/colorama/winterm.py +../../../tools/android/modularization/convenience/lookup_dep.py +../../action_helpers.py ../../gn_helpers.py ../../print_python_deps.py +../../zip_helpers.py +../list_java_targets.py +../pylib/__init__.py +../pylib/constants/__init__.py compile_java.py +javac_output_processor.py util/__init__.py util/build_utils.py util/jar_info_utils.py
diff --git a/build/android/gyp/compile_kt.py b/build/android/gyp/compile_kt.py new file mode 100755 index 0000000..4c7eb6f --- /dev/null +++ b/build/android/gyp/compile_kt.py
@@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import logging +import os +import shutil +import sys +import time + +import compile_java + +from util import build_utils +import action_helpers # build_utils adds //build to sys.path. + + +def _RunCompiler(args, + kotlinc_cmd, + source_files, + jar_path, + intermediates_out_dir=None): + """Runs the Kotlin compiler.""" + logging.info('Starting _RunCompiler') + + source_files = source_files.copy() + kt_files = [f for f in source_files if f.endswith('.kt')] + assert len(kt_files) > 0, 'At least one .kt file must be passed in.' + + java_srcjars = args.java_srcjars + + # Use jar_path's directory to ensure paths are relative (needed for goma). + temp_dir = jar_path + '.staging' + build_utils.DeleteDirectory(temp_dir) + os.makedirs(temp_dir) + try: + classes_dir = os.path.join(temp_dir, 'classes') + os.makedirs(classes_dir) + + input_srcjars_dir = os.path.join(intermediates_out_dir or temp_dir, + 'input_srcjars') + + if java_srcjars: + logging.info('Extracting srcjars to %s', input_srcjars_dir) + build_utils.MakeDirectory(input_srcjars_dir) + for srcjar in args.java_srcjars: + source_files += build_utils.ExtractAll(srcjar, + no_clobber=True, + path=input_srcjars_dir, + pattern='*.java') + logging.info('Done extracting srcjars') + + # Don't include the output directory in the initial set of args since it + # being in a temp dir makes it unstable (breaks md5 stamping). + cmd = list(kotlinc_cmd) + cmd += ['-d', classes_dir] + + if args.classpath: + cmd += ['-classpath', ':'.join(args.classpath)] + + # This a kotlinc plugin to generate header files for .kt files, similar to + # turbine for .java files. + jvm_abi_path = os.path.join(build_utils.KOTLIN_HOME, 'lib', + 'jvm-abi-gen.jar') + cmd += [ + f'-Xplugin={jvm_abi_path}', '-P', + 'plugin:org.jetbrains.kotlin.jvm.abi:outputDir=' + + args.interface_jar_path + ] + + # Pass source paths as response files to avoid extremely long command + # lines that are tedius to debug. + source_files_rsp_path = os.path.join(temp_dir, 'files_list.txt') + with open(source_files_rsp_path, 'w') as f: + f.write(' '.join(source_files)) + cmd += ['@' + source_files_rsp_path] + + # Explicitly set JAVA_HOME since some bots do not have this already set. + env = os.environ.copy() + env['JAVA_HOME'] = build_utils.JAVA_HOME + + logging.debug('Build command %s', cmd) + start = time.time() + build_utils.CheckOutput(cmd, + env=env, + print_stdout=args.chromium_code, + fail_on_output=args.warnings_as_errors) + logging.info('Kotlin compilation took %ss', time.time() - start) + + compile_java.CreateJarFile(jar_path, classes_dir) + + logging.info('Completed all steps in _RunCompiler') + finally: + shutil.rmtree(temp_dir) + + +def _ParseOptions(argv): + parser = argparse.ArgumentParser() + action_helpers.add_depfile_arg(parser) + + parser.add_argument('--java-srcjars', + action='append', + default=[], + help='List of srcjars to include in compilation.') + parser.add_argument( + '--generated-dir', + help='Subdirectory within target_gen_dir to place extracted srcjars and ' + 'annotation processor output for codesearch to find.') + parser.add_argument('--classpath', action='append', help='Classpath to use.') + parser.add_argument( + '--chromium-code', + action='store_true', + help='Whether code being compiled should be built with stricter ' + 'warnings for chromium code.') + parser.add_argument('--gomacc-path', + help='When set, prefix kotlinc command with gomacc') + parser.add_argument('--warnings-as-errors', + action='store_true', + help='Treat all warnings as errors.') + parser.add_argument('--jar-path', help='Jar output path.', required=True) + parser.add_argument('--interface-jar-path', + help='Interface jar output path.', + required=True) + + args, extra_args = parser.parse_known_args(argv) + + args.classpath = action_helpers.parse_gn_list(args.classpath) + args.java_srcjars = action_helpers.parse_gn_list(args.java_srcjars) + + source_files = [] + for arg in extra_args: + # Interpret a path prefixed with @ as a file containing a list of sources. + if arg.startswith('@'): + source_files.extend(build_utils.ReadSourcesList(arg[1:])) + else: + assert not arg.startswith('--'), f'Undefined option {arg}' + source_files.append(arg) + + return args, source_files + + +def main(argv): + build_utils.InitLogging('KOTLINC_DEBUG') + argv = build_utils.ExpandFileArgs(argv) + args, source_files = _ParseOptions(argv) + + kotlinc_cmd = [] + if args.gomacc_path: + kotlinc_cmd.append(args.gomacc_path) + kotlinc_cmd.append(build_utils.KOTLINC_PATH) + + kotlinc_cmd += [ + '-no-jdk', # Avoid depending on the bundled JDK. + # Avoid depending on the bundled Kotlin stdlib. This may have a version + # skew with the one in //third_party/android_deps (which is the one we + # prefer to use). + '-no-stdlib', + # Avoid depending on the bundled Kotlin reflect libs. + '-no-reflect', + ] + + if args.generated_dir: + # Delete any stale files in the generated directory. The purpose of + # args.generated_dir is for codesearch. + shutil.rmtree(args.generated_dir, True) + + _RunCompiler(args, + kotlinc_cmd, + source_files, + args.jar_path, + intermediates_out_dir=args.generated_dir) + + if args.depfile: + # GN already knows of the source files, so avoid listing individual files + # in the depfile. + action_helpers.write_depfile(args.depfile, args.jar_path, args.classpath) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))
diff --git a/build/android/gyp/compile_kt.pydeps b/build/android/gyp/compile_kt.pydeps new file mode 100644 index 0000000..818bca8 --- /dev/null +++ b/build/android/gyp/compile_kt.pydeps
@@ -0,0 +1,33 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/compile_kt.pydeps build/android/gyp/compile_kt.py +../../../third_party/catapult/devil/devil/__init__.py +../../../third_party/catapult/devil/devil/android/__init__.py +../../../third_party/catapult/devil/devil/android/constants/__init__.py +../../../third_party/catapult/devil/devil/android/constants/chrome.py +../../../third_party/catapult/devil/devil/android/sdk/__init__.py +../../../third_party/catapult/devil/devil/android/sdk/keyevent.py +../../../third_party/catapult/devil/devil/android/sdk/version_codes.py +../../../third_party/catapult/devil/devil/constants/__init__.py +../../../third_party/catapult/devil/devil/constants/exit_codes.py +../../../third_party/colorama/src/colorama/__init__.py +../../../third_party/colorama/src/colorama/ansi.py +../../../third_party/colorama/src/colorama/ansitowin32.py +../../../third_party/colorama/src/colorama/initialise.py +../../../third_party/colorama/src/colorama/win32.py +../../../third_party/colorama/src/colorama/winterm.py +../../../tools/android/modularization/convenience/lookup_dep.py +../../action_helpers.py +../../gn_helpers.py +../../print_python_deps.py +../../zip_helpers.py +../list_java_targets.py +../pylib/__init__.py +../pylib/constants/__init__.py +compile_java.py +compile_kt.py +javac_output_processor.py +util/__init__.py +util/build_utils.py +util/jar_info_utils.py +util/md5_check.py +util/server_utils.py
diff --git a/build/android/gyp/compile_resources.py b/build/android/gyp/compile_resources.py index 8a668e7..3b1fe73 100755 --- a/build/android/gyp/compile_resources.py +++ b/build/android/gyp/compile_resources.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,13 +19,12 @@ import hashlib import logging import os +import pathlib import re import shutil import subprocess import sys -import tempfile import textwrap -import zipfile from xml.etree import ElementTree from util import build_utils @@ -34,6 +33,8 @@ from util import parallel from util import protoresources from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers # Pngs that we shouldn't convert to webp. Please add rationale when updating. @@ -53,8 +54,24 @@ Returns: An options object as from argparse.ArgumentParser.parse_args() """ - parser, input_opts, output_opts = resource_utils.ResourceArgsParser() + parser = argparse.ArgumentParser(description=__doc__) + input_opts = parser.add_argument_group('Input options') + output_opts = parser.add_argument_group('Output options') + + input_opts.add_argument('--include-resources', + action='append', + required=True, + help='Paths to arsc resource files used to link ' + 'against. Can be specified multiple times.') + input_opts.add_argument( + '--dependencies-res-zips', + default=[], + help='Resources zip archives from dependents. Required to ' + 'resolve @type/foo references into dependent libraries.') + input_opts.add_argument( + '--extra-res-packages', + help='Additional package names to generate R.java files for.') input_opts.add_argument( '--aapt2-path', required=True, help='Path to the Android aapt2 tool.') input_opts.add_argument( @@ -76,31 +93,25 @@ action='store_true', help='Same as --shared-resources, but also ensures all resource IDs are ' 'directly usable from the APK loaded as an application.') - input_opts.add_argument( '--package-id', type=int, help='Decimal integer representing custom package ID for resources ' '(instead of 127==0x7f). Cannot be used with --shared-resources.') - input_opts.add_argument( '--package-name', help='Package name that will be used to create R class.') - input_opts.add_argument( '--rename-manifest-package', help='Package name to force AAPT to use.') - input_opts.add_argument( '--arsc-package-name', help='Package name to set in manifest of resources.arsc file. This is ' 'only used for apks under test.') - input_opts.add_argument( '--shared-resources-allowlist', help='An R.txt file acting as a allowlist for resources that should be ' 'non-final and have their package ID changed at runtime in R.java. ' 'Implies and overrides --shared-resources.') - input_opts.add_argument( '--shared-resources-allowlist-locales', default='[]', @@ -108,28 +119,13 @@ ' to this locale list will be kept in the final output for the ' 'resources identified through --shared-resources-allowlist, even ' 'if --locale-allowlist is being used.') - input_opts.add_argument( '--use-resource-ids-path', help='Use resource IDs generated by aapt --emit-ids.') - - input_opts.add_argument( - '--extra-main-r-text-files', - help='Additional R.txt files that will be added to the root R.java file, ' - 'but not packaged in the generated resources.arsc. If these resources ' - 'entries contain duplicate resources with the generated R.txt file, they ' - 'must be identical.') - - input_opts.add_argument( - '--support-zh-hk', - action='store_true', - help='Use zh-rTW resources for zh-rHK.') - input_opts.add_argument( '--debuggable', action='store_true', help='Whether to add android:debuggable="true".') - input_opts.add_argument('--version-code', help='Version code for apk.') input_opts.add_argument('--version-name', help='Version name for apk.') input_opts.add_argument( @@ -143,7 +139,6 @@ help="android:maxSdkVersion expected in AndroidManifest.xml.") input_opts.add_argument( '--manifest-package', help='Package name of the AndroidManifest.xml.') - input_opts.add_argument( '--locale-allowlist', default='[]', @@ -159,17 +154,14 @@ default='[]', help='GN list of globs that say which files to include even ' 'when --resource-exclusion-regex is set.') - input_opts.add_argument( '--dependencies-res-zip-overlays', help='GN list with subset of --dependencies-res-zips to use overlay ' 'semantics for.') - input_opts.add_argument( '--values-filter-rules', help='GN list of source_glob:regex for filtering resources after they ' 'are compiled. Use this to filter out entries within values/ files.') - input_opts.add_argument('--png-to-webp', action='store_true', help='Convert png files to webp format.') @@ -177,105 +169,61 @@ help='Path to the cwebp binary.') input_opts.add_argument( '--webp-cache-dir', help='The directory to store webp image cache.') - - input_opts.add_argument( - '--no-xml-namespaces', - action='store_true', - help='Whether to strip xml namespaces from processed xml resources.') - input_opts.add_argument( - '--short-resource-paths', - action='store_true', - help='Whether to shorten resource paths inside the apk or module.') - input_opts.add_argument( - '--strip-resource-names', - action='store_true', - help='Whether to strip resource names from the resource table of the apk ' - 'or module.') - - output_opts.add_argument('--arsc-path', help='Apk output for arsc format.') - output_opts.add_argument('--proto-path', help='Apk output for proto format.') - group = input_opts.add_mutually_exclusive_group() - group.add_argument( - '--optimized-arsc-path', - help='Output for `aapt2 optimize` for arsc format (enables the step).') - group.add_argument( - '--optimized-proto-path', - help='Output for `aapt2 optimize` for proto format (enables the step).') - input_opts.add_argument( - '--resources-config-paths', - default='[]', - help='GN list of paths to aapt2 resources config files.') - - output_opts.add_argument( - '--info-path', help='Path to output info file for the partial apk.') - - output_opts.add_argument( - '--srcjar-out', - required=True, - help='Path to srcjar to contain generated R.java.') - - output_opts.add_argument('--r-text-out', - help='Path to store the generated R.txt file.') - - output_opts.add_argument( - '--proguard-file', help='Path to proguard.txt generated file.') - - output_opts.add_argument( - '--proguard-file-main-dex', - help='Path to proguard.txt generated file for main dex.') - - output_opts.add_argument( - '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.') - - output_opts.add_argument( - '--resources-path-map-out-path', - help='Path to file produced by aapt2 that maps original resource paths ' - 'to shortened resource paths inside the apk or module.') - input_opts.add_argument( '--is-bundle-module', action='store_true', help='Whether resources are being generated for a bundle module.') - input_opts.add_argument( '--uses-split', help='Value to set uses-split to in the AndroidManifest.xml.') - input_opts.add_argument( - '--extra-verification-manifest', - help='Path to AndroidManifest.xml which should be merged into base ' - 'manifest when performing verification.') + '--verification-version-code-offset', + help='Subtract this from versionCode for expectation files') + input_opts.add_argument( + '--verification-library-version-offset', + help='Subtract this from static-library version for expectation files') + + action_helpers.add_depfile_arg(output_opts) + output_opts.add_argument('--arsc-path', help='Apk output for arsc format.') + output_opts.add_argument('--proto-path', help='Apk output for proto format.') + output_opts.add_argument( + '--info-path', help='Path to output info file for the partial apk.') + output_opts.add_argument( + '--srcjar-out', + help='Path to srcjar to contain generated R.java.') + output_opts.add_argument('--r-text-out', + help='Path to store the generated R.txt file.') + output_opts.add_argument( + '--proguard-file', help='Path to proguard.txt generated file.') + output_opts.add_argument( + '--proguard-file-main-dex', + help='Path to proguard.txt generated file for main dex.') + output_opts.add_argument( + '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.') diff_utils.AddCommandLineFlags(parser) options = parser.parse_args(args) - resource_utils.HandleCommonOptions(options) - - options.locale_allowlist = build_utils.ParseGnList(options.locale_allowlist) - options.shared_resources_allowlist_locales = build_utils.ParseGnList( + options.include_resources = action_helpers.parse_gn_list( + options.include_resources) + options.dependencies_res_zips = action_helpers.parse_gn_list( + options.dependencies_res_zips) + options.extra_res_packages = action_helpers.parse_gn_list( + options.extra_res_packages) + options.locale_allowlist = action_helpers.parse_gn_list( + options.locale_allowlist) + options.shared_resources_allowlist_locales = action_helpers.parse_gn_list( options.shared_resources_allowlist_locales) - options.resource_exclusion_exceptions = build_utils.ParseGnList( + options.resource_exclusion_exceptions = action_helpers.parse_gn_list( options.resource_exclusion_exceptions) - options.dependencies_res_zip_overlays = build_utils.ParseGnList( + options.dependencies_res_zip_overlays = action_helpers.parse_gn_list( options.dependencies_res_zip_overlays) - options.values_filter_rules = build_utils.ParseGnList( + options.values_filter_rules = action_helpers.parse_gn_list( options.values_filter_rules) - options.extra_main_r_text_files = build_utils.ParseGnList( - options.extra_main_r_text_files) - options.resources_config_paths = build_utils.ParseGnList( - options.resources_config_paths) - - if options.optimized_proto_path and not options.proto_path: - # We could write to a temp file, but it's simpler to require it. - parser.error('--optimized-proto-path requires --proto-path') if not options.arsc_path and not options.proto_path: parser.error('One of --arsc-path or --proto-path is required.') - if options.resources_path_map_out_path and not options.short_resource_paths: - parser.error( - '--resources-path-map-out-path requires --short-resource-paths') - if options.package_id and options.shared_resources: parser.error('--package-id and --shared-resources are mutually exclusive') @@ -288,20 +236,6 @@ yield os.path.join(root, f) -def _DuplicateZhResources(resource_dirs, path_info): - """Duplicate Taiwanese resources into Hong-Kong specific directory.""" - for resource_dir in resource_dirs: - # We use zh-TW resources for zh-HK (if we have zh-TW resources). - for path in _IterFiles(resource_dir): - if 'zh-rTW' in path: - hk_path = path.replace('zh-rTW', 'zh-rHK') - build_utils.MakeDirectory(os.path.dirname(hk_path)) - shutil.copyfile(path, hk_path) - path_info.RegisterRename( - os.path.relpath(path, resource_dir), - os.path.relpath(hk_path, resource_dir)) - - def _RenameLocaleResourceDirs(resource_dirs, path_info): """Rename locale resource directories into standard names when necessary. @@ -324,15 +258,12 @@ * BCP 47 langauge tags will be renamed to an equivalent ISO 639-1 locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS'). - Though this is not necessary at the moment, because no third-party - package that Chromium links against uses these for the current list of - supported locales, this may change when the list is extended in the - future). Args: resource_dirs: list of top-level resource directories. """ for resource_dir in resource_dirs: + ignore_dirs = {} for path in _IterFiles(resource_dir): locale = resource_utils.FindLocaleInStringResourceFilePath(path) if not locale: @@ -346,10 +277,24 @@ if path == path2: raise Exception('Could not substitute locale %s for %s in %s' % (locale, locale2, path)) - if os.path.exists(path2): - # This happens sometimes, e.g. some libraries provide both - # values-nb/ and values-no/ with the same content. + + # Ignore rather than rename when the destination resources config + # already exists. + # e.g. some libraries provide both values-nb/ and values-no/. + # e.g. material design provides: + # * res/values-rUS/values-rUS.xml + # * res/values-b+es+419/values-b+es+419.xml + config_dir = os.path.dirname(path2) + already_has_renamed_config = ignore_dirs.get(config_dir) + if already_has_renamed_config is None: + # Cache the result of the first time the directory is encountered + # since subsequent encounters will find the directory already exists + # (due to the rename). + already_has_renamed_config = os.path.exists(config_dir) + ignore_dirs[config_dir] = already_has_renamed_config + if already_has_renamed_config: continue + build_utils.MakeDirectory(os.path.dirname(path2)) shutil.move(path, path2) path_info.RegisterRename( @@ -357,13 +302,11 @@ os.path.relpath(path2, resource_dir)) -def _ToAndroidLocales(locale_allowlist, support_zh_hk): +def _ToAndroidLocales(locale_allowlist): """Converts the list of Chrome locales to Android config locale qualifiers. Args: locale_allowlist: A list of Chromium locale names. - support_zh_hk: True if we need to support zh-HK by duplicating - the zh-TW strings. Returns: A set of matching Android config locale qualifier names. """ @@ -377,14 +320,7 @@ language = locale.split('-')[0] ret.add(language) - # We don't actually support zh-HK in Chrome on Android, but we mimic the - # native side behavior where we use zh-TW resources when the locale is set to - # zh-HK. See https://crbug.com/780847. - if support_zh_hk: - assert not any('HK' in l for l in locale_allowlist), ( - 'Remove special logic if zh-HK is now supported (crbug.com/780847).') - ret.add('zh-rHK') - return set(ret) + return ret def _MoveImagesToNonMdpiFolders(res_root, path_info): @@ -416,7 +352,37 @@ os.path.relpath(dst_file, res_root)) -def _FixManifest(options, temp_dir, extra_manifest=None): +def _DeterminePlatformVersion(aapt2_path, jar_candidates): + def maybe_extract_version(j): + try: + return resource_utils.ExtractBinaryManifestValues(aapt2_path, j) + except build_utils.CalledProcessError: + return None + + def is_sdk_jar(jar_name): + if jar_name in ('android.jar', 'android_system.jar'): + return True + # Robolectric jar looks a bit different. + return 'android-all' in jar_name and 'robolectric' in jar_name + + android_sdk_jars = [ + j for j in jar_candidates if is_sdk_jar(os.path.basename(j)) + ] + extract_all = [maybe_extract_version(j) for j in android_sdk_jars] + extract_all = [x for x in extract_all if x] + if len(extract_all) == 0: + raise Exception( + 'Unable to find android SDK jar among candidates: %s' + % ', '.join(android_sdk_jars)) + if len(extract_all) > 1: + raise Exception( + 'Found multiple android SDK jars among candidates: %s' + % ', '.join(android_sdk_jars)) + platform_version_code, platform_version_name = extract_all.pop()[:2] + return platform_version_code, platform_version_name + + +def _FixManifest(options, temp_dir): """Fix the APK's AndroidManifest.xml. This adds any missing namespaces for 'android' and 'tools', and @@ -426,74 +392,40 @@ Args: options: The command-line arguments tuple. temp_dir: A temporary directory where the fixed manifest will be written to. - extra_manifest: Path to an AndroidManifest.xml file which will get merged - into the application node of the base manifest. Returns: Tuple of: * Manifest path within |temp_dir|. * Original package_name. + * Manifest package name. """ - def maybe_extract_version(j): - try: - return resource_utils.ExtractBinaryManifestValues(options.aapt2_path, j) - except build_utils.CalledProcessError: - return None - - android_sdk_jars = [j for j in options.include_resources - if os.path.basename(j) in ('android.jar', - 'android_system.jar')] - extract_all = [maybe_extract_version(j) for j in android_sdk_jars] - successful_extractions = [x for x in extract_all if x] - if len(successful_extractions) == 0: - raise Exception( - 'Unable to find android SDK jar among candidates: %s' - % ', '.join(android_sdk_jars)) - elif len(successful_extractions) > 1: - raise Exception( - 'Found multiple android SDK jars among candidates: %s' - % ', '.join(android_sdk_jars)) - version_code, version_name = successful_extractions.pop()[:2] - - debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml') doc, manifest_node, app_node = manifest_utils.ParseManifest( options.android_manifest) - if extra_manifest: - _, extra_manifest_node, extra_app_node = manifest_utils.ParseManifest( - extra_manifest) - for node in extra_app_node: - app_node.append(node) - for node in extra_manifest_node: - # DFM manifests have a bunch of tags we don't care about inside - # <manifest>, so only take <queries>. - if node.tag == 'queries': - manifest_node.append(node) + # merge_manifest.py also sets package & <uses-sdk>. We may want to ensure + # manifest merger is always enabled and remove these command-line arguments. + manifest_utils.SetUsesSdk(manifest_node, options.target_sdk_version, + options.min_sdk_version, options.max_sdk_version) + orig_package = manifest_node.get('package') or options.manifest_package + fixed_package = (options.arsc_package_name or options.manifest_package + or orig_package) + manifest_node.set('package', fixed_package) - manifest_utils.AssertUsesSdk(manifest_node, options.min_sdk_version, - options.target_sdk_version) - # We explicitly check that maxSdkVersion is set in the manifest since we don't - # add it later like minSdkVersion and targetSdkVersion. - manifest_utils.AssertUsesSdk( - manifest_node, - max_sdk_version=options.max_sdk_version, - fail_if_not_exist=True) - manifest_utils.AssertPackage(manifest_node, options.manifest_package) - - manifest_node.set('platformBuildVersionCode', version_code) - manifest_node.set('platformBuildVersionName', version_name) - - orig_package = manifest_node.get('package') - if options.arsc_package_name: - manifest_node.set('package', options.arsc_package_name) - + platform_version_code, platform_version_name = _DeterminePlatformVersion( + options.aapt2_path, options.include_resources) + manifest_node.set('platformBuildVersionCode', platform_version_code) + manifest_node.set('platformBuildVersionName', platform_version_name) + if options.version_code: + manifest_utils.NamespacedSet(manifest_node, 'versionCode', + options.version_code) + if options.version_name: + manifest_utils.NamespacedSet(manifest_node, 'versionName', + options.version_name) if options.debuggable: - app_node.set('{%s}%s' % (manifest_utils.ANDROID_NAMESPACE, 'debuggable'), - 'true') + manifest_utils.NamespacedSet(app_node, 'debuggable', 'true') if options.uses_split: uses_split = ElementTree.SubElement(manifest_node, 'uses-split') - uses_split.set('{%s}name' % manifest_utils.ANDROID_NAMESPACE, - options.uses_split) + manifest_utils.NamespacedSet(uses_split, 'name', options.uses_split) # Make sure the min-sdk condition is not less than the min-sdk of the bundle. for min_sdk_node in manifest_node.iter('{%s}min-sdk' % @@ -502,8 +434,9 @@ if int(min_sdk_node.get(dist_value)) < int(options.min_sdk_version): min_sdk_node.set(dist_value, options.min_sdk_version) + debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml') manifest_utils.SaveManifest(doc, debug_manifest_path) - return debug_manifest_path, orig_package + return debug_manifest_path, orig_package, fixed_package def _CreateKeepPredicate(resource_exclusion_regex, @@ -710,8 +643,7 @@ # list provided by --locale-allowlist. wanted_locales = all_locales if options.locale_allowlist: - wanted_locales = _ToAndroidLocales(options.locale_allowlist, - options.support_zh_hk) + wanted_locales = _ToAndroidLocales(options.locale_allowlist) # Set B: shared resources locales, which is either set A # or the list provided by --shared-resources-allowlist-locales @@ -723,7 +655,7 @@ options.shared_resources_allowlist)) shared_resources_locales = _ToAndroidLocales( - options.shared_resources_allowlist_locales, options.support_zh_hk) + options.shared_resources_allowlist_locales) # Remove any file that belongs to a locale not covered by # either A or B. @@ -783,8 +715,6 @@ logging.debug('Applying locale transformations') path_info = resource_utils.ResourceInfoFile() - if options.support_zh_hk: - _DuplicateZhResources(dep_subdirs, path_info) _RenameLocaleResourceDirs(dep_subdirs, path_info) logging.debug('Applying file-based exclusions') @@ -816,19 +746,12 @@ 'link', '--auto-add-overlay', '--no-version-vectors', - # Set SDK versions in case they are not set in the Android manifest. - '--min-sdk-version', - options.min_sdk_version, - '--target-sdk-version', - options.target_sdk_version, + '--output-text-symbols', + build.r_txt_path, ] for j in options.include_resources: link_command += ['-I', j] - if options.version_code: - link_command += ['--version-code', options.version_code] - if options.version_name: - link_command += ['--version-name', options.version_name] if options.proguard_file: link_command += ['--proguard', build.proguard_path] link_command += ['--proguard-minimal-keep-rules'] @@ -836,28 +759,24 @@ link_command += ['--proguard-main-dex', build.proguard_main_dex_path] if options.emit_ids_out: link_command += ['--emit-ids', build.emit_ids_path] - if options.r_text_in: - shutil.copyfile(options.r_text_in, build.r_txt_path) - else: - link_command += ['--output-text-symbols', build.r_txt_path] # Note: only one of --proto-format, --shared-lib or --app-as-shared-lib # can be used with recent versions of aapt2. if options.shared_resources: link_command.append('--shared-lib') - if options.no_xml_namespaces: + if int(options.min_sdk_version) > 21: link_command.append('--no-xml-namespaces') if options.package_id: link_command += [ '--package-id', - hex(options.package_id), + '0x%02x' % options.package_id, '--allow-reserved-package-id', ] - fixed_manifest, desired_manifest_package_name = _FixManifest( - options, build.temp_dir) + fixed_manifest, desired_manifest_package_name, fixed_manifest_package = ( + _FixManifest(options, build.temp_dir)) if options.rename_manifest_package: desired_manifest_package_name = options.rename_manifest_package @@ -866,32 +785,41 @@ desired_manifest_package_name ] - # Creates a .zip with AndroidManifest.xml, resources.arsc, res/* - # Also creates R.txt - if options.use_resource_ids_path: - _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path, - desired_manifest_package_name) - link_command += ['--stable-ids', build.stable_ids_path] + if options.package_id is not None: + package_id = options.package_id + elif options.shared_resources: + package_id = 0 + else: + package_id = 0x7f + _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path, + fixed_manifest_package, package_id) + link_command += ['--stable-ids', build.stable_ids_path] link_command += partials # We always create a binary arsc file first, then convert to proto, so flags # such as --shared-lib can be supported. - arsc_path = build.arsc_path - if arsc_path is None: - _, arsc_path = tempfile.mkstmp() link_command += ['-o', build.arsc_path] logging.debug('Starting: aapt2 link') link_proc = subprocess.Popen(link_command) # Create .res.info file in parallel. - _CreateResourceInfoFile(path_info, build.info_path, - options.dependencies_res_zips) - logging.debug('Created .res.info file') + if options.info_path: + logging.debug('Creating .res.info file') + _CreateResourceInfoFile(path_info, build.info_path, + options.dependencies_res_zips) exit_code = link_proc.wait() + assert exit_code == 0, f'aapt2 link cmd failed with {exit_code=}' logging.debug('Finished: aapt2 link') + + if options.shared_resources: + logging.debug('Resolving styleables in R.txt') + # Need to resolve references because unused resource removal tool does not + # support references in R.txt files. + resource_utils.ResolveStyleableReferences(build.r_txt_path) + if exit_code: raise subprocess.CalledProcessError(exit_code, link_command) @@ -902,7 +830,7 @@ # can call it in the case where the APK is being loaded as a library. with open(build.proguard_path, 'a') as proguard_file: keep_rule = ''' - -keep class {package}.R {{ + -keep,allowoptimization class {package}.R {{ public static void onResourcesLoaded(int); }} '''.format(package=desired_manifest_package_name) @@ -927,120 +855,38 @@ build.arsc_path, build.proto_path ]) - if build.arsc_path is None: - os.remove(arsc_path) - - if options.optimized_proto_path: - _OptimizeApk(build.optimized_proto_path, options, build.temp_dir, - build.proto_path, build.r_txt_path) - elif options.optimized_arsc_path: - _OptimizeApk(build.optimized_arsc_path, options, build.temp_dir, - build.arsc_path, build.r_txt_path) + # Sanity check that the created resources have the expected package ID. + logging.debug('Performing sanity check') + _, actual_package_id = resource_utils.ExtractArscPackage( + options.aapt2_path, + build.arsc_path if options.arsc_path else build.proto_path) + # When there are no resources, ExtractArscPackage returns (None, None), in + # this case there is no need to check for matching package ID. + if actual_package_id is not None and actual_package_id != package_id: + raise Exception('Invalid package ID 0x%x (expected 0x%x)' % + (actual_package_id, package_id)) return desired_manifest_package_name -def _CombineResourceConfigs(resources_config_paths, out_config_path): - with open(out_config_path, 'w') as out_config: - for config_path in resources_config_paths: - with open(config_path) as config: - out_config.write(config.read()) - out_config.write('\n') - - -def _OptimizeApk(output, options, temp_dir, unoptimized_path, r_txt_path): - """Optimize intermediate .ap_ file with aapt2. - - Args: - output: Path to write to. - options: The command-line options. - temp_dir: A temporary directory. - unoptimized_path: path of the apk to optimize. - r_txt_path: path to the R.txt file of the unoptimized apk. - """ - optimize_command = [ - options.aapt2_path, - 'optimize', - unoptimized_path, - '-o', - output, - ] - - # Optimize the resources.arsc file by obfuscating resource names and only - # allow usage via R.java constant. - if options.strip_resource_names: - no_collapse_resources = _ExtractNonCollapsableResources(r_txt_path) - gen_config_path = os.path.join(temp_dir, 'aapt2.config') - if options.resources_config_paths: - _CombineResourceConfigs(options.resources_config_paths, gen_config_path) - with open(gen_config_path, 'a') as config: - for resource in no_collapse_resources: - config.write('{}#no_collapse\n'.format(resource)) - - optimize_command += [ - '--collapse-resource-names', - '--resources-config-path', - gen_config_path, - ] - - if options.short_resource_paths: - optimize_command += ['--shorten-resource-paths'] - if options.resources_path_map_out_path: - optimize_command += [ - '--resource-path-shortening-map', options.resources_path_map_out_path - ] - - logging.debug('Running aapt2 optimize') - build_utils.CheckOutput( - optimize_command, print_stdout=False, print_stderr=False) - - -def _ExtractNonCollapsableResources(rtxt_path): - """Extract resources that should not be collapsed from the R.txt file - - Resources of type ID are references to UI elements/views. They are used by - UI automation testing frameworks. They are kept in so that they don't break - tests, even though they may not actually be used during runtime. See - https://crbug.com/900993 - App icons (aka mipmaps) are sometimes referenced by other apps by name so must - be keps as well. See https://b/161564466 - - Args: - rtxt_path: Path to R.txt file with all the resources - Returns: - List of resources in the form of <resource_type>/<resource_name> - """ - resources = [] - _NO_COLLAPSE_TYPES = ['id', 'mipmap'] - with open(rtxt_path) as rtxt: - for line in rtxt: - for resource_type in _NO_COLLAPSE_TYPES: - if ' {} '.format(resource_type) in line: - resource_name = line.split()[2] - resources.append('{}/{}'.format(resource_type, resource_name)) - return resources - - -@contextlib.contextmanager -def _CreateStableIdsFile(in_path, out_path, package_name): +def _CreateStableIdsFile(in_path, out_path, package_name, package_id): """Transforms a file generated by --emit-ids from another package. --stable-ids is generally meant to be used by different versions of the same package. To make it work for other packages, we need to transform the package name references to match the package that resources are being generated for. - - Note: This will fail if the package ID of the resources in - |options.use_resource_ids_path| does not match the package ID of the - resources being linked. """ - with open(in_path) as stable_ids_file: - with open(out_path, 'w') as output_ids_file: - output_stable_ids = re.sub( - r'^.*?:', - package_name + ':', - stable_ids_file.read(), - flags=re.MULTILINE) - output_ids_file.write(output_stable_ids) + if in_path: + data = pathlib.Path(in_path).read_text() + else: + # Force IDs to use 0x01 for the type byte in order to ensure they are + # different from IDs generated by other apps. https://crbug.com/1293336 + data = 'pkg:id/fake_resource_id = 0x7f010000\n' + # Replace "pkg:" with correct package name. + data = re.sub(r'^.*?:', package_name + ':', data, flags=re.MULTILINE) + # Replace "0x7f" with correct package id. + data = re.sub(r'0x..', '0x%02x' % package_id, data) + pathlib.Path(out_path).write_text(data) def _WriteOutputs(options, build): @@ -1049,8 +895,6 @@ (options.r_text_out, build.r_txt_path), (options.arsc_path, build.arsc_path), (options.proto_path, build.proto_path), - (options.optimized_arsc_path, build.optimized_arsc_path), - (options.optimized_proto_path, build.optimized_proto_path), (options.proguard_file, build.proguard_path), (options.proguard_file_main_dex, build.proguard_main_dex_path), (options.emit_ids_out, build.emit_ids_path), @@ -1065,10 +909,11 @@ def _CreateNormalizedManifestForVerification(options): with build_utils.TempDir() as tempdir: - fixed_manifest, _ = _FixManifest( - options, tempdir, extra_manifest=options.extra_verification_manifest) + fixed_manifest, _, _ = _FixManifest(options, tempdir) with open(fixed_manifest) as f: - return manifest_utils.NormalizeManifest(f.read()) + return manifest_utils.NormalizeManifest( + f.read(), options.verification_version_code_offset, + options.verification_library_version_offset) def main(args): @@ -1142,37 +987,27 @@ # will be created in the base module. apk_package_name = None - logging.debug('Creating R.srcjar') - resource_utils.CreateRJavaFiles( - build.srcjar_dir, apk_package_name, build.r_txt_path, - options.extra_res_packages, rjava_build_options, options.srcjar_out, - custom_root_package_name, grandparent_custom_package_name, - options.extra_main_r_text_files) - build_utils.ZipDir(build.srcjar_path, build.srcjar_dir) - - # Sanity check that the created resources have the expected package ID. - logging.debug('Performing sanity check') - if options.package_id: - expected_id = options.package_id - elif options.shared_resources: - expected_id = 0 - else: - expected_id = 127 # == '0x7f'. - _, package_id = resource_utils.ExtractArscPackage( - options.aapt2_path, - build.arsc_path if options.arsc_path else build.proto_path) - if package_id != expected_id: - raise Exception( - 'Invalid package ID 0x%x (expected 0x%x)' % (package_id, expected_id)) + if options.srcjar_out: + logging.debug('Creating R.srcjar') + resource_utils.CreateRJavaFiles(build.srcjar_dir, apk_package_name, + build.r_txt_path, + options.extra_res_packages, + rjava_build_options, options.srcjar_out, + custom_root_package_name, + grandparent_custom_package_name) + with action_helpers.atomic_output(build.srcjar_path) as f: + zip_helpers.zip_directory(f, build.srcjar_dir) logging.debug('Copying outputs') _WriteOutputs(options, build) if options.depfile: + assert options.srcjar_out, 'Update first output below and remove assert.' depfile_deps = (options.dependencies_res_zips + options.dependencies_res_zip_overlays + - options.extra_main_r_text_files + options.include_resources) - build_utils.WriteDepfile(options.depfile, options.srcjar_out, depfile_deps) + options.include_resources) + action_helpers.write_depfile(options.depfile, options.srcjar_out, + depfile_deps) if __name__ == '__main__':
diff --git a/build/android/gyp/compile_resources.pydeps b/build/android/gyp/compile_resources.pydeps index 174b526..458a772 100644 --- a/build/android/gyp/compile_resources.pydeps +++ b/build/android/gyp/compile_resources.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/compile_resources.pydeps build/android/gyp/compile_resources.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,31 +22,9 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py -../../../third_party/protobuf/python/google/__init__.py -../../../third_party/protobuf/python/google/protobuf/__init__.py -../../../third_party/protobuf/python/google/protobuf/descriptor.py -../../../third_party/protobuf/python/google/protobuf/descriptor_database.py -../../../third_party/protobuf/python/google/protobuf/descriptor_pool.py -../../../third_party/protobuf/python/google/protobuf/internal/__init__.py -../../../third_party/protobuf/python/google/protobuf/internal/api_implementation.py -../../../third_party/protobuf/python/google/protobuf/internal/containers.py -../../../third_party/protobuf/python/google/protobuf/internal/decoder.py -../../../third_party/protobuf/python/google/protobuf/internal/encoder.py -../../../third_party/protobuf/python/google/protobuf/internal/enum_type_wrapper.py -../../../third_party/protobuf/python/google/protobuf/internal/extension_dict.py -../../../third_party/protobuf/python/google/protobuf/internal/message_listener.py -../../../third_party/protobuf/python/google/protobuf/internal/python_message.py -../../../third_party/protobuf/python/google/protobuf/internal/type_checkers.py -../../../third_party/protobuf/python/google/protobuf/internal/well_known_types.py -../../../third_party/protobuf/python/google/protobuf/internal/wire_format.py -../../../third_party/protobuf/python/google/protobuf/message.py -../../../third_party/protobuf/python/google/protobuf/message_factory.py -../../../third_party/protobuf/python/google/protobuf/reflection.py -../../../third_party/protobuf/python/google/protobuf/symbol_database.py -../../../third_party/protobuf/python/google/protobuf/text_encoding.py -../../../third_party/protobuf/python/google/protobuf/text_format.py -../../../third_party/six/src/six.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py compile_resources.py proto/Configuration_pb2.py proto/Resources_pb2.py
diff --git a/build/android/gyp/copy_ex.py b/build/android/gyp/copy_ex.py index 41604c4..542a08c 100755 --- a/build/android/gyp/copy_ex.py +++ b/build/android/gyp/copy_ex.py
@@ -1,12 +1,11 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Copies files to a directory.""" -from __future__ import print_function import filecmp import itertools @@ -16,6 +15,7 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. def _get_all_files(base): @@ -50,8 +50,9 @@ def DoCopy(options, deps): """Copy files or directories given in options.files and update deps.""" - files = list(itertools.chain.from_iterable(build_utils.ParseGnList(f) - for f in options.files)) + files = list( + itertools.chain.from_iterable( + action_helpers.parse_gn_list(f) for f in options.files)) for f in files: if os.path.isdir(f) and not options.clear: @@ -62,13 +63,14 @@ def DoRenaming(options, deps): """Copy and rename files given in options.renaming_sources and update deps.""" - src_files = list(itertools.chain.from_iterable( - build_utils.ParseGnList(f) - for f in options.renaming_sources)) + src_files = list( + itertools.chain.from_iterable( + action_helpers.parse_gn_list(f) for f in options.renaming_sources)) - dest_files = list(itertools.chain.from_iterable( - build_utils.ParseGnList(f) - for f in options.renaming_destinations)) + dest_files = list( + itertools.chain.from_iterable( + action_helpers.parse_gn_list(f) + for f in options.renaming_destinations)) if (len(src_files) != len(dest_files)): print('Renaming source and destination files not match.') @@ -85,7 +87,7 @@ args = build_utils.ExpandFileArgs(args) parser = optparse.OptionParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_option('--dest', help='Directory to copy files to.') parser.add_option('--files', action='append', @@ -119,7 +121,7 @@ DoRenaming(options, deps) if options.depfile: - build_utils.WriteDepfile(options.depfile, options.stamp, deps) + action_helpers.write_depfile(options.depfile, options.stamp, deps) if options.stamp: build_utils.Touch(options.stamp)
diff --git a/build/android/gyp/copy_ex.pydeps b/build/android/gyp/copy_ex.pydeps index 3735251..5d75f9a 100644 --- a/build/android/gyp/copy_ex.pydeps +++ b/build/android/gyp/copy_ex.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/copy_ex.pydeps build/android/gyp/copy_ex.py +../../action_helpers.py ../../gn_helpers.py copy_ex.py util/__init__.py
diff --git a/build/android/gyp/create_apk_operations_script.py b/build/android/gyp/create_apk_operations_script.py index 660567f..1d1cb5d 100755 --- a/build/android/gyp/create_apk_operations_script.py +++ b/build/android/gyp/create_apk_operations_script.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,7 +12,7 @@ from util import build_utils SCRIPT_TEMPLATE = string.Template("""\ -#!/usr/bin/env python +#!/usr/bin/env python3 # # This file was generated by build/android/gyp/create_apk_operations_script.py @@ -26,21 +26,14 @@ sys.path.append(resolve(${APK_OPERATIONS_DIR})) import apk_operations output_dir = resolve(${OUTPUT_DIR}) - try: - apk_operations.Run( - output_dir, - resolve(${APK_PATH}), - [resolve(p) for p in ${ADDITIONAL_APK_PATHS}], - resolve(${INC_JSON_PATH}), - ${FLAGS_FILE}, - ${TARGET_CPU}, - resolve(${MAPPING_PATH})) - except TypeError: - rel_output_dir = os.path.relpath(output_dir) - rel_script_path = os.path.relpath(sys.argv[0], output_dir) - sys.stderr.write('Script out-of-date. Rebuild via:\\n') - sys.stderr.write(' ninja -C %s %s\\n' % (rel_output_dir, rel_script_path)) - return 1 + apk_operations.Run( + output_dir, + resolve(${APK_PATH}), + [resolve(p) for p in ${ADDITIONAL_APK_PATHS}], + resolve(${INC_JSON_PATH}), + ${FLAGS_FILE}, + ${TARGET_CPU}, + resolve(${MAPPING_PATH})) if __name__ == '__main__':
diff --git a/build/android/gyp/create_app_bundle.py b/build/android/gyp/create_app_bundle.py index 0b44c16..1282608 100755 --- a/build/android/gyp/create_app_bundle.py +++ b/build/android/gyp/create_app_bundle.py
@@ -1,28 +1,33 @@ #!/usr/bin/env python3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Create an Android application bundle from one or more bundle modules.""" import argparse +import concurrent.futures import json +import logging import os +import posixpath import shutil import sys +from xml.etree import ElementTree import zipfile sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) from pylib.utils import dexdump +import bundletool from util import build_utils from util import manifest_utils from util import resource_utils -from xml.etree import ElementTree +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers -import bundletool # Location of language-based assets in bundle modules. _LOCALES_SUBDIR = 'assets/locales/' @@ -60,6 +65,11 @@ 'xmf' ] +_COMPONENT_TYPES = ('activity', 'provider', 'receiver', 'service') +_DEDUPE_ENTRY_TYPES = _COMPONENT_TYPES + ('activity-alias', 'meta-data') + +_ROTATION_METADATA_KEY = 'com.google.play.apps.signing/RotationConfig.textproto' + def _ParseArgs(args): parser = argparse.ArgumentParser() @@ -88,6 +98,9 @@ '--compress-shared-libraries', action='store_true', help='Whether to store native libraries compressed.') + parser.add_argument('--compress-dex', + action='store_true', + help='Compress .dex files') parser.add_argument('--split-dimensions', help="GN-list of split dimensions to support.") parser.add_argument( @@ -100,6 +113,8 @@ 'listed there _and_ in --base-module-rtxt-path will ' 'be kept in the base bundle module, even if language' ' splitting is enabled.') + parser.add_argument('--rotation-config', + help='Path to a RotationConfig.textproto') parser.add_argument('--warnings-as-errors', action='store_true', help='Treat all warnings as errors.') @@ -110,30 +125,47 @@ help='Check if services are in base module if isolatedSplits is enabled.') options = parser.parse_args(args) - options.module_zips = build_utils.ParseGnList(options.module_zips) - options.rtxt_in_paths = build_utils.ParseGnList(options.rtxt_in_paths) - options.pathmap_in_paths = build_utils.ParseGnList(options.pathmap_in_paths) + options.module_zips = action_helpers.parse_gn_list(options.module_zips) if len(options.module_zips) == 0: - raise Exception('The module zip list cannot be empty.') + parser.error('The module zip list cannot be empty.') + if len(options.module_zips) != len(options.module_names): + parser.error('# module zips != # names.') + if 'base' not in options.module_names: + parser.error('Missing base module.') + + # Sort modules for more stable outputs. + per_module_values = list( + zip(options.module_names, options.module_zips, + options.uncompressed_assets, options.rtxt_in_paths, + options.pathmap_in_paths)) + per_module_values.sort(key=lambda x: (x[0] != 'base', x[0])) + options.module_names = [x[0] for x in per_module_values] + options.module_zips = [x[1] for x in per_module_values] + options.uncompressed_assets = [x[2] for x in per_module_values] + options.rtxt_in_paths = [x[3] for x in per_module_values] + options.pathmap_in_paths = [x[4] for x in per_module_values] + + options.rtxt_in_paths = action_helpers.parse_gn_list(options.rtxt_in_paths) + options.pathmap_in_paths = action_helpers.parse_gn_list( + options.pathmap_in_paths) # Merge all uncompressed assets into a set. uncompressed_list = [] - if options.uncompressed_assets: - for l in options.uncompressed_assets: - for entry in build_utils.ParseGnList(l): - # Each entry has the following format: 'zipPath' or 'srcPath:zipPath' - pos = entry.find(':') - if pos >= 0: - uncompressed_list.append(entry[pos + 1:]) - else: - uncompressed_list.append(entry) + for entry in action_helpers.parse_gn_list(options.uncompressed_assets): + # Each entry has the following format: 'zipPath' or 'srcPath:zipPath' + pos = entry.find(':') + if pos >= 0: + uncompressed_list.append(entry[pos + 1:]) + else: + uncompressed_list.append(entry) options.uncompressed_assets = set(uncompressed_list) # Check that all split dimensions are valid if options.split_dimensions: - options.split_dimensions = build_utils.ParseGnList(options.split_dimensions) + options.split_dimensions = action_helpers.parse_gn_list( + options.split_dimensions) for dim in options.split_dimensions: if dim.upper() not in _ALL_SPLIT_DIMENSIONS: parser.error('Invalid split dimension "%s" (expected one of: %s)' % ( @@ -162,13 +194,15 @@ return {'value': value, 'negate': not enabled} -def _GenerateBundleConfigJson(uncompressed_assets, compress_shared_libraries, - split_dimensions, base_master_resource_ids): +def _GenerateBundleConfigJson(uncompressed_assets, compress_dex, + compress_shared_libraries, split_dimensions, + base_master_resource_ids): """Generate a dictionary that can be written to a JSON BuildConfig. Args: uncompressed_assets: A list or set of file paths under assets/ that always be stored uncompressed. + compressed_dex: Boolean, whether to compress .dex. compress_shared_libraries: Boolean, whether to compress native libs. split_dimensions: list of split dimensions. base_master_resource_ids: Optional list of 32-bit resource IDs to keep @@ -185,16 +219,22 @@ split_dimensions = [ _MakeSplitDimension(dim, dim in split_dimensions) for dim in _ALL_SPLIT_DIMENSIONS ] - # Native libraries loaded by the crazy linker. - # Whether other .so files are compressed is controlled by - # "uncompressNativeLibraries". - uncompressed_globs = ['lib/*/crazy.*'] # Locale-specific pak files stored in bundle splits need not be compressed. + uncompressed_globs = [ + 'assets/locales#lang_*/*.pak', 'assets/fallback-locales/*.pak' + ] + # normpath to allow for ../ prefix. uncompressed_globs.extend( - ['assets/locales#lang_*/*.pak', 'assets/fallback-locales/*.pak']) - uncompressed_globs.extend('assets/' + x for x in uncompressed_assets) + posixpath.normpath('assets/' + x) for x in uncompressed_assets) # NOTE: Use '**' instead of '*' to work through directories! uncompressed_globs.extend('**.' + ext for ext in _UNCOMPRESSED_FILE_EXTS) + if not compress_dex: + # Explicit glob required only when using bundletool to create .apks files. + # Play Store looks for and respects "uncompressDexFiles" set below. + # b/176198991 + # This is added as a placeholder entry in order to have no effect unless + # processed with app_bundle_utils.GenerateBundleApks(). + uncompressed_globs.append('classesX.dex') data = { 'optimizations': { @@ -298,11 +338,10 @@ if src_path in language_files: dst_path = _RewriteLanguageAssetPath(src_path) - build_utils.AddToZipHermetic( - dst_zip, - dst_path, - data=src_zip.read(src_path), - compress=is_compressed) + zip_helpers.add_to_zip_hermetic(dst_zip, + dst_path, + data=src_zip.read(src_path), + compress=is_compressed) return tmp_zip @@ -382,10 +421,14 @@ def _GetManifestForModule(bundle_path, module_name): - return ElementTree.fromstring( - bundletool.RunBundleTool([ - 'dump', 'manifest', '--bundle', bundle_path, '--module', module_name - ])) + data = bundletool.RunBundleTool( + ['dump', 'manifest', '--bundle', bundle_path, '--module', module_name]) + try: + return ElementTree.fromstring(data) + except ElementTree.ParseError: + sys.stderr.write('Failed to parse:\n') + sys.stderr.write(data) + raise def _GetComponentNames(manifest, tag_name): @@ -393,77 +436,89 @@ return [s.attrib.get(android_name) for s in manifest.iter(tag_name)] -def _MaybeCheckServicesAndProvidersPresentInBase(bundle_path, module_zips): - """Checks bundles with isolated splits define all services in the base module. +def _ClassesFromZip(module_zip): + classes = set() + for package in dexdump.Dump(module_zip): + for java_package, package_dict in package.items(): + java_package += '.' if java_package else '' + classes.update(java_package + c for c in package_dict['classes']) + return classes - Due to b/169196314, service classes are not found if they are not present in - the base module. Providers are also checked because they are loaded early in - startup, and keeping them in the base module gives more time for the chrome - split to load. - """ - base_manifest = _GetManifestForModule(bundle_path, 'base') - isolated_splits = base_manifest.get('{%s}isolatedSplits' % - manifest_utils.ANDROID_NAMESPACE) - if isolated_splits != 'true': - return + +def _ValidateSplits(bundle_path, module_zips): + logging.info('Reading manifests and running dexdump') + base_zip = next(p for p in module_zips if os.path.basename(p) == 'base.zip') + module_names = sorted(os.path.basename(p)[:-len('.zip')] for p in module_zips) + # Using threads makes these step go from 7s -> 1s on my machine. + with concurrent.futures.ThreadPoolExecutor() as executor: + # Create list of classes from the base module's dex. + classes_future = executor.submit(_ClassesFromZip, base_zip) + + # Create xmltrees of all module manifests. + manifest_futures = [ + executor.submit(_GetManifestForModule, bundle_path, n) + for n in module_names + ] + manifests_by_name = dict( + zip(module_names, (f.result() for f in manifest_futures))) + base_classes = classes_future.result() # Collect service names from all split manifests. - base_zip = None - service_names = _GetComponentNames(base_manifest, 'service') - provider_names = _GetComponentNames(base_manifest, 'provider') - for module_zip in module_zips: - name = os.path.basename(module_zip)[:-len('.zip')] - if name == 'base': - base_zip = module_zip - else: - service_names.extend( - _GetComponentNames(_GetManifestForModule(bundle_path, name), - 'service')) - module_providers = _GetComponentNames( - _GetManifestForModule(bundle_path, name), 'provider') - if module_providers: - raise Exception("Providers should all be declared in the base manifest." - " '%s' module declared: %s" % (name, module_providers)) + logging.info('Performing checks') + errors = [] - # Extract classes from the base module's dex. - classes = set() - base_package_name = manifest_utils.GetPackage(base_manifest) - for package in dexdump.Dump(base_zip): - for name, package_dict in package.items(): - if not name: - name = base_package_name - classes.update('%s.%s' % (name, c) - for c in package_dict['classes'].keys()) + # Ensure there are no components defined in multiple splits. + splits_by_component = {} + for module_name, cur_manifest in manifests_by_name.items(): + for kind in _DEDUPE_ENTRY_TYPES: + for component in _GetComponentNames(cur_manifest, kind): + owner_module_name = splits_by_component.setdefault((kind, component), + module_name) + # Allow services that exist only to keep <meta-data> out of + # ApplicationInfo. + if (owner_module_name != module_name + and not component.endswith('HolderService')): + errors.append(f'The {kind} "{component}" appeared in both ' + f'{owner_module_name} and {module_name}.') - ignored_service_names = { - # Defined in the chime DFM manifest, but unused. - # org.chromium.chrome.browser.chime.ScheduledTaskService is used instead. - ("com.google.android.libraries.notifications.entrypoints.scheduled." - "ScheduledTaskService"), + # Ensure components defined in base manifest exist in base dex. + for (kind, component), module_name in splits_by_component.items(): + if module_name == 'base' and kind in _COMPONENT_TYPES: + if component not in base_classes: + errors.append(f"{component} is defined in the base manfiest, " + f"but the class does not exist in the base splits' dex") - # Defined in the chime DFM manifest, only used pre-O (where isolated - # splits are not supported). - ("com.google.android.libraries.notifications.executor.impl.basic." - "ChimeExecutorApiService"), - } + # Remaining checks apply only when isolatedSplits="true". + isolated_splits = manifests_by_name['base'].get( + f'{manifest_utils.ANDROID_NAMESPACE}isolatedSplits') + if isolated_splits != 'true': + return errors - # Ensure all services are present in base module. - for service_name in service_names: - if service_name not in classes: - if service_name in ignored_service_names: - continue - raise Exception("Service %s should be present in the base module's dex." + # Ensure all providers are present in base module. We enforce this because + # providers are loaded early in startup, and keeping them in the base module + # gives more time for the chrome split to load. + for module_name, cur_manifest in manifests_by_name.items(): + if module_name == 'base': + continue + provider_names = _GetComponentNames(cur_manifest, 'provider') + if provider_names: + errors.append('Providers should all be declared in the base manifest.' + ' "%s" module declared: %s' % (module_name, provider_names)) + + # Ensure all services are present in base module because service classes are + # not found if they are not present in the base module. b/169196314 + # It is fine if they are defined in split manifests though. + for cur_manifest in manifests_by_name.values(): + for service_name in _GetComponentNames(cur_manifest, 'service'): + if service_name not in base_classes: + errors.append("Service %s should be present in the base module's dex." " See b/169196314 for more details." % service_name) - # Ensure all providers are present in base module. - for provider_name in provider_names: - if provider_name not in classes: - raise Exception( - "Provider %s should be present in the base module's dex." % - provider_name) + return errors def main(args): + build_utils.InitLogging('AAB_DEBUG') args = build_utils.ExpandFileArgs(args) options = _ParseArgs(args) @@ -473,18 +528,23 @@ with build_utils.TempDir() as tmp_dir: + logging.info('Splitting locale assets') module_zips = [ _SplitModuleForAssetTargeting(module, tmp_dir, split_dimensions) \ for module in options.module_zips] base_master_resource_ids = None if options.base_module_rtxt_path: + logging.info('Creating R.txt allowlist') base_master_resource_ids = _GenerateBaseResourcesAllowList( options.base_module_rtxt_path, options.base_allowlist_rtxt_path) - bundle_config = _GenerateBundleConfigJson( - options.uncompressed_assets, options.compress_shared_libraries, - split_dimensions, base_master_resource_ids) + logging.info('Creating BundleConfig.pb.json') + bundle_config = _GenerateBundleConfigJson(options.uncompressed_assets, + options.compress_dex, + options.compress_shared_libraries, + split_dimensions, + base_master_resource_ids) tmp_bundle = os.path.join(tmp_dir, 'tmp_bundle') @@ -495,7 +555,8 @@ with open(tmp_bundle_config, 'w') as f: f.write(bundle_config) - cmd_args = build_utils.JavaCmd(options.warnings_as_errors) + [ + logging.info('Running bundletool') + cmd_args = build_utils.JavaCmd() + [ '-jar', bundletool.BUNDLETOOL_JAR_PATH, 'build-bundle', @@ -504,6 +565,11 @@ '--config=' + tmp_bundle_config, ] + if options.rotation_config: + cmd_args += [ + f'--metadata-file={_ROTATION_METADATA_KEY}:{options.rotation_config}' + ] + build_utils.CheckOutput( cmd_args, print_stdout=True, @@ -516,8 +582,15 @@ # isolated splits disabled and 2s for bundles with isolated splits # enabled. Consider making this run in parallel or move into a separate # step before enabling isolated splits by default. - _MaybeCheckServicesAndProvidersPresentInBase(tmp_bundle, module_zips) + logging.info('Validating isolated split manifests') + errors = _ValidateSplits(tmp_bundle, module_zips) + if errors: + sys.stderr.write('Bundle failed sanity checks:\n ') + sys.stderr.write('\n '.join(errors)) + sys.stderr.write('\n') + sys.exit(1) + logging.info('Writing final output artifacts') shutil.move(tmp_bundle, options.out_bundle) if options.rtxt_out_path:
diff --git a/build/android/gyp/create_app_bundle.pydeps b/build/android/gyp/create_app_bundle.pydeps index cbb471a..5e7a79f 100644 --- a/build/android/gyp/create_app_bundle.pydeps +++ b/build/android/gyp/create_app_bundle.pydeps
@@ -13,9 +13,8 @@ ../../../third_party/catapult/devil/devil/utils/__init__.py ../../../third_party/catapult/devil/devil/utils/cmd_helper.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -35,7 +34,9 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py ../pylib/__init__.py ../pylib/constants/__init__.py ../pylib/utils/__init__.py
diff --git a/build/android/gyp/create_app_bundle_apks.py b/build/android/gyp/create_app_bundle_apks.py index 5950696..2f0dc51 100755 --- a/build/android/gyp/create_app_bundle_apks.py +++ b/build/android/gyp/create_app_bundle_apks.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -32,18 +32,21 @@ '--minimal', action='store_true', help='Create APKs archive with minimal language support.') + parser.add_argument('--local-testing', + action='store_true', + help='Create APKs archive with local testing support.') args = parser.parse_args() - app_bundle_utils.GenerateBundleApks( - args.bundle, - args.output, - args.aapt2_path, - args.keystore_path, - args.keystore_password, - args.keystore_name, - minimal=args.minimal, - check_for_noop=False) + app_bundle_utils.GenerateBundleApks(args.bundle, + args.output, + args.aapt2_path, + args.keystore_path, + args.keystore_password, + args.keystore_name, + local_testing=args.local_testing, + minimal=args.minimal, + check_for_noop=False) if __name__ == '__main__':
diff --git a/build/android/gyp/create_app_bundle_apks.pydeps b/build/android/gyp/create_app_bundle_apks.pydeps index 20d8ffe..65810c3 100644 --- a/build/android/gyp/create_app_bundle_apks.pydeps +++ b/build/android/gyp/create_app_bundle_apks.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_app_bundle_apks.pydeps build/android/gyp/create_app_bundle_apks.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,6 +22,7 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py ../../print_python_deps.py ../pylib/__init__.py
diff --git a/build/android/gyp/create_bundle_wrapper_script.py b/build/android/gyp/create_bundle_wrapper_script.py index 282e206..a3870bf 100755 --- a/build/android/gyp/create_bundle_wrapper_script.py +++ b/build/android/gyp/create_bundle_wrapper_script.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -11,9 +11,10 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. SCRIPT_TEMPLATE = string.Template("""\ -#!/usr/bin/env python +#!/usr/bin/env python3 # # This file was generated by build/android/gyp/create_bundle_wrapper_script.py @@ -109,7 +110,7 @@ 'TARGET_CPU': repr(args.target_cpu), 'SYSTEM_IMAGE_LOCALES': - repr(build_utils.ParseGnList(args.system_image_locales)), + repr(action_helpers.parse_gn_list(args.system_image_locales)), 'DEFAULT_MODULES': repr(args.default_modules), }
diff --git a/build/android/gyp/create_bundle_wrapper_script.pydeps b/build/android/gyp/create_bundle_wrapper_script.pydeps index 7758ed6..51d912c 100644 --- a/build/android/gyp/create_bundle_wrapper_script.pydeps +++ b/build/android/gyp/create_bundle_wrapper_script.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_bundle_wrapper_script.pydeps build/android/gyp/create_bundle_wrapper_script.py +../../action_helpers.py ../../gn_helpers.py create_bundle_wrapper_script.py util/__init__.py
diff --git a/build/android/gyp/create_java_binary_script.py b/build/android/gyp/create_java_binary_script.py index 5bc9d08..f9e665f 100755 --- a/build/android/gyp/create_java_binary_script.py +++ b/build/android/gyp/create_java_binary_script.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,18 +10,19 @@ jar. This includes correctly setting the classpath and the main class. """ -import optparse +import argparse import os import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. # The java command must be executed in the current directory because there may # be user-supplied paths in the args. The script receives the classpath relative # to the directory that the script is written in and then, when run, must # recalculate the paths relative to the current directory. script_template = """\ -#!/usr/bin/env python +#!/usr/bin/env python3 # # This file was generated by build/android/gyp/create_java_binary_script.py @@ -76,44 +77,60 @@ def main(argv): argv = build_utils.ExpandFileArgs(argv) - parser = optparse.OptionParser() - parser.add_option('--output', help='Output path for executable script.') - parser.add_option('--main-class', + parser = argparse.ArgumentParser() + parser.add_argument('--output', + required=True, + help='Output path for executable script.') + parser.add_argument( + '--main-class', + required=True, help='Name of the java class with the "main" entry point.') - parser.add_option('--classpath', action='append', default=[], - help='Classpath for running the jar.') - parser.add_option('--noverify', action='store_true', - help='JVM flag: noverify.') - parser.add_option('--tiered-stop-at-level-one', - action='store_true', - help='JVM flag: -XX:TieredStopAtLevel=1.') + parser.add_argument('--max-heap-size', + required=True, + help='Argument for -Xmx') + parser.add_argument('--classpath', + action='append', + default=[], + help='Classpath for running the jar.') + parser.add_argument('--tiered-stop-at-level-one', + action='store_true', + help='JVM flag: -XX:TieredStopAtLevel=1.') + parser.add_argument('--use-jdk-11', + action='store_true', + help='Use older JDK11 instead of modern JDK.') + parser.add_argument('extra_program_args', + nargs='*', + help='This captures all ' + 'args after "--" to pass as extra args to the java cmd.') - options, extra_program_args = parser.parse_args(argv) + args = parser.parse_args(argv) - extra_flags = [] - if options.noverify: - extra_flags.append('java_cmd.append("-noverify")') - if options.tiered_stop_at_level_one: + extra_flags = [f'java_cmd.append("-Xmx{args.max_heap_size}")'] + if args.tiered_stop_at_level_one: extra_flags.append('java_cmd.append("-XX:TieredStopAtLevel=1")') classpath = [] - for cp_arg in options.classpath: - classpath += build_utils.ParseGnList(cp_arg) + for cp_arg in args.classpath: + classpath += action_helpers.parse_gn_list(cp_arg) - run_dir = os.path.dirname(options.output) + run_dir = os.path.dirname(args.output) classpath = [os.path.relpath(p, run_dir) for p in classpath] - java_path = os.path.relpath( - os.path.join(build_utils.JAVA_HOME, 'bin', 'java'), run_dir) - with build_utils.AtomicOutput(options.output, mode='w') as script: + if args.use_jdk_11: + java_home = build_utils.JAVA_11_HOME_DEPRECATED + else: + java_home = build_utils.JAVA_HOME + java_path = os.path.relpath(os.path.join(java_home, 'bin', 'java'), run_dir) + + with action_helpers.atomic_output(args.output, mode='w') as script: script.write( script_template.format(classpath=('"%s"' % '", "'.join(classpath)), java_path=repr(java_path), - main_class=options.main_class, - extra_program_args=repr(extra_program_args), + main_class=args.main_class, + extra_program_args=repr(args.extra_program_args), extra_flags='\n'.join(extra_flags))) - os.chmod(options.output, 0o750) + os.chmod(args.output, 0o750) if __name__ == '__main__':
diff --git a/build/android/gyp/create_java_binary_script.pydeps b/build/android/gyp/create_java_binary_script.pydeps index 6bc21fa..a0a740d 100644 --- a/build/android/gyp/create_java_binary_script.pydeps +++ b/build/android/gyp/create_java_binary_script.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_java_binary_script.pydeps build/android/gyp/create_java_binary_script.py +../../action_helpers.py ../../gn_helpers.py create_java_binary_script.py util/__init__.py
diff --git a/build/android/gyp/create_r_java.py b/build/android/gyp/create_r_java.py index 97e512d..b662a39 100755 --- a/build/android/gyp/create_r_java.py +++ b/build/android/gyp/create_r_java.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Writes a dummy R.java file from a list of R.txt files.""" @@ -9,6 +9,8 @@ from util import build_utils from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers def _ConcatRTxts(rtxt_in_paths, combined_out_path): @@ -34,12 +36,13 @@ rjava_build_options=rjava_build_options, srcjar_out=srcjar_out, ignore_mismatched_values=True) - build_utils.ZipDir(srcjar_out, build.srcjar_dir) + with action_helpers.atomic_output(srcjar_out) as f: + zip_helpers.zip_directory(f, build.srcjar_dir) def main(args): parser = argparse.ArgumentParser(description='Create an R.java srcjar.') - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--srcjar-out', required=True, help='Path to output srcjar.') @@ -50,12 +53,12 @@ required=True, help='R.java package to use.') options = parser.parse_args(build_utils.ExpandFileArgs(args)) - options.deps_rtxts = build_utils.ParseGnList(options.deps_rtxts) + options.deps_rtxts = action_helpers.parse_gn_list(options.deps_rtxts) _CreateRJava(options.deps_rtxts, options.r_package, options.srcjar_out) - build_utils.WriteDepfile(options.depfile, - options.srcjar_out, - inputs=options.deps_rtxts) + action_helpers.write_depfile(options.depfile, + options.srcjar_out, + inputs=options.deps_rtxts) if __name__ == "__main__":
diff --git a/build/android/gyp/create_r_java.pydeps b/build/android/gyp/create_r_java.pydeps index 45121e3..20fd1f8 100644 --- a/build/android/gyp/create_r_java.pydeps +++ b/build/android/gyp/create_r_java.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_r_java.pydeps build/android/gyp/create_r_java.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,7 +22,9 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py create_r_java.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/create_r_txt.py b/build/android/gyp/create_r_txt.py index 2adde5d..429f62f 100755 --- a/build/android/gyp/create_r_txt.py +++ b/build/android/gyp/create_r_txt.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Writes a dummy R.txt file from a resource zip."""
diff --git a/build/android/gyp/create_r_txt.pydeps b/build/android/gyp/create_r_txt.pydeps index c7698ee..65378f0 100644 --- a/build/android/gyp/create_r_txt.pydeps +++ b/build/android/gyp/create_r_txt.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_r_txt.pydeps build/android/gyp/create_r_txt.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,6 +22,7 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py create_r_txt.py util/__init__.py
diff --git a/build/android/gyp/create_size_info_files.py b/build/android/gyp/create_size_info_files.py index c60b02d..24fcf8d 100755 --- a/build/android/gyp/create_size_info_files.py +++ b/build/android/gyp/create_size_info_files.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -15,6 +15,7 @@ from util import build_utils from util import jar_info_utils +import action_helpers # build_utils adds //build to sys.path. _AAR_VERSION_PATTERN = re.compile(r'/[^/]*?(\.aar/|\.jar/)') @@ -40,9 +41,7 @@ def _MergeResInfoFiles(res_info_path, info_paths): # Concatenate them all. - # only_if_changed=False since no build rules depend on this as an input. - with build_utils.AtomicOutput(res_info_path, only_if_changed=False, - mode='w+') as dst: + with action_helpers.atomic_output(res_info_path, 'w+') as dst: for p in info_paths: with open(p) as src: dst.writelines(_TransformAarPaths(l) for l in src) @@ -58,8 +57,9 @@ with open(pak_info_path, 'r') as src_info_file: info_lines.update(_TransformAarPaths(x) for x in src_info_file) # only_if_changed=False since no build rules depend on this as an input. - with build_utils.AtomicOutput(merged_path, only_if_changed=False, - mode='w+') as f: + with action_helpers.atomic_output(merged_path, + only_if_changed=False, + mode='w+') as f: f.writelines(sorted(info_lines)) @@ -121,7 +121,7 @@ attributed_path, name)) # only_if_changed=False since no build rules depend on this as an input. - with build_utils.AtomicOutput(output, only_if_changed=False) as f: + with action_helpers.atomic_output(output, only_if_changed=False) as f: jar_info_utils.WriteJarInfoFile(f, info_data) @@ -139,7 +139,7 @@ def main(args): args = build_utils.ExpandFileArgs(args) parser = argparse.ArgumentParser(description=__doc__) - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument( '--jar-info-path', required=True, help='Output .jar.info file') parser.add_argument( @@ -170,9 +170,9 @@ options = parser.parse_args(args) - options.jar_files = build_utils.ParseGnList(options.jar_files) - options.assets = build_utils.ParseGnList(options.assets) - options.uncompressed_assets = build_utils.ParseGnList( + options.jar_files = action_helpers.parse_gn_list(options.jar_files) + options.assets = action_helpers.parse_gn_list(options.assets) + options.uncompressed_assets = action_helpers.parse_gn_list( options.uncompressed_assets) jar_inputs = _FindJarInputs(_RemoveDuplicatesFromList(options.jar_files)) @@ -186,9 +186,9 @@ _MergeResInfoFiles(options.res_info_path, res_inputs) all_inputs = jar_inputs + pak_inputs + res_inputs - build_utils.WriteDepfile(options.depfile, - options.jar_info_path, - inputs=all_inputs) + action_helpers.write_depfile(options.depfile, + options.jar_info_path, + inputs=all_inputs) if __name__ == '__main__':
diff --git a/build/android/gyp/create_size_info_files.pydeps b/build/android/gyp/create_size_info_files.pydeps index 1a69c55..0dd61cb 100644 --- a/build/android/gyp/create_size_info_files.pydeps +++ b/build/android/gyp/create_size_info_files.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_size_info_files.pydeps build/android/gyp/create_size_info_files.py +../../action_helpers.py ../../gn_helpers.py create_size_info_files.py util/__init__.py
diff --git a/build/android/gyp/create_stub_manifest.py b/build/android/gyp/create_stub_manifest.py new file mode 100755 index 0000000..889fa26 --- /dev/null +++ b/build/android/gyp/create_stub_manifest.py
@@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Generates AndroidManifest.xml for a -Stub.apk.""" + +import argparse +import pathlib + +_MAIN_TEMPLATE = """\ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="will.be.replaced"> + <application android:label="APK Stub">{}</application> +</manifest> +""" + +_STATIC_LIBRARY_TEMPLATE = """ + <static-library android:name="{}" android:version="{}" /> +""" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--static-library-name') + parser.add_argument('--static-library-version') + parser.add_argument('--output', required=True) + args = parser.parse_args() + + static_library_part = '' + if args.static_library_name: + static_library_part = _STATIC_LIBRARY_TEMPLATE.format( + args.static_library_name, args.static_library_version) + + data = _MAIN_TEMPLATE.format(static_library_part) + pathlib.Path(args.output).write_text(data, encoding='utf8') + + +if __name__ == '__main__': + main()
diff --git a/build/android/gyp/create_test_apk_wrapper_script.py b/build/android/gyp/create_test_apk_wrapper_script.py new file mode 100755 index 0000000..1e63748 --- /dev/null +++ b/build/android/gyp/create_test_apk_wrapper_script.py
@@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Create a wrapper script to run a test apk using apk_operations.py.""" + +import argparse +import os +import string +import sys + +from util import build_utils + +SCRIPT_TEMPLATE = string.Template("""\ +#!/usr/bin/env python3 +# +# This file was generated by build/android/gyp/create_test_apk_wrapper_script.py + +import os +import sys + +def main(): + script_directory = os.path.dirname(__file__) + resolve = lambda p: p if p is None else os.path.abspath(os.path.join( + script_directory, p)) + sys.path.append(resolve(${WRAPPED_SCRIPT_DIR})) + import apk_operations + + additional_apk_paths = [resolve(p) for p in ${ADDITIONAL_APKS}] + apk_operations.RunForTestApk( + output_directory=resolve(${OUTPUT_DIR}), + package_name=${PACKAGE_NAME}, + test_apk_path=resolve(${TEST_APK}), + test_apk_json=resolve(${TEST_APK_JSON}), + proguard_mapping_path=resolve(${MAPPING_PATH}), + additional_apk_paths=additional_apk_paths) + +if __name__ == '__main__': + sys.exit(main()) +""") + + +def main(args): + args = build_utils.ExpandFileArgs(args) + parser = argparse.ArgumentParser() + parser.add_argument('--script-output-path', + required=True, + help='Output path for executable script.') + parser.add_argument('--package-name', required=True) + parser.add_argument('--test-apk') + parser.add_argument('--test-apk-incremental-install-json') + parser.add_argument('--proguard-mapping-path') + parser.add_argument('--additional-apk', + action='append', + dest='additional_apks', + default=[], + help='Paths to APKs to be installed prior to --apk-path.') + args = parser.parse_args(args) + + def relativize(path): + """Returns the path relative to the output script directory.""" + if path is None: + return path + return os.path.relpath(path, os.path.dirname(args.script_output_path)) + + wrapped_script_dir = os.path.join(os.path.dirname(__file__), os.path.pardir) + wrapped_script_dir = relativize(wrapped_script_dir) + with open(args.script_output_path, 'w') as script: + script_dict = { + 'WRAPPED_SCRIPT_DIR': repr(wrapped_script_dir), + 'OUTPUT_DIR': repr(relativize('.')), + 'PACKAGE_NAME': repr(args.package_name), + 'TEST_APK': repr(relativize(args.test_apk)), + 'TEST_APK_JSON': + repr(relativize(args.test_apk_incremental_install_json)), + 'MAPPING_PATH': repr(relativize(args.proguard_mapping_path)), + 'ADDITIONAL_APKS': [relativize(p) for p in args.additional_apks], + } + script.write(SCRIPT_TEMPLATE.substitute(script_dict)) + os.chmod(args.script_output_path, 0o750) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))
diff --git a/build/android/gyp/create_test_apk_wrapper_script.pydeps b/build/android/gyp/create_test_apk_wrapper_script.pydeps new file mode 100644 index 0000000..d52f343 --- /dev/null +++ b/build/android/gyp/create_test_apk_wrapper_script.pydeps
@@ -0,0 +1,6 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_test_apk_wrapper_script.pydeps build/android/gyp/create_test_apk_wrapper_script.py +../../gn_helpers.py +create_test_apk_wrapper_script.py +util/__init__.py +util/build_utils.py
diff --git a/build/android/gyp/create_ui_locale_resources.py b/build/android/gyp/create_ui_locale_resources.py index 772dab7..c767bc5 100755 --- a/build/android/gyp/create_ui_locale_resources.py +++ b/build/android/gyp/create_ui_locale_resources.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -27,6 +27,9 @@ from util import build_utils from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers + # A small string template for the content of each strings.xml file. # NOTE: The name is chosen to avoid any conflicts with other string defined @@ -52,8 +55,10 @@ zip_path = 'values-%s/strings.xml' % android_locale else: zip_path = 'values/strings.xml' - build_utils.AddToZipHermetic( - out_zip, zip_path, data=locale_data, compress=False) + zip_helpers.add_to_zip_hermetic(out_zip, + zip_path, + data=locale_data, + compress=False) def main(): @@ -69,11 +74,11 @@ args = parser.parse_args() - locale_list = build_utils.ParseGnList(args.locale_list) + locale_list = action_helpers.parse_gn_list(args.locale_list) if not locale_list: raise Exception('Locale list cannot be empty!') - with build_utils.AtomicOutput(args.output_zip) as tmp_file: + with action_helpers.atomic_output(args.output_zip) as tmp_file: with zipfile.ZipFile(tmp_file, 'w') as out_zip: # First, write the default value, since aapt requires one. _AddLocaleResourceFileToZip(out_zip, '', _DEFAULT_CHROME_LOCALE)
diff --git a/build/android/gyp/create_ui_locale_resources.pydeps b/build/android/gyp/create_ui_locale_resources.pydeps index 6bb98dd..5cffc79 100644 --- a/build/android/gyp/create_ui_locale_resources.pydeps +++ b/build/android/gyp/create_ui_locale_resources.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/create_ui_locale_resources.pydeps build/android/gyp/create_ui_locale_resources.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,7 +22,9 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py create_ui_locale_resources.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/create_unwind_table.py b/build/android/gyp/create_unwind_table.py new file mode 100755 index 0000000..83cd73d --- /dev/null +++ b/build/android/gyp/create_unwind_table.py
@@ -0,0 +1,1095 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Creates a table of unwind information in Android Chrome's bespoke format.""" + +import abc +import argparse +import collections +import enum +import json +import logging +import re +import struct +import subprocess +import sys +from typing import (Dict, Iterable, List, NamedTuple, Sequence, TextIO, Tuple, + Union) + +from util import build_utils + +_STACK_CFI_INIT_REGEX = re.compile( + r'^STACK CFI INIT ([0-9a-f]+) ([0-9a-f]+) (.+)$') +_STACK_CFI_REGEX = re.compile(r'^STACK CFI ([0-9a-f]+) (.+)$') + + +class AddressCfi(NamedTuple): + """Record representing CFI for an address within a function. + + Represents the Call Frame Information required to unwind from an address in a + function. + + Attributes: + address: The address. + unwind_instructions: The unwind instructions for the address. + + """ + address: int + unwind_instructions: str + + +class FunctionCfi(NamedTuple): + """Record representing CFI for a function. + + Note: address_cfi[0].address is the start address of the function. + + Attributes: + size: The function size in bytes. + address_cfi: The CFI at each address in the function. + + """ + size: int + address_cfi: Tuple[AddressCfi, ...] + + +def FilterToNonTombstoneCfi(stream: TextIO) -> Iterable[str]: + """Generates non-tombstone STACK CFI lines from the stream. + + STACK CFI functions with address 0 correspond are a 'tombstone' record + associated with dead code and can be ignored. See + https://bugs.llvm.org/show_bug.cgi?id=47148#c2. + + Args: + stream: A file object. + + Returns: + An iterable over the non-tombstone STACK CFI lines in the stream. + """ + in_tombstone_function = False + for line in stream: + if not line.startswith('STACK CFI '): + continue + + if line.startswith('STACK CFI INIT 0 '): + in_tombstone_function = True + elif line.startswith('STACK CFI INIT '): + in_tombstone_function = False + + if not in_tombstone_function: + yield line + + +def ReadFunctionCfi(stream: TextIO) -> Iterable[FunctionCfi]: + """Generates FunctionCfi records from the stream. + + Args: + stream: A file object. + + Returns: + An iterable over FunctionCfi corresponding to the non-tombstone STACK CFI + lines in the stream. + """ + current_function_address = None + current_function_size = None + current_function_address_cfi = [] + for line in FilterToNonTombstoneCfi(stream): + cfi_init_match = _STACK_CFI_INIT_REGEX.search(line) + if cfi_init_match: + # Function CFI with address 0 are tombstone entries per + # https://bugs.llvm.org/show_bug.cgi?id=47148#c2 and should have been + # filtered in `FilterToNonTombstoneCfi`. + assert current_function_address != 0 + if (current_function_address is not None + and current_function_size is not None): + yield FunctionCfi(current_function_size, + tuple(current_function_address_cfi)) + current_function_address = int(cfi_init_match.group(1), 16) + current_function_size = int(cfi_init_match.group(2), 16) + current_function_address_cfi = [ + AddressCfi(int(cfi_init_match.group(1), 16), cfi_init_match.group(3)) + ] + else: + cfi_match = _STACK_CFI_REGEX.search(line) + assert cfi_match + current_function_address_cfi.append( + AddressCfi(int(cfi_match.group(1), 16), cfi_match.group(2))) + + assert current_function_address is not None + assert current_function_size is not None + yield FunctionCfi(current_function_size, tuple(current_function_address_cfi)) + + +def EncodeAsBytes(*values: int) -> bytes: + """Encodes the argument ints as bytes. + + This function validates that the inputs are within the range that can be + represented as bytes. + + Args: + values: Integers in range [0, 255]. + + Returns: + The values encoded as bytes. + """ + for i, value in enumerate(values): + if not 0 <= value <= 255: + raise ValueError('value = %d out of bounds at byte %d' % (value, i)) + return bytes(values) + + +def Uleb128Encode(value: int) -> bytes: + """Encodes the argument int to ULEB128 format. + + Args: + value: Unsigned integer. + + Returns: + The values encoded as ULEB128 bytes. + """ + if value < 0: + raise ValueError(f'Cannot uleb128 encode negative value ({value}).') + + uleb128_bytes = [] + done = False + while not done: + value, lowest_seven_bits = divmod(value, 0x80) + done = value == 0 + uleb128_bytes.append(lowest_seven_bits | (0x80 if not done else 0x00)) + return EncodeAsBytes(*uleb128_bytes) + + +def EncodeStackPointerUpdate(offset: int) -> bytes: + """Encodes a stack pointer update as arm unwind instructions. + + Args: + offset: Offset to apply on stack pointer. Should be in range [-0x204, inf). + + Returns: + A list of arm unwind instructions as bytes. + """ + assert offset % 4 == 0 + + abs_offset = abs(offset) + instruction_code = 0b01000000 if offset < 0 else 0b00000000 + if 0x04 <= abs_offset <= 0x200: + instructions = [ + # vsp = vsp + (xxxxxx << 2) + 4. Covers range 0x04-0x100 inclusive. + instruction_code | ((min(abs_offset, 0x100) - 4) >> 2) + ] + # For vsp increments of 0x104-0x200 we use 00xxxxxx twice. + if abs_offset >= 0x104: + instructions.append(instruction_code | ((abs_offset - 0x100 - 4) >> 2)) + try: + return EncodeAsBytes(*instructions) + except ValueError as e: + raise RuntimeError('offset = %d produced out of range value' % + offset) from e + else: + # This only encodes positive sp movement. + assert offset > 0, offset + return EncodeAsBytes(0b10110010 # vsp = vsp + 0x204 + (uleb128 << 2) + ) + Uleb128Encode((offset - 0x204) >> 2) + + +def EncodePop(registers: Sequence[int]) -> bytes: + """Encodes popping of a sequence of registers as arm unwind instructions. + + Args: + registers: Collection of target registers to accept values popped from + stack. Register value order in the sequence does not matter. Values are + popped based on register index order. + + Returns: + A list of arm unwind instructions as bytes. + """ + assert all( + r in range(4, 16) + for r in registers), f'Can only pop r4 ~ r15. Registers:\n{registers}.' + assert len(registers) > 0, 'Register sequence cannot be empty.' + + instructions: List[int] = [] + + # Check if the pushed registers are continuous set starting from r4 (and + # ending prior to r12). This scenario has its own encoding. + pop_lr = 14 in registers + non_lr_registers = [r for r in registers if r != 14] + non_lr_registers_continuous_from_r4 = \ + sorted(non_lr_registers) == list(range(4, 4 + len(non_lr_registers))) + + if (pop_lr and 0 < len(non_lr_registers) <= 8 + and non_lr_registers_continuous_from_r4): + instructions.append(0b10101000 + | (len(non_lr_registers) - 1) # Pop r4-r[4+nnn], r14. + ) + else: + register_bits = 0 + for register in registers: + register_bits |= 1 << register + register_bits = register_bits >> 4 # Skip r0 ~ r3. + instructions.extend([ + # Pop up to 12 integer registers under masks {r15-r12}, {r11-r4}. + 0b10000000 | (register_bits >> 8), + register_bits & 0xff + ]) + + return EncodeAsBytes(*instructions) + + +class UnwindType(enum.Enum): + """ + The type of unwind action to perform. + """ + + # Use lr as the return address. + RETURN_TO_LR = 1 + + # Increment or decrement the stack pointer and/or pop registers (r4 ~ r15). + # If both, the increment/decrement occurs first. + UPDATE_SP_AND_OR_POP_REGISTERS = 2 + + # Restore the stack pointer from a register then increment/decrement the stack + # pointer. + RESTORE_SP_FROM_REGISTER = 3 + + # No action necessary. Used for floating point register pops. + NO_ACTION = 4 + + +class AddressUnwind(NamedTuple): + """Record representing unwind information for an address within a function. + + Attributes: + address_offset: The offset of the address from the start of the function. + unwind_type: The type of unwind to perform from the address. + sp_offset: The offset to apply to the stack pointer. + registers: The registers involved in the unwind. + """ + address_offset: int + unwind_type: UnwindType + sp_offset: int + registers: Tuple[int, ...] + + +class FunctionUnwind(NamedTuple): + """Record representing unwind information for a function. + + Attributes: + address: The address of the function. + size: The function size in bytes. + address_unwind_info: The unwind info at each address in the function. + """ + + address: int + size: int + address_unwinds: Tuple[AddressUnwind, ...] + + +def EncodeAddressUnwind(address_unwind: AddressUnwind) -> bytes: + """Encodes an `AddressUnwind` object as arm unwind instructions. + + Args: + address_unwind: Record representing unwind information for an address within + a function. + + Returns: + A list of arm unwind instructions as bytes. + """ + if address_unwind.unwind_type == UnwindType.RETURN_TO_LR: + return EncodeAsBytes(0b10110000) # Finish. + if address_unwind.unwind_type == UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS: + return ((EncodeStackPointerUpdate(address_unwind.sp_offset) + if address_unwind.sp_offset else b'') + + (EncodePop(address_unwind.registers) + if address_unwind.registers else b'')) + + if address_unwind.unwind_type == UnwindType.RESTORE_SP_FROM_REGISTER: + assert len(address_unwind.registers) == 1 + return (EncodeAsBytes(0b10010000 + | address_unwind.registers[0] # Set vsp = r[nnnn]. + ) + + (EncodeStackPointerUpdate(address_unwind.sp_offset) + if address_unwind.sp_offset else b'')) + + if address_unwind.unwind_type == UnwindType.NO_ACTION: + return b'' + + assert False, 'unknown unwind type' + return b'' + + +class UnwindInstructionsParser(abc.ABC): + """Base class for parsers of breakpad unwind instruction sequences. + + Provides regexes matching breakpad instruction sequences understood by the + parser, and parsing of the sequences from the regex match. + """ + + @abc.abstractmethod + def GetBreakpadInstructionsRegex(self) -> re.Pattern: + pass + + @abc.abstractmethod + def ParseFromMatch(self, address_offset: int, cfa_sp_offset: int, + match: re.Match) -> Tuple[AddressUnwind, int]: + """Returns the regex matching the breakpad instructions. + + Args: + address_offset: Offset from function start address. + cfa_sp_offset: CFA stack pointer offset. + + Returns: + The unwind info for the address plus the new cfa_sp_offset. + """ + + +class NullParser(UnwindInstructionsParser): + """Translates the state before any instruction has been executed.""" + + regex = re.compile(r'^\.cfa: sp 0 \+ \.ra: lr$') + + def GetBreakpadInstructionsRegex(self) -> re.Pattern: + return self.regex + + def ParseFromMatch(self, address_offset: int, cfa_sp_offset: int, + match: re.Match) -> Tuple[AddressUnwind, int]: + return AddressUnwind(address_offset, UnwindType.RETURN_TO_LR, 0, ()), 0 + + +class PushOrSubSpParser(UnwindInstructionsParser): + """Translates unwinds from push or sub sp, #constant instructions.""" + + # We expect at least one of the three outer groups to be non-empty. Cases: + # + # Standard prologue pushes. + # Match the first two and optionally the third. + # + # Standard prologue sub sp, #constant. + # Match only the first. + # + # Pushes in dynamic stack allocation functions after saving sp. + # Match only the third since they don't alter the stack pointer or store the + # return address. + # + # Leaf functions that use callee-save registers. + # Match the first and third but not the second. + regex = re.compile(r'^(?:\.cfa: sp (\d+) \+ ?)?' + r'(?:\.ra: \.cfa (-\d+) \+ \^ ?)?' + r'((?:r\d+: \.cfa -\d+ \+ \^ ?)*)$') + + # 'r' followed by digits, with 'r' matched via positive lookbehind so only the + # number appears in the match. + register_regex = re.compile('(?<=r)(\d+)') + + def GetBreakpadInstructionsRegex(self) -> re.Pattern: + return self.regex + + def ParseFromMatch(self, address_offset: int, cfa_sp_offset: int, + match: re.Match) -> Tuple[AddressUnwind, int]: + # The group will be None if the outer non-capturing groups for the(\d+) and + # (-\d+) expressions are not matched. + new_cfa_sp_offset, ra_cfa_offset = (int(group) if group else None + for group in match.groups()[:2]) + + # Registers are pushed in reverse order by register number so are popped in + # order. Sort them to ensure the proper order. + registers = sorted([ + int(register) + for register in self.register_regex.findall(match.group(3)) + # `UpdateSpAndOrPopRegisters` only supports popping of register + # r4 ~ r15. The ignored registers are translated to sp increments by + # the following calculation on `sp_offset`. + if int(register) in range(4, 16) + ] + + # Also pop lr (ra in breakpad terms) if it was stored. + ([14] if ra_cfa_offset is not None else [])) + + sp_offset = 0 + if new_cfa_sp_offset is not None: + sp_offset = new_cfa_sp_offset - cfa_sp_offset + assert sp_offset % 4 == 0 + if sp_offset >= len(registers) * 4: + # Handles the sub sp, #constant case, and push instructions that push + # caller-save registers r0-r3 which don't get encoded in the unwind + # instructions. In the latter case we need to move the stack pointer up + # to the first pushed register. + sp_offset -= len(registers) * 4 + + return AddressUnwind(address_offset, + UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, sp_offset, + tuple(registers)), new_cfa_sp_offset or cfa_sp_offset + + +class VPushParser(UnwindInstructionsParser): + # VPushes that occur in dynamic stack allocation functions after storing the + # stack pointer don't change the stack pointer or push any register that we + # care about. The first group will not match in those cases. + # + # Breakpad doesn't seem to understand how to name the floating point + # registers so calls them unnamed_register. + regex = re.compile(r'^(?:\.cfa: sp (\d+) \+ )?' + r'(?:unnamed_register\d+: \.cfa -\d+ \+ \^ ?)+$') + + def GetBreakpadInstructionsRegex(self) -> re.Pattern: + return self.regex + + def ParseFromMatch(self, address_offset: int, cfa_sp_offset: int, + match: re.Match) -> Tuple[AddressUnwind, int]: + # `match.group(1)`, which corresponds to the (\d+) expression, will be None + # if the first outer non-capturing group is not matched. + new_cfa_sp_offset = int(match.group(1)) if match.group(1) else None + if new_cfa_sp_offset is None: + return (AddressUnwind(address_offset, UnwindType.NO_ACTION, 0, + ()), cfa_sp_offset) + + sp_offset = new_cfa_sp_offset - cfa_sp_offset + assert sp_offset % 4 == 0 + return AddressUnwind(address_offset, + UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, sp_offset, + ()), new_cfa_sp_offset + + +class StoreSpParser(UnwindInstructionsParser): + regex = re.compile(r'^\.cfa: r(\d+) (\d+) \+$') + + def GetBreakpadInstructionsRegex(self) -> re.Pattern: + return self.regex + + def ParseFromMatch(self, address_offset: int, cfa_sp_offset: int, + match: re.Match) -> Tuple[AddressUnwind, int]: + register = int(match.group(1)) + new_cfa_sp_offset = int(match.group(2)) + sp_offset = new_cfa_sp_offset - cfa_sp_offset + assert sp_offset % 4 == 0 + return AddressUnwind(address_offset, UnwindType.RESTORE_SP_FROM_REGISTER, + sp_offset, (register, )), new_cfa_sp_offset + + +def EncodeUnwindInstructionTable(complete_instruction_sequences: Iterable[bytes] + ) -> Tuple[bytes, Dict[bytes, int]]: + """Encodes the unwind instruction table. + + Deduplicates the encoded unwind instruction sequences. Generates the table and + a dictionary mapping a function to its starting index in the table. + + The instruction table is used by the unwinder to provide the sequence of + unwind instructions to execute for each function, separated by offset + into the function. + + Args: + complete_instruction_sequences: An iterable of encoded unwind instruction + sequences. The sequences represent the series of unwind instructions to + execute corresponding to offsets within each function. + + Returns: + A tuple containing: + - The unwind instruction table as bytes. + - The mapping from the instruction sequence to the offset in the unwind + instruction table. This mapping is used to construct the function offset + table, which references entries in the unwind instruction table. + """ + # As the function offset table uses variable length number encoding (uleb128), + # which means smaller number uses fewer bytes to represent, we should sort + # the unwind instruction table by number of references from the function + # offset table in order to minimize the size of the function offset table. + ref_counts: Dict[bytes, int] = collections.defaultdict(int) + for sequence in complete_instruction_sequences: + ref_counts[sequence] += 1 + + def ComputeScore(sequence): + """ Score for each sequence is computed as ref_count / size_of_sequence. + + According to greedy algorithm, items with higher value / space cost ratio + should be prioritized. Here value is bytes saved in the function offset + table, represetned by ref_count. Space cost is the space taken in the + unwind instruction table, represented by size_of_sequence. + + Note: In order to ensure build-time determinism, `sequence` is also returned + to resolve sorting order when scores are the same. + """ + return ref_counts[sequence] / len(sequence), sequence + + ordered_sequences = sorted(ref_counts.keys(), key=ComputeScore, reverse=True) + offsets: Dict[bytes, int] = {} + current_offset = 0 + for sequence in ordered_sequences: + offsets[sequence] = current_offset + current_offset += len(sequence) + return b''.join(ordered_sequences), offsets + + +class EncodedAddressUnwind(NamedTuple): + """Record representing unwind information for an address within a function. + + This structure represents the same concept as `AddressUnwind`. The only + difference is that how to unwind from the address is represented as + encoded ARM unwind instructions. + + Attributes: + address_offset: The offset of the address from the start address of the + function. + complete_instruction_sequence: The full ARM unwind instruction sequence to + unwind from the `address_offset`. + """ + address_offset: int + complete_instruction_sequence: bytes + + +def EncodeAddressUnwinds(address_unwinds: Tuple[AddressUnwind, ...] + ) -> Tuple[EncodedAddressUnwind, ...]: + """Encodes the unwind instructions and offset for the addresses within a + function. + + Args: + address_unwinds: A tuple of unwind state for addresses within a function. + + Returns: + The encoded unwind instructions and offsets for the addresses within a + function, ordered by decreasing offset. + """ + sorted_address_unwinds: List[AddressUnwind] = sorted( + address_unwinds, + key=lambda address_unwind: address_unwind.address_offset, + reverse=True) + unwind_instructions: List[bytes] = [ + EncodeAddressUnwind(address_unwind) + for address_unwind in sorted_address_unwinds + ] + + # A complete instruction sequence contains all the unwind instructions + # necessary to unwind from an offset within a function. For a given offset + # this includes the offset's instructions plus the instructions for all + # earlier offsets. The offsets are stored in reverse order, hence the i: + # range rather than :i+1. + complete_instruction_sequences = [ + b''.join(unwind_instructions[i:]) for i in range(len(unwind_instructions)) + ] + + encoded_unwinds: List[EncodedAddressUnwind] = [] + for address_unwind, sequence in zip(sorted_address_unwinds, + complete_instruction_sequences): + encoded_unwinds.append( + EncodedAddressUnwind(address_unwind.address_offset, sequence)) + return tuple(encoded_unwinds) + + +class EncodedFunctionUnwind(NamedTuple): + """Record representing unwind information for a function. + + This structure represents the same concept as `FunctionUnwind`, but with + some differences: + - Attribute `address` is split into 2 attributes: `page_number` and + `page_offset`. + - Attribute `size` is dropped. + - Attribute `address_unwinds` becomes a collection of `EncodedAddressUnwind`s, + instead of a collection of `AddressUnwind`s. + + Attributes: + page_number: The upper bits (17 ~ 31bits) of byte offset from text section + start. + page_offset: The lower bits (1 ~ 16bits) of instruction offset from text + section start. + address_unwinds: A collection of `EncodedAddressUnwind`s. + + """ + + page_number: int + page_offset: int + address_unwinds: Tuple[EncodedAddressUnwind, ...] + + +# The trivial unwind is defined as a single `RETURN_TO_LR` instruction +# at the start of the function. +TRIVIAL_UNWIND: Tuple[EncodedAddressUnwind, ...] = EncodeAddressUnwinds( + (AddressUnwind(address_offset=0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=()), )) + +# The refuse to unwind filler unwind is used to fill the invalid space +# before the first function in the first page and after the last function +# in the last page. +REFUSE_TO_UNWIND: Tuple[EncodedAddressUnwind, ...] = (EncodedAddressUnwind( + address_offset=0, + complete_instruction_sequence=bytes([0b10000000, 0b00000000])), ) + + +def EncodeFunctionUnwinds(function_unwinds: Iterable[FunctionUnwind], + text_section_start_address: int + ) -> Iterable[EncodedFunctionUnwind]: + """Encodes the unwind state for all functions defined in the binary. + + This function + - sorts the collection of `FunctionUnwind`s by address. + - fills in gaps between functions with trivial unwind. + - fills the space in the last page after last function with refuse to unwind. + - fills the space in the first page before the first function with refuse + to unwind. + + Args: + function_unwinds: An iterable of function unwind states. + text_section_start_address: The address of .text section in ELF file. + + Returns: + The encoded function unwind states with no gaps between functions, ordered + by ascending address. + """ + + def GetPageNumber(address: int) -> int: + """Calculates the page number. + + Page number is calculated as byte_offset_from_text_section_start >> 17, + i.e. the upper bits (17 ~ 31bits) of byte offset from text section start. + """ + return (address - text_section_start_address) >> 17 + + def GetPageOffset(address: int) -> int: + """Calculates the page offset. + + Page offset is calculated as (byte_offset_from_text_section_start >> 1) + & 0xffff, i.e. the lower bits (1 ~ 16bits) of instruction offset from + text section start. + """ + return ((address - text_section_start_address) >> 1) & 0xffff + + sorted_function_unwinds: List[FunctionUnwind] = sorted( + function_unwinds, key=lambda function_unwind: function_unwind.address) + + if sorted_function_unwinds[0].address > text_section_start_address: + yield EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=REFUSE_TO_UNWIND) + + prev_func_end_address: int = sorted_function_unwinds[0].address + + gaps = 0 + for unwind in sorted_function_unwinds: + assert prev_func_end_address <= unwind.address, ( + 'Detected overlap between functions.') + + if prev_func_end_address < unwind.address: + # Gaps between functions are typically filled by regions of thunks which + # do not alter the stack pointer. Filling these gaps with TRIVIAL_UNWIND + # is the appropriate unwind strategy. + gaps += 1 + yield EncodedFunctionUnwind(GetPageNumber(prev_func_end_address), + GetPageOffset(prev_func_end_address), + TRIVIAL_UNWIND) + + yield EncodedFunctionUnwind(GetPageNumber(unwind.address), + GetPageOffset(unwind.address), + EncodeAddressUnwinds(unwind.address_unwinds)) + + prev_func_end_address = unwind.address + unwind.size + + if GetPageOffset(prev_func_end_address) != 0: + yield EncodedFunctionUnwind(GetPageNumber(prev_func_end_address), + GetPageOffset(prev_func_end_address), + REFUSE_TO_UNWIND) + + logging.info('%d/%d gaps between functions filled with trivial unwind.', gaps, + len(sorted_function_unwinds)) + + +def EncodeFunctionOffsetTable( + encoded_address_unwind_sequences: Iterable[ + Tuple[EncodedAddressUnwind, ...]], + unwind_instruction_table_offsets: Dict[bytes, int] +) -> Tuple[bytes, Dict[Tuple[EncodedAddressUnwind, ...], int]]: + """Encodes the function offset table. + + The function offset table maps local instruction offset from function + start to the location in the unwind instruction table. + + Args: + encoded_address_unwind_sequences: An iterable of encoded address unwind + sequences. + unwind_instruction_table_offsets: The offset mapping returned from + `EncodeUnwindInstructionTable`. + + Returns: + A tuple containing: + - The function offset table as bytes. + - The mapping from the `EncodedAddressUnwind`s to the offset in the function + offset table. This mapping is used to construct the function table, which + references entries in the function offset table. + """ + function_offset_table = bytearray() + offsets: Dict[Tuple[EncodedAddressUnwind, ...], int] = {} + + for sequence in encoded_address_unwind_sequences: + if sequence in offsets: + continue + + offsets[sequence] = len(function_offset_table) + for address_offset, complete_instruction_sequence in sequence: + # Note: address_offset is the number of bytes from one address to another, + # while the instruction_offset is the number of 2-byte instructions + # from one address to another. + instruction_offset = address_offset >> 1 + function_offset_table += ( + Uleb128Encode(instruction_offset) + Uleb128Encode( + unwind_instruction_table_offsets[complete_instruction_sequence])) + + return bytes(function_offset_table), offsets + + +def EncodePageTableAndFunctionTable( + function_unwinds: Iterable[EncodedFunctionUnwind], + function_offset_table_offsets: Dict[Tuple[EncodedAddressUnwind, ...], int] +) -> Tuple[bytes, bytes]: + """Encode page table and function table as bytes. + + Page table: + A table that contains the mapping from page_number to the location of the + entry for the first function on the page in the function table. + + Function table: + A table that contains the mapping from page_offset to the location of an entry + in the function offset table. + + Args: + function_unwinds: All encoded function unwinds in the module. + function_offset_table_offsets: The offset mapping returned from + `EncodeFunctionOffsetTable`. + + Returns: + A tuple containing: + - The page table as bytes. + - The function table as bytes. + """ + page_function_unwinds: Dict[ + int, List[EncodedFunctionUnwind]] = collections.defaultdict(list) + for function_unwind in function_unwinds: + page_function_unwinds[function_unwind.page_number].append(function_unwind) + + raw_page_table: List[int] = [] + function_table = bytearray() + + for page_number, same_page_function_unwinds in sorted( + page_function_unwinds.items(), key=lambda item: item[0]): + # Pad empty pages. + # Empty pages can occur when a function spans over multiple pages. + # Example: + # A page table with a starting function that spans 3 over pages. + # page_table: + # [0, 1, 1, 1] + # function_table: + # [ + # # Page 0 + # (0, 20) # This function spans from page 0 offset 0 to page 3 offset 5. + # # Page 1 is empty. + # # Page 2 is empty. + # # Page 3 + # (6, 70) + # ] + assert page_number > len(raw_page_table) - 1 + number_of_empty_pages = page_number - len(raw_page_table) + # The function table is represented as `base::FunctionTableEntry[]`, + # where `base::FunctionTableEntry` is 4 bytes. + function_table_index = len(function_table) // 4 + raw_page_table.extend([function_table_index] * (number_of_empty_pages + 1)) + assert page_number == len(raw_page_table) - 1 + + for function_unwind in sorted( + same_page_function_unwinds, + key=lambda function_unwind: function_unwind.page_offset): + function_table += struct.pack( + 'HH', function_unwind.page_offset, + function_offset_table_offsets[function_unwind.address_unwinds]) + + page_table = struct.pack(f'{len(raw_page_table)}I', *raw_page_table) + + return page_table, bytes(function_table) + + +ALL_PARSERS: Tuple[UnwindInstructionsParser, ...] = ( + NullParser(), + PushOrSubSpParser(), + StoreSpParser(), + VPushParser(), +) + + +def ParseAddressCfi(address_cfi: AddressCfi, function_start_address: int, + parsers: Tuple[UnwindInstructionsParser, ...], + prev_cfa_sp_offset: int + ) -> Tuple[Union[AddressUnwind, None], bool, int]: + """Parses address CFI with given parsers. + + Args: + address_cfi: The CFI for an address in the function. + function_start_address: The start address of the function. + parsers: Available parsers to try on CFI data. + prev_cfa_sp_offset: Previous CFA stack pointer offset. + + Returns: + A tuple containing: + - An `AddressUnwind` object when the parse is successful, None otherwise. + - Whether the address is in function epilogue. + - The new cfa_sp_offset. + """ + for parser in parsers: + match = parser.GetBreakpadInstructionsRegex().search( + address_cfi.unwind_instructions) + if not match: + continue + + address_unwind, cfa_sp_offset = parser.ParseFromMatch( + address_cfi.address - function_start_address, prev_cfa_sp_offset, match) + + in_epilogue = ( + prev_cfa_sp_offset > cfa_sp_offset + and address_unwind.unwind_type != UnwindType.RESTORE_SP_FROM_REGISTER) + + return (address_unwind if not in_epilogue else None, in_epilogue, + cfa_sp_offset) + + return None, False, prev_cfa_sp_offset + + +def GenerateUnwinds(function_cfis: Iterable[FunctionCfi], + parsers: Tuple[UnwindInstructionsParser, ...] + ) -> Iterable[FunctionUnwind]: + """Generates parsed function unwind states from breakpad CFI data. + + This function parses `FunctionCfi`s to `FunctionUnwind`s using + `UnwindInstructionParser`. + + Args: + function_cfis: An iterable of function CFI data. + parsers: Available parsers to try on CFI address data. + + Returns: + An iterable of parsed function unwind states. + """ + functions = 0 + addresses = 0 + handled_addresses = 0 + epilogues_seen = 0 + + for function_cfi in function_cfis: + functions += 1 + address_unwinds: List[AddressUnwind] = [] + cfa_sp_offset = 0 + for address_cfi in function_cfi.address_cfi: + addresses += 1 + + address_unwind, in_epilogue, cfa_sp_offset = ParseAddressCfi( + address_cfi, function_cfi.address_cfi[0].address, parsers, + cfa_sp_offset) + + if address_unwind: + handled_addresses += 1 + address_unwinds.append(address_unwind) + continue + + if in_epilogue: + epilogues_seen += 1 + break + + logging.info('unrecognized CFI: %x %s.', address_cfi.address, + address_cfi.unwind_instructions) + + if address_unwinds: + # We expect that the unwind information for every function starts with a + # trivial unwind (RETURN_TO_LR) prior to the execution of any code in the + # function. This is required by the arm calling convention which involves + # setting lr to the return address on calling into a function. + assert address_unwinds[0].address_offset == 0 + assert address_unwinds[0].unwind_type == UnwindType.RETURN_TO_LR + + yield FunctionUnwind(function_cfi.address_cfi[0].address, + function_cfi.size, tuple(address_unwinds)) + + logging.info('%d functions.', functions) + logging.info('%d/%d addresses handled.', handled_addresses, addresses) + logging.info('epilogues_seen: %d.', epilogues_seen) + + +def EncodeUnwindInfo(page_table: bytes, function_table: bytes, + function_offset_table: bytes, + unwind_instruction_table: bytes) -> bytes: + """Encodes all unwind tables as a single binary. + + Concats all unwind table binaries together and attach a header at the start + with a offset-size pair for each table. + + offset: The offset to the target table from the start of the unwind info + binary in bytes. + size: The declared size of the target table. + + Both offset and size are represented as 32bit integers. + See `base::ChromeUnwindInfoHeaderAndroid` for more details. + + Args: + page_table: The page table as bytes. + function_table: The function table as bytes. + function_offset_table: The function offset table as bytes. + unwind_instruction_table: The unwind instruction table as bytes. + + Returns: + A single binary containing + - A header that points to the location of each table. + - All unwind tables. + """ + unwind_info_header = bytearray() + # Each table is represented as (offset, size) pair, both offset and size + # are represented as 4 byte integer. + unwind_info_header_size = 4 * 2 * 4 + unwind_info_body = bytearray() + + # Both the page_table and the function table need to be aligned because their + # contents are interpreted as multi-byte integers. However, the byte size of + # the header, the page table, the function table are all multiples of 4 and + # the resource will be memory mapped at a 4 byte boundary, so no extra care + # is required to align the page table and the function table. + # + # The function offset table and the unwind instruction table are accessed + # byte by byte, so they only need 1 byte alignment. + + assert len(page_table) % 4 == 0, ( + 'Each entry in the page table should be 4-byte integer.') + assert len(function_table) % 4 == 0, ( + 'Each entry in the function table should be a pair of 2 2-byte integers.') + + for table in page_table, function_table: + offset = unwind_info_header_size + len(unwind_info_body) + # For the page table and the function_table, declared size is the number of + # entries in each table. The tables will be aligned to a 4 byte boundary + # because the resource will be memory mapped at a 4 byte boundary and the + # header is a multiple of 4 bytes. + declared_size = len(table) // 4 + unwind_info_header += struct.pack('II', offset, declared_size) + unwind_info_body += table + + for table in function_offset_table, unwind_instruction_table: + offset = unwind_info_header_size + len(unwind_info_body) + # Because both the function offset table and the unwind instruction table + # contain variable length encoded numbers, the declared size is simply the + # number of bytes in each table. The tables only require 1 byte alignment. + declared_size = len(table) + unwind_info_header += struct.pack('II', offset, declared_size) + unwind_info_body += table + + return bytes(unwind_info_header + unwind_info_body) + + +def GenerateUnwindTables( + encoded_function_unwinds_iterable: Iterable[EncodedFunctionUnwind] +) -> Tuple[bytes, bytes, bytes, bytes]: + """Generates all unwind tables as bytes. + + Args: + encoded_function_unwinds_iterable: Encoded function unwinds for all + functions in the ELF binary. + + Returns: + A tuple containing: + - The page table as bytes. + - The function table as bytes. + - The function offset table as bytes. + - The unwind instruction table as bytes. + """ + encoded_function_unwinds: List[EncodedFunctionUnwind] = list( + encoded_function_unwinds_iterable) + complete_instruction_sequences: List[bytes] = [] + encoded_address_unwind_sequences: List[Tuple[EncodedAddressUnwind, ...]] = [] + + for encoded_function_unwind in encoded_function_unwinds: + encoded_address_unwind_sequences.append( + encoded_function_unwind.address_unwinds) + for address_unwind in encoded_function_unwind.address_unwinds: + complete_instruction_sequences.append( + address_unwind.complete_instruction_sequence) + + unwind_instruction_table, unwind_instruction_table_offsets = ( + EncodeUnwindInstructionTable(complete_instruction_sequences)) + + function_offset_table, function_offset_table_offsets = ( + EncodeFunctionOffsetTable(encoded_address_unwind_sequences, + unwind_instruction_table_offsets)) + + page_table, function_table = EncodePageTableAndFunctionTable( + encoded_function_unwinds, function_offset_table_offsets) + + return (page_table, function_table, function_offset_table, + unwind_instruction_table) + + +def ReadTextSectionStartAddress(readobj_path: str, libchrome_path: str) -> int: + """Reads the .text section start address of libchrome ELF. + + Arguments: + readobj_path: Path to llvm-obj binary. + libchrome_path: Path to libchrome binary. + + Returns: + The text section start address as a number. + """ + def GetSectionName(section) -> str: + # See crbug.com/1426287 for context on different JSON names. + if 'Name' in section['Section']['Name']: + return section['Section']['Name']['Name'] + return section['Section']['Name']['Value'] + + proc = subprocess.Popen( + [readobj_path, '--sections', '--elf-output-style=JSON', libchrome_path], + stdout=subprocess.PIPE, + encoding='ascii') + + elfs = json.loads(proc.stdout.read())[0] + sections = elfs['Sections'] + + return next(s['Section']['Address'] for s in sections + if GetSectionName(s) == '.text') + + +def main(): + build_utils.InitLogging('CREATE_UNWIND_TABLE_DEBUG') + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--input_path', + help='Path to the unstripped binary.', + required=True, + metavar='FILE') + parser.add_argument('--output_path', + help='Path to unwind info binary output.', + required=True, + metavar='FILE') + parser.add_argument('--dump_syms_path', + required=True, + help='The path of the dump_syms binary.', + metavar='FILE') + parser.add_argument('--readobj_path', + required=True, + help='The path of the llvm-readobj binary.', + metavar='FILE') + + args = parser.parse_args() + proc = subprocess.Popen(['./' + args.dump_syms_path, args.input_path, '-v'], + stdout=subprocess.PIPE, + encoding='ascii') + + function_cfis = ReadFunctionCfi(proc.stdout) + function_unwinds = GenerateUnwinds(function_cfis, parsers=ALL_PARSERS) + encoded_function_unwinds = EncodeFunctionUnwinds( + function_unwinds, + ReadTextSectionStartAddress(args.readobj_path, args.input_path)) + (page_table, function_table, function_offset_table, + unwind_instruction_table) = GenerateUnwindTables(encoded_function_unwinds) + unwind_info: bytes = EncodeUnwindInfo(page_table, function_table, + function_offset_table, + unwind_instruction_table) + + if proc.wait(): + logging.critical('dump_syms exited with return code %d', proc.returncode) + sys.exit(proc.returncode) + + with open(args.output_path, 'wb') as f: + f.write(unwind_info) + + return 0 + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/build/android/gyp/create_unwind_table_tests.py b/build/android/gyp/create_unwind_table_tests.py new file mode 100755 index 0000000..14fbc22 --- /dev/null +++ b/build/android/gyp/create_unwind_table_tests.py
@@ -0,0 +1,1182 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Tests for create_unwind_table.py. + +This test suite contains tests for the custom unwind table creation for 32-bit +arm builds. +""" + +import io +import struct + +import unittest +import unittest.mock +import re + +from create_unwind_table import ( + AddressCfi, AddressUnwind, FilterToNonTombstoneCfi, FunctionCfi, + FunctionUnwind, EncodeAddressUnwind, EncodeAddressUnwinds, + EncodedAddressUnwind, EncodeAsBytes, EncodeFunctionOffsetTable, + EncodedFunctionUnwind, EncodeFunctionUnwinds, EncodeStackPointerUpdate, + EncodePop, EncodePageTableAndFunctionTable, EncodeUnwindInfo, + EncodeUnwindInstructionTable, GenerateUnwinds, GenerateUnwindTables, + NullParser, ParseAddressCfi, PushOrSubSpParser, ReadFunctionCfi, + REFUSE_TO_UNWIND, StoreSpParser, TRIVIAL_UNWIND, Uleb128Encode, + UnwindInstructionsParser, UnwindType, VPushParser) + + +class _TestReadFunctionCfi(unittest.TestCase): + def testFilterTombstone(self): + input_lines = [ + 'file name', + 'STACK CFI INIT 0 ', + 'STACK CFI 100 ', + 'STACK CFI INIT 1 ', + 'STACK CFI 200 ', + ] + + f = io.StringIO(''.join(line + '\n' for line in input_lines)) + + self.assertEqual([ + 'STACK CFI INIT 1 \n', + 'STACK CFI 200 \n', + ], list(FilterToNonTombstoneCfi(f))) + + def testReadFunctionCfiTombstoneFiltered(self): + input_lines = [ + 'STACK CFI INIT 0 50 .cfa: sp 0 + .ra: lr', # Tombstone function. + 'STACK CFI 2 .cfa: sp 24 + .ra: .cfa - 4 + ^ r4: .cfa - 16 + ^ ' + 'r5: .cfa - 12 + ^ r7: .cfa - 8 + ^', + 'STACK CFI INIT 15b6490 4 .cfa: sp 0 + .ra: lr', + ] + + f = io.StringIO(''.join(line + '\n' for line in input_lines)) + + self.assertEqual( + [FunctionCfi(4, (AddressCfi(0x15b6490, '.cfa: sp 0 + .ra: lr'), ))], + list(ReadFunctionCfi(f))) + + def testReadFunctionCfiSingleFunction(self): + input_lines = [ + 'STACK CFI INIT 15b6490 4 .cfa: sp 0 + .ra: lr', + 'STACK CFI 2 .cfa: sp 24 + .ra: .cfa - 4 + ^ r4: .cfa - 16 + ^ ' + 'r5: .cfa - 12 + ^ r7: .cfa - 8 + ^', + ] + + f = io.StringIO(''.join(line + '\n' for line in input_lines)) + + self.assertEqual([ + FunctionCfi(4, ( + AddressCfi(0x15b6490, '.cfa: sp 0 + .ra: lr'), + AddressCfi( + 0x2, '.cfa: sp 24 + .ra: .cfa - 4 + ^ r4: .cfa - 16 + ^ ' + 'r5: .cfa - 12 + ^ r7: .cfa - 8 + ^'), + )) + ], list(ReadFunctionCfi(f))) + + def testReadFunctionCfiMultipleFunctions(self): + input_lines = [ + 'STACK CFI INIT 15b6490 4 .cfa: sp 0 + .ra: lr', + 'STACK CFI 2 .cfa: sp 24 + .ra: .cfa - 4 + ^ r4: .cfa - 16 + ^ ' + 'r5: .cfa - 12 + ^ r7: .cfa - 8 + ^', + 'STACK CFI INIT 15b655a 26 .cfa: sp 0 + .ra: lr', + 'STACK CFI 15b655c .cfa: sp 8 + .ra: .cfa - 4 + ^ r4: .cfa - 8 + ^', + ] + + f = io.StringIO(''.join(line + '\n' for line in input_lines)) + + self.assertEqual([ + FunctionCfi(0x4, ( + AddressCfi(0x15b6490, '.cfa: sp 0 + .ra: lr'), + AddressCfi( + 0x2, '.cfa: sp 24 + .ra: .cfa - 4 + ^ r4: .cfa - 16 + ^ ' + 'r5: .cfa - 12 + ^ r7: .cfa - 8 + ^'), + )), + FunctionCfi(0x26, ( + AddressCfi(0x15b655a, '.cfa: sp 0 + .ra: lr'), + AddressCfi(0x15b655c, + '.cfa: sp 8 + .ra: .cfa - 4 + ^ r4: .cfa - 8 + ^'), + )), + ], list(ReadFunctionCfi(f))) + + +class _TestEncodeAsBytes(unittest.TestCase): + def testOutOfBounds(self): + self.assertRaises(ValueError, lambda: EncodeAsBytes(1024)) + self.assertRaises(ValueError, lambda: EncodeAsBytes(256)) + self.assertRaises(ValueError, lambda: EncodeAsBytes(-1)) + + def testEncode(self): + self.assertEqual(bytes([0]), EncodeAsBytes(0)) + self.assertEqual(bytes([255]), EncodeAsBytes(255)) + self.assertEqual(bytes([0, 1]), EncodeAsBytes(0, 1)) + + +class _TestUleb128Encode(unittest.TestCase): + def testNegativeValue(self): + self.assertRaises(ValueError, lambda: Uleb128Encode(-1)) + + def testSingleByte(self): + self.assertEqual(bytes([0]), Uleb128Encode(0)) + self.assertEqual(bytes([1]), Uleb128Encode(1)) + self.assertEqual(bytes([127]), Uleb128Encode(127)) + + def testMultiBytes(self): + self.assertEqual(bytes([0b10000000, 0b1]), Uleb128Encode(128)) + self.assertEqual(bytes([0b10000000, 0b10000000, 0b1]), + Uleb128Encode(128**2)) + + +class _TestEncodeStackPointerUpdate(unittest.TestCase): + def testSingleByte(self): + self.assertEqual(bytes([0b00000000 | 0]), EncodeStackPointerUpdate(4)) + self.assertEqual(bytes([0b01000000 | 0]), EncodeStackPointerUpdate(-4)) + + self.assertEqual(bytes([0b00000000 | 0b00111111]), + EncodeStackPointerUpdate(0x100)) + self.assertEqual(bytes([0b01000000 | 0b00111111]), + EncodeStackPointerUpdate(-0x100)) + + self.assertEqual(bytes([0b00000000 | 3]), EncodeStackPointerUpdate(16)) + self.assertEqual(bytes([0b01000000 | 3]), EncodeStackPointerUpdate(-16)) + + self.assertEqual(bytes([0b00111111]), EncodeStackPointerUpdate(0x100)) + + # 10110010 uleb128 + # vsp = vsp + 0x204 + (uleb128 << 2) + self.assertEqual(bytes([0b10110010, 0b00000000]), + EncodeStackPointerUpdate(0x204)) + self.assertEqual(bytes([0b10110010, 0b00000001]), + EncodeStackPointerUpdate(0x208)) + + # For vsp increments of 0x104-0x200, use 00xxxxxx twice. + self.assertEqual(bytes([0b00111111, 0b00000000]), + EncodeStackPointerUpdate(0x104)) + self.assertEqual(bytes([0b00111111, 0b00111111]), + EncodeStackPointerUpdate(0x200)) + self.assertEqual(bytes([0b01111111, 0b01111111]), + EncodeStackPointerUpdate(-0x200)) + + # Not multiple of 4. + self.assertRaises(AssertionError, lambda: EncodeStackPointerUpdate(101)) + # offset=0 is meaningless. + self.assertRaises(AssertionError, lambda: EncodeStackPointerUpdate(0)) + + +class _TestEncodePop(unittest.TestCase): + def testSingleRegister(self): + # Should reject registers outside r4 ~ r15 range. + for r in 0, 1, 2, 3, 16: + self.assertRaises(AssertionError, lambda: EncodePop([r])) + # Should use + # 1000iiii iiiiiiii + # Pop up to 12 integer registers under masks {r15-r12}, {r11-r4}. + self.assertEqual(bytes([0b10000000, 0b00000001]), EncodePop([4])) + self.assertEqual(bytes([0b10000000, 0b00001000]), EncodePop([7])) + self.assertEqual(bytes([0b10000100, 0b00000000]), EncodePop([14])) + self.assertEqual(bytes([0b10001000, 0b00000000]), EncodePop([15])) + + def testContinuousRegisters(self): + # 10101nnn + # Pop r4-r[4+nnn], r14. + self.assertEqual(bytes([0b10101000]), EncodePop([4, 14])) + self.assertEqual(bytes([0b10101001]), EncodePop([4, 5, 14])) + self.assertEqual(bytes([0b10101111]), + EncodePop([4, 5, 6, 7, 8, 9, 10, 11, 14])) + + def testDiscontinuousRegisters(self): + # 1000iiii iiiiiiii + # Pop up to 12 integer registers under masks {r15-r12}, {r11-r4}. + self.assertEqual(bytes([0b10001000, 0b00000001]), EncodePop([4, 15])) + self.assertEqual(bytes([0b10000100, 0b00011000]), EncodePop([7, 8, 14])) + self.assertEqual(bytes([0b10000111, 0b11111111]), + EncodePop([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])) + self.assertEqual(bytes([0b10000100, 0b10111111]), + EncodePop([4, 5, 6, 7, 8, 9, 11, 14])) + + +class _TestEncodeAddressUnwind(unittest.TestCase): + def testReturnToLr(self): + self.assertEqual( + bytes([0b10110000]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=tuple()))) + + def testNoAction(self): + self.assertEqual( + bytes([]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.NO_ACTION, + sp_offset=0, + registers=tuple()))) + + def testUpdateSpAndOrPopRegisters(self): + self.assertEqual( + bytes([0b0, 0b10101000]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0x4, + registers=(4, 14)))) + + self.assertEqual( + bytes([0b0]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0x4, + registers=tuple()))) + + self.assertEqual( + bytes([0b10101000]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(4, 14)))) + + def testRestoreSpFromRegisters(self): + self.assertEqual( + bytes([0b10010100, 0b0]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.RESTORE_SP_FROM_REGISTER, + sp_offset=0x4, + registers=(4, )))) + + self.assertEqual( + bytes([0b10010100]), + EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.RESTORE_SP_FROM_REGISTER, + sp_offset=0, + registers=(4, )))) + + self.assertRaises( + AssertionError, lambda: EncodeAddressUnwind( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.RESTORE_SP_FROM_REGISTER, + sp_offset=0x4, + registers=tuple()))) + + +class _TestEncodeAddressUnwinds(unittest.TestCase): + def testEncodeOrder(self): + address_unwind1 = AddressUnwind(address_offset=0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=tuple()) + address_unwind2 = AddressUnwind( + address_offset=4, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(4, 14)) + + def MockEncodeAddressUnwind(address_unwind): + return { + address_unwind1: bytes([1]), + address_unwind2: bytes([2]), + }[address_unwind] + + with unittest.mock.patch("create_unwind_table.EncodeAddressUnwind", + side_effect=MockEncodeAddressUnwind): + encoded_unwinds = EncodeAddressUnwinds((address_unwind1, address_unwind2)) + self.assertEqual(( + EncodedAddressUnwind(4, + bytes([2]) + bytes([1])), + EncodedAddressUnwind(0, bytes([1])), + ), encoded_unwinds) + + +PAGE_SIZE = 1 << 17 + + +class _TestEncodeFunctionUnwinds(unittest.TestCase): + @unittest.mock.patch('create_unwind_table.EncodeAddressUnwinds') + def testEncodeOrder(self, MockEncodeAddressUnwinds): + MockEncodeAddressUnwinds.return_value = EncodedAddressUnwind(0, b'\x00') + + self.assertEqual([ + EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=EncodedAddressUnwind(0, b'\x00')), + EncodedFunctionUnwind(page_number=0, + page_offset=100 >> 1, + address_unwinds=EncodedAddressUnwind(0, b'\x00')), + ], + list( + EncodeFunctionUnwinds([ + FunctionUnwind(address=100, + size=PAGE_SIZE - 100, + address_unwinds=()), + FunctionUnwind( + address=0, size=100, address_unwinds=()), + ], + text_section_start_address=0))) + + @unittest.mock.patch('create_unwind_table.EncodeAddressUnwinds') + def testFillingGaps(self, MockEncodeAddressUnwinds): + MockEncodeAddressUnwinds.return_value = EncodedAddressUnwind(0, b'\x00') + + self.assertEqual([ + EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=EncodedAddressUnwind(0, b'\x00')), + EncodedFunctionUnwind( + page_number=0, page_offset=50 >> 1, address_unwinds=TRIVIAL_UNWIND), + EncodedFunctionUnwind(page_number=0, + page_offset=100 >> 1, + address_unwinds=EncodedAddressUnwind(0, b'\x00')), + ], + list( + EncodeFunctionUnwinds([ + FunctionUnwind( + address=0, size=50, address_unwinds=()), + FunctionUnwind(address=100, + size=PAGE_SIZE - 100, + address_unwinds=()), + ], + text_section_start_address=0))) + + @unittest.mock.patch('create_unwind_table.EncodeAddressUnwinds') + def testFillingLastPage(self, MockEncodeAddressUnwinds): + MockEncodeAddressUnwinds.return_value = EncodedAddressUnwind(0, b'\x00') + + self.assertEqual( + [ + EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=EncodedAddressUnwind( + 0, b'\x00')), + EncodedFunctionUnwind(page_number=0, + page_offset=100 >> 1, + address_unwinds=EncodedAddressUnwind( + 0, b'\x00')), + EncodedFunctionUnwind(page_number=0, + page_offset=200 >> 1, + address_unwinds=REFUSE_TO_UNWIND), + ], + list( + EncodeFunctionUnwinds([ + FunctionUnwind(address=1100, size=100, address_unwinds=()), + FunctionUnwind(address=1200, size=100, address_unwinds=()), + ], + text_section_start_address=1100))) + + @unittest.mock.patch('create_unwind_table.EncodeAddressUnwinds') + def testFillingFirstPage(self, MockEncodeAddressUnwinds): + MockEncodeAddressUnwinds.return_value = EncodedAddressUnwind(0, b'\x00') + + self.assertEqual( + [ + EncodedFunctionUnwind( + page_number=0, page_offset=0, address_unwinds=REFUSE_TO_UNWIND), + EncodedFunctionUnwind(page_number=0, + page_offset=100 >> 1, + address_unwinds=EncodedAddressUnwind( + 0, b'\x00')), + EncodedFunctionUnwind(page_number=0, + page_offset=200 >> 1, + address_unwinds=EncodedAddressUnwind( + 0, b'\x00')), + EncodedFunctionUnwind(page_number=0, + page_offset=300 >> 1, + address_unwinds=REFUSE_TO_UNWIND), + ], + list( + EncodeFunctionUnwinds([ + FunctionUnwind(address=1100, size=100, address_unwinds=()), + FunctionUnwind(address=1200, size=100, address_unwinds=()), + ], + text_section_start_address=1000))) + + @unittest.mock.patch('create_unwind_table.EncodeAddressUnwinds') + def testOverlappedFunctions(self, _): + self.assertRaises( + # Eval generator with `list`. Otherwise the code will not execute. + AssertionError, + lambda: list( + EncodeFunctionUnwinds([ + FunctionUnwind(address=0, size=100, address_unwinds=()), + FunctionUnwind(address=50, size=100, address_unwinds=()), + ], + text_section_start_address=0))) + + +class _TestNullParser(unittest.TestCase): + def testCfaChange(self): + parser = NullParser() + match = parser.GetBreakpadInstructionsRegex().search('.cfa: sp 0 + .ra: lr') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=0, + cfa_sp_offset=0, + match=match) + + self.assertEqual(0, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=()), address_unwind) + + +class _TestPushOrSubSpParser(unittest.TestCase): + def testCfaChange(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search('.cfa: sp 4 +') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(4, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=4, + registers=()), address_unwind) + + def testCfaAndRaChangePopOnly(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 4 + .ra: .cfa -4 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(4, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(14, )), address_unwind) + + def testCfaAndRaChangePopAndSpUpdate(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 8 + .ra: .cfa -4 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(8, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=4, + registers=(14, )), address_unwind) + + def testCfaAndRaAndRegistersChangePopOnly(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 12 + .ra: .cfa -4 + ^ r4: .cfa -12 + ^ r7: .cfa -8 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(12, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(4, 7, 14)), address_unwind) + + def testCfaAndRaAndRegistersChangePopAndSpUpdate(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 16 + .ra: .cfa -4 + ^ r4: .cfa -12 + ^ r7: .cfa -8 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(16, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=4, + registers=(4, 7, 14)), address_unwind) + + def testRegistersChange(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + 'r4: .cfa -8 + ^ r7: .cfa -4 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(0, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(4, 7)), address_unwind) + + def testCfaAndRegistersChange(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 8 + r4: .cfa -8 + ^ r7: .cfa -4 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(8, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(4, 7)), address_unwind) + + def testRegistersOrdering(self): + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + 'r10: .cfa -8 + ^ r7: .cfa -4 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(0, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=0, + registers=(7, 10)), address_unwind) + + def testPoppingCallerSaveRegisters(self): + """Regression test for pop unwinds that encode caller-save registers. + + Callee-save registers: r0 ~ r3. + """ + parser = PushOrSubSpParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 16 + .ra: .cfa -4 + ^ ' + 'r3: .cfa -16 + ^ r4: .cfa -12 + ^ r5: .cfa -8 + ^') + + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=0, + match=match) + + self.assertEqual(16, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=4, + registers=(4, 5, 14)), address_unwind) + + +class _TestVPushParser(unittest.TestCase): + def testCfaAndRegistersChange(self): + parser = VPushParser() + match = parser.GetBreakpadInstructionsRegex().search( + '.cfa: sp 40 + unnamed_register264: .cfa -40 + ^ ' + 'unnamed_register265: .cfa -32 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=24, + match=match) + + self.assertEqual(40, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, + sp_offset=16, + registers=()), address_unwind) + + def testRegistersChange(self): + parser = VPushParser() + match = parser.GetBreakpadInstructionsRegex().search( + 'unnamed_register264: .cfa -40 + ^ unnamed_register265: .cfa -32 + ^') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=24, + match=match) + + self.assertEqual(24, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.NO_ACTION, + sp_offset=0, + registers=()), address_unwind) + + +class _TestStoreSpParser(unittest.TestCase): + def testCfaAndRegistersChange(self): + parser = StoreSpParser() + match = parser.GetBreakpadInstructionsRegex().search('.cfa: r7 8 +') + self.assertIsNotNone(match) + + address_unwind, new_cfa_sp_offset = parser.ParseFromMatch(address_offset=20, + cfa_sp_offset=12, + match=match) + + self.assertEqual(8, new_cfa_sp_offset) + self.assertEqual( + AddressUnwind(address_offset=20, + unwind_type=UnwindType.RESTORE_SP_FROM_REGISTER, + sp_offset=-4, + registers=(7, )), address_unwind) + + +class _TestEncodeUnwindInstructionTable(unittest.TestCase): + def testSingleEntry(self): + table, offsets = EncodeUnwindInstructionTable([bytes([3])]) + + self.assertEqual(bytes([3]), table) + self.assertDictEqual({ + bytes([3]): 0, + }, offsets) + + def testMultipleEntries(self): + self.maxDiff = None + # Result should be sorted by score descending. + table, offsets = EncodeUnwindInstructionTable([ + bytes([1, 2, 3]), + bytes([0, 3]), + bytes([3]), + ]) + self.assertEqual(bytes([3, 0, 3, 1, 2, 3]), table) + self.assertDictEqual( + { + bytes([1, 2, 3]): 3, # score = 1 / 3 = 0.67 + bytes([0, 3]): 1, # score = 1 / 2 = 0.5 + bytes([3]): 0, # score = 1 / 1 = 1 + }, + offsets) + + # When scores are same, sort by sequence descending. + table, offsets = EncodeUnwindInstructionTable([ + bytes([3]), + bytes([0, 3]), + bytes([0, 3]), + bytes([1, 2, 3]), + bytes([1, 2, 3]), + bytes([1, 2, 3]), + ]) + self.assertEqual(bytes([3, 1, 2, 3, 0, 3]), table) + self.assertDictEqual( + { + bytes([3]): 0, # score = 1 / 1 = 1 + bytes([1, 2, 3]): 1, # score = 3 / 3 = 1 + bytes([0, 3]): 4, # score = 2 / 2 = 1 + }, + offsets) + + +class _TestFunctionOffsetTable(unittest.TestCase): + def testSingleEntry(self): + self.maxDiff = None + complete_instruction_sequence0 = bytes([3]) + complete_instruction_sequence1 = bytes([1, 3]) + + sequence1 = ( + EncodedAddressUnwind(0x400, complete_instruction_sequence1), + EncodedAddressUnwind(0x0, complete_instruction_sequence0), + ) + + address_unwind_sequences = [sequence1] + + table, offsets = EncodeFunctionOffsetTable( + address_unwind_sequences, { + complete_instruction_sequence0: 52, + complete_instruction_sequence1: 50, + }) + + self.assertEqual( + bytes([ + # (0x200, 50) + 128, + 4, + 50, + # (0, 52) + 0, + 52, + ]), + table) + + self.assertDictEqual({ + sequence1: 0, + }, offsets) + + def testMultipleEntry(self): + self.maxDiff = None + complete_instruction_sequence0 = bytes([3]) + complete_instruction_sequence1 = bytes([1, 3]) + complete_instruction_sequence2 = bytes([2, 3]) + + sequence1 = ( + EncodedAddressUnwind(0x20, complete_instruction_sequence1), + EncodedAddressUnwind(0x0, complete_instruction_sequence0), + ) + sequence2 = ( + EncodedAddressUnwind(0x400, complete_instruction_sequence2), + EncodedAddressUnwind(0x0, complete_instruction_sequence0), + ) + address_unwind_sequences = [sequence1, sequence2] + + table, offsets = EncodeFunctionOffsetTable( + address_unwind_sequences, { + complete_instruction_sequence0: 52, + complete_instruction_sequence1: 50, + complete_instruction_sequence2: 80, + }) + + self.assertEqual( + bytes([ + # (0x10, 50) + 0x10, + 50, + # (0, 52) + 0, + 52, + # (0x200, 80) + 128, + 4, + 80, + # (0, 52) + 0, + 52, + ]), + table) + + self.assertDictEqual({ + sequence1: 0, + sequence2: 4, + }, offsets) + + def testDuplicatedEntry(self): + self.maxDiff = None + complete_instruction_sequence0 = bytes([3]) + complete_instruction_sequence1 = bytes([1, 3]) + complete_instruction_sequence2 = bytes([2, 3]) + + sequence1 = ( + EncodedAddressUnwind(0x20, complete_instruction_sequence1), + EncodedAddressUnwind(0x0, complete_instruction_sequence0), + ) + sequence2 = ( + EncodedAddressUnwind(0x400, complete_instruction_sequence2), + EncodedAddressUnwind(0x0, complete_instruction_sequence0), + ) + sequence3 = sequence1 + + address_unwind_sequences = [sequence1, sequence2, sequence3] + + table, offsets = EncodeFunctionOffsetTable( + address_unwind_sequences, { + complete_instruction_sequence0: 52, + complete_instruction_sequence1: 50, + complete_instruction_sequence2: 80, + }) + + self.assertEqual( + bytes([ + # (0x10, 50) + 0x10, + 50, + # (0, 52) + 0, + 52, + # (0x200, 80) + 128, + 4, + 80, + # (0, 52) + 0, + 52, + ]), + table) + + self.assertDictEqual({ + sequence1: 0, + sequence2: 4, + }, offsets) + + +class _TestEncodePageTableAndFunctionTable(unittest.TestCase): + def testMultipleFunctionUnwinds(self): + address_unwind_sequence0 = ( + EncodedAddressUnwind(0x10, bytes([0, 3])), + EncodedAddressUnwind(0x0, bytes([3])), + ) + address_unwind_sequence1 = ( + EncodedAddressUnwind(0x10, bytes([1, 3])), + EncodedAddressUnwind(0x0, bytes([3])), + ) + address_unwind_sequence2 = ( + EncodedAddressUnwind(0x200, bytes([2, 3])), + EncodedAddressUnwind(0x0, bytes([3])), + ) + + function_unwinds = [ + EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=address_unwind_sequence0), + EncodedFunctionUnwind(page_number=0, + page_offset=0x8000, + address_unwinds=address_unwind_sequence1), + EncodedFunctionUnwind(page_number=1, + page_offset=0x8000, + address_unwinds=address_unwind_sequence2), + ] + + function_offset_table_offsets = { + address_unwind_sequence0: 0x100, + address_unwind_sequence1: 0x200, + address_unwind_sequence2: 0x300, + } + + page_table, function_table = EncodePageTableAndFunctionTable( + function_unwinds, function_offset_table_offsets) + + self.assertEqual(2 * 4, len(page_table)) + self.assertEqual((0, 2), struct.unpack('2I', page_table)) + + self.assertEqual(6 * 2, len(function_table)) + self.assertEqual((0, 0x100, 0x8000, 0x200, 0x8000, 0x300), + struct.unpack('6H', function_table)) + + def testMultiPageFunction(self): + address_unwind_sequence0 = ( + EncodedAddressUnwind(0x10, bytes([0, 3])), + EncodedAddressUnwind(0x0, bytes([3])), + ) + address_unwind_sequence1 = ( + EncodedAddressUnwind(0x10, bytes([1, 3])), + EncodedAddressUnwind(0x0, bytes([3])), + ) + address_unwind_sequence2 = ( + EncodedAddressUnwind(0x200, bytes([2, 3])), + EncodedAddressUnwind(0x0, bytes([3])), + ) + + function_unwinds = [ + EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=address_unwind_sequence0), + # Large function. + EncodedFunctionUnwind(page_number=0, + page_offset=0x8000, + address_unwinds=address_unwind_sequence1), + EncodedFunctionUnwind(page_number=4, + page_offset=0x8000, + address_unwinds=address_unwind_sequence2), + ] + + function_offset_table_offsets = { + address_unwind_sequence0: 0x100, + address_unwind_sequence1: 0x200, + address_unwind_sequence2: 0x300, + } + + page_table, function_table = EncodePageTableAndFunctionTable( + function_unwinds, function_offset_table_offsets) + + self.assertEqual(5 * 4, len(page_table)) + self.assertEqual((0, 2, 2, 2, 2), struct.unpack('5I', page_table)) + + self.assertEqual(6 * 2, len(function_table)) + self.assertEqual((0, 0x100, 0x8000, 0x200, 0x8000, 0x300), + struct.unpack('6H', function_table)) + + +class MockReturnParser(UnwindInstructionsParser): + def GetBreakpadInstructionsRegex(self): + return re.compile(r'^RETURN$') + + def ParseFromMatch(self, address_offset, cfa_sp_offset, match): + return AddressUnwind(address_offset, UnwindType.RETURN_TO_LR, 0, ()), 0 + + +class MockEpilogueUnwindParser(UnwindInstructionsParser): + def GetBreakpadInstructionsRegex(self): + return re.compile(r'^EPILOGUE_UNWIND$') + + def ParseFromMatch(self, address_offset, cfa_sp_offset, match): + return AddressUnwind(address_offset, + UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, 0, ()), -100 + + +class MockWildcardParser(UnwindInstructionsParser): + def GetBreakpadInstructionsRegex(self): + return re.compile(r'.*') + + def ParseFromMatch(self, address_offset, cfa_sp_offset, match): + return AddressUnwind(address_offset, + UnwindType.UPDATE_SP_AND_OR_POP_REGISTERS, 0, ()), -200 + + +class _TestParseAddressCfi(unittest.TestCase): + def testSuccessParse(self): + address_unwind = AddressUnwind( + address_offset=0x300, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=(), + ) + + self.assertEqual((address_unwind, False, 0), + ParseAddressCfi(AddressCfi(address=0x800, + unwind_instructions='RETURN'), + function_start_address=0x500, + parsers=(MockReturnParser(), ), + prev_cfa_sp_offset=0)) + + def testUnhandledAddress(self): + self.assertEqual((None, False, 100), + ParseAddressCfi(AddressCfi(address=0x800, + unwind_instructions='UNKNOWN'), + function_start_address=0x500, + parsers=(MockReturnParser(), ), + prev_cfa_sp_offset=100)) + + def testEpilogueUnwind(self): + self.assertEqual( + (None, True, -100), + ParseAddressCfi(AddressCfi(address=0x800, + unwind_instructions='EPILOGUE_UNWIND'), + function_start_address=0x500, + parsers=(MockEpilogueUnwindParser(), ), + prev_cfa_sp_offset=100)) + + def testParsePrecedence(self): + address_unwind = AddressUnwind( + address_offset=0x300, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=(), + ) + + self.assertEqual( + (address_unwind, False, 0), + ParseAddressCfi(AddressCfi(address=0x800, unwind_instructions='RETURN'), + function_start_address=0x500, + parsers=(MockReturnParser(), MockWildcardParser()), + prev_cfa_sp_offset=0)) + + +class _TestGenerateUnwinds(unittest.TestCase): + def testSuccessUnwind(self): + self.assertEqual( + [ + FunctionUnwind(address=0x100, + size=1024, + address_unwinds=( + AddressUnwind( + address_offset=0x0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=(), + ), + AddressUnwind( + address_offset=0x200, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=(), + ), + )) + ], + list( + GenerateUnwinds([ + FunctionCfi( + size=1024, + address_cfi=( + AddressCfi(address=0x100, unwind_instructions='RETURN'), + AddressCfi(address=0x300, unwind_instructions='RETURN'), + )) + ], + parsers=[MockReturnParser()]))) + + def testUnhandledAddress(self): + self.assertEqual( + [ + FunctionUnwind(address=0x100, + size=1024, + address_unwinds=(AddressUnwind( + address_offset=0x0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=(), + ), )) + ], + list( + GenerateUnwinds([ + FunctionCfi(size=1024, + address_cfi=( + AddressCfi(address=0x100, + unwind_instructions='RETURN'), + AddressCfi(address=0x300, + unwind_instructions='UNKNOWN'), + )) + ], + parsers=[MockReturnParser()]))) + + def testEpilogueUnwind(self): + self.assertEqual( + [ + FunctionUnwind(address=0x100, + size=1024, + address_unwinds=(AddressUnwind( + address_offset=0x0, + unwind_type=UnwindType.RETURN_TO_LR, + sp_offset=0, + registers=(), + ), )) + ], + list( + GenerateUnwinds([ + FunctionCfi( + size=1024, + address_cfi=( + AddressCfi(address=0x100, unwind_instructions='RETURN'), + AddressCfi(address=0x300, + unwind_instructions='EPILOGUE_UNWIND'), + )) + ], + parsers=[ + MockReturnParser(), + MockEpilogueUnwindParser() + ]))) + + def testInvalidInitialUnwindInstructionAsserts(self): + self.assertRaises( + AssertionError, lambda: list( + GenerateUnwinds([ + FunctionCfi(size=1024, + address_cfi=( + AddressCfi(address=0x100, + unwind_instructions='UNKNOWN'), + AddressCfi(address=0x200, + unwind_instructions='RETURN'), + )) + ], + parsers=[MockReturnParser()]))) + + +class _TestEncodeUnwindInfo(unittest.TestCase): + def testEncodeTables(self): + page_table = struct.pack('I', 0) + function_table = struct.pack('4H', 1, 2, 3, 4) + function_offset_table = bytes([1, 2]) + unwind_instruction_table = bytes([1, 2, 3]) + + unwind_info = EncodeUnwindInfo( + page_table, + function_table, + function_offset_table, + unwind_instruction_table, + ) + + self.assertEqual( + 32 + len(page_table) + len(function_table) + + len(function_offset_table) + len(unwind_instruction_table), + len(unwind_info)) + # Header. + self.assertEqual((32, 1, 36, 2, 44, 2, 46, 3), + struct.unpack('8I', unwind_info[:32])) + # Body. + self.assertEqual( + page_table + function_table + function_offset_table + + unwind_instruction_table, unwind_info[32:]) + + def testUnalignedTables(self): + self.assertRaises( + AssertionError, lambda: EncodeUnwindInfo(bytes([1]), b'', b'', b'')) + self.assertRaises( + AssertionError, lambda: EncodeUnwindInfo(b'', bytes([1]), b'', b'')) + + +class _TestGenerateUnwindTables(unittest.TestCase): + def testGenerateUnwindTables(self): + """This is an integration test that hooks everything together. """ + address_unwind_sequence0 = ( + EncodedAddressUnwind(0x20, bytes([0, 0xb0])), + EncodedAddressUnwind(0x0, bytes([0xb0])), + ) + address_unwind_sequence1 = ( + EncodedAddressUnwind(0x20, bytes([1, 0xb0])), + EncodedAddressUnwind(0x0, bytes([0xb0])), + ) + address_unwind_sequence2 = ( + EncodedAddressUnwind(0x200, bytes([2, 0xb0])), + EncodedAddressUnwind(0x0, bytes([0xb0])), + ) + + (page_table, function_table, function_offset_table, + unwind_instruction_table) = GenerateUnwindTables([ + EncodedFunctionUnwind(page_number=0, + page_offset=0, + address_unwinds=TRIVIAL_UNWIND), + EncodedFunctionUnwind(page_number=0, + page_offset=0x1000, + address_unwinds=address_unwind_sequence0), + EncodedFunctionUnwind(page_number=1, + page_offset=0x2000, + address_unwinds=address_unwind_sequence1), + EncodedFunctionUnwind(page_number=3, + page_offset=0x1000, + address_unwinds=address_unwind_sequence2), + ]) + + # Complete instruction sequences and their frequencies. + # [0xb0]: 4 + # [0, 0xb0]: 1 + # [1, 0xb0]: 1 + # [2, 0xb0]: 1 + self.assertEqual(bytes([0xb0, 2, 0xb0, 1, 0xb0, 0, 0xb0]), + unwind_instruction_table) + + self.assertEqual( + bytes([ + # Trivial unwind. + 0, + 0, + # Address unwind sequence 0. + 0x10, + 5, + 0, + 0, + # Address unwind sequence 1. + 0x10, + 3, + 0, + 0, + # Address unwind sequence 2. + 0x80, + 2, + 1, + 0, + 0, + ]), + function_offset_table) + + self.assertEqual(8 * 2, len(function_table)) + self.assertEqual((0, 0, 0x1000, 2, 0x2000, 6, 0x1000, 10), + struct.unpack('8H', function_table)) + + self.assertEqual(4 * 4, len(page_table)) + self.assertEqual((0, 2, 3, 3), struct.unpack('4I', page_table))
diff --git a/build/android/gyp/desugar.py b/build/android/gyp/desugar.py deleted file mode 100755 index 87eb159..0000000 --- a/build/android/gyp/desugar.py +++ /dev/null
@@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2017 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import argparse -import os -import sys - -from util import build_utils - - -def main(): - args = build_utils.ExpandFileArgs(sys.argv[1:]) - parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) - parser.add_argument('--desugar-jar', required=True, - help='Path to Desugar.jar.') - parser.add_argument('--input-jar', required=True, - help='Jar input path to include .class files from.') - parser.add_argument('--output-jar', required=True, - help='Jar output path.') - parser.add_argument('--classpath', - action='append', - required=True, - help='Classpath.') - parser.add_argument('--bootclasspath', required=True, - help='Path to javac bootclasspath interface jar.') - parser.add_argument('--warnings-as-errors', - action='store_true', - help='Treat all warnings as errors.') - options = parser.parse_args(args) - - options.bootclasspath = build_utils.ParseGnList(options.bootclasspath) - options.classpath = build_utils.ParseGnList(options.classpath) - - cmd = build_utils.JavaCmd(options.warnings_as_errors) + [ - '-jar', - options.desugar_jar, - '--input', - options.input_jar, - '--output', - options.output_jar, - '--generate_base_classes_for_default_methods', - # Don't include try-with-resources files in every .jar. Instead, they - # are included via //third_party/bazel/desugar:desugar_runtime_java. - '--desugar_try_with_resources_omit_runtime_classes', - ] - for path in options.bootclasspath: - cmd += ['--bootclasspath_entry', path] - for path in options.classpath: - cmd += ['--classpath_entry', path] - build_utils.CheckOutput( - cmd, - print_stdout=False, - stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings, - fail_on_output=options.warnings_as_errors) - - if options.depfile: - build_utils.WriteDepfile(options.depfile, - options.output_jar, - inputs=options.bootclasspath + options.classpath) - - -if __name__ == '__main__': - sys.exit(main())
diff --git a/build/android/gyp/desugar.pydeps b/build/android/gyp/desugar.pydeps deleted file mode 100644 index 3e5c9ea..0000000 --- a/build/android/gyp/desugar.pydeps +++ /dev/null
@@ -1,6 +0,0 @@ -# Generated by running: -# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/desugar.pydeps build/android/gyp/desugar.py -../../gn_helpers.py -desugar.py -util/__init__.py -util/build_utils.py
diff --git a/build/android/gyp/dex.py b/build/android/gyp/dex.py index 9664922..a7f024a 100755 --- a/build/android/gyp/dex.py +++ b/build/android/gyp/dex.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,38 +10,58 @@ import os import re import shutil +import shlex import sys import tempfile import zipfile from util import build_utils from util import md5_check -from util import zipalign +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers -sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.path.pardir)) -import convert_dex_profile - +_DEX_XMX = '2G' # Increase this when __final_dex OOMs. _IGNORE_WARNINGS = ( - # Caused by Play Services: - r'Type `libcore.io.Memory` was not found', - # Caused by a missing final class in flogger: - r'Type `dalvik.system.VMStack` was not found', - # Caused by jacoco code coverage: - r'Type `java.lang.management.ManagementFactory` was not found', - # TODO(wnwen): Remove this after R8 version 3.0.26-dev: - r'Missing class sun.misc.Unsafe', - # Caused when the test apk and the apk under test do not having native libs. + # Warning: Running R8 version main (build engineering), which cannot be + # represented as a semantic version. Using an artificial version newer than + # any known version for selecting Proguard configurations embedded under + # META-INF/. This means that all rules with a '-upto-' qualifier will be + # excluded and all rules with a -from- qualifier will be included. + r'Running R8 version main', + # E.g. Triggers for weblayer_instrumentation_test_apk since both it and its + # apk_under_test have no shared_libraries. + # https://crbug.com/1364192 << To fix this in a better way. r'Missing class org.chromium.build.NativeLibraries', - # Caused by internal annotation: https://crbug.com/1180222 - r'Missing class com.google.errorprone.annotations.RestrictedInheritance', # Caused by internal protobuf package: https://crbug.com/1183971 r'referenced from: com.google.protobuf.GeneratedMessageLite$GeneratedExtension', # pylint: disable=line-too-long - # Caused by using Bazel desugar instead of D8 for desugar, since Bazel - # desugar doesn't preserve interfaces in the same way. This should be - # removed when D8 is used for desugaring. - r'Warning: Cannot emulate interface ', + # Desugaring configs may occasionally not match types in our program. This + # may happen temporarily until we move over to the new desugared library + # json flags. See crbug.com/1302088 - this should be removed when this bug + # is fixed. + r'Warning: Specification conversion: The following', + # Caused by protobuf runtime using -identifiernamestring in a way that + # doesn't work with R8. Looks like: + # Rule matches the static final field `...`, which may have been inlined... + # com.google.protobuf.*GeneratedExtensionRegistryLite { + # static java.lang.String CONTAINING_TYPE_*; + # } + r'GeneratedExtensionRegistryLite.CONTAINING_TYPE_', + # Relevant for R8 when optimizing an app that doesn't use protobuf. + r'Ignoring -shrinkunusedprotofields since the protobuf-lite runtime is', + # Ignore Unused Rule Warnings in third_party libraries. + r'/third_party/.*Proguard configuration rule does not match anything', + # Ignore Unused Rule Warnings for system classes (aapt2 generates these). + r'Proguard configuration rule does not match anything:.*class android\.', + # TODO(crbug.com/1303951): Don't ignore all such warnings. + r'Proguard configuration rule does not match anything:', + # TODO(agrieve): Remove once we update to U SDK. + r'OnBackAnimationCallback', +) + +_SKIPPED_CLASS_FILE_NAMES = ( + 'module-info.class', # Explicitly skipped by r8/utils/FileUtils#isClassFile ) @@ -49,7 +69,7 @@ args = build_utils.ExpandFileArgs(args) parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--output', required=True, help='Dex output path.') parser.add_argument( '--class-inputs', @@ -94,8 +114,6 @@ '--bootclasspath', action='append', help='GN-list of bootclasspath. Needed for --desugar') - parser.add_argument( - '--desugar-jdk-libs-json', help='Path to desugar_jdk_libs.json.') parser.add_argument('--show-desugar-default-interface-warnings', action='store_true', help='Enable desugaring warnings.') @@ -114,6 +132,8 @@ parser.add_argument('--force-enable-assertions', action='store_true', help='Forcefully enable javac generated assertion code.') + parser.add_argument('--assertion-handler', + help='The class name of the assertion handler class.') parser.add_argument('--warnings-as-errors', action='store_true', help='Treat all warnings as errors.') @@ -121,47 +141,22 @@ action='store_true', help='Use when filing D8 bugs to capture inputs.' ' Stores inputs to d8inputs.zip') - - group = parser.add_argument_group('Dexlayout') - group.add_argument( - '--dexlayout-profile', - help=('Text profile for dexlayout. If present, a dexlayout ' - 'pass will happen')) - group.add_argument( - '--profman-path', - help=('Path to ART profman binary. There should be a lib/ directory at ' - 'the same path with shared libraries (shared with dexlayout).')) - group.add_argument( - '--dexlayout-path', - help=('Path to ART dexlayout binary. There should be a lib/ directory at ' - 'the same path with shared libraries (shared with dexlayout).')) - group.add_argument('--dexdump-path', help='Path to dexdump binary.') - group.add_argument( - '--proguard-mapping-path', - help=('Path to proguard map from obfuscated symbols in the jar to ' - 'unobfuscated symbols present in the code. If not present, the jar ' - 'is assumed not to be obfuscated.')) - options = parser.parse_args(args) - if options.dexlayout_profile: - build_utils.CheckOptions( - options, - parser, - required=('profman_path', 'dexlayout_path', 'dexdump_path')) - elif options.proguard_mapping_path is not None: - parser.error('Unexpected proguard mapping without dexlayout') - if options.main_dex_rules_path and not options.multi_dex: parser.error('--main-dex-rules-path is unused if multidex is not enabled') - options.class_inputs = build_utils.ParseGnList(options.class_inputs) - options.class_inputs_filearg = build_utils.ParseGnList( + if options.force_enable_assertions and options.assertion_handler: + parser.error('Cannot use both --force-enable-assertions and ' + '--assertion-handler') + + options.class_inputs = action_helpers.parse_gn_list(options.class_inputs) + options.class_inputs_filearg = action_helpers.parse_gn_list( options.class_inputs_filearg) - options.bootclasspath = build_utils.ParseGnList(options.bootclasspath) - options.classpath = build_utils.ParseGnList(options.classpath) - options.dex_inputs = build_utils.ParseGnList(options.dex_inputs) - options.dex_inputs_filearg = build_utils.ParseGnList( + options.bootclasspath = action_helpers.parse_gn_list(options.bootclasspath) + options.classpath = action_helpers.parse_gn_list(options.classpath) + options.dex_inputs = action_helpers.parse_gn_list(options.dex_inputs) + options.dex_inputs_filearg = action_helpers.parse_gn_list( options.dex_inputs_filearg) return options @@ -169,28 +164,27 @@ def CreateStderrFilter(show_desugar_default_interface_warnings): def filter_stderr(output): + # Set this when debugging R8 output. + if os.environ.get('R8_SHOW_ALL_OUTPUT', '0') != '0': + return output + + warnings = re.split(r'^(?=Warning|Error)', output, flags=re.MULTILINE) + preamble, *warnings = warnings + patterns = list(_IGNORE_WARNINGS) - # When using Bazel's Desugar tool to desugar lambdas and interface methods, - # we do not provide D8 with a classpath, which causes a lot of warnings from - # D8's default interface desugaring pass. Not having a classpath makes - # incremental dexing much more effective. D8 still does backported method - # desugaring. - # These warnings are also turned off when bytecode checks are turned off. + # Missing deps can happen for prebuilts that are missing transitive deps + # and have set enable_bytecode_checks=false. if not show_desugar_default_interface_warnings: patterns += ['default or static interface methods'] combined_pattern = '|'.join(re.escape(p) for p in patterns) - output = build_utils.FilterLines(output, combined_pattern) + preamble = build_utils.FilterLines(preamble, combined_pattern) - # Each warning has a prefix line of the file it's from. If we've filtered - # out the warning, then also filter out the file header. - # E.g.: - # Warning in path/to/Foo.class: - # Error message #1 indented here. - # Error message #2 indented here. - output = re.sub(r'^Warning in .*?:\n(?! )', '', output, flags=re.MULTILINE) - return output + compiled_re = re.compile(combined_pattern, re.DOTALL) + warnings = [w for w in warnings if not compiled_re.search(w)] + + return preamble + ''.join(warnings) return filter_stderr @@ -201,142 +195,35 @@ stderr_filter = CreateStderrFilter(show_desugar_default_interface_warnings) - with tempfile.NamedTemporaryFile(mode='w') as flag_file: + is_debug = logging.getLogger().isEnabledFor(logging.DEBUG) + + # Avoid deleting the flag file when DEX_DEBUG is set in case the flag file + # needs to be examined after the build. + with tempfile.NamedTemporaryFile(mode='w', delete=not is_debug) as flag_file: # Chosen arbitrarily. Needed to avoid command-line length limits. MAX_ARGS = 50 + orig_dex_cmd = dex_cmd if len(dex_cmd) > MAX_ARGS: - flag_file.write('\n'.join(dex_cmd[MAX_ARGS:])) - flag_file.flush() - dex_cmd = dex_cmd[:MAX_ARGS] - dex_cmd.append('@' + flag_file.name) + # Add all flags to D8 (anything after the first --) as well as all + # positional args at the end to the flag file. + for idx, cmd in enumerate(dex_cmd): + if cmd.startswith('--'): + flag_file.write('\n'.join(dex_cmd[idx:])) + flag_file.flush() + dex_cmd = dex_cmd[:idx] + dex_cmd.append('@' + flag_file.name) + break # stdout sometimes spams with things like: # Stripped invalid locals information from 1 method. - build_utils.CheckOutput(dex_cmd, - stderr_filter=stderr_filter, - fail_on_output=warnings_as_errors) - - -def _EnvWithArtLibPath(binary_path): - """Return an environment dictionary for ART host shared libraries. - - Args: - binary_path: the path to an ART host binary. - - Returns: - An environment dictionary where LD_LIBRARY_PATH has been augmented with the - shared library path for the binary. This assumes that there is a lib/ - directory in the same location as the binary. - """ - lib_path = os.path.join(os.path.dirname(binary_path), 'lib') - env = os.environ.copy() - libraries = [l for l in env.get('LD_LIBRARY_PATH', '').split(':') if l] - libraries.append(lib_path) - env['LD_LIBRARY_PATH'] = ':'.join(libraries) - return env - - -def _CreateBinaryProfile(text_profile, input_dex, profman_path, temp_dir): - """Create a binary profile for dexlayout. - - Args: - text_profile: The ART text profile that will be converted to a binary - profile. - input_dex: The input dex file to layout. - profman_path: Path to the profman binary. - temp_dir: Directory to work in. - - Returns: - The name of the binary profile, which will live in temp_dir. - """ - binary_profile = os.path.join( - temp_dir, 'binary_profile-for-' + os.path.basename(text_profile)) - open(binary_profile, 'w').close() # Touch binary_profile. - profman_cmd = [profman_path, - '--apk=' + input_dex, - '--dex-location=' + input_dex, - '--create-profile-from=' + text_profile, - '--reference-profile-file=' + binary_profile] - build_utils.CheckOutput( - profman_cmd, - env=_EnvWithArtLibPath(profman_path), - stderr_filter=lambda output: - build_utils.FilterLines(output, '|'.join( - [r'Could not find (method_id|proto_id|name):', - r'Could not create type list']))) - return binary_profile - - -def _LayoutDex(binary_profile, input_dex, dexlayout_path, temp_dir): - """Layout a dexfile using a profile. - - Args: - binary_profile: An ART binary profile, eg output from _CreateBinaryProfile. - input_dex: The dex file used to create the binary profile. - dexlayout_path: Path to the dexlayout binary. - temp_dir: Directory to work in. - - Returns: - List of output files produced by dexlayout. This will be one if the input - was a single dexfile, or multiple files if the input was a multidex - zip. These output files are located in temp_dir. - """ - dexlayout_output_dir = os.path.join(temp_dir, 'dexlayout_output') - os.mkdir(dexlayout_output_dir) - dexlayout_cmd = [ dexlayout_path, - '-u', # Update checksum - '-p', binary_profile, - '-w', dexlayout_output_dir, - input_dex ] - build_utils.CheckOutput( - dexlayout_cmd, - env=_EnvWithArtLibPath(dexlayout_path), - stderr_filter=lambda output: - build_utils.FilterLines(output, - r'Can.t mmap dex file.*please zipalign')) - output_files = os.listdir(dexlayout_output_dir) - if not output_files: - raise Exception('dexlayout unexpectedly produced no output') - return sorted([os.path.join(dexlayout_output_dir, f) for f in output_files]) - - -def _ZipMultidex(file_dir, dex_files): - """Zip dex files into a multidex. - - Args: - file_dir: The directory into which to write the output. - dex_files: The dexfiles forming the multizip. Their names must end with - classes.dex, classes2.dex, ... - - Returns: - The name of the multidex file, which will live in file_dir. - """ - ordered_files = [] # List of (archive name, file name) - for f in dex_files: - if f.endswith('dex.jar'): - ordered_files.append(('classes.dex', f)) - break - if not ordered_files: - raise Exception('Could not find classes.dex multidex file in %s', - dex_files) - for dex_idx in xrange(2, len(dex_files) + 1): - archive_name = 'classes%d.dex' % dex_idx - for f in dex_files: - if f.endswith(archive_name): - ordered_files.append((archive_name, f)) - break - else: - raise Exception('Could not find classes%d.dex multidex file in %s', - dex_files) - if len(set(f[1] for f in ordered_files)) != len(ordered_files): - raise Exception('Unexpected clashing filenames for multidex in %s', - dex_files) - - zip_name = os.path.join(file_dir, 'multidex_classes.zip') - build_utils.DoZip(((archive_name, os.path.join(file_dir, file_name)) - for archive_name, file_name in ordered_files), - zip_name) - return zip_name + try: + build_utils.CheckOutput(dex_cmd, + stderr_filter=stderr_filter, + fail_on_output=warnings_as_errors) + except Exception: + if orig_dex_cmd is not dex_cmd: + sys.stderr.write('Full command: ' + shlex.join(orig_dex_cmd) + '\n') + raise def _ZipAligned(dex_files, output_path): @@ -349,31 +236,7 @@ with zipfile.ZipFile(output_path, 'w') as z: for i, dex_file in enumerate(dex_files): name = 'classes{}.dex'.format(i + 1 if i > 0 else '') - zipalign.AddToZipHermetic(z, name, src_path=dex_file, alignment=4) - - -def _PerformDexlayout(tmp_dir, tmp_dex_output, options): - if options.proguard_mapping_path is not None: - matching_profile = os.path.join(tmp_dir, 'obfuscated_profile') - convert_dex_profile.ObfuscateProfile( - options.dexlayout_profile, tmp_dex_output, - options.proguard_mapping_path, options.dexdump_path, matching_profile) - else: - logging.warning('No obfuscation for %s', options.dexlayout_profile) - matching_profile = options.dexlayout_profile - binary_profile = _CreateBinaryProfile(matching_profile, tmp_dex_output, - options.profman_path, tmp_dir) - output_files = _LayoutDex(binary_profile, tmp_dex_output, - options.dexlayout_path, tmp_dir) - if len(output_files) > 1: - return _ZipMultidex(tmp_dir, output_files) - - if zipfile.is_zipfile(output_files[0]): - return output_files[0] - - final_output = os.path.join(tmp_dir, 'dex_classes.zip') - _ZipAligned(output_files, final_output) - return final_output + zip_helpers.add_to_zip_hermetic(z, name, src_path=dex_file, alignment=4) def _CreateFinalDex(d8_inputs, output, tmp_dir, dex_cmd, options=None): @@ -406,9 +269,6 @@ _ZipAligned(sorted(d8_inputs), tmp_dex_output) logging.debug('Quick-zipped %d files', len(d8_inputs)) - if options and options.dexlayout_profile: - tmp_dex_output = _PerformDexlayout(tmp_dir, tmp_dex_output, options) - # The dex file is complete and can be moved out of tmp_dir. shutil.move(tmp_dex_output, output) @@ -419,7 +279,7 @@ for jar in class_inputs: with zipfile.ZipFile(jar, 'r') as z: for subpath in z.namelist(): - if subpath.endswith('.class'): + if _IsClassFile(subpath): subpath = subpath[:-5] + 'dex' dex_files.append(os.path.join(incremental_dir, subpath)) return dex_files @@ -435,12 +295,34 @@ def _ParseDesugarDeps(desugar_dependencies_file): + # pylint: disable=line-too-long + """Returns a dict of dependent/dependency mapping parsed from the file. + + Example file format: + $ tail out/Debug/gen/base/base_java__dex.desugardeps + org/chromium/base/task/SingleThreadTaskRunnerImpl.class + <- org/chromium/base/task/SingleThreadTaskRunner.class + <- org/chromium/base/task/TaskRunnerImpl.class + org/chromium/base/task/TaskRunnerImpl.class + <- org/chromium/base/task/TaskRunner.class + org/chromium/base/task/TaskRunnerImplJni$1.class + <- obj/base/jni_java.turbine.jar:org/chromium/base/JniStaticTestMocker.class + org/chromium/base/task/TaskRunnerImplJni.class + <- org/chromium/base/task/TaskRunnerImpl$Natives.class + """ + # pylint: enable=line-too-long dependents_from_dependency = collections.defaultdict(set) if desugar_dependencies_file and os.path.exists(desugar_dependencies_file): with open(desugar_dependencies_file, 'r') as f: + dependent = None for line in f: - dependent, dependency = line.rstrip().split(' -> ') - dependents_from_dependency[dependency].add(dependent) + line = line.rstrip() + if line.startswith(' <- '): + dependency = line[len(' <- '):] + # Note that this is a reversed mapping from the one in CustomD8.java. + dependents_from_dependency[dependency].add(dependent) + else: + dependent = line return dependents_from_dependency @@ -461,15 +343,21 @@ return required_classes +def _IsClassFile(path): + if os.path.basename(path) in _SKIPPED_CLASS_FILE_NAMES: + return False + return path.endswith('.class') + + def _ExtractClassFiles(changes, tmp_dir, class_inputs, required_classes_set): classes_list = [] for jar in class_inputs: if changes: changed_class_list = (set(changes.IterChangedSubpaths(jar)) | required_classes_set) - predicate = lambda x: x in changed_class_list and x.endswith('.class') + predicate = lambda x: x in changed_class_list and _IsClassFile(x) else: - predicate = lambda x: x.endswith('.class') + predicate = _IsClassFile classes_list.extend( build_utils.ExtractAll(jar, path=tmp_dir, predicate=predicate)) @@ -494,14 +382,14 @@ strings_changed, non_direct_input_changed) changes = None - if changes: + if changes is None: + required_desugar_classes_set = set() + else: required_desugar_classes_set = _ComputeRequiredDesugarClasses( changes, options.desugar_dependencies, options.class_inputs, options.classpath) logging.debug('Class files needing re-desugar: %d', len(required_desugar_classes_set)) - else: - required_desugar_classes_set = set() class_files = _ExtractClassFiles(changes, tmp_extract_dir, options.class_inputs, required_desugar_classes_set) @@ -512,7 +400,13 @@ # Dex necessary classes into intermediate dex files. dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file'] if options.desugar_dependencies and not options.skip_custom_d8: - dex_cmd += ['--file-tmp-prefix', tmp_extract_dir] + # Adding os.sep to remove the entire prefix. + dex_cmd += ['--file-tmp-prefix', tmp_extract_dir + os.sep] + if changes is None and os.path.exists(options.desugar_dependencies): + # Since incremental dexing only ever adds to the desugar_dependencies + # file, whenever full dexes are required the .desugardeps files need to + # be manually removed. + os.unlink(options.desugar_dependencies) _RunD8(dex_cmd, class_files, options.incremental_dir, options.warnings_as_errors, options.show_desugar_default_interface_warnings) @@ -537,7 +431,7 @@ def MergeDexForIncrementalInstall(r8_jar_path, src_paths, dest_dex_jar, min_api): - dex_cmd = build_utils.JavaCmd(verify=False) + [ + dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) + [ '-cp', r8_jar_path, 'com.android.tools.r8.D8', @@ -575,7 +469,7 @@ final_dex_inputs = list(options.class_inputs) final_dex_inputs += options.dex_inputs - dex_cmd = build_utils.JavaCmd(options.warnings_as_errors) + dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) if options.dump_inputs: dex_cmd += ['-Dcom.android.tools.r8.dumpinputtofile=d8inputs.zip'] @@ -622,8 +516,8 @@ input_paths += options.bootclasspath - if options.desugar_jdk_libs_json: - dex_cmd += ['--desugared-lib', options.desugar_jdk_libs_json] + if options.assertion_handler: + dex_cmd += ['--force-assertions-handler:' + options.assertion_handler] if options.force_enable_assertions: dex_cmd += ['--force-enable-assertions'] @@ -633,7 +527,7 @@ lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, dex_cmd), options, input_paths=input_paths, - input_strings=dex_cmd + [bool(options.incremental_dir)], + input_strings=dex_cmd + [str(bool(options.incremental_dir))], output_paths=output_paths, pass_changes=True, track_subpaths_allowlist=track_subpaths_allowlist,
diff --git a/build/android/gyp/dex.pydeps b/build/android/gyp/dex.pydeps index 23856f3..d920e24 100644 --- a/build/android/gyp/dex.pydeps +++ b/build/android/gyp/dex.pydeps
@@ -1,10 +1,10 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dex.pydeps build/android/gyp/dex.py +../../action_helpers.py ../../gn_helpers.py ../../print_python_deps.py -../convert_dex_profile.py +../../zip_helpers.py dex.py util/__init__.py util/build_utils.py util/md5_check.py -util/zipalign.py
diff --git a/build/android/gyp/dex_jdk_libs.py b/build/android/gyp/dex_jdk_libs.py deleted file mode 100755 index 6304779..0000000 --- a/build/android/gyp/dex_jdk_libs.py +++ /dev/null
@@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2020 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import argparse -import os -import shutil -import subprocess -import sys -import zipfile - -from util import build_utils - - -def _ParseArgs(args): - args = build_utils.ExpandFileArgs(args) - parser = argparse.ArgumentParser() - - parser.add_argument('--output', required=True, help='Dex output path.') - parser.add_argument('--r8-path', required=True, help='Path to R8 jar.') - parser.add_argument( - '--desugar-jdk-libs-json', help='Path to desugar_jdk_libs.json.') - parser.add_argument( - '--desugar-jdk-libs-jar', help='Path to desugar_jdk_libs.jar.') - parser.add_argument('--desugar-jdk-libs-configuration-jar', - help='Path to desugar_jdk_libs_configuration.jar.') - parser.add_argument('--min-api', help='minSdkVersion', required=True) - parser.add_argument('--warnings-as-errors', - action='store_true', - help='Treat all warnings as errors.') - options = parser.parse_args(args) - return options - - -def DexJdkLibJar(r8_path, - min_api, - desugar_jdk_libs_json, - desugar_jdk_libs_jar, - desugar_jdk_libs_configuration_jar, - output, - warnings_as_errors, - config_paths=None): - # TODO(agrieve): Spews a lot of stderr about missing classes. - with build_utils.TempDir() as tmp_dir: - cmd = build_utils.JavaCmd(warnings_as_errors) + [ - '-cp', - r8_path, - 'com.android.tools.r8.L8', - '--min-api', - min_api, - '--lib', - build_utils.JAVA_HOME, - '--desugared-lib', - desugar_jdk_libs_json, - ] - - # If no desugaring is required, no keep rules are generated, and the keep - # file will not be created. - if config_paths is not None: - for path in config_paths: - cmd += ['--pg-conf', path] - - cmd += [ - '--output', tmp_dir, desugar_jdk_libs_jar, - desugar_jdk_libs_configuration_jar - ] - - build_utils.CheckOutput(cmd, - print_stdout=True, - fail_on_output=warnings_as_errors) - if os.path.exists(os.path.join(tmp_dir, 'classes2.dex')): - raise Exception('Achievement unlocked: desugar_jdk_libs is multidex!') - - # classes.dex might not exists if the "desugar_jdk_libs_jar" is not used - # at all. - if os.path.exists(os.path.join(tmp_dir, 'classes.dex')): - shutil.move(os.path.join(tmp_dir, 'classes.dex'), output) - return True - return False - - -def main(args): - options = _ParseArgs(args) - DexJdkLibJar(options.r8_path, options.min_api, options.desugar_jdk_libs_json, - options.desugar_jdk_libs_jar, - options.desugar_jdk_libs_configuration_jar, options.output, - options.warnings_as_errors) - - -if __name__ == '__main__': - main(sys.argv[1:])
diff --git a/build/android/gyp/dex_jdk_libs.pydeps b/build/android/gyp/dex_jdk_libs.pydeps deleted file mode 100644 index 28d181f..0000000 --- a/build/android/gyp/dex_jdk_libs.pydeps +++ /dev/null
@@ -1,6 +0,0 @@ -# Generated by running: -# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dex_jdk_libs.pydeps build/android/gyp/dex_jdk_libs.py -../../gn_helpers.py -dex_jdk_libs.py -util/__init__.py -util/build_utils.py
diff --git a/build/android/gyp/dex_test.py b/build/android/gyp/dex_test.py new file mode 100755 index 0000000..5042e5f --- /dev/null +++ b/build/android/gyp/dex_test.py
@@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import unittest + +import dex + + +class DexTest(unittest.TestCase): + def testStdErrFilter(self): + # pylint: disable=line-too-long + output = """\ +some initial message +Warning: Specification conversion: The following prefixes do not match any type: [Ljava/util/Desugar] +Warning in ../../clank/third_party/google3/pg_confs/java_com_google_protobuf_lite_proguard.pgcfg: +Rule matches the static final field `java.lang.String com.google.protobuf.BaseGeneratedExtensionRegistryLite.CONTAINING_TYPE_0`, which may have been inlined: -identifiernamestring class com.google.protobuf.*GeneratedExtensionRegistryLite { + static java.lang.String CONTAINING_TYPE_*; +} +Warning: some message +Warning in gen/.../Foo.jar:Bar.class: + Type `libcore.io.Memory` was not found, it is required for default or static interface methods desugaring of `void Bar.a(long, byte)` +Warning: Missing class com.google.android.apps.gsa.search.shared.service.proto.PublicStopClientEvent (referenced from: com.google.protobuf.GeneratedMessageLite$GeneratedExtension com.google.protobuf.BaseGeneratedExtensionRegistryLite.findLiteExtensionByNumber(com.google.protobuf.MessageLite, int)) +Missing class com.google.android.gms.feedback.ApplicationProperties (referenced from: com.google.protobuf.GeneratedMessageLite$GeneratedExtension com.google.protobuf.BaseGeneratedExtensionRegistryLite.findLiteExtensionByNumber(com.google.protobuf.MessageLite, int)) +""" + expected = """\ +some initial message +Warning: some message +""" + # pylint: enable=line-too-long + filter_func = dex.CreateStderrFilter( + show_desugar_default_interface_warnings=False) + self.assertEqual(filter_func(output), expected) + + # Test no preamble, not filtered. + output = """Warning: hi""" + expected = output + self.assertEqual(filter_func(output), expected) + + # Test no preamble, filtered + output = """\ +Warning: Specification conversion: The following prefixes do not ... +""" + expected = '' + self.assertEqual(filter_func(output), expected) + + +if __name__ == '__main__': + unittest.main()
diff --git a/build/android/gyp/dexsplitter.py b/build/android/gyp/dexsplitter.py deleted file mode 100755 index 149e994..0000000 --- a/build/android/gyp/dexsplitter.py +++ /dev/null
@@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import argparse -import os -import shutil -import sys -import zipfile - -from util import build_utils - - -def _ParseOptions(args): - parser = argparse.ArgumentParser() - parser.add_argument('--depfile', help='Path to the depfile to write to.') - parser.add_argument('--stamp', help='Path to stamp to mark when finished.') - parser.add_argument('--r8-path', help='Path to the r8.jar to use.') - parser.add_argument( - '--input-dex-zip', help='Path to dex files in zip being split.') - parser.add_argument( - '--proguard-mapping-file', help='Path to proguard mapping file.') - parser.add_argument( - '--feature-name', - action='append', - dest='feature_names', - help='The name of the feature module.') - parser.add_argument( - '--feature-jars', - action='append', - help='GN list of path to jars which compirse the corresponding feature.') - parser.add_argument( - '--dex-dest', - action='append', - dest='dex_dests', - help='Destination for dex file of the corresponding feature.') - options = parser.parse_args(args) - - assert len(options.feature_names) == len(options.feature_jars) and len( - options.feature_names) == len(options.dex_dests) - options.features = {} - for i, name in enumerate(options.feature_names): - options.features[name] = build_utils.ParseGnList(options.feature_jars[i]) - - return options - - -def _RunDexsplitter(options, output_dir): - cmd = build_utils.JavaCmd() + [ - '-cp', - options.r8_path, - 'com.android.tools.r8.dexsplitter.DexSplitter', - '--output', - output_dir, - '--proguard-map', - options.proguard_mapping_file, - ] - - for base_jar in options.features['base']: - cmd += ['--base-jar', base_jar] - - base_jars_lookup = set(options.features['base']) - for feature in options.features: - if feature == 'base': - continue - for feature_jar in options.features[feature]: - if feature_jar not in base_jars_lookup: - cmd += ['--feature-jar', feature_jar + ':' + feature] - - with build_utils.TempDir() as temp_dir: - unzipped_files = build_utils.ExtractAll(options.input_dex_zip, temp_dir) - for file_name in unzipped_files: - cmd += ['--input', file_name] - build_utils.CheckOutput(cmd) - - -def main(args): - args = build_utils.ExpandFileArgs(args) - options = _ParseOptions(args) - - input_paths = [options.input_dex_zip] - for feature_jars in options.features.itervalues(): - for feature_jar in feature_jars: - input_paths.append(feature_jar) - - with build_utils.TempDir() as dexsplitter_output_dir: - curr_location_to_dest = [] - if len(options.features) == 1: - # Don't run dexsplitter since it needs at least 1 feature module. - curr_location_to_dest.append((options.input_dex_zip, - options.dex_dests[0])) - else: - _RunDexsplitter(options, dexsplitter_output_dir) - - for i, dest in enumerate(options.dex_dests): - module_dex_file = os.path.join(dexsplitter_output_dir, - options.feature_names[i], 'classes.dex') - if os.path.exists(module_dex_file): - curr_location_to_dest.append((module_dex_file, dest)) - else: - module_dex_file += '.jar' - assert os.path.exists( - module_dex_file), 'Dexsplitter tool output not found.' - curr_location_to_dest.append((module_dex_file + '.jar', dest)) - - for curr_location, dest in curr_location_to_dest: - with build_utils.AtomicOutput(dest) as f: - if curr_location.endswith('.jar'): - if dest.endswith('.jar'): - shutil.copy(curr_location, f.name) - else: - with zipfile.ZipFile(curr_location, 'r') as z: - namelist = z.namelist() - assert len(namelist) == 1, ( - 'Unzipping to single dex file, but not single dex file in ' + - options.input_dex_zip) - z.extract(namelist[0], f.name) - else: - if dest.endswith('.jar'): - build_utils.ZipDir( - f.name, os.path.abspath(os.path.join(curr_location, os.pardir))) - else: - shutil.move(curr_location, f.name) - - build_utils.Touch(options.stamp) - build_utils.WriteDepfile(options.depfile, options.stamp, inputs=input_paths) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:]))
diff --git a/build/android/gyp/dexsplitter.pydeps b/build/android/gyp/dexsplitter.pydeps deleted file mode 100644 index cefc572..0000000 --- a/build/android/gyp/dexsplitter.pydeps +++ /dev/null
@@ -1,6 +0,0 @@ -# Generated by running: -# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dexsplitter.pydeps build/android/gyp/dexsplitter.py -../../gn_helpers.py -dexsplitter.py -util/__init__.py -util/build_utils.py
diff --git a/build/android/gyp/dist_aar.py b/build/android/gyp/dist_aar.py index 7f0de1d..507d0c3 100755 --- a/build/android/gyp/dist_aar.py +++ b/build/android/gyp/dist_aar.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,6 +16,8 @@ import filter_zip from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers _ANDROID_BUILD_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -65,7 +67,7 @@ def main(args): args = build_utils.ExpandFileArgs(args) parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--output', required=True, help='Path to output aar.') parser.add_argument('--jars', required=True, help='GN list of jar inputs.') parser.add_argument('--dependencies-res-zips', required=True, @@ -98,52 +100,60 @@ if options.native_libraries and not options.abi: parser.error('You must provide --abi if you have native libs') - options.jars = build_utils.ParseGnList(options.jars) - options.dependencies_res_zips = build_utils.ParseGnList( + options.jars = action_helpers.parse_gn_list(options.jars) + options.dependencies_res_zips = action_helpers.parse_gn_list( options.dependencies_res_zips) - options.r_text_files = build_utils.ParseGnList(options.r_text_files) - options.proguard_configs = build_utils.ParseGnList(options.proguard_configs) - options.native_libraries = build_utils.ParseGnList(options.native_libraries) - options.jar_excluded_globs = build_utils.ParseGnList( + options.r_text_files = action_helpers.parse_gn_list(options.r_text_files) + options.proguard_configs = action_helpers.parse_gn_list( + options.proguard_configs) + options.native_libraries = action_helpers.parse_gn_list( + options.native_libraries) + options.jar_excluded_globs = action_helpers.parse_gn_list( options.jar_excluded_globs) - options.jar_included_globs = build_utils.ParseGnList( + options.jar_included_globs = action_helpers.parse_gn_list( options.jar_included_globs) - options.resource_included_globs = build_utils.ParseGnList( + options.resource_included_globs = action_helpers.parse_gn_list( options.resource_included_globs) with tempfile.NamedTemporaryFile(delete=False) as staging_file: try: with zipfile.ZipFile(staging_file.name, 'w') as z: - build_utils.AddToZipHermetic( - z, 'AndroidManifest.xml', src_path=options.android_manifest) + zip_helpers.add_to_zip_hermetic(z, + 'AndroidManifest.xml', + src_path=options.android_manifest) path_transform = filter_zip.CreatePathTransform( - options.jar_excluded_globs, options.jar_included_globs, []) + options.jar_excluded_globs, options.jar_included_globs) with tempfile.NamedTemporaryFile() as jar_file: - build_utils.MergeZips( - jar_file.name, options.jars, path_transform=path_transform) - build_utils.AddToZipHermetic(z, 'classes.jar', src_path=jar_file.name) + zip_helpers.merge_zips(jar_file.name, + options.jars, + path_transform=path_transform) + zip_helpers.add_to_zip_hermetic(z, + 'classes.jar', + src_path=jar_file.name) - build_utils.AddToZipHermetic( - z, - 'R.txt', - data=_MergeRTxt(options.r_text_files, - options.resource_included_globs)) - build_utils.AddToZipHermetic(z, 'public.txt', data='') + zip_helpers.add_to_zip_hermetic(z, + 'R.txt', + data=_MergeRTxt( + options.r_text_files, + options.resource_included_globs)) + zip_helpers.add_to_zip_hermetic(z, 'public.txt', data='') if options.proguard_configs: - build_utils.AddToZipHermetic( - z, 'proguard.txt', - data=_MergeProguardConfigs(options.proguard_configs)) + zip_helpers.add_to_zip_hermetic(z, + 'proguard.txt', + data=_MergeProguardConfigs( + options.proguard_configs)) _AddResources(z, options.dependencies_res_zips, options.resource_included_globs) for native_library in options.native_libraries: libname = os.path.basename(native_library) - build_utils.AddToZipHermetic( - z, os.path.join('jni', options.abi, libname), - src_path=native_library) + zip_helpers.add_to_zip_hermetic(z, + os.path.join('jni', options.abi, + libname), + src_path=native_library) except: os.unlink(staging_file.name) raise @@ -152,7 +162,7 @@ if options.depfile: all_inputs = (options.jars + options.dependencies_res_zips + options.r_text_files + options.proguard_configs) - build_utils.WriteDepfile(options.depfile, options.output, all_inputs) + action_helpers.write_depfile(options.depfile, options.output, all_inputs) if __name__ == '__main__':
diff --git a/build/android/gyp/dist_aar.pydeps b/build/android/gyp/dist_aar.pydeps index 3182580..ba0dd52 100644 --- a/build/android/gyp/dist_aar.pydeps +++ b/build/android/gyp/dist_aar.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/dist_aar.pydeps build/android/gyp/dist_aar.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py dist_aar.py filter_zip.py util/__init__.py
diff --git a/build/android/gyp/extract_unwind_tables.py b/build/android/gyp/extract_unwind_tables.py index 25c3130..de0f016 100755 --- a/build/android/gyp/extract_unwind_tables.py +++ b/build/android/gyp/extract_unwind_tables.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,7 +10,7 @@ The output file is a binary file containing CFI rows ordered based on function address. The output file only contains rows that match the most popular rule type in CFI table, to reduce the output size and specify data in compact format. -See doc https://github.com/google/breakpad/blob/master/docs/symbol_files.md. +See doc https://github.com/google/breakpad/blob/main/docs/symbol_files.md. 1. The CFA rules should be of postfix form "SP <val> +". 2. The RA rules should be of postfix form "CFA <val> + ^". Note: breakpad represents dereferencing address with '^' operator. @@ -255,12 +255,6 @@ _Write2Bytes(out_file, data) -def _ParseCfiData(sym_stream, output_path): - cfi_data = _GetAllCfiRows(sym_stream) - with open(output_path, 'wb') as out_file: - _WriteCfiData(cfi_data, out_file) - - def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -274,12 +268,16 @@ help='The path of the dump_syms binary') args = parser.parse_args() - cmd = ['./' + args.dump_syms_path, args.input_path] - proc = subprocess.Popen(cmd, bufsize=-1, stdout=subprocess.PIPE) - _ParseCfiData(proc.stdout, args.output_path) - assert proc.wait() == 0 + cmd = ['./' + args.dump_syms_path, args.input_path, '-v'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + cfi_data = _GetAllCfiRows(proc.stdout) + if proc.wait(): + sys.stderr.write('dump_syms exited with code {} after {} symbols\n'.format( + proc.returncode, len(cfi_data))) + sys.exit(proc.returncode) + with open(args.output_path, 'wb') as out_file: + _WriteCfiData(cfi_data, out_file) - return 0 if __name__ == '__main__': - sys.exit(main()) + main()
diff --git a/build/android/gyp/extract_unwind_tables_tests.py b/build/android/gyp/extract_unwind_tables_tests.py index 59436ff..dd716bf 100755 --- a/build/android/gyp/extract_unwind_tables_tests.py +++ b/build/android/gyp/extract_unwind_tables_tests.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,6 +9,7 @@ symbol files. """ +import io import optparse import os import struct @@ -24,8 +25,7 @@ class TestExtractUnwindTables(unittest.TestCase): def testExtractCfi(self): - with tempfile.NamedTemporaryFile() as output_file: - test_data_lines = """ + test_data_lines = """ MODULE Linux arm CDE12FE1DF2B37A9C6560B4CBEE056420 lib_chrome.so INFO CODE_ID E12FE1CD2BDFA937C6560B4CBEE05642 FILE 0 ../../base/allocator/allocator_check.cc @@ -63,57 +63,58 @@ STACK CFI INIT 3b93214 fffff .cfa: sp 0 + .ra: lr STACK CFI 3b93218 .cfa: r7 16 + .ra: .cfa -4 + ^ """.splitlines() - extract_unwind_tables._ParseCfiData( - [l.encode('utf8') for l in test_data_lines], output_file.name) + cfi_data = extract_unwind_tables._GetAllCfiRows( + [l.encode('utf8') for l in test_data_lines]) + out_file = io.BytesIO() + extract_unwind_tables._WriteCfiData(cfi_data, out_file) - expected_cfi_data = { - 0xe1a1e4 : [0x2, 0x11, 0x4, 0x50], - 0xe1a296 : [], - 0xe1a96e : [0x2, 0x4, 0x4, 0xe, 0x6, 0x10], - 0xe1a990 : [], + expected_cfi_data = { + 0xe1a1e4: [0x2, 0x11, 0x4, 0x50], + 0xe1a296: [], + 0xe1a96e: [0x2, 0x4, 0x4, 0xe, 0x6, 0x10], + 0xe1a990: [], 0x3b92e24: [0x28, 0x13], 0x3b92e62: [], - } - expected_function_count = len(expected_cfi_data) + } + expected_function_count = len(expected_cfi_data) - actual_output = [] - with open(output_file.name, 'rb') as f: - while True: - read = f.read(2) - if not read: - break - actual_output.append(struct.unpack('H', read)[0]) + actual_output = [] + out_file.seek(0) + while True: + read = out_file.read(2) + if not read: + break + actual_output.append(struct.unpack('H', read)[0]) - # First value is size of unw_index table. - unw_index_size = actual_output[1] << 16 | actual_output[0] - # |unw_index_size| should match entry count. - self.assertEqual(expected_function_count, unw_index_size) - # |actual_output| is in blocks of 2 bytes. Skip first 4 bytes representing - # size. - unw_index_start = 2 - unw_index_addr_end = unw_index_start + expected_function_count * 2 - unw_index_end = unw_index_addr_end + expected_function_count - unw_index_addr_col = actual_output[unw_index_start : unw_index_addr_end] - unw_index_index_col = actual_output[unw_index_addr_end : unw_index_end] + # First value is size of unw_index table. + unw_index_size = actual_output[1] << 16 | actual_output[0] + # |unw_index_size| should match entry count. + self.assertEqual(expected_function_count, unw_index_size) + # |actual_output| is in blocks of 2 bytes. Skip first 4 bytes representing + # size. + unw_index_start = 2 + unw_index_addr_end = unw_index_start + expected_function_count * 2 + unw_index_end = unw_index_addr_end + expected_function_count + unw_index_addr_col = actual_output[unw_index_start:unw_index_addr_end] + unw_index_index_col = actual_output[unw_index_addr_end:unw_index_end] - unw_data_start = unw_index_end - unw_data = actual_output[unw_data_start:] + unw_data_start = unw_index_end + unw_data = actual_output[unw_data_start:] - for func_iter in range(0, expected_function_count): - func_addr = (unw_index_addr_col[func_iter * 2 + 1] << 16 | - unw_index_addr_col[func_iter * 2]) - index = unw_index_index_col[func_iter] - # If index is CANT_UNWIND then invalid function. - if index == 0xFFFF: - self.assertEqual(expected_cfi_data[func_addr], []) - continue + for func_iter in range(0, expected_function_count): + func_addr = (unw_index_addr_col[func_iter * 2 + 1] << 16 + | unw_index_addr_col[func_iter * 2]) + index = unw_index_index_col[func_iter] + # If index is CANT_UNWIND then invalid function. + if index == 0xFFFF: + self.assertEqual(expected_cfi_data[func_addr], []) + continue - func_start = index + 1 - func_end = func_start + unw_data[index] * 2 - self.assertEqual(len(expected_cfi_data[func_addr]), - func_end - func_start) - func_cfi = unw_data[func_start : func_end] - self.assertEqual(expected_cfi_data[func_addr], func_cfi) + func_start = index + 1 + func_end = func_start + unw_data[index] * 2 + self.assertEqual(len(expected_cfi_data[func_addr]), func_end - func_start) + func_cfi = unw_data[func_start:func_end] + self.assertEqual(expected_cfi_data[func_addr], func_cfi) if __name__ == '__main__':
diff --git a/build/android/gyp/filter_zip.py b/build/android/gyp/filter_zip.py index 068ff03..0382651 100755 --- a/build/android/gyp/filter_zip.py +++ b/build/android/gyp/filter_zip.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,38 +9,25 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers -_RESOURCE_CLASSES = [ - "R.class", - "R##*.class", - "Manifest.class", - "Manifest##*.class", -] - - -def CreatePathTransform(exclude_globs, include_globs, - strip_resource_classes_for): +def CreatePathTransform(exclude_globs, include_globs): """Returns a function to strip paths for the given patterns. Args: exclude_globs: List of globs that if matched should be excluded. include_globs: List of globs that if not matched should be excluded. - strip_resource_classes_for: List of Java packages for which to strip - R.java classes from. Returns: * None if no filters are needed. * A function "(path) -> path" that returns None when |path| should be stripped, or |path| otherwise. """ - if not (exclude_globs or include_globs or strip_resource_classes_for): + if not (exclude_globs or include_globs): return None exclude_globs = list(exclude_globs or []) - if strip_resource_classes_for: - exclude_globs.extend(p.replace('.', '/') + '/' + f - for p in strip_resource_classes_for - for f in _RESOURCE_CLASSES) def path_transform(path): # Exclude filters take precidence over include filters. if build_utils.MatchesGlob(path, exclude_globs): @@ -62,22 +49,17 @@ help='GN list of exclude globs') parser.add_argument('--include-globs', help='GN list of include globs') - parser.add_argument('--strip-resource-classes-for', - help='GN list of java package names exclude R.class files in.') - argv = build_utils.ExpandFileArgs(sys.argv[1:]) args = parser.parse_args(argv) - args.exclude_globs = build_utils.ParseGnList(args.exclude_globs) - args.include_globs = build_utils.ParseGnList(args.include_globs) - args.strip_resource_classes_for = build_utils.ParseGnList( - args.strip_resource_classes_for) + args.exclude_globs = action_helpers.parse_gn_list(args.exclude_globs) + args.include_globs = action_helpers.parse_gn_list(args.include_globs) - path_transform = CreatePathTransform(args.exclude_globs, args.include_globs, - args.strip_resource_classes_for) - with build_utils.AtomicOutput(args.output) as f: + path_transform = CreatePathTransform(args.exclude_globs, args.include_globs) + with action_helpers.atomic_output(args.output) as f: if path_transform: - build_utils.MergeZips(f.name, [args.input], path_transform=path_transform) + zip_helpers.merge_zips(f.name, [args.input], + path_transform=path_transform) else: shutil.copy(args.input, f.name)
diff --git a/build/android/gyp/filter_zip.pydeps b/build/android/gyp/filter_zip.pydeps index f561e05..4905fd5 100644 --- a/build/android/gyp/filter_zip.pydeps +++ b/build/android/gyp/filter_zip.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/filter_zip.pydeps build/android/gyp/filter_zip.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py filter_zip.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/finalize_apk.py b/build/android/gyp/finalize_apk.py index b465f71..aaf66c2 100644 --- a/build/android/gyp/finalize_apk.py +++ b/build/android/gyp/finalize_apk.py
@@ -1,4 +1,4 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Signs and aligns an APK.""" @@ -38,7 +38,7 @@ else: signer_input_path = unsigned_apk_path - sign_cmd = build_utils.JavaCmd(warnings_as_errors) + [ + sign_cmd = build_utils.JavaCmd() + [ '-jar', apksigner_path, 'sign',
diff --git a/build/android/gyp/find.py b/build/android/gyp/find.py index b05874b..617efef 100755 --- a/build/android/gyp/find.py +++ b/build/android/gyp/find.py
@@ -1,13 +1,12 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Finds files in directories. """ -from __future__ import print_function import fnmatch import optparse @@ -27,6 +26,7 @@ for root, _, filenames in os.walk(d): for f in fnmatch.filter(filenames, options.pattern): print(os.path.join(root, f)) + return 0 if __name__ == '__main__':
diff --git a/build/android/gyp/flatc_java.py b/build/android/gyp/flatc_java.py new file mode 100755 index 0000000..003f820 --- /dev/null +++ b/build/android/gyp/flatc_java.py
@@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Generate java source files from flatbuffer files. + +This is the action script for the flatbuffer_java_library template. +""" + +import argparse +import sys + +from util import build_utils +import action_helpers +import zip_helpers + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('--flatc', required=True, help='Path to flatc binary.') + parser.add_argument('--srcjar', required=True, help='Path to output srcjar.') + parser.add_argument( + '--import-dir', + action='append', + default=[], + help='Extra import directory for flatbuffers, can be repeated.') + parser.add_argument('flatbuffers', nargs='+', help='flatbuffer source files') + options = parser.parse_args(argv) + + import_args = [] + for path in options.import_dir: + import_args += ['-I', path] + with build_utils.TempDir() as temp_dir: + build_utils.CheckOutput([options.flatc, '-j', '-o', temp_dir] + + import_args + options.flatbuffers) + + with action_helpers.atomic_output(options.srcjar) as f: + zip_helpers.zip_directory(f, temp_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))
diff --git a/build/android/gyp/flatc_java.pydeps b/build/android/gyp/flatc_java.pydeps new file mode 100644 index 0000000..8c0c4f0 --- /dev/null +++ b/build/android/gyp/flatc_java.pydeps
@@ -0,0 +1,8 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/flatc_java.pydeps build/android/gyp/flatc_java.py +../../action_helpers.py +../../gn_helpers.py +../../zip_helpers.py +flatc_java.py +util/__init__.py +util/build_utils.py
diff --git a/build/android/gyp/gcc_preprocess.py b/build/android/gyp/gcc_preprocess.py index 70ae10f..2e5b3b3 100755 --- a/build/android/gyp/gcc_preprocess.py +++ b/build/android/gyp/gcc_preprocess.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,6 +12,8 @@ import zipfile from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers def _ParsePackageName(data): @@ -32,8 +34,8 @@ parser.add_argument('templates', nargs='+', help='Template files.') options = parser.parse_args(args) - options.defines = build_utils.ParseGnList(options.defines) - options.include_dirs = build_utils.ParseGnList(options.include_dirs) + options.defines = action_helpers.parse_gn_list(options.defines) + options.include_dirs = action_helpers.parse_gn_list(options.include_dirs) gcc_cmd = [ 'gcc', @@ -46,7 +48,7 @@ gcc_cmd.extend('-D' + x for x in options.defines) gcc_cmd.extend('-I' + x for x in options.include_dirs) - with build_utils.AtomicOutput(options.output) as f: + with action_helpers.atomic_output(options.output) as f: with zipfile.ZipFile(f, 'w') as z: for template in options.templates: data = build_utils.CheckOutput(gcc_cmd + [template]) @@ -56,7 +58,7 @@ zip_path = posixpath.join( package_name.replace('.', '/'), os.path.splitext(os.path.basename(template))[0]) + '.java' - build_utils.AddToZipHermetic(z, zip_path, data=data) + zip_helpers.add_to_zip_hermetic(z, zip_path, data=data) if __name__ == '__main__':
diff --git a/build/android/gyp/gcc_preprocess.pydeps b/build/android/gyp/gcc_preprocess.pydeps index 39e56f7..b57d400 100644 --- a/build/android/gyp/gcc_preprocess.pydeps +++ b/build/android/gyp/gcc_preprocess.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/gcc_preprocess.pydeps build/android/gyp/gcc_preprocess.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py gcc_preprocess.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/generate_android_wrapper.py b/build/android/gyp/generate_android_wrapper.py index c8b762c..46c7afe 100755 --- a/build/android/gyp/generate_android_wrapper.py +++ b/build/android/gyp/generate_android_wrapper.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,6 +8,7 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. sys.path.append( os.path.abspath( @@ -23,7 +24,7 @@ for arg in args: m = _WRAPPED_PATH_LIST_RE.match(arg) if m: - for p in build_utils.ParseGnList(m.group(2)): + for p in action_helpers.parse_gn_list(m.group(2)): expanded_args.extend([m.group(1), '@WrappedPath(%s)' % p]) else: expanded_args.append(arg)
diff --git a/build/android/gyp/generate_linker_version_script.py b/build/android/gyp/generate_linker_version_script.py index 995fcd7..4f34457 100755 --- a/build/android/gyp/generate_linker_version_script.py +++ b/build/android/gyp/generate_linker_version_script.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Generate linker version scripts for Chrome on Android shared libraries.""" @@ -8,6 +8,7 @@ import os from util import build_utils +import action_helpers # build_utils adds //build to sys.path. _SCRIPT_HEADER = """\ # AUTO-GENERATED FILE. DO NOT MODIFY. @@ -32,9 +33,12 @@ required=True, help='Path to output linker version script file.') parser.add_argument( - '--export-java-symbols', + '--jni-multiplexing', action='store_true', - help='Export Java_* JNI methods') + help='Export only the JNI methods generated by multiplexing') + parser.add_argument('--export-fortesting-java-symbols', + action='store_true', + help='Export Java_*_ForTesting JNI methods') parser.add_argument( '--export-symbol-allowlist-file', action='append', @@ -53,8 +57,30 @@ # for libcrashpad_handler_trampoline.so. symbol_list = ['CrashpadHandlerMain', 'JNI_OnLoad'] - if options.export_java_symbols: + if options.jni_multiplexing: + symbol_list.append('Java_*_resolve_1for_*') + elif options.export_fortesting_java_symbols: symbol_list.append('Java_*') + else: + # The linker uses unix shell globbing patterns, not regex. So, we have to + # include everything that doesn't end in "ForTest(ing)" with this set of + # globs. + symbol_list.append('Java_*[!F]orTesting') + symbol_list.append('Java_*[!o]rTesting') + symbol_list.append('Java_*[!r]Testing') + symbol_list.append('Java_*[!T]esting') + symbol_list.append('Java_*[!e]sting') + symbol_list.append('Java_*[!s]ting') + symbol_list.append('Java_*[!t]ing') + symbol_list.append('Java_*[!i]ng') + symbol_list.append('Java_*[!n]g') + symbol_list.append('Java_*[!F]orTest') + symbol_list.append('Java_*[!o]rTest') + symbol_list.append('Java_*[!r]Test') + symbol_list.append('Java_*[!T]est') + symbol_list.append('Java_*[!e]st') + symbol_list.append('Java_*[!s]t') + symbol_list.append('Java_*[!gt]') if options.export_feature_registrations: symbol_list.append('JNI_OnLoad_*') @@ -74,7 +100,7 @@ script = ''.join(script_content) - with build_utils.AtomicOutput(options.output, mode='w') as f: + with action_helpers.atomic_output(options.output, mode='w') as f: f.write(script)
diff --git a/build/android/gyp/generate_linker_version_script.pydeps b/build/android/gyp/generate_linker_version_script.pydeps index de9fa56..03ac25d 100644 --- a/build/android/gyp/generate_linker_version_script.pydeps +++ b/build/android/gyp/generate_linker_version_script.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/generate_linker_version_script.pydeps build/android/gyp/generate_linker_version_script.py +../../action_helpers.py ../../gn_helpers.py generate_linker_version_script.py util/__init__.py
diff --git a/build/android/gyp/ijar.py b/build/android/gyp/ijar.py index 45413f6..ec12cec 100755 --- a/build/android/gyp/ijar.py +++ b/build/android/gyp/ijar.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,6 +10,7 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. # python -c "import zipfile; zipfile.ZipFile('test.jar', 'w')" @@ -20,8 +21,10 @@ def main(): # The point of this wrapper is to use AtomicOutput so that output timestamps # are not updated when outputs are unchanged. - ijar_bin, in_jar, out_jar = sys.argv[1:] - with build_utils.AtomicOutput(out_jar) as f: + if len(sys.argv) != 4: + raise ValueError('unexpected arguments were given. %s' % sys.argv) + ijar_bin, in_jar, out_jar = sys.argv[1], sys.argv[2], sys.argv[3] + with action_helpers.atomic_output(out_jar) as f: # ijar fails on empty jars: https://github.com/bazelbuild/bazel/issues/10162 if os.path.getsize(in_jar) <= _EMPTY_JAR_SIZE: with open(in_jar, 'rb') as in_f:
diff --git a/build/android/gyp/ijar.pydeps b/build/android/gyp/ijar.pydeps index e9ecb66..530aabe 100644 --- a/build/android/gyp/ijar.pydeps +++ b/build/android/gyp/ijar.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/ijar.pydeps build/android/gyp/ijar.py +../../action_helpers.py ../../gn_helpers.py ijar.py util/__init__.py
diff --git a/build/android/gyp/jacoco_instr.py b/build/android/gyp/jacoco_instr.py index 8e5f29c..f32d6e8 100755 --- a/build/android/gyp/jacoco_instr.py +++ b/build/android/gyp/jacoco_instr.py
@@ -1,9 +1,8 @@ #!/usr/bin/env python3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Instruments classes and jar files. This script corresponds to the 'jacoco_instr' action in the Java build process. @@ -13,17 +12,20 @@ """ -from __future__ import print_function - import argparse import json import os import shutil import sys -import tempfile import zipfile from util import build_utils +import action_helpers +import zip_helpers + + +# This should be same as recipe side token. See bit.ly/3STSPcE. +INSTRUMENT_ALL_JACOCO_OVERRIDE_TOKEN = 'INSTRUMENT_ALL_JACOCO' def _AddArguments(parser): @@ -49,9 +51,9 @@ help='File to create with the list of source directories ' 'and input path.') parser.add_argument( - '--java-sources-file', + '--target-sources-file', required=True, - help='File containing newline-separated .java paths') + help='File containing newline-separated .java and .kt paths') parser.add_argument( '--jacococli-jar', required=True, help='Path to jacococli.jar.') parser.add_argument( @@ -101,10 +103,12 @@ data = {} data['source_dirs'] = relative_sources data['input_path'] = [] + data['output_dir'] = src_root if input_path: data['input_path'].append(os.path.abspath(input_path)) with open(sources_json_file, 'w') as f: json.dump(data, f) + return 0 def _GetAffectedClasses(jar_file, source_files): @@ -133,7 +137,8 @@ if index == -1: index = member.find('.class') for source_file in source_files: - if source_file.endswith(member[:index] + '.java'): + if source_file.endswith( + (member[:index] + '.java', member[:index] + '.kt')): affected_classes.append(member) is_affected = True break @@ -180,7 +185,8 @@ f.extractall(instrumented_dir, unaffected_members) # Zip all files to output_path - build_utils.ZipDir(output_path, instrumented_dir) + with action_helpers.atomic_output(output_path) as f: + zip_helpers.zip_directory(f, instrumented_dir) def _RunInstrumentCommand(parser): @@ -195,8 +201,8 @@ args = parser.parse_args() source_files = [] - if args.java_sources_file: - source_files.extend(build_utils.ReadSourcesList(args.java_sources_file)) + if args.target_sources_file: + source_files.extend(build_utils.ReadSourcesList(args.target_sources_file)) with build_utils.TempDir() as temp_dir: instrument_cmd = build_utils.JavaCmd() + [ @@ -204,23 +210,32 @@ ] if not args.files_to_instrument: - _InstrumentClassFiles(instrument_cmd, args.input_path, args.output_path, - temp_dir) + affected_source_files = None else: affected_files = build_utils.ReadSourcesList(args.files_to_instrument) - source_set = set(source_files) - affected_source_files = [f for f in affected_files if f in source_set] + # Check if coverage recipe decided to instrument everything by overriding + # the try builder default setting(selective instrumentation). This can + # happen in cases like a DEPS roll of jacoco library - # Copy input_path to output_path and return if no source file affected. - if not affected_source_files: - shutil.copyfile(args.input_path, args.output_path) - # Create a dummy sources_json_file. - _CreateSourcesJsonFile([], None, args.sources_json_file, - build_utils.DIR_SOURCE_ROOT) - return 0 + # Note: This token is preceded by ../../ because the paths to be + # instrumented are expected to be relative to the build directory. + # See _rebase_paths() at https://bit.ly/40oiixX + token = '../../' + INSTRUMENT_ALL_JACOCO_OVERRIDE_TOKEN + if token in affected_files: + affected_source_files = None else: - _InstrumentClassFiles(instrument_cmd, args.input_path, args.output_path, - temp_dir, affected_source_files) + source_set = set(source_files) + affected_source_files = [f for f in affected_files if f in source_set] + + # Copy input_path to output_path and return if no source file affected. + if not affected_source_files: + shutil.copyfile(args.input_path, args.output_path) + # Create a dummy sources_json_file. + _CreateSourcesJsonFile([], None, args.sources_json_file, + build_utils.DIR_SOURCE_ROOT) + return 0 + _InstrumentClassFiles(instrument_cmd, args.input_path, args.output_path, + temp_dir, affected_source_files) source_dirs = _GetSourceDirsFromSourceFiles(source_files) # TODO(GYP): In GN, we are passed the list of sources, detecting source
diff --git a/build/android/gyp/jacoco_instr.pydeps b/build/android/gyp/jacoco_instr.pydeps index d7fec19..9c763fc 100644 --- a/build/android/gyp/jacoco_instr.pydeps +++ b/build/android/gyp/jacoco_instr.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/jacoco_instr.pydeps build/android/gyp/jacoco_instr.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py jacoco_instr.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/java_cpp_enum.py b/build/android/gyp/java_cpp_enum.py index 08a381a..9098cfc 100755 --- a/build/android/gyp/java_cpp_enum.py +++ b/build/android/gyp/java_cpp_enum.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,6 +16,9 @@ from util import build_utils from util import java_cpp_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers + # List of C++ types that are compatible with the Java code generated by this # script. @@ -28,7 +31,7 @@ ] -class EnumDefinition(object): +class EnumDefinition: def __init__(self, original_enum_name=None, class_name_override=None, enum_package=None, entries=None, comments=None, fixed_type=None): self.original_enum_name = original_enum_name @@ -79,9 +82,9 @@ else: try: self.entries[key] = int(value) - except ValueError: + except ValueError as e: raise Exception('Could not interpret integer from enum value "%s" ' - 'for key %s.' % (value, key)) + 'for key %s.' % (value, key)) from e prev_enum_value = self.entries[key] @@ -96,7 +99,7 @@ 'k' + self.original_enum_name] for prefix in prefixes: - if all([w.startswith(prefix) for w in self.entries.keys()]): + if all(w.startswith(prefix) for w in self.entries.keys()): prefix_to_strip = prefix break else: @@ -141,7 +144,7 @@ return ret -class DirectiveSet(object): +class DirectiveSet: class_name_override_key = 'CLASS_NAME_OVERRIDE' enum_package_key = 'ENUM_PACKAGE' prefix_to_strip_key = 'PREFIX_TO_STRIP' @@ -169,7 +172,7 @@ DirectiveSet.prefix_to_strip_key) -class HeaderParser(object): +class HeaderParser: single_line_comment_re = re.compile(r'\s*//\s*([^\n]*)') multi_line_comment_start_re = re.compile(r'\s*/\*') enum_line_re = re.compile(r'^\s*(\w+)(\s*\=\s*([^,\n]+))?,?') @@ -305,7 +308,7 @@ '. Use () for multi-line directives. E.g.\n' + '// GENERATED_JAVA_ENUM_PACKAGE: (\n' + '// foo.package)') - elif generator_directive: + if generator_directive: directive_name = generator_directive.groups()[0] directive_value = generator_directive.groups()[1] self._generator_directives.Update(directive_name, directive_value) @@ -427,10 +430,10 @@ parser.error('Need to specify at least one input file') input_paths = args - with build_utils.AtomicOutput(options.srcjar) as f: + with action_helpers.atomic_output(options.srcjar) as f: with zipfile.ZipFile(f, 'w', zipfile.ZIP_STORED) as srcjar: for output_path, data in DoGenerate(input_paths): - build_utils.AddToZipHermetic(srcjar, output_path, data=data) + zip_helpers.add_to_zip_hermetic(srcjar, output_path, data=data) if __name__ == '__main__':
diff --git a/build/android/gyp/java_cpp_enum.pydeps b/build/android/gyp/java_cpp_enum.pydeps index e6aaeb7..3e63ff8 100644 --- a/build/android/gyp/java_cpp_enum.pydeps +++ b/build/android/gyp/java_cpp_enum.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/java_cpp_enum.pydeps build/android/gyp/java_cpp_enum.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py java_cpp_enum.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/java_cpp_enum_tests.py b/build/android/gyp/java_cpp_enum_tests.py index 6d5f150..c14f2a0 100755 --- a/build/android/gyp/java_cpp_enum_tests.py +++ b/build/android/gyp/java_cpp_enum_tests.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gyp/java_cpp_features.py b/build/android/gyp/java_cpp_features.py index 8e7c244..10639a5 100755 --- a/build/android/gyp/java_cpp_features.py +++ b/build/android/gyp/java_cpp_features.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,14 +12,16 @@ from util import build_utils from util import java_cpp_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers class FeatureParserDelegate(java_cpp_utils.CppConstantParser.Delegate): - # Ex. 'const base::Feature kConstantName{"StringNameOfTheFeature", ...};' + # Ex. 'BASE_FEATURE(kConstantName, "StringNameOfTheFeature", ...);' # would parse as: # ExtractConstantName() -> 'ConstantName' # ExtractValue() -> '"StringNameOfTheFeature"' - FEATURE_RE = re.compile(r'\s*const (?:base::)?Feature\s+k(\w+)\s*(?:=\s*)?{') + FEATURE_RE = re.compile(r'BASE_FEATURE\(k([^,]+),') VALUE_RE = re.compile(r'\s*("(?:\"|[^"])*")\s*,') def ExtractConstantName(self, line): @@ -100,10 +102,10 @@ metavar='INPUTFILE') args = parser.parse_args(argv) - with build_utils.AtomicOutput(args.srcjar) as f: + with action_helpers.atomic_output(args.srcjar) as f: with zipfile.ZipFile(f, 'w', zipfile.ZIP_STORED) as srcjar: data, path = _Generate(args.inputs, args.template) - build_utils.AddToZipHermetic(srcjar, path, data=data) + zip_helpers.add_to_zip_hermetic(srcjar, path, data=data) if __name__ == '__main__':
diff --git a/build/android/gyp/java_cpp_features.pydeps b/build/android/gyp/java_cpp_features.pydeps index acffae2..4faa903 100644 --- a/build/android/gyp/java_cpp_features.pydeps +++ b/build/android/gyp/java_cpp_features.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/java_cpp_features.pydeps build/android/gyp/java_cpp_features.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py java_cpp_features.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/java_cpp_features_tests.py b/build/android/gyp/java_cpp_features_tests.py index 5dcdcd8..3053955 100755 --- a/build/android/gyp/java_cpp_features_tests.py +++ b/build/android/gyp/java_cpp_features_tests.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for java_cpp_features.py. @@ -27,14 +27,14 @@ // Comment followed by unrelated code. int foo() { return 3; } -// Real comment. -const base::Feature kSomeFeature{"SomeFeature", - base::FEATURE_DISABLED_BY_DEFAULT}; +// Real comment. base::Feature intentionally split across two lines. +BASE_FEATURE(kSomeFeature, "SomeFeature", + base::FEATURE_DISABLED_BY_DEFAULT); // Real comment that spans // multiple lines. -const base::Feature kSomeOtherFeature{"SomeOtherFeature", - base::FEATURE_ENABLED_BY_DEFAULT}; +BASE_FEATURE(kSomeOtherFeature, "SomeOtherFeature", + base::FEATURE_ENABLED_BY_DEFAULT); // Comment followed by nothing. """.split('\n') @@ -52,18 +52,18 @@ def testWhitespace(self): test_data = """ // 1 line -const base::Feature kShort{"Short", base::FEATURE_DISABLED_BY_DEFAULT}; +BASE_FEATURE(kShort, "Short", base::FEATURE_DISABLED_BY_DEFAULT); // 2 lines -const base::Feature kTwoLineFeatureA{"TwoLineFeatureA", - base::FEATURE_DISABLED_BY_DEFAULT}; -const base::Feature kTwoLineFeatureB{ - "TwoLineFeatureB", base::FEATURE_DISABLED_BY_DEFAULT}; +BASE_FEATURE(kTwoLineFeatureA, "TwoLineFeatureA", + base::FEATURE_DISABLED_BY_DEFAULT); +BASE_FEATURE(kTwoLineFeatureB, + "TwoLineFeatureB", base::FEATURE_DISABLED_BY_DEFAULT); // 3 lines -const base::Feature kFeatureWithAVeryLongNameThatWillHaveToWrap{ +BASE_FEATURE(kFeatureWithAVeryLongNameThatWillHaveToWrap, "FeatureWithAVeryLongNameThatWillHaveToWrap", - base::FEATURE_DISABLED_BY_DEFAULT}; + base::FEATURE_DISABLED_BY_DEFAULT); """.split('\n') feature_file_parser = java_cpp_utils.CppConstantParser( java_cpp_features.FeatureParserDelegate(), test_data) @@ -83,64 +83,59 @@ def testCppSyntax(self): test_data = """ // Mismatched name -const base::Feature kMismatchedFeature{"MismatchedName", - base::FEATURE_DISABLED_BY_DEFAULT}; +BASE_FEATURE(kMismatchedFeature, "MismatchedName", + base::FEATURE_DISABLED_BY_DEFAULT); namespace myfeature { // In a namespace -const base::Feature kSomeFeature{"SomeFeature", - base::FEATURE_DISABLED_BY_DEFAULT}; +BASE_FEATURE(kSomeFeature, "SomeFeature", + base::FEATURE_DISABLED_BY_DEFAULT); } -// Defined with equals sign -const base::Feature kFoo = {"Foo", base::FEATURE_DISABLED_BY_DEFAULT}; - // Build config-specific base::Feature -#if defined(OS_ANDROID) -const base::Feature kAndroidOnlyFeature{"AndroidOnlyFeature", - base::FEATURE_DISABLED_BY_DEFAULT}; +#if BUILDFLAG(IS_ANDROID) +BASE_FEATURE(kAndroidOnlyFeature, "AndroidOnlyFeature", + base::FEATURE_DISABLED_BY_DEFAULT); #endif // Value depends on build config -const base::Feature kMaybeEnabled{"MaybeEnabled", -#if defined(OS_ANDROID) +BASE_FEATURE(kMaybeEnabled, "MaybeEnabled", +#if BUILDFLAG(IS_ANDROID) base::FEATURE_DISABLED_BY_DEFAULT #else base::FEATURE_ENABLED_BY_DEFAULT #endif -}; +); """.split('\n') feature_file_parser = java_cpp_utils.CppConstantParser( java_cpp_features.FeatureParserDelegate(), test_data) features = feature_file_parser.Parse() - self.assertEqual(5, len(features)) + self.assertEqual(4, len(features)) self.assertEqual('MISMATCHED_FEATURE', features[0].name) self.assertEqual('"MismatchedName"', features[0].value) self.assertEqual('SOME_FEATURE', features[1].name) self.assertEqual('"SomeFeature"', features[1].value) - self.assertEqual('FOO', features[2].name) - self.assertEqual('"Foo"', features[2].value) - self.assertEqual('ANDROID_ONLY_FEATURE', features[3].name) - self.assertEqual('"AndroidOnlyFeature"', features[3].value) - self.assertEqual('MAYBE_ENABLED', features[4].name) - self.assertEqual('"MaybeEnabled"', features[4].value) + self.assertEqual('ANDROID_ONLY_FEATURE', features[2].name) + self.assertEqual('"AndroidOnlyFeature"', features[2].value) + self.assertEqual('MAYBE_ENABLED', features[3].name) + self.assertEqual('"MaybeEnabled"', features[3].value) def testNotYetSupported(self): # Negative test for cases we don't yet support, to ensure we don't misparse # these until we intentionally add proper support. test_data = """ // Not currently supported: name depends on C++ directive -const base::Feature kNameDependsOnOs{ -#if defined(OS_ANDROID) +BASE_FEATURE(kNameDependsOnOs, +#if BUILDFLAG(IS_ANDROID) "MaybeName1", #else "MaybeName2", #endif - base::FEATURE_DISABLED_BY_DEFAULT}; + base::FEATURE_DISABLED_BY_DEFAULT); // Not currently supported: feature named with a constant instead of literal -const base::Feature kNamedAfterConstant{kNamedStringConstant, - base::FEATURE_DISABLED_BY_DEFAULT}; +BASE_FEATURE(kNamedAfterConstant, kNamedStringConstant, + base::FEATURE_DISABLED_BY_DEFAULT}; """.split('\n') feature_file_parser = java_cpp_utils.CppConstantParser( java_cpp_features.FeatureParserDelegate(), test_data) @@ -149,13 +144,13 @@ def testTreatWebViewLikeOneWord(self): test_data = """ -const base::Feature kSomeWebViewFeature{"SomeWebViewFeature", - base::FEATURE_DISABLED_BY_DEFAULT}; -const base::Feature kWebViewOtherFeature{"WebViewOtherFeature", - base::FEATURE_ENABLED_BY_DEFAULT}; -const base::Feature kFeatureWithPluralWebViews{ +BASE_FEATURE(kSomeWebViewFeature, "SomeWebViewFeature", + base::FEATURE_DISABLED_BY_DEFAULT); +BASE_FEATURE(kWebViewOtherFeature, "WebViewOtherFeature", + base::FEATURE_ENABLED_BY_DEFAULT); +BASE_FEATURE(kFeatureWithPluralWebViews, "FeatureWithPluralWebViews", - base::FEATURE_ENABLED_BY_DEFAULT}; + base::FEATURE_ENABLED_BY_DEFAULT); """.split('\n') feature_file_parser = java_cpp_utils.CppConstantParser( java_cpp_features.FeatureParserDelegate(), test_data) @@ -169,11 +164,11 @@ def testSpecialCharacters(self): test_data = r""" -const base::Feature kFeatureWithEscapes{"Weird\tfeature\"name\n", - base::FEATURE_DISABLED_BY_DEFAULT}; -const base::Feature kFeatureWithEscapes2{ +BASE_FEATURE(kFeatureWithEscapes, "Weird\tfeature\"name\n", + base::FEATURE_DISABLED_BY_DEFAULT); +BASE_FEATURE(kFeatureWithEscapes2, "Weird\tfeature\"name\n", - base::FEATURE_ENABLED_BY_DEFAULT}; + base::FEATURE_ENABLED_BY_DEFAULT); """.split('\n') feature_file_parser = java_cpp_utils.CppConstantParser( java_cpp_features.FeatureParserDelegate(), test_data) @@ -183,16 +178,6 @@ self.assertEqual('FEATURE_WITH_ESCAPES2', features[1].name) self.assertEqual(r'"Weird\tfeature\"name\n"', features[1].value) - def testNoBaseNamespacePrefix(self): - test_data = """ -const Feature kSomeFeature{"SomeFeature", FEATURE_DISABLED_BY_DEFAULT}; -""".split('\n') - feature_file_parser = java_cpp_utils.CppConstantParser( - java_cpp_features.FeatureParserDelegate(), test_data) - features = feature_file_parser.Parse() - self.assertEqual('SOME_FEATURE', features[0].name) - self.assertEqual('"SomeFeature"', features[0].value) - if __name__ == '__main__': unittest.main()
diff --git a/build/android/gyp/java_cpp_strings.py b/build/android/gyp/java_cpp_strings.py index d713599..c3d05de 100755 --- a/build/android/gyp/java_cpp_strings.py +++ b/build/android/gyp/java_cpp_strings.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,6 +12,8 @@ from util import build_utils from util import java_cpp_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers class StringParserDelegate(java_cpp_utils.CppConstantParser.Delegate): @@ -93,10 +95,10 @@ 'inputs', nargs='+', help='Input file(s)', metavar='INPUTFILE') args = parser.parse_args(argv) - with build_utils.AtomicOutput(args.srcjar) as f: + with action_helpers.atomic_output(args.srcjar) as f: with zipfile.ZipFile(f, 'w', zipfile.ZIP_STORED) as srcjar: data, path = _Generate(args.inputs, args.template) - build_utils.AddToZipHermetic(srcjar, path, data=data) + zip_helpers.add_to_zip_hermetic(srcjar, path, data=data) if __name__ == '__main__':
diff --git a/build/android/gyp/java_cpp_strings.pydeps b/build/android/gyp/java_cpp_strings.pydeps index 0a821f4..39b299e 100644 --- a/build/android/gyp/java_cpp_strings.pydeps +++ b/build/android/gyp/java_cpp_strings.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/java_cpp_strings.pydeps build/android/gyp/java_cpp_strings.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py java_cpp_strings.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/java_cpp_strings_tests.py b/build/android/gyp/java_cpp_strings_tests.py index 4cb1eee..793b2c3 100755 --- a/build/android/gyp/java_cpp_strings_tests.py +++ b/build/android/gyp/java_cpp_strings_tests.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -130,7 +130,7 @@ def testTemplateParsing(self): test_data = """ -// Copyright 2019 The Chromium Authors. All rights reserved. +// Copyright 2019 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gyp/java_google_api_keys.py b/build/android/gyp/java_google_api_keys.py index a58628a..4e4fa19 100755 --- a/build/android/gyp/java_google_api_keys.py +++ b/build/android/gyp/java_google_api_keys.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,6 +13,7 @@ import zipfile from util import build_utils +import zip_helpers sys.path.append( os.path.abspath(os.path.join(sys.path[0], '../../../google_apis'))) @@ -29,7 +30,7 @@ def GenerateOutput(constant_definitions): template = string.Template(""" -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -81,7 +82,7 @@ with zipfile.ZipFile(output_path, 'w') as srcjar: path = '%s/%s' % (PACKAGE.replace('.', '/'), CLASSNAME + '.java') data = GenerateOutput(constant_definition) - build_utils.AddToZipHermetic(srcjar, path, data=data) + zip_helpers.add_to_zip_hermetic(srcjar, path, data=data) def _DoMain(argv): @@ -95,14 +96,10 @@ values = {} values['GOOGLE_API_KEY'] = google_api_keys.GetAPIKey() - values['GOOGLE_API_KEY_PHYSICAL_WEB_TEST'] = (google_api_keys. - GetAPIKeyPhysicalWebTest()) + values['GOOGLE_API_KEY_ANDROID_NON_STABLE'] = ( + google_api_keys.GetAPIKeyAndroidNonStable()) values['GOOGLE_CLIENT_ID_MAIN'] = google_api_keys.GetClientID('MAIN') values['GOOGLE_CLIENT_SECRET_MAIN'] = google_api_keys.GetClientSecret('MAIN') - values['GOOGLE_CLIENT_ID_CLOUD_PRINT'] = google_api_keys.GetClientID( - 'CLOUD_PRINT') - values['GOOGLE_CLIENT_SECRET_CLOUD_PRINT'] = google_api_keys.GetClientSecret( - 'CLOUD_PRINT') values['GOOGLE_CLIENT_ID_REMOTING'] = google_api_keys.GetClientID('REMOTING') values['GOOGLE_CLIENT_SECRET_REMOTING'] = google_api_keys.GetClientSecret( 'REMOTING') @@ -110,8 +107,6 @@ 'REMOTING_HOST') values['GOOGLE_CLIENT_SECRET_REMOTING_HOST'] = (google_api_keys. GetClientSecret('REMOTING_HOST')) - values['GOOGLE_CLIENT_ID_REMOTING_IDENTITY_API'] = (google_api_keys. - GetClientID('REMOTING_IDENTITY_API')) if options.out: _DoWriteJavaOutput(options.out, values)
diff --git a/build/android/gyp/java_google_api_keys.pydeps b/build/android/gyp/java_google_api_keys.pydeps index ebb7172..6c027a1 100644 --- a/build/android/gyp/java_google_api_keys.pydeps +++ b/build/android/gyp/java_google_api_keys.pydeps
@@ -2,6 +2,7 @@ # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/java_google_api_keys.pydeps build/android/gyp/java_google_api_keys.py ../../../google_apis/google_api_keys.py ../../gn_helpers.py +../../zip_helpers.py java_google_api_keys.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/java_google_api_keys_tests.py b/build/android/gyp/java_google_api_keys_tests.py index e00e86c..0610178 100755 --- a/build/android/gyp/java_google_api_keys_tests.py +++ b/build/android/gyp/java_google_api_keys_tests.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,7 +19,7 @@ definition = {'E1': 'abc', 'E2': 'defgh'} output = java_google_api_keys.GenerateOutput(definition) expected = """ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gyp/javac_output_processor.py b/build/android/gyp/javac_output_processor.py new file mode 100755 index 0000000..6faf5de --- /dev/null +++ b/build/android/gyp/javac_output_processor.py
@@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Contains helper class for processing javac output.""" + +import dataclasses +import os +import pathlib +import re +import sys +import traceback +from typing import List + +from util import build_utils + +sys.path.insert( + 0, + os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'colorama', 'src')) +import colorama +sys.path.insert( + 0, + os.path.join(build_utils.DIR_SOURCE_ROOT, 'tools', 'android', + 'modularization', 'convenience')) +import lookup_dep + + +def ReplaceGmsPackageIfNeeded(target_name: str) -> str: + if target_name.startswith( + ('//third_party/android_deps:google_play_services_', + '//clank/third_party/google3:google_play_services_')): + return f'$google_play_services_package:{target_name.split(":")[1]}' + return target_name + + +def _DisambiguateDeps(class_entries: List[lookup_dep.ClassEntry]): + def filter_if_not_empty(entries, filter_func): + filtered_entries = [e for e in entries if filter_func(e)] + return filtered_entries or entries + + # When some deps are preferred, ignore all other potential deps. + class_entries = filter_if_not_empty(class_entries, lambda e: e.preferred_dep) + + # E.g. javax_annotation_jsr250_api_java. + class_entries = filter_if_not_empty(class_entries, + lambda e: 'jsr' in e.target) + + # Avoid suggesting subtargets when regular targets exist. + class_entries = filter_if_not_empty(class_entries, + lambda e: '__' not in e.target) + + # Swap out GMS package names if needed. + class_entries = [ + dataclasses.replace(e, target=ReplaceGmsPackageIfNeeded(e.target)) + for e in class_entries + ] + + # Convert to dict and then use list to get the keys back to remove dups and + # keep order the same as before. + class_entries = list({e: True for e in class_entries}) + + return class_entries + + +class JavacOutputProcessor: + def __init__(self, target_name): + self._target_name = self._RemoveSuffixesIfPresent( + ["__compile_java", "__errorprone", "__header"], target_name) + self._suggested_deps = set() + + # Example: ../../ui/android/java/src/org/chromium/ui/base/Clipboard.java:45: + fileline_prefix = ( + r'(?P<fileline>(?P<file>[-.\w/\\]+.java):(?P<line>[0-9]+):)') + + self._warning_re = re.compile( + fileline_prefix + r'(?P<full_message> warning: (?P<message>.*))$') + self._error_re = re.compile(fileline_prefix + + r'(?P<full_message> (?P<message>.*))$') + self._marker_re = re.compile(r'\s*(?P<marker>\^)\s*$') + + self._symbol_not_found_re_list = [ + # Example: + # error: package org.chromium.components.url_formatter does not exist + re.compile(fileline_prefix + + r'( error: package [\w.]+ does not exist)$'), + # Example: error: cannot find symbol + re.compile(fileline_prefix + r'( error: cannot find symbol)$'), + # Example: error: symbol not found org.chromium.url.GURL + re.compile(fileline_prefix + r'( error: symbol not found [\w.]+)$'), + ] + + # Example: import org.chromium.url.GURL; + self._import_re = re.compile(r'\s*import (?P<imported_class>[\w\.]+);$') + + self._warning_color = [ + 'full_message', colorama.Fore.YELLOW + colorama.Style.DIM + ] + self._error_color = [ + 'full_message', colorama.Fore.MAGENTA + colorama.Style.BRIGHT + ] + self._marker_color = ['marker', colorama.Fore.BLUE + colorama.Style.BRIGHT] + + self._class_lookup_index = None + + colorama.init() + + def Process(self, lines): + """ Processes javac output. + + - Applies colors to output. + - Suggests GN dep to add for 'unresolved symbol in Java import' errors. + """ + lines = self._ElaborateLinesForUnknownSymbol(iter(lines)) + for line in lines: + yield self._ApplyColors(line) + if self._suggested_deps: + + def yellow(text): + return colorama.Fore.YELLOW + text + colorama.Fore.RESET + + # Show them in quotes so they can be copy/pasted into BUILD.gn files. + yield yellow('Hint:') + ' One or more errors due to missing GN deps.' + yield (yellow('Hint:') + ' Try adding the following to ' + + yellow(self._target_name)) + for dep in sorted(self._suggested_deps): + yield ' "{}",'.format(dep) + + def _ElaborateLinesForUnknownSymbol(self, lines): + """ Elaborates passed-in javac output for unresolved symbols. + + Looks for unresolved symbols in imports. + Adds: + - Line with GN target which cannot compile. + - Mention of unresolved class if not present in error message. + - Line with suggestion of GN dep to add. + + Args: + lines: Generator with javac input. + Returns: + Generator with processed output. + """ + previous_line = next(lines, None) + line = next(lines, None) + while previous_line != None: + try: + self._LookForUnknownSymbol(previous_line, line) + except Exception: + elaborated_lines = ['Error in _LookForUnknownSymbol ---'] + elaborated_lines += traceback.format_exc().splitlines() + elaborated_lines += ['--- end _LookForUnknownSymbol error'] + for elaborated_line in elaborated_lines: + yield elaborated_line + + yield previous_line + previous_line = line + line = next(lines, None) + + def _ApplyColors(self, line): + """Adds colors to passed-in line and returns processed line.""" + if self._warning_re.match(line): + line = self._Colorize(line, self._warning_re, self._warning_color) + elif self._error_re.match(line): + line = self._Colorize(line, self._error_re, self._error_color) + elif self._marker_re.match(line): + line = self._Colorize(line, self._marker_re, self._marker_color) + return line + + def _LookForUnknownSymbol(self, line, next_line): + if not next_line: + return + + import_re_match = self._import_re.match(next_line) + if not import_re_match: + return + + for regex in self._symbol_not_found_re_list: + if regex.match(line): + break + else: + return + + if self._class_lookup_index is None: + self._class_lookup_index = lookup_dep.ClassLookupIndex( + pathlib.Path(os.getcwd()), + should_build=False, + ) + + class_to_lookup = import_re_match.group('imported_class') + suggested_deps = self._class_lookup_index.match(class_to_lookup) + + if not suggested_deps: + return + + suggested_deps = _DisambiguateDeps(suggested_deps) + suggested_deps_str = ', '.join(s.target for s in suggested_deps) + + if len(suggested_deps) > 1: + suggested_deps_str = 'one of: ' + suggested_deps_str + + self._suggested_deps.add(suggested_deps_str) + + @staticmethod + def _RemoveSuffixesIfPresent(suffixes, text): + for suffix in suffixes: + if text.endswith(suffix): + return text[:-len(suffix)] + return text + + @staticmethod + def _Colorize(line, regex, color): + match = regex.match(line) + start = match.start(color[0]) + end = match.end(color[0]) + return (line[:start] + color[1] + line[start:end] + colorama.Fore.RESET + + colorama.Style.RESET_ALL + line[end:])
diff --git a/build/android/gyp/jetify_jar.py b/build/android/gyp/jetify_jar.py deleted file mode 100755 index e97ad97..0000000 --- a/build/android/gyp/jetify_jar.py +++ /dev/null
@@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2019 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -from __future__ import print_function - -import argparse -import os -import subprocess -import sys - -from util import build_utils - - -def _AddArguments(parser): - """Adds arguments related to jetifying to parser. - - Args: - parser: ArgumentParser object. - """ - parser.add_argument( - '--input-path', - required=True, - help='Path to input file(s). Either the classes ' - 'directory, or the path to a jar.') - parser.add_argument( - '--output-path', - required=True, - help='Path to output final file(s) to. Either the ' - 'final classes directory, or the directory in ' - 'which to place the instrumented/copied jar.') - parser.add_argument( - '--jetify-path', required=True, help='Path to jetify bin.') - parser.add_argument( - '--jetify-config-path', required=True, help='Path to jetify config file.') - - -def _RunJetifyCommand(parser): - args = parser.parse_args() - cmd = [ - args.jetify_path, - '-i', - args.input_path, - '-o', - args.output_path, - # Need to suppress a lot of warning output when jar doesn't have - # any references rewritten. - '-l', - 'error' - ] - if args.jetify_config_path: - cmd.extend(['-c', args.jetify_config_path]) - # Must wait for jetify command to complete to prevent race condition. - env = os.environ.copy() - env['JAVA_HOME'] = build_utils.JAVA_HOME - subprocess.check_call(cmd, env=env) - - -def main(): - parser = argparse.ArgumentParser() - _AddArguments(parser) - _RunJetifyCommand(parser) - - -if __name__ == '__main__': - sys.exit(main())
diff --git a/build/android/gyp/jetify_jar.pydeps b/build/android/gyp/jetify_jar.pydeps deleted file mode 100644 index 6a1a589..0000000 --- a/build/android/gyp/jetify_jar.pydeps +++ /dev/null
@@ -1,6 +0,0 @@ -# Generated by running: -# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/jetify_jar.pydeps build/android/gyp/jetify_jar.py -../../gn_helpers.py -jetify_jar.py -util/__init__.py -util/build_utils.py
diff --git a/build/android/gyp/jinja_template.py b/build/android/gyp/jinja_template.py index d42189b..4a24268 100755 --- a/build/android/gyp/jinja_template.py +++ b/build/android/gyp/jinja_template.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,6 +13,8 @@ from util import build_utils from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir)) from pylib.constants import host_paths @@ -34,7 +36,7 @@ return contents, filename, uptodate -class JinjaProcessor(object): +class JinjaProcessor: """Allows easy rendering of jinja templates with input file tracking.""" def __init__(self, loader_base_dir, variables=None): self.loader_base_dir = loader_base_dir @@ -90,12 +92,13 @@ path_info.AddMapping(relpath, input_filename) path_info.Write(outputs_zip + '.info') - build_utils.ZipDir(outputs_zip, temp_dir) + with action_helpers.atomic_output(outputs_zip) as f: + zip_helpers.zip_directory(f, temp_dir) def _ParseVariables(variables_arg, error_func): variables = {} - for v in build_utils.ParseGnList(variables_arg): + for v in action_helpers.parse_gn_list(variables_arg): if '=' not in v: error_func('--variables argument must contain "=": ' + v) name, _, value = v.partition('=') @@ -128,8 +131,8 @@ help='Enable inputs and includes checks.') options = parser.parse_args() - inputs = build_utils.ParseGnList(options.inputs) - includes = build_utils.ParseGnList(options.includes) + inputs = action_helpers.parse_gn_list(options.inputs) + includes = action_helpers.parse_gn_list(options.includes) if (options.output is None) == (options.outputs_zip is None): parser.error('Exactly one of --output and --output-zip must be given')
diff --git a/build/android/gyp/jinja_template.pydeps b/build/android/gyp/jinja_template.pydeps index af22c40..1eafd88 100644 --- a/build/android/gyp/jinja_template.pydeps +++ b/build/android/gyp/jinja_template.pydeps
@@ -10,9 +10,8 @@ ../../../third_party/catapult/devil/devil/constants/__init__.py ../../../third_party/catapult/devil/devil/constants/exit_codes.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -32,7 +31,9 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py ../pylib/__init__.py ../pylib/constants/__init__.py ../pylib/constants/host_paths.py
diff --git a/build/android/gyp/lint.py b/build/android/gyp/lint.py index faad21c..ae26a18 100755 --- a/build/android/gyp/lint.py +++ b/build/android/gyp/lint.py
@@ -1,29 +1,25 @@ #!/usr/bin/env python3 # -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Runs Android's lint tool.""" -from __future__ import print_function - import argparse -import functools import logging import os -import re import shutil import sys import time -import traceback from xml.dom import minidom from xml.etree import ElementTree from util import build_utils from util import manifest_utils from util import server_utils +import action_helpers # build_utils adds //build to sys.path. -_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/master/build/android/docs/lint.md' # pylint: disable=line-too-long +_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long # These checks are not useful for chromium. _DISABLED_ALWAYS = [ @@ -32,11 +28,13 @@ "InflateParams", # Null is ok when inflating views for dialogs. "InlinedApi", # Constants are copied so they are always available. "LintBaseline", # Don't warn about using baseline.xml files. + "MissingInflatedId", # False positives https://crbug.com/1394222 "MissingApplicationIcon", # False positive for non-production targets. + "ObsoleteLintCustomCheck", # We have no control over custom lint checks. "SwitchIntDef", # Many C++ enums are not used at all in java. + "Typos", # Strings are committed in English first and later translated. "UniqueConstants", # Chromium enums allow aliases. "UnusedAttribute", # Chromium apks have various minSdkVersion values. - "ObsoleteLintCustomCheck", # We have no control over custom lint checks. ] # These checks are not useful for test targets and adds an unnecessary burden @@ -80,7 +78,8 @@ resource_sources=None, custom_lint_jars=None, custom_annotation_zips=None, - android_sdk_version=None): + android_sdk_version=None, + baseline_path=None): project = ElementTree.Element('project') root = ElementTree.SubElement(project, 'root') # Run lint from output directory: crbug.com/1115594 @@ -88,6 +87,9 @@ sdk = ElementTree.SubElement(project, 'sdk') # Lint requires that the sdk path be an absolute path. sdk.set('dir', os.path.abspath(android_sdk_root)) + if baseline_path is not None: + baseline = ElementTree.SubElement(project, 'baseline') + baseline.set('file', baseline_path) cache = ElementTree.SubElement(project, 'cache') cache.set('dir', cache_dir) main_module = ElementTree.SubElement(project, 'module') @@ -163,12 +165,6 @@ for node in extra_app_node: app_node.append(node) - if app_node.find( - '{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE) is None: - # Assume no backup is intended, appeases AllowBackup lint check and keeping - # it working for manifests that do define android:allowBackup. - app_node.set('{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE, 'false') - uses_sdk = manifest.find('./uses-sdk') if uses_sdk is None: uses_sdk = ElementTree.Element('uses-sdk') @@ -183,7 +179,7 @@ def _WriteXmlFile(root, path): logging.info('Writing xml file %s', path) build_utils.MakeDirectory(os.path.dirname(path)) - with build_utils.AtomicOutput(path) as f: + with action_helpers.atomic_output(path) as f: # Although we can write it just with ElementTree.tostring, using minidom # makes it a lot easier to read as a human (also on code search). f.write( @@ -191,7 +187,9 @@ root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8')) -def _RunLint(lint_binary_path, +def _RunLint(create_cache, + custom_lint_jar_path, + lint_jar_path, backported_methods_path, config_path, manifest_path, @@ -212,14 +210,46 @@ warnings_as_errors=False): logging.info('Lint starting') - cmd = [ - lint_binary_path, + if create_cache: + # Occasionally lint may crash due to re-using intermediate files from older + # lint runs. See https://crbug.com/1258178 for context. + logging.info('Clearing cache dir %s before creating cache.', cache_dir) + shutil.rmtree(cache_dir, ignore_errors=True) + os.makedirs(cache_dir) + + if baseline and not os.path.exists(baseline): + # Generating new baselines is only done locally, and requires more memory to + # avoid OOMs. + creating_baseline = True + lint_xmx = '4G' + else: + creating_baseline = False + lint_xmx = '2G' + + # All paths in lint are based off of relative paths from root with root as the + # prefix. Path variable substitution is based off of prefix matching so custom + # path variables need to match exactly in order to show up in baseline files. + # e.g. lint_path=path/to/output/dir/../../file/in/src + root_path = os.getcwd() # This is usually the output directory. + pathvar_src = os.path.join( + root_path, os.path.relpath(build_utils.DIR_SOURCE_ROOT, start=root_path)) + + cmd = build_utils.JavaCmd(xmx=lint_xmx) + [ + '-cp', + '{}:{}'.format(lint_jar_path, custom_lint_jar_path), + 'org.chromium.build.CustomLint', + '--sdk-home', + android_sdk_root, + '--jdk-home', + build_utils.JAVA_HOME, + '--path-variables', + f'SRC={pathvar_src}', '--quiet', # Silences lint's "." progress updates. + '--stacktrace', # Prints full stacktraces for internal lint errors. '--disable', ','.join(_DISABLED_ALWAYS), ] - if baseline: - cmd.extend(['--baseline', baseline]) + if testonly_target: cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)]) @@ -295,27 +325,12 @@ classpath, srcjar_sources, resource_sources, custom_lint_jars, custom_annotation_zips, - android_sdk_version) + android_sdk_version, baseline) project_xml_path = os.path.join(lint_gen_dir, 'project.xml') _WriteXmlFile(project_file_root, project_xml_path) cmd += ['--project', project_xml_path] - logging.info('Preparing environment variables') - env = os.environ.copy() - # It is important that lint uses the checked-in JDK11 as it is almost 50% - # faster than JDK8. - env['JAVA_HOME'] = build_utils.JAVA_HOME - # This is necessary so that lint errors print stack traces in stdout. - env['LINT_PRINT_STACKTRACE'] = 'true' - if baseline and not os.path.exists(baseline): - # Generating new baselines is only done locally, and requires more memory to - # avoid OOMs. - env['LINT_OPTS'] = '-Xmx4g' - else: - # The default set in the wrapper script is 1g, but it seems not enough :( - env['LINT_OPTS'] = '-Xmx2g' - # This filter is necessary for JDK11. stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found') @@ -323,14 +338,21 @@ start = time.time() logging.debug('Lint command %s', ' '.join(cmd)) failed = True + + if creating_baseline and not warnings_as_errors: + # Allow error code 6 when creating a baseline: ERRNO_CREATED_BASELINE + fail_func = lambda returncode, _: returncode not in (0, 6) + else: + fail_func = lambda returncode, _: returncode != 0 + try: failed = bool( build_utils.CheckOutput(cmd, - env=env, print_stdout=True, stdout_filter=stdout_filter, stderr_filter=stderr_filter, - fail_on_output=warnings_as_errors)) + fail_on_output=warnings_as_errors, + fail_func=fail_func)) finally: # When not treating warnings as errors, display the extra footer. is_debug = os.environ.get('LINT_DEBUG', '0') != '0' @@ -356,14 +378,20 @@ def _ParseArgs(argv): parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--target-name', help='Fully qualified GN target name.') parser.add_argument('--skip-build-server', action='store_true', help='Avoid using the build server.') - parser.add_argument('--lint-binary-path', + parser.add_argument('--use-build-server', + action='store_true', + help='Always use the build server.') + parser.add_argument('--lint-jar-path', required=True, - help='Path to lint executable.') + help='Path to the lint jar.') + parser.add_argument('--custom-lint-jar-path', + required=True, + help='Path to our custom lint jar.') parser.add_argument('--backported-methods', help='Path to backported methods file created by R8.') parser.add_argument('--cache-dir', @@ -395,8 +423,9 @@ parser.add_argument('--warnings-as-errors', action='store_true', help='Treat all warnings as errors.') - parser.add_argument('--java-sources', - help='File containing a list of java sources files.') + parser.add_argument('--sources', + help='A list of files containing java and kotlin source ' + 'files.') parser.add_argument('--aars', help='GN list of included aars.') parser.add_argument('--srcjars', help='GN list of included srcjars.') parser.add_argument('--manifest-path', @@ -422,13 +451,21 @@ 'on new errors.') args = parser.parse_args(build_utils.ExpandFileArgs(argv)) - args.java_sources = build_utils.ParseGnList(args.java_sources) - args.aars = build_utils.ParseGnList(args.aars) - args.srcjars = build_utils.ParseGnList(args.srcjars) - args.resource_sources = build_utils.ParseGnList(args.resource_sources) - args.extra_manifest_paths = build_utils.ParseGnList(args.extra_manifest_paths) - args.resource_zips = build_utils.ParseGnList(args.resource_zips) - args.classpath = build_utils.ParseGnList(args.classpath) + args.sources = action_helpers.parse_gn_list(args.sources) + args.aars = action_helpers.parse_gn_list(args.aars) + args.srcjars = action_helpers.parse_gn_list(args.srcjars) + args.resource_sources = action_helpers.parse_gn_list(args.resource_sources) + args.extra_manifest_paths = action_helpers.parse_gn_list( + args.extra_manifest_paths) + args.resource_zips = action_helpers.parse_gn_list(args.resource_zips) + args.classpath = action_helpers.parse_gn_list(args.classpath) + + if args.baseline: + assert os.path.basename(args.baseline) == 'lint-baseline.xml', ( + 'The baseline file needs to be named "lint-baseline.xml" in order for ' + 'the autoroller to find and update it whenever lint is rolled to a new ' + 'version.') + return args @@ -441,13 +478,15 @@ # Avoid parallelizing cache creation since lint runs without the cache defeat # the purpose of creating the cache in the first place. if (not args.create_cache and not args.skip_build_server - and server_utils.MaybeRunCommand( - name=args.target_name, argv=sys.argv, stamp_file=args.stamp)): + and server_utils.MaybeRunCommand(name=args.target_name, + argv=sys.argv, + stamp_file=args.stamp, + force=args.use_build_server)): return sources = [] - for java_sources_file in args.java_sources: - sources.extend(build_utils.ReadSourcesList(java_sources_file)) + for sources_file in args.sources: + sources.extend(build_utils.ReadSourcesList(sources_file)) resource_sources = [] for resource_sources_file in args.resource_sources: resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) @@ -459,7 +498,9 @@ ]) depfile_deps = [p for p in possible_depfile_deps if p] - _RunLint(args.lint_binary_path, + _RunLint(args.create_cache, + args.custom_lint_jar_path, + args.lint_jar_path, args.backported_methods, args.config_path, args.manifest_path, @@ -482,7 +523,7 @@ build_utils.Touch(args.stamp) if args.depfile: - build_utils.WriteDepfile(args.depfile, args.stamp, depfile_deps) + action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) if __name__ == '__main__':
diff --git a/build/android/gyp/lint.pydeps b/build/android/gyp/lint.pydeps index 0994e19..84bafde 100644 --- a/build/android/gyp/lint.pydeps +++ b/build/android/gyp/lint.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/lint.pydeps build/android/gyp/lint.py +../../action_helpers.py ../../gn_helpers.py lint.py util/__init__.py
diff --git a/build/android/gyp/merge_manifest.py b/build/android/gyp/merge_manifest.py index 53f1c11..a9c2535 100755 --- a/build/android/gyp/merge_manifest.py +++ b/build/android/gyp/merge_manifest.py
@@ -1,12 +1,13 @@ #!/usr/bin/env python3 -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Merges dependency Android manifests into a root manifest.""" import argparse +import collections import contextlib import os import sys @@ -15,57 +16,62 @@ from util import build_utils from util import manifest_utils +import action_helpers # build_utils adds //build to sys.path. _MANIFEST_MERGER_MAIN_CLASS = 'com.android.manifmerger.Merger' -_MANIFEST_MERGER_JARS = [ - os.path.join('build-system', 'manifest-merger.jar'), - os.path.join('common', 'common.jar'), - os.path.join('sdk-common', 'sdk-common.jar'), - os.path.join('sdklib', 'sdklib.jar'), - os.path.join('external', 'com', 'google', 'guava', 'guava', '28.1-jre', - 'guava-28.1-jre.jar'), - os.path.join('external', 'kotlin-plugin-ij', 'Kotlin', 'kotlinc', 'lib', - 'kotlin-stdlib.jar'), - os.path.join('external', 'com', 'google', 'code', 'gson', 'gson', '2.8.5', - 'gson-2.8.5.jar'), -] @contextlib.contextmanager -def _ProcessManifest(manifest_path, min_sdk_version, target_sdk_version, - max_sdk_version, manifest_package): - """Patches an Android manifest's package and performs assertions to ensure - correctness for the manifest. - """ +def _ProcessMainManifest(manifest_path, min_sdk_version, target_sdk_version, + max_sdk_version, manifest_package): + """Patches the main Android manifest""" doc, manifest, _ = manifest_utils.ParseManifest(manifest_path) - manifest_utils.AssertUsesSdk(manifest, min_sdk_version, target_sdk_version, - max_sdk_version) + manifest_utils.SetUsesSdk(manifest, target_sdk_version, min_sdk_version, + max_sdk_version) assert manifest_utils.GetPackage(manifest) or manifest_package, \ 'Must set manifest package in GN or in AndroidManifest.xml' - manifest_utils.AssertPackage(manifest, manifest_package) if manifest_package: manifest.set('package', manifest_package) - tmp_prefix = os.path.basename(manifest_path) + tmp_prefix = manifest_path.replace(os.path.sep, '-') with tempfile.NamedTemporaryFile(prefix=tmp_prefix) as patched_manifest: manifest_utils.SaveManifest(doc, patched_manifest.name) yield patched_manifest.name, manifest_utils.GetPackage(manifest) -def _BuildManifestMergerClasspath(android_sdk_cmdline_tools): - return ':'.join([ - os.path.join(android_sdk_cmdline_tools, 'lib', jar) - for jar in _MANIFEST_MERGER_JARS - ]) +@contextlib.contextmanager +def _ProcessOtherManifest(manifest_path, target_sdk_version, + seen_package_names): + """Patches non-main AndroidManifest.xml if necessary.""" + # 1. Ensure targetSdkVersion is set to the expected value to avoid + # spurious permissions being added (b/222331337). + # 2. Ensure all manifests have a unique package name so that the merger + # does not fail when this happens. + doc, manifest, _ = manifest_utils.ParseManifest(manifest_path) + + changed_api = manifest_utils.SetTargetApiIfUnset(manifest, target_sdk_version) + + package_name = manifest_utils.GetPackage(manifest) + package_count = seen_package_names[package_name] + seen_package_names[package_name] += 1 + if package_count > 0: + manifest.set('package', f'{package_name}_{package_count}') + + if package_count > 0 or changed_api: + tmp_prefix = manifest_path.replace(os.path.sep, '-') + with tempfile.NamedTemporaryFile(prefix=tmp_prefix) as patched_manifest: + manifest_utils.SaveManifest(doc, patched_manifest.name) + yield patched_manifest.name + else: + yield manifest_path def main(argv): argv = build_utils.ExpandFileArgs(argv) parser = argparse.ArgumentParser(description=__doc__) - build_utils.AddDepfileOption(parser) - parser.add_argument( - '--android-sdk-cmdline-tools', - help='Path to SDK\'s cmdline-tools folder.', - required=True) + action_helpers.add_depfile_arg(parser) + parser.add_argument('--manifest-merger-jar', + help='Path to SDK\'s manifest merger jar.', + required=True) parser.add_argument('--root-manifest', help='Root manifest which to merge into', required=True) @@ -90,12 +96,10 @@ help='Treat all warnings as errors.') args = parser.parse_args(argv) - classpath = _BuildManifestMergerClasspath(args.android_sdk_cmdline_tools) - - with build_utils.AtomicOutput(args.output) as output: - cmd = build_utils.JavaCmd(args.warnings_as_errors) + [ + with action_helpers.atomic_output(args.output) as output: + cmd = build_utils.JavaCmd() + [ '-cp', - classpath, + args.manifest_merger_jar, _MANIFEST_MERGER_MAIN_CLASS, '--out', output.name, @@ -111,19 +115,27 @@ 'MAX_SDK_VERSION=' + args.max_sdk_version, ] - extras = build_utils.ParseGnList(args.extras) - if extras: - cmd += ['--libs', ':'.join(extras)] + extras = action_helpers.parse_gn_list(args.extras) - with _ProcessManifest(args.root_manifest, args.min_sdk_version, - args.target_sdk_version, args.max_sdk_version, - args.manifest_package) as tup: - root_manifest, package = tup + with contextlib.ExitStack() as stack: + root_manifest, package = stack.enter_context( + _ProcessMainManifest(args.root_manifest, args.min_sdk_version, + args.target_sdk_version, args.max_sdk_version, + args.manifest_package)) + if extras: + seen_package_names = collections.Counter() + extras_processed = [ + stack.enter_context( + _ProcessOtherManifest(e, args.target_sdk_version, + seen_package_names)) for e in extras + ] + cmd += ['--libs', ':'.join(extras_processed)] cmd += [ '--main', root_manifest, '--property', 'PACKAGE=' + package, + '--remove-tools-declarations', ] build_utils.CheckOutput( cmd, @@ -133,15 +145,8 @@ IsTimeStale(output.name, [root_manifest] + extras), fail_on_output=args.warnings_as_errors) - # Check for correct output. - _, manifest, _ = manifest_utils.ParseManifest(output.name) - manifest_utils.AssertUsesSdk(manifest, args.min_sdk_version, - args.target_sdk_version) - manifest_utils.AssertPackage(manifest, package) - if args.depfile: - inputs = extras + classpath.split(':') - build_utils.WriteDepfile(args.depfile, args.output, inputs=inputs) + action_helpers.write_depfile(args.depfile, args.output, inputs=extras) if __name__ == '__main__':
diff --git a/build/android/gyp/merge_manifest.pydeps b/build/android/gyp/merge_manifest.pydeps index ef9bb34..003690f 100644 --- a/build/android/gyp/merge_manifest.pydeps +++ b/build/android/gyp/merge_manifest.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/merge_manifest.pydeps build/android/gyp/merge_manifest.py +../../action_helpers.py ../../gn_helpers.py merge_manifest.py util/__init__.py
diff --git a/build/android/gyp/native_libraries_template.py b/build/android/gyp/native_libraries_template.py deleted file mode 100644 index cf336ec..0000000 --- a/build/android/gyp/native_libraries_template.py +++ /dev/null
@@ -1,39 +0,0 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -NATIVE_LIBRARIES_TEMPLATE = """\ -// This file is autogenerated by -// build/android/gyp/write_native_libraries_java.py -// Please do not change its content. - -package org.chromium.build; - -public class NativeLibraries {{ - public static final int CPU_FAMILY_UNKNOWN = 0; - public static final int CPU_FAMILY_ARM = 1; - public static final int CPU_FAMILY_MIPS = 2; - public static final int CPU_FAMILY_X86 = 3; - - // Set to true to enable the use of the Chromium Linker. - public static {MAYBE_FINAL}boolean sUseLinker{USE_LINKER}; - public static {MAYBE_FINAL}boolean sUseLibraryInZipFile{USE_LIBRARY_IN_ZIP_FILE}; - public static {MAYBE_FINAL}boolean sUseModernLinker{USE_MODERN_LINKER}; - - // This is the list of native libraries to be loaded (in the correct order) - // by LibraryLoader.java. - // TODO(cjhopman): This is public since it is referenced by NativeTestActivity.java - // directly. The two ways of library loading should be refactored into one. - public static {MAYBE_FINAL}String[] LIBRARIES = {{{LIBRARIES}}}; - - // This is the expected version of the 'main' native library, which is the one that - // implements the initial set of base JNI functions including - // base::android::nativeGetVersionName() - // TODO(torne): This is public to work around classloader issues in Trichrome - // where NativeLibraries is not in the same dex as LibraryLoader. - // We should instead split up Java code along package boundaries. - public static {MAYBE_FINAL}String sVersionNumber = {VERSION_NUMBER}; - - public static {MAYBE_FINAL}int sCpuFamily = {CPU_FAMILY}; -}} -"""
diff --git a/build/android/gyp/nocompile_test.py b/build/android/gyp/nocompile_test.py index a5739f1..c3b02d2 100755 --- a/build/android/gyp/nocompile_test.py +++ b/build/android/gyp/nocompile_test.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Checks that compiling targets in BUILD.gn file fails.""" @@ -13,10 +13,13 @@ from util import build_utils _CHROMIUM_SRC = os.path.normpath(os.path.join(__file__, '..', '..', '..', '..')) -_NINJA_PATH = os.path.join(_CHROMIUM_SRC, 'third_party', 'depot_tools', 'ninja') +_NINJA_PATH = os.path.join(_CHROMIUM_SRC, 'third_party', 'ninja', 'ninja') # Relative to _CHROMIUM_SRC -_GN_SRC_REL_PATH = os.path.join('third_party', 'depot_tools', 'gn') +_GN_SRC_REL_PATH = os.path.join('buildtools', 'linux64', 'gn') + +# Regex for determining whether compile failed because 'gn gen' needs to be run. +_GN_GEN_REGEX = re.compile(r'ninja: (error|fatal):') def _raise_command_exception(args, returncode, output): @@ -47,17 +50,22 @@ _raise_command_exception(args, p.returncode, pout) -def _run_command_get_output(args, success_output): - """Runs shell command and returns command output.""" +def _run_command_get_failure_output(args): + """Runs shell command. + + Returns: + Command output if command fails, None if command succeeds. + """ p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pout, _ = p.communicate() + if p.returncode == 0: - return success_output + return None # For Python3 only: if isinstance(pout, bytes) and sys.version_info >= (3, ): pout = pout.decode('utf-8') - return pout + return '' if pout is None else pout def _copy_and_append_gn_args(src_args_path, dest_args_path, extra_args): @@ -74,21 +82,57 @@ f_out.write('\n'.join(extra_args)) -def _find_lines_after_prefix(text, prefix, num_lines): - """Searches |text| for a line which starts with |prefix|. +def _find_regex_in_test_failure_output(test_output, regex): + """Searches for regex in test output. - Args: - text: String to search in. - prefix: Prefix to search for. - num_lines: Number of lines, starting with line with prefix, to return. - Returns: - Matched lines. Returns None otherwise. + Args: + test_output: test output. + regex: regular expression to search for. + Returns: + Whether the regular expression was found in the part of the test output + after the 'FAILED' message. + + If the regex does not contain '\n': + the first 5 lines after the 'FAILED' message (including the text on the + line after the 'FAILED' message) is searched. + Otherwise: + the entire test output after the 'FAILED' message is searched. """ - lines = text.split('\n') - for i, line in enumerate(lines): - if line.startswith(prefix): - return lines[i:i + num_lines] - return None + if test_output is None: + return False + + failed_index = test_output.find('FAILED') + if failed_index < 0: + return False + + failure_message = test_output[failed_index:] + if regex.find('\n') >= 0: + return re.search(regex, failure_message) + + return _search_regex_in_list(failure_message.split('\n')[:5], regex) + + +def _search_regex_in_list(value, regex): + for line in value: + if re.search(regex, line): + return True + return False + + +def _do_build_get_failure_output(gn_path, gn_cmd, options): + # Extract directory from test target. As all of the test targets are declared + # in the same BUILD.gn file, it does not matter which test target is used. + target_dir = gn_path.rsplit(':', 1)[0] + + if gn_cmd is not None: + gn_args = [ + _GN_SRC_REL_PATH, '--root-target=' + target_dir, gn_cmd, + os.path.relpath(options.out_dir, _CHROMIUM_SRC) + ] + _run_command(gn_args, cwd=_CHROMIUM_SRC) + + ninja_args = [_NINJA_PATH, '-C', options.out_dir, gn_path] + return _run_command_get_failure_output(ninja_args) def main(): @@ -106,7 +150,10 @@ options = parser.parse_args() with open(options.test_configs_path) as f: - test_configs = json.loads(f.read()) + # Escape '\' in '\.' now. This avoids having to do the escaping in the test + # specification. + config_text = f.read().replace(r'\.', r'\\.') + test_configs = json.loads(config_text) if not os.path.exists(options.out_dir): os.makedirs(options.out_dir) @@ -121,34 +168,34 @@ _copy_and_append_gn_args(options.gn_args_path, out_gn_args_path, extra_gn_args) - # As all of the test targets are declared in the same BUILD.gn file, it does - # not matter which test target is used as the root target. - gn_args = [ - _GN_SRC_REL_PATH, '--root-target=' + test_configs[0]['target'], 'gen', - os.path.relpath(options.out_dir, _CHROMIUM_SRC) - ] - _run_command(gn_args, cwd=_CHROMIUM_SRC) - + ran_gn_gen = False + did_clean_build = False error_messages = [] for config in test_configs: # Strip leading '//' gn_path = config['target'][2:] expect_regex = config['expect_regex'] - ninja_args = [_NINJA_PATH, '-C', options.out_dir, gn_path] - # Purpose of quotes at beginning of message is to make it clear that - # "Compile successful." is not a compiler log message. - test_output = _run_command_get_output(ninja_args, '""\nCompile successful.') + test_output = _do_build_get_failure_output(gn_path, None, options) - failure_message_lines = _find_lines_after_prefix(test_output, 'FAILED:', 5) + # 'gn gen' takes > 1s to run. Only run 'gn gen' if it is needed for compile. + if (test_output + and _search_regex_in_list(test_output.split('\n'), _GN_GEN_REGEX)): + assert not ran_gn_gen + ran_gn_gen = True + test_output = _do_build_get_failure_output(gn_path, 'gen', options) - found_expect_regex = False - if failure_message_lines: - for line in failure_message_lines: - if re.search(expect_regex, line): - found_expect_regex = True - break - if not found_expect_regex: + if (not _find_regex_in_test_failure_output(test_output, expect_regex) + and not did_clean_build): + # Ensure the failure is not due to incremental build. + did_clean_build = True + test_output = _do_build_get_failure_output(gn_path, 'clean', options) + + if not _find_regex_in_test_failure_output(test_output, expect_regex): + if test_output is None: + # Purpose of quotes at beginning of message is to make it clear that + # "Compile successful." is not a compiler log message. + test_output = '""\nCompile successful.' error_message = '//{} failed.\nExpected compile output pattern:\n'\ '{}\nActual compile output:\n{}'.format( gn_path, expect_regex, test_output)
diff --git a/build/android/gyp/optimize_resources.py b/build/android/gyp/optimize_resources.py new file mode 100755 index 0000000..f1be4cc --- /dev/null +++ b/build/android/gyp/optimize_resources.py
@@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import logging +import os +import sys + +from util import build_utils +import action_helpers # build_utils adds //build to sys.path. + + +def _ParseArgs(args): + """Parses command line options. + + Returns: + An options object as from argparse.ArgumentParser.parse_args() + """ + parser = argparse.ArgumentParser() + parser.add_argument('--aapt2-path', + required=True, + help='Path to the Android aapt2 tool.') + parser.add_argument( + '--short-resource-paths', + action='store_true', + help='Whether to shorten resource paths inside the apk or module.') + parser.add_argument( + '--strip-resource-names', + action='store_true', + help='Whether to strip resource names from the resource table of the apk ' + 'or module.') + parser.add_argument('--proto-path', + required=True, + help='Input proto format resources APK.') + parser.add_argument('--resources-config-paths', + default='[]', + help='GN list of paths to aapt2 resources config files.') + parser.add_argument('--r-text-in', + required=True, + help='Path to R.txt. Used to exclude id/ resources.') + parser.add_argument( + '--resources-path-map-out-path', + help='Path to file produced by aapt2 that maps original resource paths ' + 'to shortened resource paths inside the apk or module.') + parser.add_argument('--optimized-proto-path', + required=True, + help='Output for `aapt2 optimize`.') + options = parser.parse_args(args) + + options.resources_config_paths = action_helpers.parse_gn_list( + options.resources_config_paths) + + if options.resources_path_map_out_path and not options.short_resource_paths: + parser.error( + '--resources-path-map-out-path requires --short-resource-paths') + return options + + +def _CombineResourceConfigs(resources_config_paths, out_config_path): + with open(out_config_path, 'w') as out_config: + for config_path in resources_config_paths: + with open(config_path) as config: + out_config.write(config.read()) + out_config.write('\n') + + +def _ExtractNonCollapsableResources(rtxt_path): + """Extract resources that should not be collapsed from the R.txt file + + Resources of type ID are references to UI elements/views. They are used by + UI automation testing frameworks. They are kept in so that they don't break + tests, even though they may not actually be used during runtime. See + https://crbug.com/900993 + App icons (aka mipmaps) are sometimes referenced by other apps by name so must + be keps as well. See https://b/161564466 + + Args: + rtxt_path: Path to R.txt file with all the resources + Returns: + List of resources in the form of <resource_type>/<resource_name> + """ + resources = [] + _NO_COLLAPSE_TYPES = ['id', 'mipmap'] + with open(rtxt_path) as rtxt: + for line in rtxt: + for resource_type in _NO_COLLAPSE_TYPES: + if ' {} '.format(resource_type) in line: + resource_name = line.split()[2] + resources.append('{}/{}'.format(resource_type, resource_name)) + return resources + + +def _OptimizeApk(output, options, temp_dir, unoptimized_path, r_txt_path): + """Optimize intermediate .ap_ file with aapt2. + + Args: + output: Path to write to. + options: The command-line options. + temp_dir: A temporary directory. + unoptimized_path: path of the apk to optimize. + r_txt_path: path to the R.txt file of the unoptimized apk. + """ + optimize_command = [ + options.aapt2_path, + 'optimize', + unoptimized_path, + '-o', + output, + ] + + # Optimize the resources.pb file by obfuscating resource names and only + # allow usage via R.java constant. + if options.strip_resource_names: + no_collapse_resources = _ExtractNonCollapsableResources(r_txt_path) + gen_config_path = os.path.join(temp_dir, 'aapt2.config') + if options.resources_config_paths: + _CombineResourceConfigs(options.resources_config_paths, gen_config_path) + with open(gen_config_path, 'a') as config: + for resource in no_collapse_resources: + config.write('{}#no_collapse\n'.format(resource)) + + optimize_command += [ + '--collapse-resource-names', + '--resources-config-path', + gen_config_path, + ] + + if options.short_resource_paths: + optimize_command += ['--shorten-resource-paths'] + if options.resources_path_map_out_path: + optimize_command += [ + '--resource-path-shortening-map', options.resources_path_map_out_path + ] + + logging.debug('Running aapt2 optimize') + build_utils.CheckOutput(optimize_command, + print_stdout=False, + print_stderr=False) + + +def main(args): + options = _ParseArgs(args) + with build_utils.TempDir() as temp_dir: + _OptimizeApk(options.optimized_proto_path, options, temp_dir, + options.proto_path, options.r_text_in) + + +if __name__ == '__main__': + main(sys.argv[1:])
diff --git a/build/android/gyp/optimize_resources.pydeps b/build/android/gyp/optimize_resources.pydeps new file mode 100644 index 0000000..be3e8e7 --- /dev/null +++ b/build/android/gyp/optimize_resources.pydeps
@@ -0,0 +1,7 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/optimize_resources.pydeps build/android/gyp/optimize_resources.py +../../action_helpers.py +../../gn_helpers.py +optimize_resources.py +util/__init__.py +util/build_utils.py
diff --git a/build/android/gyp/prepare_resources.py b/build/android/gyp/prepare_resources.py index 93fe9f9..e86711c3 100755 --- a/build/android/gyp/prepare_resources.py +++ b/build/android/gyp/prepare_resources.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -18,6 +18,8 @@ from util import md5_check from util import resources_parser from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers def _ParseArgs(args): @@ -27,7 +29,7 @@ An options object as from argparse.ArgumentParser.parse_args() """ parser = argparse.ArgumentParser(description=__doc__) - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--res-sources-path', required=True, @@ -39,6 +41,12 @@ 'in the generated R.txt when generating R.java.') parser.add_argument( + '--allow-missing-resources', + action='store_true', + help='Do not fail if some resources exist in the res/ dir but are not ' + 'listed in the sources.') + + parser.add_argument( '--resource-zip-out', help='Path to a zip archive containing all resources from ' '--resource-dirs, merged into a single directory tree.') @@ -110,7 +118,7 @@ # the contents of possibly multiple res/ dirs each within an encapsulating # directory within the zip. z.comment = resource_utils.MULTIPLE_RES_MAGIC_STRING - build_utils.DoZip(files_to_zip, z) + zip_helpers.add_files_to_zip(files_to_zip, z) def _GenerateRTxt(options, r_txt_path): @@ -130,7 +138,7 @@ def _OnStaleMd5(options): with resource_utils.BuildContext() as build: - if options.sources: + if options.sources and not options.allow_missing_resources: _CheckAllFilesListed(options.sources, options.resource_dirs) if options.r_text_in: r_txt_path = options.r_text_in
diff --git a/build/android/gyp/prepare_resources.pydeps b/build/android/gyp/prepare_resources.pydeps index b225918..5c7c441 100644 --- a/build/android/gyp/prepare_resources.pydeps +++ b/build/android/gyp/prepare_resources.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/prepare_resources.pydeps build/android/gyp/prepare_resources.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,8 +22,10 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py ../../print_python_deps.py +../../zip_helpers.py prepare_resources.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/process_native_prebuilt.py b/build/android/gyp/process_native_prebuilt.py index 52645d9..060adae 100755 --- a/build/android/gyp/process_native_prebuilt.py +++ b/build/android/gyp/process_native_prebuilt.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,6 +10,7 @@ import sys from util import build_utils +import action_helpers # build_utils adds //build to sys.path. def main(args): @@ -23,7 +24,7 @@ # eu-strip's output keeps mode from source file which might not be writable # thus it fails to override its output on the next run. AtomicOutput fixes # the issue. - with build_utils.AtomicOutput(options.stripped_output_path) as out: + with action_helpers.atomic_output(options.stripped_output_path) as out: cmd = [ options.strip_path, options.input_path,
diff --git a/build/android/gyp/process_native_prebuilt.pydeps b/build/android/gyp/process_native_prebuilt.pydeps index 8e2012a..baf9eff 100644 --- a/build/android/gyp/process_native_prebuilt.pydeps +++ b/build/android/gyp/process_native_prebuilt.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/process_native_prebuilt.pydeps build/android/gyp/process_native_prebuilt.py +../../action_helpers.py ../../gn_helpers.py process_native_prebuilt.py util/__init__.py
diff --git a/build/android/gyp/proguard.py b/build/android/gyp/proguard.py index 7f59769..579501c 100755 --- a/build/android/gyp/proguard.py +++ b/build/android/gyp/proguard.py
@@ -1,57 +1,43 @@ #!/usr/bin/env python3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse -from collections import defaultdict import logging import os +import pathlib import re import shutil import sys -import tempfile import zipfile import dex -import dex_jdk_libs -from pylib.dex import dex_parser from util import build_utils from util import diff_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers -_API_LEVEL_VERSION_CODE = [ - (21, 'L'), - (22, 'LollipopMR1'), - (23, 'M'), - (24, 'N'), - (25, 'NMR1'), - (26, 'O'), - (27, 'OMR1'), - (28, 'P'), - (29, 'Q'), - (30, 'R'), - (31, 'S'), +_BLOCKLISTED_EXPECTATION_PATHS = [ + # A separate expectation file is created for these files. + 'clank/third_party/google3/pg_confs/', ] +_DUMP_DIR_NAME = 'r8inputs_dir' + def _ParseOptions(): args = build_utils.ExpandFileArgs(sys.argv[1:]) parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--r8-path', required=True, help='Path to the R8.jar to use.') - parser.add_argument( - '--desugar-jdk-libs-json', help='Path to desugar_jdk_libs.json.') parser.add_argument('--input-paths', action='append', required=True, help='GN-list of .jar files to optimize.') - parser.add_argument('--desugar-jdk-libs-jar', - help='Path to desugar_jdk_libs.jar.') - parser.add_argument('--desugar-jdk-libs-configuration-jar', - help='Path to desugar_jdk_libs_configuration.jar.') parser.add_argument('--output-path', help='Path to the generated .jar file.') parser.add_argument( '--proguard-configs', @@ -84,18 +70,18 @@ parser.add_argument( '--repackage-classes', help='Package all optimized classes are put in.') parser.add_argument( - '--disable-outlining', - action='store_true', - help='Disable the outlining optimization provided by R8.') - parser.add_argument( '--disable-checks', action='store_true', help='Disable -checkdiscard directives and missing symbols check') - parser.add_argument('--sourcefile', help='Value for source file attribute') + parser.add_argument('--source-file', help='Value for source file attribute.') + parser.add_argument('--package-name', + help='Goes into a comment in the mapping file.') parser.add_argument( '--force-enable-assertions', action='store_true', help='Forcefully enable javac generated assertion code.') + parser.add_argument('--assertion-handler', + help='The class name of the assertion handler class.') parser.add_argument( '--feature-jars', action='append', @@ -135,6 +121,10 @@ help='Use when filing R8 bugs to capture inputs.' ' Stores inputs to r8inputs.zip') parser.add_argument( + '--dump-unknown-refs', + action='store_true', + help='Log all reasons why API modelling cannot determine API level') + parser.add_argument( '--stamp', help='File to touch upon success. Mutually exclusive with --output-path') parser.add_argument('--desugared-library-keep-rule-output', @@ -153,13 +143,18 @@ if bool(options.keep_rules_targets_regex) != bool( options.keep_rules_output_path): - raise Exception('You must path both --keep-rules-targets-regex and ' - '--keep-rules-output-path') + parser.error('You must path both --keep-rules-targets-regex and ' + '--keep-rules-output-path') - options.classpath = build_utils.ParseGnList(options.classpath) - options.proguard_configs = build_utils.ParseGnList(options.proguard_configs) - options.input_paths = build_utils.ParseGnList(options.input_paths) - options.extra_mapping_output_paths = build_utils.ParseGnList( + if options.force_enable_assertions and options.assertion_handler: + parser.error('Cannot use both --force-enable-assertions and ' + '--assertion-handler') + + options.classpath = action_helpers.parse_gn_list(options.classpath) + options.proguard_configs = action_helpers.parse_gn_list( + options.proguard_configs) + options.input_paths = action_helpers.parse_gn_list(options.input_paths) + options.extra_mapping_output_paths = action_helpers.parse_gn_list( options.extra_mapping_output_paths) if options.feature_names: @@ -170,7 +165,7 @@ parser.error('Invalid feature argument lengths.') options.feature_jars = [ - build_utils.ParseGnList(x) for x in options.feature_jars + action_helpers.parse_gn_list(x) for x in options.feature_jars ] split_map = {} @@ -186,7 +181,7 @@ return options -class _SplitContext(object): +class _SplitContext: def __init__(self, name, output_path, input_jars, work_dir, parent_name=None): self.name = name self.parent_name = parent_name @@ -195,18 +190,12 @@ self.staging_dir = os.path.join(work_dir, name) os.mkdir(self.staging_dir) - def CreateOutput(self, has_imported_lib=False, keep_rule_output=None): + def CreateOutput(self): found_files = build_utils.FindInDirectory(self.staging_dir) if not found_files: raise Exception('Missing dex outputs in {}'.format(self.staging_dir)) if self.final_output_path.endswith('.dex'): - if has_imported_lib: - raise Exception( - 'Trying to create a single .dex file, but a dependency requires ' - 'JDK Library Desugaring (which necessitates a second file).' - 'Refer to %s to see what desugaring was required' % - keep_rule_output) if len(found_files) != 1: raise Exception('Expected exactly 1 dex file output, found: {}'.format( '\t'.join(found_files))) @@ -216,52 +205,12 @@ # Add to .jar using Python rather than having R8 output to a .zip directly # in order to disable compression of the .jar, saving ~500ms. tmp_jar_output = self.staging_dir + '.jar' - build_utils.DoZip(found_files, tmp_jar_output, base_dir=self.staging_dir) + zip_helpers.add_files_to_zip(found_files, + tmp_jar_output, + base_dir=self.staging_dir) shutil.move(tmp_jar_output, self.final_output_path) -def _DeDupeInputJars(split_contexts_by_name): - """Moves jars used by multiple splits into common ancestors. - - Updates |input_jars| for each _SplitContext. - """ - - def count_ancestors(split_context): - ret = 0 - if split_context.parent_name: - ret += 1 - ret += count_ancestors(split_contexts_by_name[split_context.parent_name]) - return ret - - base_context = split_contexts_by_name['base'] - # Sort by tree depth so that ensure children are visited before their parents. - sorted_contexts = list(split_contexts_by_name.values()) - sorted_contexts.remove(base_context) - sorted_contexts.sort(key=count_ancestors, reverse=True) - - # If a jar is present in multiple siblings, promote it to their parent. - seen_jars_by_parent = defaultdict(set) - for split_context in sorted_contexts: - seen_jars = seen_jars_by_parent[split_context.parent_name] - new_dupes = seen_jars.intersection(split_context.input_jars) - parent_context = split_contexts_by_name[split_context.parent_name] - parent_context.input_jars.update(new_dupes) - seen_jars.update(split_context.input_jars) - - def ancestor_jars(parent_name, dest=None): - dest = dest or set() - if not parent_name: - return dest - parent_context = split_contexts_by_name[parent_name] - dest.update(parent_context.input_jars) - return ancestor_jars(parent_context.parent_name, dest) - - # Now that jars have been moved up the tree, remove those that appear in - # ancestors. - for split_context in sorted_contexts: - split_context.input_jars -= ancestor_jars(split_context.parent_name) - - def _OptimizeWithR8(options, config_paths, libraries, @@ -304,20 +253,27 @@ base_context = split_contexts_by_name['base'] # R8 OOMs with the default xmx=1G. - cmd = build_utils.JavaCmd(options.warnings_as_errors, xmx='2G') + [ - '-Dcom.android.tools.r8.allowTestProguardOptions=1', - '-Dcom.android.tools.r8.verticalClassMerging=1', - '-Dcom.android.tools.r8.disableHorizontalClassMerging=1', + cmd = build_utils.JavaCmd(xmx='2G') + [ + # Allows -whyareyounotinlining, which we don't have by default, but + # which is useful for one-off queries. + '-Dcom.android.tools.r8.experimental.enablewhyareyounotinlining=1', + # Restricts horizontal class merging to apply only to classes that + # share a .java file (nested classes). https://crbug.com/1363709 + '-Dcom.android.tools.r8.enableSameFilePolicy=1', ] - if options.disable_outlining: - cmd += ['-Dcom.android.tools.r8.disableOutlining=1'] if options.dump_inputs: - cmd += ['-Dcom.android.tools.r8.dumpinputtofile=r8inputs.zip'] + cmd += [f'-Dcom.android.tools.r8.dumpinputtodirectory={_DUMP_DIR_NAME}'] + if options.dump_unknown_refs: + cmd += ['-Dcom.android.tools.r8.reportUnknownApiReferences=1'] cmd += [ '-cp', options.r8_path, 'com.android.tools.r8.R8', '--no-data-resources', + '--map-id-template', + f'{options.source_file} ({options.package_name})', + '--source-file-template', + options.source_file, '--output', base_context.staging_dir, '--pg-map-output', @@ -325,21 +281,18 @@ ] if options.disable_checks: - # Info level priority logs are not printed by default. - cmd += ['--map-diagnostics:CheckDiscardDiagnostic', 'error', 'info'] - - if options.desugar_jdk_libs_json: - cmd += [ - '--desugared-lib', - options.desugar_jdk_libs_json, - '--desugared-lib-pg-conf-output', - options.desugared_library_keep_rule_output, - ] + cmd += ['--map-diagnostics:CheckDiscardDiagnostic', 'error', 'none'] + cmd += ['--map-diagnostics', 'info', 'warning'] + # An "error" level diagnostic causes r8 to return an error exit code. Doing + # this allows our filter to decide what should/shouldn't break our build. + cmd += ['--map-diagnostics', 'error', 'warning'] if options.min_api: cmd += ['--min-api', options.min_api] - if options.force_enable_assertions: + if options.assertion_handler: + cmd += ['--force-assertions-handler:' + options.assertion_handler] + elif options.force_enable_assertions: cmd += ['--force-enable-assertions'] for lib in libraries: @@ -352,8 +305,6 @@ for main_dex_rule in options.main_dex_rules_path: cmd += ['--main-dex-rules', main_dex_rule] - _DeDupeInputJars(split_contexts_by_name) - # Add any extra inputs to the base context (e.g. desugar runtime). extra_jars = set(options.input_paths) for split_context in split_contexts_by_name.values(): @@ -376,61 +327,26 @@ print_stdout=print_stdout, stderr_filter=stderr_filter, fail_on_output=options.warnings_as_errors) - except build_utils.CalledProcessError as err: - debugging_link = ('\n\nR8 failed. Please see {}.'.format( - 'https://chromium.googlesource.com/chromium/src/+/HEAD/build/' - 'android/docs/java_optimization.md#Debugging-common-failures\n')) - raise build_utils.CalledProcessError(err.cwd, err.args, - err.output + debugging_link) - - base_has_imported_lib = False - if options.desugar_jdk_libs_json: - logging.debug('Running L8') - existing_files = build_utils.FindInDirectory(base_context.staging_dir) - jdk_dex_output = os.path.join(base_context.staging_dir, - 'classes%d.dex' % (len(existing_files) + 1)) - # Use -applymapping to avoid name collisions. - l8_dynamic_config_path = os.path.join(tmp_dir, 'l8_dynamic_config.flags') - with open(l8_dynamic_config_path, 'w') as f: - f.write("-applymapping '{}'\n".format(tmp_mapping_path)) - # Pass the dynamic config so that obfuscation options are picked up. - l8_config_paths = [dynamic_config_path, l8_dynamic_config_path] - if os.path.exists(options.desugared_library_keep_rule_output): - l8_config_paths.append(options.desugared_library_keep_rule_output) - - base_has_imported_lib = dex_jdk_libs.DexJdkLibJar( - options.r8_path, options.min_api, options.desugar_jdk_libs_json, - options.desugar_jdk_libs_jar, - options.desugar_jdk_libs_configuration_jar, jdk_dex_output, - options.warnings_as_errors, l8_config_paths) - if int(options.min_api) >= 24 and base_has_imported_lib: - with open(jdk_dex_output, 'rb') as f: - dexfile = dex_parser.DexFile(bytearray(f.read())) - for m in dexfile.IterMethodSignatureParts(): - print('{}#{}'.format(m[0], m[2])) - assert False, ( - 'Desugared JDK libs are disabled on Monochrome and newer - see ' - 'crbug.com/1159984 for details, and see above list for desugared ' - 'classes and methods.') + except build_utils.CalledProcessError as e: + # Do not output command line because it is massive and makes the actual + # error message hard to find. + sys.stderr.write(e.output) + sys.exit(1) logging.debug('Collecting ouputs') - base_context.CreateOutput(base_has_imported_lib, - options.desugared_library_keep_rule_output) + base_context.CreateOutput() for split_context in split_contexts_by_name.values(): if split_context is not base_context: split_context.CreateOutput() - with open(options.mapping_output, 'w') as out_file, \ - open(tmp_mapping_path) as in_file: - # Mapping files generated by R8 include comments that may break - # some of our tooling so remove those (specifically: apkanalyzer). - out_file.writelines(l for l in in_file if not l.startswith('#')) - return base_context + shutil.move(tmp_mapping_path, options.mapping_output) + return split_contexts_by_name def _OutputKeepRules(r8_path, input_paths, classpath, targets_re_string, keep_rules_output): - cmd = build_utils.JavaCmd(False) + [ + + cmd = build_utils.JavaCmd() + [ '-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences', '--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning', '--keep-rules', '--output', keep_rules_output @@ -448,8 +364,13 @@ def _CheckForMissingSymbols(r8_path, dex_files, classpath, warnings_as_errors, - error_title): - cmd = build_utils.JavaCmd(warnings_as_errors) + [ + dump_inputs, error_title): + cmd = build_utils.JavaCmd() + + if dump_inputs: + cmd += [f'-Dcom.android.tools.r8.dumpinputtodirectory={_DUMP_DIR_NAME}'] + + cmd += [ '-cp', r8_path, 'com.android.tools.r8.tracereferences.TraceReferences', '--map-diagnostics:MissingDefinitionsDiagnostic', 'error', 'warning', '--check' @@ -460,6 +381,8 @@ for path in dex_files: cmd += ['--source', path] + failed_holder = [False] + def stderr_filter(stderr): ignored_lines = [ # Summary contains warning count, which our filtering makes wrong. @@ -467,54 +390,46 @@ # TODO(agrieve): Create interface jars for these missing classes rather # than allowlisting here. - 'dalvik/system', - 'libcore/io', - 'sun/misc/Unsafe', + 'dalvik.system', + 'libcore.io', + 'sun.misc.Unsafe', # Found in: com/facebook/fbui/textlayoutbuilder/StaticLayoutHelper - ('android/text/StaticLayout;<init>(Ljava/lang/CharSequence;IILandroid' - '/text/TextPaint;ILandroid/text/Layout$Alignment;Landroid/text/' - 'TextDirectionHeuristic;FFZLandroid/text/TextUtils$TruncateAt;II)V'), - - # Found in - # com/google/android/gms/cast/framework/media/internal/ResourceProvider - # Missing due to setting "strip_resources = true". - 'com/google/android/gms/cast/framework/R', - - # Found in com/google/android/gms/common/GoogleApiAvailability - # Missing due to setting "strip_drawables = true". - 'com/google/android/gms/base/R$drawable', + 'android.text.StaticLayout.<init>', + # TODO(crbug/1426964): Remove once chrome builds with Android U SDK. + 'android.adservices.measurement', # Explicictly guarded by try (NoClassDefFoundError) in Flogger's # PlatformProvider. - 'com/google/common/flogger/backend/google/GooglePlatform', - 'com/google/common/flogger/backend/system/DefaultPlatform', - - # trichrome_webview_google_bundle contains this missing reference. - # TODO(crbug.com/1142530): Fix this missing reference properly. - 'org/chromium/build/NativeLibraries', + 'com.google.common.flogger.backend.google.GooglePlatform', + 'com.google.common.flogger.backend.system.DefaultPlatform', # TODO(agrieve): Exclude these only when use_jacoco_coverage=true. - 'Ljava/lang/instrument/ClassFileTransformer', - 'Ljava/lang/instrument/IllegalClassFormatException', - 'Ljava/lang/instrument/Instrumentation', - 'Ljava/lang/management/ManagementFactory', - 'Ljavax/management/MBeanServer', - 'Ljavax/management/ObjectInstance', - 'Ljavax/management/ObjectName', - 'Ljavax/management/StandardMBean', + 'java.lang.instrument.ClassFileTransformer', + 'java.lang.instrument.IllegalClassFormatException', + 'java.lang.instrument.Instrumentation', + 'java.lang.management.ManagementFactory', + 'javax.management.MBeanServer', + 'javax.management.ObjectInstance', + 'javax.management.ObjectName', + 'javax.management.StandardMBean', # Explicitly guarded by try (NoClassDefFoundError) in Firebase's # KotlinDetector: com.google.firebase.platforminfo.KotlinDetector. - 'Lkotlin/KotlinVersion', + 'kotlin.KotlinVersion', + + # TODO(agrieve): Remove once we move to Android U SDK. + 'android.window.BackEvent', + 'android.window.OnBackAnimationCallback', ] had_unfiltered_items = ' ' in stderr stderr = build_utils.FilterLines( stderr, '|'.join(re.escape(x) for x in ignored_lines)) if stderr: - if ' ' in stderr: - stderr = error_title + """ + if 'Missing' in stderr: + failed_holder[0] = True + stderr = 'TraceReferences failed: ' + error_title + """ Tip: Build with: is_java_debug=false treat_warnings_as_errors=false @@ -530,47 +445,67 @@ stderr += """ You may need to update build configs to run FragmentActivityReplacer for additional targets. See -https://chromium.googlesource.com/chromium/src.git/+/master/docs/ui/android/bytecode_rewriting.md. +https://chromium.googlesource.com/chromium/src.git/+/main/docs/ui/android/bytecode_rewriting.md. """ elif had_unfiltered_items: # Left only with empty headings. All indented items filtered out. stderr = '' return stderr - logging.debug('cmd: %s', ' '.join(cmd)) - build_utils.CheckOutput(cmd, - print_stdout=True, - stderr_filter=stderr_filter, - fail_on_output=warnings_as_errors) + try: + build_utils.CheckOutput(cmd, + print_stdout=True, + stderr_filter=stderr_filter, + fail_on_output=warnings_as_errors) + except build_utils.CalledProcessError as e: + # Do not output command line because it is massive and makes the actual + # error message hard to find. + sys.stderr.write(e.output) + sys.exit(1) + return failed_holder[0] -def _CombineConfigs(configs, dynamic_config_data, exclude_generated=False): - ret = [] - +def _CombineConfigs(configs, + dynamic_config_data, + embedded_configs, + exclude_generated=False): # Sort in this way so //clank versions of the same libraries will sort # to the same spot in the file. def sort_key(path): return tuple(reversed(path.split(os.path.sep))) - for config in sorted(configs, key=sort_key): - if exclude_generated and config.endswith('.resources.proguard.txt'): - continue - - with open(config) as config_file: - contents = config_file.read().rstrip() - + def format_config_contents(path, contents): + formatted_contents = [] if not contents.strip(): - # Ignore empty files. - continue + return [] # Fix up line endings (third_party configs can have windows endings). contents = contents.replace('\r', '') # Remove numbers from generated rule comments to make file more # diff'able. contents = re.sub(r' #generated:\d+', '', contents) - ret.append('# File: ' + config) - ret.append(contents) - ret.append('') + formatted_contents.append('# File: ' + path) + formatted_contents.append(contents) + formatted_contents.append('') + return formatted_contents + + ret = [] + for config in sorted(configs, key=sort_key): + if exclude_generated and config.endswith('.resources.proguard.txt'): + continue + + # Exclude some confs from expectations. + if any(entry in config for entry in _BLOCKLISTED_EXPECTATION_PATHS): + continue + + with open(config) as config_file: + contents = config_file.read().rstrip() + + ret.extend(format_config_contents(config, contents)) + + for path, contents in sorted(embedded_configs.items()): + ret.extend(format_config_contents(path, contents)) + if dynamic_config_data: ret.append('# File: //build/android/gyp/proguard.py (generated rules)') @@ -580,15 +515,7 @@ def _CreateDynamicConfig(options): - # Our scripts already fail on output. Adding -ignorewarnings makes R8 output - # warnings rather than throw exceptions so we can selectively ignore them via - # dex.py's ignore list. Context: https://crbug.com/1180222 - ret = ["-ignorewarnings"] - - if options.sourcefile: - ret.append("-renamesourcefileattribute '%s' # OMIT FROM EXPECTATIONS" % - options.sourcefile) - + ret = [] if options.enable_obfuscation: ret.append("-repackageclasses ''") else: @@ -597,39 +524,30 @@ if options.apply_mapping: ret.append("-applymapping '%s'" % options.apply_mapping) - _min_api = int(options.min_api) if options.min_api else 0 - for api_level, version_code in _API_LEVEL_VERSION_CODE: - annotation_name = 'org.chromium.base.annotations.VerifiesOn' + version_code - if api_level > _min_api: - ret.append('-keep @interface %s' % annotation_name) - ret.append("""\ --if @%s class * { - *** *(...); -} --keep,allowobfuscation class <1> { - *** <2>(...); -}""" % annotation_name) - ret.append("""\ --keepclassmembers,allowobfuscation class ** { - @%s <methods>; -}""" % annotation_name) return '\n'.join(ret) -def _VerifyNoEmbeddedConfigs(jar_paths): - failed = False - for jar_path in jar_paths: - with zipfile.ZipFile(jar_path) as z: - for name in z.namelist(): - if name.startswith('META-INF/proguard/'): - failed = True - sys.stderr.write("""\ -Found embedded proguard config within {}. -Embedded configs are not permitted (https://crbug.com/989505) -""".format(jar_path)) - break - if failed: - sys.exit(1) +def _ExtractEmbeddedConfigs(jar_path, embedded_configs): + with zipfile.ZipFile(jar_path) as z: + proguard_names = [] + r8_names = [] + for info in z.infolist(): + if info.is_dir(): + continue + if info.filename.startswith('META-INF/proguard/'): + proguard_names.append(info.filename) + elif info.filename.startswith('META-INF/com.android.tools/r8/'): + r8_names.append(info.filename) + elif info.filename.startswith('META-INF/com.android.tools/r8-from'): + # Assume our version of R8 is always latest. + if '-upto-' not in info.filename: + r8_names.append(info.filename) + + # Give preference to r8-from-*, then r8/, then proguard/. + active = r8_names or proguard_names + for filename in active: + config_path = '{}:{}'.format(jar_path, filename) + embedded_configs[config_path] = z.read(filename).decode('utf-8').rstrip() def _ContainsDebuggingConfig(config_str): @@ -643,79 +561,130 @@ build_utils.Touch(options.stamp) output = options.stamp if options.depfile: - build_utils.WriteDepfile(options.depfile, output, inputs=inputs) + action_helpers.write_depfile(options.depfile, output, inputs=inputs) -def main(): - build_utils.InitLogging('PROGUARD_DEBUG') - options = _ParseOptions() +def _IterParentContexts(context_name, split_contexts_by_name): + while context_name: + context = split_contexts_by_name[context_name] + yield context + context_name = context.parent_name - logging.debug('Preparing configs') - proguard_configs = options.proguard_configs +def _DoTraceReferencesChecks(options, split_contexts_by_name): + # Set of all contexts that are a parent to another. + parent_splits_context_names = { + c.parent_name + for c in split_contexts_by_name.values() if c.parent_name + } + context_sets = [ + list(_IterParentContexts(n, split_contexts_by_name)) + for n in parent_splits_context_names + ] + # Visit them in order of: base, base+chrome, base+chrome+thing. + context_sets.sort(key=lambda x: (len(x), x[0].name)) + + # Ensure there are no missing references when considering all dex files. + error_title = 'DEX contains references to non-existent symbols after R8.' + dex_files = sorted(c.final_output_path + for c in split_contexts_by_name.values()) + if _CheckForMissingSymbols(options.r8_path, dex_files, options.classpath, + options.warnings_as_errors, options.dump_inputs, + error_title): + # Failed but didn't raise due to warnings_as_errors=False + return + + for context_set in context_sets: + # Ensure there are no references from base -> chrome module, or from + # chrome -> feature modules. + error_title = (f'DEX within module "{context_set[0].name}" contains ' + 'reference(s) to symbols within child splits') + dex_files = [c.final_output_path for c in context_set] + # Each check currently takes about 3 seconds on a fast dev machine, and we + # run 3 of them (all, base, base+chrome). + # We could run them concurrently, to shave off 5-6 seconds, but would need + # to make sure that the order is maintained. + if _CheckForMissingSymbols(options.r8_path, dex_files, options.classpath, + options.warnings_as_errors, options.dump_inputs, + error_title): + # Failed but didn't raise due to warnings_as_errors=False + return + + +def _Run(options): # ProGuard configs that are derived from flags. + logging.debug('Preparing configs') dynamic_config_data = _CreateDynamicConfig(options) + logging.debug('Looking for embedded configs') + # If a jar is part of input no need to include it as library jar. + libraries = [p for p in options.classpath if p not in options.input_paths] + + embedded_configs = {} + for jar_path in options.input_paths + libraries: + _ExtractEmbeddedConfigs(jar_path, embedded_configs) + # ProGuard configs that are derived from flags. - merged_configs = _CombineConfigs( - proguard_configs, dynamic_config_data, exclude_generated=True) + merged_configs = _CombineConfigs(options.proguard_configs, + dynamic_config_data, + embedded_configs, + exclude_generated=True) print_stdout = _ContainsDebuggingConfig(merged_configs) or options.verbose + depfile_inputs = options.proguard_configs + options.input_paths + libraries if options.expected_file: diff_utils.CheckExpectations(merged_configs, options) if options.only_verify_expectations: - build_utils.WriteDepfile(options.depfile, - options.actual_file, - inputs=options.proguard_configs) + action_helpers.write_depfile(options.depfile, + options.actual_file, + inputs=depfile_inputs) return - logging.debug('Looking for embedded configs') - libraries = [] - for p in options.classpath: - # TODO(bjoyce): Remove filter once old android support libraries are gone. - # Fix for having Library class extend program class dependency problem. - if 'com_android_support' in p or 'android_support_test' in p: - continue - # If a jar is part of input no need to include it as library jar. - if p not in libraries and p not in options.input_paths: - libraries.append(p) - _VerifyNoEmbeddedConfigs(options.input_paths + libraries) if options.keep_rules_output_path: _OutputKeepRules(options.r8_path, options.input_paths, options.classpath, options.keep_rules_targets_regex, options.keep_rules_output_path) return - base_context = _OptimizeWithR8(options, proguard_configs, libraries, - dynamic_config_data, print_stdout) + split_contexts_by_name = _OptimizeWithR8(options, options.proguard_configs, + libraries, dynamic_config_data, + print_stdout) if not options.disable_checks: logging.debug('Running tracereferences') - all_dex_files = [] - if options.output_path: - all_dex_files.append(options.output_path) - if options.dex_dests: - all_dex_files.extend(options.dex_dests) - error_title = 'DEX contains references to non-existent symbols after R8.' - _CheckForMissingSymbols(options.r8_path, all_dex_files, options.classpath, - options.warnings_as_errors, error_title) - # Also ensure that base module doesn't have any references to child dex - # symbols. - # TODO(agrieve): Remove this check once r8 desugaring is fixed to not put - # synthesized classes in the base module. - error_title = 'Base module DEX contains references symbols within DFMs.' - _CheckForMissingSymbols(options.r8_path, [base_context.final_output_path], - options.classpath, options.warnings_as_errors, - error_title) + _DoTraceReferencesChecks(options, split_contexts_by_name) for output in options.extra_mapping_output_paths: shutil.copy(options.mapping_output, output) - inputs = options.proguard_configs + options.input_paths + libraries if options.apply_mapping: - inputs.append(options.apply_mapping) + depfile_inputs.append(options.apply_mapping) - _MaybeWriteStampAndDepFile(options, inputs) + _MaybeWriteStampAndDepFile(options, depfile_inputs) + + +def main(): + build_utils.InitLogging('PROGUARD_DEBUG') + options = _ParseOptions() + + if options.dump_inputs: + # Dumping inputs causes output to be emitted, avoid failing due to stdout. + options.warnings_as_errors = False + # Use dumpinputtodirectory instead of dumpinputtofile to avoid failing the + # build and keep running tracereferences. + dump_dir_name = _DUMP_DIR_NAME + dump_dir_path = pathlib.Path(dump_dir_name) + if dump_dir_path.exists(): + shutil.rmtree(dump_dir_path) + # The directory needs to exist before r8 adds the zip files in it. + dump_dir_path.mkdir() + + # This ensure that the final outputs are zipped and easily uploaded to a bug. + try: + _Run(options) + finally: + if options.dump_inputs: + zip_helpers.zip_directory('r8inputs.zip', _DUMP_DIR_NAME) if __name__ == '__main__':
diff --git a/build/android/gyp/proguard.pydeps b/build/android/gyp/proguard.pydeps index c1de73b..7ee251b 100644 --- a/build/android/gyp/proguard.pydeps +++ b/build/android/gyp/proguard.pydeps
@@ -1,16 +1,12 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/proguard.pydeps build/android/gyp/proguard.py +../../action_helpers.py ../../gn_helpers.py ../../print_python_deps.py -../convert_dex_profile.py -../pylib/__init__.py -../pylib/dex/__init__.py -../pylib/dex/dex_parser.py +../../zip_helpers.py dex.py -dex_jdk_libs.py proguard.py util/__init__.py util/build_utils.py util/diff_utils.py util/md5_check.py -util/zipalign.py
diff --git a/build/android/gyp/resources_shrinker/BUILD.gn b/build/android/gyp/resources_shrinker/BUILD.gn deleted file mode 100644 index e6381e1..0000000 --- a/build/android/gyp/resources_shrinker/BUILD.gn +++ /dev/null
@@ -1,15 +0,0 @@ -import("//build/config/android/rules.gni") - -java_binary("resources_shrinker") { - sources = [ "//build/android/gyp/resources_shrinker/Shrinker.java" ] - main_class = "build.android.gyp.resources_shrinker.Shrinker" - deps = [ - "//third_party/android_deps:com_android_tools_common_java", - "//third_party/android_deps:com_android_tools_layoutlib_layoutlib_api_java", - "//third_party/android_deps:com_android_tools_sdk_common_java", - "//third_party/android_deps:com_google_guava_guava_java", - "//third_party/android_deps:org_jetbrains_kotlin_kotlin_stdlib_java", - "//third_party/r8:r8_java", - ] - wrapper_script_name = "helper/resources_shrinker" -}
diff --git a/build/android/gyp/resources_shrinker/Shrinker.java b/build/android/gyp/resources_shrinker/Shrinker.java deleted file mode 100644 index 50e2f93..0000000 --- a/build/android/gyp/resources_shrinker/Shrinker.java +++ /dev/null
@@ -1,599 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Modifications are owned by the Chromium Authors. -// Copyright 2021 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package build.android.gyp.resources_shrinker; - -import static com.android.ide.common.symbols.SymbolIo.readFromAapt; -import static com.android.utils.SdkUtils.endsWithIgnoreCase; -import static com.google.common.base.Charsets.UTF_8; - -import com.android.ide.common.resources.usage.ResourceUsageModel; -import com.android.ide.common.resources.usage.ResourceUsageModel.Resource; -import com.android.ide.common.symbols.Symbol; -import com.android.ide.common.symbols.SymbolTable; -import com.android.resources.ResourceFolderType; -import com.android.resources.ResourceType; -import com.android.tools.r8.CompilationFailedException; -import com.android.tools.r8.ProgramResource; -import com.android.tools.r8.ProgramResourceProvider; -import com.android.tools.r8.ResourceShrinker; -import com.android.tools.r8.ResourceShrinker.Command; -import com.android.tools.r8.ResourceShrinker.ReferenceChecker; -import com.android.tools.r8.origin.PathOrigin; -import com.android.utils.XmlUtils; -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; -import com.google.common.collect.Maps; -import com.google.common.io.ByteStreams; -import com.google.common.io.Closeables; - -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.xml.sax.SAXException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import javax.xml.parsers.ParserConfigurationException; - -/** - Copied with modifications from gradle core source - https://android.googlesource.com/platform/tools/base/+/master/build-system/gradle-core/src/main/groovy/com/android/build/gradle/tasks/ResourceUsageAnalyzer.java - - Modifications are mostly to: - - Remove unused code paths to reduce complexity. - - Reduce dependencies unless absolutely required. -*/ - -public class Shrinker { - private static final String ANDROID_RES = "android_res/"; - private static final String DOT_DEX = ".dex"; - private static final String DOT_CLASS = ".class"; - private static final String DOT_XML = ".xml"; - private static final String DOT_JAR = ".jar"; - private static final String FN_RESOURCE_TEXT = "R.txt"; - - /* A source of resource classes to track, can be either a folder or a jar */ - private final Iterable<File> mRTxtFiles; - private final File mProguardMapping; - /** These can be class or dex files. */ - private final Iterable<File> mClasses; - private final Iterable<File> mManifests; - private final Iterable<File> mResourceDirs; - - private final File mReportFile; - private final StringWriter mDebugOutput; - private final PrintWriter mDebugPrinter; - - /** Easy way to invoke more verbose output for debugging */ - private boolean mDebug = false; - - /** The computed set of unused resources */ - private List<Resource> mUnused; - - /** - * Map from resource class owners (VM format class) to corresponding resource entries. - * This lets us map back from code references (obfuscated class and possibly obfuscated field - * reference) back to the corresponding resource type and name. - */ - private Map<String, Pair<ResourceType, Map<String, String>>> mResourceObfuscation = - Maps.newHashMapWithExpectedSize(30); - - /** Obfuscated name of android/support/v7/widget/SuggestionsAdapter.java */ - private String mSuggestionsAdapter; - - /** Obfuscated name of android/support/v7/internal/widget/ResourcesWrapper.java */ - private String mResourcesWrapper; - - /* A Pair class because java does not come with batteries included. */ - private static class Pair<U, V> { - private U mFirst; - private V mSecond; - - Pair(U first, V second) { - this.mFirst = first; - this.mSecond = second; - } - - public U getFirst() { - return mFirst; - } - - public V getSecond() { - return mSecond; - } - } - - public Shrinker(Iterable<File> rTxtFiles, Iterable<File> classes, Iterable<File> manifests, - File mapping, Iterable<File> resources, File reportFile) { - mRTxtFiles = rTxtFiles; - mProguardMapping = mapping; - mClasses = classes; - mManifests = manifests; - mResourceDirs = resources; - - mReportFile = reportFile; - if (reportFile != null) { - mDebugOutput = new StringWriter(8 * 1024); - mDebugPrinter = new PrintWriter(mDebugOutput); - } else { - mDebugOutput = null; - mDebugPrinter = null; - } - } - - public void close() { - if (mDebugOutput != null) { - String output = mDebugOutput.toString(); - - if (mReportFile != null) { - File dir = mReportFile.getParentFile(); - if (dir != null) { - if ((dir.exists() || dir.mkdir()) && dir.canWrite()) { - try { - Files.asCharSink(mReportFile, Charsets.UTF_8).write(output); - } catch (IOException ignore) { - } - } - } - } - } - } - - public void analyze() throws IOException, ParserConfigurationException, SAXException { - gatherResourceValues(mRTxtFiles); - recordMapping(mProguardMapping); - - for (File jarOrDir : mClasses) { - recordClassUsages(jarOrDir); - } - recordManifestUsages(mManifests); - recordResources(mResourceDirs); - dumpReferences(); - mModel.processToolsAttributes(); - mUnused = mModel.findUnused(); - } - - public void emitConfig(Path destination) throws IOException { - File destinationFile = destination.toFile(); - if (!destinationFile.exists()) { - destinationFile.getParentFile().mkdirs(); - boolean success = destinationFile.createNewFile(); - if (!success) { - throw new IOException("Could not create " + destination); - } - } - StringBuilder sb = new StringBuilder(); - Collections.sort(mUnused); - for (Resource resource : mUnused) { - sb.append(resource.type + "/" + resource.name + "#remove\n"); - } - Files.asCharSink(destinationFile, UTF_8).write(sb.toString()); - } - - private void dumpReferences() { - if (mDebugPrinter != null) { - mDebugPrinter.print(mModel.dumpReferences()); - } - } - - private void recordResources(Iterable<File> resources) - throws IOException, SAXException, ParserConfigurationException { - for (File resDir : resources) { - File[] resourceFolders = resDir.listFiles(); - if (resourceFolders != null) { - for (File folder : resourceFolders) { - ResourceFolderType folderType = - ResourceFolderType.getFolderType(folder.getName()); - if (folderType != null) { - recordResources(folderType, folder); - } - } - } - } - } - - private void recordResources(ResourceFolderType folderType, File folder) - throws ParserConfigurationException, SAXException, IOException { - File[] files = folder.listFiles(); - if (files != null) { - for (File file : files) { - String path = file.getPath(); - mModel.file = file; - try { - boolean isXml = endsWithIgnoreCase(path, DOT_XML); - if (isXml) { - String xml = Files.toString(file, UTF_8); - Document document = XmlUtils.parseDocument(xml, true); - mModel.visitXmlDocument(file, folderType, document); - } else { - mModel.visitBinaryResource(folderType, file); - } - } finally { - mModel.file = null; - } - } - } - } - - void recordMapping(File mapping) throws IOException { - if (mapping == null || !mapping.exists()) { - return; - } - final String arrowString = " -> "; - final String resourceString = ".R$"; - Map<String, String> nameMap = null; - for (String line : Files.readLines(mapping, UTF_8)) { - if (line.startsWith(" ") || line.startsWith("\t")) { - if (nameMap != null) { - // We're processing the members of a resource class: record names into the map - int n = line.length(); - int i = 0; - for (; i < n; i++) { - if (!Character.isWhitespace(line.charAt(i))) { - break; - } - } - if (i < n && line.startsWith("int", i)) { // int or int[] - int start = line.indexOf(' ', i + 3) + 1; - int arrow = line.indexOf(arrowString); - if (start > 0 && arrow != -1) { - int end = line.indexOf(' ', start + 1); - if (end != -1) { - String oldName = line.substring(start, end); - String newName = - line.substring(arrow + arrowString.length()).trim(); - if (!newName.equals(oldName)) { - nameMap.put(newName, oldName); - } - } - } - } - } - continue; - } else { - nameMap = null; - } - int index = line.indexOf(resourceString); - if (index == -1) { - // Record obfuscated names of a few known appcompat usages of - // Resources#getIdentifier that are unlikely to be used for general - // resource name reflection - if (line.startsWith("android.support.v7.widget.SuggestionsAdapter ")) { - mSuggestionsAdapter = - line.substring(line.indexOf(arrowString) + arrowString.length(), - line.indexOf(':') != -1 ? line.indexOf(':') : line.length()) - .trim() - .replace('.', '/') - + DOT_CLASS; - } else if (line.startsWith("android.support.v7.internal.widget.ResourcesWrapper ") - || line.startsWith("android.support.v7.widget.ResourcesWrapper ") - || (mResourcesWrapper == null // Recently wrapper moved - && line.startsWith( - "android.support.v7.widget.TintContextWrapper$TintResources "))) { - mResourcesWrapper = - line.substring(line.indexOf(arrowString) + arrowString.length(), - line.indexOf(':') != -1 ? line.indexOf(':') : line.length()) - .trim() - .replace('.', '/') - + DOT_CLASS; - } - continue; - } - int arrow = line.indexOf(arrowString, index + 3); - if (arrow == -1) { - continue; - } - String typeName = line.substring(index + resourceString.length(), arrow); - ResourceType type = ResourceType.fromClassName(typeName); - if (type == null) { - continue; - } - int end = line.indexOf(':', arrow + arrowString.length()); - if (end == -1) { - end = line.length(); - } - String target = line.substring(arrow + arrowString.length(), end).trim(); - String ownerName = target.replace('.', '/'); - - nameMap = Maps.newHashMap(); - Pair<ResourceType, Map<String, String>> pair = new Pair(type, nameMap); - mResourceObfuscation.put(ownerName, pair); - // For fast lookup in isResourceClass - mResourceObfuscation.put(ownerName + DOT_CLASS, pair); - } - } - - private void recordManifestUsages(File manifest) - throws IOException, ParserConfigurationException, SAXException { - String xml = Files.toString(manifest, UTF_8); - Document document = XmlUtils.parseDocument(xml, true); - mModel.visitXmlDocument(manifest, null, document); - } - - private void recordManifestUsages(Iterable<File> manifests) - throws IOException, ParserConfigurationException, SAXException { - for (File manifest : manifests) { - recordManifestUsages(manifest); - } - } - - private void recordClassUsages(File file) throws IOException { - assert file.isFile(); - if (file.getPath().endsWith(DOT_DEX)) { - byte[] bytes = Files.toByteArray(file); - recordClassUsages(file, file.getName(), bytes); - } else if (file.getPath().endsWith(DOT_JAR)) { - ZipInputStream zis = null; - try { - FileInputStream fis = new FileInputStream(file); - try { - zis = new ZipInputStream(fis); - ZipEntry entry = zis.getNextEntry(); - while (entry != null) { - String name = entry.getName(); - if (name.endsWith(DOT_DEX)) { - byte[] bytes = ByteStreams.toByteArray(zis); - if (bytes != null) { - recordClassUsages(file, name, bytes); - } - } - - entry = zis.getNextEntry(); - } - } finally { - Closeables.close(fis, true); - } - } finally { - Closeables.close(zis, true); - } - } - } - - private void recordClassUsages(File file, String name, byte[] bytes) { - assert name.endsWith(DOT_DEX); - ReferenceChecker callback = new ReferenceChecker() { - @Override - public boolean shouldProcess(String internalName) { - return !isResourceClass(internalName + DOT_CLASS); - } - - @Override - public void referencedInt(int value) { - Shrinker.this.referencedInt("dex", value, file, name); - } - - @Override - public void referencedString(String value) { - // do nothing. - } - - @Override - public void referencedStaticField(String internalName, String fieldName) { - Resource resource = getResourceFromCode(internalName, fieldName); - if (resource != null) { - ResourceUsageModel.markReachable(resource); - } - } - - @Override - public void referencedMethod( - String internalName, String methodName, String methodDescriptor) { - // Do nothing. - } - }; - ProgramResource resource = ProgramResource.fromBytes( - new PathOrigin(file.toPath()), ProgramResource.Kind.DEX, bytes, null); - ProgramResourceProvider provider = () -> Arrays.asList(resource); - try { - Command command = - (new ResourceShrinker.Builder()).addProgramResourceProvider(provider).build(); - ResourceShrinker.run(command, callback); - } catch (CompilationFailedException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (ExecutionException e) { - e.printStackTrace(); - } - } - - /** Returns whether the given class file name points to an aapt-generated compiled R class. */ - boolean isResourceClass(String name) { - if (mResourceObfuscation.containsKey(name)) { - return true; - } - int index = name.lastIndexOf('/'); - if (index != -1 && name.startsWith("R$", index + 1) && name.endsWith(DOT_CLASS)) { - String typeName = name.substring(index + 3, name.length() - DOT_CLASS.length()); - return ResourceType.fromClassName(typeName) != null; - } - return false; - } - - Resource getResourceFromCode(String owner, String name) { - Pair<ResourceType, Map<String, String>> pair = mResourceObfuscation.get(owner); - if (pair != null) { - ResourceType type = pair.getFirst(); - Map<String, String> nameMap = pair.getSecond(); - String renamedField = nameMap.get(name); - if (renamedField != null) { - name = renamedField; - } - return mModel.getResource(type, name); - } - if (isValidResourceType(owner)) { - ResourceType type = - ResourceType.fromClassName(owner.substring(owner.lastIndexOf('$') + 1)); - if (type != null) { - return mModel.getResource(type, name); - } - } - return null; - } - - private Boolean isValidResourceType(String candidateString) { - return candidateString.contains("/") - && candidateString.substring(candidateString.lastIndexOf('/') + 1).contains("$"); - } - - private void gatherResourceValues(Iterable<File> rTxts) throws IOException { - for (File rTxt : rTxts) { - assert rTxt.isFile(); - assert rTxt.getName().endsWith(FN_RESOURCE_TEXT); - addResourcesFromRTxtFile(rTxt); - } - } - - private void addResourcesFromRTxtFile(File file) { - try { - SymbolTable st = readFromAapt(file, null); - for (Symbol symbol : st.getSymbols().values()) { - String symbolValue = symbol.getValue(); - if (symbol.getResourceType() == ResourceType.STYLEABLE) { - if (symbolValue.trim().startsWith("{")) { - // Only add the styleable parent, styleable children are not yet supported. - mModel.addResource(symbol.getResourceType(), symbol.getName(), null); - } - } else { - mModel.addResource(symbol.getResourceType(), symbol.getName(), symbolValue); - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - ResourceUsageModel getModel() { - return mModel; - } - - private void referencedInt(String context, int value, File file, String currentClass) { - Resource resource = mModel.getResource(value); - if (ResourceUsageModel.markReachable(resource) && mDebug) { - assert mDebugPrinter != null : "mDebug is true, but mDebugPrinter is null."; - mDebugPrinter.println("Marking " + resource + " reachable: referenced from " + context - + " in " + file + ":" + currentClass); - } - } - - private final ResourceShrinkerUsageModel mModel = new ResourceShrinkerUsageModel(); - - private class ResourceShrinkerUsageModel extends ResourceUsageModel { - public File file; - - /** - * Whether we should ignore tools attribute resource references. - * <p> - * For example, for resource shrinking we want to ignore tools attributes, - * whereas for resource refactoring on the source code we do not. - * - * @return whether tools attributes should be ignored - */ - @Override - protected boolean ignoreToolsAttributes() { - return true; - } - - @Override - protected void onRootResourcesFound(List<Resource> roots) { - if (mDebugPrinter != null) { - mDebugPrinter.println( - "\nThe root reachable resources are:\n" + Joiner.on(",\n ").join(roots)); - } - } - - @Override - protected Resource declareResource(ResourceType type, String name, Node node) { - Resource resource = super.declareResource(type, name, node); - resource.addLocation(file); - return resource; - } - - @Override - protected void referencedString(String string) { - // Do nothing - } - } - - public static void main(String[] args) throws Exception { - List<File> rTxtFiles = null; // R.txt files - List<File> classes = null; // Dex/jar w dex - List<File> manifests = null; // manifests - File mapping = null; // mapping - List<File> resources = null; // resources dirs - File log = null; // output log for debugging - Path configPath = null; // output config - for (int i = 0; i < args.length; i += 2) { - switch (args[i]) { - case "--rtxts": - rTxtFiles = Arrays.stream(args[i + 1].split(":")) - .map(s -> new File(s)) - .collect(Collectors.toList()); - break; - case "--dex": - classes = Arrays.stream(args[i + 1].split(":")) - .map(s -> new File(s)) - .collect(Collectors.toList()); - break; - case "--manifests": - manifests = Arrays.stream(args[i + 1].split(":")) - .map(s -> new File(s)) - .collect(Collectors.toList()); - break; - case "--mapping": - mapping = new File(args[i + 1]); - break; - case "--resourceDirs": - resources = Arrays.stream(args[i + 1].split(":")) - .map(s -> new File(s)) - .collect(Collectors.toList()); - break; - case "--log": - log = new File(args[i + 1]); - break; - case "--outputConfig": - configPath = Paths.get(args[i + 1]); - break; - default: - throw new IllegalArgumentException(args[i] + " is not a valid arg."); - } - } - Shrinker shrinker = new Shrinker(rTxtFiles, classes, manifests, mapping, resources, log); - shrinker.analyze(); - shrinker.close(); - shrinker.emitConfig(configPath); - } -}
diff --git a/build/android/gyp/resources_shrinker/shrinker.py b/build/android/gyp/resources_shrinker/shrinker.py deleted file mode 100755 index 2800ce2..0000000 --- a/build/android/gyp/resources_shrinker/shrinker.py +++ /dev/null
@@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 -# Copyright (c) 2021 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import argparse -import os -import sys - -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) -from util import build_utils -from util import resource_utils - - -def main(args): - parser = argparse.ArgumentParser() - - build_utils.AddDepfileOption(parser) - parser.add_argument('--script', - required=True, - help='Path to the unused resources detector script.') - parser.add_argument( - '--dependencies-res-zips', - required=True, - help='Resources zip archives to investigate for unused resources.') - parser.add_argument('--dex', - required=True, - help='Path to dex file, or zip with dex files.') - parser.add_argument( - '--proguard-mapping', - required=True, - help='Path to proguard mapping file for the optimized dex.') - parser.add_argument('--r-text', required=True, help='Path to R.txt') - parser.add_argument('--android-manifest', - required=True, - help='Path to AndroidManifest') - parser.add_argument('--output-config', - required=True, - help='Path to output the aapt2 config to.') - args = build_utils.ExpandFileArgs(args) - options = parser.parse_args(args) - options.dependencies_res_zips = (build_utils.ParseGnList( - options.dependencies_res_zips)) - - # in case of no resources, short circuit early. - if not options.dependencies_res_zips: - build_utils.Touch(options.output_config) - return - - with build_utils.TempDir() as temp_dir: - dep_subdirs = [] - for dependency_res_zip in options.dependencies_res_zips: - dep_subdirs += resource_utils.ExtractDeps([dependency_res_zip], temp_dir) - - build_utils.CheckOutput([ - options.script, '--rtxts', options.r_text, '--manifests', - options.android_manifest, '--resourceDirs', ':'.join(dep_subdirs), - '--dex', options.dex, '--mapping', options.proguard_mapping, - '--outputConfig', options.output_config - ]) - - if options.depfile: - depfile_deps = options.dependencies_res_zips + [ - options.r_text, - options.android_manifest, - options.dex, - options.proguard_mapping, - ] - build_utils.WriteDepfile(options.depfile, options.output_config, - depfile_deps) - - -if __name__ == '__main__': - main(sys.argv[1:])
diff --git a/build/android/gyp/resources_shrinker/shrinker.pydeps b/build/android/gyp/resources_shrinker/shrinker.pydeps deleted file mode 100644 index 92c8905..0000000 --- a/build/android/gyp/resources_shrinker/shrinker.pydeps +++ /dev/null
@@ -1,30 +0,0 @@ -# Generated by running: -# build/print_python_deps.py --root build/android/gyp/resources_shrinker --output build/android/gyp/resources_shrinker/shrinker.pydeps build/android/gyp/resources_shrinker/shrinker.py -../../../../third_party/jinja2/__init__.py -../../../../third_party/jinja2/_compat.py -../../../../third_party/jinja2/asyncfilters.py -../../../../third_party/jinja2/asyncsupport.py -../../../../third_party/jinja2/bccache.py -../../../../third_party/jinja2/compiler.py -../../../../third_party/jinja2/defaults.py -../../../../third_party/jinja2/environment.py -../../../../third_party/jinja2/exceptions.py -../../../../third_party/jinja2/filters.py -../../../../third_party/jinja2/idtracking.py -../../../../third_party/jinja2/lexer.py -../../../../third_party/jinja2/loaders.py -../../../../third_party/jinja2/nodes.py -../../../../third_party/jinja2/optimizer.py -../../../../third_party/jinja2/parser.py -../../../../third_party/jinja2/runtime.py -../../../../third_party/jinja2/tests.py -../../../../third_party/jinja2/utils.py -../../../../third_party/jinja2/visitor.py -../../../../third_party/markupsafe/__init__.py -../../../../third_party/markupsafe/_compat.py -../../../../third_party/markupsafe/_native.py -../../../gn_helpers.py -../util/__init__.py -../util/build_utils.py -../util/resource_utils.py -shrinker.py
diff --git a/build/android/gyp/system_image_apks.py b/build/android/gyp/system_image_apks.py new file mode 100755 index 0000000..0b6804b --- /dev/null +++ b/build/android/gyp/system_image_apks.py
@@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Generates APKs for use on system images.""" + +import argparse +import os +import pathlib +import tempfile +import shutil +import sys +import zipfile + +_DIR_SOURCE_ROOT = str(pathlib.Path(__file__).parents[2]) +sys.path.append(os.path.join(_DIR_SOURCE_ROOT, 'build', 'android', 'gyp')) +from util import build_utils + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--input', required=True, help='Input path') + parser.add_argument('--output', required=True, help='Output path') + parser.add_argument('--bundle-wrapper', help='APK operations script path') + parser.add_argument('--fuse-apk', + help='Create single .apk rather than using apk splits', + action='store_true') + args = parser.parse_args() + + if not args.bundle_wrapper: + shutil.copyfile(args.input, args.output) + return + + with tempfile.NamedTemporaryFile(suffix='.apks') as tmp_file: + cmd = [ + args.bundle_wrapper, 'build-bundle-apks', '--output-apks', tmp_file.name + ] + cmd += ['--build-mode', 'system' if args.fuse_apk else 'system_apks'] + + # Creates a .apks zip file that contains the system image APK(s). + build_utils.CheckOutput(cmd) + + if args.fuse_apk: + with zipfile.ZipFile(tmp_file.name) as z: + pathlib.Path(args.output).write_bytes(z.read('system/system.apk')) + return + + # Rename .apk files and remove toc.pb to make it clear that system apks + # should not be installed via bundletool. + with zipfile.ZipFile(tmp_file.name) as z_input, \ + zipfile.ZipFile(args.output, 'w') as z_output: + for info in z_input.infolist(): + if info.filename.endswith('.apk'): + data = z_input.read(info) + info.filename = (info.filename.replace('splits/', + '').replace('-master', '')) + z_output.writestr(info, data) + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/build/android/gyp/system_image_apks.pydeps b/build/android/gyp/system_image_apks.pydeps new file mode 100644 index 0000000..35f1dc9 --- /dev/null +++ b/build/android/gyp/system_image_apks.pydeps
@@ -0,0 +1,6 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/system_image_apks.pydeps build/android/gyp/system_image_apks.py +../../gn_helpers.py +system_image_apks.py +util/__init__.py +util/build_utils.py
diff --git a/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldMain.java b/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldMain.java index 10860d8..2c4d9a2 100644 --- a/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldMain.java +++ b/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldMain.java
@@ -1,4 +1,4 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldPrinter.java b/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldPrinter.java index b09673e..2762b4f 100644 --- a/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldPrinter.java +++ b/build/android/gyp/test/java/org/chromium/helloworld/HelloWorldPrinter.java
@@ -1,4 +1,4 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/gyp/trace_event_bytecode_rewriter.py b/build/android/gyp/trace_event_bytecode_rewriter.py new file mode 100755 index 0000000..3e0e696 --- /dev/null +++ b/build/android/gyp/trace_event_bytecode_rewriter.py
@@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Wrapper script around TraceEventAdder script.""" + +import argparse +import sys +import os + +from util import build_utils +import action_helpers # build_utils adds //build to sys.path. + + +def main(argv): + argv = build_utils.ExpandFileArgs(argv[1:]) + parser = argparse.ArgumentParser() + action_helpers.add_depfile_arg(parser) + parser.add_argument('--script', + required=True, + help='Path to the java binary wrapper script.') + parser.add_argument('--stamp', help='Path to stamp to mark when finished.') + parser.add_argument('--classpath', action='append', nargs='+') + parser.add_argument('--input-jars', action='append', nargs='+') + parser.add_argument('--output-jars', action='append', nargs='+') + args = parser.parse_args(argv) + + args.classpath = action_helpers.parse_gn_list(args.classpath) + args.input_jars = action_helpers.parse_gn_list(args.input_jars) + args.output_jars = action_helpers.parse_gn_list(args.output_jars) + + for output_jar in args.output_jars: + jar_dir = os.path.dirname(output_jar) + if not os.path.exists(jar_dir): + os.makedirs(jar_dir) + + all_input_jars = set(args.classpath + args.input_jars) + cmd = [ + args.script, '--classpath', ':'.join(sorted(all_input_jars)), + ':'.join(args.input_jars), ':'.join(args.output_jars) + ] + build_utils.CheckOutput(cmd, print_stdout=True) + + build_utils.Touch(args.stamp) + + action_helpers.write_depfile(args.depfile, args.stamp, inputs=all_input_jars) + + +if __name__ == '__main__': + sys.exit(main(sys.argv))
diff --git a/build/android/gyp/trace_event_bytecode_rewriter.pydeps b/build/android/gyp/trace_event_bytecode_rewriter.pydeps new file mode 100644 index 0000000..e03fc0c --- /dev/null +++ b/build/android/gyp/trace_event_bytecode_rewriter.pydeps
@@ -0,0 +1,7 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/trace_event_bytecode_rewriter.pydeps build/android/gyp/trace_event_bytecode_rewriter.py +../../action_helpers.py +../../gn_helpers.py +trace_event_bytecode_rewriter.py +util/__init__.py +util/build_utils.py
diff --git a/build/android/gyp/turbine.py b/build/android/gyp/turbine.py index 208cc76..2de92f4 100755 --- a/build/android/gyp/turbine.py +++ b/build/android/gyp/turbine.py
@@ -1,24 +1,35 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Wraps the turbine jar and expands @FileArgs.""" import argparse +import functools import logging -import os -import shutil import sys import time +import zipfile +import compile_java +import javac_output_processor from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers + + +def ProcessJavacOutput(output, target_name): + output_processor = javac_output_processor.JavacOutputProcessor(target_name) + lines = output_processor.Process(output.split('\n')) + return '\n'.join(lines) def main(argv): build_utils.InitLogging('TURBINE_DEBUG') argv = build_utils.ExpandFileArgs(argv[1:]) parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) + parser.add_argument('--target-name', help='Fully qualified GN target name.') parser.add_argument( '--turbine-jar-path', required=True, help='Path to the turbine jar file.') parser.add_argument( @@ -26,15 +37,6 @@ action='append', default=[], help='List of srcjars to include in compilation.') - parser.add_argument( - '--bootclasspath', - action='append', - default=[], - help='Boot classpath for javac. If this is specified multiple times, ' - 'they will all be appended to construct the classpath.') - parser.add_argument( - '--java-version', - help='Java language version to use in -source and -target args to javac.') parser.add_argument('--classpath', action='append', help='Classpath to use.') parser.add_argument( '--processors', @@ -57,13 +59,14 @@ parser.add_argument('--warnings-as-errors', action='store_true', help='Treat all warnings as errors.') + parser.add_argument('--kotlin-jar-path', + help='Kotlin jar to be merged into the output jar.') options, unknown_args = parser.parse_known_args(argv) - options.bootclasspath = build_utils.ParseGnList(options.bootclasspath) - options.classpath = build_utils.ParseGnList(options.classpath) - options.processorpath = build_utils.ParseGnList(options.processorpath) - options.processors = build_utils.ParseGnList(options.processors) - options.java_srcjars = build_utils.ParseGnList(options.java_srcjars) + options.classpath = action_helpers.parse_gn_list(options.classpath) + options.processorpath = action_helpers.parse_gn_list(options.processorpath) + options.processors = action_helpers.parse_gn_list(options.processors) + options.java_srcjars = action_helpers.parse_gn_list(options.java_srcjars) files = [] for arg in unknown_args: @@ -71,10 +74,19 @@ if arg.startswith('@'): files.extend(build_utils.ReadSourcesList(arg[1:])) - cmd = build_utils.JavaCmd(options.warnings_as_errors) + [ + # The target's .sources file contains both Java and Kotlin files. We use + # compile_kt.py to compile the Kotlin files to .class and header jars. + # Turbine is run only on .java files. + java_files = [f for f in files if f.endswith('.java')] + + cmd = build_utils.JavaCmd() + [ '-classpath', options.turbine_jar_path, 'com.google.turbine.main.Main' ] - javac_cmd = [] + javac_cmd = [ + # We currently target JDK 11 everywhere. + '--release', + '11', + ] # Turbine reads lists from command line args by consuming args until one # starts with double dash (--). Thus command line args should be grouped @@ -83,22 +95,6 @@ cmd += ['--processors'] cmd += options.processors - if options.java_version: - javac_cmd.extend([ - '-source', - options.java_version, - '-target', - options.java_version, - ]) - if options.java_version == '1.8': - # Android's boot jar doesn't contain all java 8 classes. - options.bootclasspath.append(build_utils.RT_JAR_PATH) - - if options.bootclasspath: - cmd += ['--bootclasspath'] - for bootclasspath in options.bootclasspath: - cmd += bootclasspath.split(':') - if options.processorpath: cmd += ['--processorpath'] cmd += options.processorpath @@ -115,40 +111,57 @@ cmd += ['--source_jars'] cmd += options.java_srcjars - if files: + if java_files: # Use jar_path to ensure paths are relative (needed for goma). - files_rsp_path = options.jar_path + '.files_list.txt' + files_rsp_path = options.jar_path + '.java_files_list.txt' with open(files_rsp_path, 'w') as f: - f.write(' '.join(files)) - # Pass source paths as response files to avoid extremely long command lines - # that are tedius to debug. + f.write(' '.join(java_files)) + # Pass source paths as response files to avoid extremely long command + # lines that are tedius to debug. cmd += ['--sources'] cmd += ['@' + files_rsp_path] - if javac_cmd: - cmd.append('--javacopts') - cmd += javac_cmd - cmd.append('--') # Terminate javacopts + cmd += ['--javacopts'] + cmd += javac_cmd + cmd += ['--'] # Terminate javacopts # Use AtomicOutput so that output timestamps are not updated when outputs # are not changed. - with build_utils.AtomicOutput(options.jar_path) as output_jar, \ - build_utils.AtomicOutput(options.generated_jar_path) as generated_jar: - cmd += ['--output', output_jar.name, '--gensrc_output', generated_jar.name] + with action_helpers.atomic_output(options.jar_path) as output_jar, \ + action_helpers.atomic_output(options.generated_jar_path) as gensrc_jar: + cmd += ['--output', output_jar.name, '--gensrc_output', gensrc_jar.name] + process_javac_output_partial = functools.partial( + ProcessJavacOutput, target_name=options.target_name) + logging.debug('Command: %s', cmd) start = time.time() - build_utils.CheckOutput(cmd, - print_stdout=True, - fail_on_output=options.warnings_as_errors) + try: + build_utils.CheckOutput(cmd, + print_stdout=True, + stdout_filter=process_javac_output_partial, + stderr_filter=process_javac_output_partial, + fail_on_output=options.warnings_as_errors) + except build_utils.CalledProcessError as e: + # Do not output stacktrace as it takes up space on gerrit UI, forcing + # you to click though to find the actual compilation error. It's never + # interesting to see the Python stacktrace for a Java compilation error. + sys.stderr.write(e.output) + sys.exit(1) end = time.time() - start logging.info('Header compilation took %ss', end) + if options.kotlin_jar_path: + with zipfile.ZipFile(output_jar.name, 'a') as out_zip: + path_transform = lambda p: p if p.endswith('.class') else None + zip_helpers.merge_zips(out_zip, [options.kotlin_jar_path], + path_transform=path_transform) if options.depfile: # GN already knows of the java files, so avoid listing individual java files # in the depfile. - depfile_deps = (options.bootclasspath + options.classpath + - options.processorpath + options.java_srcjars) - build_utils.WriteDepfile(options.depfile, options.jar_path, depfile_deps) + depfile_deps = (options.classpath + options.processorpath + + options.java_srcjars) + action_helpers.write_depfile(options.depfile, options.jar_path, + depfile_deps) if __name__ == '__main__':
diff --git a/build/android/gyp/turbine.pydeps b/build/android/gyp/turbine.pydeps index f0b2411..3d20f2e 100644 --- a/build/android/gyp/turbine.pydeps +++ b/build/android/gyp/turbine.pydeps
@@ -1,6 +1,33 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/turbine.pydeps build/android/gyp/turbine.py +../../../third_party/catapult/devil/devil/__init__.py +../../../third_party/catapult/devil/devil/android/__init__.py +../../../third_party/catapult/devil/devil/android/constants/__init__.py +../../../third_party/catapult/devil/devil/android/constants/chrome.py +../../../third_party/catapult/devil/devil/android/sdk/__init__.py +../../../third_party/catapult/devil/devil/android/sdk/keyevent.py +../../../third_party/catapult/devil/devil/android/sdk/version_codes.py +../../../third_party/catapult/devil/devil/constants/__init__.py +../../../third_party/catapult/devil/devil/constants/exit_codes.py +../../../third_party/colorama/src/colorama/__init__.py +../../../third_party/colorama/src/colorama/ansi.py +../../../third_party/colorama/src/colorama/ansitowin32.py +../../../third_party/colorama/src/colorama/initialise.py +../../../third_party/colorama/src/colorama/win32.py +../../../third_party/colorama/src/colorama/winterm.py +../../../tools/android/modularization/convenience/lookup_dep.py +../../action_helpers.py ../../gn_helpers.py +../../print_python_deps.py +../../zip_helpers.py +../list_java_targets.py +../pylib/__init__.py +../pylib/constants/__init__.py +compile_java.py +javac_output_processor.py turbine.py util/__init__.py util/build_utils.py +util/jar_info_utils.py +util/md5_check.py +util/server_utils.py
diff --git a/build/android/gyp/unused_resources.py b/build/android/gyp/unused_resources.py new file mode 100755 index 0000000..d7578ce --- /dev/null +++ b/build/android/gyp/unused_resources.py
@@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import sys + +from util import build_utils +from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. + + +def _FilterUnusedResources(r_text_in, r_text_out, unused_resources_config): + removed_resources = set() + with open(unused_resources_config, encoding='utf-8') as output_config: + for line in output_config: + # example line: attr/line_height#remove + resource = line.split('#')[0] + resource_type, resource_name = resource.split('/') + removed_resources.add((resource_type, resource_name)) + kept_lines = [] + with open(r_text_in, encoding='utf-8') as infile: + for line in infile: + # example line: int attr line_height 0x7f0014ee + resource_type, resource_name = line.split(' ')[1:3] + if (resource_type, resource_name) not in removed_resources: + kept_lines.append(line) + + with open(r_text_out, 'w', encoding='utf-8') as out_file: + out_file.writelines(kept_lines) + + +def main(args): + parser = argparse.ArgumentParser() + + action_helpers.add_depfile_arg(parser) + parser.add_argument('--script', + required=True, + help='Path to the unused resources detector script.') + parser.add_argument( + '--dependencies-res-zips', + required=True, + action='append', + help='Resources zip archives to investigate for unused resources.') + parser.add_argument('--dexes', + action='append', + required=True, + help='Path to dex file, or zip with dex files.') + parser.add_argument( + '--proguard-mapping', + help='Path to proguard mapping file for the optimized dex.') + parser.add_argument('--r-text-in', required=True, help='Path to input R.txt') + parser.add_argument( + '--r-text-out', + help='Path to output R.txt with unused resources removed.') + parser.add_argument('--android-manifests', + action='append', + required=True, + help='Path to AndroidManifest') + parser.add_argument('--output-config', + required=True, + help='Path to output the aapt2 config to.') + args = build_utils.ExpandFileArgs(args) + options = parser.parse_args(args) + options.dependencies_res_zips = (action_helpers.parse_gn_list( + options.dependencies_res_zips)) + + # in case of no resources, short circuit early. + if not options.dependencies_res_zips: + build_utils.Touch(options.output_config) + return + + with build_utils.TempDir() as temp_dir: + dep_subdirs = [] + for dependency_res_zip in options.dependencies_res_zips: + dep_subdirs += resource_utils.ExtractDeps([dependency_res_zip], temp_dir) + + cmd = [ + options.script, + '--rtxts', + options.r_text_in, + '--manifests', + ':'.join(options.android_manifests), + '--resourceDirs', + ':'.join(dep_subdirs), + '--dexes', + ':'.join(options.dexes), + '--outputConfig', + options.output_config, + ] + if options.proguard_mapping: + cmd += [ + '--mapping', + options.proguard_mapping, + ] + build_utils.CheckOutput(cmd) + + if options.r_text_out: + _FilterUnusedResources(options.r_text_in, options.r_text_out, + options.output_config) + + if options.depfile: + depfile_deps = (options.dependencies_res_zips + options.android_manifests + + options.dexes) + [options.r_text_in] + if options.proguard_mapping: + depfile_deps.append(options.proguard_mapping) + action_helpers.write_depfile(options.depfile, options.output_config, + depfile_deps) + + +if __name__ == '__main__': + main(sys.argv[1:])
diff --git a/build/android/gyp/unused_resources.pydeps b/build/android/gyp/unused_resources.pydeps new file mode 100644 index 0000000..b4da89a --- /dev/null +++ b/build/android/gyp/unused_resources.pydeps
@@ -0,0 +1,30 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/gyp --output build/android/gyp/unused_resources.pydeps build/android/gyp/unused_resources.py +../../../third_party/jinja2/__init__.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py +../../../third_party/jinja2/bccache.py +../../../third_party/jinja2/compiler.py +../../../third_party/jinja2/defaults.py +../../../third_party/jinja2/environment.py +../../../third_party/jinja2/exceptions.py +../../../third_party/jinja2/filters.py +../../../third_party/jinja2/idtracking.py +../../../third_party/jinja2/lexer.py +../../../third_party/jinja2/loaders.py +../../../third_party/jinja2/nodes.py +../../../third_party/jinja2/optimizer.py +../../../third_party/jinja2/parser.py +../../../third_party/jinja2/runtime.py +../../../third_party/jinja2/tests.py +../../../third_party/jinja2/utils.py +../../../third_party/jinja2/visitor.py +../../../third_party/markupsafe/__init__.py +../../../third_party/markupsafe/_compat.py +../../../third_party/markupsafe/_native.py +../../action_helpers.py +../../gn_helpers.py +unused_resources.py +util/__init__.py +util/build_utils.py +util/resource_utils.py
diff --git a/build/android/gyp/util/__init__.py b/build/android/gyp/util/__init__.py index 96196cf..5ffa284 100644 --- a/build/android/gyp/util/__init__.py +++ b/build/android/gyp/util/__init__.py
@@ -1,3 +1,3 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gyp/util/build_utils.py b/build/android/gyp/util/build_utils.py index d1d3a72..f885182 100644 --- a/build/android/gyp/util/build_utils.py +++ b/build/android/gyp/util/build_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -14,11 +14,13 @@ import os import pipes import re +import shlex import shutil import stat import subprocess import sys import tempfile +import textwrap import time import zipfile @@ -36,26 +38,19 @@ JAVA_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current') JAVAC_PATH = os.path.join(JAVA_HOME, 'bin', 'javac') JAVAP_PATH = os.path.join(JAVA_HOME, 'bin', 'javap') -RT_JAR_PATH = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'extras', - 'java_8', 'jre', 'lib', 'rt.jar') +KOTLIN_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'kotlinc', 'current') +KOTLINC_PATH = os.path.join(KOTLIN_HOME, 'bin', 'kotlinc') +# Please avoid using this. Our JAVA_HOME is using a newer and actively patched +# JDK. +JAVA_11_HOME_DEPRECATED = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk11', + 'current') -try: - string_types = basestring -except NameError: - string_types = (str, bytes) - - -def JavaCmd(verify=True, xmx='1G'): +def JavaCmd(xmx='1G'): ret = [os.path.join(JAVA_HOME, 'bin', 'java')] # Limit heap to avoid Java not GC'ing when it should, and causing # bots to OOM when many java commands are runnig at the same time # https://crbug.com/1098333 ret += ['-Xmx' + xmx] - - # Disable bytecode verification for local builds gives a ~2% speed-up. - if not verify: - ret += ['-noverify'] - return ret @@ -97,35 +92,6 @@ return files -def ParseGnList(value): - """Converts a "GN-list" command-line parameter into a list. - - Conversions handled: - * None -> [] - * '' -> [] - * 'asdf' -> ['asdf'] - * '["a", "b"]' -> ['a', 'b'] - * ['["a", "b"]', 'c'] -> ['a', 'b', 'c'] (flattened list) - - The common use for this behavior is in the Android build where things can - take lists of @FileArg references that are expanded via ExpandFileArgs. - """ - # Convert None to []. - if not value: - return [] - # Convert a list of GN lists to a flattened list. - if isinstance(value, list): - ret = [] - for arg in value: - ret.extend(ParseGnList(arg)) - return ret - # Convert normal GN list. - if value.startswith('['): - return gn_helpers.GNValueParser(value).ParseList() - # Convert a single string value to a list. - return [value] - - def CheckOptions(options, parser, required=None): if not required: return @@ -148,24 +114,7 @@ @contextlib.contextmanager -def AtomicOutput(path, only_if_changed=True, mode='w+b'): - """Helper to prevent half-written outputs. - - Args: - path: Path to the final output file, which will be written atomically. - only_if_changed: If True (the default), do not touch the filesystem - if the content has not changed. - mode: The mode to open the file in (str). - Returns: - A python context manager that yelds a NamedTemporaryFile instance - that must be used by clients to write the data to. On exit, the - manager will try to replace the final output file with the - temporary one if necessary. The temporary file is always destroyed - on exit. - Example: - with build_utils.AtomicOutput(output_path) as tmp_file: - subprocess.check_call(['prog', '--output', tmp_file.name]) - """ +def _AtomicOutput(path, only_if_changed=True, mode='w+b'): # Create in same directory to ensure same filesystem when moving. dirname = os.path.dirname(path) if not os.path.exists(dirname): @@ -190,16 +139,21 @@ exits with a non-zero exit code.""" def __init__(self, cwd, args, output): - super(CalledProcessError, self).__init__() + super().__init__() self.cwd = cwd self.args = args self.output = output def __str__(self): # A user should be able to simply copy and paste the command that failed - # into their shell. + # into their shell (unless it is more than 200 chars). + # User can set PRINT_FULL_COMMAND=1 to always print the full command. + print_full = os.environ.get('PRINT_FULL_COMMAND', '0') != '0' + full_cmd = shlex.join(self.args) + short_cmd = textwrap.shorten(full_cmd, width=200) + printed_cmd = full_cmd if print_full else short_cmd copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd), - ' '.join(map(pipes.quote, self.args))) + printed_cmd) return 'Command failed: {}\n{}'.format(copyable_command, self.output) @@ -254,6 +208,7 @@ if not cwd: cwd = os.getcwd() + logging.info('CheckOutput: %s', ' '.join(args)) child = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) stdout, stderr = child.communicate() @@ -279,18 +234,25 @@ has_stdout = print_stdout and stdout has_stderr = print_stderr and stderr - if fail_on_output and (has_stdout or has_stderr): - MSG = """\ + if has_stdout or has_stderr: + if has_stdout and has_stderr: + stream_name = 'stdout and stderr' + elif has_stdout: + stream_name = 'stdout' + else: + stream_name = 'stderr' + + if fail_on_output: + MSG = """ Command failed because it wrote to {}. You can often set treat_warnings_as_errors=false to not treat output as \ -failure (useful when developing locally).""" - if has_stdout and has_stderr: - stream_string = 'stdout and stderr' - elif has_stdout: - stream_string = 'stdout' - else: - stream_string = 'stderr' - raise CalledProcessError(cwd, args, MSG.format(stream_string)) +failure (useful when developing locally). +""" + raise CalledProcessError(cwd, args, MSG.format(stream_name)) + + short_cmd = textwrap.shorten(shlex.join(args), width=200) + sys.stderr.write( + f'\nThe above {stream_name} output was from: {short_cmd}\n') return stdout @@ -367,183 +329,6 @@ return extracted -def HermeticDateTime(timestamp=None): - """Returns a constant ZipInfo.date_time tuple. - - Args: - timestamp: Unix timestamp to use for files in the archive. - - Returns: - A ZipInfo.date_time tuple for Jan 1, 2001, or the given timestamp. - """ - if not timestamp: - return (2001, 1, 1, 0, 0, 0) - utc_time = time.gmtime(timestamp) - return (utc_time.tm_year, utc_time.tm_mon, utc_time.tm_mday, utc_time.tm_hour, - utc_time.tm_min, utc_time.tm_sec) - - -def HermeticZipInfo(*args, **kwargs): - """Creates a zipfile.ZipInfo with a constant timestamp and external_attr. - - If a date_time value is not provided in the positional or keyword arguments, - the default value from HermeticDateTime is used. - - Args: - See zipfile.ZipInfo. - - Returns: - A zipfile.ZipInfo. - """ - # The caller may have provided a date_time either as a positional parameter - # (args[1]) or as a keyword parameter. Use the default hermetic date_time if - # none was provided. - date_time = None - if len(args) >= 2: - date_time = args[1] - elif 'date_time' in kwargs: - date_time = kwargs['date_time'] - if not date_time: - kwargs['date_time'] = HermeticDateTime() - ret = zipfile.ZipInfo(*args, **kwargs) - ret.external_attr = (0o644 << 16) - return ret - - -def AddToZipHermetic(zip_file, - zip_path, - src_path=None, - data=None, - compress=None, - date_time=None): - """Adds a file to the given ZipFile with a hard-coded modified time. - - Args: - zip_file: ZipFile instance to add the file to. - zip_path: Destination path within the zip file (or ZipInfo instance). - src_path: Path of the source file. Mutually exclusive with |data|. - data: File data as a string. - compress: Whether to enable compression. Default is taken from ZipFile - constructor. - date_time: The last modification date and time for the archive member. - """ - assert (src_path is None) != (data is None), ( - '|src_path| and |data| are mutually exclusive.') - if isinstance(zip_path, zipfile.ZipInfo): - zipinfo = zip_path - zip_path = zipinfo.filename - else: - zipinfo = HermeticZipInfo(filename=zip_path, date_time=date_time) - - _CheckZipPath(zip_path) - - if src_path and os.path.islink(src_path): - zipinfo.filename = zip_path - zipinfo.external_attr |= stat.S_IFLNK << 16 # mark as a symlink - zip_file.writestr(zipinfo, os.readlink(src_path)) - return - - # zipfile.write() does - # external_attr = (os.stat(src_path)[0] & 0xFFFF) << 16 - # but we want to use _HERMETIC_FILE_ATTR, so manually set - # the few attr bits we care about. - if src_path: - st = os.stat(src_path) - for mode in (stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH): - if st.st_mode & mode: - zipinfo.external_attr |= mode << 16 - - if src_path: - with open(src_path, 'rb') as f: - data = f.read() - - # zipfile will deflate even when it makes the file bigger. To avoid - # growing files, disable compression at an arbitrary cut off point. - if len(data) < 16: - compress = False - - # None converts to ZIP_STORED, when passed explicitly rather than the - # default passed to the ZipFile constructor. - compress_type = zip_file.compression - if compress is not None: - compress_type = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED - zip_file.writestr(zipinfo, data, compress_type) - - -def DoZip(inputs, - output, - base_dir=None, - compress_fn=None, - zip_prefix_path=None, - timestamp=None): - """Creates a zip file from a list of files. - - Args: - inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples. - output: Path, fileobj, or ZipFile instance to add files to. - base_dir: Prefix to strip from inputs. - compress_fn: Applied to each input to determine whether or not to compress. - By default, items will be |zipfile.ZIP_STORED|. - zip_prefix_path: Path prepended to file path in zip file. - timestamp: Unix timestamp to use for files in the archive. - """ - if base_dir is None: - base_dir = '.' - input_tuples = [] - for tup in inputs: - if isinstance(tup, string_types): - tup = (os.path.relpath(tup, base_dir), tup) - if tup[0].startswith('..'): - raise Exception('Invalid zip_path: ' + tup[0]) - input_tuples.append(tup) - - # Sort by zip path to ensure stable zip ordering. - input_tuples.sort(key=lambda tup: tup[0]) - - out_zip = output - if not isinstance(output, zipfile.ZipFile): - out_zip = zipfile.ZipFile(output, 'w') - - date_time = HermeticDateTime(timestamp) - try: - for zip_path, fs_path in input_tuples: - if zip_prefix_path: - zip_path = os.path.join(zip_prefix_path, zip_path) - compress = compress_fn(zip_path) if compress_fn else None - AddToZipHermetic(out_zip, - zip_path, - src_path=fs_path, - compress=compress, - date_time=date_time) - finally: - if output is not out_zip: - out_zip.close() - - -def ZipDir(output, base_dir, compress_fn=None, zip_prefix_path=None): - """Creates a zip file from a directory.""" - inputs = [] - for root, _, files in os.walk(base_dir): - for f in files: - inputs.append(os.path.join(root, f)) - - if isinstance(output, zipfile.ZipFile): - DoZip( - inputs, - output, - base_dir, - compress_fn=compress_fn, - zip_prefix_path=zip_prefix_path) - else: - with AtomicOutput(output) as f: - DoZip( - inputs, - f, - base_dir, - compress_fn=compress_fn, - zip_prefix_path=zip_prefix_path) - - def MatchesGlob(path, filters): """Returns whether the given path matches any of the given glob patterns.""" return filters and any(fnmatch.fnmatch(path, f) for f in filters) @@ -560,12 +345,14 @@ compress: Overrides compression setting from origin zip entries. """ path_transform = path_transform or (lambda p: p) - added_names = set() out_zip = output if not isinstance(output, zipfile.ZipFile): out_zip = zipfile.ZipFile(output, 'w') + # Include paths in the existing zip here to avoid adding duplicate files. + added_names = set(out_zip.namelist()) + try: for in_file in input_zips: with zipfile.ZipFile(in_file, 'r') as in_zip: @@ -637,29 +424,6 @@ atexit.register(log_exit) -def AddDepfileOption(parser): - # TODO(agrieve): Get rid of this once we've moved to argparse. - if hasattr(parser, 'add_option'): - func = parser.add_option - else: - func = parser.add_argument - func('--depfile', - help='Path to depfile (refer to `gn help depfile`)') - - -def WriteDepfile(depfile_path, first_gn_output, inputs=None): - assert depfile_path != first_gn_output # http://crbug.com/646165 - assert not isinstance(inputs, string_types) # Easy mistake to make - inputs = inputs or [] - MakeDirectory(os.path.dirname(depfile_path)) - # Ninja does not support multiple outputs in depfiles. - with open(depfile_path, 'w') as depfile: - depfile.write(first_gn_output.replace(' ', '\\ ')) - depfile.write(': ') - depfile.write(' '.join(i.replace(' ', '\\ ') for i in inputs)) - depfile.write('\n') - - def ExpandFileArgs(args): """Replaces file-arg placeholders in args. @@ -704,7 +468,7 @@ raise Exception('Expected single item list but got %s' % expansion) expansion = expansion[0] - # This should match ParseGnList. The output is either a GN-formatted list + # This should match parse_gn_list. The output is either a GN-formatted list # or a literal (with no quotes). if isinstance(expansion, list): new_args[i] = (arg[:match.start()] + gn_helpers.ToGNString(expansion) +
diff --git a/build/android/gyp/util/build_utils_test.py b/build/android/gyp/util/build_utils_test.py index 008ea11..44528c9 100755 --- a/build/android/gyp/util/build_utils_test.py +++ b/build/android/gyp/util/build_utils_test.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gyp/util/diff_utils.py b/build/android/gyp/util/diff_utils.py index 530a688..445bbe3 100644 --- a/build/android/gyp/util/diff_utils.py +++ b/build/android/gyp/util/diff_utils.py
@@ -1,12 +1,13 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import difflib import os import sys -import difflib from util import build_utils +import action_helpers # build_utils adds //build to sys.path. def _SkipOmitted(line): @@ -34,7 +35,15 @@ '{}\n'.format(l.rstrip()) for l in actual_data.splitlines() if l.strip() ] - diff = difflib.ndiff(expected_lines, actual_lines) + # This helps the diff to not over-anchor on comments or closing braces in + # proguard configs. + def is_junk_line(l): + l = l.strip() + if l.startswith('# File:'): + return False + return l == '' or l == '}' or l.startswith('#') + + diff = difflib.ndiff(expected_lines, actual_lines, linejunk=is_junk_line) filtered_diff = (l for l in diff if l.startswith('+')) return ''.join(filtered_diff) @@ -88,7 +97,7 @@ def CheckExpectations(actual_data, options, custom_msg=''): if options.actual_file: - with build_utils.AtomicOutput(options.actual_file) as f: + with action_helpers.atomic_output(options.actual_file) as f: f.write(actual_data.encode('utf8')) if options.expected_file_base: actual_data = _GenerateDiffWithOnlyAdditons(options.expected_file_base,
diff --git a/build/android/gyp/util/jar_info_utils.py b/build/android/gyp/util/jar_info_utils.py index 9759455..3a895c2 100644 --- a/build/android/gyp/util/jar_info_utils.py +++ b/build/android/gyp/util/jar_info_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gyp/util/java_cpp_utils.py b/build/android/gyp/util/java_cpp_utils.py index 5180400..46f05f6 100644 --- a/build/android/gyp/util/java_cpp_utils.py +++ b/build/android/gyp/util/java_cpp_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -38,7 +38,7 @@ return s.upper() -class JavaString(object): +class JavaString: def __init__(self, name, value, comments): self.name = KCamelToShouty(name) self.value = value @@ -67,7 +67,7 @@ # TODO(crbug.com/937282): Work will be needed if we want to annotate specific # constants in the file to be parsed. -class CppConstantParser(object): +class CppConstantParser: """Parses C++ constants, retaining their comments. The Delegate subclass is responsible for matching and extracting the @@ -76,7 +76,7 @@ """ SINGLE_LINE_COMMENT_RE = re.compile(r'\s*(// [^\n]*)') - class Delegate(object): + class Delegate: def ExtractConstantName(self, line): """Extracts a constant's name from line or None if not a match.""" raise NotImplementedError() @@ -149,9 +149,8 @@ self._in_comment = True self._in_variable = True return True - else: - self._in_comment = False - return False + self._in_comment = False + return False def _ParseVariable(self, line): current_name = self._delegate.ExtractConstantName(line) @@ -164,9 +163,8 @@ else: self._in_variable = True return True - else: - self._in_variable = False - return False + self._in_variable = False + return False def _ParseLine(self, line): if not self._in_variable:
diff --git a/build/android/gyp/util/manifest_utils.py b/build/android/gyp/util/manifest_utils.py index a517708..3202058 100644 --- a/build/android/gyp/util/manifest_utils.py +++ b/build/android/gyp/util/manifest_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,9 +10,10 @@ import shlex import sys import xml.dom.minidom as minidom +from xml.etree import ElementTree from util import build_utils -from xml.etree import ElementTree +import action_helpers # build_utils adds //build to sys.path. ANDROID_NAMESPACE = 'http://schemas.android.com/apk/res/android' TOOLS_NAMESPACE = 'http://schemas.android.com/tools' @@ -45,6 +46,14 @@ ElementTree.register_namespace('dist', DIST_NAMESPACE) +def NamespacedGet(node, key): + return node.get('{%s}%s' % (ANDROID_NAMESPACE, key)) + + +def NamespacedSet(node, key, value): + node.set('{%s}%s' % (ANDROID_NAMESPACE, key), value) + + def ParseManifest(path): """Parses an AndroidManifest.xml using ElementTree. @@ -63,6 +72,7 @@ manifest_node = doc.getroot() else: manifest_node = doc.find('manifest') + assert manifest_node is not None, 'Manifest is none for path ' + path app_node = doc.find('application') if app_node is None: @@ -72,7 +82,7 @@ def SaveManifest(doc, path): - with build_utils.AtomicOutput(path) as f: + with action_helpers.atomic_output(path) as f: f.write(ElementTree.tostring(doc.getroot(), encoding='UTF-8')) @@ -80,47 +90,27 @@ return manifest_node.get('package') -def AssertUsesSdk(manifest_node, - min_sdk_version=None, - target_sdk_version=None, - max_sdk_version=None, - fail_if_not_exist=False): - """Asserts values of attributes of <uses-sdk> element. - - Unless |fail_if_not_exist| is true, will only assert if both the passed value - is not None and the value of attribute exist. If |fail_if_not_exist| is true - will fail if passed value is not None but attribute does not exist. - """ +def SetUsesSdk(manifest_node, + target_sdk_version, + min_sdk_version, + max_sdk_version=None): uses_sdk_node = manifest_node.find('./uses-sdk') if uses_sdk_node is None: - return - for prefix, sdk_version in (('min', min_sdk_version), ('target', - target_sdk_version), - ('max', max_sdk_version)): - value = uses_sdk_node.get('{%s}%sSdkVersion' % (ANDROID_NAMESPACE, prefix)) - if fail_if_not_exist and not value and sdk_version: - assert False, ( - '%sSdkVersion in Android manifest does not exist but we expect %s' % - (prefix, sdk_version)) - if not value or not sdk_version: - continue - assert value == sdk_version, ( - '%sSdkVersion in Android manifest is %s but we expect %s' % - (prefix, value, sdk_version)) + uses_sdk_node = ElementTree.SubElement(manifest_node, 'uses-sdk') + NamespacedSet(uses_sdk_node, 'targetSdkVersion', target_sdk_version) + NamespacedSet(uses_sdk_node, 'minSdkVersion', min_sdk_version) + if max_sdk_version: + NamespacedSet(uses_sdk_node, 'maxSdkVersion', max_sdk_version) -def AssertPackage(manifest_node, package): - """Asserts that manifest package has desired value. - - Will only assert if both |package| is not None and the package is set in the - manifest. - """ - package_value = GetPackage(manifest_node) - if package_value is None or package is None: - return - assert package_value == package, ( - 'Package in Android manifest is %s but we expect %s' % (package_value, - package)) +def SetTargetApiIfUnset(manifest_node, target_sdk_version): + uses_sdk_node = manifest_node.find('./uses-sdk') + if uses_sdk_node is None: + uses_sdk_node = ElementTree.SubElement(manifest_node, 'uses-sdk') + curr_target_sdk_version = NamespacedGet(uses_sdk_node, 'targetSdkVersion') + if curr_target_sdk_version is None: + NamespacedSet(uses_sdk_node, 'targetSdkVersion', target_sdk_version) + return curr_target_sdk_version is None def _SortAndStripElementTree(root): @@ -195,7 +185,7 @@ if cur_indent != -1 and cur_indent <= target_indent: tag_lines = lines[:i + 1] break - elif not tag_closed and 'android:name="' in l: + if not tag_closed and 'android:name="' in l: # To reduce noise of node tags changing, use android:name as the # basis the hash since they usually unique. tag_lines = [l] @@ -214,7 +204,7 @@ idx = l.find('>') if idx != -1: return l[idx - 1] == '/' - assert False, 'Did not find end of tag:\n' + '\n'.join(lines) + raise RuntimeError('Did not find end of tag:\n%s' % '\n'.join(lines)) def _AddDiffTags(lines): @@ -251,7 +241,8 @@ assert not hash_stack, 'hash_stack was not empty:\n' + '\n'.join(hash_stack) -def NormalizeManifest(manifest_contents): +def NormalizeManifest(manifest_contents, version_code_offset, + library_version_offset): _RegisterElementTreeNamespaces() # This also strips comments and sorts node attributes alphabetically. root = ElementTree.fromstring(manifest_contents) @@ -266,14 +257,24 @@ if debuggable_name in app_node.attrib: del app_node.attrib[debuggable_name] + version_code = NamespacedGet(root, 'versionCode') + if version_code and version_code_offset: + version_code = int(version_code) - int(version_code_offset) + NamespacedSet(root, 'versionCode', f'OFFSET={version_code}') + version_name = NamespacedGet(root, 'versionName') + if version_name: + version_name = re.sub(r'\d+', '#', version_name) + NamespacedSet(root, 'versionName', version_name) + # Trichrome's static library version number is updated daily. To avoid # frequent manifest check failures, we remove the exact version number # during normalization. for node in app_node: - if (node.tag in ['uses-static-library', 'static-library'] - and '{%s}version' % ANDROID_NAMESPACE in node.keys() - and '{%s}name' % ANDROID_NAMESPACE in node.keys()): - node.set('{%s}version' % ANDROID_NAMESPACE, '$VERSION_NUMBER') + if node.tag in ['uses-static-library', 'static-library']: + version = NamespacedGet(node, 'version') + if version and library_version_offset: + version = int(version) - int(library_version_offset) + NamespacedSet(node, 'version', f'OFFSET={version}') # We also remove the exact package name (except the one at the root level) # to avoid noise during manifest comparison.
diff --git a/build/android/gyp/util/manifest_utils_test.py b/build/android/gyp/util/manifest_utils_test.py index 52bf458..165df4c 100755 --- a/build/android/gyp/util/manifest_utils_test.py +++ b/build/android/gyp/util/manifest_utils_test.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -14,6 +14,8 @@ _TEST_MANIFEST = """\ <?xml version="1.0" ?> <manifest package="test.pkg" + android:versionCode="1234" + android:versionName="1.2.33.4" tools:ignore="MissingVersion" xmlns:android="http://schemas.android.com/apk/res/android" xmlns:tools="http://schemas.android.com/tools"> @@ -52,6 +54,8 @@ xmlns:android="http://schemas.android.com/apk/res/android" xmlns:tools="http://schemas.android.com/tools" package="test.pkg" + android:versionCode="OFFSET=4" + android:versionName="#.#.#.#" tools:ignore="MissingVersion"> <uses-feature android:name="android.hardware.vr.headtracking" \ android:required="false" android:version="1"/> @@ -106,19 +110,19 @@ def testNormalizeManifest_golden(self): test_manifest, expected = _CreateTestData() - actual = manifest_utils.NormalizeManifest(test_manifest) + actual = manifest_utils.NormalizeManifest(test_manifest, 1230, None) self.assertMultiLineEqual(expected, actual) def testNormalizeManifest_nameUsedForActivity(self): test_manifest, expected = _CreateTestData(extra_activity_attr='a="b"') - actual = manifest_utils.NormalizeManifest(test_manifest) + actual = manifest_utils.NormalizeManifest(test_manifest, 1230, None) # Checks that the DIFF-ANCHOR does not change with the added attribute. self.assertMultiLineEqual(expected, actual) def testNormalizeManifest_nameNotUsedForIntentFilter(self): test_manifest, expected = _CreateTestData( extra_intent_filter_elem='<a/>', intent_filter_diff_anchor='5f5c8a70') - actual = manifest_utils.NormalizeManifest(test_manifest) + actual = manifest_utils.NormalizeManifest(test_manifest, 1230, None) # Checks that the DIFF-ANCHOR does change with the added element despite # having a nested element with an android:name set. self.assertMultiLineEqual(expected, actual)
diff --git a/build/android/gyp/util/md5_check.py b/build/android/gyp/util/md5_check.py index 87ee723..269ae28 100644 --- a/build/android/gyp/util/md5_check.py +++ b/build/android/gyp/util/md5_check.py
@@ -1,8 +1,7 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import difflib import hashlib @@ -13,8 +12,7 @@ import zipfile from util import build_utils - -sys.path.insert(1, os.path.join(build_utils.DIR_SOURCE_ROOT, 'build')) +import action_helpers # build_utils adds //build to sys.path. import print_python_deps # When set and a difference is detected, a diff of what changed is printed. @@ -67,7 +65,7 @@ # on bots that build with & without patch, and the patch changes the depfile # location. if hasattr(options, 'depfile') and options.depfile: - build_utils.WriteDepfile(options.depfile, output_paths[0], depfile_deps) + action_helpers.write_depfile(options.depfile, output_paths[0], depfile_deps) def CallAndRecordIfStale(function, @@ -158,7 +156,7 @@ new_metadata.ToFile(f) -class Changes(object): +class Changes: """Provides and API for querying what changed between runs.""" def __init__(self, old_metadata, new_metadata, force, missing_outputs, @@ -262,11 +260,11 @@ """Returns a human-readable description of what changed.""" if self.force: return 'force=True' - elif self.missing_outputs: + if self.missing_outputs: return 'Outputs do not exist:\n ' + '\n '.join(self.missing_outputs) - elif self.too_new: + if self.too_new: return 'Outputs newer than stamp file:\n ' + '\n '.join(self.too_new) - elif self.old_metadata is None: + if self.old_metadata is None: return 'Previous stamp file not found.' if self.old_metadata.StringsMd5() != self.new_metadata.StringsMd5(): @@ -294,7 +292,7 @@ return 'I have no idea what changed (there is a bug).' -class _Metadata(object): +class _Metadata: """Data model for tracking change metadata. Args:
diff --git a/build/android/gyp/util/md5_check_test.py b/build/android/gyp/util/md5_check_test.py index e11bbd5..e1e940b 100755 --- a/build/android/gyp/util/md5_check_test.py +++ b/build/android/gyp/util/md5_check_test.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/gyp/util/parallel.py b/build/android/gyp/util/parallel.py index c26875a..dec94c7 100644 --- a/build/android/gyp/util/parallel.py +++ b/build/android/gyp/util/parallel.py
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Helpers related to multiprocessing. @@ -27,7 +27,7 @@ _fork_kwargs = None -class _ImmediateResult(object): +class _ImmediateResult: def __init__(self, value): self._value = value @@ -44,7 +44,7 @@ return True -class _ExceptionWrapper(object): +class _ExceptionWrapper: """Used to marshal exception messages back to main process.""" def __init__(self, msg, exception_type=None): @@ -57,7 +57,7 @@ self.exception_type)('Originally caused by: ' + self.msg) -class _FuncWrapper(object): +class _FuncWrapper: """Runs on the fork()'ed side to catch exceptions and spread *args.""" def __init__(self, func): @@ -66,7 +66,10 @@ self._func = func def __call__(self, index, _=None): + global _fork_kwargs try: + if _fork_kwargs is None: # Clarifies _fork_kwargs is map for pylint. + _fork_kwargs = {} return self._func(*_fork_params[index], **_fork_kwargs) except Exception as e: # Only keep the exception type for builtin exception types or else risk @@ -81,7 +84,7 @@ return _ExceptionWrapper(traceback.format_exc()) -class _WrappedResult(object): +class _WrappedResult: """Allows for host-side logic to be run after child process has terminated. * Unregisters associated pool _all_pools.
diff --git a/build/android/gyp/util/protoresources.py b/build/android/gyp/util/protoresources.py index 272574f..11f8778 100644 --- a/build/android/gyp/util/protoresources.py +++ b/build/android/gyp/util/protoresources.py
@@ -1,10 +1,10 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Functions that modify resources in protobuf format. Format reference: -https://cs.android.com/android/platform/superproject/+/master:frameworks/base/tools/aapt2/Resources.proto +https://cs.android.com/search?q=f:aapt2.*Resources.proto """ import logging @@ -211,7 +211,7 @@ _ProcessZip(zip_path, process_func) -class _ResourceStripper(object): +class _ResourceStripper: def __init__(self, partial_path, keep_predicate): self.partial_path = partial_path self.keep_predicate = keep_predicate @@ -231,12 +231,12 @@ for style in self._IterStyles(entry): entries = style.entry new_entries = [] - for entry in entries: - full_name = '{}/{}'.format(type_and_name, entry.key.name) + for e in entries: + full_name = '{}/{}'.format(type_and_name, e.key.name) if not self.keep_predicate(full_name): logging.debug('Stripped %s/%s', self.partial_path, full_name) else: - new_entries.append(entry) + new_entries.append(e) if len(new_entries) != len(entries): self._has_changes = True @@ -267,7 +267,7 @@ def _TableFromFlatBytes(data): - # https://cs.android.com/android/platform/superproject/+/master:frameworks/base/tools/aapt2/format/Container.cpp + # https://cs.android.com/search?q=f:aapt2.*Container.cpp size_idx = len(_FLAT_ARSC_HEADER) proto_idx = size_idx + 8 if data[:size_idx] != _FLAT_ARSC_HEADER:
diff --git a/build/android/gyp/util/resource_utils.py b/build/android/gyp/util/resource_utils.py index 263b7c2..dac0ae7 100644 --- a/build/android/gyp/util/resource_utils.py +++ b/build/android/gyp/util/resource_utils.py
@@ -1,8 +1,7 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import argparse import collections import contextlib import itertools @@ -43,11 +42,11 @@ 'no': 'nb', # 'no' is not a real language. http://crbug.com/920960 } -_ALL_RESOURCE_TYPES = { +ALL_RESOURCE_TYPES = { 'anim', 'animator', 'array', 'attr', 'bool', 'color', 'dimen', 'drawable', - 'font', 'fraction', 'id', 'integer', 'interpolator', 'layout', 'menu', - 'mipmap', 'plurals', 'raw', 'string', 'style', 'styleable', 'transition', - 'xml' + 'font', 'fraction', 'id', 'integer', 'interpolator', 'layout', 'macro', + 'menu', 'mipmap', 'plurals', 'raw', 'string', 'style', 'styleable', + 'transition', 'xml' } AAPT_IGNORE_PATTERN = ':'.join([ @@ -238,7 +237,7 @@ yield path, archive_path -class ResourceInfoFile(object): +class ResourceInfoFile: """Helper for building up .res.info files.""" def __init__(self): @@ -348,6 +347,39 @@ return resource_value.replace('0x00', '0x7f') +def ResolveStyleableReferences(r_txt_path): + # Convert lines like: + # int[] styleable ViewBack { 0x010100d4, com.android.webview.R.attr.backTint } + # to: + # int[] styleable ViewBack { 0x010100d4, 0xREALVALUE } + entries = _ParseTextSymbolsFile(r_txt_path) + lookup_table = {(e.resource_type, e.name): e.value for e in entries} + + sb = [] + with open(r_txt_path, encoding='utf8') as f: + for l in f: + if l.startswith('int[] styleable'): + brace_start = l.index('{') + 2 + brace_end = l.index('}') - 1 + values = [x for x in l[brace_start:brace_end].split(', ') if x] + new_values = [] + for v in values: + try: + if not v.startswith('0x'): + resource_type, name = v.split('.')[-2:] + new_values.append(lookup_table[(resource_type, name)]) + else: + new_values.append(v) + except: + logging.warning('Failed line: %r %r', l, v) + raise + l = l[:brace_start] + ', '.join(new_values) + l[brace_end:] + sb.append(l) + + with open(r_txt_path, 'w', encoding='utf8') as f: + f.writelines(sb) + + def _GetRTxtResourceNames(r_txt_path): """Parse an R.txt file and extract the set of resource names from it.""" return {entry.name for entry in _ParseTextSymbolsFile(r_txt_path)} @@ -486,15 +518,14 @@ if entry.resource_type == 'styleable' and entry.java_type != 'int[]': # A styleable constant may be exported as non-final after all. return not self.export_const_styleable - elif not self.has_constant_ids: + if not self.has_constant_ids: # Every resource is non-final return False - elif not self.resources_allowlist: + if not self.resources_allowlist: # No allowlist means all IDs are non-final. return True - else: - # Otherwise, only those in the - return entry.name not in self.resources_allowlist + # Otherwise, only those in the + return entry.name not in self.resources_allowlist def CreateRJavaFiles(srcjar_dir, @@ -505,7 +536,6 @@ srcjar_out, custom_root_package_name=None, grandparent_custom_package_name=None, - extra_main_r_text_files=None, ignore_mismatched_values=False): """Create all R.java files for a set of packages and R.txt files. @@ -526,7 +556,6 @@ as the grandparent_custom_package_name. The format of this package name is identical to custom_root_package_name. (eg. for vr grandparent_custom_package_name would be "base") - extra_main_r_text_files: R.txt files to be added to the root R.java file. ignore_mismatched_values: If True, ignores if a resource appears multiple times with different entry values (useful when all the values are dummy anyways). @@ -548,8 +577,6 @@ all_resources_by_type = collections.defaultdict(list) main_r_text_files = [main_r_txt_file] - if extra_main_r_text_files: - main_r_text_files.extend(extra_main_r_text_files) for r_txt_file in main_r_text_files: for entry in _ParseTextSymbolsFile(r_txt_file, fix_package_ids=True): entry_key = (entry.resource_type, entry.name) @@ -562,8 +589,8 @@ else: all_resources[entry_key] = entry all_resources_by_type[entry.resource_type].append(entry) - assert entry.resource_type in _ALL_RESOURCE_TYPES, ( - 'Unknown resource type: %s, add to _ALL_RESOURCE_TYPES!' % + assert entry.resource_type in ALL_RESOURCE_TYPES, ( + 'Unknown resource type: %s, add to ALL_RESOURCE_TYPES!' % entry.resource_type) if custom_root_package_name: @@ -583,8 +610,8 @@ with open(root_r_java_path, 'w') as f: f.write(root_java_file_contents) - for package in packages: - _CreateRJavaSourceFile(srcjar_dir, package, root_r_java_package, + for p in packages: + _CreateRJavaSourceFile(srcjar_dir, p, root_r_java_package, rjava_build_options) @@ -639,7 +666,7 @@ return template.render( package=package, - resource_types=sorted(_ALL_RESOURCE_TYPES), + resource_types=sorted(ALL_RESOURCE_TYPES), root_package=root_r_java_package, has_on_resources_loaded=rjava_build_options.has_on_resources_loaded) @@ -662,14 +689,6 @@ else: non_final_resources_by_type[res_type].append(entry) - # Keep these assignments all on one line to make diffing against regular - # aapt-generated files easier. - create_id = ('{{ e.resource_type }}.{{ e.name }} ^= packageIdTransform;') - create_id_arr = ('{{ e.resource_type }}.{{ e.name }}[i] ^=' - ' packageIdTransform;') - for_loop_condition = ('int i = {{ startIndex(e) }}; i < ' - '{{ e.resource_type }}.{{ e.name }}.length; ++i') - # Here we diverge from what aapt does. Because we have so many # resources, the onResourcesLoaded method was exceeding the 64KB limit that # Java imposes. For this reason we split onResourcesLoaded into different @@ -680,6 +699,10 @@ extends_string = 'extends {{ parent_path }}.R.{{ resource_type }} ' dep_path = GetCustomPackagePath(grandparent_custom_package_name) + # Don't actually mark fields as "final" or else R8 complain when aapt2 uses + # --proguard-conditional-keep-rules. E.g.: + # Rule precondition matches static final fields javac has inlined. + # Such rules are unsound as the shrinker cannot infer the inlining precisely. template = Template("""/* AUTO-GENERATED FILE. DO NOT MODIFY. */ package {{ package }}; @@ -688,7 +711,7 @@ {% for resource_type in resource_types %} public static class {{ resource_type }} """ + extends_string + """ { {% for e in final_resources[resource_type] %} - public static final {{ e.java_type }} {{ e.name }} = {{ e.value }}; + public static {{ e.java_type }} {{ e.name }} = {{ e.value }}; {% endfor %} {% for e in non_final_resources[resource_type] %} {% if e.value != '0' %} @@ -705,29 +728,44 @@ } {% else %} private static boolean sResourcesDidLoad; + + private static void patchArray( + int[] arr, int startIndex, int packageIdTransform) { + for (int i = startIndex; i < arr.length; ++i) { + arr[i] ^= packageIdTransform; + } + } + public static void onResourcesLoaded(int packageId) { if (sResourcesDidLoad) { return; } sResourcesDidLoad = true; int packageIdTransform = (packageId ^ 0x7f) << 24; + {# aapt2 makes int[] resources refer to other resources by reference + rather than by value. Thus, need to transform the int[] resources + first, before the referenced resources are transformed in order to + ensure the transform applies exactly once. + See https://crbug.com/1237059 for context. + #} {% for resource_type in resource_types %} - onResourcesLoaded{{ resource_type|title }}(packageIdTransform); {% for e in non_final_resources[resource_type] %} {% if e.java_type == 'int[]' %} - for(""" + for_loop_condition + """) { - """ + create_id_arr + """ - } + patchArray({{ e.resource_type }}.{{ e.name }}, {{ startIndex(e) }}, \ +packageIdTransform); {% endif %} {% endfor %} {% endfor %} + {% for resource_type in resource_types %} + onResourcesLoaded{{ resource_type|title }}(packageIdTransform); + {% endfor %} } {% for res_type in resource_types %} private static void onResourcesLoaded{{ res_type|title }} ( int packageIdTransform) { {% for e in non_final_resources[res_type] %} {% if res_type != 'styleable' and e.java_type != 'int[]' %} - """ + create_id + """ + {{ e.resource_type }}.{{ e.name }} ^= packageIdTransform; {% endif %} {% endfor %} } @@ -740,7 +778,7 @@ lstrip_blocks=True) return template.render( package=package, - resource_types=sorted(_ALL_RESOURCE_TYPES), + resource_types=sorted(ALL_RESOURCE_TYPES), has_on_resources_loaded=rjava_build_options.has_on_resources_loaded, fake_on_resources_loaded=rjava_build_options.fake_on_resources_loaded, final_resources=final_resources_by_type, @@ -761,7 +799,14 @@ def ExtractArscPackage(aapt2_path, apk_path): - """Returns (package_name, package_id) of resources.arsc from apk_path.""" + """Returns (package_name, package_id) of resources.arsc from apk_path. + + When the apk does not have any entries in its resources file, in recent aapt2 + versions it will not contain a "Package" line. The package is not even in the + actual resources.arsc/resources.pb file (which itself is mostly empty). Thus + return (None, None) when dump succeeds and there are no errors to indicate + that the package name does not exist in the resources file. + """ proc = subprocess.Popen([aapt2_path, 'dump', 'resources', apk_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -777,8 +822,11 @@ # aapt2 currently crashes when dumping webview resources, but not until after # it prints the "Package" line (b/130553900). - sys.stderr.write(proc.stderr.read()) - raise Exception('Failed to find arsc package name') + stderr_output = proc.stderr.read().decode('utf-8') + if stderr_output: + sys.stderr.write(stderr_output) + raise Exception('Failed to find arsc package name') + return None, None def _RenameSubdirsWithPrefix(dir_path, prefix): @@ -840,7 +888,7 @@ return dep_subdirs -class _ResourceBuildContext(object): +class _ResourceBuildContext: """A temporary directory for packaging and compiling Android resources. Args: @@ -898,65 +946,6 @@ context.Close() -def ResourceArgsParser(): - """Create an argparse.ArgumentParser instance with common argument groups. - - Returns: - A tuple of (parser, in_group, out_group) corresponding to the parser - instance, and the input and output argument groups for it, respectively. - """ - parser = argparse.ArgumentParser(description=__doc__) - - input_opts = parser.add_argument_group('Input options') - output_opts = parser.add_argument_group('Output options') - - build_utils.AddDepfileOption(output_opts) - - input_opts.add_argument('--include-resources', required=True, action="append", - help='Paths to arsc resource files used to link ' - 'against. Can be specified multiple times.') - - input_opts.add_argument('--dependencies-res-zips', required=True, - help='Resources zip archives from dependents. Required to ' - 'resolve @type/foo references into dependent ' - 'libraries.') - - input_opts.add_argument( - '--r-text-in', - help='Path to pre-existing R.txt. Its resource IDs override those found ' - 'in the aapt-generated R.txt when generating R.java.') - - input_opts.add_argument( - '--extra-res-packages', - help='Additional package names to generate R.java files for.') - - return (parser, input_opts, output_opts) - - -def HandleCommonOptions(options): - """Handle common command-line options after parsing. - - Args: - options: the result of parse_args() on the parser returned by - ResourceArgsParser(). This function updates a few common fields. - """ - options.include_resources = [build_utils.ParseGnList(r) for r in - options.include_resources] - # Flatten list of include resources list to make it easier to use. - options.include_resources = [r for resources in options.include_resources - for r in resources] - - options.dependencies_res_zips = ( - build_utils.ParseGnList(options.dependencies_res_zips)) - - # Don't use [] as default value since some script explicitly pass "". - if options.extra_res_packages: - options.extra_res_packages = ( - build_utils.ParseGnList(options.extra_res_packages)) - else: - options.extra_res_packages = [] - - def ParseAndroidResourceStringsFromXml(xml_data): """Parse and Android xml resource file and extract strings from it. @@ -1005,7 +994,7 @@ raise Exception('Expected closing string tag: ' + input_data) text = input_data[:m2.start()] input_data = input_data[m2.end():] - if len(text) and text[0] == '"' and text[-1] == '"': + if len(text) != 0 and text[0] == '"' and text[-1] == '"': text = text[1:-1] result[name] = text
diff --git a/build/android/gyp/util/resource_utils_test.py b/build/android/gyp/util/resource_utils_test.py index 62d5b43..4b31e92 100755 --- a/build/android/gyp/util/resource_utils_test.py +++ b/build/android/gyp/util/resource_utils_test.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # coding: utf-8 -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,7 +19,7 @@ os.path.join(os.path.dirname(__file__), os.pardir)) sys.path.insert(1, _BUILD_ANDROID_GYP_ROOT) -import resource_utils # pylint: disable=relative-import +import resource_utils # pylint: disable=line-too-long
diff --git a/build/android/gyp/util/resources_parser.py b/build/android/gyp/util/resources_parser.py index 8d8d69c..86d8540 100644 --- a/build/android/gyp/util/resources_parser.py +++ b/build/android/gyp/util/resources_parser.py
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,6 +9,7 @@ from util import build_utils from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. _TextSymbolEntry = collections.namedtuple( 'RTextEntry', ('java_type', 'resource_type', 'name', 'value')) @@ -21,7 +22,7 @@ return re.sub('[\.:]', '_', resource_name) -class RTxtGenerator(object): +class RTxtGenerator: def __init__(self, res_dirs, ignore_pattern=resource_utils.AAPT_IGNORE_PATTERN): @@ -73,13 +74,19 @@ ret.update(self._ExtractNewIdsFromNode(child)) return ret + def _ParseXml(self, xml_path): + try: + return ElementTree.parse(xml_path).getroot() + except Exception as e: + raise RuntimeError('Failure parsing {}:\n'.format(xml_path)) from e + def _ExtractNewIdsFromXml(self, xml_path): - root = ElementTree.parse(xml_path).getroot() - return self._ExtractNewIdsFromNode(root) + return self._ExtractNewIdsFromNode(self._ParseXml(xml_path)) def _ParseValuesXml(self, xml_path): ret = set() - root = ElementTree.parse(xml_path).getroot() + root = self._ParseXml(xml_path) + assert root.tag == 'resources' for child in root: if child.tag == 'eat-comment': @@ -91,12 +98,18 @@ if child.tag == 'declare-styleable': ret.update(self._ParseDeclareStyleable(child)) else: - if child.tag == 'item': + if child.tag in ('item', 'public'): resource_type = child.attrib['type'] elif child.tag in ('array', 'integer-array', 'string-array'): resource_type = 'array' else: resource_type = child.tag + parsed_element = ElementTree.tostring(child, encoding='unicode').strip() + assert resource_type in resource_utils.ALL_RESOURCE_TYPES, ( + f'Infered resource type ({resource_type}) from xml entry ' + f'({parsed_element}) (found in {xml_path}) is not listed in ' + 'resource_utils.ALL_RESOURCE_TYPES. Teach resources_parser.py how ' + 'to parse this entry and/or add to the list.') name = _ResourceNameToJavaSymbol(child.attrib['name']) ret.add(_TextSymbolEntry('int', resource_type, name, _DUMMY_RTXT_ID)) return ret @@ -131,11 +144,11 @@ ret = set() for res_dir in self.res_dirs: ret.update(self._CollectResourcesListFromDirectory(res_dir)) - return ret + return sorted(ret) def WriteRTxtFile(self, rtxt_path): resources = self._CollectResourcesListFromDirectories() - with build_utils.AtomicOutput(rtxt_path, mode='w') as f: + with action_helpers.atomic_output(rtxt_path, mode='w') as f: for resource in resources: line = '{0.java_type} {0.resource_type} {0.name} {0.value}\n'.format( resource)
diff --git a/build/android/gyp/util/server_utils.py b/build/android/gyp/util/server_utils.py index e050ef6..b634cf9 100644 --- a/build/android/gyp/util/server_utils.py +++ b/build/android/gyp/util/server_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,7 +13,7 @@ BUILD_SERVER_ENV_VARIABLE = 'INVOKED_BY_BUILD_SERVER' -def MaybeRunCommand(name, argv, stamp_file): +def MaybeRunCommand(name, argv, stamp_file, force): """Returns True if the command was successfully sent to the build server.""" # When the build server runs a command, it sets this environment variable. @@ -36,6 +36,12 @@ # [Errno 111] Connection refused. Either the server has not been started # or the server is not currently accepting new connections. if e.errno == 111: + if force: + raise RuntimeError( + '\n\nBuild server is not running and ' + 'android_static_analysis="build_server" is set.\nPlease run ' + 'this command in a separate terminal:\n\n' + '$ build/android/fast_local_dev_server.py\n\n') from None return False raise e return True
diff --git a/build/android/gyp/util/zipalign.py b/build/android/gyp/util/zipalign.py deleted file mode 100644 index c5c4ea8..0000000 --- a/build/android/gyp/util/zipalign.py +++ /dev/null
@@ -1,97 +0,0 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import struct -import sys -import zipfile - -from util import build_utils - -_FIXED_ZIP_HEADER_LEN = 30 - - -def _PatchedDecodeExtra(self): - # Try to decode the extra field. - extra = self.extra - unpack = struct.unpack - while len(extra) >= 4: - tp, ln = unpack('<HH', extra[:4]) - if tp == 1: - if ln >= 24: - counts = unpack('<QQQ', extra[4:28]) - elif ln == 16: - counts = unpack('<QQ', extra[4:20]) - elif ln == 8: - counts = unpack('<Q', extra[4:12]) - elif ln == 0: - counts = () - else: - raise RuntimeError("Corrupt extra field %s" % (ln, )) - - idx = 0 - - # ZIP64 extension (large files and/or large archives) - if self.file_size in (0xffffffffffffffff, 0xffffffff): - self.file_size = counts[idx] - idx += 1 - - if self.compress_size == 0xffffffff: - self.compress_size = counts[idx] - idx += 1 - - if self.header_offset == 0xffffffff: - self.header_offset = counts[idx] - idx += 1 - - extra = extra[ln + 4:] - - -def ApplyZipFileZipAlignFix(): - """Fix zipfile.ZipFile() to be able to open zipaligned .zip files. - - Android's zip alignment uses not-quite-valid zip headers to perform alignment. - Python < 3.4 crashes when trying to load them. - https://bugs.python.org/issue14315 - """ - if sys.version_info < (3, 4): - zipfile.ZipInfo._decodeExtra = ( # pylint: disable=protected-access - _PatchedDecodeExtra) - - -def _SetAlignment(zip_obj, zip_info, alignment): - """Sets a ZipInfo's extra field such that the file will be aligned. - - Args: - zip_obj: The ZipFile object that is being written. - zip_info: The ZipInfo object about to be written. - alignment: The amount of alignment (e.g. 4, or 4*1024). - """ - cur_offset = zip_obj.fp.tell() - header_size = _FIXED_ZIP_HEADER_LEN + len(zip_info.filename) - padding_needed = (alignment - ( - (cur_offset + header_size) % alignment)) % alignment - - - # Python writes |extra| to both the local file header and the central - # directory's file header. Android's zipalign tool writes only to the - # local file header, so there is more overhead in using python to align. - zip_info.extra = b'\0' * padding_needed - - -def AddToZipHermetic(zip_file, - zip_path, - src_path=None, - data=None, - compress=None, - alignment=None): - """Same as build_utils.AddToZipHermetic(), but with alignment. - - Args: - alignment: If set, align the data of the entry to this many bytes. - """ - zipinfo = build_utils.HermeticZipInfo(filename=zip_path) - if alignment: - _SetAlignment(zip_file, zipinfo, alignment) - build_utils.AddToZipHermetic( - zip_file, zipinfo, src_path=src_path, data=data, compress=compress)
diff --git a/build/android/gyp/validate_inputs.py b/build/android/gyp/validate_inputs.py new file mode 100755 index 0000000..e6435d6 --- /dev/null +++ b/build/android/gyp/validate_inputs.py
@@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Ensures inputs exist and writes a stamp file.""" + +import argparse +import pathlib +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--stamp', help='Path to touch on success.') + parser.add_argument('inputs', nargs='+', help='Files to check.') + + args = parser.parse_args() + + for path in args.inputs: + path_obj = pathlib.Path(path) + if not path_obj.is_file(): + if not path_obj.exists(): + sys.stderr.write(f'File not found: {path}\n') + else: + sys.stderr.write(f'Not a file: {path}\n') + sys.exit(1) + + if args.stamp: + pathlib.Path(args.stamp).touch() + + +if __name__ == '__main__': + main()
diff --git a/build/android/gyp/validate_static_library_dex_references.py b/build/android/gyp/validate_static_library_dex_references.py index b14ca3c..419776e 100755 --- a/build/android/gyp/validate_static_library_dex_references.py +++ b/build/android/gyp/validate_static_library_dex_references.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,6 +12,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir)) from pylib.dex import dex_parser from util import build_utils +import action_helpers # build_utils adds //build to sys.path. _FLAGS_PATH = ( '//chrome/android/java/static_library_dex_reference_workarounds.flags') @@ -49,8 +50,7 @@ def main(args): args = build_utils.ExpandFileArgs(args) parser = argparse.ArgumentParser() - parser.add_argument( - '--depfile', required=True, help='Path to output depfile.') + action_helpers.add_depfile_arg(parser) parser.add_argument( '--stamp', required=True, help='Path to file to touch upon success.') parser.add_argument( @@ -86,7 +86,7 @@ input_paths = [args.static_library_dex] + args.static_library_dependent_dexes build_utils.Touch(args.stamp) - build_utils.WriteDepfile(args.depfile, args.stamp, inputs=input_paths) + action_helpers.write_depfile(args.depfile, args.stamp, inputs=input_paths) if __name__ == '__main__':
diff --git a/build/android/gyp/validate_static_library_dex_references.pydeps b/build/android/gyp/validate_static_library_dex_references.pydeps index e57172d..7fd91c2 100644 --- a/build/android/gyp/validate_static_library_dex_references.pydeps +++ b/build/android/gyp/validate_static_library_dex_references.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/validate_static_library_dex_references.pydeps build/android/gyp/validate_static_library_dex_references.py +../../action_helpers.py ../../gn_helpers.py ../pylib/__init__.py ../pylib/dex/__init__.py
diff --git a/build/android/gyp/write_build_config.py b/build/android/gyp/write_build_config.py index 0600fdc..7976dd8 100755 --- a/build/android/gyp/write_build_config.py +++ b/build/android/gyp/write_build_config.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -57,7 +57,7 @@ * [java_binary](#target_java_binary) * [java_annotation_processor](#target_java_annotation_processor) - * [junit_binary](#target_junit_binary) + * [robolectric_binary](#target_robolectric_binary) * [java_library](#target_java_library) * [android_assets](#target_android_assets) * [android_resources](#target_android_resources) @@ -114,10 +114,10 @@ Only seen for the [`android_app_bundle`](#target_android_app_bundle) type. Path to the base module for the bundle. -* `deps_info['is_base_module']`: +* `deps_info['module_name']`: Only seen for the [`android_app_bundle_module`](#target_android_app_bundle_module) -type. Whether or not this module is the base module for some bundle. +type. The name of the feature module. * `deps_info['dependency_zips']`: List of `deps_info['resources_zip']` entries for all `android_resources` @@ -154,17 +154,16 @@ * `deps_info['res_sources_path']`: Path to file containing a list of resource source files used by the -android_resources target. This replaces `deps_info['resource_dirs']` which is -now no longer used. +android_resources target. * `deps_info['resources_zip']`: *Required*. Path to the `.resources.zip` file that contains all raw/uncompiled resource files for this target (and also no `R.txt`, `R.java` or `R.class`). - If `deps_info['resource_dirs']` is missing, this must point to a prebuilt - `.aar` archive containing resources. Otherwise, this will point to a - zip archive generated at build time, wrapping the content of - `deps_info['resource_dirs']` into a single zip file. + If `deps_info['res_sources_path']` is missing, this must point to a prebuilt + `.aar` archive containing resources. Otherwise, this will point to a zip + archive generated at build time, wrapping the sources listed in + `deps_info['res_sources_path']` into a single zip file. * `deps_info['package_name']`: Java package name that the R class for this target belongs to. @@ -192,15 +191,6 @@ *always* generated from the content of `deps_info['r_text_path']` by the `build/android/gyp/process_resources.py` script. -* `deps_info['static_library_dependent_classpath_configs']`: -Sub dictionary mapping .build_config paths to lists of jar files. For static -library APKs, this defines which input jars belong to each -static_library_dependent_target. - -* `deps_info['static_library_proguard_mapping_output_paths']`: -Additional paths to copy the ProGuard mapping file to for static library -APKs. - ## <a name="target_android_assets">Target type `android_assets`</a>: This type corresponds to targets used to group Android assets, i.e. liberal @@ -244,11 +234,7 @@ * `deps_info['public_deps_configs']`: List of paths to the `.build_config` files of *direct* dependencies of the current target which are exposed as part of the -current target's public API. This should be a subset of -deps_info['deps_configs']. - -* `deps_info['ignore_dependency_public_deps']`: If true, 'public_deps' will not -be collected from the current target's direct deps. +current target's public API. * `deps_info['unprocessed_jar_path']`: Path to the original .jar file for this target, before any kind of processing @@ -278,17 +264,18 @@ In this case, `deps_info['unprocessed_jar_path']` will point to the source `.jar` file. Otherwise, it will be point to a build-generated file. -* `deps_info['java_sources_file']`: -Path to a single `.sources` file listing all the Java sources that were used -to generate the library (simple text format, one `.jar` path per line). +* `deps_info['target_sources_file']`: +Path to a single `.sources` file listing all the Java and Kotlin sources that +were used to generate the library (simple text format, one `.jar` path per +line). * `deps_info['lint_android_manifest']`: Path to an AndroidManifest.xml file to use for this lint target. -* `deps_info['lint_java_sources']`: -The list of all `deps_info['java_sources_file']` entries for all library +* `deps_info['lint_sources']`: +The list of all `deps_info['target_sources_file']` entries for all library dependencies that are chromium code. Note: this is a list of files, where each -file contains a list of Java source files. This is used for lint. +file contains a list of Java and Kotlin source files. This is used for lint. * `deps_info['lint_aars']`: List of all aars from transitive java dependencies. This allows lint to collect @@ -343,8 +330,18 @@ collection of all `deps_info['device_jar_path']` entries for the target and all its dependencies. +* `deps_info['all_dex_files']`: +The list of paths to all `deps_info['dex_path']` entries for all libraries +that comprise this APK. Valid only for debug builds. -## <a name="target_junit_binary">Target type `junit_binary`</a>: +* `deps_info['preferred_dep']`: +Whether the target should be the preferred dep. This is usually the case when we +have a java_group that depends on either the public or internal dep accordingly, +and it is better to depend on the group rather than the underlying dep. Another +case is for android_library_factory targets, the factory target should be +preferred instead of the actual implementation. + +## <a name="target_robolectric_binary">Target type `robolectric_binary`</a>: A target type for JUnit-specific binaries. Identical to [`java_binary`](#target_java_binary) in the context of `.build_config` files, @@ -381,11 +378,7 @@ * `deps_info['final_dex']['path']`: Path to the final classes.dex file (or classes.zip in case of multi-dex) -for this APK. - -* `deps_info['final_dex']['all_dex_files']`: -The list of paths to all `deps_info['dex_path']` entries for all libraries -that comprise this APK. Valid only for debug builds. +for this APK - only used for proguarded builds. * `native['libraries']` List of native libraries for the primary ABI to be embedded in this APK. @@ -401,10 +394,6 @@ List of native libraries for the secondary ABI to be embedded in this APK. Empty if only a single ABI is supported. -* `native['uncompress_shared_libraries']` -A boolean indicating whether native libraries are stored uncompressed in the -APK. - * `native['loadable_modules']` A list of native libraries to store within the APK, in addition to those from `native['libraries']`. These correspond to things like the Chromium linker @@ -416,9 +405,6 @@ * `native['library_always_compress']` A list of library files that we always compress. -* `native['library_renames']` -A list of library files that we prepend "crazy." to their file names. - * `assets` A list of assets stored compressed in the APK. Each entry has the format `<source-path>:<destination-path>`, where `<source-path>` is relative to @@ -451,10 +437,11 @@ NOTE: This has nothing to do with *Android* resources. -* `jni['all_source']` -The list of all `deps_info['java_sources_file']` entries for all library +* `deps_info['jni_all_source']` +The list of all `deps_info['target_sources_file']` entries for all library dependencies for this APK. Note: this is a list of files, where each file -contains a list of Java source files. This is used for JNI registration. +contains a list of Java and Kotlin source files. This is used for JNI +registration. * `deps_info['proguard_all_configs']`: The collection of all 'deps_info['proguard_configs']` values from this target @@ -544,11 +531,6 @@ `android_apk` and others), and contains information related to the compilation of Java sources, class files, and jars. -* `javac['resource_packages']` -For `java_library` targets, this is the list of package names for all resource -dependencies for the current target. Order must match the one from -`javac['srcjars']`. For other target types, this key does not exist. - * `javac['classpath']` The classpath used to compile this target when annotation processors are present. @@ -573,38 +555,33 @@ --------------- END_MARKDOWN --------------------------------------------------- """ -from __future__ import print_function - import collections import itertools import json import optparse import os +import shutil import sys import xml.dom.minidom from util import build_utils from util import resource_utils - -# TODO(crbug.com/1174969): Remove this once Python2 is obsoleted. -if sys.version_info.major == 2: - zip_longest = itertools.izip_longest -else: - zip_longest = itertools.zip_longest +import action_helpers # build_utils adds //build to sys.path. # Types that should never be used as a dependency of another build config. _ROOT_TYPES = ('android_apk', 'java_binary', 'java_annotation_processor', - 'junit_binary', 'android_app_bundle') + 'robolectric_binary', 'android_app_bundle') # Types that should not allow code deps to pass through. _RESOURCE_TYPES = ('android_assets', 'android_resources', 'system_java_library') +# Cache of path -> JSON dict. +_dep_config_cache = {} + class OrderedSet(collections.OrderedDict): - # Value |parameter| is present to avoid presubmit warning due to different - # number of parameters from overridden method. @staticmethod - def fromkeys(iterable, value=None): + def fromkeys(iterable): out = OrderedSet() out.update(iterable) return out @@ -636,7 +613,8 @@ return result -class AndroidManifest(object): + +class AndroidManifest: def __init__(self, path): self.path = path dom = xml.dom.minidom.parse(path) @@ -666,26 +644,40 @@ return self.manifest.getAttribute('package') -dep_config_cache = {} -def GetDepConfig(path): - if not path in dep_config_cache: +def GetDepConfigRoot(path): + if not path in _dep_config_cache: with open(path) as jsonfile: - dep_config_cache[path] = json.load(jsonfile)['deps_info'] - return dep_config_cache[path] + _dep_config_cache[path] = json.load(jsonfile) + return _dep_config_cache[path] + + +def GetDepConfig(path): + return GetDepConfigRoot(path)['deps_info'] def DepsOfType(wanted_type, configs): return [c for c in configs if c['type'] == wanted_type] -def GetAllDepsConfigsInOrder(deps_config_paths, filter_func=None): - def GetDeps(path): - config = GetDepConfig(path) - if filter_func and not filter_func(config): - return [] - return config['deps_configs'] +def DepPathsOfType(wanted_type, config_paths): + return [p for p in config_paths if GetDepConfig(p)['type'] == wanted_type] - return build_utils.GetSortedTransitiveDependencies(deps_config_paths, GetDeps) + +def GetAllDepsConfigsInOrder(deps_config_paths, filter_func=None): + def apply_filter(paths): + if filter_func: + return [p for p in paths if filter_func(GetDepConfig(p))] + return paths + + def discover(path): + config = GetDepConfig(path) + all_deps = config['deps_configs'] + config.get('public_deps_configs', []) + return apply_filter(all_deps) + + deps_config_paths = apply_filter(deps_config_paths) + deps_config_paths = build_utils.GetSortedTransitiveDependencies( + deps_config_paths, discover) + return deps_config_paths def GetObjectByPath(obj, key_path): @@ -703,7 +695,7 @@ target[:] = [x for x in target if x not in base_target] -class Deps(object): +class Deps: def __init__(self, direct_deps_config_paths): self._all_deps_config_paths = GetAllDepsConfigsInOrder( direct_deps_config_paths) @@ -725,18 +717,6 @@ return self._direct_deps_configs return DepsOfType(wanted_type, self._direct_deps_configs) - def DirectAndChildPublicDeps(self, wanted_type=None): - """Returns direct dependencies and dependencies exported via public_deps of - direct dependencies. - """ - dep_paths = set(self._direct_deps_config_paths) - for direct_dep in self._direct_deps_configs: - dep_paths.update(direct_dep.get('public_deps_configs', [])) - deps_list = [GetDepConfig(p) for p in dep_paths] - if wanted_type is None: - return deps_list - return DepsOfType(wanted_type, deps_list) - def AllConfigPaths(self): return self._all_deps_config_paths @@ -760,7 +740,9 @@ if config['is_prebuilt']: pass elif config['gradle_treat_as_prebuilt']: - helper(Deps(config['deps_configs'])) + all_deps = config['deps_configs'] + config.get( + 'public_deps_configs', []) + helper(Deps(all_deps)) elif config not in ret: ret.append(config) @@ -789,7 +771,7 @@ dest_map = uncompressed if disable_compression else compressed other_map = compressed if disable_compression else uncompressed outputs = entry.get('outputs', []) - for src, dest in zip_longest(entry['sources'], outputs): + for src, dest in itertools.zip_longest(entry['sources'], outputs): if not dest: dest = os.path.basename(src) # Merge so that each path shows up in only one of the lists, and that @@ -800,25 +782,30 @@ locale_paks.add(dest) def create_list(asset_map): - ret = ['%s:%s' % (src, dest) for dest, src in asset_map.items()] # Sort to ensure deterministic ordering. - ret.sort() - return ret + items = sorted(asset_map.items()) + return [f'{src}:{dest}' for dest, src in items] return create_list(compressed), create_list(uncompressed), locale_paks -def _ResolveGroups(configs): +def _ResolveGroupsAndPublicDeps(config_paths): """Returns a list of configs with all groups inlined.""" - ret = list(configs) - while True: - groups = DepsOfType('group', ret) - if not groups: - return ret - for config in groups: - index = ret.index(config) - expanded_configs = [GetDepConfig(p) for p in config['deps_configs']] - ret[index:index + 1] = expanded_configs + + def helper(config_path): + config = GetDepConfig(config_path) + if config['type'] == 'group': + # Groups combine public_deps with deps_configs, so no need to check + # public_config_paths separately. + return config['deps_configs'] + if config['type'] == 'android_resources': + # android_resources targets do not support public_deps, but instead treat + # all resource deps as public deps. + return DepPathsOfType('android_resources', config['deps_configs']) + + return config.get('public_deps_configs', []) + + return build_utils.GetSortedTransitiveDependencies(config_paths, helper) def _DepsFromPaths(dep_paths, @@ -860,6 +847,18 @@ return _DepsFromPathsWithFilters(dep_paths, blocklist, allowlist) +def _FilterConfigPaths(dep_paths, blocklist=None, allowlist=None): + if not blocklist and not allowlist: + return dep_paths + configs = [GetDepConfig(p) for p in dep_paths] + if blocklist: + configs = [c for c in configs if c['type'] not in blocklist] + if allowlist: + configs = [c for c in configs if c['type'] in allowlist] + + return [c['path'] for c in configs] + + def _DepsFromPathsWithFilters(dep_paths, blocklist=None, allowlist=None): """Resolves all groups and trims dependency branches that we never want. @@ -872,16 +871,17 @@ about (i.e. we wish to prune all other branches that do not start from one of these). """ - configs = [GetDepConfig(p) for p in dep_paths] - groups = DepsOfType('group', configs) - configs = _ResolveGroups(configs) - configs += groups - if blocklist: - configs = [c for c in configs if c['type'] not in blocklist] + # Filter both before and after so that public_deps of blocked targets are not + # added. + allowlist_with_groups = None if allowlist: - configs = [c for c in configs if c['type'] in allowlist] + allowlist_with_groups = set(allowlist) + allowlist_with_groups.add('group') + dep_paths = _FilterConfigPaths(dep_paths, blocklist, allowlist_with_groups) + dep_paths = _ResolveGroupsAndPublicDeps(dep_paths) + dep_paths = _FilterConfigPaths(dep_paths, blocklist, allowlist) - return Deps([c['path'] for c in configs]) + return Deps(dep_paths) def _ExtractSharedLibsFromRuntimeDeps(runtime_deps_file): @@ -937,10 +937,103 @@ return 1 if dep.get('low_classpath_priority') else 0 +def _DedupFeatureModuleSharedCode(uses_split_arg, modules, + field_names_to_dedup): + child_to_ancestors = collections.defaultdict(list) + if uses_split_arg: + for split_pair in uses_split_arg: + child, parent = split_pair.split(':') + assert child in modules + assert parent in modules + child_to_ancestors[child] = [parent] + + # Create a full list of ancestors for each module. + for name in modules: + if name == 'base': + continue + curr_name = name + while curr_name in child_to_ancestors: + parent = child_to_ancestors[curr_name][0] + if parent not in child_to_ancestors[name]: + child_to_ancestors[name].append(parent) + curr_name = parent + + if curr_name != 'base': + child_to_ancestors[name].append('base') + + # Strip out duplicates from ancestors. + for name, module in modules.items(): + if name == 'base': + continue + # Make sure we get all ancestors, not just direct parent. + for ancestor in child_to_ancestors[name]: + for f in field_names_to_dedup: + if f in module: + RemoveObjDups(module, modules[ancestor], f) + + # Strip out duplicates from siblings/cousins. + for f in field_names_to_dedup: + _PromoteToCommonAncestor(modules, child_to_ancestors, f) + + +def _PromoteToCommonAncestor(modules, child_to_ancestors, field_name): + module_to_fields_set = {} + for module_name, module in modules.items(): + if field_name in module: + module_to_fields_set[module_name] = set(module[field_name]) + + seen = set() + dupes = set() + for fields in module_to_fields_set.values(): + new_dupes = seen & fields + if new_dupes: + dupes |= new_dupes + seen |= fields + + for d in dupes: + owning_modules = [] + for module_name, fields in module_to_fields_set.items(): + if d in fields: + owning_modules.append(module_name) + assert len(owning_modules) >= 2 + # Rely on the fact that ancestors are inserted from closest to + # farthest, where "base" should always be the last element. + # Arbitrarily using the first owning module - any would work. + for ancestor in child_to_ancestors[owning_modules[0]]: + ancestor_is_shared_with_all = True + for o in owning_modules[1:]: + if ancestor not in child_to_ancestors[o]: + ancestor_is_shared_with_all = False + break + if ancestor_is_shared_with_all: + common_ancestor = ancestor + break + for o in owning_modules: + module_to_fields_set[o].remove(d) + module_to_fields_set[common_ancestor].add(d) + + for module_name, module in modules.items(): + if field_name in module: + module[field_name] = sorted(list(module_to_fields_set[module_name])) + + +def _CopyBuildConfigsForDebugging(debug_dir): + shutil.rmtree(debug_dir, ignore_errors=True) + os.makedirs(debug_dir) + for src_path in _dep_config_cache: + dst_path = os.path.join(debug_dir, src_path) + assert dst_path.startswith(debug_dir), dst_path + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy(src_path, dst_path) + print(f'Copied {len(_dep_config_cache)} .build_config.json into {debug_dir}') + + def main(argv): parser = optparse.OptionParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_option('--build-config', help='Path to build_config output.') + parser.add_option('--store-deps-for-debugging-to', + help='Path to copy all transitive build config files to.') parser.add_option( '--type', help='Type of this target (e.g. android_library).') @@ -957,7 +1050,10 @@ parser.add_option('--resources-zip', help='Path to target\'s resources zip.') parser.add_option('--package-name', help='Java package name for these resources.') - parser.add_option('--android-manifest', help='Path to android manifest.') + parser.add_option('--android-manifest', + help='Path to the root android manifest.') + parser.add_option('--merged-android-manifest', + help='Path to the merged android manifest.') parser.add_option('--resource-dirs', action='append', default=[], help='GYP-list of resource dirs') parser.add_option( @@ -984,16 +1080,15 @@ parser.add_option('--treat-as-locale-paks', action='store_true', help='Consider the assets as locale paks in BuildConfig.java') - # java library options + # java library and group options + parser.add_option('--preferred-dep', + action='store_true', + help='Whether the target should be preferred as a dep.') + # java library options parser.add_option('--public-deps-configs', help='GN list of config files of deps which are exposed as ' 'part of the target\'s public API.') - parser.add_option( - '--ignore-dependency-public-deps', - action='store_true', - help='If true, \'public_deps\' will not be collected from the current ' - 'target\'s direct deps.') parser.add_option('--aar-path', help='Path to containing .aar file.') parser.add_option('--device-jar-path', help='Path to .jar for dexing.') parser.add_option('--host-jar-path', help='Path to .jar for java_binary.') @@ -1004,7 +1099,7 @@ help='Path to the interface .jar to use for javac classpath purposes.') parser.add_option('--is-prebuilt', action='store_true', help='Whether the jar was compiled or pre-compiled.') - parser.add_option('--java-sources-file', help='Path to .sources file') + parser.add_option('--target-sources-file', help='Path to .sources file') parser.add_option('--bundled-srcjars', help='GYP-list of .srcjars that have been included in this java_library.') parser.add_option('--supports-android', action='store_true', @@ -1038,6 +1133,11 @@ action='store_true', help='True if a java library is not chromium code, used for lint.') + # robolectric_library options + parser.add_option('--is-robolectric', + action='store_true', + help='Whether this is a host side android test library.') + # android library options parser.add_option('--dex-path', help='Path to target\'s dex output.') @@ -1077,10 +1177,6 @@ parser.add_option( '--library-always-compress', help='The list of library files that we always compress.') - parser.add_option( - '--library-renames', - default=[], - help='The list of library files that we prepend crazy. to their names.') # apk options parser.add_option('--apk-path', help='Path to the target\'s apk output.') @@ -1128,24 +1224,33 @@ parser.add_option( '--base-allowlist-rtxt-path', help='Path to R.txt file for the base resources allowlist.') - parser.add_option( - '--is-base-module', - action='store_true', - help='Specifies that this module is a base module for some app bundle.') parser.add_option('--generate-markdown-format-doc', action='store_true', help='Dump the Markdown .build_config format documentation ' 'then exit immediately.') + parser.add_option('--module-name', help='The name of this feature module.') parser.add_option( '--base-module-build-config', help='Path to the base module\'s build config ' 'if this is a feature module.') + parser.add_option('--parent-module-build-config', + help='Path to the parent module\'s build config ' + 'when not using base module as parent.') parser.add_option( '--module-build-configs', help='For bundles, the paths of all non-async module .build_configs ' 'for modules that are part of the bundle.') + parser.add_option( + '--uses-split', + action='append', + help='List of name pairs separated by : mapping a feature module to a ' + 'dependent feature module.') + + parser.add_option( + '--trace-events-jar-dir', + help='Directory of rewritten .jar files for trace event rewriting.') parser.add_option('--version-name', help='Version name for this APK.') parser.add_option('--version-code', help='Version code for this APK.') @@ -1162,15 +1267,14 @@ return 0 if options.fail: - parser.error('\n'.join(build_utils.ParseGnList(options.fail))) + parser.error('\n'.join(action_helpers.parse_gn_list(options.fail))) lib_options = ['unprocessed_jar_path', 'interface_jar_path'] device_lib_options = ['device_jar_path', 'dex_path'] required_options_map = { 'android_apk': ['build_config'] + lib_options + device_lib_options, 'android_app_bundle_module': - ['build_config', 'final_dex_path', 'res_size_info'] + lib_options + - device_lib_options, + ['build_config', 'res_size_info'] + lib_options + device_lib_options, 'android_assets': ['build_config'], 'android_resources': ['build_config', 'resources_zip'], 'dist_aar': ['build_config'], @@ -1179,7 +1283,7 @@ 'java_annotation_processor': ['build_config', 'main_class'], 'java_binary': ['build_config'], 'java_library': ['build_config', 'host_jar_path'] + lib_options, - 'junit_binary': ['build_config'], + 'robolectric_binary': ['build_config'], 'system_java_library': ['build_config', 'unprocessed_jar_path'], 'android_app_bundle': ['build_config', 'module_build_configs'], } @@ -1199,26 +1303,18 @@ if options.base_allowlist_rtxt_path: raise Exception('--base-allowlist-rtxt-path can only be used with ' '--type=android_app_bundle_module') - if options.is_base_module: - raise Exception('--is-base-module can only be used with ' + if options.module_name: + raise Exception('--module-name can only be used with ' '--type=android_app_bundle_module') is_apk_or_module_target = options.type in ('android_apk', 'android_app_bundle_module') if not is_apk_or_module_target: - if options.uncompress_shared_libraries: - raise Exception('--uncompressed-shared-libraries can only be used ' - 'with --type=android_apk or ' - '--type=android_app_bundle_module') if options.library_always_compress: raise Exception( '--library-always-compress can only be used with --type=android_apk ' 'or --type=android_app_bundle_module') - if options.library_renames: - raise Exception( - '--library-renames can only be used with --type=android_apk or ' - '--type=android_app_bundle_module') if options.device_jar_path and not options.dex_path: raise Exception('java_library that supports Android requires a dex path.') @@ -1231,34 +1327,26 @@ raise Exception( '--supports-android is required when using --requires-android') - is_java_target = options.type in ( - 'java_binary', 'junit_binary', 'java_annotation_processor', - 'java_library', 'android_apk', 'dist_aar', 'dist_jar', - 'system_java_library', 'android_app_bundle_module') + is_java_target = options.type in ('java_binary', 'robolectric_binary', + 'java_annotation_processor', 'java_library', + 'android_apk', 'dist_aar', 'dist_jar', + 'system_java_library', + 'android_app_bundle_module') - is_static_library_dex_provider_target = ( - options.static_library_dependent_configs and options.proguard_enabled) - if is_static_library_dex_provider_target: - if options.type != 'android_apk': - raise Exception( - '--static-library-dependent-configs only supports --type=android_apk') - options.static_library_dependent_configs = build_utils.ParseGnList( - options.static_library_dependent_configs) - static_library_dependent_configs_by_path = { - p: GetDepConfig(p) - for p in options.static_library_dependent_configs - } - - deps_configs_paths = build_utils.ParseGnList(options.deps_configs) + deps_configs_paths = action_helpers.parse_gn_list(options.deps_configs) + public_deps_configs_paths = action_helpers.parse_gn_list( + options.public_deps_configs) + deps_configs_paths += public_deps_configs_paths deps = _DepsFromPaths(deps_configs_paths, options.type, recursive_resource_deps=options.recursive_resource_deps) - processor_deps = _DepsFromPaths( - build_utils.ParseGnList(options.annotation_processor_configs or ''), - options.type, filter_root_targets=False) + public_deps = _DepsFromPaths(public_deps_configs_paths, options.type) + processor_deps = _DepsFromPaths(action_helpers.parse_gn_list( + options.annotation_processor_configs or ''), + options.type, + filter_root_targets=False) - all_inputs = (deps.AllConfigPaths() + processor_deps.AllConfigPaths() + - list(static_library_dependent_configs_by_path)) + all_inputs = (deps.AllConfigPaths() + processor_deps.AllConfigPaths()) if options.recursive_resource_deps: # Include java_library targets since changes to these targets can remove @@ -1269,41 +1357,35 @@ allowlist=['java_library']) all_inputs.extend(recursive_java_deps.AllConfigPaths()) - direct_deps = deps.Direct() system_library_deps = deps.Direct('system_java_library') all_deps = deps.All() all_library_deps = deps.All('java_library') - all_resources_deps = deps.All('android_resources') if options.type == 'java_library': - java_library_deps = _DepsFromPathsWithFilters( - deps_configs_paths, allowlist=['android_resources']) - # for java libraries, we only care about resources that are directly - # reachable without going through another java_library. - all_resources_deps = java_library_deps.All('android_resources') + # For Java libraries, restrict to resource targets that are direct deps, or + # are indirect via other resource targets. + # The indirect-through-other-targets ones are picked up because + # _ResolveGroupsAndPublicDeps() treats resource deps of resource targets as + # public_deps. + all_resources_deps = deps.Direct('android_resources') + else: + all_resources_deps = deps.All('android_resources') + if options.type == 'android_resources' and options.recursive_resource_deps: # android_resources targets that want recursive resource deps also need to # collect package_names from all library deps. This ensures the R.java files # for these libraries will get pulled in along with the resources. android_resources_library_deps = _DepsFromPathsWithFilters( deps_configs_paths, allowlist=['java_library']).All('java_library') - if is_apk_or_module_target: - # android_resources deps which had recursive_resource_deps set should not - # have the manifests from the recursively collected deps added to this - # module. This keeps the manifest declarations in the child DFMs, since they - # will have the Java implementations. - def ExcludeRecursiveResourcesDeps(config): - return not config.get('includes_recursive_resources', False) - - extra_manifest_deps = [ - GetDepConfig(p) for p in GetAllDepsConfigsInOrder( - deps_configs_paths, filter_func=ExcludeRecursiveResourcesDeps) - ] base_module_build_config = None if options.base_module_build_config: - with open(options.base_module_build_config, 'r') as f: - base_module_build_config = json.load(f) + base_module_build_config = GetDepConfigRoot( + options.base_module_build_config) + parent_module_build_config = base_module_build_config + if options.parent_module_build_config: + parent_module_build_config = GetDepConfigRoot( + options.parent_module_build_config) # Initialize some common config. # Any value that needs to be queryable by dependents must go within deps_info. @@ -1313,7 +1395,6 @@ 'path': options.build_config, 'type': options.type, 'gn_target': options.gn_target, - 'deps_configs': [d['path'] for d in direct_deps], 'chromium_code': not options.non_chromium_code, }, # Info needed only by generate_gradle.py. @@ -1322,45 +1403,72 @@ deps_info = config['deps_info'] gradle = config['gradle'] + # The paths we record as deps can differ from deps_config_paths: + # 1) Paths can be removed when blocked by _ROOT_TYPES / _RESOURCE_TYPES. + # 2) Paths can be added when promoted from group deps or public_deps of deps. + # Deps are promoted from groups/public_deps in order to make the filtering + # of 1) work through group() targets (which themselves are not resource + # targets, but should be treated as such when depended on by a resource + # target. A more involved filtering implementation could work to maintain + # the semantics of 1) without the need to promote deps, but we've avoided + # such an undertaking so far. + public_deps_set = set() + if public_deps_configs_paths: + deps_info['public_deps_configs'] = [d['path'] for d in public_deps.Direct()] + public_deps_set = set(deps_info['public_deps_configs']) + + deps_info['deps_configs'] = [ + d['path'] for d in deps.Direct() if d['path'] not in public_deps_set + ] + if options.type == 'android_apk' and options.tested_apk_config: tested_apk_deps = Deps([options.tested_apk_config]) tested_apk_config = tested_apk_deps.Direct()[0] gradle['apk_under_test'] = tested_apk_config['name'] if options.type == 'android_app_bundle_module': - deps_info['is_base_module'] = bool(options.is_base_module) + deps_info['module_name'] = options.module_name # Required for generating gradle files. if options.type == 'java_library': deps_info['is_prebuilt'] = bool(options.is_prebuilt) deps_info['gradle_treat_as_prebuilt'] = options.gradle_treat_as_prebuilt + if options.preferred_dep: + deps_info['preferred_dep'] = bool(options.preferred_dep) + if options.android_manifest: deps_info['android_manifest'] = options.android_manifest + if options.merged_android_manifest: + deps_info['merged_android_manifest'] = options.merged_android_manifest + if options.bundled_srcjars: - deps_info['bundled_srcjars'] = build_utils.ParseGnList( + deps_info['bundled_srcjars'] = action_helpers.parse_gn_list( options.bundled_srcjars) - if options.java_sources_file: - deps_info['java_sources_file'] = options.java_sources_file + if options.target_sources_file: + deps_info['target_sources_file'] = options.target_sources_file if is_java_target: - if options.bundled_srcjars: - gradle['bundled_srcjars'] = deps_info['bundled_srcjars'] - - gradle['dependent_android_projects'] = [] - gradle['dependent_java_projects'] = [] - gradle['dependent_prebuilt_jars'] = deps.GradlePrebuiltJarPaths() - if options.main_class: deps_info['main_class'] = options.main_class + dependent_prebuilt_jars = deps.GradlePrebuiltJarPaths() + dependent_prebuilt_jars.sort() + if dependent_prebuilt_jars: + gradle['dependent_prebuilt_jars'] = dependent_prebuilt_jars + + dependent_android_projects = [] + dependent_java_projects = [] for c in deps.GradleLibraryProjectDeps(): if c['requires_android']: - gradle['dependent_android_projects'].append(c['path']) + dependent_android_projects.append(c['path']) else: - gradle['dependent_java_projects'].append(c['path']) + dependent_java_projects.append(c['path']) + + gradle['dependent_android_projects'] = dependent_android_projects + gradle['dependent_java_projects'] = dependent_java_projects if options.r_text_path: deps_info['r_text_path'] = options.r_text_path @@ -1368,14 +1476,18 @@ # TODO(tiborg): Remove creation of JNI info for type group and java_library # once we can generate the JNI registration based on APK / module targets as # opposed to groups and libraries. - if is_apk_or_module_target or options.type in ( - 'group', 'java_library', 'junit_binary'): - deps_info['jni'] = {} - all_java_sources = [c['java_sources_file'] for c in all_library_deps - if 'java_sources_file' in c] - if options.java_sources_file: - all_java_sources.append(options.java_sources_file) + if is_apk_or_module_target or options.type in ('group', 'java_library', + 'robolectric_binary', + 'dist_aar'): + all_target_sources = [ + c['target_sources_file'] for c in all_library_deps + if 'target_sources_file' in c + ] + if options.target_sources_file: + all_target_sources.append(options.target_sources_file) + if is_apk_or_module_target or options.type in ('group', 'java_library', + 'robolectric_binary'): if options.apk_proto_resources: deps_info['proto_resources_path'] = options.apk_proto_resources @@ -1400,7 +1512,10 @@ deps_info['requires_android'] = bool(options.requires_android) deps_info['supports_android'] = bool(options.supports_android) - if not options.bypass_platform_checks: + # robolectric is special in that its an android target that runs on host. + # You are allowed to depend on both android |deps_require_android| and + # non-android |deps_not_support_android| targets. + if not options.bypass_platform_checks and not options.is_robolectric: deps_require_android = (all_resources_deps + [d['name'] for d in all_library_deps if d['requires_android']]) deps_not_support_android = ( @@ -1425,9 +1540,6 @@ if options.unprocessed_jar_path: deps_info['unprocessed_jar_path'] = options.unprocessed_jar_path deps_info['interface_jar_path'] = options.interface_jar_path - if options.public_deps_configs: - deps_info['public_deps_configs'] = build_utils.ParseGnList( - options.public_deps_configs) if options.device_jar_path: deps_info['device_jar_path'] = options.device_jar_path if options.host_jar_path: @@ -1448,16 +1560,17 @@ all_asset_sources = [] if options.asset_renaming_sources: all_asset_sources.extend( - build_utils.ParseGnList(options.asset_renaming_sources)) + action_helpers.parse_gn_list(options.asset_renaming_sources)) if options.asset_sources: - all_asset_sources.extend(build_utils.ParseGnList(options.asset_sources)) + all_asset_sources.extend( + action_helpers.parse_gn_list(options.asset_sources)) deps_info['assets'] = { 'sources': all_asset_sources } if options.asset_renaming_destinations: - deps_info['assets']['outputs'] = ( - build_utils.ParseGnList(options.asset_renaming_destinations)) + deps_info['assets']['outputs'] = (action_helpers.parse_gn_list( + options.asset_renaming_destinations)) if options.disable_asset_compression: deps_info['assets']['disable_compression'] = True if options.treat_as_locale_paks: @@ -1478,15 +1591,12 @@ if options.res_sources_path: deps_info['res_sources_path'] = options.res_sources_path - if options.requires_android and options.type == 'java_library': - # Used to strip out R.class for android_prebuilt()s. - config['javac']['resource_packages'] = [ - c['package_name'] for c in all_resources_deps if 'package_name' in c - ] + if (options.requires_android + and options.type == 'java_library') or options.is_robolectric: if options.package_name: deps_info['package_name'] = options.package_name - if options.type in ('android_resources', 'android_apk', 'junit_binary', + if options.type in ('android_resources', 'android_apk', 'robolectric_binary', 'dist_aar', 'android_app_bundle_module', 'java_library'): dependency_zips = [] dependency_zip_overlays = [] @@ -1504,6 +1614,8 @@ extra_package_names = [ c['package_name'] for c in all_resources_deps if 'package_name' in c ] + if options.package_name: + extra_package_names += [options.package_name] # android_resources targets which specified recursive_resource_deps may # have extra_package_names. @@ -1532,22 +1644,6 @@ ] deps_info['dependency_r_txt_files'] = r_text_files - # For feature modules, remove any resources that already exist in the base - # module. - if base_module_build_config: - dependency_zips = [ - c for c in dependency_zips - if c not in base_module_build_config['deps_info']['dependency_zips'] - ] - dependency_zip_overlays = [ - c for c in dependency_zip_overlays if c not in - base_module_build_config['deps_info']['dependency_zip_overlays'] - ] - extra_package_names = [ - c for c in extra_package_names if c not in - base_module_build_config['deps_info']['extra_package_names'] - ] - if options.type == 'android_apk' and options.tested_apk_config: config['deps_info']['arsc_package_name'] = ( tested_apk_config['package_name']) @@ -1560,35 +1656,37 @@ if options.res_size_info: config['deps_info']['res_size_info'] = options.res_size_info + # Safe to sort: Build checks that non-overlay resource have no overlap. + dependency_zips.sort() config['deps_info']['dependency_zips'] = dependency_zips config['deps_info']['dependency_zip_overlays'] = dependency_zip_overlays + # Order doesn't matter, so make stable. + extra_package_names.sort() config['deps_info']['extra_package_names'] = extra_package_names # These are .jars to add to javac classpath but not to runtime classpath. - extra_classpath_jars = build_utils.ParseGnList(options.extra_classpath_jars) + extra_classpath_jars = action_helpers.parse_gn_list( + options.extra_classpath_jars) if extra_classpath_jars: + extra_classpath_jars.sort() deps_info['extra_classpath_jars'] = extra_classpath_jars - mergeable_android_manifests = build_utils.ParseGnList( + mergeable_android_manifests = action_helpers.parse_gn_list( options.mergeable_android_manifests) + mergeable_android_manifests.sort() if mergeable_android_manifests: deps_info['mergeable_android_manifests'] = mergeable_android_manifests extra_proguard_classpath_jars = [] - proguard_configs = build_utils.ParseGnList(options.proguard_configs) + proguard_configs = action_helpers.parse_gn_list(options.proguard_configs) if proguard_configs: # Make a copy of |proguard_configs| since it's mutated below. deps_info['proguard_configs'] = list(proguard_configs) if is_java_target: - if options.ignore_dependency_public_deps: - classpath_direct_deps = deps.Direct() - classpath_direct_library_deps = deps.Direct('java_library') - else: - classpath_direct_deps = deps.DirectAndChildPublicDeps() - classpath_direct_library_deps = deps.DirectAndChildPublicDeps( - 'java_library') + classpath_direct_deps = deps.Direct() + classpath_direct_library_deps = deps.Direct('java_library') # The classpath used to compile this target when annotation processors are # present. @@ -1656,7 +1754,7 @@ device_classpath.extend(c for c in d.get('device_classpath', []) if c not in device_classpath) - if options.type in ('dist_jar', 'java_binary', 'junit_binary'): + if options.type in ('dist_jar', 'java_binary', 'robolectric_binary'): # The classpath to use to run this target. host_classpath = [] if options.host_jar_path: @@ -1675,18 +1773,18 @@ # Collect all sources and resources at the apk/bundle_module level. lint_aars = set() lint_srcjars = set() - lint_java_sources = set() + lint_sources = set() lint_resource_sources = set() lint_resource_zips = set() - if options.java_sources_file: - lint_java_sources.add(options.java_sources_file) + if options.target_sources_file: + lint_sources.add(options.target_sources_file) if options.bundled_srcjars: lint_srcjars.update(deps_info['bundled_srcjars']) for c in all_library_deps: if c['chromium_code'] and c['requires_android']: - if 'java_sources_file' in c: - lint_java_sources.add(c['java_sources_file']) + if 'target_sources_file' in c: + lint_sources.add(c['target_sources_file']) lint_srcjars.update(c['bundled_srcjars']) if 'aar_path' in c: lint_aars.add(c['aar_path']) @@ -1706,7 +1804,7 @@ deps_info['lint_aars'] = sorted(lint_aars) deps_info['lint_srcjars'] = sorted(lint_srcjars) - deps_info['lint_java_sources'] = sorted(lint_java_sources) + deps_info['lint_sources'] = sorted(lint_sources) deps_info['lint_resource_sources'] = sorted(lint_resource_sources) deps_info['lint_resource_zips'] = sorted(lint_resource_zips) deps_info['lint_extra_android_manifests'] = [] @@ -1716,96 +1814,61 @@ deps_info['lint_android_manifest'] = options.android_manifest if options.type == 'android_app_bundle': - module_configs = [ - GetDepConfig(c) - for c in build_utils.ParseGnList(options.module_build_configs) + module_config_paths = action_helpers.parse_gn_list( + options.module_build_configs) + module_configs = [GetDepConfig(c) for c in module_config_paths] + module_configs_by_name = {d['module_name']: d for d in module_configs} + per_module_fields = [ + 'device_classpath', 'trace_event_rewritten_device_classpath', + 'all_dex_files' ] jni_all_source = set() lint_aars = set() lint_srcjars = set() - lint_java_sources = set() + lint_sources = set() lint_resource_sources = set() lint_resource_zips = set() lint_extra_android_manifests = set() - for c in module_configs: - if c['is_base_module']: + config['modules'] = {} + modules = config['modules'] + for n, c in module_configs_by_name.items(): + if n == 'base': assert 'base_module_config' not in deps_info, ( 'Must have exactly 1 base module!') + deps_info['package_name'] = c['package_name'] + deps_info['version_code'] = c['version_code'] + deps_info['version_name'] = c['version_name'] deps_info['base_module_config'] = c['path'] # Use the base module's android manifest for linting. deps_info['lint_android_manifest'] = c['android_manifest'] else: lint_extra_android_manifests.add(c['android_manifest']) - jni_all_source.update(c['jni']['all_source']) + jni_all_source.update(c['jni_all_source']) lint_aars.update(c['lint_aars']) lint_srcjars.update(c['lint_srcjars']) - lint_java_sources.update(c['lint_java_sources']) + lint_sources.update(c['lint_sources']) lint_resource_sources.update(c['lint_resource_sources']) lint_resource_zips.update(c['lint_resource_zips']) - deps_info['jni'] = {'all_source': sorted(jni_all_source)} + module = modules[n] = {} + for f in per_module_fields: + if f in c: + module[f] = c[f] + deps_info['jni_all_source'] = sorted(jni_all_source) deps_info['lint_aars'] = sorted(lint_aars) deps_info['lint_srcjars'] = sorted(lint_srcjars) - deps_info['lint_java_sources'] = sorted(lint_java_sources) + deps_info['lint_sources'] = sorted(lint_sources) deps_info['lint_resource_sources'] = sorted(lint_resource_sources) deps_info['lint_resource_zips'] = sorted(lint_resource_zips) deps_info['lint_extra_android_manifests'] = sorted( lint_extra_android_manifests) - # Map configs to classpath entries that should be included in their final dex. - classpath_entries_by_owning_config = collections.defaultdict(list) - extra_main_r_text_files = [] - if is_static_library_dex_provider_target: - # Map classpath entries to configs that include them in their classpath. - configs_by_classpath_entry = collections.defaultdict(list) - static_lib_jar_paths = {} - for config_path, dep_config in (sorted( - static_library_dependent_configs_by_path.items())): - # For bundles, only the jar path and jni sources of the base module - # are relevant for proguard. Should be updated when bundle feature - # modules support JNI. - base_config = dep_config - if dep_config['type'] == 'android_app_bundle': - base_config = GetDepConfig(dep_config['base_module_config']) - extra_main_r_text_files.append(base_config['r_text_path']) - static_lib_jar_paths[config_path] = base_config['device_jar_path'] - proguard_configs.extend(dep_config['proguard_all_configs']) - extra_proguard_classpath_jars.extend( - dep_config['proguard_classpath_jars']) - all_java_sources.extend(base_config['jni']['all_source']) - - # The srcjars containing the generated R.java files are excluded for APK - # targets the use static libraries, so we add them here to ensure the - # union of resource IDs are available in the static library APK. - for package in base_config['extra_package_names']: - if package not in extra_package_names: - extra_package_names.append(package) - for cp_entry in dep_config['device_classpath']: - configs_by_classpath_entry[cp_entry].append(config_path) - - for cp_entry in device_classpath: - configs_by_classpath_entry[cp_entry].append(options.build_config) - - for cp_entry, candidate_configs in configs_by_classpath_entry.items(): - config_path = (candidate_configs[0] - if len(candidate_configs) == 1 else options.build_config) - classpath_entries_by_owning_config[config_path].append(cp_entry) - device_classpath.append(cp_entry) - - device_classpath = sorted(set(device_classpath)) - - deps_info['static_library_proguard_mapping_output_paths'] = sorted([ - d['proguard_mapping_path'] - for d in static_library_dependent_configs_by_path.values() - ]) - deps_info['static_library_dependent_classpath_configs'] = { - path: sorted(set(classpath)) - for path, classpath in classpath_entries_by_owning_config.items() - } - deps_info['extra_main_r_text_files'] = sorted(extra_main_r_text_files) + _DedupFeatureModuleSharedCode(options.uses_split, modules, + per_module_fields) if is_apk_or_module_target or options.type in ('group', 'java_library', - 'junit_binary'): - deps_info['jni']['all_source'] = sorted(set(all_java_sources)) + 'robolectric_binary', + 'dist_aar'): + deps_info['jni_all_source'] = sorted(set(all_target_sources)) system_jars = [c['unprocessed_jar_path'] for c in system_library_deps] system_interface_jars = [c['interface_jar_path'] for c in system_library_deps] @@ -1911,14 +1974,11 @@ deps_info['proguard_classpath_jars'] = sorted( set(extra_proguard_classpath_jars)) - # Dependencies for the final dex file of an apk. - if (is_apk_or_module_target or options.final_dex_path - or options.type == 'dist_jar'): - config['final_dex'] = {} - dex_config = config['final_dex'] - dex_config['path'] = options.final_dex_path + if options.final_dex_path: + config['final_dex'] = {'path': options.final_dex_path} if is_apk_or_module_target or options.type == 'dist_jar': - dex_config['all_dex_files'] = all_dex_files + # Dependencies for the final dex file of an apk. + deps_info['all_dex_files'] = all_dex_files if is_java_target: config['javac']['classpath'] = sorted(javac_classpath) @@ -1933,8 +1993,8 @@ config['javac']['processor_classpath'] += [ c['host_jar_path'] for c in processor_deps.All('java_library') ] - config['javac']['processor_classes'] = [ - c['main_class'] for c in processor_deps.Direct()] + config['javac']['processor_classes'] = sorted( + c['main_class'] for c in processor_deps.Direct()) deps_info['javac_full_classpath'] = list(javac_full_classpath) deps_info['javac_full_interface_classpath'] = list( javac_full_interface_classpath) @@ -1952,9 +2012,23 @@ deps_info['javac_full_interface_classpath'] = list( javac_full_interface_classpath) - if options.type in ('android_apk', 'dist_jar', 'android_app_bundle_module', - 'android_app_bundle'): + if options.type in ('android_apk', 'android_app_bundle', + 'android_app_bundle_module', 'dist_aar', 'dist_jar'): deps_info['device_classpath'] = device_classpath + if options.trace_events_jar_dir: + trace_event_rewritten_device_classpath = [] + for jar_path in device_classpath: + file_path = jar_path.replace('../', '') + file_path = file_path.replace('obj/', '') + file_path = file_path.replace('gen/', '') + file_path = file_path.replace('.jar', '.tracing_rewritten.jar') + rewritten_jar_path = os.path.join(options.trace_events_jar_dir, + file_path) + trace_event_rewritten_device_classpath.append(rewritten_jar_path) + + deps_info['trace_event_rewritten_device_classpath'] = ( + trace_event_rewritten_device_classpath) + if options.tested_apk_config: deps_info['device_classpath_extended'] = device_classpath_extended @@ -1987,17 +2061,34 @@ if options.secondary_abi_shared_libraries_runtime_deps: secondary_abi_library_paths = _ExtractSharedLibsFromRuntimeDeps( options.secondary_abi_shared_libraries_runtime_deps) + secondary_abi_library_paths.sort() + paths_without_parent_dirs = [ + p for p in secondary_abi_library_paths if os.path.sep not in p + ] + if paths_without_parent_dirs: + sys.stderr.write('Found secondary native libraries from primary ' + 'toolchain directory. This is a bug!\n') + sys.stderr.write('\n'.join(paths_without_parent_dirs)) + sys.stderr.write('\n\nIt may be helpful to run: \n') + sys.stderr.write(' gn path out/Default //chrome/android:' + 'monochrome_secondary_abi_lib //base:base\n') + sys.exit(1) + all_inputs.append(options.secondary_abi_shared_libraries_runtime_deps) - native_library_placeholder_paths = build_utils.ParseGnList( + native_library_placeholder_paths = action_helpers.parse_gn_list( options.native_lib_placeholders) + native_library_placeholder_paths.sort() - secondary_native_library_placeholder_paths = build_utils.ParseGnList( + secondary_native_library_placeholder_paths = action_helpers.parse_gn_list( options.secondary_native_lib_placeholders) + secondary_native_library_placeholder_paths.sort() - loadable_modules = build_utils.ParseGnList(options.loadable_modules) - secondary_abi_loadable_modules = build_utils.ParseGnList( + loadable_modules = action_helpers.parse_gn_list(options.loadable_modules) + loadable_modules.sort() + secondary_abi_loadable_modules = action_helpers.parse_gn_list( options.secondary_abi_loadable_modules) + secondary_abi_loadable_modules.sort() config['native'] = { 'libraries': @@ -2010,27 +2101,13 @@ secondary_native_library_placeholder_paths, 'java_libraries_list': java_libraries_list, - 'uncompress_shared_libraries': - options.uncompress_shared_libraries, 'library_always_compress': options.library_always_compress, - 'library_renames': - options.library_renames, 'loadable_modules': loadable_modules, 'secondary_abi_loadable_modules': secondary_abi_loadable_modules, } - config['assets'], config['uncompressed_assets'], locale_paks = ( - _MergeAssets(deps.All('android_assets'))) - - deps_info['locales_java_list'] = _CreateJavaLocaleListFromAssets( - config['uncompressed_assets'], locale_paks) - - config['extra_android_manifests'] = [] - for c in extra_manifest_deps: - config['extra_android_manifests'].extend( - c.get('mergeable_android_manifests', [])) # Collect java resources java_resources_jars = [d['java_resources_jar'] for d in all_library_deps @@ -2041,22 +2118,63 @@ if 'java_resources_jar' in d] java_resources_jars = [jar for jar in java_resources_jars if jar not in tested_apk_resource_jars] + java_resources_jars.sort() config['java_resources_jars'] = java_resources_jars + if is_apk_or_module_target or options.type == 'robolectric_binary': + # android_resources deps which had recursive_resource_deps set should not + # have the manifests from the recursively collected deps added to this + # module. This keeps the manifest declarations in the child DFMs, since they + # will have the Java implementations. + def ExcludeRecursiveResourcesDeps(config): + return not config.get('includes_recursive_resources', False) + + extra_manifest_deps = [ + GetDepConfig(p) for p in GetAllDepsConfigsInOrder( + deps_configs_paths, filter_func=ExcludeRecursiveResourcesDeps) + ] + # Manifests are listed from highest priority to lowest priority. + # Ensure directly manfifests come first, and then sort the rest by name. + # https://developer.android.com/build/manage-manifests#merge_priorities + config['extra_android_manifests'] = list(mergeable_android_manifests) + manifests_from_deps = [] + for c in extra_manifest_deps: + manifests_from_deps += c.get('mergeable_android_manifests', []) + manifests_from_deps.sort(key=lambda p: (os.path.basename(p), p)) + config['extra_android_manifests'] += manifests_from_deps + + config['assets'], config['uncompressed_assets'], locale_paks = ( + _MergeAssets(deps.All('android_assets'))) + deps_info['locales_java_list'] = _CreateJavaLocaleListFromAssets( + config['uncompressed_assets'], locale_paks) + if options.java_resources_jar_path: deps_info['java_resources_jar'] = options.java_resources_jar_path # DYNAMIC FEATURE MODULES: - # Make sure that dependencies that exist on the base module - # are not duplicated on the feature module. + # There are two approaches to dealing with modules dependencies: + # 1) Perform steps in android_apk_or_module(), with only the knowledge of + # ancesstor splits. Our implementation currently allows only for 2 levels: + # base -> parent -> leaf + # Bundletool normally fails if two leaf nodes merge the same manifest or + # resources. The fix is to add the common dep to the chrome or base module + # so that our deduplication logic will work. + # RemoveObjDups() implements this approach. + # 2) Perform steps in android_app_bundle(), with knowledge of full set of + # modules. This is required for dex because it can handle the case of two + # leaf nodes having the same dep, and promoting that dep to their common + # parent. + # _DedupFeatureModuleSharedCode() implements this approach. if base_module_build_config: - base = base_module_build_config - RemoveObjDups(config, base, 'deps_info', 'device_classpath') - RemoveObjDups(config, base, 'deps_info', 'javac_full_classpath') - RemoveObjDups(config, base, 'deps_info', 'javac_full_interface_classpath') - RemoveObjDups(config, base, 'deps_info', 'jni', 'all_source') - RemoveObjDups(config, base, 'final_dex', 'all_dex_files') - RemoveObjDups(config, base, 'extra_android_manifests') + ancestors = [base_module_build_config] + if parent_module_build_config is not base_module_build_config: + ancestors += [parent_module_build_config] + for ancestor in ancestors: + RemoveObjDups(config, ancestor, 'deps_info', 'dependency_zips') + RemoveObjDups(config, ancestor, 'deps_info', 'dependency_zip_overlays') + RemoveObjDups(config, ancestor, 'deps_info', 'extra_package_names') + RemoveObjDups(config, ancestor, 'deps_info', 'jni_all_source') + RemoveObjDups(config, ancestor, 'extra_android_manifests') if is_java_target: jar_to_target = {} @@ -2064,6 +2182,8 @@ _AddJarMapping(jar_to_target, all_deps) if base_module_build_config: _AddJarMapping(jar_to_target, [base_module_build_config['deps_info']]) + if parent_module_build_config is not base_module_build_config: + _AddJarMapping(jar_to_target, [parent_module_build_config['deps_info']]) if options.tested_apk_config: _AddJarMapping(jar_to_target, [tested_apk_config]) for jar, target in zip(tested_apk_config['javac_full_classpath'], @@ -2071,7 +2191,9 @@ jar_to_target[jar] = target # Used by bytecode_processor to give better error message when missing - # deps are found. + # deps are found. Both javac_full_classpath_targets and javac_full_classpath + # must be in identical orders, as they get passed as separate arrays and + # then paired up based on index. config['deps_info']['javac_full_classpath_targets'] = [ jar_to_target[x] for x in deps_info['javac_full_classpath'] ] @@ -2079,8 +2201,14 @@ build_utils.WriteJson(config, options.build_config, only_if_changed=True) if options.depfile: - build_utils.WriteDepfile(options.depfile, options.build_config, - sorted(set(all_inputs))) + action_helpers.write_depfile(options.depfile, options.build_config, + sorted(set(all_inputs))) + + if options.store_deps_for_debugging_to: + GetDepConfig(options.build_config) # Add it to cache. + _CopyBuildConfigsForDebugging(options.store_deps_for_debugging_to) + + return 0 if __name__ == '__main__':
diff --git a/build/android/gyp/write_build_config.pydeps b/build/android/gyp/write_build_config.pydeps index b1276bc..fa7209c 100644 --- a/build/android/gyp/write_build_config.pydeps +++ b/build/android/gyp/write_build_config.pydeps
@@ -1,9 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/write_build_config.pydeps build/android/gyp/write_build_config.py ../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/asyncfilters.py -../../../third_party/jinja2/asyncsupport.py +../../../third_party/jinja2/_identifier.py +../../../third_party/jinja2/async_utils.py ../../../third_party/jinja2/bccache.py ../../../third_party/jinja2/compiler.py ../../../third_party/jinja2/defaults.py @@ -23,6 +22,7 @@ ../../../third_party/markupsafe/__init__.py ../../../third_party/markupsafe/_compat.py ../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py util/__init__.py util/build_utils.py
diff --git a/build/android/gyp/write_native_libraries_java.py b/build/android/gyp/write_native_libraries_java.py index 322b8b2..fb4d2ad 100755 --- a/build/android/gyp/write_native_libraries_java.py +++ b/build/android/gyp/write_native_libraries_java.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,6 +12,8 @@ import zipfile from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers _NATIVE_LIBRARIES_TEMPLATE = """\ @@ -29,8 +31,6 @@ // Set to true to enable the use of the Chromium Linker. public static {MAYBE_FINAL}boolean sUseLinker{USE_LINKER}; - public static {MAYBE_FINAL}boolean sUseLibraryInZipFile{USE_LIBRARY_IN_ZIP_FILE}; - public static {MAYBE_FINAL}boolean sUseModernLinker{USE_MODERN_LINKER}; // This is the list of native libraries to be loaded (in the correct order) // by LibraryLoader.java. @@ -52,19 +52,13 @@ def main(): parser = argparse.ArgumentParser() - build_utils.AddDepfileOption(parser) + action_helpers.add_depfile_arg(parser) parser.add_argument('--final', action='store_true', help='Use final fields.') parser.add_argument( '--enable-chromium-linker', action='store_true', help='Enable Chromium linker.') parser.add_argument( - '--load-library-from-apk', - action='store_true', - help='Load libaries from APK without uncompressing.') - parser.add_argument( - '--use-modern-linker', action='store_true', help='To use ModernLinker.') - parser.add_argument( '--native-libraries-list', help='File with list of native libraries.') parser.add_argument( '--cpu-family', @@ -85,45 +79,45 @@ options = parser.parse_args(build_utils.ExpandFileArgs(sys.argv[1:])) - assert (options.enable_chromium_linker or not options.load_library_from_apk) - - native_libraries_list = [] + native_libraries = [] if options.main_component_library: - native_libraries_list.append( - _FormatLibraryName(options.main_component_library)) + native_libraries.append(options.main_component_library) elif options.native_libraries_list: with open(options.native_libraries_list) as f: - for path in f: - path = path.strip() - native_libraries_list.append(_FormatLibraryName(path)) + native_libraries.extend(l.strip() for l in f) + + if options.enable_chromium_linker and len(native_libraries) > 1: + sys.stderr.write( + 'Multiple libraries not supported when using chromium linker. Found:\n') + sys.stderr.write('\n'.join(native_libraries)) + sys.stderr.write('\n') + sys.exit(1) def bool_str(value): if value: return ' = true' - elif options.final: + if options.final: return ' = false' return '' format_dict = { 'MAYBE_FINAL': 'final ' if options.final else '', 'USE_LINKER': bool_str(options.enable_chromium_linker), - 'USE_LIBRARY_IN_ZIP_FILE': bool_str(options.load_library_from_apk), - 'USE_MODERN_LINKER': bool_str(options.use_modern_linker), - 'LIBRARIES': ','.join(native_libraries_list), + 'LIBRARIES': ','.join(_FormatLibraryName(n) for n in native_libraries), 'CPU_FAMILY': options.cpu_family, } - with build_utils.AtomicOutput(options.output) as f: + with action_helpers.atomic_output(options.output) as f: with zipfile.ZipFile(f.name, 'w') as srcjar_file: - build_utils.AddToZipHermetic( + zip_helpers.add_to_zip_hermetic( zip_file=srcjar_file, zip_path='org/chromium/build/NativeLibraries.java', data=_NATIVE_LIBRARIES_TEMPLATE.format(**format_dict)) if options.depfile: assert options.native_libraries_list - build_utils.WriteDepfile(options.depfile, - options.output, - inputs=[options.native_libraries_list]) + action_helpers.write_depfile(options.depfile, + options.output, + inputs=[options.native_libraries_list]) if __name__ == '__main__':
diff --git a/build/android/gyp/write_native_libraries_java.pydeps b/build/android/gyp/write_native_libraries_java.pydeps index f5176ef..c47e165 100644 --- a/build/android/gyp/write_native_libraries_java.pydeps +++ b/build/android/gyp/write_native_libraries_java.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/write_native_libraries_java.pydeps build/android/gyp/write_native_libraries_java.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py util/__init__.py util/build_utils.py write_native_libraries_java.py
diff --git a/build/android/gyp/zip.py b/build/android/gyp/zip.py index 6b40540..f4b4acf 100755 --- a/build/android/gyp/zip.py +++ b/build/android/gyp/zip.py
@@ -1,16 +1,19 @@ #!/usr/bin/env python3 # -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Archives a set of files.""" import argparse +import json import os import sys import zipfile from util import build_utils +import action_helpers # build_utils adds //build to sys.path. +import zip_helpers def main(args): @@ -33,39 +36,47 @@ action='store_false', dest='compress', help='Do not compress entries') - build_utils.AddDepfileOption(parser) + parser.add_argument('--comment-json', + action='append', + metavar='KEY=VALUE', + type=lambda x: x.split('=', 1), + help='Entry to store in JSON-encoded archive comment.') + action_helpers.add_depfile_arg(parser) options = parser.parse_args(args) - with build_utils.AtomicOutput(options.output) as f: + with action_helpers.atomic_output(options.output) as f: with zipfile.ZipFile(f.name, 'w') as out_zip: depfile_deps = None if options.input_files: - files = build_utils.ParseGnList(options.input_files) - build_utils.DoZip( - files, - out_zip, - base_dir=options.input_files_base_dir, - compress_fn=lambda _: options.compress) + files = action_helpers.parse_gn_list(options.input_files) + zip_helpers.add_files_to_zip(files, + out_zip, + base_dir=options.input_files_base_dir, + compress=options.compress) if options.input_zips: - files = build_utils.ParseGnList(options.input_zips) + files = action_helpers.parse_gn_list(options.input_zips) depfile_deps = files path_transform = None if options.input_zips_excluded_globs: - globs = build_utils.ParseGnList(options.input_zips_excluded_globs) + globs = action_helpers.parse_gn_list( + options.input_zips_excluded_globs) path_transform = ( lambda p: None if build_utils.MatchesGlob(p, globs) else p) - build_utils.MergeZips( - out_zip, - files, - path_transform=path_transform, - compress=options.compress) + zip_helpers.merge_zips(out_zip, + files, + path_transform=path_transform, + compress=options.compress) + + if options.comment_json: + out_zip.comment = json.dumps(dict(options.comment_json), + sort_keys=True).encode('utf-8') # Depfile used only by dist_jar(). if options.depfile: - build_utils.WriteDepfile(options.depfile, - options.output, - inputs=depfile_deps) + action_helpers.write_depfile(options.depfile, + options.output, + inputs=depfile_deps) if __name__ == '__main__':
diff --git a/build/android/gyp/zip.pydeps b/build/android/gyp/zip.pydeps index 36affd1..973fe43 100644 --- a/build/android/gyp/zip.pydeps +++ b/build/android/gyp/zip.pydeps
@@ -1,6 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/gyp --output build/android/gyp/zip.pydeps build/android/gyp/zip.py +../../action_helpers.py ../../gn_helpers.py +../../zip_helpers.py util/__init__.py util/build_utils.py zip.py
diff --git a/build/android/host_heartbeat.py b/build/android/host_heartbeat.py index 4e11c5c..f22c2d7 100755 --- a/build/android/host_heartbeat.py +++ b/build/android/host_heartbeat.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/incremental_install/BUILD.gn b/build/android/incremental_install/BUILD.gn index 8d26e96..e2134dd 100644 --- a/build/android/incremental_install/BUILD.gn +++ b/build/android/incremental_install/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,6 +13,7 @@ "java/org/chromium/incrementalinstall/Reflect.java", "java/org/chromium/incrementalinstall/SecondInstrumentation.java", ] + deps = [ "third_party/AndroidHiddenApiBypass:hidden_api_bypass_java" ] jacoco_never_instrument = true no_build_hooks = true }
diff --git a/build/android/incremental_install/__init__.py b/build/android/incremental_install/__init__.py index 50b23df..a43e6af 100644 --- a/build/android/incremental_install/__init__.py +++ b/build/android/incremental_install/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/incremental_install/generate_android_manifest.py b/build/android/incremental_install/generate_android_manifest.py index e069dab..ffa26c2 100755 --- a/build/android/incremental_install/generate_android_manifest.py +++ b/build/android/incremental_install/generate_android_manifest.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Creates an AndroidManifest.xml for an incremental APK. @@ -11,16 +11,13 @@ import argparse import os -import subprocess import sys -import tempfile -import zipfile from xml.etree import ElementTree sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, 'gyp')) from util import build_utils from util import manifest_utils -from util import resource_utils +import action_helpers # build_utils adds //build to sys.path. _INCREMENTAL_APP_NAME = 'org.chromium.incrementalinstall.BootstrapApplication' _META_DATA_APP_NAME = 'incremental-install-real-app' @@ -42,23 +39,18 @@ def _ParseArgs(args): parser = argparse.ArgumentParser() - parser.add_argument( - '--src-manifest', required=True, help='The main manifest of the app') + parser.add_argument('--src-manifest', + required=True, + help='The main manifest of the app.') + parser.add_argument('--dst-manifest', + required=True, + help='The output modified manifest.') parser.add_argument('--disable-isolated-processes', help='Changes all android:isolatedProcess to false. ' 'This is required on Android M+', action='store_true') - parser.add_argument( - '--out-apk', required=True, help='Path to output .ap_ file') - parser.add_argument( - '--in-apk', required=True, help='Path to non-incremental .ap_ file') - parser.add_argument( - '--aapt2-path', required=True, help='Path to the Android aapt tool') - parser.add_argument( - '--android-sdk-jars', help='GN List of resource apks to include.') ret = parser.parse_args(build_utils.ExpandFileArgs(args)) - ret.android_sdk_jars = build_utils.ParseGnList(ret.android_sdk_jars) return ret @@ -68,13 +60,8 @@ meta_data_node.set(_AddNamespace('value'), value) -def _ProcessManifest(path, arsc_package_name, disable_isolated_processes): - doc, manifest_node, app_node = manifest_utils.ParseManifest(path) - - # Ensure the manifest package matches that of the apk's arsc package - # So that resource references resolve correctly. The actual manifest - # package name is set via --rename-manifest-package. - manifest_node.set('package', arsc_package_name) +def _ProcessManifest(path, disable_isolated_processes): + doc, _, app_node = manifest_utils.ParseManifest(path) # Pylint for some reason things app_node is an int. # pylint: disable=no-member @@ -100,40 +87,19 @@ ret = ret.replace(b'extractNativeLibs="false"', b'extractNativeLibs="true"') if disable_isolated_processes: ret = ret.replace(b'isolatedProcess="true"', b'isolatedProcess="false"') + # externalService only matters for isolatedProcess="true". See: + # https://developer.android.com/reference/android/R.attr#externalService + ret = ret.replace(b'externalService="true"', b'externalService="false"') return ret def main(raw_args): options = _ParseArgs(raw_args) - arsc_package, _ = resource_utils.ExtractArscPackage(options.aapt2_path, - options.in_apk) - # Extract version from the compiled manifest since it might have been set - # via aapt, and not exist in the manifest's text form. - version_code, version_name, manifest_package = ( - resource_utils.ExtractBinaryManifestValues(options.aapt2_path, - options.in_apk)) - - new_manifest_data = _ProcessManifest(options.src_manifest, arsc_package, + new_manifest_data = _ProcessManifest(options.src_manifest, options.disable_isolated_processes) - with tempfile.NamedTemporaryFile() as tmp_manifest, \ - tempfile.NamedTemporaryFile() as tmp_apk: - tmp_manifest.write(new_manifest_data) - tmp_manifest.flush() - cmd = [ - options.aapt2_path, 'link', '-o', tmp_apk.name, '--manifest', - tmp_manifest.name, '-I', options.in_apk, '--replace-version', - '--version-code', version_code, '--version-name', version_name, - '--rename-manifest-package', manifest_package, '--debug-mode' - ] - for j in options.android_sdk_jars: - cmd += ['-I', j] - subprocess.check_call(cmd) - with zipfile.ZipFile(options.out_apk, 'w') as z: - path_transform = lambda p: None if p != 'AndroidManifest.xml' else p - build_utils.MergeZips(z, [tmp_apk.name], path_transform=path_transform) - path_transform = lambda p: None if p == 'AndroidManifest.xml' else p - build_utils.MergeZips(z, [options.in_apk], path_transform=path_transform) + with action_helpers.atomic_output(options.dst_manifest) as out_manifest: + out_manifest.write(new_manifest_data) if __name__ == '__main__':
diff --git a/build/android/incremental_install/generate_android_manifest.pydeps b/build/android/incremental_install/generate_android_manifest.pydeps index 568ea1e..68c832b 100644 --- a/build/android/incremental_install/generate_android_manifest.pydeps +++ b/build/android/incremental_install/generate_android_manifest.pydeps
@@ -1,29 +1,8 @@ # Generated by running: # build/print_python_deps.py --root build/android/incremental_install --output build/android/incremental_install/generate_android_manifest.pydeps build/android/incremental_install/generate_android_manifest.py -../../../third_party/jinja2/__init__.py -../../../third_party/jinja2/_compat.py -../../../third_party/jinja2/bccache.py -../../../third_party/jinja2/compiler.py -../../../third_party/jinja2/defaults.py -../../../third_party/jinja2/environment.py -../../../third_party/jinja2/exceptions.py -../../../third_party/jinja2/filters.py -../../../third_party/jinja2/idtracking.py -../../../third_party/jinja2/lexer.py -../../../third_party/jinja2/loaders.py -../../../third_party/jinja2/nodes.py -../../../third_party/jinja2/optimizer.py -../../../third_party/jinja2/parser.py -../../../third_party/jinja2/runtime.py -../../../third_party/jinja2/tests.py -../../../third_party/jinja2/utils.py -../../../third_party/jinja2/visitor.py -../../../third_party/markupsafe/__init__.py -../../../third_party/markupsafe/_compat.py -../../../third_party/markupsafe/_native.py +../../action_helpers.py ../../gn_helpers.py ../gyp/util/__init__.py ../gyp/util/build_utils.py ../gyp/util/manifest_utils.py -../gyp/util/resource_utils.py generate_android_manifest.py
diff --git a/build/android/incremental_install/installer.py b/build/android/incremental_install/installer.py index 9625822..68e28b4 100755 --- a/build/android/incremental_install/installer.py +++ b/build/android/incremental_install/installer.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,6 +10,7 @@ import collections import functools import glob +import hashlib import json import logging import os @@ -36,6 +37,7 @@ _R8_PATH = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'r8', 'lib', 'r8.jar') +_SHARD_JSON_FILENAME = 'shards.json' def _DeviceCachePath(device): @@ -60,17 +62,34 @@ return '/data/local/tmp/incremental-app-%s' % package -def _IsStale(src_paths, dest): +def _IsStale(src_paths, old_src_paths, dest_path): """Returns if |dest| is older than any of |src_paths|, or missing.""" - if not os.path.exists(dest): + if not os.path.exists(dest_path): return True - dest_time = os.path.getmtime(dest) + # Always mark as stale if any paths were added or removed. + if set(src_paths) != set(old_src_paths): + return True + dest_time = os.path.getmtime(dest_path) for path in src_paths: if os.path.getmtime(path) > dest_time: return True return False +def _LoadPrevShards(dex_staging_dir): + shards_json_path = os.path.join(dex_staging_dir, _SHARD_JSON_FILENAME) + if not os.path.exists(shards_json_path): + return {} + with open(shards_json_path) as f: + return json.load(f) + + +def _SaveNewShards(shards, dex_staging_dir): + shards_json_path = os.path.join(dex_staging_dir, _SHARD_JSON_FILENAME) + with open(shards_json_path, 'w') as f: + json.dump(shards, f) + + def _AllocateDexShards(dex_files): """Divides input dex files into buckets.""" # Goals: @@ -91,19 +110,26 @@ os.sep, '.') shards[name].append(src_path) else: - name = 'shard{}.dex.jar'.format(hash(src_path) % NUM_CORE_SHARDS) + # The stdlib hash(string) function is salted differently across python3 + # invocations. Thus we use md5 instead to consistently shard the same + # file to the same shard across runs. + hex_hash = hashlib.md5(src_path.encode('utf-8')).hexdigest() + name = 'shard{}.dex.jar'.format(int(hex_hash, 16) % NUM_CORE_SHARDS) shards[name].append(src_path) logging.info('Sharding %d dex files into %d buckets', len(dex_files), len(shards)) return shards -def _CreateDexFiles(shards, dex_staging_dir, min_api, use_concurrency): +def _CreateDexFiles(shards, prev_shards, dex_staging_dir, min_api, + use_concurrency): """Creates dex files within |dex_staging_dir| defined by |shards|.""" tasks = [] - for name, src_paths in shards.iteritems(): + for name, src_paths in shards.items(): dest_path = os.path.join(dex_staging_dir, name) - if _IsStale(src_paths, dest_path): + if _IsStale(src_paths=src_paths, + old_src_paths=prev_shards.get(name, []), + dest_path=dest_path): tasks.append( functools.partial(dex.MergeDexForIncrementalInstall, _R8_PATH, src_paths, dest_path, min_api)) @@ -146,7 +172,7 @@ permissions: A list of the permissions to grant, or None to grant all non-denylisted permissions in the manifest. """ - if isinstance(install_json, basestring): + if isinstance(install_json, str): with open(install_json) as f: install_dict = json.load(f) else: @@ -212,10 +238,14 @@ def do_merge_dex(): merge_dex_timer.Start() + prev_shards = _LoadPrevShards(dex_staging_dir) shards = _AllocateDexShards(dex_files) build_utils.MakeDirectory(dex_staging_dir) - _CreateDexFiles(shards, dex_staging_dir, apk.GetMinSdkVersion(), - use_concurrency) + _CreateDexFiles(shards, prev_shards, dex_staging_dir, + apk.GetMinSdkVersion(), use_concurrency) + # New shard information must be saved after _CreateDexFiles since + # _CreateDexFiles removes all non-dex files from the staging dir. + _SaveNewShards(shards, dex_staging_dir) merge_dex_timer.Stop(log=False) def do_push_dex(): @@ -227,33 +257,6 @@ _Execute(use_concurrency, do_push_native, do_merge_dex) do_push_dex() - def check_device_configured(): - target_sdk_version = int(apk.GetTargetSdkVersion()) - # Beta Q builds apply allowlist to targetSdk=28 as well. - if target_sdk_version >= 28 and device.build_version_sdk >= 28: - # In P, there are two settings: - # * hidden_api_policy_p_apps - # * hidden_api_policy_pre_p_apps - # In Q, there is just one: - # * hidden_api_policy - if device.build_version_sdk == 28: - setting_name = 'hidden_api_policy_p_apps' - else: - setting_name = 'hidden_api_policy' - apis_allowed = ''.join( - device.RunShellCommand(['settings', 'get', 'global', setting_name], - check_return=True)) - if apis_allowed.strip() not in '01': - msg = """\ -Cannot use incremental installs on Android P+ without first enabling access to -non-SDK interfaces (https://developer.android.com/preview/non-sdk-q). - -To enable access: - adb -s {0} shell settings put global {1} 0 -To restore back to default: - adb -s {0} shell settings delete global {1}""" - raise Exception(msg.format(device.serial, setting_name)) - cache_path = _DeviceCachePath(device) def restore_cache(): if not enable_device_cache: @@ -294,8 +297,7 @@ # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't # been designed for multi-threading. Enabling only because this is a # developer-only tool. - setup_timer = _Execute(use_concurrency, create_lock_files, restore_cache, - check_device_configured) + setup_timer = _Execute(use_concurrency, create_lock_files, restore_cache) _Execute(use_concurrency, do_install, do_push_files)
diff --git a/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapApplication.java b/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapApplication.java index f7003f2..f882970 100644 --- a/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapApplication.java +++ b/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapApplication.java
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapInstrumentation.java b/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapInstrumentation.java index f197406..f1f507a 100644 --- a/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapInstrumentation.java +++ b/build/android/incremental_install/java/org/chromium/incrementalinstall/BootstrapInstrumentation.java
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/incremental_install/java/org/chromium/incrementalinstall/ClassLoaderPatcher.java b/build/android/incremental_install/java/org/chromium/incrementalinstall/ClassLoaderPatcher.java index b6d7522..53e926e 100644 --- a/build/android/incremental_install/java/org/chromium/incrementalinstall/ClassLoaderPatcher.java +++ b/build/android/incremental_install/java/org/chromium/incrementalinstall/ClassLoaderPatcher.java
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -153,34 +153,30 @@ @SuppressLint("SetWorldReadable") private void safeCopyAllFiles(File srcDir, File dstDir) throws IOException { + if (!mIsPrimaryProcess) { + // TODO: Work around this issue by using APK splits to install each dex / lib. + throw new RuntimeException("Incremental install does not work on Android M+ " + + "with isolated processes. Build system should have removed this. " + + "Please file a bug."); + } + // The library copying is not necessary on older devices, but we do it anyways to // simplify things (it's fast compared to dexing). // https://code.google.com/p/android/issues/detail?id=79480 + ensureAppFilesSubDirExists(); File lockFile = new File(mAppFilesSubDir, dstDir.getName() + ".lock"); - if (mIsPrimaryProcess) { - ensureAppFilesSubDirExists(); - LockFile lock = LockFile.acquireRuntimeLock(lockFile); - if (lock == null) { - LockFile.waitForRuntimeLock(lockFile, 10 * 1000); - } else { - try { - dstDir.mkdir(); - dstDir.setReadable(true, false); - dstDir.setExecutable(true, false); - copyChangedFiles(srcDir, dstDir); - } finally { - lock.release(); - } - } - } else { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - // TODO: Work around this issue by using APK splits to install each dex / lib. - throw new RuntimeException("Incremental install does not work on Android M+ " - + "with isolated processes. Build system should have removed this. " - + "Please file a bug."); - } - // Other processes: Waits for primary process to finish copying. + LockFile lock = LockFile.acquireRuntimeLock(lockFile); + if (lock == null) { LockFile.waitForRuntimeLock(lockFile, 10 * 1000); + } else { + try { + dstDir.mkdir(); + dstDir.setReadable(true, false); + dstDir.setExecutable(true, false); + copyChangedFiles(srcDir, dstDir); + } finally { + lock.release(); + } } } @@ -291,14 +287,9 @@ File emptyDir = new File(""); for (int i = 0; i < files.length; ++i) { File file = files[i]; - Object dexFile; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - // loadDexFile requires that ret contain all previously added elements. - dexFile = Reflect.invokeMethod(clazz, "loadDexFile", file, optimizedDirectory, - mClassLoader, ret); - } else { - dexFile = Reflect.invokeMethod(clazz, "loadDexFile", file, optimizedDirectory); - } + // loadDexFile requires that ret contain all previously added elements. + Object dexFile = Reflect.invokeMethod( + clazz, "loadDexFile", file, optimizedDirectory, mClassLoader, ret); Object dexElement; if (Build.VERSION.SDK_INT >= 26) { dexElement = Reflect.newInstance(entryClazz, dexFile, file);
diff --git a/build/android/incremental_install/java/org/chromium/incrementalinstall/LockFile.java b/build/android/incremental_install/java/org/chromium/incrementalinstall/LockFile.java index 19d1f76..08d4c66 100644 --- a/build/android/incremental_install/java/org/chromium/incrementalinstall/LockFile.java +++ b/build/android/incremental_install/java/org/chromium/incrementalinstall/LockFile.java
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/incremental_install/java/org/chromium/incrementalinstall/Reflect.java b/build/android/incremental_install/java/org/chromium/incrementalinstall/Reflect.java index c64dc1e..6ce74eb 100644 --- a/build/android/incremental_install/java/org/chromium/incrementalinstall/Reflect.java +++ b/build/android/incremental_install/java/org/chromium/incrementalinstall/Reflect.java
@@ -1,14 +1,19 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. package org.chromium.incrementalinstall; +import android.os.Build; + +import org.lsposed.hiddenapibypass.HiddenApiBypass; + import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.List; /** * Reflection helper methods. @@ -79,12 +84,22 @@ private static Field findField(Object instance, String name) throws NoSuchFieldException { boolean isStatic = instance instanceof Class; - Class<?> clazz = isStatic ? (Class<?>) instance : instance.getClass(); + Class<?> clazz = isStatic ? (Class<?>) instance : instance.getClass(); for (; clazz != null; clazz = clazz.getSuperclass()) { - try { - return clazz.getDeclaredField(name); - } catch (NoSuchFieldException e) { - // Need to look in the super class. + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { + try { + return clazz.getDeclaredField(name); + } catch (NoSuchFieldException e) { + // Need to look in the super class. + } + } else { + List<Field> fields = isStatic ? HiddenApiBypass.getStaticFields(clazz) + : HiddenApiBypass.getInstanceFields(clazz); + for (Field field : fields) { + if (field.getName().equals(name)) { + return field; + } + } } } throw new NoSuchFieldException("Field " + name + " not found in " + instance.getClass());
diff --git a/build/android/incremental_install/java/org/chromium/incrementalinstall/SecondInstrumentation.java b/build/android/incremental_install/java/org/chromium/incrementalinstall/SecondInstrumentation.java index 3e0df05..ecf4870 100644 --- a/build/android/incremental_install/java/org/chromium/incrementalinstall/SecondInstrumentation.java +++ b/build/android/incremental_install/java/org/chromium/incrementalinstall/SecondInstrumentation.java
@@ -1,4 +1,4 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. +// Copyright 2017 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/BUILD.gn b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/BUILD.gn new file mode 100644 index 0000000..86e1466 --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/BUILD.gn
@@ -0,0 +1,29 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") + +# Approved by chrome-security@ only for use by incremental install. +visibility = [ + ":*", + "//build/android/incremental_install:*", +] + +android_library("stub_java") { + sources = [ "stub/src/main/java/dalvik/system/VMRuntime.java" ] + jar_excluded_patterns = [ "*" ] +} + +android_library("hidden_api_bypass_java") { + sources = [ + "library/src/main/java/org/lsposed/hiddenapibypass/Helper.java", + "library/src/main/java/org/lsposed/hiddenapibypass/HiddenApiBypass.java", + "local_modifications/org/lsposed/hiddenapibypass/library/BuildConfig.java", + ] + deps = [ + ":stub_java", + "//third_party/androidx:androidx_annotation_annotation_jvm_java", + ] + jacoco_never_instrument = true +}
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/LICENSE b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/LICENSE
@@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/README.chromium b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/README.chromium new file mode 100644 index 0000000..b1fdc9c --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/README.chromium
@@ -0,0 +1,16 @@ +Name: AndroidHiddenApiBypass +URL: https://github.com/LSPosed/AndroidHiddenApiBypass +Version: b16cc3934a27e55e51f00f5504c7f49e7c8cfab7 +License: Apache 2.0 +License File: NOT_SHIPPED +Security Critical: no + +Description: +AndroidHiddenApiBypass enables reflection on APIs that are meant to be guarded +by Android's API Blocklist. + +Local Modifications: +* Removed files related to Gradle. +* Added local_modifications/.../BuildConfig.java to replace what Gradle would + have generated. +* Added BUILD.gn
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/README.md b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/README.md new file mode 100644 index 0000000..c7e0681 --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/README.md
@@ -0,0 +1,84 @@ +# AndroidHiddenApiBypass + +[](https://github.com/LSPosed/AndroidHiddenApiBypass/actions/workflows/android.yml) + +Bypass restrictions on non-SDK interfaces. + +## Why AndroidHiddenApiBypass? + +- Pure Java: no native code used. +- Reliable: does not rely on specific behaviors, so it will not be blocked like meta-reflection or `dexfile`. +- Stable: `unsafe`, art structs and `setHiddenApiExemptions` are stable APIs. + +[How it works (Chinese)](https://lovesykun.cn/archives/android-hidden-api-bypass.html) + +## Integration + +Gradle: + +```gradle +repositories { + mavenCentral() +} +dependencies { + implementation 'org.lsposed.hiddenapibypass:hiddenapibypass:4.3' +} +``` + +## Usage + +1. Invoke a restricted method: + ```java + HiddenApiBypass.invoke(ApplicationInfo.class, new ApplicationInfo(), "usesNonSdkApi"/*, args*/) + ``` +1. Invoke restricted constructor: + ```java + Object instance = HiddenApiBypass.newInstance(Class.forName("android.app.IActivityManager$Default")/*, args*/); + ``` +1. Get all methods including restricted ones from a class: + ```java + var allMethods = HiddenApiBypass.getDeclaredMethods(ApplicationInfo.class); + ((Method).stream(allMethods).filter(e -> e.getName().equals("usesNonSdkApi")).findFirst().get()).invoke(new ApplicationInfo()); + ``` +1. Get all non-static fields including restricted ones from a class: + ```java + var allInstanceFields = HiddenApiBypass.getInstanceFields(ApplicationInfo.class); + ((Method).stream(allInstanceFields).filter(e -> e.getName().equals("longVersionCode")).findFirst().get()).get(new ApplicationInfo()); + ``` +1. Get all static fields including restricted ones from a class: + ```java + var allStaticFields = HiddenApiBypass.getStaticFields(ApplicationInfo.class); + ((Method).stream(allInstanceFields).filter(e -> e.getName().equals("HIDDEN_API_ENFORCEMENT_DEFAULT")).findFirst().get()).get(null); + ``` +1. Get specific class method or class constructor + ```java + var ctor = HiddenApiBypass.getDeclaredConstructor(ClipDrawable.class /*, args */); + var method = HiddenApiBypass.getDeclaredMethod(ApplicationInfo.class, "getHiddenApiEnforcementPolicy" /*, args */); + ``` +1. Add a class to exemption list: + ```java + HiddenApiBypass.addHiddenApiExemptions( + "Landroid/content/pm/ApplicationInfo;", // one specific class + "Ldalvik/system" // all classes in packages dalvik.system + "Lx" // all classes whose full name is started with x + ); + ``` + if you are going to add all classes to exemption list, just leave an empty prefix: + ```java + HiddenApiBypass.addHiddenApiExemptions(""); + ``` +## License + + Copyright 2021 LSPosed + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/library/src/main/java/org/lsposed/hiddenapibypass/Helper.java b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/library/src/main/java/org/lsposed/hiddenapibypass/Helper.java new file mode 100644 index 0000000..07d130d --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/library/src/main/java/org/lsposed/hiddenapibypass/Helper.java
@@ -0,0 +1,108 @@ +/* + * Copyright (C) 2021 LSPosed + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lsposed.hiddenapibypass; + +import java.lang.invoke.MethodHandleInfo; +import java.lang.invoke.MethodType; +import java.lang.reflect.Member; + +@SuppressWarnings("unused") +public class Helper { + static public class MethodHandle { + private final MethodType type = null; + private MethodType nominalType; + private MethodHandle cachedSpreadInvoker; + protected final int handleKind = 0; + + // The ArtMethod* or ArtField* associated with this method handle (used by the runtime). + protected final long artFieldOrMethod = 0; + } + + static final public class MethodHandleImpl extends MethodHandle { + private final MethodHandleInfo info = null; + } + + static final public class HandleInfo { + private final Member member = null; + private final MethodHandle handle = null; + } + + static final public class Class { + private transient ClassLoader classLoader; + private transient java.lang.Class<?> componentType; + private transient Object dexCache; + private transient Object extData; + private transient Object[] ifTable; + private transient String name; + private transient java.lang.Class<?> superClass; + private transient Object vtable; + private transient long iFields; + private transient long methods; + private transient long sFields; + private transient int accessFlags; + private transient int classFlags; + private transient int classSize; + private transient int clinitThreadId; + private transient int dexClassDefIndex; + private transient volatile int dexTypeIndex; + private transient int numReferenceInstanceFields; + private transient int numReferenceStaticFields; + private transient int objectSize; + private transient int objectSizeAllocFastPath; + private transient int primitiveType; + private transient int referenceInstanceOffsets; + private transient int status; + private transient short copiedMethodsOffset; + private transient short virtualMethodsOffset; + } + + static public class AccessibleObject { + private boolean override; + } + + static final public class Executable extends AccessibleObject { + private Class declaringClass; + private Class declaringClassOfOverriddenMethod; + private Object[] parameters; + private long artMethod; + private int accessFlags; + } + + @SuppressWarnings("EmptyMethod") + public static class NeverCall { + private static void a() { + } + + private static void b() { + } + + private static int s; + private static int t; + private int i; + private int j; + } + + public static class InvokeStub { + private static Object invoke(Object... args) { + throw new IllegalStateException("Failed to invoke the method"); + } + + private InvokeStub(Object... args) { + throw new IllegalStateException("Failed to new a instance"); + } + } +}
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/library/src/main/java/org/lsposed/hiddenapibypass/HiddenApiBypass.java b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/library/src/main/java/org/lsposed/hiddenapibypass/HiddenApiBypass.java new file mode 100644 index 0000000..2344acf --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/library/src/main/java/org/lsposed/hiddenapibypass/HiddenApiBypass.java
@@ -0,0 +1,415 @@ +/* + * Copyright (C) 2021 LSPosed + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lsposed.hiddenapibypass; + +import android.os.Build; +import android.util.Log; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; + +import org.lsposed.hiddenapibypass.library.BuildConfig; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandleInfo; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import dalvik.system.VMRuntime; +import sun.misc.Unsafe; + +@RequiresApi(Build.VERSION_CODES.P) +public final class HiddenApiBypass { + private static final String TAG = "HiddenApiBypass"; + private static final Unsafe unsafe; + private static final long methodOffset; + private static final long classOffset; + private static final long artOffset; + private static final long infoOffset; + private static final long methodsOffset; + private static final long iFieldOffset; + private static final long sFieldOffset; + private static final long memberOffset; + private static final long artMethodSize; + private static final long artMethodBias; + private static final long artFieldSize; + private static final long artFieldBias; + private static final Set<String> signaturePrefixes = new HashSet<>(); + + static { + try { + //noinspection JavaReflectionMemberAccess DiscouragedPrivateApi + unsafe = (Unsafe) Unsafe.class.getDeclaredMethod("getUnsafe").invoke(null); + assert unsafe != null; + methodOffset = unsafe.objectFieldOffset(Helper.Executable.class.getDeclaredField("artMethod")); + classOffset = unsafe.objectFieldOffset(Helper.Executable.class.getDeclaredField("declaringClass")); + artOffset = unsafe.objectFieldOffset(Helper.MethodHandle.class.getDeclaredField("artFieldOrMethod")); + infoOffset = unsafe.objectFieldOffset(Helper.MethodHandleImpl.class.getDeclaredField("info")); + methodsOffset = unsafe.objectFieldOffset(Helper.Class.class.getDeclaredField("methods")); + iFieldOffset = unsafe.objectFieldOffset(Helper.Class.class.getDeclaredField("iFields")); + sFieldOffset = unsafe.objectFieldOffset(Helper.Class.class.getDeclaredField("sFields")); + memberOffset = unsafe.objectFieldOffset(Helper.HandleInfo.class.getDeclaredField("member")); + Method mA = Helper.NeverCall.class.getDeclaredMethod("a"); + Method mB = Helper.NeverCall.class.getDeclaredMethod("b"); + mA.setAccessible(true); + mB.setAccessible(true); + MethodHandle mhA = MethodHandles.lookup().unreflect(mA); + MethodHandle mhB = MethodHandles.lookup().unreflect(mB); + long aAddr = unsafe.getLong(mhA, artOffset); + long bAddr = unsafe.getLong(mhB, artOffset); + long aMethods = unsafe.getLong(Helper.NeverCall.class, methodsOffset); + artMethodSize = bAddr - aAddr; + if (BuildConfig.DEBUG) Log.v(TAG, artMethodSize + " " + + Long.toString(aAddr, 16) + ", " + + Long.toString(bAddr, 16) + ", " + + Long.toString(aMethods, 16)); + artMethodBias = aAddr - aMethods - artMethodSize; + Field fI = Helper.NeverCall.class.getDeclaredField("i"); + Field fJ = Helper.NeverCall.class.getDeclaredField("j"); + fI.setAccessible(true); + fJ.setAccessible(true); + MethodHandle mhI = MethodHandles.lookup().unreflectGetter(fI); + MethodHandle mhJ = MethodHandles.lookup().unreflectGetter(fJ); + long iAddr = unsafe.getLong(mhI, artOffset); + long jAddr = unsafe.getLong(mhJ, artOffset); + long iFields = unsafe.getLong(Helper.NeverCall.class, iFieldOffset); + artFieldSize = jAddr - iAddr; + if (BuildConfig.DEBUG) Log.v(TAG, artFieldSize + " " + + Long.toString(iAddr, 16) + ", " + + Long.toString(jAddr, 16) + ", " + + Long.toString(iFields, 16)); + artFieldBias = iAddr - iFields; + } catch (ReflectiveOperationException e) { + Log.e(TAG, "Initialize error", e); + throw new ExceptionInInitializerError(e); + } + } + + @VisibleForTesting + static boolean checkArgsForInvokeMethod(Class<?>[] params, Object[] args) { + if (params.length != args.length) return false; + for (int i = 0; i < params.length; ++i) { + if (params[i].isPrimitive()) { + if (params[i] == int.class && !(args[i] instanceof Integer)) return false; + else if (params[i] == byte.class && !(args[i] instanceof Byte)) return false; + else if (params[i] == char.class && !(args[i] instanceof Character)) return false; + else if (params[i] == boolean.class && !(args[i] instanceof Boolean)) return false; + else if (params[i] == double.class && !(args[i] instanceof Double)) return false; + else if (params[i] == float.class && !(args[i] instanceof Float)) return false; + else if (params[i] == long.class && !(args[i] instanceof Long)) return false; + else if (params[i] == short.class && !(args[i] instanceof Short)) return false; + } else if (args[i] != null && !params[i].isInstance(args[i])) return false; + } + return true; + } + + /** + * create an instance of the given class {@code clazz} calling the restricted constructor with arguments {@code args} + * + * @param clazz the class of the instance to new + * @param initargs arguments to call constructor + * @return the new instance + * @see Constructor#newInstance(Object...) + */ + public static Object newInstance(@NonNull Class<?> clazz, Object... initargs) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + Method stub = Helper.InvokeStub.class.getDeclaredMethod("invoke", Object[].class); + Constructor<?> ctor = Helper.InvokeStub.class.getDeclaredConstructor(Object[].class); + ctor.setAccessible(true); + long methods = unsafe.getLong(clazz, methodsOffset); + if (methods == 0) throw new NoSuchMethodException("Cannot find matching constructor"); + int numMethods = unsafe.getInt(methods); + if (BuildConfig.DEBUG) Log.d(TAG, clazz + " has " + numMethods + " methods"); + for (int i = 0; i < numMethods; i++) { + long method = methods + i * artMethodSize + artMethodBias; + unsafe.putLong(stub, methodOffset, method); + if (BuildConfig.DEBUG) Log.v(TAG, "got " + clazz.getTypeName() + "." + stub.getName() + + "(" + Arrays.stream(stub.getParameterTypes()).map(Type::getTypeName).collect(Collectors.joining()) + ")"); + if ("<init>".equals(stub.getName())) { + unsafe.putLong(ctor, methodOffset, method); + unsafe.putObject(ctor, classOffset, clazz); + Class<?>[] params = ctor.getParameterTypes(); + if (checkArgsForInvokeMethod(params, initargs)) + return ctor.newInstance(initargs); + } + } + throw new NoSuchMethodException("Cannot find matching constructor"); + } + + /** + * invoke a restrict method named {@code methodName} of the given class {@code clazz} with this object {@code thiz} and arguments {@code args} + * + * @param clazz the class call the method on (this parameter is required because this method cannot call inherit method) + * @param thiz this object, which can be {@code null} if the target method is static + * @param methodName the method name + * @param args arguments to call the method with name {@code methodName} + * @return the return value of the method + * @see Method#invoke(Object, Object...) + */ + public static Object invoke(@NonNull Class<?> clazz, @Nullable Object thiz, @NonNull String methodName, Object... args) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + if (thiz != null && !clazz.isInstance(thiz)) { + throw new IllegalArgumentException("this object is not an instance of the given class"); + } + Method stub = Helper.InvokeStub.class.getDeclaredMethod("invoke", Object[].class); + stub.setAccessible(true); + long methods = unsafe.getLong(clazz, methodsOffset); + if (methods == 0) throw new NoSuchMethodException("Cannot find matching method"); + int numMethods = unsafe.getInt(methods); + if (BuildConfig.DEBUG) Log.d(TAG, clazz + " has " + numMethods + " methods"); + for (int i = 0; i < numMethods; i++) { + long method = methods + i * artMethodSize + artMethodBias; + unsafe.putLong(stub, methodOffset, method); + if (BuildConfig.DEBUG) Log.v(TAG, "got " + clazz.getTypeName() + "." + stub.getName() + + "(" + Arrays.stream(stub.getParameterTypes()).map(Type::getTypeName).collect(Collectors.joining()) + ")"); + if (methodName.equals(stub.getName())) { + Class<?>[] params = stub.getParameterTypes(); + if (checkArgsForInvokeMethod(params, args)) + return stub.invoke(thiz, args); + } + } + throw new NoSuchMethodException("Cannot find matching method"); + } + + /** + * get declared methods of given class without hidden api restriction + * + * @param clazz the class to fetch declared methods (including constructors with name `<init>`) + * @return list of declared methods of {@code clazz} + */ + @NonNull + public static List<Executable> getDeclaredMethods(@NonNull Class<?> clazz) { + ArrayList<Executable> list = new ArrayList<>(); + if (clazz.isPrimitive() || clazz.isArray()) return list; + MethodHandle mh; + try { + Method mA = Helper.NeverCall.class.getDeclaredMethod("a"); + mA.setAccessible(true); + mh = MethodHandles.lookup().unreflect(mA); + } catch (NoSuchMethodException | IllegalAccessException e) { + return list; + } + long methods = unsafe.getLong(clazz, methodsOffset); + if (methods == 0) return list; + int numMethods = unsafe.getInt(methods); + if (BuildConfig.DEBUG) Log.d(TAG, clazz + " has " + numMethods + " methods"); + for (int i = 0; i < numMethods; i++) { + long method = methods + i * artMethodSize + artMethodBias; + unsafe.putLong(mh, artOffset, method); + unsafe.putObject(mh, infoOffset, null); + try { + MethodHandles.lookup().revealDirect(mh); + } catch (Throwable ignored) { + } + MethodHandleInfo info = (MethodHandleInfo) unsafe.getObject(mh, infoOffset); + Executable member = (Executable) unsafe.getObject(info, memberOffset); + if (BuildConfig.DEBUG) + Log.v(TAG, "got " + clazz.getTypeName() + "." + member.getName() + + "(" + Arrays.stream(member.getParameterTypes()).map(Type::getTypeName).collect(Collectors.joining()) + ")"); + list.add(member); + } + return list; + } + + /** + * get a restrict method named {@code methodName} of the given class {@code clazz} with argument types {@code parameterTypes} + * + * @param clazz the class where the expected method declares + * @param methodName the expected method's name + * @param parameterTypes argument types of the expected method with name {@code methodName} + * @return the found method + * @throws NoSuchMethodException when no method matches the given parameters + * @see Class#getDeclaredMethod(String, Class[]) + */ + @NonNull + public static Method getDeclaredMethod(@NonNull Class<?> clazz, @NonNull String methodName, @NonNull Class<?>... parameterTypes) throws NoSuchMethodException { + List<Executable> methods = getDeclaredMethods(clazz); + allMethods: + for (Executable method : methods) { + if (!method.getName().equals(methodName)) continue; + if (!(method instanceof Method)) continue; + Class<?>[] expectedTypes = method.getParameterTypes(); + if (expectedTypes.length != parameterTypes.length) continue; + for (int i = 0; i < parameterTypes.length; ++i) { + if (parameterTypes[i] != expectedTypes[i]) continue allMethods; + } + return (Method) method; + } + throw new NoSuchMethodException("Cannot find matching method"); + } + + /** + * get a restrict constructor of the given class {@code clazz} with argument types {@code parameterTypes} + * + * @param clazz the class where the expected constructor declares + * @param parameterTypes argument types of the expected constructor + * @return the found constructor + * @throws NoSuchMethodException when no constructor matches the given parameters + * @see Class#getDeclaredConstructor(Class[]) + */ + @NonNull + public static Constructor<?> getDeclaredConstructor(@NonNull Class<?> clazz, @NonNull Class<?>... parameterTypes) throws NoSuchMethodException { + List<Executable> methods = getDeclaredMethods(clazz); + allMethods: + for (Executable method : methods) { + if (!(method instanceof Constructor)) continue; + Class<?>[] expectedTypes = method.getParameterTypes(); + if (expectedTypes.length != parameterTypes.length) continue; + for (int i = 0; i < parameterTypes.length; ++i) { + if (parameterTypes[i] != expectedTypes[i]) continue allMethods; + } + return (Constructor<?>) method; + } + throw new NoSuchMethodException("Cannot find matching constructor"); + } + + + /** + * get declared non-static fields of given class without hidden api restriction + * + * @param clazz the class to fetch declared methods + * @return list of declared non-static fields of {@code clazz} + */ + @NonNull + public static List<Field> getInstanceFields(@NonNull Class<?> clazz) { + ArrayList<Field> list = new ArrayList<>(); + if (clazz.isPrimitive() || clazz.isArray()) return list; + MethodHandle mh; + try { + Field fI = Helper.NeverCall.class.getDeclaredField("i"); + fI.setAccessible(true); + mh = MethodHandles.lookup().unreflectGetter(fI); + } catch (IllegalAccessException | NoSuchFieldException e) { + return list; + } + long fields = unsafe.getLong(clazz, iFieldOffset); + if (fields == 0) return list; + int numFields = unsafe.getInt(fields); + if (BuildConfig.DEBUG) Log.d(TAG, clazz + " has " + numFields + " instance fields"); + for (int i = 0; i < numFields; i++) { + long field = fields + i * artFieldSize + artFieldBias; + unsafe.putLong(mh, artOffset, field); + unsafe.putObject(mh, infoOffset, null); + try { + MethodHandles.lookup().revealDirect(mh); + } catch (Throwable ignored) { + } + MethodHandleInfo info = (MethodHandleInfo) unsafe.getObject(mh, infoOffset); + Field member = (Field) unsafe.getObject(info, memberOffset); + if (BuildConfig.DEBUG) + Log.v(TAG, "got " + member.getType() + " " + clazz.getTypeName() + "." + member.getName()); + list.add(member); + } + return list; + } + + /** + * get declared static fields of given class without hidden api restriction + * + * @param clazz the class to fetch declared methods + * @return list of declared static fields of {@code clazz} + */ + @NonNull + public static List<Field> getStaticFields(@NonNull Class<?> clazz) { + ArrayList<Field> list = new ArrayList<>(); + if (clazz.isPrimitive() || clazz.isArray()) return list; + MethodHandle mh; + try { + Field fS = Helper.NeverCall.class.getDeclaredField("s"); + fS.setAccessible(true); + mh = MethodHandles.lookup().unreflectGetter(fS); + } catch (IllegalAccessException | NoSuchFieldException e) { + return list; + } + long fields = unsafe.getLong(clazz, sFieldOffset); + if (fields == 0) return list; + int numFields = unsafe.getInt(fields); + if (BuildConfig.DEBUG) Log.d(TAG, clazz + " has " + numFields + " static fields"); + for (int i = 0; i < numFields; i++) { + long field = fields + i * artFieldSize + artFieldBias; + unsafe.putLong(mh, artOffset, field); + unsafe.putObject(mh, infoOffset, null); + try { + MethodHandles.lookup().revealDirect(mh); + } catch (Throwable ignored) { + } + MethodHandleInfo info = (MethodHandleInfo) unsafe.getObject(mh, infoOffset); + Field member = (Field) unsafe.getObject(info, memberOffset); + if (BuildConfig.DEBUG) + Log.v(TAG, "got " + member.getType() + " " + clazz.getTypeName() + "." + member.getName()); + list.add(member); + } + return list; + } + + /** + * Sets the list of exemptions from hidden API access enforcement. + * + * @param signaturePrefixes A list of class signature prefixes. Each item in the list is a prefix match on the type + * signature of a blacklisted API. All matching APIs are treated as if they were on + * the whitelist: access permitted, and no logging.. + * @return whether the operation is successful + */ + public static boolean setHiddenApiExemptions(@NonNull String... signaturePrefixes) { + try { + Object runtime = invoke(VMRuntime.class, null, "getRuntime"); + invoke(VMRuntime.class, runtime, "setHiddenApiExemptions", (Object) signaturePrefixes); + return true; + } catch (Throwable e) { + Log.w(TAG, "setHiddenApiExemptions", e); + return false; + } + } + + /** + * Adds the list of exemptions from hidden API access enforcement. + * + * @param signaturePrefixes A list of class signature prefixes. Each item in the list is a prefix match on the type + * signature of a blacklisted API. All matching APIs are treated as if they were on + * the whitelist: access permitted, and no logging.. + * @return whether the operation is successful + */ + public static boolean addHiddenApiExemptions(String... signaturePrefixes) { + HiddenApiBypass.signaturePrefixes.addAll(Arrays.asList(signaturePrefixes)); + String[] strings = new String[HiddenApiBypass.signaturePrefixes.size()]; + HiddenApiBypass.signaturePrefixes.toArray(strings); + return setHiddenApiExemptions(strings); + } + + /** + * Clear the list of exemptions from hidden API access enforcement. + * Android runtime will cache access flags, so if a hidden API has been accessed unrestrictedly, + * running this method will not restore the restriction on it. + * + * @return whether the operation is successful + */ + public static boolean clearHiddenApiExemptions() { + HiddenApiBypass.signaturePrefixes.clear(); + return setHiddenApiExemptions(); + } +}
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/local_modifications/org/lsposed/hiddenapibypass/library/BuildConfig.java b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/local_modifications/org/lsposed/hiddenapibypass/library/BuildConfig.java new file mode 100644 index 0000000..9788a8e --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/local_modifications/org/lsposed/hiddenapibypass/library/BuildConfig.java
@@ -0,0 +1,9 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +package org.lsposed.hiddenapibypass.library; + +/** When building with Gradle, this file would be generated. */ +public class BuildConfig { + public static final boolean DEBUG = false; +}
diff --git a/build/android/incremental_install/third_party/AndroidHiddenApiBypass/stub/src/main/java/dalvik/system/VMRuntime.java b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/stub/src/main/java/dalvik/system/VMRuntime.java new file mode 100644 index 0000000..87db1ec --- /dev/null +++ b/build/android/incremental_install/third_party/AndroidHiddenApiBypass/stub/src/main/java/dalvik/system/VMRuntime.java
@@ -0,0 +1,9 @@ +package dalvik.system; + +@SuppressWarnings("unused") +public class VMRuntime { + public static VMRuntime getRuntime() { + throw new IllegalArgumentException("stub"); + } + public native void setHiddenApiExemptions(String[] signaturePrefixes); +}
diff --git a/build/android/incremental_install/write_installer_json.py b/build/android/incremental_install/write_installer_json.py index cf1d2d4..4825a80 100755 --- a/build/android/incremental_install/write_installer_json.py +++ b/build/android/incremental_install/write_installer_json.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -14,6 +14,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'gyp')) from util import build_utils +import action_helpers # build_utils adds //build to sys.path. def _ParseArgs(args): @@ -44,8 +45,8 @@ help='Print a warning about proguard being disabled') options = parser.parse_args(args) - options.dex_files = build_utils.ParseGnList(options.dex_files) - options.native_libs = build_utils.ParseGnList(options.native_libs) + options.dex_files = action_helpers.parse_gn_list(options.dex_files) + options.native_libs = action_helpers.parse_gn_list(options.native_libs) return options @@ -60,7 +61,7 @@ 'split_globs': options.split_globs, } - with build_utils.AtomicOutput(options.output_path, mode='w+') as f: + with action_helpers.atomic_output(options.output_path, mode='w+') as f: json.dump(data, f, indent=2, sort_keys=True)
diff --git a/build/android/incremental_install/write_installer_json.pydeps b/build/android/incremental_install/write_installer_json.pydeps index 11a263f..519281f 100644 --- a/build/android/incremental_install/write_installer_json.pydeps +++ b/build/android/incremental_install/write_installer_json.pydeps
@@ -1,5 +1,6 @@ # Generated by running: # build/print_python_deps.py --root build/android/incremental_install --output build/android/incremental_install/write_installer_json.pydeps build/android/incremental_install/write_installer_json.py +../../action_helpers.py ../../gn_helpers.py ../gyp/util/__init__.py ../gyp/util/build_utils.py
diff --git a/build/android/java/src/org/chromium/build/annotations/AlwaysInline.java b/build/android/java/src/org/chromium/build/annotations/AlwaysInline.java new file mode 100644 index 0000000..e79bfe7 --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/AlwaysInline.java
@@ -0,0 +1,17 @@ +// Copyright 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Tells R8 to always inline the annotated method/constructor. + */ +@Target({ElementType.CONSTRUCTOR, ElementType.METHOD}) +@Retention(RetentionPolicy.CLASS) +public @interface AlwaysInline {}
diff --git a/build/android/java/src/org/chromium/build/annotations/CheckDiscard.java b/build/android/java/src/org/chromium/build/annotations/CheckDiscard.java new file mode 100644 index 0000000..897067e --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/CheckDiscard.java
@@ -0,0 +1,24 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Causes build to assert that annotated classes / methods / fields are + * optimized away in release builds (without dcheck_always_on). + */ +@Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.CLASS) +public @interface CheckDiscard { + /** + * Describes why the element should be discarded. + * @return reason for discarding (crbug links are preferred unless reason is trivial). + */ + String value(); +}
diff --git a/build/android/java/src/org/chromium/build/annotations/DoNotClassMerge.java b/build/android/java/src/org/chromium/build/annotations/DoNotClassMerge.java new file mode 100644 index 0000000..94c9fa3 --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/DoNotClassMerge.java
@@ -0,0 +1,20 @@ +// Copyright 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * The annotated class should never be horizontally or vertically merged. + * + * The annotated classes are guaranteed not to be horizontally or vertically + * merged by Proguard. Other optimizations may still apply. + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.CLASS) +public @interface DoNotClassMerge {}
diff --git a/build/android/java/src/org/chromium/build/annotations/DoNotInline.java b/build/android/java/src/org/chromium/build/annotations/DoNotInline.java new file mode 100644 index 0000000..4dd1933 --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/DoNotInline.java
@@ -0,0 +1,20 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * The annotated method or class should never be inlined. + * + * The annotated method (or methods on the annotated class) are guaranteed not to be inlined by + * Proguard. Other optimizations may still apply. + */ +@Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.CLASS) +public @interface DoNotInline {}
diff --git a/build/android/java/src/org/chromium/build/annotations/DoNotStripLogs.java b/build/android/java/src/org/chromium/build/annotations/DoNotStripLogs.java new file mode 100644 index 0000000..be96d9a --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/DoNotStripLogs.java
@@ -0,0 +1,17 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * The annotated method or class will have -maximumremovedandroidloglevel 0 applied to it. + */ +@Target({ElementType.CONSTRUCTOR, ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.CLASS) +public @interface DoNotStripLogs {}
diff --git a/build/android/java/src/org/chromium/build/annotations/IdentifierNameString.java b/build/android/java/src/org/chromium/build/annotations/IdentifierNameString.java new file mode 100644 index 0000000..ca8b2df --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/IdentifierNameString.java
@@ -0,0 +1,35 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation used to mark field that may contain Strings referring to fully qualified class names + * and methods whose arguments may be fully qualified class names. These classes may then be + * obfuscated by R8. A couple caveats when using this: + * - This only obfuscates the string, it does not actually check that the class exists. + * - If a field has this annotation, it must be non-final, otherwise javac will inline the constant + * and R8 won't obfuscate it. + * - Any field/method must be assigned/called with a String literal or a variable R8 can easily + * trace to a String literal. + * + * <p>Usage example:<br> + * {@code + * @IdentifierNameString + * public static final String LOGGING_TAG = "com.google.android.apps.foo.FooActivity"; + * + * // In this example, both className and message are treated as identifier name strings, but will + * // only be obfuscated if the string points to a real class. + * @IdentifierNameString + * public void doSomeLogging(String className, String message) { ... } + * } + */ +@Target({ElementType.FIELD, ElementType.METHOD}) +@Retention(RetentionPolicy.CLASS) +public @interface IdentifierNameString {}
diff --git a/build/android/java/src/org/chromium/build/annotations/MainDex.java b/build/android/java/src/org/chromium/build/annotations/MainDex.java new file mode 100644 index 0000000..5eedb0b --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/MainDex.java
@@ -0,0 +1,23 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Classes with native methods (contain @NativeMethods interfaces) that are used within renderer + * processes must be annotated with with @MainDex in order for their native methods work. + * + * Applies only for Chrome/ChromeModern (not needed for Monochrome+). + * + * For Cronet builds, which use a default_min_sdk_version of less than 21, this annotation also + * causes classes to appear in the main dex file (for "Legacy multidex"). + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface MainDex {}
diff --git a/build/android/java/src/org/chromium/build/annotations/MockedInTests.java b/build/android/java/src/org/chromium/build/annotations/MockedInTests.java new file mode 100644 index 0000000..6b486f7 --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/MockedInTests.java
@@ -0,0 +1,17 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; + +/** + * See b/147584922. Proguard and Mockito don't play nicely together, and proguard rules make it + * impossible to keep the base class/interface for a mocked class without providing additional + * explicit information, like this annotation. This annotation should only need to be used on a + * class/interface that is extended/implemented by another class/interface that is then mocked. + */ +@Target(ElementType.TYPE) +public @interface MockedInTests {}
diff --git a/build/android/java/src/org/chromium/build/annotations/UsedByReflection.java b/build/android/java/src/org/chromium/build/annotations/UsedByReflection.java new file mode 100644 index 0000000..f28f383 --- /dev/null +++ b/build/android/java/src/org/chromium/build/annotations/UsedByReflection.java
@@ -0,0 +1,22 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; + +/** + * Annotation used for marking methods and fields that are called by reflection. + * Useful for keeping components that would otherwise be removed by Proguard. + * Use the value parameter to mention a file that calls this method. + * + * Note that adding this annotation to a method is not enough to guarantee that + * it is kept - either its class must be referenced elsewhere in the program, or + * the class must be annotated with this as well. + */ +@Target({ElementType.METHOD, ElementType.FIELD, ElementType.TYPE, ElementType.CONSTRUCTOR}) +public @interface UsedByReflection { + String value(); +}
diff --git a/build/android/java/templates/BuildConfig.template b/build/android/java/templates/BuildConfig.template index 8953ad5..cfecb6f 100644 --- a/build/android/java/templates/BuildConfig.template +++ b/build/android/java/templates/BuildConfig.template
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -81,15 +81,15 @@ public static MAYBE_FINAL boolean IS_INCREMENTAL_INSTALL MAYBE_FALSE; #endif -#if defined(_IS_CHROMECAST_BRANDING_INTERNAL) - public static MAYBE_FINAL boolean IS_CHROMECAST_BRANDING_INTERNAL = true; -#else - public static MAYBE_FINAL boolean IS_CHROMECAST_BRANDING_INTERNAL MAYBE_FALSE; -#endif - #if defined(_ISOLATED_SPLITS_ENABLED) public static MAYBE_FINAL boolean ISOLATED_SPLITS_ENABLED = true; #else public static MAYBE_FINAL boolean ISOLATED_SPLITS_ENABLED MAYBE_FALSE; #endif + +#if defined(_IS_FOR_TEST) + public static MAYBE_FINAL boolean IS_FOR_TEST = true; +#else + public static MAYBE_FINAL boolean IS_FOR_TEST MAYBE_FALSE; +#endif }
diff --git a/build/android/java/templates/ProductConfig.template b/build/android/java/templates/ProductConfig.template index 4bc0d52..d6e1236 100644 --- a/build/android/java/templates/ProductConfig.template +++ b/build/android/java/templates/ProductConfig.template
@@ -1,4 +1,4 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. +// Copyright 2019 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -7,12 +7,10 @@ #if defined(USE_FINAL) #define MAYBE_FINAL final #define MAYBE_USE_CHROMIUM_LINKER = USE_CHROMIUM_LINKER_VALUE -#define MAYBE_USE_MODERN_LINKER = USE_MODERN_LINKER_VALUE #define MAYBE_IS_BUNDLE = IS_BUNDLE_VALUE #else #define MAYBE_FINAL #define MAYBE_USE_CHROMIUM_LINKER -#define MAYBE_USE_MODERN_LINKER #define MAYBE_IS_BUNDLE #endif @@ -29,6 +27,5 @@ #endif public static MAYBE_FINAL boolean USE_CHROMIUM_LINKER MAYBE_USE_CHROMIUM_LINKER; - public static MAYBE_FINAL boolean USE_MODERN_LINKER MAYBE_USE_MODERN_LINKER; public static MAYBE_FINAL boolean IS_BUNDLE MAYBE_IS_BUNDLE; }
diff --git a/build/android/java/test/DefaultLocaleLintTest.java b/build/android/java/test/DefaultLocaleLintTest.java index 2193429..76f9ea5 100644 --- a/build/android/java/test/DefaultLocaleLintTest.java +++ b/build/android/java/test/DefaultLocaleLintTest.java
@@ -1,4 +1,4 @@ -// Copyright 2021 The Chromium Authors. All rights reserved. +// Copyright 2021 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/java/test/NewApiLintTest.java b/build/android/java/test/NewApiLintTest.java index 6c68dd8..66d576a 100644 --- a/build/android/java/test/NewApiLintTest.java +++ b/build/android/java/test/NewApiLintTest.java
@@ -1,4 +1,4 @@ -// Copyright 2021 The Chromium Authors. All rights reserved. +// Copyright 2021 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/java/test/NoSignatureChangeIncrementalJavacTestHelper.template b/build/android/java/test/NoSignatureChangeIncrementalJavacTestHelper.template new file mode 100644 index 0000000..b51a67d --- /dev/null +++ b/build/android/java/test/NoSignatureChangeIncrementalJavacTestHelper.template
@@ -0,0 +1,18 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test; + +public class NoSignatureChangeIncrementalJavacTestHelper { + private NoSignatureChangeIncrementalJavacTestHelper2 mHelper2 = + new NoSignatureChangeIncrementalJavacTestHelper2(); + + public String foo() { + return "{{foo_return_value}}"; + } + + public String bar() { + return mHelper2.bar(); + } +}
diff --git a/build/android/java/test/NoSignatureChangeIncrementalJavacTestHelper2.java b/build/android/java/test/NoSignatureChangeIncrementalJavacTestHelper2.java new file mode 100644 index 0000000..9694f3f --- /dev/null +++ b/build/android/java/test/NoSignatureChangeIncrementalJavacTestHelper2.java
@@ -0,0 +1,11 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test; + +public class NoSignatureChangeIncrementalJavacTestHelper2 { + public String bar() { + return "bar"; + } +}
diff --git a/build/android/java/test/missing_symbol/B.java b/build/android/java/test/missing_symbol/B.java new file mode 100644 index 0000000..639a744 --- /dev/null +++ b/build/android/java/test/missing_symbol/B.java
@@ -0,0 +1,9 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test.missing_symbol; + +public class B { + public void foo() {} +}
diff --git a/build/android/java/test/missing_symbol/D.template b/build/android/java/test/missing_symbol/D.template new file mode 100644 index 0000000..3f7eef3 --- /dev/null +++ b/build/android/java/test/missing_symbol/D.template
@@ -0,0 +1,9 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test.missing_symbol; + +public class D { + public void foo() {} +}
diff --git a/build/android/java/test/missing_symbol/Importer.template b/build/android/java/test/missing_symbol/Importer.template new file mode 100644 index 0000000..a1fd881 --- /dev/null +++ b/build/android/java/test/missing_symbol/Importer.template
@@ -0,0 +1,13 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package _IMPORTER_PACKAGE; + +import _IMPORTEE_PACKAGE._IMPORTEE_CLASS_NAME; + +public class Importer { + public Importer() { + new _IMPORTEE_CLASS_NAME().foo(); + } +}
diff --git a/build/android/java/test/missing_symbol/ImportsSubB.java b/build/android/java/test/missing_symbol/ImportsSubB.java new file mode 100644 index 0000000..2422b4a --- /dev/null +++ b/build/android/java/test/missing_symbol/ImportsSubB.java
@@ -0,0 +1,13 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test.missing_symbol; + +import test.missing_symbol.sub.SubB; + +public class ImportsSubB { + public ImportsSubB() { + new SubB().foo(); + } +}
diff --git a/build/android/java/test/missing_symbol/c.jar b/build/android/java/test/missing_symbol/c.jar new file mode 100644 index 0000000..5f30be8 --- /dev/null +++ b/build/android/java/test/missing_symbol/c.jar Binary files differ
diff --git a/build/android/java/test/missing_symbol/sub/BInMethodSignature.java b/build/android/java/test/missing_symbol/sub/BInMethodSignature.java new file mode 100644 index 0000000..36b6ba2 --- /dev/null +++ b/build/android/java/test/missing_symbol/sub/BInMethodSignature.java
@@ -0,0 +1,13 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test.missing_symbol.sub; + +import test.missing_symbol.B; + +public class BInMethodSignature { + public B foo() { + return new B(); + } +}
diff --git a/build/android/java/test/missing_symbol/sub/SubB.java b/build/android/java/test/missing_symbol/sub/SubB.java new file mode 100644 index 0000000..1e58378 --- /dev/null +++ b/build/android/java/test/missing_symbol/sub/SubB.java
@@ -0,0 +1,9 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package test.missing_symbol.sub; + +public class SubB { + public void foo() {} +}
diff --git a/build/android/junit/AndroidManifest_mergetest.xml b/build/android/junit/AndroidManifest_mergetest.xml new file mode 100644 index 0000000..2541b8d --- /dev/null +++ b/build/android/junit/AndroidManifest_mergetest.xml
@@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright 2022 The Chromium Authors + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="test.merged.manifest"> + <application> + <meta-data android:name="test-metadata" android:value="Hello World" /> + </application> +</manifest>
diff --git a/build/android/junit/res/values/strings.xml b/build/android/junit/res/values/strings.xml new file mode 100644 index 0000000..9b9c078 --- /dev/null +++ b/build/android/junit/res/values/strings.xml
@@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2022 The Chromium Authors + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. +--> +<resources> + <string name="test_string">Hello World</string> +</resources>
diff --git a/build/android/junit/src/org/chromium/build/AndroidAssetsTest.java b/build/android/junit/src/org/chromium/build/AndroidAssetsTest.java new file mode 100644 index 0000000..8ff149e --- /dev/null +++ b/build/android/junit/src/org/chromium/build/AndroidAssetsTest.java
@@ -0,0 +1,58 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build; + +import android.content.Context; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageManager; +import android.content.pm.PackageManager.NameNotFoundException; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.RuntimeEnvironment; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Checks that Robolectric tests can use android assets. + */ +@RunWith(RobolectricTestRunner.class) +public class AndroidAssetsTest { + private static final String TEST_ASSET_NAME = "AndroidAssetsTest.java"; + + public String readTestAsset() throws IOException { + try (InputStream stream = + RuntimeEnvironment.getApplication().getAssets().open(TEST_ASSET_NAME)) { + byte[] buffer = new byte[stream.available()]; + stream.read(buffer); + return new String(buffer); + } + } + + @Test + public void testAssetsExist() throws IOException { + String myselfAsAssetData = readTestAsset(); + Assert.assertTrue("asset not correct. It had length=" + myselfAsAssetData.length(), + myselfAsAssetData.contains("String myselfAsAssetData = ")); + } + + @Test + public void testResourcesExist() { + String actual = RuntimeEnvironment.getApplication().getString(R.string.test_string); + Assert.assertEquals("Hello World", actual); + } + + @Test + public void testManifestMerged() throws NameNotFoundException { + Context context = RuntimeEnvironment.getApplication(); + ApplicationInfo info = context.getPackageManager().getApplicationInfo( + context.getPackageName(), PackageManager.GET_META_DATA); + String actual = info.metaData.getString("test-metadata"); + Assert.assertEquals("Hello World", actual); + } +}
diff --git a/build/android/junit/src/org/chromium/build/IncrementalJavacTest.java b/build/android/junit/src/org/chromium/build/IncrementalJavacTest.java new file mode 100644 index 0000000..b15b7df --- /dev/null +++ b/build/android/junit/src/org/chromium/build/IncrementalJavacTest.java
@@ -0,0 +1,33 @@ +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.build; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.annotation.Config; + +import test.NoSignatureChangeIncrementalJavacTestHelper; + +/** + * Checks that build picked up changes to + * {@link NoSignatureChangeIncrementalJavacTestHelper#foo()}. + */ +@RunWith(RobolectricTestRunner.class) +@Config(manifest = Config.NONE) +public final class IncrementalJavacTest { + @Test + public void testNoSignatureChange() { + NoSignatureChangeIncrementalJavacTestHelper helper = + new NoSignatureChangeIncrementalJavacTestHelper(); + // #foo() should return updated value. + assertEquals("foo2", helper.foo()); + + // #bar() should not crash. + assertEquals("bar", helper.bar()); + } +}
diff --git a/build/android/lighttpd_server.py b/build/android/lighttpd_server.py index 42fbcdb..9950253 100755 --- a/build/android/lighttpd_server.py +++ b/build/android/lighttpd_server.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,11 +10,9 @@ lighttpd_server PATH_TO_DOC_ROOT """ -from __future__ import print_function import codecs import contextlib -import httplib import os import random import shutil @@ -24,10 +22,14 @@ import tempfile import time +from six.moves import http_client +from six.moves import input # pylint: disable=redefined-builtin + from pylib import constants from pylib import pexpect -class LighttpdServer(object): + +class LighttpdServer: """Wraps lighttpd server, providing robust startup. Args: @@ -122,11 +124,12 @@ def _TestServerConnection(self): # Wait for server to start server_msg = '' - for timeout in xrange(1, 5): + for timeout in range(1, 5): client_error = None try: - with contextlib.closing(httplib.HTTPConnection( - '127.0.0.1', self.port, timeout=timeout)) as http: + with contextlib.closing( + http_client.HTTPConnection('127.0.0.1', self.port, + timeout=timeout)) as http: http.set_debuglevel(timeout > 3) http.request('HEAD', '/') r = http.getresponse() @@ -137,7 +140,7 @@ client_error = ('Bad response: %s %s version %s\n ' % (r.status, r.reason, r.version) + '\n '.join([': '.join(h) for h in r.getheaders()])) - except (httplib.HTTPException, socket.error) as client_error: + except (http_client.HTTPException, socket.error) as client_error: pass # Probably too quick connecting: try again # Check for server startup error messages # pylint: disable=no-member @@ -248,8 +251,8 @@ server = LighttpdServer(*argv[1:]) try: if server.StartupHttpServer(): - raw_input('Server running at http://127.0.0.1:%s -' - ' press Enter to exit it.' % server.port) + input('Server running at http://127.0.0.1:%s -' + ' press Enter to exit it.' % server.port) else: print('Server exit code:', server.process.exitstatus) finally:
diff --git a/build/android/list_class_verification_failures.py b/build/android/list_class_verification_failures.py index 508e831..9c94e30 100755 --- a/build/android/list_class_verification_failures.py +++ b/build/android/list_class_verification_failures.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2018 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,10 +9,10 @@ and accommodating API-level-specific details, such as file paths. """ -from __future__ import print_function + import argparse -import exceptions +import dataclasses # pylint: disable=wrong-import-order import logging import os import re @@ -63,12 +63,10 @@ class DeviceOSError(Exception): """Raised when a file is missing from the device, or something similar.""" - pass class UnsupportedDeviceError(Exception): """Raised when the device is not supported by this script.""" - pass def _GetFormattedArch(device): @@ -77,63 +75,61 @@ return {abis.ARM_64: 'arm64', abis.ARM: 'arm'}.get(abi, abi) -def PathToDexForPlatformVersion(device, package_name): - """Gets the full path to the dex file on the device.""" +def FindOdexFiles(device, package_name): + """Gets the full paths to the dex files on the device.""" sdk_level = device.build_version_sdk paths_to_apk = device.GetApplicationPaths(package_name) if not paths_to_apk: raise DeviceOSError( 'Could not find data directory for {}. Is it installed?'.format( package_name)) - if len(paths_to_apk) != 1: - raise DeviceOSError( - 'Expected exactly one path for {} but found {}'.format( - package_name, - paths_to_apk)) - path_to_apk = paths_to_apk[0] - if version_codes.LOLLIPOP <= sdk_level <= version_codes.LOLLIPOP_MR1: - # Of the form "com.example.foo-\d", where \d is some digit (usually 1 or 2) - package_with_suffix = os.path.basename(os.path.dirname(path_to_apk)) - arch = _GetFormattedArch(device) - dalvik_prefix = '/data/dalvik-cache/{arch}'.format(arch=arch) - odex_file = '{prefix}/data@app@{package}@base.apk@classes.dex'.format( - prefix=dalvik_prefix, - package=package_with_suffix) - elif sdk_level >= version_codes.MARSHMALLOW: - arch = _GetFormattedArch(device) - odex_file = '{data_dir}/oat/{arch}/base.odex'.format( - data_dir=os.path.dirname(path_to_apk), arch=arch) - else: - raise UnsupportedDeviceError('Unsupported API level: {}'.format(sdk_level)) + ret = [] + for path_to_apk in paths_to_apk: + if version_codes.LOLLIPOP <= sdk_level <= version_codes.LOLLIPOP_MR1: + # Of the form "com.example.foo-\d", where \d is a digit (usually 1 or 2). + package_with_suffix = os.path.basename(os.path.dirname(path_to_apk)) + arch = _GetFormattedArch(device) + dalvik_prefix = '/data/dalvik-cache/{arch}'.format(arch=arch) + odex_file = '{prefix}/data@app@{package}@base.apk@classes.dex'.format( + prefix=dalvik_prefix, package=package_with_suffix) + elif sdk_level >= version_codes.MARSHMALLOW: + arch = _GetFormattedArch(device) + odex_file = '{data_dir}/oat/{arch}/base.odex'.format( + data_dir=os.path.dirname(path_to_apk), arch=arch) + else: + raise UnsupportedDeviceError( + 'Unsupported API level: {}'.format(sdk_level)) - odex_file_exists = device.FileExists(odex_file) - if odex_file_exists: - return odex_file - elif sdk_level >= version_codes.PIE: - raise DeviceOSError( - 'Unable to find odex file: you must run dex2oat on debuggable apps ' - 'on >= P after installation.') - raise DeviceOSError('Unable to find odex file ' + odex_file) + odex_file_exists = device.FileExists(odex_file) + if odex_file_exists: + ret.append(odex_file) + elif sdk_level >= version_codes.PIE: + raise DeviceOSError( + 'Unable to find odex file: you must run dex2oat on debuggable apps ' + 'on >= P after installation.') + else: + raise DeviceOSError('Unable to find odex file ' + odex_file) + return ret -def _AdbOatDumpForPackage(device, package_name, out_file): +def _AdbOatDump(device, odex_file, out_file): """Runs oatdump on the device.""" # Get the path to the odex file. - odex_file = PathToDexForPlatformVersion(device, package_name) - device.RunShellCommand( - ['oatdump', '--oat-file=' + odex_file, '--output=' + out_file], - timeout=420, - shell=True, - check_return=True) + with device_temp_file.DeviceTempFile(device.adb) as device_file: + device.RunShellCommand( + ['oatdump', '--oat-file=' + odex_file, '--output=' + device_file.name], + timeout=420, + shell=True, + check_return=True) + device.PullFile(device_file.name, out_file, timeout=220) -class JavaClass(object): +@dataclasses.dataclass(order=True, frozen=True) +class JavaClass: """This represents a Java Class and its ART Class Verification status.""" - - def __init__(self, name, verification_status): - self.name = name - self.verification_status = verification_status + name: str + verification_status: str def _ParseMappingFile(proguard_map_file): @@ -158,11 +154,10 @@ obfuscated_name = dex_code_name.replace('/', '.') if proguard_mappings is not None: return _DeobfuscateJavaClassName(obfuscated_name, proguard_mappings) - else: - return obfuscated_name + return obfuscated_name -def ListClassesAndVerificationStatus(oatdump_output, proguard_mappings): +def ParseOatdump(oatdump_output, proguard_mappings): """Lists all Java classes in the dex along with verification status.""" java_classes = [] pattern = re.compile(r'\d+: L([^;]+).*\(type_idx=[^(]+\((\w+)\).*') @@ -189,10 +184,9 @@ if java_class.verification_status == target_status: print(java_class.name) if java_class.verification_status not in d: - raise exceptions.RuntimeError('Unexpected status: {0}'.format( + raise RuntimeError('Unexpected status: {0}'.format( java_class.verification_status)) - else: - d[java_class.verification_status] += 1 + d[java_class.verification_status] += 1 if show_summary: for status in d: @@ -205,18 +199,20 @@ def RealMain(mapping, device_arg, package, status, hide_summary, workdir): if mapping is None: - logging.warn('Skipping deobfuscation because no map file was provided.') + logging.warning('Skipping deobfuscation because no map file was provided.') + proguard_mappings = None + else: + proguard_mappings = _ParseMappingFile(mapping) device = DetermineDeviceToUse(device_arg) + host_tempfile = os.path.join(workdir, 'out.dump') device.EnableRoot() - with device_temp_file.DeviceTempFile( - device.adb) as file_on_device: - _AdbOatDumpForPackage(device, package, file_on_device.name) - file_on_host = os.path.join(workdir, 'out.dump') - device.PullFile(file_on_device.name, file_on_host, timeout=220) - proguard_mappings = (_ParseMappingFile(mapping) if mapping else None) - with open(file_on_host, 'r') as f: - java_classes = ListClassesAndVerificationStatus(f, proguard_mappings) - _PrintVerificationResults(status, java_classes, not hide_summary) + odex_files = FindOdexFiles(device, package) + java_classes = set() + for odex_file in odex_files: + _AdbOatDump(device, odex_file, host_tempfile) + with open(host_tempfile, 'r') as f: + java_classes.update(ParseOatdump(f, proguard_mappings)) + _PrintVerificationResults(status, sorted(java_classes), not hide_summary) def main(): @@ -271,8 +267,8 @@ RealMain(args.mapping, args.devices, args.package, args.status, args.hide_summary, args.workdir) # Assume the user wants the workdir to persist (useful for debugging). - logging.warn('Not cleaning up explicitly-specified workdir: %s', - args.workdir) + logging.warning('Not cleaning up explicitly-specified workdir: %s', + args.workdir) else: with tempfile_ext.NamedTemporaryDirectory() as workdir: RealMain(args.mapping, args.devices, args.package, args.status,
diff --git a/build/android/list_class_verification_failures_test.py b/build/android/list_class_verification_failures_test.py old mode 100644 new mode 100755 index 4248064..1499436 --- a/build/android/list_class_verification_failures_test.py +++ b/build/android/list_class_verification_failures_test.py
@@ -1,4 +1,5 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -76,7 +77,7 @@ device.GetApplicationPaths = mock.MagicMock(return_value=paths_to_apk) with self.assertRaises(list_verification.DeviceOSError) as cm: - list_verification.PathToDexForPlatformVersion(device, package_name) + list_verification.FindOdexFiles(device, package_name) message = str(cm.exception) self.assertIn('Could not find data directory', message) @@ -89,10 +90,11 @@ device = mock.Mock(build_version_sdk=sdk_int, product_cpu_abi=arch) device.GetApplicationPaths = mock.MagicMock(return_value=paths_to_apk) - with self.assertRaises(list_verification.DeviceOSError) as cm: - list_verification.PathToDexForPlatformVersion(device, package_name) - message = str(cm.exception) - self.assertIn('Expected exactly one path for', message) + odex_files = list_verification.FindOdexFiles(device, package_name) + self.assertEqual(odex_files, [ + '/data/dalvik-cache/arm64/data@app@first@base.apk@classes.dex', + '/data/dalvik-cache/arm64/data@app@second@base.apk@classes.dex' + ]) def testPathToDexForPlatformVersion_dalvikApiLevel(self): sdk_int = version_codes.KITKAT @@ -104,7 +106,7 @@ device.GetApplicationPaths = mock.MagicMock(return_value=paths_to_apk) with self.assertRaises(list_verification.UnsupportedDeviceError) as _: - list_verification.PathToDexForPlatformVersion(device, package_name) + list_verification.FindOdexFiles(device, package_name) def testPathToDexForPlatformVersion_lollipopArm(self): sdk_int = version_codes.LOLLIPOP @@ -116,11 +118,10 @@ device.GetApplicationPaths = mock.MagicMock(return_value=paths_to_apk) device.FileExists = mock.MagicMock(return_value=True) - odex_file = list_verification.PathToDexForPlatformVersion(device, - package_name) - self.assertEqual(odex_file, - ('/data/dalvik-cache/arm/data@app' - '@package.name-1@base.apk@classes.dex')) + odex_files = list_verification.FindOdexFiles(device, package_name) + self.assertEqual( + odex_files, + ['/data/dalvik-cache/arm/data@app@package.name-1@base.apk@classes.dex']) def testPathToDexForPlatformVersion_mashmallowArm(self): sdk_int = version_codes.MARSHMALLOW @@ -132,10 +133,9 @@ device.GetApplicationPaths = mock.MagicMock(return_value=paths_to_apk) device.FileExists = mock.MagicMock(return_value=True) - odex_file = list_verification.PathToDexForPlatformVersion(device, - package_name) - self.assertEqual(odex_file, - '/some/path/package.name-1/oat/arm/base.odex') + odex_files = list_verification.FindOdexFiles(device, package_name) + self.assertEqual(odex_files, + ['/some/path/package.name-1/oat/arm/base.odex']) def testPathToDexForPlatformVersion_mashmallowArm64(self): sdk_int = version_codes.MARSHMALLOW @@ -147,10 +147,9 @@ device.GetApplicationPaths = mock.MagicMock(return_value=paths_to_apk) device.FileExists = mock.MagicMock(return_value=True) - odex_file = list_verification.PathToDexForPlatformVersion(device, - package_name) - self.assertEqual(odex_file, - '/some/path/package.name-1/oat/arm64/base.odex') + odex_files = list_verification.FindOdexFiles(device, package_name) + self.assertEqual(odex_files, + ['/some/path/package.name-1/oat/arm64/base.odex']) def testPathToDexForPlatformVersion_pieNoOdexFile(self): sdk_int = version_codes.PIE @@ -163,7 +162,7 @@ device.FileExists = mock.MagicMock(return_value=False) with self.assertRaises(list_verification.DeviceOSError) as cm: - list_verification.PathToDexForPlatformVersion(device, package_name) + list_verification.FindOdexFiles(device, package_name) message = str(cm.exception) self.assertIn('you must run dex2oat on debuggable apps on >= P', message) @@ -178,7 +177,7 @@ device.FileExists = mock.MagicMock(return_value=False) with self.assertRaises(list_verification.DeviceOSError) as _: - list_verification.PathToDexForPlatformVersion(device, package_name) + list_verification.FindOdexFiles(device, package_name) def testListClasses_noProguardMap(self): oatdump_output = [ @@ -187,8 +186,7 @@ 'StatusRetryVerificationAtRuntime'), ] - classes = list_verification.ListClassesAndVerificationStatus(oatdump_output, - None) + classes = list_verification.ParseOatdump(oatdump_output, None) self.assertEqual(2, len(classes)) java_class_1 = _ClassForName('a.b.JavaClass1', classes) java_class_2 = _ClassForName('a.b.JavaClass2', classes) @@ -207,8 +205,7 @@ 'a.b.ObfuscatedJavaClass1': 'a.b.JavaClass1', 'a.b.ObfuscatedJavaClass2': 'a.b.JavaClass2', } - classes = list_verification.ListClassesAndVerificationStatus(oatdump_output, - mapping) + classes = list_verification.ParseOatdump(oatdump_output, mapping) self.assertEqual(2, len(classes)) java_class_1 = _ClassForName('a.b.JavaClass1', classes) java_class_2 = _ClassForName('a.b.JavaClass2', classes) @@ -222,8 +219,7 @@ _CreateOdexLine('a.b.JavaClass2', 7, 'RetryVerificationAtRuntime'), ] - classes = list_verification.ListClassesAndVerificationStatus(oatdump_output, - None) + classes = list_verification.ParseOatdump(oatdump_output, None) self.assertEqual(2, len(classes)) java_class_1 = _ClassForName('a.b.JavaClass1', classes) java_class_2 = _ClassForName('a.b.JavaClass2', classes)
diff --git a/build/android/list_java_targets.py b/build/android/list_java_targets.py index d0689a6..b135b0f 100755 --- a/build/android/list_java_targets.py +++ b/build/android/list_java_targets.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython3 -# Copyright 2020 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -28,6 +28,7 @@ import json import logging import os +import shutil import subprocess import sys @@ -48,25 +49,49 @@ 'java_annotation_processor', 'java_binary', 'java_library', - 'junit_binary', + 'robolectric_binary', 'system_java_library', ) -def _run_ninja(output_dir, args): - cmd = [ - 'autoninja', +def _resolve_ninja(): + # Prefer the version on PATH, but fallback to known version if PATH doesn't + # have one (e.g. on bots). + if shutil.which('ninja') is None: + return os.path.join(_SRC_ROOT, 'third_party', 'ninja', 'ninja') + return 'ninja' + + +def _resolve_autoninja(): + # Prefer the version on PATH, but fallback to known version if PATH doesn't + # have one (e.g. on bots). + if shutil.which('autoninja') is None: + return os.path.join(_SRC_ROOT, 'third_party', 'depot_tools', 'autoninja') + return 'autoninja' + + +def _run_ninja(output_dir, args, j_value=None, quiet=False): + if j_value: + cmd = [_resolve_ninja(), '-j', j_value] + else: + cmd = [_resolve_autoninja()] + cmd += [ '-C', output_dir, ] cmd.extend(args) logging.info('Running: %r', cmd) - subprocess.run(cmd, check=True, stdout=sys.stderr) + if quiet: + subprocess.run(cmd, check=True, capture_output=True) + else: + subprocess.run(cmd, check=True, stdout=sys.stderr) def _query_for_build_config_targets(output_dir): # Query ninja rather than GN since it's faster. - cmd = ['ninja', '-C', output_dir, '-t', 'targets'] + # Use ninja rather than autoninja to avoid extra output if user has set the + # NINJA_SUMMARIZE_BUILD environment variable. + cmd = [_resolve_ninja(), '-C', output_dir, '-t', 'targets'] logging.info('Running: %r', cmd) ninja_output = subprocess.run(cmd, check=True, @@ -83,7 +108,45 @@ return ret -class _TargetEntry(object): +def _query_json(*, json_dict: dict, query: str, path: str): + """Traverses through the json dictionary according to the query. + + If at any point a key does not exist, return the empty string, but raise an + error if a key exists but is the wrong type. + + This is roughly equivalent to returning + json_dict[queries[0]]?[queries[1]]?...[queries[N]]? where the ? means that if + the key doesn't exist, the empty string is returned. + + Example: + Given json_dict = {'a': {'b': 'c'}} + - If queries = ['a', 'b'] + Return: 'c' + - If queries = ['a', 'd'] + Return '' + - If queries = ['x'] + Return '' + - If queries = ['a', 'b', 'x'] + Raise an error since json_dict['a']['b'] is the string 'c' instead of an + expected dict that can be indexed into. + + Returns the final result after exhausting all the queries. + """ + queries = query.split('.') + value = json_dict + try: + for key in queries: + value = value.get(key) + if value is None: + return '' + except AttributeError as e: + raise Exception( + f'Failed when attempting to get {queries} from {path}') from e + return value + + +class _TargetEntry: + def __init__(self, gn_target): assert gn_target.startswith('//'), f'{gn_target} does not start with //' assert ':' in gn_target, f'Non-root {gn_target} required' @@ -100,23 +163,23 @@ @property def build_config_path(self): - """Returns the filepath of the project's .build_config.""" + """Returns the filepath of the project's .build_config.json.""" ninja_target = self.ninja_target # Support targets at the root level. e.g. //:foo if ninja_target[0] == ':': ninja_target = ninja_target[1:] - subpath = ninja_target.replace(':', os.path.sep) + '.build_config' + subpath = ninja_target.replace(':', os.path.sep) + '.build_config.json' return os.path.join(constants.GetOutDirectory(), 'gen', subpath) def build_config(self): - """Reads and returns the project's .build_config JSON.""" + """Reads and returns the project's .build_config.json JSON.""" if not self._build_config: with open(self.build_config_path) as jsonfile: self._build_config = json.load(jsonfile) return self._build_config def get_type(self): - """Returns the target type from its .build_config.""" + """Returns the target type from its .build_config.json.""" return self.build_config()['deps_info']['type'] def proguard_enabled(self): @@ -145,12 +208,13 @@ parser.add_argument('--print-types', action='store_true', help='Print type of each target') - parser.add_argument('--print-build-config-paths', - action='store_true', - help='Print path to the .build_config of each target') + parser.add_argument( + '--print-build-config-paths', + action='store_true', + help='Print path to the .build_config.json of each target') parser.add_argument('--build', action='store_true', - help='Build all .build_config files.') + help='Build all .build_config.json files.') parser.add_argument('--type', action='append', help='Restrict to targets of given type', @@ -160,14 +224,22 @@ help='Print counts of each target type.') parser.add_argument('--proguard-enabled', action='store_true', - help='Restrict to targets that have proguard enabled') + help='Restrict to targets that have proguard enabled.') + parser.add_argument('--query', + help='A dot separated string specifying a query for a ' + 'build config json value of each target. Example: Use ' + '--query deps_info.unprocessed_jar_path to show a list ' + 'of all targets that have a non-empty deps_info dict and ' + 'non-empty "unprocessed_jar_path" value in that dict.') + parser.add_argument('-j', help='Use -j with ninja instead of autoninja.') parser.add_argument('-v', '--verbose', default=0, action='count') + parser.add_argument('-q', '--quiet', default=0, action='count') args = parser.parse_args() args.build |= bool(args.type or args.proguard_enabled or args.print_types - or args.stats) + or args.stats or args.query) - logging.basicConfig(level=logging.WARNING - (10 * args.verbose), + logging.basicConfig(level=logging.WARNING + 10 * (args.quiet - args.verbose), format='%(levelname).1s %(relativeCreated)6d %(message)s') if args.output_directory: @@ -180,8 +252,10 @@ entries = [_TargetEntry(t) for t in targets] if args.build: - logging.warning('Building %d .build_config files...', len(entries)) - _run_ninja(output_dir, [e.ninja_build_config_target for e in entries]) + logging.warning('Building %d .build_config.json files...', len(entries)) + _run_ninja(output_dir, [e.ninja_build_config_target for e in entries], + j_value=args.j, + quiet=args.quiet) if args.type: entries = [e for e in entries if e.get_type() in args.type] @@ -208,6 +282,13 @@ to_print = f'{to_print}: {e.get_type()}' elif args.print_build_config_paths: to_print = f'{to_print}: {e.build_config_path}' + elif args.query: + value = _query_json(json_dict=e.build_config(), + query=args.query, + path=e.build_config_path) + if not value: + continue + to_print = f'{to_print}: {value}' print(to_print)
diff --git a/build/android/main_dex_classes.flags b/build/android/main_dex_classes.flags index 31dbdd6..7e04756 100644 --- a/build/android/main_dex_classes.flags +++ b/build/android/main_dex_classes.flags
@@ -1,16 +1,16 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # Proguard flags for what should be kept in the main dex. Only used # during main dex list determination, not during actual proguarding. --keep @org.chromium.base.annotations.MainDex class * { +-keep @org.chromium.build.annotations.MainDex class * { *; } -keepclasseswithmembers class * { - @org.chromium.base.annotations.MainDex <methods>; + @org.chromium.build.annotations.MainDex <methods>; } # Assume all IDL-generated classes should be kept. They can't reference other @@ -29,11 +29,6 @@ *; } -# Used by tests for secondary dex extraction. --keep class android.support.v4.content.ContextCompat { - *; -} - # The following are based on $SDK_BUILD_TOOLS/mainDexClasses.rules # Ours differ in that: # 1. It omits -keeps for application / instrumentation / backupagents (these are
diff --git a/build/android/method_count.py b/build/android/method_count.py index a39a390..8556b22 100755 --- a/build/android/method_count.py +++ b/build/android/method_count.py
@@ -1,9 +1,8 @@ -#! /usr/bin/env python -# Copyright 2015 The Chromium Authors. All rights reserved. +#! /usr/bin/env python3 +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import argparse import os @@ -13,7 +12,7 @@ from pylib.dex import dex_parser -class DexStatsCollector(object): +class DexStatsCollector: """Tracks count of method/field/string/type as well as unique methods.""" def __init__(self):
diff --git a/build/android/multidex.flags b/build/android/multidex.flags deleted file mode 100644 index e3543c1..0000000 --- a/build/android/multidex.flags +++ /dev/null
@@ -1,8 +0,0 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# When multidex is enabled, need to keep the @MainDex annotation so that it -# can be used to create the main dex list. --keepattributes *Annotations* --keep @interface org.chromium.base.annotations.MainDex
diff --git a/build/android/native_flags/BUILD.gn b/build/android/native_flags/BUILD.gn index 9c5be70..3171030 100644 --- a/build/android/native_flags/BUILD.gn +++ b/build/android/native_flags/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/native_flags/argcapture.py b/build/android/native_flags/argcapture.py index 159b03a..b590fff 100755 --- a/build/android/native_flags/argcapture.py +++ b/build/android/native_flags/argcapture.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2021 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Writes arguments to a file."""
diff --git a/build/android/native_flags/empty.cc b/build/android/native_flags/empty.cc index 94aac14..29dfc78 100644 --- a/build/android/native_flags/empty.cc +++ b/build/android/native_flags/empty.cc
@@ -1,4 +1,4 @@ -// Copyright 2021 The Chromium Authors. All rights reserved. +// Copyright 2021 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/print_cipd_version.py b/build/android/print_cipd_version.py new file mode 100755 index 0000000..581295d --- /dev/null +++ b/build/android/print_cipd_version.py
@@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import pathlib +import re +import subprocess + +_DIR_SOURCE_ROOT = str(pathlib.Path(__file__).absolute().parents[2]) + + +def main(): + parser = argparse.ArgumentParser() + # Hide args set by wrappers so that using --help with the wrappers does not + # show them. + parser.add_argument('--subdir', required=True, help=argparse.SUPPRESS) + parser.add_argument('--cipd-package', required=True, help=argparse.SUPPRESS) + parser.add_argument('--git-log-url', help=argparse.SUPPRESS) + parser.add_argument('--cipd-instance', help='Uses value from DEPS by default') + args = parser.parse_args() + + if not args.cipd_instance: + args.cipd_instance = subprocess.check_output( + ['gclient', 'getdep', '-r', f'src/{args.subdir}:{args.cipd_package}'], + cwd=_DIR_SOURCE_ROOT, + text=True) + + cmd = ['cipd', 'describe', args.cipd_package, '-version', args.cipd_instance] + print(' '.join(cmd)) + output = subprocess.check_output(cmd, text=True) + print(output, end='') + if args.git_log_url: + git_hashes = re.findall(r'version:.*?@(\w+)', output) + if not git_hashes: + print('Could not find git hash from output.') + else: + # Multiple version tags exist when multiple versions have the same sha1. + last_version = git_hashes[-1] + print() + print('Recent commits:', args.git_log_url.format(last_version)) + + +if __name__ == '__main__': + main()
diff --git a/build/android/provision_devices.py b/build/android/provision_devices.py index 5fb4d93..428d9b3 100755 --- a/build/android/provision_devices.py +++ b/build/android/provision_devices.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -46,7 +46,7 @@ _TOMBSTONE_REGEX = re.compile('tombstone.*') -class _DEFAULT_TIMEOUTS(object): +class _DEFAULT_TIMEOUTS: # L can take a while to reboot after a wipe. LOLLIPOP = 600 PRE_LOLLIPOP = 180 @@ -54,7 +54,7 @@ HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP) -class _PHASES(object): +class _PHASES: WIPE = 'wipe' PROPERTIES = 'properties' FINISH = 'finish' @@ -67,7 +67,7 @@ if args.denylist_file else None) devices = [ d for d in device_utils.DeviceUtils.HealthyDevices(denylist) - if not args.emulators or d.adb.is_emulator + if not args.emulators or d.is_emulator ] if args.device: devices = [d for d in devices if d == args.device] @@ -394,14 +394,13 @@ get_date_command, as_root=True, single_line=True).replace('"', '') device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S") correct_time = datetime.datetime.strptime(strgmtime, date_format) - tdelta = (correct_time - device_time).seconds + tdelta = abs(correct_time - device_time).seconds if tdelta <= 1: logging.info('Date/time successfully set on %s', device) return True - else: - logging.error('Date mismatch. Device: %s Correct: %s', - device_time.isoformat(), correct_time.isoformat()) - return False + logging.error('Date mismatch. Device: %s Correct: %s', + device_time.isoformat(), correct_time.isoformat()) + return False # Sometimes the date is not set correctly on the devices. Retry on failure. if device.IsUserBuild():
diff --git a/build/android/pylib/__init__.py b/build/android/pylib/__init__.py index c9a4c03..2e6d65f 100644 --- a/build/android/pylib/__init__.py +++ b/build/android/pylib/__init__.py
@@ -1,13 +1,16 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import os import sys -_THIRD_PARTY_PATH = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', '..', '..', 'third_party')) +_SRC_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) + +_THIRD_PARTY_PATH = os.path.join(_SRC_PATH, 'third_party') _CATAPULT_PATH = os.path.join(_THIRD_PARTY_PATH, 'catapult') @@ -21,6 +24,7 @@ _TRACE2HTML_PATH = os.path.join(_CATAPULT_PATH, 'tracing') +_BUILD_UTIL_PATH = os.path.join(_SRC_PATH, 'build', 'util') if _DEVIL_PATH not in sys.path: sys.path.append(_DEVIL_PATH) @@ -36,3 +40,6 @@ if _SIX_PATH not in sys.path: sys.path.append(_SIX_PATH) + +if _BUILD_UTIL_PATH not in sys.path: + sys.path.insert(0, _BUILD_UTIL_PATH)
diff --git a/build/android/pylib/android/__init__.py b/build/android/pylib/android/__init__.py index a67c350..68130d5 100644 --- a/build/android/pylib/android/__init__.py +++ b/build/android/pylib/android/__init__.py
@@ -1,3 +1,3 @@ -# Copyright (c) 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/android/logcat_symbolizer.py b/build/android/pylib/android/logcat_symbolizer.py index 720629b..84d812c 100644 --- a/build/android/pylib/android/logcat_symbolizer.py +++ b/build/android/pylib/android/logcat_symbolizer.py
@@ -1,7 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import re from devil.android import logcat_monitor
diff --git a/build/android/pylib/base/__init__.py b/build/android/pylib/base/__init__.py index 96196cf..5ffa284 100644 --- a/build/android/pylib/base/__init__.py +++ b/build/android/pylib/base/__init__.py
@@ -1,3 +1,3 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/base/base_test_result.py b/build/android/pylib/base/base_test_result.py index 03f00f2..e5fbab5 100644 --- a/build/android/pylib/base/base_test_result.py +++ b/build/android/pylib/base/base_test_result.py
@@ -1,36 +1,32 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Module containing base test results classes.""" -from __future__ import absolute_import + +import functools +import sys import threading -import six + +from lib.results import result_types # pylint: disable=import-error + +# This must match the source adding the suffix: bit.ly/3Zmwwyx +_MULTIPROCESS_SUFFIX = '__multiprocess_mode' -class ResultType(object): - """Class enumerating test types.""" - # The test passed. - PASS = 'SUCCESS' +class ResultType: + """Class enumerating test types. - # The test was intentionally skipped. - SKIP = 'SKIPPED' - - # The test failed. - FAIL = 'FAILURE' - - # The test caused the containing process to crash. - CRASH = 'CRASH' - - # The test timed out. - TIMEOUT = 'TIMEOUT' - - # The test ran, but we couldn't determine what happened. - UNKNOWN = 'UNKNOWN' - - # The test did not run. - NOTRUN = 'NOTRUN' + Wraps the results defined in //build/util/lib/results/. + """ + PASS = result_types.PASS + SKIP = result_types.SKIP + FAIL = result_types.FAIL + CRASH = result_types.CRASH + TIMEOUT = result_types.TIMEOUT + UNKNOWN = result_types.UNKNOWN + NOTRUN = result_types.NOTRUN @staticmethod def GetTypes(): @@ -40,10 +36,11 @@ ResultType.NOTRUN] -class BaseTestResult(object): +@functools.total_ordering +class BaseTestResult: """Base class for a single test result.""" - def __init__(self, name, test_type, duration=0, log=''): + def __init__(self, name, test_type, duration=0, log='', failure_reason=None): """Construct a BaseTestResult. Args: @@ -58,7 +55,9 @@ self._test_type = test_type self._duration = duration self._log = log + self._failure_reason = failure_reason self._links = {} + self._webview_multiprocess_mode = name.endswith(_MULTIPROCESS_SUFFIX) def __str__(self): return self._name @@ -66,9 +65,11 @@ def __repr__(self): return self._name - def __cmp__(self, other): - # pylint: disable=W0212 - return cmp(self._name, other._name) + def __eq__(self, other): + return self.GetName() == other.GetName() + + def __lt__(self, other): + return self.GetName() == other.GetName() def __hash__(self): return hash(self._name) @@ -85,6 +86,16 @@ """Get the test name.""" return self._name + def GetNameForResultSink(self): + """Get the test name to be reported to resultsink.""" + raw_name = self.GetName() + if self._webview_multiprocess_mode: + assert raw_name.endswith( + _MULTIPROCESS_SUFFIX + ), 'multiprocess mode test raw name should have the corresponding suffix' + return raw_name[:-len(_MULTIPROCESS_SUFFIX)] + return raw_name + def SetType(self, test_type): """Set the test result type.""" assert test_type in ResultType.GetTypes() @@ -106,6 +117,22 @@ """Get the test log.""" return self._log + def SetFailureReason(self, failure_reason): + """Set the reason the test failed. + + This should be the first failure the test encounters and exclude any stack + trace. + """ + self._failure_reason = failure_reason + + def GetFailureReason(self): + """Get the reason the test failed. + + Returns None if the test did not fail or if the reason the test failed is + unknown. + """ + return self._failure_reason + def SetLink(self, name, link_url): """Set link with test result data.""" self._links[name] = link_url @@ -114,8 +141,14 @@ """Get dict containing links to test result data.""" return self._links + def GetVariantForResultSink(self): + """Get the variant dict to be reported to result sink.""" + if self._webview_multiprocess_mode: + return {'webview_multiprocess_mode': 'Yes'} + return None -class TestRunResults(object): + +class TestRunResults: """Set of results for a test run.""" def __init__(self): @@ -141,7 +174,10 @@ log = t.GetLog() if log: s.append('[%s] %s:' % (test_type, t)) - s.append(six.text_type(log, 'utf-8')) + s.append(log) + if sys.version_info.major == 2: + decoded = [u.decode(encoding='utf-8', errors='ignore') for u in s] + return '\n'.join(decoded) return '\n'.join(s) def GetGtestForm(self):
diff --git a/build/android/pylib/base/base_test_result_unittest.py b/build/android/pylib/base/base_test_result_unittest.py index 31a1f60..955a59f 100644 --- a/build/android/pylib/base/base_test_result_unittest.py +++ b/build/android/pylib/base/base_test_result_unittest.py
@@ -1,10 +1,10 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Unittests for TestRunResults.""" -from __future__ import absolute_import + import unittest from pylib.base.base_test_result import BaseTestResult
diff --git a/build/android/pylib/base/environment.py b/build/android/pylib/base/environment.py index 744c392..0c4326a 100644 --- a/build/android/pylib/base/environment.py +++ b/build/android/pylib/base/environment.py
@@ -1,8 +1,11 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# TODO(1262303): After Telemetry is supported by python3 we can remove +# object inheritance from this script. +# pylint: disable=useless-object-inheritance class Environment(object): """An environment in which tests can be run.
diff --git a/build/android/pylib/base/environment_factory.py b/build/android/pylib/base/environment_factory.py index 2ff93f3..377e0f7 100644 --- a/build/android/pylib/base/environment_factory.py +++ b/build/android/pylib/base/environment_factory.py
@@ -1,8 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + from pylib import constants from pylib.local.device import local_device_environment from pylib.local.machine import local_machine_environment @@ -23,12 +23,13 @@ if args.avd_config: if not local_emulator_environment: error_func('emulator environment requested but not available.') + raise RuntimeError('error_func must call exit inside.') return local_emulator_environment.LocalEmulatorEnvironment( args, output_manager, error_func) return local_device_environment.LocalDeviceEnvironment( args, output_manager, error_func) - else: - return local_machine_environment.LocalMachineEnvironment( - args, output_manager, error_func) + return local_machine_environment.LocalMachineEnvironment( + args, output_manager, error_func) error_func('Unable to create %s environment.' % args.environment) + raise RuntimeError('error_func must call exit inside.')
diff --git a/build/android/pylib/base/mock_environment.py b/build/android/pylib/base/mock_environment.py index d7293c7..c537f05 100644 --- a/build/android/pylib/base/mock_environment.py +++ b/build/android/pylib/base/mock_environment.py
@@ -1,8 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + from pylib.base import environment import mock # pylint: disable=import-error
diff --git a/build/android/pylib/base/mock_test_instance.py b/build/android/pylib/base/mock_test_instance.py index 19a1d7e..547a84b 100644 --- a/build/android/pylib/base/mock_test_instance.py +++ b/build/android/pylib/base/mock_test_instance.py
@@ -1,8 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + from pylib.base import test_instance import mock # pylint: disable=import-error
diff --git a/build/android/pylib/base/output_manager.py b/build/android/pylib/base/output_manager.py index 53e5aea..f562be8 100644 --- a/build/android/pylib/base/output_manager.py +++ b/build/android/pylib/base/output_manager.py
@@ -1,8 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import contextlib import logging import os @@ -11,14 +11,14 @@ from devil.utils import reraiser_thread -class Datatype(object): +class Datatype: HTML = 'text/html' JSON = 'application/json' PNG = 'image/png' TEXT = 'text/plain' -class OutputManager(object): +class OutputManager: def __init__(self): """OutputManager Constructor. @@ -51,26 +51,38 @@ if not self._allow_upload: raise Exception('Must run |SetUp| before attempting to upload!') - f = self._CreateArchivedFile(out_filename, out_subdir, datatype) + f = self.CreateArchivedFile(out_filename, out_subdir, datatype) try: yield f finally: - f.PrepareArchive() + self.ArchiveArchivedFile(f, delete=True) - def archive(): - try: - f.Archive() - finally: - f.Delete() - - thread = reraiser_thread.ReraiserThread(func=archive) - thread.start() - self._thread_group.Add(thread) + def CreateArchivedFile(self, out_filename, out_subdir, + datatype=Datatype.TEXT): + """Returns an instance of ArchivedFile.""" + return self._CreateArchivedFile(out_filename, out_subdir, datatype) def _CreateArchivedFile(self, out_filename, out_subdir, datatype): - """Returns an instance of ArchivedFile.""" raise NotImplementedError + def ArchiveArchivedFile(self, archived_file, delete=False): + """Archive an ArchivedFile instance and optionally delete it.""" + if not isinstance(archived_file, ArchivedFile): + raise Exception('Excepting an instance of ArchivedFile, got %s.' % + type(archived_file)) + archived_file.PrepareArchive() + + def archive(): + try: + archived_file.Archive() + finally: + if delete: + archived_file.Delete() + + thread = reraiser_thread.ReraiserThread(func=archive) + thread.start() + self._thread_group.Add(thread) + def SetUp(self): self._allow_upload = True self._thread_group = reraiser_thread.ReraiserThreadGroup() @@ -88,20 +100,29 @@ self.TearDown() -class ArchivedFile(object): +class ArchivedFile: def __init__(self, out_filename, out_subdir, datatype): self._out_filename = out_filename self._out_subdir = out_subdir self._datatype = datatype - self._f = tempfile.NamedTemporaryFile(delete=False) + mode = 'w+' + if datatype == Datatype.PNG: + mode = 'w+b' + self._f = tempfile.NamedTemporaryFile(mode=mode, delete=False) self._ready_to_archive = False @property def name(self): return self._f.name + def fileno(self, *args, **kwargs): + if self._ready_to_archive: + raise Exception('Cannot retrieve the integer file descriptor ' + 'after archiving has begun!') + return self._f.fileno(*args, **kwargs) + def write(self, *args, **kwargs): if self._ready_to_archive: raise Exception('Cannot write to file after archiving has begun!') @@ -141,7 +162,6 @@ content addressed files. This is called after the file is written but before archiving has begun. """ - pass def Archive(self): """Archives file."""
diff --git a/build/android/pylib/base/output_manager_factory.py b/build/android/pylib/base/output_manager_factory.py index 891692d..378a89a 100644 --- a/build/android/pylib/base/output_manager_factory.py +++ b/build/android/pylib/base/output_manager_factory.py
@@ -1,8 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + from pylib import constants from pylib.output import local_output_manager from pylib.output import remote_output_manager @@ -13,6 +13,5 @@ if args.local_output or not local_utils.IsOnSwarming(): return local_output_manager.LocalOutputManager( output_dir=constants.GetOutDirectory()) - else: - return remote_output_manager.RemoteOutputManager( - bucket=args.gs_results_bucket) + return remote_output_manager.RemoteOutputManager( + bucket=args.gs_results_bucket)
diff --git a/build/android/pylib/base/output_manager_test_case.py b/build/android/pylib/base/output_manager_test_case.py index 7b7e462..7349fd1 100644 --- a/build/android/pylib/base/output_manager_test_case.py +++ b/build/android/pylib/base/output_manager_test_case.py
@@ -1,8 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import os.path import unittest
diff --git a/build/android/pylib/base/result_sink.py b/build/android/pylib/base/result_sink.py deleted file mode 100644 index 424b873..0000000 --- a/build/android/pylib/base/result_sink.py +++ /dev/null
@@ -1,163 +0,0 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -from __future__ import absolute_import -import base64 -import cgi -import json -import os - -import six -if not six.PY2: - import html # pylint: disable=import-error - -from pylib.base import base_test_result -import requests # pylint: disable=import-error - -# Comes from luci/resultdb/pbutil/test_result.go -MAX_REPORT_LEN = 4 * 1024 - -# Maps base_test_results to the luci test-result.proto. -# https://godoc.org/go.chromium.org/luci/resultdb/proto/v1#TestStatus -RESULT_MAP = { - base_test_result.ResultType.UNKNOWN: 'ABORT', - base_test_result.ResultType.PASS: 'PASS', - base_test_result.ResultType.FAIL: 'FAIL', - base_test_result.ResultType.CRASH: 'CRASH', - base_test_result.ResultType.TIMEOUT: 'ABORT', - base_test_result.ResultType.SKIP: 'SKIP', - base_test_result.ResultType.NOTRUN: 'SKIP', -} - - -def TryInitClient(): - """Tries to initialize a result_sink_client object. - - Assumes that rdb stream is already running. - - Returns: - A ResultSinkClient for the result_sink server else returns None. - """ - try: - with open(os.environ['LUCI_CONTEXT']) as f: - sink = json.load(f)['result_sink'] - return ResultSinkClient(sink) - except KeyError: - return None - - -class ResultSinkClient(object): - """A class to store the sink's post configurations and make post requests. - - This assumes that the rdb stream has been called already and that the - server is listening. - """ - def __init__(self, context): - base_url = 'http://%s/prpc/luci.resultsink.v1.Sink' % context['address'] - self.test_results_url = base_url + '/ReportTestResults' - self.report_artifacts_url = base_url + '/ReportInvocationLevelArtifacts' - - self.headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'Authorization': 'ResultSink %s' % context['auth_token'], - } - - def Post(self, test_id, status, duration, test_log, test_file, - artifacts=None): - """Uploads the test result to the ResultSink server. - - This assumes that the rdb stream has been called already and that - server is ready listening. - - Args: - test_id: A string representing the test's name. - status: A string representing if the test passed, failed, etc... - duration: An int representing time in ms. - test_log: A string representing the test's output. - test_file: A string representing the file location of the test. - artifacts: An optional dict of artifacts to attach to the test. - - Returns: - N/A - """ - assert status in RESULT_MAP - expected = status in (base_test_result.ResultType.PASS, - base_test_result.ResultType.SKIP) - result_db_status = RESULT_MAP[status] - - # Slightly smaller to allow addition of <pre> tags and message. - report_check_size = MAX_REPORT_LEN - 45 - if six.PY2: - test_log_escaped = cgi.escape(test_log) - else: - test_log_escaped = html.escape(test_log) - if len(test_log_escaped) > report_check_size: - test_log_formatted = ('<pre>' + test_log_escaped[:report_check_size] + - '...Full output in Artifact.</pre>') - else: - test_log_formatted = '<pre>' + test_log_escaped + '</pre>' - - tr = { - 'expected': - expected, - 'status': - result_db_status, - 'summaryHtml': - test_log_formatted, - 'tags': [ - { - 'key': 'test_name', - 'value': test_id, - }, - { - # Status before getting mapped to result_db statuses. - 'key': 'android_test_runner_status', - 'value': status, - } - ], - 'testId': - test_id, - } - artifacts = artifacts or {} - if len(test_log_escaped) > report_check_size: - # Upload the original log without any modifications. - b64_log = six.ensure_str(base64.b64encode(six.ensure_binary(test_log))) - artifacts.update({'Test Log': {'contents': b64_log}}) - if artifacts: - tr['artifacts'] = artifacts - - if duration is not None: - # Duration must be formatted to avoid scientific notation in case - # number is too small or too large. Result_db takes seconds, not ms. - # Need to use float() otherwise it does substitution first then divides. - tr['duration'] = '%.9fs' % float(duration / 1000.0) - - if test_file and str(test_file).startswith('//'): - tr['testMetadata'] = { - 'name': test_id, - 'location': { - 'file_name': test_file, - 'repo': 'https://chromium.googlesource.com/chromium/src', - } - } - - res = requests.post(url=self.test_results_url, - headers=self.headers, - data=json.dumps({'testResults': [tr]})) - res.raise_for_status() - - def ReportInvocationLevelArtifacts(self, artifacts): - """Uploads invocation-level artifacts to the ResultSink server. - - This is for artifacts that don't apply to a single test but to the test - invocation as a whole (eg: system logs). - - Args: - artifacts: A dict of artifacts to attach to the invocation. - """ - req = {'artifacts': artifacts} - res = requests.post(url=self.report_artifacts_url, - headers=self.headers, - data=json.dumps(req)) - res.raise_for_status()
diff --git a/build/android/pylib/base/test_collection.py b/build/android/pylib/base/test_collection.py index 83b3bf8..3b9fec0 100644 --- a/build/android/pylib/base/test_collection.py +++ b/build/android/pylib/base/test_collection.py
@@ -1,11 +1,12 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import threading -class TestCollection(object): + +class TestCollection: """A threadsafe collection of tests. Args:
diff --git a/build/android/pylib/base/test_exception.py b/build/android/pylib/base/test_exception.py index c98d2cb..6dd31cd 100644 --- a/build/android/pylib/base/test_exception.py +++ b/build/android/pylib/base/test_exception.py
@@ -1,8 +1,7 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. class TestException(Exception): """Base class for exceptions thrown by the test runner.""" - pass
diff --git a/build/android/pylib/base/test_instance.py b/build/android/pylib/base/test_instance.py index 7b1099c..9a4e922 100644 --- a/build/android/pylib/base/test_instance.py +++ b/build/android/pylib/base/test_instance.py
@@ -1,9 +1,9 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -class TestInstance(object): +class TestInstance: """A type of test. This is expected to handle all logic that is test-type specific but
diff --git a/build/android/pylib/base/test_instance_factory.py b/build/android/pylib/base/test_instance_factory.py index f47242a..3b12974 100644 --- a/build/android/pylib/base/test_instance_factory.py +++ b/build/android/pylib/base/test_instance_factory.py
@@ -1,8 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + from pylib.gtest import gtest_test_instance from pylib.instrumentation import instrumentation_test_instance from pylib.junit import junit_test_instance @@ -15,12 +15,13 @@ if args.command == 'gtest': return gtest_test_instance.GtestTestInstance( args, device_dependencies.GetDataDependencies, error_func) - elif args.command == 'instrumentation': + if args.command == 'instrumentation': return instrumentation_test_instance.InstrumentationTestInstance( args, device_dependencies.GetDataDependencies, error_func) - elif args.command == 'junit': + if args.command == 'junit': return junit_test_instance.JunitTestInstance(args, error_func) - elif args.command == 'monkey': + if args.command == 'monkey': return monkey_test_instance.MonkeyTestInstance(args, error_func) error_func('Unable to create %s test instance.' % args.command) + raise RuntimeError('error_func must call exit inside.')
diff --git a/build/android/pylib/base/test_run.py b/build/android/pylib/base/test_run.py index fc72d3a..36aca96 100644 --- a/build/android/pylib/base/test_run.py +++ b/build/android/pylib/base/test_run.py
@@ -1,9 +1,9 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -class TestRun(object): +class TestRun: """An execution of a particular test on a particular device. This is expected to handle all logic that is specific to the combination of @@ -27,15 +27,20 @@ def SetUp(self): raise NotImplementedError - def RunTests(self, results): + def RunTests(self, results, raw_logs_fh=None): """Runs Tests and populates |results|. Args: results: An array that should be populated with |base_test_result.TestRunResults| objects. + raw_logs_fh: An optional file handle to write raw logs to. """ raise NotImplementedError + def GetTestsForListing(self): + """Returns a list of test names.""" + raise NotImplementedError + def TearDown(self): raise NotImplementedError
diff --git a/build/android/pylib/base/test_run_factory.py b/build/android/pylib/base/test_run_factory.py index 35d5494..5806a4f 100644 --- a/build/android/pylib/base/test_run_factory.py +++ b/build/android/pylib/base/test_run_factory.py
@@ -1,8 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + from pylib.gtest import gtest_test_instance from pylib.instrumentation import instrumentation_test_instance from pylib.junit import junit_test_instance @@ -34,3 +34,4 @@ error_func('Unable to create test run for %s tests in %s environment' % (str(test_instance), str(env))) + raise RuntimeError('error_func must call exit inside.')
diff --git a/build/android/pylib/base/test_server.py b/build/android/pylib/base/test_server.py index 763e121..d1fda4b 100644 --- a/build/android/pylib/base/test_server.py +++ b/build/android/pylib/base/test_server.py
@@ -1,8 +1,9 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -class TestServer(object): + +class TestServer: """Base class for any server that needs to be set up for the tests.""" def __init__(self, *args, **kwargs):
diff --git a/build/android/pylib/constants/__init__.py b/build/android/pylib/constants/__init__.py index 2d1be26..cf57d9f 100644 --- a/build/android/pylib/constants/__init__.py +++ b/build/android/pylib/constants/__init__.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,12 +8,10 @@ # pylint: disable=W0212 -from __future__ import absolute_import -import collections + import glob import logging import os -import subprocess import devil.android.sdk.keyevent from devil.android.constants import chrome @@ -27,6 +25,7 @@ DIR_SOURCE_ROOT = os.environ.get('CHECKOUT_SOURCE_ROOT', os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir))) +JAVA_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current') PACKAGE_INFO = dict(chrome.PACKAGE_INFO) PACKAGE_INFO.update({ @@ -72,9 +71,9 @@ chrome.PackageInfo('com.google.android.webview', 'com.android.cts.webkit.WebViewStartupCtsActivity', 'webview-command-line', None), - 'android_system_webview_shell': - chrome.PackageInfo('org.chromium.webview_shell', - 'org.chromium.webview_shell.WebViewBrowserActivity', + 'android_google_webview_cts_debug': + chrome.PackageInfo('com.google.android.webview.debug', + 'com.android.cts.webkit.WebViewStartupCtsActivity', 'webview-command-line', None), 'android_webview_ui_test': chrome.PackageInfo('org.chromium.webview_ui_test', @@ -115,7 +114,7 @@ SCREENSHOTS_DIR = os.path.join(DIR_SOURCE_ROOT, 'out_screenshots') -ANDROID_SDK_BUILD_TOOLS_VERSION = '30.0.1' +ANDROID_SDK_BUILD_TOOLS_VERSION = '33.0.0' ANDROID_SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'android_sdk', 'public') ANDROID_SDK_TOOLS = os.path.join(ANDROID_SDK_ROOT, @@ -151,13 +150,13 @@ 'devil.android.md5sum_test', 'devil.utils.cmd_helper_test', 'pylib.results.json_results_test', - 'pylib.utils.proguard_test', ] }, 'gyp_py_unittests': { 'path': os.path.join(DIR_SOURCE_ROOT, 'build', 'android', 'gyp'), 'test_modules': [ + 'create_unwind_table_tests', 'java_cpp_enum_tests', 'java_cpp_strings_tests', 'java_google_api_keys_tests', @@ -202,7 +201,7 @@ CheckOutputDirectory(). Typically by providing an --output-dir or --chromium-output-dir option. """ - os.environ['CHROMIUM_OUTPUT_DIR'] = output_directory + os.environ['CHROMIUM_OUTPUT_DIR'] = os.path.abspath(output_directory) # The message that is printed when the Chromium output directory cannot
diff --git a/build/android/pylib/constants/host_paths.py b/build/android/pylib/constants/host_paths.py index a38d28e..4b71264 100644 --- a/build/android/pylib/constants/host_paths.py +++ b/build/android/pylib/constants/host_paths.py
@@ -1,8 +1,8 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import contextlib import os import sys
diff --git a/build/android/pylib/constants/host_paths_unittest.py b/build/android/pylib/constants/host_paths_unittest.py index 72be4ed..3ce406f 100755 --- a/build/android/pylib/constants/host_paths_unittest.py +++ b/build/android/pylib/constants/host_paths_unittest.py
@@ -1,9 +1,9 @@ -#!/usr/bin/env python -# Copyright 2018 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import logging import os import unittest
diff --git a/build/android/pylib/content_settings.py b/build/android/pylib/content_settings.py index 3bf11bc..ddd663f 100644 --- a/build/android/pylib/content_settings.py +++ b/build/android/pylib/content_settings.py
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -11,7 +11,7 @@ """ def __init__(self, table, device): - super(ContentSettings, self).__init__() + super().__init__() self._table = table self._device = device @@ -23,7 +23,7 @@ return 'f' if isinstance(value, int): return 'i' - if isinstance(value, long): + if isinstance(value, int): return 'l' if isinstance(value, str): return 's'
diff --git a/build/android/pylib/device/commands/BUILD.gn b/build/android/pylib/device/commands/BUILD.gn index 13b69f6..2f02734 100644 --- a/build/android/pylib/device/commands/BUILD.gn +++ b/build/android/pylib/device/commands/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/device/commands/java/src/org/chromium/android/commands/unzip/Unzip.java b/build/android/pylib/device/commands/java/src/org/chromium/android/commands/unzip/Unzip.java index cf0ff67..b322e32 100644 --- a/build/android/pylib/device/commands/java/src/org/chromium/android/commands/unzip/Unzip.java +++ b/build/android/pylib/device/commands/java/src/org/chromium/android/commands/unzip/Unzip.java
@@ -1,4 +1,4 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/pylib/device_settings.py b/build/android/pylib/device_settings.py index ab4ad1b..2e1abe8 100644 --- a/build/android/pylib/device_settings.py +++ b/build/android/pylib/device_settings.py
@@ -1,8 +1,10 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import logging +import six from pylib import content_settings @@ -33,7 +35,7 @@ for key, value in key_value: settings[key] = value logging.info('\n%s %s', table, (80 - len(table)) * '-') - for key, value in sorted(settings.iteritems()): + for key, value in sorted(six.iteritems(settings)): logging.info('\t%s: %s', key, value)
diff --git a/build/android/pylib/dex/__init__.py b/build/android/pylib/dex/__init__.py index 4a12e35..401c54b 100644 --- a/build/android/pylib/dex/__init__.py +++ b/build/android/pylib/dex/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/dex/dex_parser.py b/build/android/pylib/dex/dex_parser.py index 3f2ed6f..9002917 100755 --- a/build/android/pylib/dex/dex_parser.py +++ b/build/android/pylib/dex/dex_parser.py
@@ -1,8 +1,7 @@ -#!/usr/bin/env python -# Copyright 2019 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Utilities for optimistically parsing dex files. This file is not meant to provide a generic tool for analyzing dex files. @@ -10,8 +9,6 @@ is provided, but it does not include error handling or validation. """ -from __future__ import print_function - import argparse import collections import errno @@ -65,7 +62,7 @@ 'annotations_off,class_data_off,static_values_off') -class _MemoryItemList(object): +class _MemoryItemList: """Base class for repeated memory items.""" def __init__(self, @@ -91,7 +88,7 @@ self.offset = offset self.size = size reader.Seek(first_item_offset or offset) - self._items = [factory(reader) for _ in xrange(size)] + self._items = [factory(reader) for _ in range(size)] if alignment: reader.AlignUpTo(alignment) @@ -116,43 +113,38 @@ class _TypeIdItemList(_MemoryItemList): - def __init__(self, reader, offset, size): factory = lambda x: _TypeIdItem(x.ReadUInt()) - super(_TypeIdItemList, self).__init__(reader, offset, size, factory) + super().__init__(reader, offset, size, factory) class _ProtoIdItemList(_MemoryItemList): - def __init__(self, reader, offset, size): factory = lambda x: _ProtoIdItem(x.ReadUInt(), x.ReadUInt(), x.ReadUInt()) - super(_ProtoIdItemList, self).__init__(reader, offset, size, factory) + super().__init__(reader, offset, size, factory) class _MethodIdItemList(_MemoryItemList): - def __init__(self, reader, offset, size): factory = ( lambda x: _MethodIdItem(x.ReadUShort(), x.ReadUShort(), x.ReadUInt())) - super(_MethodIdItemList, self).__init__(reader, offset, size, factory) + super().__init__(reader, offset, size, factory) class _StringItemList(_MemoryItemList): - def __init__(self, reader, offset, size): reader.Seek(offset) - string_item_offsets = iter([reader.ReadUInt() for _ in xrange(size)]) + string_item_offsets = iter([reader.ReadUInt() for _ in range(size)]) def factory(x): data_offset = next(string_item_offsets) string = x.ReadString(data_offset) return _StringDataItem(len(string), string) - super(_StringItemList, self).__init__(reader, offset, size, factory) + super().__init__(reader, offset, size, factory) class _TypeListItem(_MemoryItemList): - def __init__(self, reader): offset = reader.Tell() size = reader.ReadUInt() @@ -160,35 +152,31 @@ # This is necessary because we need to extract the size of the type list # (in other cases the list size is provided in the header). first_item_offset = reader.Tell() - super(_TypeListItem, self).__init__( - reader, - offset, - size, - factory, - alignment=4, - first_item_offset=first_item_offset) + super().__init__(reader, + offset, + size, + factory, + alignment=4, + first_item_offset=first_item_offset) class _TypeListItemList(_MemoryItemList): - def __init__(self, reader, offset, size): - super(_TypeListItemList, self).__init__(reader, offset, size, _TypeListItem) + super().__init__(reader, offset, size, _TypeListItem) class _ClassDefItemList(_MemoryItemList): - def __init__(self, reader, offset, size): reader.Seek(offset) def factory(x): return _ClassDefItem(*(x.ReadUInt() - for _ in xrange(len(_ClassDefItem._fields)))) + for _ in range(len(_ClassDefItem._fields)))) - super(_ClassDefItemList, self).__init__(reader, offset, size, factory) + super().__init__(reader, offset, size, factory) -class _DexMapItem(object): - +class _DexMapItem: def __init__(self, reader): self.type = reader.ReadUShort() reader.ReadUShort() @@ -200,7 +188,7 @@ self.type, self.size, self.offset) -class _DexMapList(object): +class _DexMapList: # Full list of type codes: # https://source.android.com/devices/tech/dalvik/dex-format#type-codes TYPE_TYPE_LIST = 0x1001 @@ -209,7 +197,7 @@ self._map = {} reader.Seek(offset) self._size = reader.ReadUInt() - for _ in xrange(self._size): + for _ in range(self._size): item = _DexMapItem(reader) self._map[item.type] = item @@ -223,8 +211,7 @@ return '_DexMapList(size={}, items={})'.format(self._size, self._map) -class _DexReader(object): - +class _DexReader: def __init__(self, data): self._data = data self._pos = 0 @@ -299,7 +286,7 @@ self.Seek(offset) ret = '' - for _ in xrange(string_length): + for _ in range(string_length): a = self.ReadUByte() if a == 0: raise _MUTf8DecodeError('Early string termination encountered', @@ -319,8 +306,7 @@ code = ((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f) else: raise _MUTf8DecodeError('Bad byte', string_length, offset) - - ret += unichr(code) + ret += chr(code) if self.ReadUByte() != 0x00: raise _MUTf8DecodeError('Expected string termination', string_length, @@ -330,14 +316,13 @@ class _MUTf8DecodeError(Exception): - def __init__(self, message, length, offset): message += ' (decoded string length: {}, string data offset: {:#x})'.format( length, offset) - super(_MUTf8DecodeError, self).__init__(message) + super().__init__(message) -class DexFile(object): +class DexFile: """Represents a single dex file. Parses and exposes access to dex file structure and contents, as described @@ -380,20 +365,25 @@ self.map_list = _DexMapList(self.reader, self.header.map_off) self.type_item_list = _TypeIdItemList(self.reader, self.header.type_ids_off, self.header.type_ids_size) - self.proto_item_list = _ProtoIdItemList( - self.reader, self.header.proto_ids_off, self.header.proto_ids_size) - self.method_item_list = _MethodIdItemList( - self.reader, self.header.method_ids_off, self.header.method_ids_size) - self.string_item_list = _StringItemList( - self.reader, self.header.string_ids_off, self.header.string_ids_size) - self.class_def_item_list = _ClassDefItemList( - self.reader, self.header.class_defs_off, self.header.class_defs_size) + self.proto_item_list = _ProtoIdItemList(self.reader, + self.header.proto_ids_off, + self.header.proto_ids_size) + self.method_item_list = _MethodIdItemList(self.reader, + self.header.method_ids_off, + self.header.method_ids_size) + self.string_item_list = _StringItemList(self.reader, + self.header.string_ids_off, + self.header.string_ids_size) + self.class_def_item_list = _ClassDefItemList(self.reader, + self.header.class_defs_off, + self.header.class_defs_size) type_list_key = _DexMapList.TYPE_TYPE_LIST if type_list_key in self.map_list: map_list_item = self.map_list[type_list_key] - self.type_list_item_list = _TypeListItemList( - self.reader, map_list_item.offset, map_list_item.size) + self.type_list_item_list = _TypeListItemList(self.reader, + map_list_item.offset, + map_list_item.size) else: self.type_list_item_list = _TypeListItemList(self.reader, 0, 0) self._type_lists_by_offset = { @@ -417,10 +407,9 @@ @staticmethod def ResolveClassAccessFlags(access_flags): - return tuple( - flag_string - for flag, flag_string in DexFile._CLASS_ACCESS_FLAGS.iteritems() - if flag & access_flags) + return tuple(flag_string + for flag, flag_string in DexFile._CLASS_ACCESS_FLAGS.items() + if flag & access_flags) def IterMethodSignatureParts(self): """Yields the string components of dex methods in a dex file. @@ -453,8 +442,7 @@ return '\n'.join(str(item) for item in items) -class _DumpCommand(object): - +class _DumpCommand: def __init__(self, dexfile): self._dexfile = dexfile @@ -463,7 +451,6 @@ class _DumpMethods(_DumpCommand): - def Run(self): for parts in self._dexfile.IterMethodSignatureParts(): class_type, return_type, method_name, parameter_types = parts @@ -472,7 +459,6 @@ class _DumpStrings(_DumpCommand): - def Run(self): for string_item in self._dexfile.string_item_list: # Some strings are likely to be non-ascii (vs. methods/classes). @@ -480,7 +466,6 @@ class _DumpClasses(_DumpCommand): - def Run(self): for class_item in self._dexfile.class_def_item_list: class_string = self._dexfile.GetTypeString(class_item.class_idx) @@ -493,7 +478,6 @@ class _DumpSummary(_DumpCommand): - def Run(self): print(self._dexfile) @@ -517,14 +501,13 @@ def main(): parser = argparse.ArgumentParser(description='Dump dex contents to stdout.') - parser.add_argument( - 'input', help='Input (.dex, .jar, .zip, .aab, .apk) file path.') - parser.add_argument( - 'item', - choices=('methods', 'strings', 'classes', 'summary'), - help='Item to dump', - nargs='?', - default='summary') + parser.add_argument('input', + help='Input (.dex, .jar, .zip, .aab, .apk) file path.') + parser.add_argument('item', + choices=('methods', 'strings', 'classes', 'summary'), + help='Item to dump', + nargs='?', + default='summary') args = parser.parse_args() if os.path.splitext(args.input)[1] in ('.apk', '.jar', '.zip', '.aab'): @@ -541,7 +524,7 @@ _DumpDexItems(z.read(path), path, args.item) else: - with open(args.input) as f: + with open(args.input, 'rb') as f: _DumpDexItems(f.read(), args.input, args.item)
diff --git a/build/android/pylib/gtest/__init__.py b/build/android/pylib/gtest/__init__.py index 96196cf..5ffa284 100644 --- a/build/android/pylib/gtest/__init__.py +++ b/build/android/pylib/gtest/__init__.py
@@ -1,3 +1,3 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/gtest/filter/unit_tests_disabled b/build/android/pylib/gtest/filter/unit_tests_disabled index 97811c8..c8564bf 100644 --- a/build/android/pylib/gtest/filter/unit_tests_disabled +++ b/build/android/pylib/gtest/filter/unit_tests_disabled
@@ -47,23 +47,13 @@ SQLiteServerBoundCertStoreTest.TestUpgradeV1 SQLiteServerBoundCertStoreTest.TestUpgradeV2 -ProfileSyncComponentsFactoryImplTest.* PermissionsTest.GetWarningMessages_Plugins ImageOperations.ResizeShouldAverageColors -# crbug.com/139643 -VariationsUtilTest.DisableAfterInitialization -VariationsUtilTest.AssociateGoogleVariationID -VariationsUtilTest.NoAssociation - # crbug.com/141473 AutofillManagerTest.UpdatePasswordSyncState AutofillManagerTest.UpdatePasswordGenerationState -# crbug.com/145843 -EntropyProviderTest.UseOneTimeRandomizationSHA1 -EntropyProviderTest.UseOneTimeRandomizationPermuted - # crbug.com/147500 ManifestTest.RestrictedKeys
diff --git a/build/android/pylib/gtest/gtest_config.py b/build/android/pylib/gtest/gtest_config.py index 3ac1955..a7b0a04 100644 --- a/build/android/pylib/gtest/gtest_config.py +++ b/build/android/pylib/gtest/gtest_config.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -22,7 +22,6 @@ 'android_webview_unittests', 'base_unittests', 'blink_unittests', - 'breakpad_unittests', 'cc_unittests', 'components_unittests', 'content_browsertests', @@ -48,7 +47,6 @@ # Tests fail in component=shared_library build, which is required for ASan. # http://crbug.com/344868 ASAN_EXCLUDED_TEST_SUITES = [ - 'breakpad_unittests', 'sandbox_linux_unittests', # The internal ASAN recipe cannot run step "unit_tests_apk", this is the
diff --git a/build/android/pylib/gtest/gtest_test_instance.py b/build/android/pylib/gtest/gtest_test_instance.py index a88c365..a62e3e4 100644 --- a/build/android/pylib/gtest/gtest_test_instance.py +++ b/build/android/pylib/gtest/gtest_test_instance.py
@@ -1,8 +1,9 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import HTMLParser + + import json import logging import os @@ -11,6 +12,7 @@ import threading import xml.etree.ElementTree +import six from devil.android import apk_helper from pylib import constants from pylib.constants import host_paths @@ -19,6 +21,7 @@ from pylib.symbols import stack_symbolizer from pylib.utils import test_filter + with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): import unittest_util # pylint: disable=import-error @@ -246,7 +249,7 @@ if not xml_content: return results - html = HTMLParser.HTMLParser() + html = six.moves.html_parser.HTMLParser() testsuites = xml.etree.ElementTree.fromstring(xml_content) for testsuite in testsuites: @@ -276,17 +279,25 @@ json_data = json.loads(json_content) - openstack = json_data['tests'].items() + openstack = list(json_data['tests'].items()) while openstack: name, value = openstack.pop() if 'expected' in value and 'actual' in value: - result_type = base_test_result.ResultType.PASS if value[ - 'actual'] == 'PASS' else base_test_result.ResultType.FAIL + if value['actual'] == 'PASS': + result_type = base_test_result.ResultType.PASS + elif value['actual'] == 'SKIP': + result_type = base_test_result.ResultType.SKIP + elif value['actual'] == 'CRASH': + result_type = base_test_result.ResultType.CRASH + elif value['actual'] == 'TIMEOUT': + result_type = base_test_result.ResultType.TIMEOUT + else: + result_type = base_test_result.ResultType.FAIL results.append(base_test_result.BaseTestResult(name, result_type)) else: - openstack += [("%s.%s" % (name, k), v) for k, v in value.iteritems()] + openstack += [("%s.%s" % (name, k), v) for k, v in six.iteritems(value)] return results @@ -308,7 +319,7 @@ class GtestTestInstance(test_instance.TestInstance): def __init__(self, args, data_deps_delegate, error_func): - super(GtestTestInstance, self).__init__() + super().__init__() # TODO(jbudorick): Support multiple test suites. if len(args.suite_name) > 1: raise ValueError('Platform mode currently supports only 1 gtest suite') @@ -328,6 +339,7 @@ self._symbolizer = stack_symbolizer.Symbolizer(None) self._total_external_shards = args.test_launcher_total_shards self._wait_for_java_debugger = args.wait_for_java_debugger + self._use_existing_test_data = args.use_existing_test_data # GYP: if args.executable_dist_dir: @@ -374,7 +386,7 @@ error_func('Could not find apk or executable for %s' % self._suite) self._data_deps = [] - self._gtest_filter = test_filter.InitializeFilterFromArgs(args) + self._gtest_filters = test_filter.InitializeFiltersFromArgs(args) self._run_disabled = args.run_disabled self._data_deps_delegate = data_deps_delegate @@ -463,8 +475,8 @@ return self._gs_test_artifacts_bucket @property - def gtest_filter(self): - return self._gtest_filter + def gtest_filters(self): + return self._gtest_filters @property def isolated_script_test_output(self): @@ -522,6 +534,10 @@ def wait_for_java_debugger(self): return self._wait_for_java_debugger + @property + def use_existing_test_data(self): + return self._use_existing_test_data + #override def TestType(self): return 'gtest' @@ -559,8 +575,8 @@ """ gtest_filter_strings = [ self._GenerateDisabledFilterString(disabled_prefixes)] - if self._gtest_filter: - gtest_filter_strings.append(self._gtest_filter) + if self._gtest_filters: + gtest_filter_strings.extend(self._gtest_filters) filtered_test_list = test_list # This lock is required because on older versions of Python @@ -571,12 +587,16 @@ filtered_test_list = unittest_util.FilterTestNames( filtered_test_list, gtest_filter_string) - if self._run_disabled and self._gtest_filter: + if self._run_disabled and self._gtest_filters: out_filtered_test_list = list(set(test_list)-set(filtered_test_list)) for test in out_filtered_test_list: test_name_no_disabled = TestNameWithoutDisabledPrefix(test) - if test_name_no_disabled != test and unittest_util.FilterTestNames( - [test_name_no_disabled], self._gtest_filter): + if test_name_no_disabled == test: + continue + if all( + unittest_util.FilterTestNames([test_name_no_disabled], + gtest_filter) + for gtest_filter in self._gtest_filters): filtered_test_list.append(test) return filtered_test_list @@ -607,4 +627,3 @@ #override def TearDown(self): """Do nothing.""" - pass
diff --git a/build/android/pylib/gtest/gtest_test_instance_test.py b/build/android/pylib/gtest/gtest_test_instance_test.py index 1429e3d..c714ba0 100755 --- a/build/android/pylib/gtest/gtest_test_instance_test.py +++ b/build/android/pylib/gtest/gtest_test_instance_test.py
@@ -1,8 +1,9 @@ -#!/usr/bin/env vpython -# Copyright 2014 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import unittest from pylib.base import base_test_result @@ -99,10 +100,10 @@ '[ OK ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.PASS, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.PASS, actual[0].GetType()) def testParseGTestOutput_fail(self): raw_output = [ @@ -110,10 +111,10 @@ '[ FAILED ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.FAIL, actual[0].GetType()) def testParseGTestOutput_crash(self): raw_output = [ @@ -121,10 +122,10 @@ '[ CRASHED ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.CRASH, actual[0].GetType()) def testParseGTestOutput_errorCrash(self): raw_output = [ @@ -132,10 +133,10 @@ '[ERROR:blah] Currently running: FooTest.Bar', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) self.assertIsNone(actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + self.assertEqual(base_test_result.ResultType.CRASH, actual[0].GetType()) def testParseGTestOutput_fatalDcheck(self): raw_output = [ @@ -143,20 +144,20 @@ '[0324/183029.116334:FATAL:test_timeouts.cc(103)] Check failed: !init', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) self.assertIsNone(actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + self.assertEqual(base_test_result.ResultType.CRASH, actual[0].GetType()) def testParseGTestOutput_unknown(self): raw_output = [ '[ RUN ] FooTest.Bar', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(0, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(0, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.CRASH, actual[0].GetType()) def testParseGTestOutput_nonterminalUnknown(self): raw_output = [ @@ -165,15 +166,15 @@ '[ OK ] FooTest.Baz (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(2, len(actual)) + self.assertEqual(2, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(0, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType()) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(0, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.CRASH, actual[0].GetType()) - self.assertEquals('FooTest.Baz', actual[1].GetName()) - self.assertEquals(1, actual[1].GetDuration()) - self.assertEquals(base_test_result.ResultType.PASS, actual[1].GetType()) + self.assertEqual('FooTest.Baz', actual[1].GetName()) + self.assertEqual(1, actual[1].GetDuration()) + self.assertEqual(base_test_result.ResultType.PASS, actual[1].GetType()) def testParseGTestOutput_deathTestCrashOk(self): raw_output = [ @@ -182,11 +183,11 @@ '[ OK ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) + self.assertEqual(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.PASS, actual[0].GetType()) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.PASS, actual[0].GetType()) def testParseGTestOutput_typeParameterized(self): raw_output = [ @@ -194,10 +195,10 @@ '[ FAILED ] Baz/FooTest.Bar/0, where TypeParam = (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('Baz/FooTest.Bar/0', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('Baz/FooTest.Bar/0', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.FAIL, actual[0].GetType()) def testParseGTestOutput_valueParameterized(self): raw_output = [ @@ -206,10 +207,10 @@ ' where GetParam() = 4-byte object <00-00 00-00> (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('Baz/FooTest.Bar/0', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('Baz/FooTest.Bar/0', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.FAIL, actual[0].GetType()) def testParseGTestOutput_typeAndValueParameterized(self): raw_output = [ @@ -218,10 +219,10 @@ ' where TypeParam = and GetParam() = (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('Baz/FooTest.Bar/0', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('Baz/FooTest.Bar/0', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.FAIL, actual[0].GetType()) def testParseGTestOutput_skippedTest(self): raw_output = [ @@ -229,18 +230,18 @@ '[ SKIPPED ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) - self.assertEquals(1, len(actual)) - self.assertEquals('FooTest.Bar', actual[0].GetName()) - self.assertEquals(1, actual[0].GetDuration()) - self.assertEquals(base_test_result.ResultType.SKIP, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('FooTest.Bar', actual[0].GetName()) + self.assertEqual(1, actual[0].GetDuration()) + self.assertEqual(base_test_result.ResultType.SKIP, actual[0].GetType()) def testParseGTestXML_none(self): actual = gtest_test_instance.ParseGTestXML(None) - self.assertEquals([], actual) + self.assertEqual([], actual) def testParseGTestJSON_none(self): actual = gtest_test_instance.ParseGTestJSON(None) - self.assertEquals([], actual) + self.assertEqual([], actual) def testParseGTestJSON_example(self): raw_json = """ @@ -275,10 +276,41 @@ } }""" actual = gtest_test_instance.ParseGTestJSON(raw_json) - self.assertEquals(1, len(actual)) - self.assertEquals('mojom_tests.parse.ast_unittest.ASTTest.testNodeBase', - actual[0].GetName()) - self.assertEquals(base_test_result.ResultType.PASS, actual[0].GetType()) + self.assertEqual(1, len(actual)) + self.assertEqual('mojom_tests.parse.ast_unittest.ASTTest.testNodeBase', + actual[0].GetName()) + self.assertEqual(base_test_result.ResultType.PASS, actual[0].GetType()) + + def testParseGTestJSON_skippedTest_example(self): + raw_json = """ + { + "tests": { + "mojom_tests": { + "parse": { + "ast_unittest": { + "ASTTest": { + "testNodeBase": { + "expected": "SKIP", + "actual": "SKIP" + } + } + } + } + } + }, + "interrupted": false, + "path_delimiter": ".", + "version": 3, + "seconds_since_epoch": 1406662283.764424, + "num_failures_by_type": { + "SKIP": 1 + } + }""" + actual = gtest_test_instance.ParseGTestJSON(raw_json) + self.assertEqual(1, len(actual)) + self.assertEqual('mojom_tests.parse.ast_unittest.ASTTest.testNodeBase', + actual[0].GetName()) + self.assertEqual(base_test_result.ResultType.SKIP, actual[0].GetType()) def testTestNameWithoutDisabledPrefix_disabled(self): test_name_list = [ @@ -290,7 +322,7 @@ actual = gtest_test_instance \ .TestNameWithoutDisabledPrefix(test_name) expected = 'A.B' - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def testTestNameWithoutDisabledPrefix_flaky(self): test_name_list = [ @@ -302,14 +334,14 @@ actual = gtest_test_instance \ .TestNameWithoutDisabledPrefix(test_name) expected = 'A.B' - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def testTestNameWithoutDisabledPrefix_notDisabledOrFlaky(self): test_name = 'A.B' actual = gtest_test_instance \ .TestNameWithoutDisabledPrefix(test_name) expected = 'A.B' - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) if __name__ == '__main__':
diff --git a/build/android/pylib/instrumentation/__init__.py b/build/android/pylib/instrumentation/__init__.py index 96196cf..5ffa284 100644 --- a/build/android/pylib/instrumentation/__init__.py +++ b/build/android/pylib/instrumentation/__init__.py
@@ -1,3 +1,3 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/instrumentation/instrumentation_parser.py b/build/android/pylib/instrumentation/instrumentation_parser.py index dd9f9cc..700d241 100644 --- a/build/android/pylib/instrumentation/instrumentation_parser.py +++ b/build/android/pylib/instrumentation/instrumentation_parser.py
@@ -1,7 +1,8 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import logging import re @@ -33,7 +34,7 @@ _INSTR_LINE_RE = re.compile(r'^\s*INSTRUMENTATION_([A-Z_]+): (.*)$') -class InstrumentationParser(object): +class InstrumentationParser: def __init__(self, stream): """An incremental parser for the output of Android instrumentation tests.
diff --git a/build/android/pylib/instrumentation/instrumentation_parser_test.py b/build/android/pylib/instrumentation/instrumentation_parser_test.py index d664455..dccb58a 100755 --- a/build/android/pylib/instrumentation/instrumentation_parser_test.py +++ b/build/android/pylib/instrumentation/instrumentation_parser_test.py
@@ -1,11 +1,12 @@ -#!/usr/bin/env vpython -# Copyright 2015 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Unit tests for instrumentation.InstrumentationParser.""" + import unittest from pylib.instrumentation import instrumentation_parser
diff --git a/build/android/pylib/instrumentation/instrumentation_test_instance.py b/build/android/pylib/instrumentation/instrumentation_test_instance.py index 5493c36..f520879 100644 --- a/build/android/pylib/instrumentation/instrumentation_test_instance.py +++ b/build/android/pylib/instrumentation/instrumentation_test_instance.py
@@ -1,13 +1,15 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import copy import logging import os import pickle import re +import six from devil.android import apk_helper from pylib import constants from pylib.base import base_test_result @@ -20,8 +22,6 @@ from pylib.symbols import stack_symbolizer from pylib.utils import dexdump from pylib.utils import gold_utils -from pylib.utils import instrumentation_tracing -from pylib.utils import proguard from pylib.utils import shared_preference_utils from pylib.utils import test_filter @@ -36,9 +36,11 @@ _COMMAND_LINE_PARAMETER = 'cmdlinearg-parameter' _DEFAULT_ANNOTATIONS = [ 'SmallTest', 'MediumTest', 'LargeTest', 'EnormousTest', 'IntegrationTest'] +# This annotation is for disabled tests that should not be run in Test Reviver. +_DO_NOT_REVIVE_ANNOTATIONS = ['DoNotRevive', 'Manual'] _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [ 'DisabledTest', 'FlakyTest', 'Manual'] -_VALID_ANNOTATIONS = set(_DEFAULT_ANNOTATIONS + +_VALID_ANNOTATIONS = set(_DEFAULT_ANNOTATIONS + _DO_NOT_REVIVE_ANNOTATIONS + _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS) _TEST_LIST_JUNIT4_RUNNERS = [ @@ -70,15 +72,14 @@ class MissingSizeAnnotationError(test_exception.TestException): def __init__(self, class_name): - super(MissingSizeAnnotationError, self).__init__(class_name + + super().__init__( + class_name + ': Test method is missing required size annotation. Add one of: ' + ', '.join('@' + a for a in _VALID_ANNOTATIONS)) class CommandLineParameterizationException(test_exception.TestException): - - def __init__(self, msg): - super(CommandLineParameterizationException, self).__init__(msg) + pass class TestListPickleException(test_exception.TestException): @@ -186,9 +187,8 @@ if current_result: if current_result.GetType() == base_test_result.ResultType.UNKNOWN: - crashed = (result_code == _ACTIVITY_RESULT_CANCELED - and any(_NATIVE_CRASH_RE.search(l) - for l in result_bundle.itervalues())) + crashed = (result_code == _ACTIVITY_RESULT_CANCELED and any( + _NATIVE_CRASH_RE.search(l) for l in six.itervalues(result_bundle))) if crashed: current_result.SetType(base_test_result.ResultType.CRASH) @@ -204,15 +204,36 @@ def _MaybeSetLog(bundle, current_result, symbolizer, device_abi): if _BUNDLE_STACK_ID in bundle: + stack = bundle[_BUNDLE_STACK_ID] if symbolizer and device_abi: - current_result.SetLog('%s\n%s' % (bundle[_BUNDLE_STACK_ID], '\n'.join( - symbolizer.ExtractAndResolveNativeStackTraces( - bundle[_BUNDLE_STACK_ID], device_abi)))) + current_result.SetLog('%s\n%s' % (stack, '\n'.join( + symbolizer.ExtractAndResolveNativeStackTraces(stack, device_abi)))) else: - current_result.SetLog(bundle[_BUNDLE_STACK_ID]) + current_result.SetLog(stack) + + current_result.SetFailureReason(_ParseExceptionMessage(stack)) -def FilterTests(tests, filter_str=None, annotations=None, +def _ParseExceptionMessage(stack): + """Extracts the exception message from the given stack trace. + """ + # This interprets stack traces reported via InstrumentationResultPrinter: + # https://source.chromium.org/chromium/chromium/src/+/main:third_party/android_support_test_runner/runner/src/main/java/android/support/test/internal/runner/listener/InstrumentationResultPrinter.java;l=181?q=InstrumentationResultPrinter&type=cs + # This is a standard Java stack trace, of the form: + # <Result of Exception.toString()> + # at SomeClass.SomeMethod(...) + # at ... + lines = stack.split('\n') + for i, line in enumerate(lines): + if line.startswith('\tat'): + return '\n'.join(lines[0:i]) + # No call stack found, so assume everything is the exception message. + return stack + + +def FilterTests(tests, + filter_strs=None, + annotations=None, excluded_annotations=None): """Filter a list of tests @@ -220,41 +241,149 @@ tests: a list of tests. e.g. [ {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] - filter_str: googletest-style filter string. + filter_strs: list of googletest-style filter string. annotations: a dict of wanted annotations for test methods. - exclude_annotations: a dict of annotations to exclude. + excluded_annotations: a dict of annotations to exclude. Return: A list of filtered tests """ - def gtest_filter(t): - if not filter_str: - return True + + def test_names_from_pattern(combined_pattern, test_names): + patterns = combined_pattern.split(':') + + hashable_patterns = set() + filename_patterns = [] + for pattern in patterns: + if ('*' in pattern or '?' in pattern or '[' in pattern): + filename_patterns.append(pattern) + else: + hashable_patterns.add(pattern) + + filter_test_names = set( + unittest_util.FilterTestNames(test_names, ':'.join( + filename_patterns))) if len(filename_patterns) > 0 else set() + + for test_name in test_names: + if test_name in hashable_patterns: + filter_test_names.add(test_name) + + return filter_test_names + + def get_test_names(test): + test_names = set() # Allow fully-qualified name as well as an omitted package. unqualified_class_test = { - 'class': t['class'].split('.')[-1], - 'method': t['method'] + 'class': test['class'].split('.')[-1], + 'method': test['method'] } - names = [ - GetTestName(t, sep='.'), - GetTestName(unqualified_class_test, sep='.'), - GetUniqueTestName(t, sep='.') - ] - if t['is_junit4']: - names += [ - GetTestNameWithoutParameterPostfix(t, sep='.'), - GetTestNameWithoutParameterPostfix(unqualified_class_test, sep='.') - ] + test_name = GetTestName(test, sep='.') + test_names.add(test_name) - pattern_groups = filter_str.split('-') - if len(pattern_groups) > 1: - negative_filter = pattern_groups[1] - if unittest_util.FilterTestNames(names, negative_filter): - return [] + unqualified_class_test_name = GetTestName(unqualified_class_test, sep='.') + test_names.add(unqualified_class_test_name) - positive_filter = pattern_groups[0] - return unittest_util.FilterTestNames(names, positive_filter) + unique_test_name = GetUniqueTestName(test, sep='.') + test_names.add(unique_test_name) + + if test['is_junit4']: + junit4_test_name = GetTestNameWithoutParameterPostfix(test, sep='.') + test_names.add(junit4_test_name) + + unqualified_junit4_test_name = \ + GetTestNameWithoutParameterPostfix(unqualified_class_test, sep='.') + test_names.add(unqualified_junit4_test_name) + return test_names + + def get_tests_from_names(tests, test_names, tests_to_names): + ''' Returns the tests for which the given names apply + + Args: + tests: a list of tests. e.g. [ + {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, + {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] + test_names: a collection of names determining tests to return. + + Return: + A list of tests that match the given test names + ''' + filtered_tests = [] + for t in tests: + current_test_names = tests_to_names[id(t)] + + for current_test_name in current_test_names: + if current_test_name in test_names: + filtered_tests.append(t) + break + + return filtered_tests + + def remove_tests_from_names(tests, remove_test_names, tests_to_names): + ''' Returns the tests from the given list with given names removed + + Args: + tests: a list of tests. e.g. [ + {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, + {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] + remove_test_names: a collection of names determining tests to remove. + tests_to_names: a dcitionary of test ids to a collection of applicable + names for that test + + Return: + A list of tests that don't match the given test names + ''' + filtered_tests = [] + + for t in tests: + for name in tests_to_names[id(t)]: + if name in remove_test_names: + break + else: + filtered_tests.append(t) + return filtered_tests + + def gtests_filter(tests, combined_filters): + ''' Returns the tests after the combined_filters have been applied + + Args: + tests: a list of tests. e.g. [ + {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, + {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] + combined_filters: the filter string representing tests to exclude + + Return: + A list of tests that should still be included after the combined_filters + are applied to their names + ''' + + if not combined_filters: + return tests + + # Collect all test names + all_test_names = set() + tests_to_names = {} + for t in tests: + tests_to_names[id(t)] = get_test_names(t) + for name in tests_to_names[id(t)]: + all_test_names.add(name) + + for combined_filter in combined_filters: + pattern_groups = combined_filter.split('-') + negative_pattern = pattern_groups[1] if len(pattern_groups) > 1 else None + positive_pattern = pattern_groups[0] + if positive_pattern: + # Only use the test names that match the positive pattern + positive_test_names = test_names_from_pattern(positive_pattern, + all_test_names) + tests = get_tests_from_names(tests, positive_test_names, tests_to_names) + + if negative_pattern: + # Remove any test the negative filter matches + remove_names = test_names_from_pattern(negative_pattern, all_test_names) + tests = remove_tests_from_names(tests, remove_names, tests_to_names) + + return tests def annotation_filter(all_annotations): if not annotations: @@ -276,24 +405,19 @@ def annotation_value_matches(filter_av, av): if filter_av is None: return True - elif isinstance(av, dict): + if isinstance(av, dict): tav_from_dict = av['value'] # If tav_from_dict is an int, the 'in' operator breaks, so convert # filter_av and manually compare. See https://crbug.com/1019707 if isinstance(tav_from_dict, int): return int(filter_av) == tav_from_dict - else: - return filter_av in tav_from_dict - elif isinstance(av, list): + return filter_av in tav_from_dict + if isinstance(av, list): return filter_av in av return filter_av == av - filtered_tests = [] - for t in tests: - # Gtest filtering - if not gtest_filter(t): - continue - + return_tests = [] + for t in gtests_filter(tests, filter_strs): # Enforce that all tests declare their size. if not any(a in _VALID_ANNOTATIONS for a in t['annotations']): raise MissingSizeAnnotationError(GetTestName(t)) @@ -301,23 +425,9 @@ if (not annotation_filter(t['annotations']) or not excluded_annotation_filter(t['annotations'])): continue + return_tests.append(t) - filtered_tests.append(t) - - return filtered_tests - - -# TODO(yolandyan): remove this once the tests are converted to junit4 -def GetAllTestsFromJar(test_jar): - pickle_path = '%s-proguard.pickle' % test_jar - try: - tests = GetTestsFromPickle(pickle_path, os.path.getmtime(test_jar)) - except TestListPickleException as e: - logging.info('Could not get tests from pickle: %s', e) - logging.info('Getting tests from JAR via proguard.') - tests = _GetTestsFromProguard(test_jar) - SaveTestsToPickle(pickle_path, tests) - return tests + return return_tests def GetAllTestsFromApk(test_apk): @@ -331,74 +441,59 @@ SaveTestsToPickle(pickle_path, tests) return tests + def GetTestsFromPickle(pickle_path, test_mtime): if not os.path.exists(pickle_path): raise TestListPickleException('%s does not exist.' % pickle_path) if os.path.getmtime(pickle_path) <= test_mtime: raise TestListPickleException('File is stale: %s' % pickle_path) - with open(pickle_path, 'r') as f: + with open(pickle_path, 'rb') as f: pickle_data = pickle.load(f) if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION: raise TestListPickleException('PICKLE_FORMAT_VERSION has changed.') return pickle_data['TEST_METHODS'] -# TODO(yolandyan): remove this once the test listing from java runner lands -@instrumentation_tracing.no_tracing -def _GetTestsFromProguard(jar_path): - p = proguard.Dump(jar_path) - class_lookup = dict((c['class'], c) for c in p['classes']) - - def is_test_class(c): - return c['class'].endswith('Test') - - def is_test_method(m): - return m['method'].startswith('test') - - def recursive_class_annotations(c): - s = c['superclass'] - if s in class_lookup: - a = recursive_class_annotations(class_lookup[s]) - else: - a = {} - a.update(c['annotations']) - return a - - def stripped_test_class(c): - return { - 'class': c['class'], - 'annotations': recursive_class_annotations(c), - 'methods': [m for m in c['methods'] if is_test_method(m)], - 'superclass': c['superclass'], - } - - return [stripped_test_class(c) for c in p['classes'] - if is_test_class(c)] - - def _GetTestsFromDexdump(test_apk): dex_dumps = dexdump.Dump(test_apk) tests = [] - def get_test_methods(methods): - return [ - { - 'method': m, - # No annotation info is available from dexdump. - # Set MediumTest annotation for default. - 'annotations': {'MediumTest': None}, - } for m in methods if m.startswith('test')] + def get_test_methods(methods, annotations): + test_methods = [] + + for method in methods: + if method.startswith('test'): + method_annotations = annotations.get(method, {}) + + # Dexdump used to not return any annotation info + # So MediumTest annotation was added to all methods + # Preserving this behaviour by adding MediumTest if none of the + # size annotations are included in these annotations + if not any(valid in method_annotations for valid in _VALID_ANNOTATIONS): + method_annotations.update({'MediumTest': None}) + + test_methods.append({ + 'method': method, + 'annotations': method_annotations + }) + + return test_methods for dump in dex_dumps: - for package_name, package_info in dump.iteritems(): - for class_name, class_info in package_info['classes'].iteritems(): - if class_name.endswith('Test'): + for package_name, package_info in six.iteritems(dump): + for class_name, class_info in six.iteritems(package_info['classes']): + if class_name.endswith('Test') and not class_info['is_abstract']: + classAnnotations, methodsAnnotations = class_info['annotations'] tests.append({ - 'class': '%s.%s' % (package_name, class_name), - 'annotations': {}, - 'methods': get_test_methods(class_info['methods']), - 'superclass': class_info['superclass'], + 'class': + '%s.%s' % (package_name, class_name), + 'annotations': + classAnnotations, + 'methods': + get_test_methods(class_info['methods'], methodsAnnotations), + 'superclass': + class_info['superclass'], }) return tests @@ -407,7 +502,7 @@ 'VERSION': _PICKLE_FORMAT_VERSION, 'TEST_METHODS': tests, } - with open(pickle_path, 'w') as pickle_file: + with open(pickle_path, 'wb') as pickle_file: pickle.dump(pickle_data, pickle_file) @@ -415,7 +510,7 @@ """Raised when JUnit4 runner is not provided or specified in apk manifest""" def __init__(self): - super(MissingJUnit4RunnerException, self).__init__( + super().__init__( 'JUnit4 runner is not provided or specified in test apk manifest.') @@ -487,9 +582,12 @@ class InstrumentationTestInstance(test_instance.TestInstance): def __init__(self, args, data_deps_delegate, error_func): - super(InstrumentationTestInstance, self).__init__() + super().__init__() self._additional_apks = [] + self._additional_apexs = [] + self._forced_queryable_additional_apks = [] + self._instant_additional_apks = [] self._apk_under_test = None self._apk_under_test_incremental_install_json = None self._modules = None @@ -498,8 +596,8 @@ self._package_info = None self._suite = None self._test_apk = None + self._test_apk_as_instant = False self._test_apk_incremental_install_json = None - self._test_jar = None self._test_package = None self._junit3_runner_class = None self._junit4_runner_class = None @@ -510,13 +608,18 @@ self._data_deps = None self._data_deps_delegate = None self._runtime_deps_path = None + self._store_data_in_app_directory = False self._initializeDataDependencyAttributes(args, data_deps_delegate) self._annotations = None self._excluded_annotations = None - self._test_filter = None + self._test_filters = None self._initializeTestFilterAttributes(args) + self._run_setup_commands = [] + self._run_teardown_commands = [] + self._initializeSetupTeardownCommandAttributes(args) + self._flags = None self._use_apk_under_test_flags_file = False self._initializeFlagAttributes(args) @@ -527,12 +630,12 @@ self._initializeTestControlAttributes(args) self._coverage_directory = None - self._jacoco_coverage_type = None self._initializeTestCoverageAttributes(args) self._store_tombstones = False self._symbolizer = None - self._enable_java_deobfuscation = False + self._enable_breakpad_dump = False + self._proguard_mapping_path = None self._deobfuscator = None self._initializeLogAttributes(args) @@ -545,17 +648,30 @@ self._system_packages_to_remove = None self._initializeSystemPackagesToRemoveAttributes(args) + self._use_voice_interaction_service = None + self._initializeUseVoiceInteractionService(args) + self._use_webview_provider = None self._initializeUseWebviewProviderAttributes(args) self._skia_gold_properties = None self._initializeSkiaGoldAttributes(args) + self._test_launcher_batch_limit = None + self._initializeTestLauncherAttributes(args) + + self._approve_app_links_domain = None + self._approve_app_links_package = None + self._initializeApproveAppLinksAttributes(args) + self._wpr_enable_record = args.wpr_enable_record self._external_shard_index = args.test_launcher_shard_index self._total_external_shards = args.test_launcher_total_shards + self._is_unit_test = False + self._initializeUnitTestFlag(args) + def _initializeApkAttributes(self, args, error_func): if args.apk_under_test: apk_under_test_path = args.apk_under_test @@ -589,6 +705,8 @@ self._test_apk = apk_helper.ToHelper(test_apk_path) self._suite = os.path.splitext(os.path.basename(args.test_apk))[0] + self._test_apk_as_instant = args.test_apk_as_instant + self._apk_under_test_incremental_install_json = ( args.apk_under_test_incremental_install_json) self._test_apk_incremental_install_json = ( @@ -602,18 +720,10 @@ self._fake_modules = args.fake_modules self._additional_locales = args.additional_locales - self._test_jar = args.test_jar self._test_support_apk = apk_helper.ToHelper(os.path.join( constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, '%sSupport.apk' % self._suite)) - if not self._test_jar: - logging.warning('Test jar not specified. Test runner will not have ' - 'Java annotation info available. May not handle test ' - 'timeouts correctly.') - elif not os.path.exists(self._test_jar): - error_func('Unable to find test JAR: %s' % self._test_jar) - self._test_package = self._test_apk.GetPackageName() all_instrumentations = self._test_apk.GetAllInstrumentations() all_junit3_runner_classes = [ @@ -649,32 +759,42 @@ self._package_info = None if self._apk_under_test: package_under_test = self._apk_under_test.GetPackageName() - for package_info in constants.PACKAGE_INFO.itervalues(): + for package_info in six.itervalues(constants.PACKAGE_INFO): if package_under_test == package_info.package: self._package_info = package_info break if not self._package_info: - logging.warning(("Unable to find package info for %s. " + - "(This may just mean that the test package is " + - "currently being installed.)"), - self._test_package) + logging.warning( + 'Unable to find package info for %s. ' + '(This may just mean that the test package is ' + 'currently being installed.)', self._test_package) - for apk in args.additional_apks: - if not os.path.exists(apk): - error_func('Unable to find additional APK: %s' % apk) - self._additional_apks = ( - [apk_helper.ToHelper(x) for x in args.additional_apks]) + for x in set(args.additional_apks + args.forced_queryable_additional_apks + + args.instant_additional_apks): + if not os.path.exists(x): + error_func('Unable to find additional APK: %s' % x) + + apk = apk_helper.ToHelper(x) + self._additional_apks.append(apk) + + if x in args.forced_queryable_additional_apks: + self._forced_queryable_additional_apks.append(apk) + + if x in args.instant_additional_apks: + self._instant_additional_apks.append(apk) + + self._additional_apexs = args.additional_apexs def _initializeDataDependencyAttributes(self, args, data_deps_delegate): self._data_deps = [] self._data_deps_delegate = data_deps_delegate self._runtime_deps_path = args.runtime_deps_path - + self._store_data_in_app_directory = args.store_data_in_app_directory if not self._runtime_deps_path: logging.warning('No data dependencies will be pushed.') def _initializeTestFilterAttributes(self, args): - self._test_filter = test_filter.InitializeFilterFromArgs(args) + self._test_filters = test_filter.InitializeFiltersFromArgs(args) def annotation_element(a): a = a.split('=', 1) @@ -683,7 +803,7 @@ if args.annotation_str: self._annotations = [ annotation_element(a) for a in args.annotation_str.split(',')] - elif not self._test_filter: + elif not self._test_filters: self._annotations = [ annotation_element(a) for a in _DEFAULT_ANNOTATIONS] else: @@ -696,11 +816,19 @@ self._excluded_annotations = [] requested_annotations = set(a[0] for a in self._annotations) - if not args.run_disabled: + if args.run_disabled: + self._excluded_annotations.extend( + annotation_element(a) for a in _DO_NOT_REVIVE_ANNOTATIONS + if a not in requested_annotations) + else: self._excluded_annotations.extend( annotation_element(a) for a in _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS if a not in requested_annotations) + def _initializeSetupTeardownCommandAttributes(self, args): + self._run_setup_commands = args.run_setup_commands + self._run_teardown_commands = args.run_teardown_commands + def _initializeFlagAttributes(self, args): self._use_apk_under_test_flags_file = args.use_apk_under_test_flags_file self._flags = ['--enable-test-intents'] @@ -723,15 +851,10 @@ def _initializeTestCoverageAttributes(self, args): self._coverage_directory = args.coverage_dir - if ("Batch", "UnitTests") in self._annotations and ( - "Batch", "UnitTests") not in self._excluded_annotations: - self._jacoco_coverage_type = "unit_tests_only" - elif ("Batch", "UnitTests") not in self._annotations and ( - "Batch", "UnitTests") in self._excluded_annotations: - self._jacoco_coverage_type = "unit_tests_excluded" def _initializeLogAttributes(self, args): - self._enable_java_deobfuscation = args.enable_java_deobfuscation + self._enable_breakpad_dump = args.enable_breakpad_dump + self._proguard_mapping_path = args.proguard_mapping_path self._store_tombstones = args.store_tombstones self._symbolizer = stack_symbolizer.Symbolizer( self.apk_under_test.path if self.apk_under_test else None) @@ -757,6 +880,12 @@ return self._system_packages_to_remove = args.system_packages_to_remove + def _initializeUseVoiceInteractionService(self, args): + if (not hasattr(args, 'use_voice_interaction_service') + or not args.use_voice_interaction_service): + return + self._use_voice_interaction_service = args.use_voice_interaction_service + def _initializeUseWebviewProviderAttributes(self, args): if (not hasattr(args, 'use_webview_provider') or not args.use_webview_provider): @@ -766,11 +895,36 @@ def _initializeSkiaGoldAttributes(self, args): self._skia_gold_properties = gold_utils.AndroidSkiaGoldProperties(args) + def _initializeTestLauncherAttributes(self, args): + if hasattr(args, 'test_launcher_batch_limit'): + self._test_launcher_batch_limit = args.test_launcher_batch_limit + + def _initializeApproveAppLinksAttributes(self, args): + if (not hasattr(args, 'approve_app_links') or not args.approve_app_links): + return + + # The argument will be formatted as com.android.thing:www.example.com . + app_links = args.approve_app_links.split(':') + + if (len(app_links) != 2 or not app_links[0] or not app_links[1]): + logging.warning('--approve_app_links option provided, but malformed.') + return + + self._approve_app_links_package = app_links[0] + self._approve_app_links_domain = app_links[1] + + def _initializeUnitTestFlag(self, args): + self._is_unit_test = args.is_unit_test + @property def additional_apks(self): return self._additional_apks @property + def additional_apexs(self): + return self._additional_apexs + + @property def apk_under_test(self): return self._apk_under_test @@ -779,6 +933,14 @@ return self._apk_under_test_incremental_install_json @property + def approve_app_links_package(self): + return self._approve_app_links_package + + @property + def approve_app_links_domain(self): + return self._approve_app_links_domain + + @property def modules(self): return self._modules @@ -799,6 +961,10 @@ return self._edit_shared_prefs @property + def enable_breakpad_dump(self): + return self._enable_breakpad_dump + + @property def external_shard_index(self): return self._external_shard_index @@ -807,8 +973,8 @@ return self._flags @property - def jacoco_coverage_type(self): - return self._jacoco_coverage_type + def is_unit_test(self): + return self._is_unit_test @property def junit3_runner_class(self): @@ -831,6 +997,18 @@ return self._replace_system_package @property + def run_setup_commands(self): + return self._run_setup_commands + + @property + def run_teardown_commands(self): + return self._run_teardown_commands + + @property + def use_voice_interaction_service(self): + return self._use_voice_interaction_service + + @property def use_webview_provider(self): return self._use_webview_provider @@ -843,6 +1021,10 @@ return self._skia_gold_properties @property + def store_data_in_app_directory(self): + return self._store_data_in_app_directory + + @property def store_tombstones(self): return self._store_tombstones @@ -863,12 +1045,20 @@ return self._test_apk @property + def test_apk_as_instant(self): + return self._test_apk_as_instant + + @property def test_apk_incremental_install_json(self): return self._test_apk_incremental_install_json @property - def test_jar(self): - return self._test_jar + def test_filters(self): + return self._test_filters + + @property + def test_launcher_batch_limit(self): + return self._test_launcher_batch_limit @property def test_support_apk(self): @@ -922,18 +1112,20 @@ def SetUp(self): self._data_deps.extend( self._data_deps_delegate(self._runtime_deps_path)) - if self._enable_java_deobfuscation: + if self._proguard_mapping_path: self._deobfuscator = deobfuscator.DeobfuscatorPool( - self.test_apk.path + '.mapping') + self._proguard_mapping_path) def GetDataDependencies(self): return self._data_deps def GetTests(self): - if self.test_jar: - raw_tests = GetAllTestsFromJar(self.test_jar) - else: - raw_tests = GetAllTestsFromApk(self.test_apk.path) + if self._test_apk_incremental_install_json: + # Would likely just be a matter of calling GetAllTestsFromApk on all + # .dex files listed in the .json. + raise Exception('Support not implemented for incremental_install=true on ' + 'tests that do not use //base\'s test runner.') + raw_tests = GetAllTestsFromApk(self.test_apk.path) return self.ProcessRawTests(raw_tests) def MaybeDeobfuscateLines(self, lines): @@ -947,15 +1139,20 @@ if self._junit4_runner_class is None and any( t['is_junit4'] for t in inflated_tests): raise MissingJUnit4RunnerException() - filtered_tests = FilterTests( - inflated_tests, self._test_filter, self._annotations, - self._excluded_annotations) - if self._test_filter and not filtered_tests: + filtered_tests = FilterTests(inflated_tests, self._test_filters, + self._annotations, self._excluded_annotations) + if self._test_filters and not filtered_tests: for t in inflated_tests: logging.debug(' %s', GetUniqueTestName(t)) - logging.warning('Unmatched Filter: %s', self._test_filter) + logging.warning('Unmatched Filters: %s', self._test_filters) return filtered_tests + def IsApkForceQueryable(self, apk): + return apk in self._forced_queryable_additional_apks + + def IsApkInstant(self, apk): + return apk in self._instant_additional_apks + # pylint: disable=no-self-use def _InflateTests(self, tests): inflated_tests = [] @@ -990,14 +1187,13 @@ def _annotationToSwitches(clazz, methods): if clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES: return [methods['value']] - elif clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS: + if clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS: list_of_switches = [] for annotation in methods['value']: - for clazz, methods in annotation.iteritems(): - list_of_switches += _annotationToSwitches(clazz, methods) + for c, m in six.iteritems(annotation): + list_of_switches += _annotationToSwitches(c, m) return list_of_switches - else: - return [] + return [] def _setTestFlags(test, flags): if flags: @@ -1011,7 +1207,7 @@ list_of_switches = [] _checkParameterization(annotations) if _SKIP_PARAMETERIZATION not in annotations: - for clazz, methods in annotations.iteritems(): + for clazz, methods in six.iteritems(annotations): list_of_switches += _annotationToSwitches(clazz, methods) if list_of_switches: _setTestFlags(t, _switchesToFlags(list_of_switches[0]))
diff --git a/build/android/pylib/instrumentation/instrumentation_test_instance_test.py b/build/android/pylib/instrumentation/instrumentation_test_instance_test.py index 77918bb..945c404 100755 --- a/build/android/pylib/instrumentation/instrumentation_test_instance_test.py +++ b/build/android/pylib/instrumentation/instrumentation_test_instance_test.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2014 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,10 +7,12 @@ # pylint: disable=protected-access + import collections import tempfile import unittest +from six.moves import range # pylint: disable=redefined-builtin from pylib.base import base_test_result from pylib.instrumentation import instrumentation_test_instance @@ -59,36 +61,36 @@ o = self.createTestInstance() args = self.createFlagAttributesArgs(command_line_flags=['--foo', '--bar']) o._initializeFlagAttributes(args) - self.assertEquals(o._flags, ['--enable-test-intents', '--foo', '--bar']) + self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) def test_initializeFlagAttributes_deviceFlagsFile(self): o = self.createTestInstance() - with tempfile.NamedTemporaryFile() as flags_file: + with tempfile.NamedTemporaryFile(mode='w') as flags_file: flags_file.write('\n'.join(['--foo', '--bar'])) flags_file.flush() args = self.createFlagAttributesArgs(device_flags_file=flags_file.name) o._initializeFlagAttributes(args) - self.assertEquals(o._flags, ['--enable-test-intents', '--foo', '--bar']) + self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) def test_initializeFlagAttributes_strictModeOn(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(strict_mode='on') o._initializeFlagAttributes(args) - self.assertEquals(o._flags, ['--enable-test-intents', '--strict-mode=on']) + self.assertEqual(o._flags, ['--enable-test-intents', '--strict-mode=on']) def test_initializeFlagAttributes_strictModeOn_coverageOn(self): o = self.createTestInstance() args = self.createFlagAttributesArgs( strict_mode='on', coverage_dir='/coverage/dir') o._initializeFlagAttributes(args) - self.assertEquals(o._flags, ['--enable-test-intents']) + self.assertEqual(o._flags, ['--enable-test-intents']) def test_initializeFlagAttributes_strictModeOff(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(strict_mode='off') o._initializeFlagAttributes(args) - self.assertEquals(o._flags, ['--enable-test-intents']) + self.assertEqual(o._flags, ['--enable-test-intents']) def testGetTests_noFilter(self): o = self.createTestInstance() @@ -151,11 +153,10 @@ }, ] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestFilter(self): o = self.createTestInstance() @@ -189,12 +190,146 @@ }, ] - o._test_filter = 'org.chromium.test.SampleTest.testMethod1' - o._test_jar = 'path/to/test.jar' + o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) + + def testGetTests_simpleGtestPositiveAndNegativeFilter(self): + o = self.createTestInstance() + raw_tests = [{ + 'annotations': { + 'Feature': { + 'value': ['Foo'] + } + }, + 'class': + 'org.chromium.test.SampleTest', + 'superclass': + 'java.lang.Object', + 'methods': [ + { + 'annotations': { + 'SmallTest': None + }, + 'method': 'testMethod1', + }, + { + 'annotations': { + 'MediumTest': None + }, + 'method': 'testMethod2', + }, + ], + }, { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + } + }, + 'class': + 'org.chromium.test.SampleTest2', + 'superclass': + 'java.lang.Object', + 'methods': [{ + 'annotations': { + 'SmallTest': None + }, + 'method': 'testMethod1', + }], + }] + + expected_tests = [ + { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + }, + 'SmallTest': None, + }, + 'class': 'org.chromium.test.SampleTest', + 'is_junit4': True, + 'method': 'testMethod1', + }, + ] + + o._test_filters = [ + 'org.chromium.test.SampleTest.*'\ + '-org.chromium.test.SampleTest.testMethod2' + ] + o._junit4_runner_class = 'J4Runner' + actual_tests = o.ProcessRawTests(raw_tests) + + self.assertEqual(actual_tests, expected_tests) + + def testGetTests_multipleGtestPositiveAndNegativeFilter(self): + o = self.createTestInstance() + raw_tests = [{ + 'annotations': { + 'Feature': { + 'value': ['Foo'] + } + }, + 'class': + 'org.chromium.test.SampleTest', + 'superclass': + 'java.lang.Object', + 'methods': [ + { + 'annotations': { + 'SmallTest': None + }, + 'method': 'testMethod1', + }, + { + 'annotations': { + 'MediumTest': None + }, + 'method': 'testMethod2', + }, + ], + }, { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + } + }, + 'class': + 'org.chromium.test.SampleTest2', + 'superclass': + 'java.lang.Object', + 'methods': [{ + 'annotations': { + 'SmallTest': None + }, + 'method': 'testMethod1', + }], + }] + + expected_tests = [ + { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + }, + 'SmallTest': None, + }, + 'class': 'org.chromium.test.SampleTest', + 'is_junit4': True, + 'method': 'testMethod1', + }, + ] + + o._test_filters = [ + 'org.chromium.test.SampleTest*testMethod1', + 'org.chromium.test.SampleTest.*'\ + '-org.chromium.test.SampleTest.testMethod2' + ] + o._junit4_runner_class = 'J4Runner' + actual_tests = o.ProcessRawTests(raw_tests) + + self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestUnqualifiedNameFilter(self): o = self.createTestInstance() @@ -228,12 +363,11 @@ }, ] - o._test_filter = 'SampleTest.testMethod1' - o._test_jar = 'path/to/test.jar' + o._test_filters = ['SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_parameterizedTestGtestFilter(self): o = self.createTestInstance() @@ -287,12 +421,11 @@ }, ] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' - o._test_filter = 'org.chromium.test.SampleTest.testMethod1' + o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_wildcardGtestFilter(self): o = self.createTestInstance() @@ -337,12 +470,11 @@ }, ] - o._test_filter = 'org.chromium.test.SampleTest2.*' - o._test_jar = 'path/to/test.jar' + o._test_filters = ['org.chromium.test.SampleTest2.*'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_negativeGtestFilter(self): o = self.createTestInstance() @@ -396,12 +528,11 @@ }, ] - o._test_filter = '*-org.chromium.test.SampleTest.testMethod1' - o._test_jar = 'path/to/test.jar' + o._test_filters = ['*-org.chromium.test.SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationFilter(self): o = self.createTestInstance() @@ -456,11 +587,10 @@ ] o._annotations = [('SmallTest', None)] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_excludedAnnotationFilter(self): o = self.createTestInstance() @@ -508,11 +638,103 @@ ] o._excluded_annotations = [('SmallTest', None)] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) + + def testGetTests_excludedDoNotReviveAnnotation(self): + o = self.createTestInstance() + raw_tests = [{ + 'annotations': { + 'Feature': { + 'value': ['Foo'] + } + }, + 'class': + 'org.chromium.test.SampleTest', + 'superclass': + 'junit.framework.TestCase', + 'methods': [ + { + 'annotations': { + 'DisabledTest': None, + 'DoNotRevive': { + 'reason': 'sample reason' + }, + }, + 'method': 'testMethod1', + }, + { + 'annotations': { + 'FlakyTest': None, + }, + 'method': 'testMethod2', + }, + ], + }, { + 'annotations': { + 'Feature': { + 'value': ['Bar'] + } + }, + 'class': + 'org.chromium.test.SampleTest2', + 'superclass': + 'junit.framework.TestCase', + 'methods': [ + { + 'annotations': { + 'FlakyTest': None, + 'DoNotRevive': { + 'reason': 'sample reason' + }, + }, + 'method': 'testMethod1', + }, + ], + }, { + 'annotations': { + 'Feature': { + 'value': ['Baz'] + } + }, + 'class': + 'org.chromium.test.SampleTest3', + 'superclass': + 'junit.framework.TestCase', + 'methods': [ + { + 'annotations': { + 'FlakyTest': None, + 'Manual': { + 'message': 'sample message' + }, + }, + 'method': 'testMethod1', + }, + ], + }] + + expected_tests = [ + { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + }, + 'FlakyTest': None, + }, + 'class': 'org.chromium.test.SampleTest', + 'is_junit4': True, + 'method': 'testMethod2', + }, + ] + + o._excluded_annotations = [('DoNotRevive', None), ('Manual', None)] + o._junit4_runner_class = 'J4Runner' + actual_tests = o.ProcessRawTests(raw_tests) + + self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationSimpleValueFilter(self): o = self.createTestInstance() @@ -570,11 +792,10 @@ ] o._annotations = [('TestValue', '1')] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationDictValueFilter(self): o = self.createTestInstance() @@ -620,11 +841,10 @@ ] o._annotations = [('Feature', 'Bar')] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGetTestName(self): test = { @@ -642,13 +862,11 @@ 'method': test['method'] } - self.assertEquals( - instrumentation_test_instance.GetTestName(test, sep='.'), - 'org.chromium.TestA.testSimple') - self.assertEquals( - instrumentation_test_instance.GetTestName( - unqualified_class_test, sep='.'), - 'TestA.testSimple') + self.assertEqual(instrumentation_test_instance.GetTestName(test, sep='.'), + 'org.chromium.TestA.testSimple') + self.assertEqual( + instrumentation_test_instance.GetTestName(unqualified_class_test, + sep='.'), 'TestA.testSimple') def testGetUniqueTestName(self): test = { @@ -661,9 +879,8 @@ 'flags': ['enable_features=abc'], 'is_junit4': True, 'method': 'testSimple'} - self.assertEquals( - instrumentation_test_instance.GetUniqueTestName( - test, sep='.'), + self.assertEqual( + instrumentation_test_instance.GetUniqueTestName(test, sep='.'), 'org.chromium.TestA.testSimple_with_enable_features=abc') def testGetTestNameWithoutParameterPostfix(self): @@ -681,14 +898,12 @@ 'class': test['class'].split('.')[-1], 'method': test['method'] } - self.assertEquals( + self.assertEqual( instrumentation_test_instance.GetTestNameWithoutParameterPostfix( - test, sep='.'), - 'org.chromium.TestA') - self.assertEquals( + test, sep='.'), 'org.chromium.TestA') + self.assertEqual( instrumentation_test_instance.GetTestNameWithoutParameterPostfix( - unqualified_class_test, sep='.'), - 'TestA') + unqualified_class_test, sep='.'), 'TestA') def testGetTests_multipleAnnotationValuesRequested(self): o = self.createTestInstance() @@ -750,11 +965,10 @@ ] o._annotations = [('Feature', 'Bar'), ('Feature', 'Baz')] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testGenerateTestResults_noStatus(self): results = instrumentation_test_instance.GenerateTestResults( @@ -948,10 +1162,9 @@ expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations']) - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testParameterizedCommandLineFlags(self): o = self.createTestInstance() @@ -1071,10 +1284,9 @@ expected_tests[4]['annotations'].update( raw_tests[0]['methods'][0]['annotations']) - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testDifferentCommandLineParameterizations(self): o = self.createTestInstance() @@ -1132,10 +1344,9 @@ expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations']) - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) - self.assertEquals(actual_tests, expected_tests) + self.assertEqual(actual_tests, expected_tests) def testMultipleCommandLineParameterizations_raises(self): o = self.createTestInstance() @@ -1176,7 +1387,6 @@ }, ] - o._test_jar = 'path/to/test.jar' o._junit4_runner_class = 'J4Runner' self.assertRaises( instrumentation_test_instance.CommandLineParameterizationException,
diff --git a/build/android/pylib/instrumentation/json_perf_parser.py b/build/android/pylib/instrumentation/json_perf_parser.py index c647890..ef541f4 100644 --- a/build/android/pylib/instrumentation/json_perf_parser.py +++ b/build/android/pylib/instrumentation/json_perf_parser.py
@@ -1,10 +1,11 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A helper module for parsing JSON objects from perf tests results.""" + import json
diff --git a/build/android/pylib/instrumentation/test_result.py b/build/android/pylib/instrumentation/test_result.py index a1c7307..dc56605 100644 --- a/build/android/pylib/instrumentation/test_result.py +++ b/build/android/pylib/instrumentation/test_result.py
@@ -1,10 +1,12 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + from pylib.base import base_test_result + class InstrumentationTestResult(base_test_result.BaseTestResult): """Result information for a single instrumentation test.""" @@ -17,8 +19,7 @@ dur: Duration of the test run in milliseconds. log: A string listing any errors. """ - super(InstrumentationTestResult, self).__init__( - full_name, test_type, dur, log) + super().__init__(full_name, test_type, dur, log) name_pieces = full_name.rsplit('#') if len(name_pieces) > 1: self._test_name = name_pieces[1]
diff --git a/build/android/pylib/junit/__init__.py b/build/android/pylib/junit/__init__.py index 4d6aabb..d46d7b4 100644 --- a/build/android/pylib/junit/__init__.py +++ b/build/android/pylib/junit/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/junit/junit_test_instance.py b/build/android/pylib/junit/junit_test_instance.py index a3d18e0..f7bd49a 100644 --- a/build/android/pylib/junit/junit_test_instance.py +++ b/build/android/pylib/junit/junit_test_instance.py
@@ -1,7 +1,8 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + from pylib.base import test_instance from pylib.utils import test_filter @@ -9,17 +10,20 @@ class JunitTestInstance(test_instance.TestInstance): def __init__(self, args, _): - super(JunitTestInstance, self).__init__() + super().__init__() self._coverage_dir = args.coverage_dir self._debug_socket = args.debug_socket self._coverage_on_the_fly = args.coverage_on_the_fly + self._native_libs_dir = args.native_libs_dir self._package_filter = args.package_filter self._resource_apk = args.resource_apk self._robolectric_runtime_deps_dir = args.robolectric_runtime_deps_dir self._runner_filter = args.runner_filter self._shards = args.shards - self._test_filter = test_filter.InitializeFilterFromArgs(args) + self._test_filters = test_filter.InitializeFiltersFromArgs(args) + self._has_literal_filters = (args.isolated_script_test_filters + or args.test_filters) self._test_suite = args.test_suite #override @@ -47,6 +51,10 @@ return self._debug_socket @property + def native_libs_dir(self): + return self._native_libs_dir + + @property def package_filter(self): return self._package_filter @@ -63,8 +71,12 @@ return self._runner_filter @property - def test_filter(self): - return self._test_filter + def test_filters(self): + return self._test_filters + + @property + def has_literal_filters(self): + return self._has_literal_filters @property def shards(self):
diff --git a/build/android/pylib/local/__init__.py b/build/android/pylib/local/__init__.py index 4d6aabb..d46d7b4 100644 --- a/build/android/pylib/local/__init__.py +++ b/build/android/pylib/local/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/local/device/__init__.py b/build/android/pylib/local/device/__init__.py index 4d6aabb..d46d7b4 100644 --- a/build/android/pylib/local/device/__init__.py +++ b/build/android/pylib/local/device/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/local/device/local_device_environment.py b/build/android/pylib/local/device/local_device_environment.py index d2a9077a..a51f370 100644 --- a/build/android/pylib/local/device/local_device_environment.py +++ b/build/android/pylib/local/device/local_device_environment.py
@@ -1,8 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import datetime import functools import logging @@ -34,6 +34,8 @@ 'StrictMode:D', ] +SYSTEM_USER_ID = 0 + def _DeviceCachePath(device): file_name = 'device_cache_%s.json' % device.adb.GetDeviceSerial() @@ -84,7 +86,7 @@ return decorator -def place_nomedia_on_device(dev, device_root): +def place_nomedia_on_device(dev, device_root, run_as=None, as_root=False): """Places .nomedia file in test data root. This helps to prevent system from scanning media files inside test data. @@ -94,10 +96,19 @@ device_root: Base path on device to place .nomedia file. """ - dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True) - dev.WriteFile('%s/.nomedia' % device_root, 'https://crbug.com/796640') + dev.RunShellCommand(['mkdir', '-p', device_root], + run_as=run_as, + as_root=as_root, + check_return=True) + dev.WriteFile('%s/.nomedia' % device_root, + 'https://crbug.com/796640', + run_as=run_as, + as_root=as_root) +# TODO(1262303): After Telemetry is supported by python3 we can re-add +# super without arguments in this script. +# pylint: disable=super-with-arguments class LocalDeviceEnvironment(environment.Environment): def __init__(self, args, output_manager, _error_func): @@ -124,6 +135,8 @@ self._trace_all = None if hasattr(args, 'trace_all'): self._trace_all = args.trace_all + self._use_persistent_shell = args.use_persistent_shell + self._disable_test_server = args.disable_test_server devil_chromium.Initialize( output_directory=constants.GetOutDirectory(), @@ -163,7 +176,8 @@ enable_device_files_cache=self._enable_device_cache, default_retries=self._max_tries - 1, device_arg=device_arg, - abis=self._preferred_abis) + abis=self._preferred_abis, + persistent_shell=self._use_persistent_shell) if self._logcat_output_file: self._logcat_output_dir = tempfile.mkdtemp() @@ -171,6 +185,12 @@ @handle_shard_failures_with(on_failure=self.DenylistDevice) def prepare_device(d): d.WaitUntilFullyBooted() + if d.GetCurrentUser() != SYSTEM_USER_ID: + # Use system user to run tasks to avoid "/sdcard "accessing issue + # due to multiple-users. For details, see + # https://source.android.com/docs/devices/admin/multi-user-testing + logging.info('Switching to user with id %s', SYSTEM_USER_ID) + d.SwitchUser(SYSTEM_USER_ID) if self._enable_device_cache: cache_path = _DeviceCachePath(d) @@ -186,8 +206,10 @@ self._logcat_output_dir, '%s_%s' % (d.adb.GetDeviceSerial(), datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S'))) - monitor = logcat_monitor.LogcatMonitor( - d.adb, clear=True, output_file=logcat_file) + monitor = logcat_monitor.LogcatMonitor(d.adb, + clear=True, + output_file=logcat_file, + check_error=False) self._logcat_monitors.append(monitor) monitor.Start() @@ -243,6 +265,10 @@ def trace_output(self): return self._trace_output + @property + def disable_test_server(self): + return self._disable_test_server + #override def TearDown(self): if self.trace_output and self._trace_all:
diff --git a/build/android/pylib/local/device/local_device_gtest_run.py b/build/android/pylib/local/device/local_device_gtest_run.py index 753556d..796f614 100644 --- a/build/android/pylib/local/device/local_device_gtest_run.py +++ b/build/android/pylib/local/device/local_device_gtest_run.py
@@ -1,10 +1,11 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import contextlib import collections +import fnmatch import itertools import logging import math @@ -54,6 +55,9 @@ 'org.chromium.native_test.NativeTestInstrumentationTestRunner' '.TestList') +# Used to identify the prefix in gtests. +_GTEST_PRETEST_PREFIX = 'PRE_' + _SECONDS_TO_NANOS = int(1e9) # Tests that use SpawnedTestServer must run the LocalTestServerSpawner on the @@ -76,7 +80,7 @@ # No-op context manager. If we used Python 3, we could change this to # contextlib.ExitStack() -class _NullContextManager(object): +class _NullContextManager: def __enter__(self): pass def __exit__(self, *args): @@ -91,29 +95,39 @@ yield '%s_%d%s' % (base, i, ext) -def _ExtractTestsFromFilter(gtest_filter): - """Returns the list of tests specified by the given filter. +def _ExtractTestsFromFilters(gtest_filters): + """Returns the list of tests specified by the given filters. Returns: None if the device should be queried for the test list instead. """ - # Empty means all tests, - means exclude filter. - if not gtest_filter or '-' in gtest_filter: + # - means exclude filter. + for gtest_filter in gtest_filters: + if '-' in gtest_filter: + return None + # Empty means all tests + if not any(gtest_filters): return None - patterns = gtest_filter.split(':') - # For a single pattern, allow it even if it has a wildcard so long as the - # wildcard comes at the end and there is at least one . to prove the scope is - # not too large. - # This heuristic is not necessarily faster, but normally is. - if len(patterns) == 1 and patterns[0].endswith('*'): - no_suffix = patterns[0].rstrip('*') - if '*' not in no_suffix and '.' in no_suffix: - return patterns + if len(gtest_filters) == 1: + patterns = gtest_filters[0].split(':') + # For a single pattern, allow it even if it has a wildcard so long as the + # wildcard comes at the end and there is at least one . to prove the scope + # is not too large. + # This heuristic is not necessarily faster, but normally is. + if len(patterns) == 1 and patterns[0].endswith('*'): + no_suffix = patterns[0].rstrip('*') + if '*' not in no_suffix and '.' in no_suffix: + return patterns - if '*' in gtest_filter: - return None - return patterns + all_patterns = set(gtest_filters[0].split(':')) + for gtest_filter in gtest_filters: + patterns = gtest_filter.split(':') + for pattern in patterns: + if '*' in pattern: + return None + all_patterns = all_patterns.intersection(set(patterns)) + return list(all_patterns) def _GetDeviceTimeoutMultiplier(): @@ -232,7 +246,7 @@ str(coverage_index), '%2m.profraw'])) -class _ApkDelegate(object): +class _ApkDelegate: def __init__(self, test_instance, tool): self._activity = test_instance.activity self._apk_helper = test_instance.apk_helper @@ -248,6 +262,7 @@ self._tool = tool self._coverage_dir = test_instance.coverage_dir self._coverage_index = 0 + self._use_existing_test_data = test_instance.use_existing_test_data def GetTestDataRoot(self, device): # pylint: disable=no-self-use @@ -255,6 +270,8 @@ 'chromium_tests_root') def Install(self, device): + if self._use_existing_test_data: + return if self._test_apk_incremental_install_json: installer.Install(device, self._test_apk_incremental_install_json, apk=self._apk_helper, permissions=self._permissions) @@ -265,8 +282,8 @@ reinstall=True, permissions=self._permissions) - def ResultsDirectory(self, device): - return device.GetApplicationDataDirectory(self._package) + def ResultsDirectory(self, device): # pylint: disable=no-self-use + return device.GetExternalStoragePath() def Run(self, test, device, flags=None, **kwargs): extras = dict(self._extras) @@ -287,7 +304,6 @@ extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int( kwargs['timeout'] * _SECONDS_TO_NANOS) - # pylint: disable=redefined-variable-type command_line_file = _NullContextManager() if flags: if len(flags) > _MAX_INLINE_FLAGS_LENGTH: @@ -305,7 +321,6 @@ extras[_EXTRA_TEST_LIST] = test_list_file.name else: extras[_EXTRA_TEST] = test[0] - # pylint: enable=redefined-variable-type # We need to use GetAppWritablePath here instead of GetExternalStoragePath # since we will not have yet applied legacy storage permission workarounds @@ -362,7 +377,7 @@ device.ClearApplicationState(self._package, permissions=self._permissions) -class _ExeDelegate(object): +class _ExeDelegate: def __init__(self, tr, test_instance, tool): self._host_dist_dir = test_instance.exe_dist_dir @@ -456,14 +471,13 @@ def __init__(self, env, test_instance): assert isinstance(env, local_device_environment.LocalDeviceEnvironment) assert isinstance(test_instance, gtest_test_instance.GtestTestInstance) - super(LocalDeviceGtestRun, self).__init__(env, test_instance) + super().__init__(env, test_instance) if self._test_instance.apk_helper: self._installed_packages = [ self._test_instance.apk_helper.GetPackageName() ] - # pylint: disable=redefined-variable-type if self._test_instance.apk: self._delegate = _ApkDelegate(self._test_instance, env.tool) elif self._test_instance.exe_dist_dir: @@ -473,7 +487,6 @@ self._test_instance.isolated_script_test_perf_output) else: self._test_perf_output_filenames = itertools.repeat(None) - # pylint: enable=redefined-variable-type self._crashes = set() self._servers = collections.defaultdict(list) @@ -492,6 +505,8 @@ self._delegate.Install(dev) def push_test_data(dev): + if self._test_instance.use_existing_test_data: + return # Push data dependencies. device_root = self._delegate.GetTestDataRoot(dev) host_device_tuples_substituted = [ @@ -514,13 +529,17 @@ tool.CopyFiles(dev) tool.SetupEnvironment() + if self._env.disable_test_server: + logging.warning('Not starting test server. Some tests may fail.') + return + try: # See https://crbug.com/1030827. # This is a hack that may break in the future. We're relying on the # fact that adb doesn't use ipv6 for it's server, and so doesn't # listen on ipv6, but ssh remote forwarding does. 5037 is the port # number adb uses for its server. - if "[::1]:5037" in subprocess.check_output( + if b"[::1]:5037" in subprocess.check_output( "ss -o state listening 'sport = 5037'", shell=True): logging.error( 'Test Server cannot be started with a remote-forwarded adb ' @@ -541,15 +560,6 @@ def bind_crash_handler(step, dev): return lambda: crash_handler.RetryOnSystemCrash(step, dev) - # Explicitly enable root to ensure that tests run under deterministic - # conditions. Without this explicit call, EnableRoot() is called from - # push_test_data() when PushChangedFiles() determines that it should use - # _PushChangedFilesZipped(), which is only most of the time. - # Root is required (amongst maybe other reasons) to pull the results file - # from the device, since it lives within the application's data directory - # (via GetApplicationDataDirectory()). - device.EnableRoot() - steps = [ bind_crash_handler(s, device) for s in (install_apk, push_test_data, init_tool_and_start_servers)] @@ -564,11 +574,25 @@ self._test_instance.GetDataDependencies()) #override - def _ShouldShard(self): + def _ShouldShardTestsForDevices(self): + """Shard tests across several devices. + + Returns: + True if tests should be sharded across several devices, + False otherwise. + """ return True #override - def _CreateShards(self, tests): + def _CreateShardsForDevices(self, tests): + """Create shards of tests to run on devices. + + Args: + tests: List containing tests or test batches. + + Returns: + List of test batches. + """ # _crashes are tests that might crash and make the tests in the same shard # following the crashed testcase not run. # Thus we need to create separate shards for each crashed testcase, @@ -582,6 +606,10 @@ # Delete suspect testcase from tests. tests = [test for test in tests if not test in self._crashes] + # Sort tests by hash. + # TODO(crbug.com/1257820): Add sorting logic back to _PartitionTests. + tests = self._SortTests(tests) + max_shard_size = self._test_instance.test_launcher_batch_limit shards.extend(self._PartitionTests(tests, device_count, max_shard_size)) @@ -593,7 +621,7 @@ # When the exact list of tests to run is given via command-line (e.g. when # locally iterating on a specific test), skip querying the device (which # takes ~3 seconds). - tests = _ExtractTestsFromFilter(self._test_instance.gtest_filter) + tests = _ExtractTestsFromFilters(self._test_instance.gtest_filters) if tests: return tests @@ -609,8 +637,10 @@ timeout = None flags = [ - f for f in self._test_instance.flags - if f not in ['--wait-for-debugger', '--wait-for-java-debugger'] + f for f in self._test_instance.flags if f not in [ + '--wait-for-debugger', '--wait-for-java-debugger', + '--gtest_also_run_disabled_tests' + ] ] flags.append('--gtest_list_tests') @@ -652,6 +682,42 @@ self._test_instance.total_external_shards) return tests + #override + def _GroupTests(self, tests): + pre_tests = dict() + other_tests = [] + for test in tests: + test_name_start = max(test.find('.') + 1, 0) + test_name = test[test_name_start:] + if test_name_start == 0 or not test_name.startswith( + _GTEST_PRETEST_PREFIX): + other_tests.append(test) + else: + test_suite = test[:test_name_start - 1] + trim_test = test + trim_tests = [test] + + while test_name.startswith(_GTEST_PRETEST_PREFIX): + test_name = test_name[len(_GTEST_PRETEST_PREFIX):] + trim_test = '%s.%s' % (test_suite, test_name) + trim_tests.append(trim_test) + + if not trim_test in pre_tests or len( + pre_tests[trim_test]) < len(trim_tests): + pre_tests[trim_test] = trim_tests + + all_tests = [] + for other_test in other_tests: + if not other_test in pre_tests: + all_tests.append(other_test) + + # TODO(crbug.com/1257820): Add logic to support grouping tests. + # Once grouping logic is added, switch to 'append' from 'extend'. + for _, test_list in pre_tests.items(): + all_tests.extend(test_list) + + return all_tests + def _UploadTestArtifacts(self, device, test_artifacts_dir): # TODO(jbudorick): Reconcile this with the output manager once # https://codereview.chromium.org/2933993002/ lands. @@ -718,7 +784,7 @@ if logmon: logmon.Close() if logcat_file and logcat_file.Link(): - logging.info('Logcat saved to %s', logcat_file.Link()) + logging.critical('Logcat saved to %s', logcat_file.Link()) #override def _RunTest(self, device, test): @@ -867,6 +933,19 @@ gtest_test_instance.TestNameWithoutDisabledPrefix(t)) not_run_tests = tests_stripped_disabled_prefix.difference( set(r.GetName() for r in results)) + + if self._test_instance.extract_test_list_from_filter: + # A test string might end with a * in this mode, and so may not match any + # r.GetName() for the set difference. It's possible a filter like foo.* + # can match two tests, ie foo.baz and foo.foo. + # When running it's possible Foo.baz is ran, foo.foo is not, but the test + # list foo.* will not be reran as at least one result matched it. + not_run_tests = { + t + for t in not_run_tests + if not any(fnmatch.fnmatch(r.GetName(), t) for r in results) + } + return results, list(not_run_tests) if results else None #override
diff --git a/build/android/pylib/local/device/local_device_gtest_run_test.py b/build/android/pylib/local/device/local_device_gtest_run_test.py index b08b24b..5a485c6 100755 --- a/build/android/pylib/local/device/local_device_gtest_run_test.py +++ b/build/android/pylib/local/device/local_device_gtest_run_test.py
@@ -1,12 +1,12 @@ -#!/usr/bin/env vpython -# Copyright 2021 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for local_device_gtest_test_run.""" # pylint: disable=protected-access -from __future__ import absolute_import + import os import tempfile import unittest @@ -19,6 +19,11 @@ import mock # pylint: disable=import-error +def isSliceInList(s, l): + lenOfSlice = len(s) + return any(s == l[i:lenOfSlice + i] for i in range(len(l) - lenOfSlice + 1)) + + class LocalDeviceGtestRunTest(unittest.TestCase): def setUp(self): self._obj = local_device_gtest_run.LocalDeviceGtestRun( @@ -27,21 +32,27 @@ def testExtractTestsFromFilter(self): # Checks splitting by colons. - self.assertEqual([ - 'b17', - 'm4e3', - 'p51', - ], local_device_gtest_run._ExtractTestsFromFilter('b17:m4e3:p51')) + self.assertEqual( + set([ + 'm4e3', + 'p51', + 'b17', + ]), + set(local_device_gtest_run._ExtractTestsFromFilters(['b17:m4e3:p51']))) # Checks the '-' sign. - self.assertIsNone(local_device_gtest_run._ExtractTestsFromFilter('-mk2')) + self.assertIsNone(local_device_gtest_run._ExtractTestsFromFilters(['-mk2'])) # Checks the more than one asterick. self.assertIsNone( - local_device_gtest_run._ExtractTestsFromFilter('.mk2*:.M67*')) + local_device_gtest_run._ExtractTestsFromFilters(['.mk2*:.M67*'])) # Checks just an asterick without a period - self.assertIsNone(local_device_gtest_run._ExtractTestsFromFilter('M67*')) + self.assertIsNone(local_device_gtest_run._ExtractTestsFromFilters(['M67*'])) # Checks an asterick at the end with a period. self.assertEqual(['.M67*'], - local_device_gtest_run._ExtractTestsFromFilter('.M67*')) + local_device_gtest_run._ExtractTestsFromFilters(['.M67*'])) + # Checks multiple filters intersect + self.assertEqual(['m4e3'], + local_device_gtest_run._ExtractTestsFromFilters( + ['b17:m4e3:p51', 'b17:m4e3', 'm4e3:p51'])) def testGetLLVMProfilePath(self): path = local_device_gtest_run._GetLLVMProfilePath('test_dir', 'sr71', '5') @@ -74,6 +85,34 @@ self.assertTrue(mock_gsh.called) self.assertEqual(result, link) + def testGroupTests(self): + test = [ + "TestClass1.testcase1", + "TestClass1.otherTestCase", + "TestClass1.PRE_testcase1", + "TestClass1.abc_testcase2", + "TestClass1.PRE_PRE_testcase1", + "TestClass1.PRE_abc_testcase2", + "TestClass1.PRE_PRE_abc_testcase2", + ] + expectedTestcase1 = [ + "TestClass1.PRE_PRE_testcase1", + "TestClass1.PRE_testcase1", + "TestClass1.testcase1", + ] + expectedTestcase2 = [ + "TestClass1.PRE_PRE_abc_testcase2", + "TestClass1.PRE_abc_testcase2", + "TestClass1.abc_testcase2", + ] + expectedOtherTestcase = [ + "TestClass1.otherTestCase", + ] + actualTestCase = self._obj._GroupTests(test) + self.assertTrue(isSliceInList(expectedTestcase1, actualTestCase)) + self.assertTrue(isSliceInList(expectedTestcase2, actualTestCase)) + self.assertTrue(isSliceInList(expectedOtherTestcase, actualTestCase)) + if __name__ == '__main__': unittest.main(verbosity=2)
diff --git a/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/build/android/pylib/local/device/local_device_instrumentation_test_run.py index 7f16d6a..f479007 100644 --- a/build/android/pylib/local/device/local_device_instrumentation_test_run.py +++ b/build/android/pylib/local/device/local_device_instrumentation_test_run.py
@@ -1,7 +1,8 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import collections import contextlib import copy @@ -16,6 +17,8 @@ import tempfile import time +from six.moves import range # pylint: disable=redefined-builtin +from six.moves import zip # pylint: disable=redefined-builtin from devil import base_error from devil.android import apk_helper from devil.android import crash_handler @@ -23,6 +26,7 @@ from devil.android import device_temp_file from devil.android import flag_changer from devil.android.sdk import shared_prefs +from devil.android.sdk import version_codes from devil.android import logcat_monitor from devil.android.tools import system_app from devil.android.tools import webview_app @@ -95,23 +99,32 @@ _EXTRA_TEST_LIST = ( 'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList') +_EXTRA_TEST_IS_UNIT = ( + 'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.IsUnitTest') + _EXTRA_PACKAGE_UNDER_TEST = ('org.chromium.chrome.test.pagecontroller.rules.' 'ChromeUiApplicationTestRule.PackageUnderTest') FEATURE_ANNOTATION = 'Feature' RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest' WPR_ARCHIVE_FILE_PATH_ANNOTATION = 'WPRArchiveDirectory' +WPR_ARCHIVE_NAME_ANNOTATION = 'WPRArchiveDirectory$ArchiveName' WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION = 'WPRRecordReplayTest' _DEVICE_GOLD_DIR = 'skia_gold' # A map of Android product models to SDK ints. RENDER_TEST_MODEL_SDK_CONFIGS = { # Android x86 emulator. - 'Android SDK built for x86': [23], + 'Android SDK built for x86': [23, 24], + # We would like this to be supported, but it is currently too prone to + # introducing flakiness due to a combination of Gold and Chromium issues. + # See crbug.com/1233700 and skbug.com/12149 for more information. + # 'Pixel 2': [28], } _BATCH_SUFFIX = '_batch' -_TEST_BATCH_MAX_GROUP_SIZE = 256 +# If the batch is too big it starts to fail for command line length reasons. +_LOCAL_TEST_BATCH_MAX_GROUP_SIZE = 200 @contextlib.contextmanager @@ -126,22 +139,38 @@ ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name], check_return=True) -# TODO(jbudorick): Make this private once the instrumentation test_runner -# is deprecated. -def DidPackageCrashOnDevice(package_name, device): + +@contextlib.contextmanager +def _VoiceInteractionService(device, use_voice_interaction_service): + def set_voice_interaction_service(service): + device.RunShellCommand( + ['settings', 'put', 'secure', 'voice_interaction_service', service]) + + default_voice_interaction_service = None + try: + default_voice_interaction_service = device.RunShellCommand( + ['settings', 'get', 'secure', 'voice_interaction_service'], + single_line=True) + + set_voice_interaction_service(use_voice_interaction_service) + yield + finally: + set_voice_interaction_service(default_voice_interaction_service) + + +def DismissCrashDialogs(device): # Dismiss any error dialogs. Limit the number in case we have an error # loop or we are failing to dismiss. + packages = set() try: - for _ in xrange(10): + for _ in range(10): package = device.DismissCrashDialogIfNeeded(timeout=10, retries=1) if not package: - return False - # Assume test package convention of ".test" suffix - if package in package_name: - return True + break + packages.add(package) except device_errors.CommandFailedError: logging.exception('Error while attempting to dismiss crash dialog.') - return False + return packages _CURRENT_FOCUS_CRASH_RE = re.compile( @@ -157,8 +186,7 @@ class LocalDeviceInstrumentationTestRun( local_device_test_run.LocalDeviceTestRun): def __init__(self, env, test_instance): - super(LocalDeviceInstrumentationTestRun, self).__init__( - env, test_instance) + super().__init__(env, test_instance) self._chrome_proxy = None self._context_managers = collections.defaultdict(list) self._flag_changers = {} @@ -166,14 +194,22 @@ self._shared_prefs_to_restore = [] self._skia_gold_session_manager = None self._skia_gold_work_dir = None + self._target_package = _GetTargetPackageName(test_instance.test_apk) #override def TestPackage(self): return self._test_instance.suite + def _GetDataStorageRootDirectory(self, device): + if self._test_instance.store_data_in_app_directory: + # TODO(rmhasan): Add check to makes sure api level > 27. Selinux + # policy on Oreo does not allow app to read files from app data dir + # that were not put there by the app. + return device.GetApplicationDataDirectory(self._target_package) + return device.GetExternalStoragePath() + #override def SetUp(self): - target_package = _GetTargetPackageName(self._test_instance.test_apk) @local_device_environment.handle_shard_failures_with( self._env.DenylistDevice) @@ -193,8 +229,7 @@ # manually invoke its __enter__ and __exit__ methods in setup and # teardown. system_app_context = system_app.ReplaceSystemApp( - dev, self._test_instance.replace_system_package.package, - self._test_instance.replace_system_package.replacement_apk) + dev, self._test_instance.replace_system_package) # Pylint is not smart enough to realize that this field has # an __enter__ method, and will complain loudly. # pylint: disable=no-member @@ -219,7 +254,79 @@ # concurrent adb with this option specified, this should be safe. steps.insert(0, remove_packages) + def install_helper(apk, + modules=None, + fake_modules=None, + permissions=None, + additional_locales=None, + instant_app=False): + + @instrumentation_tracing.no_tracing + @trace_event.traced + def install_helper_internal(d, apk_path=None): + # pylint: disable=unused-argument + d.Install( + apk, + modules=modules, + fake_modules=fake_modules, + permissions=permissions, + additional_locales=additional_locales, + instant_app=instant_app, + force_queryable=self._test_instance.IsApkForceQueryable(apk)) + + return install_helper_internal + + def install_apex_helper(apex): + @instrumentation_tracing.no_tracing + @trace_event.traced + def install_helper_internal(d, apk_path=None): + # pylint: disable=unused-argument + d.InstallApex(apex) + + return install_helper_internal + + def incremental_install_helper(apk, json_path, permissions): + + @trace_event.traced + def incremental_install_helper_internal(d, apk_path=None): + # pylint: disable=unused-argument + installer.Install(d, json_path, apk=apk, permissions=permissions) + + return incremental_install_helper_internal + + steps.extend( + install_apex_helper(apex) + for apex in self._test_instance.additional_apexs) + + steps.extend( + install_helper(apk, instant_app=self._test_instance.IsApkInstant(apk)) + for apk in self._test_instance.additional_apks) + + permissions = self._test_instance.test_apk.GetPermissions() + if self._test_instance.test_apk_incremental_install_json: + if self._test_instance.test_apk_as_instant: + raise Exception('Test APK cannot be installed as an instant ' + 'app if it is incremental') + + steps.append( + incremental_install_helper( + self._test_instance.test_apk, + self._test_instance.test_apk_incremental_install_json, + permissions)) + else: + steps.append( + install_helper(self._test_instance.test_apk, + permissions=permissions, + instant_app=self._test_instance.test_apk_as_instant)) + + # We'll potentially need the package names later for setting app + # compatibility workarounds. + for apk in (self._test_instance.additional_apks + + [self._test_instance.test_apk]): + self._installed_packages.append(apk_helper.GetPackageName(apk)) + if self._test_instance.use_webview_provider: + @trace_event.traced def use_webview_provider(dev): # We need the context manager to be applied before modifying any @@ -230,6 +337,9 @@ # applying the context manager up in test_runner. Instead, we # manually invoke its __enter__ and __exit__ methods in setup and # teardown. + # We do this after installing additional APKs so that + # we can install trichrome library before installing the webview + # provider webview_context = webview_app.UseWebViewProvider( dev, self._test_instance.use_webview_provider) # Pylint is not smart enough to realize that this field has @@ -241,52 +351,21 @@ steps.append(use_webview_provider) - def install_helper(apk, - modules=None, - fake_modules=None, - permissions=None, - additional_locales=None): - - @instrumentation_tracing.no_tracing - @trace_event.traced - def install_helper_internal(d, apk_path=None): - # pylint: disable=unused-argument - d.Install(apk, - modules=modules, - fake_modules=fake_modules, - permissions=permissions, - additional_locales=additional_locales) - - return install_helper_internal - - def incremental_install_helper(apk, json_path, permissions): + if self._test_instance.use_voice_interaction_service: @trace_event.traced - def incremental_install_helper_internal(d, apk_path=None): - # pylint: disable=unused-argument - installer.Install(d, json_path, apk=apk, permissions=permissions) - return incremental_install_helper_internal + def use_voice_interaction_service(device): + voice_interaction_service_context = _VoiceInteractionService( + device, self._test_instance.use_voice_interaction_service) + # Pylint is not smart enough to realize that this field has + # an __enter__ method, and will complain loudly. + # pylint: disable=no-member + voice_interaction_service_context.__enter__() + # pylint: enable=no-member + self._context_managers[str(device)].append( + voice_interaction_service_context) - permissions = self._test_instance.test_apk.GetPermissions() - if self._test_instance.test_apk_incremental_install_json: - steps.append(incremental_install_helper( - self._test_instance.test_apk, - self._test_instance. - test_apk_incremental_install_json, - permissions)) - else: - steps.append( - install_helper( - self._test_instance.test_apk, permissions=permissions)) - - steps.extend( - install_helper(apk) for apk in self._test_instance.additional_apks) - - # We'll potentially need the package names later for setting app - # compatibility workarounds. - for apk in (self._test_instance.additional_apks + - [self._test_instance.test_apk]): - self._installed_packages.append(apk_helper.GetPackageName(apk)) + steps.append(use_voice_interaction_service) # The apk under test needs to be installed last since installing other # apks after will unintentionally clear the fake module directory. @@ -308,6 +387,17 @@ self._test_instance.fake_modules, permissions, self._test_instance.additional_locales)) + # Execute any custom setup shell commands + if self._test_instance.run_setup_commands: + + @trace_event.traced + def run_setup_commands(dev): + for cmd in self._test_instance.run_setup_commands: + logging.info('Running custom setup shell command: %s', cmd) + dev.RunShellCommand(cmd, shell=True, check_return=True) + + steps.append(run_setup_commands) + @trace_event.traced def set_debug_app(dev): # Set debug app in order to enable reading command line flags on user @@ -315,7 +405,7 @@ cmd = ['am', 'set-debug-app', '--persistent'] if self._test_instance.wait_for_java_debugger: cmd.append('-w') - cmd.append(target_package) + cmd.append(self._target_package) dev.RunShellCommand(cmd, check_return=True) @trace_event.traced @@ -332,6 +422,10 @@ shared_pref, setting) @trace_event.traced + def approve_app_links(dev): + self._ToggleAppLinks(dev, 'STATE_APPROVED') + + @trace_event.traced def set_vega_permissions(dev): # Normally, installation of VrCore automatically grants storage # permissions. However, since VrCore is part of the system image on @@ -346,20 +440,32 @@ @instrumentation_tracing.no_tracing def push_test_data(dev): - device_root = posixpath.join(dev.GetExternalStoragePath(), - 'chromium_tests_root') + test_data_root_dir = posixpath.join( + self._GetDataStorageRootDirectory(dev), 'chromium_tests_root') host_device_tuples_substituted = [ - (h, local_device_test_run.SubstituteDeviceRoot(d, device_root)) - for h, d in host_device_tuples] + (h, + local_device_test_run.SubstituteDeviceRoot(d, test_data_root_dir)) + for h, d in host_device_tuples + ] logging.info('Pushing data dependencies.') for h, d in host_device_tuples_substituted: logging.debug(' %r -> %r', h, d) - local_device_environment.place_nomedia_on_device(dev, device_root) + + as_root = self._test_instance.store_data_in_app_directory + local_device_environment.place_nomedia_on_device(dev, + test_data_root_dir, + as_root=as_root) dev.PushChangedFiles(host_device_tuples_substituted, - delete_device_stale=True) + delete_device_stale=True, + as_root=as_root) + if not host_device_tuples_substituted: - dev.RunShellCommand(['rm', '-rf', device_root], check_return=True) - dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True) + dev.RunShellCommand(['rm', '-rf', test_data_root_dir], + check_return=True, + as_root=as_root) + dev.RunShellCommand(['mkdir', '-p', test_data_root_dir], + check_return=True, + as_root=as_root) @trace_event.traced def create_flag_changer(dev): @@ -373,8 +479,8 @@ dev, self._test_instance.timeout_scale) steps += [ - set_debug_app, edit_shared_prefs, push_test_data, create_flag_changer, - set_vega_permissions + set_debug_app, edit_shared_prefs, approve_app_links, push_test_data, + create_flag_changer, set_vega_permissions, DismissCrashDialogs ] def bind_crash_handler(step, dev): @@ -421,7 +527,7 @@ if self._test_instance.wait_for_java_debugger: logging.warning('*' * 80) logging.warning('Waiting for debugger to attach to process: %s', - target_package) + self._target_package) logging.warning('*' * 80) #override @@ -447,6 +553,11 @@ # Remove package-specific configuration dev.RunShellCommand(['am', 'clear-debug-app'], check_return=True) + # Execute any custom teardown shell commands + for cmd in self._test_instance.run_teardown_commands: + logging.info('Running custom teardown shell command: %s', cmd) + dev.RunShellCommand(cmd, shell=True, check_return=True) + valgrind_tools.SetChromeTimeoutScale(dev, None) # Restore any shared preference files that we stored during setup. @@ -456,6 +567,9 @@ for pref_to_restore in self._shared_prefs_to_restore: pref_to_restore.Commit(force_commit=True) + # If we've force approved app links for a package, undo that now. + self._ToggleAppLinks(dev, 'STATE_NO_RESPONSE') + # Context manager exit handlers are applied in reverse order # of the enter handlers. for context in reversed(self._context_managers[str(dev)]): @@ -466,6 +580,24 @@ self._env.parallel_devices.pMap(individual_device_tear_down) + def _ToggleAppLinks(self, dev, state): + # The set-app-links command was added in Android 12 (sdk = 31). The + # restrictions that require us to set the app links were also added in + # Android 12, so doing nothing on earlier Android versions is fine. + if dev.build_version_sdk < version_codes.S: + return + + package = self._test_instance.approve_app_links_package + domain = self._test_instance.approve_app_links_domain + + if not package or not domain: + return + + cmd = [ + 'pm', 'set-app-links', '--package', package, state, domain + ] + dev.RunShellCommand(cmd, check_return=True) + def _CreateFlagChangerIfNeeded(self, device): if str(device) not in self._flag_changers: cmdline_file = 'test-cmdline-file' @@ -479,7 +611,16 @@ device, cmdline_file) #override - def _CreateShards(self, tests): + def _CreateShardsForDevices(self, tests): + """Create shards of tests to run on devices. + + Args: + tests: List containing tests or test batches. + + Returns: + List of tests or batches. + """ + # Each test or test batch will be a single shard. return tests #override @@ -495,6 +636,14 @@ return tests #override + def GetTestsForListing(self): + # Parent class implementation assumes _GetTests() returns strings rather + # than dicts. + test_dicts = self._GetTests() + test_dicts = local_device_test_run.FlattenTestList(test_dicts) + return sorted('{}#{}'.format(d['class'], d['method']) for d in test_dicts) + + #override def _GroupTests(self, tests): batched_tests = dict() other_tests = [] @@ -508,29 +657,60 @@ # Feature flags won't work in instrumentation tests unless the activity # is restarted. # Tests with identical features are grouped to minimize restarts. - if 'Batch$SplitByFeature' in annotations: + # UnitTests that specify flags always use Features.JUnitProcessor, so + # they don't need to be split. + if batch_name != 'UnitTests': if 'Features$EnableFeatures' in annotations: batch_name += '|enabled:' + ','.join( sorted(annotations['Features$EnableFeatures']['value'])) if 'Features$DisableFeatures' in annotations: batch_name += '|disabled:' + ','.join( sorted(annotations['Features$DisableFeatures']['value'])) + if 'CommandLineFlags$Add' in annotations: + batch_name += '|cmd_line_add:' + ','.join( + sorted(annotations['CommandLineFlags$Add']['value'])) + if 'CommandLineFlags$Remove' in annotations: + batch_name += '|cmd_line_remove:' + ','.join( + sorted(annotations['CommandLineFlags$Remove']['value'])) - if not batch_name in batched_tests: - batched_tests[batch_name] = [] - batched_tests[batch_name].append(test) + batched_tests.setdefault(batch_name, []).append(test) else: other_tests.append(test) + def dict2list(d): + if isinstance(d, dict): + return sorted([(k, dict2list(v)) for k, v in d.items()]) + if isinstance(d, list): + return [dict2list(v) for v in d] + if isinstance(d, tuple): + return tuple(dict2list(v) for v in d) + return d + + test_count = sum( + [len(test) - 1 for test in tests if self._CountTestsIndividually(test)]) + test_count += len(tests) + if self._test_instance.total_external_shards > 1: + # Calculate suitable test batch max group size based on average partition + # size. The batch size should be below partition size to balance between + # shards. Choose to divide by 3 as it works fine with most of test suite + # without increasing too much setup/teardown time for batch tests. + test_batch_max_group_size = \ + max(1, test_count // self._test_instance.total_external_shards // 3) + else: + test_batch_max_group_size = _LOCAL_TEST_BATCH_MAX_GROUP_SIZE + all_tests = [] - for _, tests in batched_tests.items(): - tests.sort() # Ensure a consistent ordering across external shards. + for _, btests in list(batched_tests.items()): + # Ensure a consistent ordering across external shards. + btests.sort(key=dict2list) all_tests.extend([ - tests[i:i + _TEST_BATCH_MAX_GROUP_SIZE] - for i in range(0, len(tests), _TEST_BATCH_MAX_GROUP_SIZE) + btests[i:i + test_batch_max_group_size] + for i in range(0, len(btests), test_batch_max_group_size) ]) all_tests.extend(other_tests) - return all_tests + # Sort all tests by hash. + # TODO(crbug.com/1257820): Add sorting logic back to _PartitionTests. + return self._SortTests(all_tests) #override def _GetUniqueTestName(self, test): @@ -540,6 +720,9 @@ def _RunTest(self, device, test): extras = {} + if self._test_instance.is_unit_test: + extras[_EXTRA_TEST_IS_UNIT] = 'true' + # Provide package name under test for apk_under_test. if self._test_instance.apk_under_test: package_name = self._test_instance.apk_under_test.GetPackageName() @@ -552,8 +735,6 @@ (test[0]['class'], test[0]['method']) if isinstance(test, list) else '%s_%s' % (test['class'], test['method'])) - if self._test_instance.jacoco_coverage_type: - coverage_basename += "_" + self._test_instance.jacoco_coverage_type extras['coverage'] = 'true' coverage_directory = os.path.join( device.GetExternalStoragePath(), 'chrome', 'test', 'coverage') @@ -563,6 +744,16 @@ coverage_device_file = os.path.join(coverage_directory, coverage_basename) coverage_device_file += '.exec' extras['coverageFile'] = coverage_device_file + + if self._test_instance.enable_breakpad_dump: + # Use external storage directory so that the breakpad dump can be accessed + # by the test APK in addition to the apk_under_test. + breakpad_dump_directory = os.path.join(device.GetExternalStoragePath(), + 'chromium_dumps') + if device.PathExists(breakpad_dump_directory): + device.RemovePath(breakpad_dump_directory, recursive=True) + flags_to_add.append('--breakpad-dump-location=' + breakpad_dump_directory) + # Save screenshot if screenshot dir is specified (save locally) or if # a GS bucket is passed (save in cloud). screenshot_device_file = device_temp_file.DeviceTempFile( @@ -592,7 +783,7 @@ i = self._GetTimeoutFromAnnotations(t['annotations'], n) return (n, i) - test_names, timeouts = zip(*(name_and_timeout(t) for t in test)) + test_names, timeouts = list(zip(*(name_and_timeout(t) for t in test))) test_name = instrumentation_test_instance.GetTestName( test[0]) + _BATCH_SUFFIX @@ -642,10 +833,12 @@ wpr_archive_path, os.path.exists(wpr_archive_path))) + file_name = _GetWPRArchiveFileName( + test) or self._GetUniqueTestName(test) + '.wprgo' + # Some linux version does not like # in the name. Replaces it with __. - archive_path = os.path.join( - wpr_archive_path, - _ReplaceUncommonChars(self._GetUniqueTestName(test)) + '.wprgo') + archive_path = os.path.join(wpr_archive_path, + _ReplaceUncommonChars(file_name)) if not os.path.exists(_WPR_GO_LINUX_X86_64_PATH): # If we got to this stage, then we should have @@ -664,6 +857,9 @@ self._CreateFlagChangerIfNeeded(device) self._flag_changers[str(device)].PushFlags(add=flags_to_add) + if self._test_instance.store_data_in_app_directory: + extras.update({'fetchTestDataFromAppDataDir': 'true'}) + time_ms = lambda: int(time.time() * 1e3) start_ms = time_ms() @@ -704,9 +900,14 @@ try: if not os.path.exists(self._test_instance.coverage_directory): os.makedirs(self._test_instance.coverage_directory) - device.PullFile(coverage_device_file, - self._test_instance.coverage_directory) - device.RemovePath(coverage_device_file, True) + # Retries add time to test execution. + if device.PathExists(coverage_device_file, retries=0): + device.PullFile(coverage_device_file, + self._test_instance.coverage_directory) + device.RemovePath(coverage_device_file, True) + else: + logging.warning('Coverage file does not exist: %s', + coverage_device_file) except (OSError, base_error.BaseError) as e: logging.warning('Failed to handle coverage data after tests: %s', e) @@ -760,6 +961,25 @@ self._chrome_proxy.wpr_archive_path) self._chrome_proxy = None + def pull_baseline_profile(): + # Search though status responses for the one with the key we are + # looking for. + for _, bundle in statuses: + baseline_profile_path = bundle.get( + 'additionalTestOutputFile_baseline-profile-ts') + if baseline_profile_path: + # Found it. + break + else: + # This test does not generate a baseline profile. + return + with self._env.output_manager.ArchivedTempfile( + 'baseline_profile.txt', 'baseline_profile') as baseline_profile: + device.PullFile(baseline_profile_path, baseline_profile.name) + _SetLinkOnResults(results, test_name, 'baseline_profile', + baseline_profile.Link()) + logging.warning('Baseline Profile Location %s', baseline_profile.Link()) + # While constructing the TestResult objects, we can parallelize several # steps that involve ADB. These steps should NOT depend on any info in @@ -767,7 +987,8 @@ # determined. post_test_steps = [ restore_flags, restore_timeout_scale, stop_chrome_proxy, - handle_coverage_data, handle_render_test_data, pull_ui_screen_captures + handle_coverage_data, handle_render_test_data, + pull_ui_screen_captures, pull_baseline_profile ] if self._env.concurrent_adb: reraiser_thread.RunAsync(post_test_steps) @@ -794,10 +1015,24 @@ # Update the result type if we detect a crash. try: - if DidPackageCrashOnDevice(self._test_instance.test_package, device): + crashed_packages = DismissCrashDialogs(device) + # Assume test package convention of ".test" suffix + if any(p in self._test_instance.test_package for p in crashed_packages): for r in results: if r.GetType() == base_test_result.ResultType.UNKNOWN: r.SetType(base_test_result.ResultType.CRASH) + elif (crashed_packages and len(results) == 1 + and results[0].GetType() != base_test_result.ResultType.PASS): + # Add log message and set failure reason if: + # 1) The app crash was likely not caused by the test. + # AND + # 2) The app crash possibly caused the test to fail. + # Crashes of the package under test are assumed to be the test's fault. + _AppendToLogForResult( + results[0], 'OS displayed error dialogs for {}'.format( + ', '.join(crashed_packages))) + results[0].SetFailureReason('{} Crashed'.format( + ','.join(crashed_packages))) except device_errors.CommandTimeoutError: logging.warning('timed out when detecting/dismissing error dialogs') # Attach screenshot to the test to help with debugging the dialog boxes. @@ -815,7 +1050,7 @@ # Handle failures by: # - optionally taking a screenshot - # - logging the raw output at INFO level + # - logging the raw output at ERROR level # - clearing the application state while persisting permissions if any(r.GetType() not in (base_test_result.ResultType.PASS, base_test_result.ResultType.SKIP) @@ -823,17 +1058,17 @@ self._SaveScreenshot(device, screenshot_device_file, test_display_name, results, 'post_test_screenshot') - logging.info('detected failure in %s. raw output:', test_display_name) + logging.error('detected failure in %s. raw output:', test_display_name) for l in output: - logging.info(' %s', l) - if (not self._env.skip_clear_data - and self._test_instance.package_info): - permissions = ( - self._test_instance.apk_under_test.GetPermissions() - if self._test_instance.apk_under_test - else None) - device.ClearApplicationState(self._test_instance.package_info.package, - permissions=permissions) + logging.error(' %s', l) + if not self._env.skip_clear_data: + if self._test_instance.package_info: + permissions = (self._test_instance.apk_under_test.GetPermissions() + if self._test_instance.apk_under_test else None) + device.ClearApplicationState(self._test_instance.package_info.package, + permissions=permissions) + if self._test_instance.enable_breakpad_dump: + device.RemovePath(breakpad_dump_directory, recursive=True) else: logging.debug('raw output from %s:', test_display_name) for l in output: @@ -931,11 +1166,15 @@ logging.info('Could not get tests from pickle: %s', e) logging.info('Getting tests by having %s list them.', self._test_instance.junit4_runner_class) + # We need to use GetAppWritablePath instead of GetExternalStoragePath + # here because we will not have applied legacy storage workarounds on R+ + # yet. + # TODO(rmhasan): Figure out how to create the temp file inside the test + # app's data directory. Currently when the temp file is created read + # permissions are only given to the app's user id. Therefore we can't + # pull the file from the device. def list_tests(d): def _run(dev): - # We need to use GetAppWritablePath instead of GetExternalStoragePath - # here because we will not have applied legacy storage workarounds on R+ - # yet. with device_temp_file.DeviceTempFile( dev.adb, suffix='.json', dir=dev.GetAppWritablePath()) as dev_test_list_json: @@ -989,24 +1228,22 @@ logcat_file = None logmon = None try: - with self._env.output_manager.ArchivedTempfile( - stream_name, 'logcat') as logcat_file: + with self._env.output_manager.ArchivedTempfile(stream_name, + 'logcat') as logcat_file: with logcat_monitor.LogcatMonitor( device.adb, filter_specs=local_device_environment.LOGCAT_FILTERS, output_file=logcat_file.name, transform_func=self._test_instance.MaybeDeobfuscateLines, check_error=False) as logmon: - with _LogTestEndpoints(device, test_name): - with contextlib_ext.Optional( - trace_event.trace(test_name), - self._env.trace_output): - yield logcat_file + with contextlib_ext.Optional(trace_event.trace(test_name), + self._env.trace_output): + yield logcat_file finally: if logmon: logmon.Close() if logcat_file and logcat_file.Link(): - logging.info('Logcat saved to %s', logcat_file.Link()) + logging.critical('Logcat saved to %s', logcat_file.Link()) def _SaveTraceData(self, trace_device_file, device, test_class): trace_host_file = self._env.trace_output @@ -1014,8 +1251,8 @@ if device.FileExists(trace_device_file.name): try: java_trace_json = device.ReadFile(trace_device_file.name) - except IOError: - raise Exception('error pulling trace file from device') + except IOError as e: + raise Exception('error pulling trace file from device') from e finally: trace_device_file.close() @@ -1125,16 +1362,17 @@ # that implies that we aren't actively maintaining baselines for the # test. This helps prevent unrelated CLs from getting comments posted to # them. - # Additionally, add the ignore if we're running on a trybot and this is - # not our final retry attempt in order to prevent unrelated CLs from - # getting spammed if a test is flaky. should_rewrite = False with open(json_path) as infile: # All the key/value pairs in the JSON file are strings, so convert # to a bool. json_dict = json.load(infile) - fail_on_unsupported = json_dict.get('fail_on_unsupported_configs', - 'false') + optional_dict = json_dict.get('optional_keys', {}) + if 'optional_keys' in json_dict: + should_rewrite = True + del json_dict['optional_keys'] + fail_on_unsupported = optional_dict.get('fail_on_unsupported_configs', + 'false') fail_on_unsupported = fail_on_unsupported.lower() == 'true' # Grab the full test name so we can associate the comparison with a # particular test, which is necessary if tests are batched together. @@ -1148,21 +1386,14 @@ running_on_unsupported = ( device.build_version_sdk not in RENDER_TEST_MODEL_SDK_CONFIGS.get( device.product_model, []) and not fail_on_unsupported) - # TODO(skbug.com/10787): Remove the ignore on non-final retry once we - # fully switch over to using the Gerrit plugin for surfacing Gold - # information since it does not spam people with emails due to automated - # comments. - not_final_retry = self._env.current_try + 1 != self._env.max_tries - tryjob_but_not_final_retry =\ - not_final_retry and gold_properties.IsTryjobRun() - should_ignore_in_gold =\ - running_on_unsupported or tryjob_but_not_final_retry + should_ignore_in_gold = running_on_unsupported # We still want to fail the test even if we're ignoring the image in # Gold if we're running on a supported configuration, so # should_ignore_in_gold != should_hide_failure. should_hide_failure = running_on_unsupported if should_ignore_in_gold: - should_rewrite = True + # This is put in the regular keys dict instead of the optional one + # because ignore rules do not apply to optional keys. json_dict['ignore'] = '1' if should_rewrite: with open(json_path, 'w') as outfile: @@ -1176,7 +1407,8 @@ name=render_name, png_file=image_path, output_manager=self._env.output_manager, - use_luci=use_luci) + use_luci=use_luci, + optional_keys=optional_dict) except Exception as e: # pylint: disable=broad-except _FailTestIfNecessary(results, full_test_name) _AppendToLog(results, full_test_name, @@ -1284,7 +1516,13 @@ return True #override - def _ShouldShard(self): + def _ShouldShardTestsForDevices(self): + """Shard tests across several devices. + + Returns: + True if tests should be sharded across several devices, + False otherwise. + """ return True @classmethod @@ -1314,10 +1552,8 @@ """Determines whether a test or a list of tests is a WPR RecordReplay Test.""" if not isinstance(test, list): test = [test] - return any([ - WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION in t['annotations'].get( - FEATURE_ANNOTATION, {}).get('value', ()) for t in test - ]) + return any(WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION in t['annotations'].get( + FEATURE_ANNOTATION, {}).get('value', ()) for t in test) def _GetWPRArchivePath(test): @@ -1326,6 +1562,13 @@ {}).get('value', ()) +def _GetWPRArchiveFileName(test): + """Retrieves the WPRArchiveDirectory.ArchiveName annotation.""" + value = test['annotations'].get(WPR_ARCHIVE_NAME_ANNOTATION, + {}).get('value', None) + return value[0] if value else None + + def _ReplaceUncommonChars(original): """Replaces uncommon characters with __.""" if not original: @@ -1341,8 +1584,8 @@ """Determines if a test or list of tests has a RenderTest amongst them.""" if not isinstance(test, list): test = [test] - return any([RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get( - FEATURE_ANNOTATION, {}).get('value', ()) for t in test]) + return any(RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get( + FEATURE_ANNOTATION, {}).get('value', ()) for t in test) def _GenerateRenderTestHtml(image_name, failure_link, golden_link, diff_link): @@ -1414,7 +1657,11 @@ for result in results: if found_matching_test and result.GetName() != full_test_name: continue - result.SetLog(result.GetLog() + '\n' + line) + _AppendToLogForResult(result, line) + + +def _AppendToLogForResult(result, line): + result.SetLog(result.GetLog() + '\n' + line) def _SetLinkOnResults(results, full_test_name, link_name, link): @@ -1450,7 +1697,7 @@ True if one of the results in |results| has the same name as |full_test_name|, otherwise False. """ - return any([r for r in results if r.GetName() == full_test_name]) + return any(r for r in results if r.GetName() == full_test_name) def _ShouldReportNoMatchingResult(full_test_name):
diff --git a/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py b/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py index 7870cd1..fb41572 100755 --- a/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py +++ b/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,8 +7,9 @@ # pylint: disable=protected-access -from __future__ import absolute_import + import unittest +import mock # pylint: disable=import-error from pylib.base import base_test_result from pylib.base import mock_environment @@ -19,7 +20,7 @@ class LocalDeviceInstrumentationTestRunTest(unittest.TestCase): def setUp(self): - super(LocalDeviceInstrumentationTestRunTest, self).setUp() + super().setUp() self._env = mock_environment.MockEnvironment() self._ti = mock_test_instance.MockTestInstance() self._obj = ( @@ -164,6 +165,33 @@ with self.assertRaises(ValueError): local_device_instrumentation_test_run._ReplaceUncommonChars(original) + def testStoreDataInAppDir(self): + env = mock.MagicMock() + test_instance = mock.MagicMock() + test_instance.store_data_in_app_directory = True + device = mock.MagicMock() + + device.GetApplicationDataDirectory.return_value = 'app_dir' + device.GetExternalStoragePath.return_value = 'external_dir' + test_run = ( + local_device_instrumentation_test_run.LocalDeviceInstrumentationTestRun( + env, test_instance)) + self.assertEqual(test_run._GetDataStorageRootDirectory(device), 'app_dir') + + def testStoreDataInExternalDir(self): + env = mock.MagicMock() + test_instance = mock.MagicMock() + test_instance.store_data_in_app_directory = False + device = mock.MagicMock() + + device.GetApplicationDataDirectory.return_value = 'app_dir' + device.GetExternalStoragePath.return_value = 'external_dir' + test_run = ( + local_device_instrumentation_test_run.LocalDeviceInstrumentationTestRun( + env, test_instance)) + self.assertEqual(test_run._GetDataStorageRootDirectory(device), + 'external_dir') + if __name__ == '__main__': unittest.main(verbosity=2)
diff --git a/build/android/pylib/local/device/local_device_monkey_test_run.py b/build/android/pylib/local/device/local_device_monkey_test_run.py index f0d2339..e90cbbd 100644 --- a/build/android/pylib/local/device/local_device_monkey_test_run.py +++ b/build/android/pylib/local/device/local_device_monkey_test_run.py
@@ -1,8 +1,8 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import logging from six.moves import range # pylint: disable=redefined-builtin @@ -16,9 +16,6 @@ _CHROME_PACKAGE = constants.PACKAGE_INFO['chrome'].package class LocalDeviceMonkeyTestRun(local_device_test_run.LocalDeviceTestRun): - def __init__(self, env, test_instance): - super(LocalDeviceMonkeyTestRun, self).__init__(env, test_instance) - def TestPackage(self): return 'monkey' @@ -91,11 +88,26 @@ pass #override - def _CreateShards(self, tests): + def _CreateShardsForDevices(self, tests): + """Create shards of tests to run on devices. + + Args: + tests: List containing tests or test batches. + + Returns: + True if tests should be sharded across several devices, + False otherwise. + """ return tests #override - def _ShouldShard(self): + def _ShouldShardTestsForDevices(self): + """Shard tests across several devices. + + Returns: + True if tests should be sharded across several devices, + False otherwise. + """ # TODO(mikecase): Run Monkey test concurrently on each attached device. return False
diff --git a/build/android/pylib/local/device/local_device_test_run.py b/build/android/pylib/local/device/local_device_test_run.py index 6fa0af7..0e0b930 100644 --- a/build/android/pylib/local/device/local_device_test_run.py +++ b/build/android/pylib/local/device/local_device_test_run.py
@@ -1,12 +1,16 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import fnmatch +import hashlib import logging import posixpath import signal -import thread +try: + import _thread as thread +except ImportError: + import thread import threading from devil import base_error @@ -30,10 +34,9 @@ def SubstituteDeviceRoot(device_path, device_root): if not device_path: return device_root - elif isinstance(device_path, list): + if isinstance(device_path, list): return posixpath.join(*(p if p else device_root for p in device_path)) - else: - return device_path + return device_path class TestsTerminated(Exception): @@ -42,22 +45,22 @@ class InvalidShardingSettings(Exception): def __init__(self, shard_index, total_shards): - super(InvalidShardingSettings, self).__init__( - 'Invalid sharding settings. shard_index: %d total_shards: %d' - % (shard_index, total_shards)) + super().__init__( + 'Invalid sharding settings. shard_index: %d total_shards: %d' % + (shard_index, total_shards)) class LocalDeviceTestRun(test_run.TestRun): def __init__(self, env, test_instance): - super(LocalDeviceTestRun, self).__init__(env, test_instance) + super().__init__(env, test_instance) self._tools = {} # This is intended to be filled by a child class. self._installed_packages = [] env.SetPreferredAbis(test_instance.GetPreferredAbis()) #override - def RunTests(self, results): + def RunTests(self, results, raw_logs_fh=None): tests = self._GetTests() exit_now = threading.Event() @@ -100,22 +103,21 @@ results.AddResults( base_test_result.BaseTestResult( self._GetUniqueTestName(t), - base_test_result.ResultType.TIMEOUT) - for t in test) + base_test_result.ResultType.TIMEOUT) for t in test) else: results.AddResult( base_test_result.BaseTestResult( self._GetUniqueTestName(test), base_test_result.ResultType.TIMEOUT)) - except Exception as e: # pylint: disable=broad-except + except device_errors.DeviceUnreachableError: + # If the device is no longer reachable then terminate this + # run_tests_on_device call. + raise + except base_error.BaseError: + # If we get a device error but believe the device is still + # reachable, attempt to continue using it. if isinstance(tests, test_collection.TestCollection): rerun = test - if (isinstance(e, device_errors.DeviceUnreachableError) - or not isinstance(e, base_error.BaseError)): - # If we get a device error but believe the device is still - # reachable, attempt to continue using it. Otherwise, raise - # the exception and terminate this run_tests_on_device call. - raise consecutive_device_errors += 1 if consecutive_device_errors >= 3: @@ -182,9 +184,9 @@ results.append(try_results) try: - if self._ShouldShard(): + if self._ShouldShardTestsForDevices(): tc = test_collection.TestCollection( - self._CreateShards(grouped_tests)) + self._CreateShardsForDevices(grouped_tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: @@ -229,17 +231,15 @@ tests_and_results = {} for test, name in tests_and_names: if name.endswith('*'): - tests_and_results[name] = ( - test, - [r for n, r in all_test_results.iteritems() - if fnmatch.fnmatch(n, name)]) + tests_and_results[name] = (test, [ + r for n, r in all_test_results.items() if fnmatch.fnmatch(n, name) + ]) else: tests_and_results[name] = (test, all_test_results.get(name)) - failed_tests_and_results = ( - (test, result) for test, result in tests_and_results.itervalues() - if is_failure_result(result) - ) + failed_tests_and_results = ((test, result) + for test, result in tests_and_results.values() + if is_failure_result(result)) return [t for t, r in failed_tests_and_results if self._ShouldRetry(t, r)] @@ -252,6 +252,10 @@ sharded_tests = [] + # Sort tests by hash. + # TODO(crbug.com/1257820): Add sorting logic back to _PartitionTests. + tests = self._SortTests(tests) + # Group tests by tests that should run in the same test invocation - either # unit tests or batched tests. grouped_tests = self._GroupTests(tests) @@ -268,6 +272,14 @@ sharded_tests.append(t) return sharded_tests + # Sort by hash so we don't put all tests in a slow suite in the same + # partition. + def _SortTests(self, tests): + return sorted(tests, + key=lambda t: hashlib.sha256( + self._GetUniqueTestName(t[0] if isinstance(t, list) else t + ).encode()).hexdigest()) + # Partition tests evenly into |num_desired_partitions| partitions where # possible. However, many constraints make partitioning perfectly impossible. # If the max_partition_size isn't large enough, extra partitions may be @@ -281,24 +293,9 @@ # pylint: disable=no-self-use partitions = [] - # Sort by hash so we don't put all tests in a slow suite in the same - # partition. - tests = sorted( - tests, - key=lambda t: hash( - self._GetUniqueTestName(t[0] if isinstance(t, list) else t))) - - def CountTestsIndividually(test): - if not isinstance(test, list): - return False - annotations = test[0]['annotations'] - # UnitTests tests are really fast, so to balance shards better, count - # UnitTests Batches as single tests. - return ('Batch' not in annotations - or annotations['Batch']['value'] != 'UnitTests') num_not_yet_allocated = sum( - [len(test) - 1 for test in tests if CountTestsIndividually(test)]) + [len(test) - 1 for test in tests if self._CountTestsIndividually(test)]) num_not_yet_allocated += len(tests) # Fast linear partition approximation capped by max_partition_size. We @@ -309,8 +306,7 @@ partitions.append([]) last_partition_size = 0 for test in tests: - test_count = len(test) if CountTestsIndividually(test) else 1 - num_not_yet_allocated -= test_count + test_count = len(test) if self._CountTestsIndividually(test) else 1 # Make a new shard whenever we would overfill the previous one. However, # if the size of the test group is larger than the max partition size on # its own, just put the group in its own shard instead of splitting up the @@ -318,9 +314,6 @@ if (last_partition_size + test_count > partition_size and last_partition_size > 0): num_desired_partitions -= 1 - partitions.append([]) - partitions[-1].append(test) - last_partition_size = test_count if num_desired_partitions <= 0: # Too many tests for number of partitions, just fill all partitions # beyond num_desired_partitions. @@ -329,21 +322,36 @@ # Re-balance remaining partitions. partition_size = min(num_not_yet_allocated // num_desired_partitions, max_partition_size) + partitions.append([]) + partitions[-1].append(test) + last_partition_size = test_count else: partitions[-1].append(test) last_partition_size += test_count + num_not_yet_allocated -= test_count + if not partitions[-1]: partitions.pop() return partitions + def _CountTestsIndividually(self, test): + # pylint: disable=no-self-use + if not isinstance(test, list): + return False + annotations = test[0]['annotations'] + # UnitTests tests are really fast, so to balance shards better, count + # UnitTests Batches as single tests. + return ('Batch' not in annotations + or annotations['Batch']['value'] != 'UnitTests') + def GetTool(self, device): if str(device) not in self._tools: self._tools[str(device)] = valgrind_tools.CreateTool( self._env.tool, device) return self._tools[str(device)] - def _CreateShards(self, tests): + def _CreateShardsForDevices(self, tests): raise NotImplementedError def _GetUniqueTestName(self, test): @@ -354,6 +362,13 @@ # pylint: disable=no-self-use,unused-argument return True + #override + def GetTestsForListing(self): + ret = self._GetTests() + ret = FlattenTestList(ret) + ret.sort() + return ret + def _GetTests(self): raise NotImplementedError @@ -364,10 +379,21 @@ def _RunTest(self, device, test): raise NotImplementedError - def _ShouldShard(self): + def _ShouldShardTestsForDevices(self): raise NotImplementedError +def FlattenTestList(values): + """Returns a list with all nested lists (shard groupings) expanded.""" + ret = [] + for v in values: + if isinstance(v, list): + ret += v + else: + ret.append(v) + return ret + + def SetAppCompatibilityFlagsIfNecessary(packages, device): """Sets app compatibility flags on the given packages and device.
diff --git a/build/android/pylib/local/device/local_device_test_run_test.py b/build/android/pylib/local/device/local_device_test_run_test.py index 77bbc2e..5f0068a 100755 --- a/build/android/pylib/local/device/local_device_test_run_test.py +++ b/build/android/pylib/local/device/local_device_test_run_test.py
@@ -1,11 +1,11 @@ -#!/usr/bin/env vpython -# Copyright 2016 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # pylint: disable=protected-access -from __future__ import absolute_import + import unittest from pylib.base import base_test_result @@ -17,29 +17,27 @@ class SubstituteDeviceRootTest(unittest.TestCase): def testNoneDevicePath(self): - self.assertEquals( + self.assertEqual( '/fake/device/root', - local_device_test_run.SubstituteDeviceRoot( - None, '/fake/device/root')) + local_device_test_run.SubstituteDeviceRoot(None, '/fake/device/root')) def testStringDevicePath(self): - self.assertEquals( + self.assertEqual( '/another/fake/device/path', - local_device_test_run.SubstituteDeviceRoot( - '/another/fake/device/path', '/fake/device/root')) + local_device_test_run.SubstituteDeviceRoot('/another/fake/device/path', + '/fake/device/root')) def testListWithNoneDevicePath(self): - self.assertEquals( + self.assertEqual( '/fake/device/root/subpath', - local_device_test_run.SubstituteDeviceRoot( - [None, 'subpath'], '/fake/device/root')) + local_device_test_run.SubstituteDeviceRoot([None, 'subpath'], + '/fake/device/root')) def testListWithoutNoneDevicePath(self): - self.assertEquals( + self.assertEqual( '/another/fake/device/path', local_device_test_run.SubstituteDeviceRoot( - ['/', 'another', 'fake', 'device', 'path'], - '/fake/device/root')) + ['/', 'another', 'fake', 'device', 'path'], '/fake/device/root')) class TestLocalDeviceTestRun(local_device_test_run.LocalDeviceTestRun): @@ -47,8 +45,7 @@ # pylint: disable=abstract-method def __init__(self): - super(TestLocalDeviceTestRun, self).__init__( - mock.MagicMock(), mock.MagicMock()) + super().__init__(mock.MagicMock(), mock.MagicMock()) class TestLocalDeviceNonStringTestRun( @@ -57,8 +54,7 @@ # pylint: disable=abstract-method def __init__(self): - super(TestLocalDeviceNonStringTestRun, self).__init__( - mock.MagicMock(), mock.MagicMock()) + super().__init__(mock.MagicMock(), mock.MagicMock()) def _GetUniqueTestName(self, test): return test['name'] @@ -66,6 +62,11 @@ class LocalDeviceTestRunTest(unittest.TestCase): + def testSortTests(self): + test_run = TestLocalDeviceTestRun() + self.assertEqual(test_run._SortTests(['a', 'b', 'c', 'd', 'e', 'f', 'g']), + ['d', 'f', 'c', 'b', 'e', 'a', 'g']) + def testGetTestsToRetry_allTestsPassed(self): results = [ base_test_result.BaseTestResult( @@ -80,7 +81,7 @@ test_run = TestLocalDeviceTestRun() tests_to_retry = test_run._GetTestsToRetry(tests, try_results) - self.assertEquals(0, len(tests_to_retry)) + self.assertEqual(0, len(tests_to_retry)) def testGetTestsToRetry_testFailed(self): results = [ @@ -96,7 +97,7 @@ test_run = TestLocalDeviceTestRun() tests_to_retry = test_run._GetTestsToRetry(tests, try_results) - self.assertEquals(1, len(tests_to_retry)) + self.assertEqual(1, len(tests_to_retry)) self.assertIn('Test1', tests_to_retry) def testGetTestsToRetry_testUnknown(self): @@ -111,7 +112,7 @@ test_run = TestLocalDeviceTestRun() tests_to_retry = test_run._GetTestsToRetry(tests, try_results) - self.assertEquals(1, len(tests_to_retry)) + self.assertEqual(1, len(tests_to_retry)) self.assertIn('Test1', tests_to_retry) def testGetTestsToRetry_wildcardFilter_allPass(self): @@ -128,7 +129,7 @@ test_run = TestLocalDeviceTestRun() tests_to_retry = test_run._GetTestsToRetry(tests, try_results) - self.assertEquals(0, len(tests_to_retry)) + self.assertEqual(0, len(tests_to_retry)) def testGetTestsToRetry_wildcardFilter_oneFails(self): results = [ @@ -144,7 +145,7 @@ test_run = TestLocalDeviceTestRun() tests_to_retry = test_run._GetTestsToRetry(tests, try_results) - self.assertEquals(1, len(tests_to_retry)) + self.assertEqual(1, len(tests_to_retry)) self.assertIn('TestCase.*', tests_to_retry) def testGetTestsToRetry_nonStringTests(self): @@ -164,9 +165,9 @@ test_run = TestLocalDeviceNonStringTestRun() tests_to_retry = test_run._GetTestsToRetry(tests, try_results) - self.assertEquals(1, len(tests_to_retry)) + self.assertEqual(1, len(tests_to_retry)) self.assertIsInstance(tests_to_retry[0], dict) - self.assertEquals(tests[1], tests_to_retry[0]) + self.assertEqual(tests[1], tests_to_retry[0]) if __name__ == '__main__':
diff --git a/build/android/pylib/local/emulator/OWNERS b/build/android/pylib/local/emulator/OWNERS index 0853590..36abc18 100644 --- a/build/android/pylib/local/emulator/OWNERS +++ b/build/android/pylib/local/emulator/OWNERS
@@ -1,4 +1,3 @@ bpastene@chromium.org hypan@google.com jbudorick@chromium.org -liaoyuke@chromium.org
diff --git a/build/android/pylib/local/emulator/__init__.py b/build/android/pylib/local/emulator/__init__.py index 4a12e35..401c54b 100644 --- a/build/android/pylib/local/emulator/__init__.py +++ b/build/android/pylib/local/emulator/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/local/emulator/avd.py b/build/android/pylib/local/emulator/avd.py index 51365eb..62db9b5 100644 --- a/build/android/pylib/local/emulator/avd.py +++ b/build/android/pylib/local/emulator/avd.py
@@ -1,9 +1,10 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import +import collections import contextlib +import glob import json import logging import os @@ -11,11 +12,15 @@ import stat import subprocess import threading +import time from google.protobuf import text_format # pylint: disable=import-error +from devil.android import apk_helper from devil.android import device_utils +from devil.android import settings from devil.android.sdk import adb_wrapper +from devil.android.tools import system_app from devil.utils import cmd_helper from devil.utils import timeout_retry from py_utils import tempfile_ext @@ -23,9 +28,23 @@ from pylib.local.emulator import ini from pylib.local.emulator.proto import avd_pb2 -_ALL_PACKAGES = object() -_DEFAULT_AVDMANAGER_PATH = os.path.join( - constants.ANDROID_SDK_ROOT, 'cmdline-tools', 'latest', 'bin', 'avdmanager') +# A common root directory to store the CIPD packages for creating or starting +# the emulator instance, e.g. emulator binary, system images, AVDs. +COMMON_CIPD_ROOT = os.path.join(constants.DIR_SOURCE_ROOT, '.android_emulator') + +# Packages that are needed for runtime. +_PACKAGES_RUNTIME = object() +# Packages that are needed during AVD creation. +_PACKAGES_CREATION = object() +# All the packages that could exist in the AVD config file. +_PACKAGES_ALL = object() + +# These files are used as backing files for corresponding qcow2 images. +_BACKING_FILES = ('system.img', 'vendor.img') + +_DEFAULT_AVDMANAGER_PATH = os.path.join(constants.ANDROID_SDK_ROOT, + 'cmdline-tools', 'latest', 'bin', + 'avdmanager') # Default to a 480dp mdpi screen (a relatively large phone). # See https://developer.android.com/training/multiscreen/screensizes # and https://developer.android.com/training/multiscreen/screendensities @@ -34,6 +53,22 @@ _DEFAULT_SCREEN_HEIGHT = 960 _DEFAULT_SCREEN_WIDTH = 480 +# Default to swiftshader_indirect since it works for most cases. +_DEFAULT_GPU_MODE = 'swiftshader_indirect' + +# The snapshot name to load/save when writable_system=False. +# This is the default name used by the emulator binary. +_DEFAULT_SNAPSHOT_NAME = 'default_boot' + +# crbug.com/1275767: Set long press timeout to 1000ms to reduce the flakiness +# caused by click being incorrectly interpreted as longclick. +_LONG_PRESS_TIMEOUT = '1000' + +# The snapshot name to load/save when writable_system=True +_SYSTEM_SNAPSHOT_NAME = 'boot_with_system' + +_SDCARD_NAME = 'cr-sdcard.img' + class AvdException(Exception): """Raised when this module has a problem interacting with an AVD.""" @@ -49,6 +84,8 @@ message_parts.append(' stderr:') message_parts.extend(' %s' % line for line in stderr.splitlines()) + # avd.py is executed with python2. + # pylint: disable=R1725 super(AvdException, self).__init__('\n'.join(message_parts)) @@ -64,7 +101,47 @@ return text_format.Merge(avd_proto_file.read(), avd_pb2.Avd()) -class _AvdManagerAgent(object): +def _FindMinSdkFile(apk_dir, min_sdk): + """Finds the apk file associated with the min_sdk file. + + This reads a version.json file located in the apk_dir to find an apk file + that is closest without going over the min_sdk. + + Args: + apk_dir: The directory to look for apk files. + min_sdk: The minimum sdk version supported by the device. + + Returns: + The path to the file that suits the minSdkFile or None + """ + json_file = os.path.join(apk_dir, 'version.json') + if not os.path.exists(json_file): + logging.error('Json version file not found: %s', json_file) + return None + + min_sdk_found = None + curr_min_sdk_version = 0 + with open(json_file) as f: + data = json.loads(f.read()) + # Finds the entry that is closest to min_sdk without going over. + for entry in data: + if (entry['min_sdk'] > curr_min_sdk_version + and entry['min_sdk'] <= min_sdk): + min_sdk_found = entry + curr_min_sdk_version = entry['min_sdk'] + + if not min_sdk_found: + logging.error('No suitable apk file found that suits the minimum sdk %d.', + min_sdk) + return None + + logging.info('Found apk file for mininum sdk %d: %r with version %r', + min_sdk, min_sdk_found['file_name'], + min_sdk_found['version_name']) + return os.path.join(apk_dir, min_sdk_found['file_name']) + + +class _AvdManagerAgent: """Private utility for interacting with avdmanager.""" def __init__(self, avd_home, sdk_root): @@ -92,6 +169,8 @@ self._avd_home, 'AVDMANAGER_OPTS': '-Dcom.android.sdkmanager.toolsdir=%s' % fake_tools_dir, + 'JAVA_HOME': + constants.JAVA_HOME, }) def Create(self, avd_name, system_image, force=False): @@ -116,19 +195,17 @@ if force: create_cmd += ['--force'] - create_proc = cmd_helper.Popen( - create_cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self._env) + create_proc = cmd_helper.Popen(create_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env) output, error = create_proc.communicate(input='\n') if create_proc.returncode != 0: - raise AvdException( - 'AVD creation failed', - command=create_cmd, - stdout=output, - stderr=error) + raise AvdException('AVD creation failed', + command=create_cmd, + stdout=output, + stderr=error) for line in output.splitlines(): logging.info(' %s', line) @@ -151,10 +228,28 @@ for line in cmd_helper.IterCmdOutputLines(delete_cmd, env=self._env): logging.info(' %s', line) except subprocess.CalledProcessError as e: + # avd.py is executed with python2. + # pylint: disable=W0707 raise AvdException('AVD deletion failed: %s' % str(e), command=delete_cmd) + def List(self): + """List existing AVDs by the name.""" + list_cmd = [ + _DEFAULT_AVDMANAGER_PATH, + '-v', + 'list', + 'avd', + '-c', + ] + output = cmd_helper.GetCmdOutput(list_cmd, env=self._env) + return output.splitlines() -class AvdConfig(object): + def IsAvailable(self, avd_name): + """Check if an AVD exists or not.""" + return avd_name in self.List() + + +class AvdConfig: """Represents a particular AVD configuration. This class supports creation, installation, and execution of an AVD @@ -168,26 +263,157 @@ Args: avd_proto_path: path to a textpb file containing an Avd message. """ + self.avd_proto_path = avd_proto_path self._config = _Load(avd_proto_path) - self._emulator_home = os.path.join(constants.DIR_SOURCE_ROOT, - self._config.avd_package.dest_path) - self._emulator_sdk_root = os.path.join( - constants.DIR_SOURCE_ROOT, self._config.emulator_package.dest_path) - self._emulator_path = os.path.join(self._emulator_sdk_root, 'emulator', - 'emulator') - self._initialized = False self._initializer_lock = threading.Lock() @property + def emulator_home(self): + """User-specific emulator configuration directory. + + It corresponds to the environment variable $ANDROID_EMULATOR_HOME. + Configs like advancedFeatures.ini are expected to be under this dir. + """ + return os.path.join(COMMON_CIPD_ROOT, self._config.avd_package.dest_path) + + @property + def emulator_sdk_root(self): + """The path to the SDK installation directory. + + It corresponds to the environment variable $ANDROID_HOME. + + To be a valid sdk root, it requires to have the subdirecotries "platforms" + and "platform-tools". See http://bit.ly/2YAkyFE for context. + + Also, it is expected to have subdirecotries "emulator" and "system-images". + """ + emulator_sdk_root = os.path.join(COMMON_CIPD_ROOT, + self._config.emulator_package.dest_path) + # Ensure this is a valid sdk root. + required_dirs = [ + os.path.join(emulator_sdk_root, 'platforms'), + os.path.join(emulator_sdk_root, 'platform-tools'), + ] + for d in required_dirs: + if not os.path.exists(d): + os.makedirs(d) + + return emulator_sdk_root + + @property + def emulator_path(self): + """The path to the emulator binary.""" + return os.path.join(self.emulator_sdk_root, 'emulator', 'emulator') + + @property + def qemu_img_path(self): + """The path to the qemu-img binary. + + This is used to rebase the paths in qcow2 images. + """ + return os.path.join(self.emulator_sdk_root, 'emulator', 'qemu-img') + + @property + def mksdcard_path(self): + """The path to the mksdcard binary. + + This is used to create a sdcard image. + """ + return os.path.join(self.emulator_sdk_root, 'emulator', 'mksdcard') + + @property def avd_settings(self): + """The AvdSettings in the avd proto file. + + This defines how to configure the AVD at creation. + """ return self._config.avd_settings + @property + def avd_name(self): + """The name of the AVD to create or use.""" + return self._config.avd_name + + @property + def avd_home(self): + """The path that contains the files of one or multiple AVDs.""" + avd_home = os.path.join(self.emulator_home, 'avd') + if not os.path.exists(avd_home): + os.makedirs(avd_home) + + return avd_home + + @property + def _avd_dir(self): + """The path that contains the files of the given AVD.""" + return os.path.join(self.avd_home, '%s.avd' % self.avd_name) + + @property + def _system_image_dir(self): + """The path of the directory that directly contains the system images. + + For example, if the system_image_name is + "system-images;android-33;google_apis;x86_64" + + The _system_image_dir will be: + <COMMON_CIPD_ROOT>/<dest_path>/system-images/android-33/google_apis/x86_64 + + This is used to rebase the paths in qcow2 images. + """ + return os.path.join(COMMON_CIPD_ROOT, + self._config.system_image_package.dest_path, + *self._config.system_image_name.split(';')) + + @property + def _root_ini_path(self): + """The <avd_name>.ini file of the given AVD.""" + return os.path.join(self.avd_home, '%s.ini' % self.avd_name) + + @property + def _config_ini_path(self): + """The config.ini file under _avd_dir.""" + return os.path.join(self._avd_dir, 'config.ini') + + @property + def _features_ini_path(self): + return os.path.join(self.emulator_home, 'advancedFeatures.ini') + + @property + def xdg_config_dir(self): + """The base directory to store qt config file. + + This dir should be added to the env variable $XDG_CONFIG_DIRS so that + _qt_config_path can take effect. See https://bit.ly/3HIQRZ3 for context. + """ + config_dir = os.path.join(self.emulator_home, '.config') + if not os.path.exists(config_dir): + os.makedirs(config_dir) + + return config_dir + + @property + def _qt_config_path(self): + """The qt config file for emulator.""" + qt_config_dir = os.path.join(self.xdg_config_dir, + 'Android Open Source Project') + if not os.path.exists(qt_config_dir): + os.makedirs(qt_config_dir) + + return os.path.join(qt_config_dir, 'Emulator.conf') + + def HasSnapshot(self, snapshot_name): + """Check if a given snapshot exists or not.""" + snapshot_path = os.path.join(self._avd_dir, 'snapshots', snapshot_name) + return os.path.exists(snapshot_path) + def Create(self, force=False, snapshot=False, keep=False, + additional_apks=None, + privileged_apk_tuples=None, cipd_json_output=None, dry_run=False): """Create an instance of the AVD CIPD package. @@ -197,7 +423,8 @@ - creates the AVD - modifies the AVD's ini files to support running chromium tests in chromium infrastructure - - optionally starts & stops the AVD for snapshotting (default no) + - optionally starts, installs additional apks and/or privileged apks, and + stops the AVD for snapshotting (default no) - By default creates and uploads an instance of the AVD CIPD package (can be turned off by dry_run flag). - optionally deletes the AVD (default yes) @@ -208,51 +435,46 @@ the CIPD package. keep: bool indicating whether to keep the AVD after creating the CIPD package. + additional_apks: a list of strings contains the paths to the APKs. These + APKs will be installed after AVD is started. + privileged_apk_tuples: a list of (apk_path, device_partition) tuples where + |apk_path| is a string containing the path to the APK, and + |device_partition| is a string indicating the system image partition on + device that contains "priv-app" directory, e.g. "/system", "/product". cipd_json_output: string path to pass to `cipd create` via -json-output. dry_run: When set to True, it will skip the CIPD package creation after creating the AVD. """ logging.info('Installing required packages.') - self._InstallCipdPackages(packages=[ - self._config.emulator_package, - self._config.system_image_package, - ]) + self._InstallCipdPackages(_PACKAGES_CREATION) - android_avd_home = os.path.join(self._emulator_home, 'avd') - - if not os.path.exists(android_avd_home): - os.makedirs(android_avd_home) - - avd_manager = _AvdManagerAgent( - avd_home=android_avd_home, sdk_root=self._emulator_sdk_root) + avd_manager = _AvdManagerAgent(avd_home=self.avd_home, + sdk_root=self.emulator_sdk_root) logging.info('Creating AVD.') - avd_manager.Create( - avd_name=self._config.avd_name, - system_image=self._config.system_image_name, - force=force) + avd_manager.Create(avd_name=self.avd_name, + system_image=self._config.system_image_name, + force=force) try: logging.info('Modifying AVD configuration.') # Clear out any previous configuration or state from this AVD. - root_ini = os.path.join(android_avd_home, - '%s.ini' % self._config.avd_name) - features_ini = os.path.join(self._emulator_home, 'advancedFeatures.ini') - avd_dir = os.path.join(android_avd_home, '%s.avd' % self._config.avd_name) - config_ini = os.path.join(avd_dir, 'config.ini') + with ini.update_ini_file(self._root_ini_path) as r_ini_contents: + r_ini_contents['path.rel'] = 'avd/%s.avd' % self.avd_name - with ini.update_ini_file(root_ini) as root_ini_contents: - root_ini_contents['path.rel'] = 'avd/%s.avd' % self._config.avd_name - - with ini.update_ini_file(features_ini) as features_ini_contents: + with ini.update_ini_file(self._features_ini_path) as f_ini_contents: # features_ini file will not be refreshed by avdmanager during # creation. So explicitly clear its content to exclude any leftover # from previous creation. - features_ini_contents.clear() - features_ini_contents.update(self.avd_settings.advanced_features) + f_ini_contents.clear() + f_ini_contents.update(self.avd_settings.advanced_features) - with ini.update_ini_file(config_ini) as config_ini_contents: + with ini.update_ini_file(self._config_ini_path) as config_ini_contents: + # Update avd_properties first so that they won't override settings + # like screen and ram_size + config_ini_contents.update(self.avd_settings.avd_properties) + height = self.avd_settings.screen.height or _DEFAULT_SCREEN_HEIGHT width = self.avd_settings.screen.width or _DEFAULT_SCREEN_WIDTH density = self.avd_settings.screen.density or _DEFAULT_SCREEN_DENSITY @@ -263,33 +485,102 @@ 'hw.lcd.density': density, 'hw.lcd.height': height, 'hw.lcd.width': width, + 'hw.mainKeys': 'no', # Show nav buttons on screen }) if self.avd_settings.ram_size: config_ini_contents['hw.ramSize'] = self.avd_settings.ram_size + config_ini_contents['hw.sdCard'] = 'yes' + if self.avd_settings.sdcard.size: + sdcard_path = os.path.join(self._avd_dir, _SDCARD_NAME) + cmd_helper.RunCmd([ + self.mksdcard_path, + self.avd_settings.sdcard.size, + sdcard_path, + ]) + config_ini_contents['hw.sdCard.path'] = sdcard_path + + if not additional_apks: + additional_apks = [] + for pkg in self._config.additional_apk: + apk_dir = os.path.join(COMMON_CIPD_ROOT, pkg.dest_path) + apk_file = _FindMinSdkFile(apk_dir, self._config.min_sdk) + # Some of these files come from chrome internal, so may not be + # available to non-internal permissioned users. + if os.path.exists(apk_file): + logging.info('Adding additional apk for install: %s', apk_file) + additional_apks.append(apk_file) + + if not privileged_apk_tuples: + privileged_apk_tuples = [] + for pkg in self._config.privileged_apk: + apk_dir = os.path.join(COMMON_CIPD_ROOT, pkg.dest_path) + apk_file = _FindMinSdkFile(apk_dir, self._config.min_sdk) + # Some of these files come from chrome internal, so may not be + # available to non-internal permissioned users. + if os.path.exists(apk_file): + logging.info('Adding privilege apk for install: %s', apk_file) + privileged_apk_tuples.append( + (apk_file, self._config.install_privileged_apk_partition)) + # Start & stop the AVD. self._Initialize() - instance = _AvdInstance(self._emulator_path, self._emulator_home, - self._config) + instance = _AvdInstance(self) # Enable debug for snapshot when it is set to True - debug_tags = 'init,snapshot' if snapshot else None - instance.Start( - read_only=False, snapshot_save=snapshot, debug_tags=debug_tags) + debug_tags = 'time,init,snapshot' if snapshot else None + # Installing privileged apks requires modifying the system + # image. + writable_system = bool(privileged_apk_tuples) + instance.Start(ensure_system_settings=False, + read_only=False, + writable_system=writable_system, + gpu_mode=_DEFAULT_GPU_MODE, + debug_tags=debug_tags) + + assert instance.device is not None, '`instance.device` not initialized.' # Android devices with full-disk encryption are encrypted on first boot, # and then get decrypted to continue the boot process (See details in # https://bit.ly/3agmjcM). # Wait for this step to complete since it can take a while for old OSs # like M, otherwise the avd may have "Encryption Unsuccessful" error. - device_utils.DeviceUtils(instance.serial).WaitUntilFullyBooted( - decrypt=True, timeout=180, retries=0) + instance.device.WaitUntilFullyBooted(decrypt=True, timeout=180, retries=0) + + if additional_apks: + for apk in additional_apks: + instance.device.Install(apk, allow_downgrade=True, reinstall=True) + package_name = apk_helper.GetPackageName(apk) + package_version = instance.device.GetApplicationVersion(package_name) + logging.info('The version for package %r on the device is %r', + package_name, package_version) + + if privileged_apk_tuples: + system_app.InstallPrivilegedApps(instance.device, privileged_apk_tuples) + for apk, _ in privileged_apk_tuples: + package_name = apk_helper.GetPackageName(apk) + package_version = instance.device.GetApplicationVersion(package_name) + logging.info('The version for package %r on the device is %r', + package_name, package_version) + + # Always disable the network to prevent built-in system apps from + # updating themselves, which could take over package manager and + # cause shell command timeout. + logging.info('Disabling the network.') + settings.ConfigureContentSettings(instance.device, + settings.NETWORK_DISABLED_SETTINGS) + + if snapshot: + # Reboot so that changes like disabling network can take effect. + instance.device.Reboot() + instance.SaveSnapshot() + instance.Stop() # The multiinstance lock file seems to interfere with the emulator's # operation in some circumstances (beyond the obvious -read-only ones), # and there seems to be no mechanism by which it gets closed or deleted. # See https://bit.ly/2pWQTH7 for context. - multiInstanceLockFile = os.path.join(avd_dir, 'multiinstance.lock') + multiInstanceLockFile = os.path.join(self._avd_dir, 'multiinstance.lock') if os.path.exists(multiInstanceLockFile): os.unlink(multiInstanceLockFile) @@ -297,21 +588,23 @@ 'package': self._config.avd_package.package_name, 'root': - self._emulator_home, + self.emulator_home, 'install_mode': 'copy', 'data': [{ - 'dir': os.path.relpath(avd_dir, self._emulator_home) + 'dir': os.path.relpath(self._avd_dir, self.emulator_home) }, { - 'file': os.path.relpath(root_ini, self._emulator_home) + 'file': + os.path.relpath(self._root_ini_path, self.emulator_home) }, { - 'file': os.path.relpath(features_ini, self._emulator_home) + 'file': + os.path.relpath(self._features_ini_path, self.emulator_home) }], } logging.info('Creating AVD CIPD package.') - logging.debug('ensure file content: %s', - json.dumps(package_def_content, indent=2)) + logging.info('ensure file content: %s', + json.dumps(package_def_content, indent=2)) with tempfile_ext.TemporaryFileName(suffix='.json') as package_def_path: with open(package_def_path, 'w') as package_def_file: @@ -341,16 +634,83 @@ for line in cmd_helper.IterCmdOutputLines(cipd_create_cmd): logging.info(' %s', line) except subprocess.CalledProcessError as e: - raise AvdException( - 'CIPD package creation failed: %s' % str(e), - command=cipd_create_cmd) + # avd.py is executed with python2. + # pylint: disable=W0707 + raise AvdException('CIPD package creation failed: %s' % str(e), + command=cipd_create_cmd) finally: if not keep: logging.info('Deleting AVD.') - avd_manager.Delete(avd_name=self._config.avd_name) + avd_manager.Delete(avd_name=self.avd_name) - def Install(self, packages=_ALL_PACKAGES): + def IsAvailable(self): + """Returns whether emulator is up-to-date.""" + if not os.path.exists(self._config_ini_path): + return False + + # Skip when no version exists to prevent "IsAvailable()" returning False + # for emualtors set up using Create() (rather than Install()). + for cipd_root, pkgs in self._IterCipdPackages(_PACKAGES_RUNTIME, + check_version=False): + stdout = subprocess.run(['cipd', 'installed', '--root', cipd_root], + capture_output=True, + check=False, + encoding='utf8').stdout + # Output looks like: + # Packages: + # name1:version1 + # name2:version2 + installed = [l.strip().split(':', 1) for l in stdout.splitlines()[1:]] + + if any([p.package_name, p.version] not in installed for p in pkgs): + return False + return True + + def Uninstall(self): + """Uninstall all the artifacts associated with the given config. + + Artifacts includes: + - CIPD packages specified in the avd config. + - The local AVD created by `Create`, if present. + + """ + # Delete any existing local AVD. This must occur before deleting CIPD + # packages because a AVD needs system image to be recognized by avdmanager. + avd_manager = _AvdManagerAgent(avd_home=self.avd_home, + sdk_root=self.emulator_sdk_root) + if avd_manager.IsAvailable(self.avd_name): + logging.info('Deleting local AVD %s', self.avd_name) + avd_manager.Delete(self.avd_name) + + # Delete installed CIPD packages. + for cipd_root, _ in self._IterCipdPackages(_PACKAGES_ALL, + check_version=False): + logging.info('Uninstalling packages in %s', cipd_root) + if not os.path.exists(cipd_root): + continue + # Create an empty ensure file to removed any installed CIPD packages. + ensure_path = os.path.join(cipd_root, '.ensure') + with open(ensure_path, 'w') as ensure_file: + ensure_file.write('$ParanoidMode CheckIntegrity\n\n') + ensure_cmd = [ + 'cipd', + 'ensure', + '-ensure-file', + ensure_path, + '-root', + cipd_root, + ] + try: + for line in cmd_helper.IterCmdOutputLines(ensure_cmd): + logging.info(' %s', line) + except subprocess.CalledProcessError as e: + # avd.py is executed with python2. + # pylint: disable=W0707 + raise AvdException('Failed to uninstall CIPD packages: %s' % str(e), + command=ensure_cmd) + + def Install(self): """Installs the requested CIPD packages and prepares them for use. This includes making files writeable and revising some of the @@ -359,26 +719,84 @@ Returns: None Raises: AvdException on failure to install. """ - self._InstallCipdPackages(packages=packages) + self._InstallCipdPackages(_PACKAGES_RUNTIME) self._MakeWriteable() - self._EditConfigs() + self._UpdateConfigs() + self._RebaseQcow2Images() - def _InstallCipdPackages(self, packages): - pkgs_by_dir = {} - if packages is _ALL_PACKAGES: + def _RebaseQcow2Images(self): + """Rebase the paths in qcow2 images. + + qcow2 files may exists in avd directory which have hard-coded paths to the + backing files, e.g., system.img, vendor.img. Such paths need to be rebased + if the avd is moved to a different directory in order to boot successfully. + """ + for f in _BACKING_FILES: + qcow2_image_path = os.path.join(self._avd_dir, '%s.qcow2' % f) + if not os.path.exists(qcow2_image_path): + continue + backing_file_path = os.path.join(self._system_image_dir, f) + logging.info('Rebasing the qcow2 image %r with the backing file %r', + qcow2_image_path, backing_file_path) + cmd_helper.RunCmd([ + self.qemu_img_path, + 'rebase', + '-u', + '-f', + 'qcow2', + '-b', + # The path to backing file must be relative to the qcow2 image. + os.path.relpath(backing_file_path, os.path.dirname(qcow2_image_path)), + qcow2_image_path, + ]) + + def _ListPackages(self, packages): + if packages is _PACKAGES_RUNTIME: packages = [ self._config.avd_package, self._config.emulator_package, self._config.system_image_package, ] - for pkg in packages: - if not pkg.dest_path in pkgs_by_dir: - pkgs_by_dir[pkg.dest_path] = [] - pkgs_by_dir[pkg.dest_path].append(pkg) + elif packages is _PACKAGES_CREATION: + packages = [ + self._config.emulator_package, + self._config.system_image_package, + *self._config.privileged_apk, + *self._config.additional_apk, + ] + elif packages is _PACKAGES_ALL: + packages = [ + self._config.avd_package, + self._config.emulator_package, + self._config.system_image_package, + *self._config.privileged_apk, + *self._config.additional_apk, + ] + return packages + + def _IterCipdPackages(self, packages, check_version=True): + """Iterate a list of CIPD packages by their CIPD roots. + + Args: + packages: a list of packages from an AVD config. + check_version: If set, raise Exception when a package has no version. + """ + pkgs_by_dir = collections.defaultdict(list) + for pkg in self._ListPackages(packages): + if pkg.version: + pkgs_by_dir[pkg.dest_path].append(pkg) + elif check_version: + raise AvdException('Expecting a version for the package %s' % + pkg.package_name) for pkg_dir, pkgs in pkgs_by_dir.items(): - logging.info('Installing packages in %s', pkg_dir) - cipd_root = os.path.join(constants.DIR_SOURCE_ROOT, pkg_dir) + cipd_root = os.path.join(COMMON_CIPD_ROOT, pkg_dir) + yield cipd_root, pkgs + + def _InstallCipdPackages(self, packages, check_version=True): + for cipd_root, pkgs in self._IterCipdPackages(packages, + check_version=check_version): + logging.info('Installing packages in %s', cipd_root) if not os.path.exists(cipd_root): os.makedirs(cipd_root) ensure_path = os.path.join(cipd_root, '.ensure') @@ -401,14 +819,14 @@ for line in cmd_helper.IterCmdOutputLines(ensure_cmd): logging.info(' %s', line) except subprocess.CalledProcessError as e: - raise AvdException( - 'Failed to install CIPD package %s: %s' % (pkg.package_name, - str(e)), - command=ensure_cmd) + # avd.py is executed with python2. + # pylint: disable=W0707 + raise AvdException('Failed to install CIPD packages: %s' % str(e), + command=ensure_cmd) def _MakeWriteable(self): # The emulator requires that some files are writable. - for dirname, _, filenames in os.walk(self._emulator_home): + for dirname, _, filenames in os.walk(self.emulator_home): for f in filenames: path = os.path.join(dirname, f) mode = os.lstat(path).st_mode @@ -416,34 +834,40 @@ mode = mode | stat.S_IWUSR os.chmod(path, mode) - def _EditConfigs(self): - android_avd_home = os.path.join(self._emulator_home, 'avd') - avd_dir = os.path.join(android_avd_home, '%s.avd' % self._config.avd_name) + def _UpdateConfigs(self): + """Update various properties in config files after installation. - config_path = os.path.join(avd_dir, 'config.ini') - if os.path.exists(config_path): - with open(config_path) as config_file: - config_contents = ini.load(config_file) - else: - config_contents = {} + AVD config files contain some properties which can be different between AVD + creation and installation, e.g. hw.sdCard.path, which is an absolute path. + Update their values so that: + * Emulator instance can be booted correctly. + * The snapshot can be loaded successfully. + """ + logging.info('Updating AVD configurations.') + # Update the absolute avd path in root_ini file + with ini.update_ini_file(self._root_ini_path) as r_ini_contents: + r_ini_contents['path'] = self._avd_dir - config_contents['hw.sdCard'] = 'true' - if self.avd_settings.sdcard.size: - sdcard_path = os.path.join(avd_dir, 'cr-sdcard.img') - if not os.path.exists(sdcard_path): - mksdcard_path = os.path.join( - os.path.dirname(self._emulator_path), 'mksdcard') - mksdcard_cmd = [ - mksdcard_path, - self.avd_settings.sdcard.size, - sdcard_path, - ] - cmd_helper.RunCmd(mksdcard_cmd) + # Update hardware settings. + config_paths = [self._config_ini_path] + # The file hardware.ini within each snapshot need to be updated as well. + hw_ini_glob_pattern = os.path.join(self._avd_dir, 'snapshots', '*', + 'hardware.ini') + config_paths.extend(glob.glob(hw_ini_glob_pattern)) - config_contents['hw.sdCard.path'] = sdcard_path + properties = {} + # Update hw.sdCard.path if applicable + sdcard_path = os.path.join(self._avd_dir, _SDCARD_NAME) + if os.path.exists(sdcard_path): + properties['hw.sdCard.path'] = sdcard_path - with open(config_path, 'w') as config_file: - ini.dump(config_contents, config_file) + for config_path in config_paths: + with ini.update_ini_file(config_path) as config_contents: + config_contents.update(properties) + + # Create qt config file to disable adb warning when launched in window mode. + with ini.update_ini_file(self._qt_config_path) as config_contents: + config_contents['set'] = {'autoFindAdb': 'false'} def _Initialize(self): if self._initialized: @@ -456,25 +880,17 @@ # Emulator start-up looks for the adb daemon. Make sure it's running. adb_wrapper.AdbWrapper.StartServer() - # Emulator start-up tries to check for the SDK root by looking for - # platforms/ and platform-tools/. Ensure they exist. - # See http://bit.ly/2YAkyFE for context. - required_dirs = [ - os.path.join(self._emulator_sdk_root, 'platforms'), - os.path.join(self._emulator_sdk_root, 'platform-tools'), - ] - for d in required_dirs: - if not os.path.exists(d): - os.makedirs(d) + # Emulator start-up requires a valid sdk root. + assert self.emulator_sdk_root - def CreateInstance(self): + def CreateInstance(self, output_manager=None): """Creates an AVD instance without starting it. Returns: An _AvdInstance. """ self._Initialize() - return _AvdInstance(self._emulator_path, self._emulator_home, self._config) + return _AvdInstance(self, output_manager=output_manager) def StartInstance(self): """Starts an AVD instance. @@ -487,39 +903,68 @@ return instance -class _AvdInstance(object): +class _AvdInstance: """Represents a single running instance of an AVD. This class should only be created directly by AvdConfig.StartInstance, but its other methods can be freely called. """ - def __init__(self, emulator_path, emulator_home, avd_config): + def __init__(self, avd_config, output_manager=None): """Create an _AvdInstance object. Args: - emulator_path: path to the emulator binary. - emulator_home: path to the emulator home directory. - avd_config: AVD config proto. + avd_config: an AvdConfig instance. + output_manager: a pylib.base.output_manager.OutputManager instance. """ self._avd_config = avd_config self._avd_name = avd_config.avd_name - self._emulator_home = emulator_home - self._emulator_path = emulator_path + self._emulator_home = avd_config.emulator_home + self._emulator_path = avd_config.emulator_path self._emulator_proc = None self._emulator_serial = None - self._sink = None + self._emulator_device = None + + self._output_manager = output_manager + self._output_file = None + + self._writable_system = False + self._debug_tags = None def __str__(self): return '%s|%s' % (self._avd_name, (self._emulator_serial or id(self))) def Start(self, + ensure_system_settings=True, read_only=True, - snapshot_save=False, window=False, writable_system=False, - debug_tags=None): - """Starts the emulator running an instance of the given AVD.""" + gpu_mode=_DEFAULT_GPU_MODE, + wipe_data=False, + debug_tags=None, + require_fast_start=False): + """Starts the emulator running an instance of the given AVD. + + Note when ensure_system_settings is True, the program will wait until the + emulator is fully booted, and then update system settings. + """ + is_slow_start = not require_fast_start + # Force to load system snapshot if detected. + if self.HasSystemSnapshot(): + if not writable_system: + logging.info('System snapshot found. Set "writable_system=True" ' + 'to load it properly.') + writable_system = True + if read_only: + logging.info('System snapshot found. Set "read_only=False" ' + 'to load it properly.') + read_only = False + elif writable_system: + is_slow_start = True + logging.warning('Emulator will be slow to start, as ' + '"writable_system=True" but system snapshot not found.') + + self._writable_system = writable_system with tempfile_ext.TemporaryFileName() as socket_path, (contextlib.closing( socket.socket(socket.AF_UNIX))) as sock: @@ -531,44 +976,78 @@ '-report-console', 'unix:%s' % socket_path, '-no-boot-anim', - # Set the gpu mode to swiftshader_indirect otherwise the avd may exit - # with the error "change of render" under window mode - '-gpu', - 'swiftshader_indirect', + # Explicitly prevent emulator from auto-saving to snapshot on exit. + '-no-snapshot-save', + # Explicitly set the snapshot name for auto-load + '-snapshot', + self.GetSnapshotName(), ] + if wipe_data: + emulator_cmd.append('-wipe-data') if read_only: emulator_cmd.append('-read-only') - if not snapshot_save: - emulator_cmd.append('-no-snapshot-save') if writable_system: emulator_cmd.append('-writable-system') + # Note when "--gpu-mode" is set to "host": + # * It needs a valid DISPLAY env, even if "--emulator-window" is false. + # Otherwise it may throw errors like "Failed to initialize backend + # EGL display". See the code in https://bit.ly/3ruiMlB as an example + # to setup the DISPLAY env with xvfb. + # * It will not work under remote sessions like chrome remote desktop. + if gpu_mode: + emulator_cmd.extend(['-gpu', gpu_mode]) if debug_tags: - emulator_cmd.extend(['-debug', debug_tags]) + self._debug_tags = set(debug_tags.split(',')) + # Always print timestamp when debug tags are set. + self._debug_tags.add('time') + emulator_cmd.extend(['-debug', ','.join(self._debug_tags)]) + if 'kernel' in self._debug_tags: + # TODO(crbug.com/1404176): newer API levels need "-virtio-console" + # as well to print kernel log. + emulator_cmd.append('-show-kernel') - emulator_env = {} - if self._emulator_home: - emulator_env['ANDROID_EMULATOR_HOME'] = self._emulator_home + emulator_env = { + # kill immediately when emulator hang. + 'ANDROID_EMULATOR_WAIT_TIME_BEFORE_KILL': '0', + # Sets the emulator configuration directory + 'ANDROID_EMULATOR_HOME': self._emulator_home, + } + if 'DISPLAY' in os.environ: + emulator_env['DISPLAY'] = os.environ.get('DISPLAY') if window: - if 'DISPLAY' in os.environ: - emulator_env['DISPLAY'] = os.environ.get('DISPLAY') - else: + if 'DISPLAY' not in emulator_env: raise AvdException('Emulator failed to start: DISPLAY not defined') else: emulator_cmd.append('-no-window') + # Need this for the qt config file to take effect. + xdg_config_dirs = [self._avd_config.xdg_config_dir] + if 'XDG_CONFIG_DIRS' in os.environ: + xdg_config_dirs.append(os.environ.get('XDG_CONFIG_DIRS')) + emulator_env['XDG_CONFIG_DIRS'] = ':'.join(xdg_config_dirs) + sock.listen(1) - logging.info('Starting emulator with commands: %s', - ' '.join(emulator_cmd)) + logging.info('Starting emulator...') + logging.info( + ' With environments: %s', + ' '.join(['%s=%s' % (k, v) for k, v in emulator_env.items()])) + logging.info(' With commands: %s', ' '.join(emulator_cmd)) - # TODO(jbudorick): Add support for logging emulator stdout & stderr at - # higher logging levels. # Enable the emulator log when debug_tags is set. - if not debug_tags: - self._sink = open('/dev/null', 'w') - self._emulator_proc = cmd_helper.Popen( - emulator_cmd, stdout=self._sink, stderr=self._sink, env=emulator_env) + if self._debug_tags: + # Write to an ArchivedFile if output manager is set, otherwise stdout. + if self._output_manager: + self._output_file = self._output_manager.CreateArchivedFile( + 'emulator_%s' % time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), + 'emulator') + else: + self._output_file = open('/dev/null', 'w') + self._emulator_proc = cmd_helper.Popen(emulator_cmd, + stdout=self._output_file, + stderr=self._output_file, + env=emulator_env) # Waits for the emulator to report its serial as requested via # -report-console. See http://bit.ly/2lK3L18 for more. @@ -580,27 +1059,103 @@ try: self._emulator_serial = timeout_retry.Run( - listen_for_serial, timeout=30, retries=0, args=[sock]) + listen_for_serial, + timeout=120 if is_slow_start else 30, + retries=0, + args=[sock]) logging.info('%s started', self._emulator_serial) - except Exception as e: - self.Stop() - raise AvdException('Emulator failed to start: %s' % str(e)) + except Exception: + self.Stop(force=True) + raise - def Stop(self): - """Stops the emulator process.""" + # Set the system settings in "Start" here instead of setting in "Create" + # because "Create" is used during AVD creation, and we want to avoid extra + # turn-around on rolling AVD. + if ensure_system_settings: + assert self.device is not None, '`instance.device` not initialized.' + logging.info('Waiting for device to be fully booted.') + self.device.WaitUntilFullyBooted(timeout=360 if is_slow_start else 90, + retries=0) + logging.info('Device fully booted, verifying system settings.') + _EnsureSystemSettings(self.device) + + def Stop(self, force=False): + """Stops the emulator process. + + When "force" is True, we will call "terminate" on the emulator process, + which is recommended when emulator is not responding to adb commands. + """ + # Close output file first in case emulator process killing goes wrong. + if self._output_file: + if self._debug_tags: + if self._output_manager: + self._output_manager.ArchiveArchivedFile(self._output_file, + delete=True) + link = self._output_file.Link() + if link: + logging.critical('Emulator logs saved to %s', link) + else: + self._output_file.close() + self._output_file = None + if self._emulator_proc: if self._emulator_proc.poll() is None: - if self._emulator_serial: - device_utils.DeviceUtils(self._emulator_serial).adb.Emu('kill') - else: + if force or not self.device: self._emulator_proc.terminate() + else: + self.device.adb.Emu('kill') self._emulator_proc.wait() self._emulator_proc = None + self._emulator_serial = None + self._emulator_device = None - if self._sink: - self._sink.close() - self._sink = None + def GetSnapshotName(self): + """Return the snapshot name to load/save. + + Emulator has a different snapshot process when '-writable-system' flag is + set (See https://issuetracker.google.com/issues/135857816#comment8). + + """ + if self._writable_system: + return _SYSTEM_SNAPSHOT_NAME + + return _DEFAULT_SNAPSHOT_NAME + + def HasSystemSnapshot(self): + """Check if the instance has the snapshot named _SYSTEM_SNAPSHOT_NAME.""" + return self._avd_config.HasSnapshot(_SYSTEM_SNAPSHOT_NAME) + + def SaveSnapshot(self): + snapshot_name = self.GetSnapshotName() + if self.device: + logging.info('Saving snapshot to %r.', snapshot_name) + self.device.adb.Emu(['avd', 'snapshot', 'save', snapshot_name]) @property def serial(self): return self._emulator_serial + + @property + def device(self): + if not self._emulator_device and self._emulator_serial: + self._emulator_device = device_utils.DeviceUtils(self._emulator_serial) + return self._emulator_device + + +# TODO(crbug.com/1275767): Refactor it to a dict-based approach. +def _EnsureSystemSettings(device): + set_long_press_timeout_cmd = [ + 'settings', 'put', 'secure', 'long_press_timeout', _LONG_PRESS_TIMEOUT + ] + device.RunShellCommand(set_long_press_timeout_cmd, check_return=True) + + # Verify if long_press_timeout is set correctly. + get_long_press_timeout_cmd = [ + 'settings', 'get', 'secure', 'long_press_timeout' + ] + adb_output = device.RunShellCommand(get_long_press_timeout_cmd, + check_return=True) + if _LONG_PRESS_TIMEOUT in adb_output: + logging.info('long_press_timeout set to %r', _LONG_PRESS_TIMEOUT) + else: + logging.warning('long_press_timeout is not set correctly')
diff --git a/build/android/pylib/local/emulator/ini.py b/build/android/pylib/local/emulator/ini.py index 8f16c33..79eb015 100644 --- a/build/android/pylib/local/emulator/ini.py +++ b/build/android/pylib/local/emulator/ini.py
@@ -1,23 +1,60 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Basic .ini encoding and decoding.""" +"""Basic .ini encoding and decoding. -from __future__ import absolute_import +The basic element in an ini file is the key. Every key is constructed by a name +and a value, delimited by an equals sign (=). + +Keys may be grouped into sections. The secetion name will be a line by itself, +in square brackets ([ and ]). All keys after the section are associated with +that section until another section occurs. + +Keys that are not under any section are considered at the top level. + +Section and key names are case sensitive. +""" + + import contextlib import os +def add_key(line, config, strict=True): + key, val = line.split('=', 1) + key = key.strip() + val = val.strip() + if strict and key in config: + raise ValueError('Multiple entries present for key "%s"' % key) + config[key] = val + + def loads(ini_str, strict=True): + """Deserialize int_str to a dict (nested dict when has sections) object. + + Duplicated sections will merge their keys. + + When there are multiple entries for a key, at the top level, or under the + same section: + - If strict is true, ValueError will be raised. + - If strict is false, only the last occurrence will be stored. + """ ret = {} + section = None for line in ini_str.splitlines(): - key, val = line.split('=', 1) - key = key.strip() - val = val.strip() - if strict and key in ret: - raise ValueError('Multiple entries present for key "%s"' % key) - ret[key] = val + # Empty line + if not line: + continue + # Section line + if line[0] == '[' and line[-1] == ']': + section = line[1:-1] + if section not in ret: + ret[section] = {} + # Key line + else: + config = ret if section is None else ret[section] + add_key(line, config, strict=strict) return ret @@ -27,10 +64,20 @@ def dumps(obj): - ret = '' + results = [] + key_str = '' + for k, v in sorted(obj.items()): - ret += '%s = %s\n' % (k, str(v)) - return ret + if isinstance(v, dict): + results.append('[%s]\n' % k + dumps(v)) + else: + key_str += '%s = %s\n' % (k, str(v)) + + # Insert key_str at the first position, before any sections + if key_str: + results.insert(0, key_str) + + return '\n'.join(results) def dump(obj, fp): @@ -46,11 +93,10 @@ Yields: The contents of the file, as a dict """ + ini_contents = {} if os.path.exists(ini_file_path): with open(ini_file_path) as ini_file: ini_contents = load(ini_file) - else: - ini_contents = {} yield ini_contents
diff --git a/build/android/pylib/local/emulator/ini_test.py b/build/android/pylib/local/emulator/ini_test.py index 0cf9250..327d6bf 100755 --- a/build/android/pylib/local/emulator/ini_test.py +++ b/build/android/pylib/local/emulator/ini_test.py
@@ -1,13 +1,17 @@ -#! /usr/bin/env vpython -# Copyright 2020 The Chromium Authors. All rights reserved. +#! /usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for ini.py.""" -from __future__ import absolute_import + +import os +import sys import textwrap import unittest +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))) from pylib.local.emulator import ini @@ -17,15 +21,35 @@ foo.bar = 1 foo.baz= example bar.bad =/path/to/thing + + [section_1] + foo.bar = 1 + foo.baz= example + + [section_2] + foo.baz= example + bar.bad =/path/to/thing + + [section_1] + bar.bad =/path/to/thing """) expected = { 'foo.bar': '1', 'foo.baz': 'example', 'bar.bad': '/path/to/thing', + 'section_1': { + 'foo.bar': '1', + 'foo.baz': 'example', + 'bar.bad': '/path/to/thing', + }, + 'section_2': { + 'foo.baz': 'example', + 'bar.bad': '/path/to/thing', + }, } self.assertEqual(expected, ini.loads(ini_str)) - def testLoadsStrictFailure(self): + def testLoadsDuplicatedKeysStrictFailure(self): ini_str = textwrap.dedent("""\ foo.bar = 1 foo.baz = example @@ -35,17 +59,39 @@ with self.assertRaises(ValueError): ini.loads(ini_str, strict=True) + def testLoadsDuplicatedKeysInSectionStrictFailure(self): + ini_str = textwrap.dedent("""\ + [section_1] + foo.bar = 1 + foo.baz = example + bar.bad = /path/to/thing + foo.bar = duplicate + """) + with self.assertRaises(ValueError): + ini.loads(ini_str, strict=True) + def testLoadsPermissive(self): ini_str = textwrap.dedent("""\ foo.bar = 1 foo.baz = example bar.bad = /path/to/thing foo.bar = duplicate + + [section_1] + foo.bar = 1 + foo.baz = example + bar.bad = /path/to/thing + foo.bar = duplicate """) expected = { 'foo.bar': 'duplicate', 'foo.baz': 'example', 'bar.bad': '/path/to/thing', + 'section_1': { + 'foo.bar': 'duplicate', + 'foo.baz': 'example', + 'bar.bad': '/path/to/thing', + }, } self.assertEqual(expected, ini.loads(ini_str, strict=False)) @@ -54,13 +100,53 @@ 'foo.bar': '1', 'foo.baz': 'example', 'bar.bad': '/path/to/thing', + 'section_2': { + 'foo.baz': 'example', + 'bar.bad': '/path/to/thing', + }, + 'section_1': { + 'foo.bar': '1', + 'foo.baz': 'example', + }, } # ini.dumps is expected to dump to string alphabetically - # by key. + # by key and section name. expected = textwrap.dedent("""\ bar.bad = /path/to/thing foo.bar = 1 foo.baz = example + + [section_1] + foo.bar = 1 + foo.baz = example + + [section_2] + bar.bad = /path/to/thing + foo.baz = example + """) + self.assertEqual(expected, ini.dumps(ini_contents)) + + def testDumpsSections(self): + ini_contents = { + 'section_2': { + 'foo.baz': 'example', + 'bar.bad': '/path/to/thing', + }, + 'section_1': { + 'foo.bar': '1', + 'foo.baz': 'example', + }, + } + # ini.dumps is expected to dump to string alphabetically + # by key first, and then by section and the associated keys + expected = textwrap.dedent("""\ + [section_1] + foo.bar = 1 + foo.baz = example + + [section_2] + bar.bad = /path/to/thing + foo.baz = example """) self.assertEqual(expected, ini.dumps(ini_contents))
diff --git a/build/android/pylib/local/emulator/local_emulator_environment.py b/build/android/pylib/local/emulator/local_emulator_environment.py index 1343d8c..d71a382 100644 --- a/build/android/pylib/local/emulator/local_emulator_environment.py +++ b/build/android/pylib/local/emulator/local_emulator_environment.py
@@ -1,14 +1,13 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import logging from six.moves import range # pylint: disable=redefined-builtin from devil import base_error from devil.android import device_errors -from devil.android import device_utils from devil.utils import parallelizer from devil.utils import reraiser_thread from devil.utils import timeout_retry @@ -19,6 +18,9 @@ _MAX_ANDROID_EMULATORS = 16 +# TODO(1262303): After Telemetry is supported by python3 we can re-add +# super without arguments in this script. +# pylint: disable=super-with-arguments class LocalEmulatorEnvironment(local_device_environment.LocalDeviceEnvironment): def __init__(self, args, output_manager, error_func): @@ -31,6 +33,7 @@ logging.warning('--emulator-count capped at 16.') self._emulator_count = min(_MAX_ANDROID_EMULATORS, args.emulator_count) self._emulator_window = args.emulator_window + self._emulator_debug_tags = args.emulator_debug_tags self._writable_system = ((hasattr(args, 'use_webview_provider') and args.use_webview_provider) or (hasattr(args, 'replace_system_package') @@ -46,36 +49,37 @@ self._avd_config.Install() emulator_instances = [ - self._avd_config.CreateInstance() for _ in range(self._emulator_count) + self._avd_config.CreateInstance(output_manager=self.output_manager) + for _ in range(self._emulator_count) ] - def start_emulator_instance(e): + def start_emulator_instance(inst): + def is_timeout_error(exc): + return isinstance( + exc, + (device_errors.CommandTimeoutError, reraiser_thread.TimeoutError)) - def impl(e): + def impl(inst): try: - e.Start( - window=self._emulator_window, - writable_system=self._writable_system) + inst.Start(window=self._emulator_window, + writable_system=self._writable_system, + debug_tags=self._emulator_debug_tags, + require_fast_start=True) except avd.AvdException: logging.exception('Failed to start emulator instance.') return None - try: - device_utils.DeviceUtils(e.serial).WaitUntilFullyBooted() - except base_error.BaseError: - e.Stop() + except base_error.BaseError as e: + # Timeout error usually indicates the emulator is not responding. + # In this case, we should stop it forcely. + inst.Stop(force=is_timeout_error(e)) raise - return e + return inst - def retry_on_timeout(exc): - return (isinstance(exc, device_errors.CommandTimeoutError) - or isinstance(exc, reraiser_thread.TimeoutError)) - - return timeout_retry.Run( - impl, - timeout=120 if self._writable_system else 30, - retries=2, - args=[e], - retry_if_func=retry_on_timeout) + return timeout_retry.Run(impl, + timeout=120 if self._writable_system else 60, + retries=2, + args=[inst], + retry_if_func=is_timeout_error) parallel_emulators = parallelizer.SyncParallelizer(emulator_instances) self._emulator_instances = [ @@ -87,7 +91,7 @@ if not self._emulator_instances: raise Exception('Failed to start any instances of the emulator.') - elif len(self._emulator_instances) < self._emulator_count: + if len(self._emulator_instances) < self._emulator_count: logging.warning( 'Running with fewer emulator instances than requested (%d vs %d)', len(self._emulator_instances), self._emulator_count)
diff --git a/build/android/pylib/local/emulator/proto/__init__.py b/build/android/pylib/local/emulator/proto/__init__.py index 4a12e35..401c54b 100644 --- a/build/android/pylib/local/emulator/proto/__init__.py +++ b/build/android/pylib/local/emulator/proto/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/local/emulator/proto/avd.proto b/build/android/pylib/local/emulator/proto/avd.proto index b06da49..957897f 100644 --- a/build/android/pylib/local/emulator/proto/avd.proto +++ b/build/android/pylib/local/emulator/proto/avd.proto
@@ -1,5 +1,4 @@ - -// Copyright 2019 The Chromium Authors. All rights reserved. +// Copyright 2019 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -14,7 +13,7 @@ // Ignored when creating AVD packages. string version = 2; // Path into which the package should be installed. - // src-relative. + // relative to pylib.local.emulator.avd.COMMON_CIPD_ROOT. string dest_path = 3; } @@ -53,6 +52,14 @@ // The physical RAM size on the device, in megabytes. uint32 ram_size = 4; + + // The properties for AVD. The <key,value> pairs here will override the + // default ones in the given system image. + // See https://bit.ly/3052c1V for all the available keys and values. + // + // Note the screen, sdcard, ram_size above are ultimately translated to + // AVD properties and they won't be overwritten by values here. + map<string, string> avd_properties = 5; } message Avd { @@ -72,4 +79,16 @@ // How to configure the AVD at creation. AvdSettings avd_settings = 6; + + // min sdk level for emulator. + uint32 min_sdk = 7; + + // The partition to install the privileged apk. + // version 27 and below is /system. After that it can be + // /system, /product, or /vendor + string install_privileged_apk_partition = 8; + + // Needed for gmscore/phonesky support. + repeated CIPDPackage privileged_apk = 9; + repeated CIPDPackage additional_apk = 10; }
diff --git a/build/android/pylib/local/emulator/proto/avd_pb2.py b/build/android/pylib/local/emulator/proto/avd_pb2.py index 49cc1aa..e43534c 100644 --- a/build/android/pylib/local/emulator/proto/avd_pb2.py +++ b/build/android/pylib/local/emulator/proto/avd_pb2.py
@@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: avd.proto +# source: build/android/pylib/local/emulator/proto/avd.proto from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -14,11 +14,12 @@ DESCRIPTOR = _descriptor.FileDescriptor( - name='avd.proto', + name='build/android/pylib/local/emulator/proto/avd.proto', package='tools.android.avd.proto', syntax='proto3', serialized_options=None, - serialized_pb=b'\n\tavd.proto\x12\x17tools.android.avd.proto\"G\n\x0b\x43IPDPackage\x12\x14\n\x0cpackage_name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x11\n\tdest_path\x18\x03 \x01(\t\"@\n\x0eScreenSettings\x12\x0e\n\x06height\x18\x01 \x01(\r\x12\r\n\x05width\x18\x02 \x01(\r\x12\x0f\n\x07\x64\x65nsity\x18\x03 \x01(\r\"\x1e\n\x0eSdcardSettings\x12\x0c\n\x04size\x18\x01 \x01(\t\"\xa1\x02\n\x0b\x41vdSettings\x12\x37\n\x06screen\x18\x01 \x01(\x0b\x32\'.tools.android.avd.proto.ScreenSettings\x12\x37\n\x06sdcard\x18\x02 \x01(\x0b\x32\'.tools.android.avd.proto.SdcardSettings\x12U\n\x11\x61\x64vanced_features\x18\x03 \x03(\x0b\x32:.tools.android.avd.proto.AvdSettings.AdvancedFeaturesEntry\x12\x10\n\x08ram_size\x18\x04 \x01(\r\x1a\x37\n\x15\x41\x64vancedFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xad\x02\n\x03\x41vd\x12>\n\x10\x65mulator_package\x18\x01 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x42\n\x14system_image_package\x18\x02 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x19\n\x11system_image_name\x18\x03 \x01(\t\x12\x39\n\x0b\x61vd_package\x18\x04 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x10\n\x08\x61vd_name\x18\x05 \x01(\t\x12:\n\x0c\x61vd_settings\x18\x06 \x01(\x0b\x32$.tools.android.avd.proto.AvdSettingsb\x06proto3' + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n2build/android/pylib/local/emulator/proto/avd.proto\x12\x17tools.android.avd.proto\"G\n\x0b\x43IPDPackage\x12\x14\n\x0cpackage_name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x11\n\tdest_path\x18\x03 \x01(\t\"@\n\x0eScreenSettings\x12\x0e\n\x06height\x18\x01 \x01(\r\x12\r\n\x05width\x18\x02 \x01(\r\x12\x0f\n\x07\x64\x65nsity\x18\x03 \x01(\r\"\x1e\n\x0eSdcardSettings\x12\x0c\n\x04size\x18\x01 \x01(\t\"\xa8\x03\n\x0b\x41vdSettings\x12\x37\n\x06screen\x18\x01 \x01(\x0b\x32\'.tools.android.avd.proto.ScreenSettings\x12\x37\n\x06sdcard\x18\x02 \x01(\x0b\x32\'.tools.android.avd.proto.SdcardSettings\x12U\n\x11\x61\x64vanced_features\x18\x03 \x03(\x0b\x32:.tools.android.avd.proto.AvdSettings.AdvancedFeaturesEntry\x12\x10\n\x08ram_size\x18\x04 \x01(\r\x12O\n\x0e\x61vd_properties\x18\x05 \x03(\x0b\x32\x37.tools.android.avd.proto.AvdSettings.AvdPropertiesEntry\x1a\x37\n\x15\x41\x64vancedFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x34\n\x12\x41vdPropertiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xe4\x03\n\x03\x41vd\x12>\n\x10\x65mulator_package\x18\x01 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x42\n\x14system_image_package\x18\x02 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x19\n\x11system_image_name\x18\x03 \x01(\t\x12\x39\n\x0b\x61vd_package\x18\x04 \x01(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12\x10\n\x08\x61vd_name\x18\x05 \x01(\t\x12:\n\x0c\x61vd_settings\x18\x06 \x01(\x0b\x32$.tools.android.avd.proto.AvdSettings\x12\x0f\n\x07min_sdk\x18\x07 \x01(\r\x12(\n install_privileged_apk_partition\x18\x08 \x01(\t\x12<\n\x0eprivileged_apk\x18\t \x03(\x0b\x32$.tools.android.avd.proto.CIPDPackage\x12<\n\x0e\x61\x64\x64itional_apk\x18\n \x03(\x0b\x32$.tools.android.avd.proto.CIPDPackageb\x06proto3' ) @@ -30,6 +31,7 @@ filename=None, file=DESCRIPTOR, containing_type=None, + create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='package_name', full_name='tools.android.avd.proto.CIPDPackage.package_name', index=0, @@ -37,21 +39,21 @@ has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='version', full_name='tools.android.avd.proto.CIPDPackage.version', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='dest_path', full_name='tools.android.avd.proto.CIPDPackage.dest_path', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -64,8 +66,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=38, - serialized_end=109, + serialized_start=79, + serialized_end=150, ) @@ -75,6 +77,7 @@ filename=None, file=DESCRIPTOR, containing_type=None, + create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='height', full_name='tools.android.avd.proto.ScreenSettings.height', index=0, @@ -82,21 +85,21 @@ has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='width', full_name='tools.android.avd.proto.ScreenSettings.width', index=1, number=2, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='density', full_name='tools.android.avd.proto.ScreenSettings.density', index=2, number=3, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -109,8 +112,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=111, - serialized_end=175, + serialized_start=152, + serialized_end=216, ) @@ -120,6 +123,7 @@ filename=None, file=DESCRIPTOR, containing_type=None, + create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='size', full_name='tools.android.avd.proto.SdcardSettings.size', index=0, @@ -127,7 +131,7 @@ has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -140,8 +144,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=177, - serialized_end=207, + serialized_start=218, + serialized_end=248, ) @@ -151,6 +155,7 @@ filename=None, file=DESCRIPTOR, containing_type=None, + create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='key', full_name='tools.android.avd.proto.AvdSettings.AdvancedFeaturesEntry.key', index=0, @@ -158,14 +163,14 @@ has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='value', full_name='tools.android.avd.proto.AvdSettings.AdvancedFeaturesEntry.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -178,8 +183,46 @@ extension_ranges=[], oneofs=[ ], - serialized_start=444, - serialized_end=499, + serialized_start=566, + serialized_end=621, +) + +_AVDSETTINGS_AVDPROPERTIESENTRY = _descriptor.Descriptor( + name='AvdPropertiesEntry', + full_name='tools.android.avd.proto.AvdSettings.AvdPropertiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='tools.android.avd.proto.AvdSettings.AvdPropertiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='tools.android.avd.proto.AvdSettings.AvdPropertiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=623, + serialized_end=675, ) _AVDSETTINGS = _descriptor.Descriptor( @@ -188,6 +231,7 @@ filename=None, file=DESCRIPTOR, containing_type=None, + create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='screen', full_name='tools.android.avd.proto.AvdSettings.screen', index=0, @@ -195,32 +239,39 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='sdcard', full_name='tools.android.avd.proto.AvdSettings.sdcard', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='advanced_features', full_name='tools.android.avd.proto.AvdSettings.advanced_features', index=2, number=3, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='ram_size', full_name='tools.android.avd.proto.AvdSettings.ram_size', index=3, number=4, type=13, cpp_type=3, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='avd_properties', full_name='tools.android.avd.proto.AvdSettings.avd_properties', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], - nested_types=[_AVDSETTINGS_ADVANCEDFEATURESENTRY, ], + nested_types=[_AVDSETTINGS_ADVANCEDFEATURESENTRY, _AVDSETTINGS_AVDPROPERTIESENTRY, ], enum_types=[ ], serialized_options=None, @@ -229,8 +280,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=210, - serialized_end=499, + serialized_start=251, + serialized_end=675, ) @@ -240,6 +291,7 @@ filename=None, file=DESCRIPTOR, containing_type=None, + create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='emulator_package', full_name='tools.android.avd.proto.Avd.emulator_package', index=0, @@ -247,42 +299,70 @@ has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='system_image_package', full_name='tools.android.avd.proto.Avd.system_image_package', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='system_image_name', full_name='tools.android.avd.proto.Avd.system_image_name', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='avd_package', full_name='tools.android.avd.proto.Avd.avd_package', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='avd_name', full_name='tools.android.avd.proto.Avd.avd_name', index=4, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), _descriptor.FieldDescriptor( name='avd_settings', full_name='tools.android.avd.proto.Avd.avd_settings', index=5, number=6, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='min_sdk', full_name='tools.android.avd.proto.Avd.min_sdk', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='install_privileged_apk_partition', full_name='tools.android.avd.proto.Avd.install_privileged_apk_partition', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='privileged_apk', full_name='tools.android.avd.proto.Avd.privileged_apk', index=8, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='additional_apk', full_name='tools.android.avd.proto.Avd.additional_apk', index=9, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -295,18 +375,22 @@ extension_ranges=[], oneofs=[ ], - serialized_start=502, - serialized_end=803, + serialized_start=678, + serialized_end=1162, ) _AVDSETTINGS_ADVANCEDFEATURESENTRY.containing_type = _AVDSETTINGS +_AVDSETTINGS_AVDPROPERTIESENTRY.containing_type = _AVDSETTINGS _AVDSETTINGS.fields_by_name['screen'].message_type = _SCREENSETTINGS _AVDSETTINGS.fields_by_name['sdcard'].message_type = _SDCARDSETTINGS _AVDSETTINGS.fields_by_name['advanced_features'].message_type = _AVDSETTINGS_ADVANCEDFEATURESENTRY +_AVDSETTINGS.fields_by_name['avd_properties'].message_type = _AVDSETTINGS_AVDPROPERTIESENTRY _AVD.fields_by_name['emulator_package'].message_type = _CIPDPACKAGE _AVD.fields_by_name['system_image_package'].message_type = _CIPDPACKAGE _AVD.fields_by_name['avd_package'].message_type = _CIPDPACKAGE _AVD.fields_by_name['avd_settings'].message_type = _AVDSETTINGS +_AVD.fields_by_name['privileged_apk'].message_type = _CIPDPACKAGE +_AVD.fields_by_name['additional_apk'].message_type = _CIPDPACKAGE DESCRIPTOR.message_types_by_name['CIPDPackage'] = _CIPDPACKAGE DESCRIPTOR.message_types_by_name['ScreenSettings'] = _SCREENSETTINGS DESCRIPTOR.message_types_by_name['SdcardSettings'] = _SDCARDSETTINGS @@ -316,21 +400,21 @@ CIPDPackage = _reflection.GeneratedProtocolMessageType('CIPDPackage', (_message.Message,), { 'DESCRIPTOR' : _CIPDPACKAGE, - '__module__' : 'avd_pb2' + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' # @@protoc_insertion_point(class_scope:tools.android.avd.proto.CIPDPackage) }) _sym_db.RegisterMessage(CIPDPackage) ScreenSettings = _reflection.GeneratedProtocolMessageType('ScreenSettings', (_message.Message,), { 'DESCRIPTOR' : _SCREENSETTINGS, - '__module__' : 'avd_pb2' + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' # @@protoc_insertion_point(class_scope:tools.android.avd.proto.ScreenSettings) }) _sym_db.RegisterMessage(ScreenSettings) SdcardSettings = _reflection.GeneratedProtocolMessageType('SdcardSettings', (_message.Message,), { 'DESCRIPTOR' : _SDCARDSETTINGS, - '__module__' : 'avd_pb2' + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' # @@protoc_insertion_point(class_scope:tools.android.avd.proto.SdcardSettings) }) _sym_db.RegisterMessage(SdcardSettings) @@ -339,24 +423,33 @@ 'AdvancedFeaturesEntry' : _reflection.GeneratedProtocolMessageType('AdvancedFeaturesEntry', (_message.Message,), { 'DESCRIPTOR' : _AVDSETTINGS_ADVANCEDFEATURESENTRY, - '__module__' : 'avd_pb2' + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' # @@protoc_insertion_point(class_scope:tools.android.avd.proto.AvdSettings.AdvancedFeaturesEntry) }) , + + 'AvdPropertiesEntry' : _reflection.GeneratedProtocolMessageType('AvdPropertiesEntry', (_message.Message,), { + 'DESCRIPTOR' : _AVDSETTINGS_AVDPROPERTIESENTRY, + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' + # @@protoc_insertion_point(class_scope:tools.android.avd.proto.AvdSettings.AvdPropertiesEntry) + }) + , 'DESCRIPTOR' : _AVDSETTINGS, - '__module__' : 'avd_pb2' + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' # @@protoc_insertion_point(class_scope:tools.android.avd.proto.AvdSettings) }) _sym_db.RegisterMessage(AvdSettings) _sym_db.RegisterMessage(AvdSettings.AdvancedFeaturesEntry) +_sym_db.RegisterMessage(AvdSettings.AvdPropertiesEntry) Avd = _reflection.GeneratedProtocolMessageType('Avd', (_message.Message,), { 'DESCRIPTOR' : _AVD, - '__module__' : 'avd_pb2' + '__module__' : 'build.android.pylib.local.emulator.proto.avd_pb2' # @@protoc_insertion_point(class_scope:tools.android.avd.proto.Avd) }) _sym_db.RegisterMessage(Avd) _AVDSETTINGS_ADVANCEDFEATURESENTRY._options = None +_AVDSETTINGS_AVDPROPERTIESENTRY._options = None # @@protoc_insertion_point(module_scope)
diff --git a/build/android/pylib/local/local_test_server_spawner.py b/build/android/pylib/local/local_test_server_spawner.py index f21f1be..453d9aa 100644 --- a/build/android/pylib/local/local_test_server_spawner.py +++ b/build/android/pylib/local/local_test_server_spawner.py
@@ -1,8 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import + import json import time @@ -62,7 +62,7 @@ class LocalTestServerSpawner(test_server.TestServer): def __init__(self, port, device, tool): - super(LocalTestServerSpawner, self).__init__() + super().__init__() self._device = device self._spawning_server = chrome_test_server_spawner.SpawningServer( port, PortForwarderAndroid(device, tool), MAX_TEST_SERVER_INSTANCES)
diff --git a/build/android/pylib/local/machine/__init__.py b/build/android/pylib/local/machine/__init__.py index ca3e206..68130d5 100644 --- a/build/android/pylib/local/machine/__init__.py +++ b/build/android/pylib/local/machine/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/local/machine/local_machine_environment.py b/build/android/pylib/local/machine/local_machine_environment.py index d198f89..d75dc88 100644 --- a/build/android/pylib/local/machine/local_machine_environment.py +++ b/build/android/pylib/local/machine/local_machine_environment.py
@@ -1,20 +1,14 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import -import devil_chromium -from pylib import constants from pylib.base import environment class LocalMachineEnvironment(environment.Environment): def __init__(self, _args, output_manager, _error_func): - super(LocalMachineEnvironment, self).__init__(output_manager) - - devil_chromium.Initialize( - output_directory=constants.GetOutDirectory()) + super().__init__(output_manager) #override def SetUp(self):
diff --git a/build/android/pylib/local/machine/local_machine_junit_test_run.py b/build/android/pylib/local/machine/local_machine_junit_test_run.py index a64b63b..a923d6a 100644 --- a/build/android/pylib/local/machine/local_machine_junit_test_run.py +++ b/build/android/pylib/local/machine/local_machine_junit_test_run.py
@@ -1,25 +1,28 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import absolute_import -import collections import json import logging import multiprocessing import os -import select +import queue +import re import subprocess import sys +import tempfile +import threading +import time import zipfile from six.moves import range # pylint: disable=redefined-builtin +from devil.utils import cmd_helper +from py_utils import tempfile_ext from pylib import constants from pylib.base import base_test_result from pylib.base import test_run from pylib.constants import host_paths from pylib.results import json_results -from py_utils import tempfile_ext # These Test classes are used for running tests and are excluded in the test @@ -42,19 +45,37 @@ # and 6 sec with 2 or more shards. _MIN_CLASSES_PER_SHARD = 8 +# Running the largest test suite with a single shard takes about 22 minutes. +_SHARD_TIMEOUT = 30 * 60 + +# RegExp to detect logcat lines, e.g., 'I/AssetManager: not found'. +_LOGCAT_RE = re.compile(r'[A-Z]/[\w\d_-]+:') + class LocalMachineJunitTestRun(test_run.TestRun): - def __init__(self, env, test_instance): - super(LocalMachineJunitTestRun, self).__init__(env, test_instance) - - #override + # override def TestPackage(self): return self._test_instance.suite - #override + # override def SetUp(self): pass + def _GetFilterArgs(self, shard_test_filter=None): + ret = [] + if shard_test_filter: + ret += ['-gtest-filter', ':'.join(shard_test_filter)] + + for test_filter in self._test_instance.test_filters: + ret += ['-gtest-filter', test_filter] + + if self._test_instance.package_filter: + ret += ['-package-filter', self._test_instance.package_filter] + if self._test_instance.runner_filter: + ret += ['-runner-filter', self._test_instance.runner_filter] + + return ret + def _CreateJarArgsList(self, json_result_file_paths, group_test_list, shards): # Creates a list of jar_args. The important thing is each jar_args list # has a different json_results file for writing test results to and that @@ -63,43 +84,41 @@ jar_args_list = [['-json-results-file', result_file] for result_file in json_result_file_paths] for index, jar_arg in enumerate(jar_args_list): - if shards > 1: - jar_arg.extend(['-gtest-filter', ':'.join(group_test_list[index])]) - elif self._test_instance.test_filter: - jar_arg.extend(['-gtest-filter', self._test_instance.test_filter]) - - if self._test_instance.package_filter: - jar_arg.extend(['-package-filter', self._test_instance.package_filter]) - if self._test_instance.runner_filter: - jar_arg.extend(['-runner-filter', self._test_instance.runner_filter]) + shard_test_filter = group_test_list[index] if shards > 1 else None + jar_arg += self._GetFilterArgs(shard_test_filter) return jar_args_list - def _CreateJvmArgsList(self): + def _CreateJvmArgsList(self, for_listing=False): # Creates a list of jvm_args (robolectric, code coverage, etc...) jvm_args = [ '-Drobolectric.dependency.dir=%s' % self._test_instance.robolectric_runtime_deps_dir, '-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT, + # Use locally available sdk jars from 'robolectric.dependency.dir' + '-Drobolectric.offline=true', '-Drobolectric.resourcesMode=binary', + '-Drobolectric.logging=stdout', + '-Djava.library.path=%s' % self._test_instance.native_libs_dir, ] - if logging.getLogger().isEnabledFor(logging.INFO): - jvm_args += ['-Drobolectric.logging=stdout'] - if self._test_instance.debug_socket: + if self._test_instance.debug_socket and not for_listing: jvm_args += [ - '-agentlib:jdwp=transport=dt_socket' - ',server=y,suspend=y,address=%s' % self._test_instance.debug_socket + '-Dchromium.jdwp_active=true', + ('-agentlib:jdwp=transport=dt_socket' + ',server=y,suspend=y,address=%s' % self._test_instance.debug_socket) ] - if self._test_instance.coverage_dir: + if self._test_instance.coverage_dir and not for_listing: if not os.path.exists(self._test_instance.coverage_dir): os.makedirs(self._test_instance.coverage_dir) elif not os.path.isdir(self._test_instance.coverage_dir): raise Exception('--coverage-dir takes a directory, not file path.') + # Jacoco supports concurrent processes using the same output file: + # https://github.com/jacoco/jacoco/blob/6cd3f0bd8e348f8fba7bffec5225407151f1cc91/org.jacoco.agent.rt/src/org/jacoco/agent/rt/internal/output/FileOutput.java#L67 + # So no need to vary the output based on shard number. + jacoco_coverage_file = os.path.join(self._test_instance.coverage_dir, + '%s.exec' % self._test_instance.suite) if self._test_instance.coverage_on_the_fly: - jacoco_coverage_file = os.path.join( - self._test_instance.coverage_dir, - '%s.exec' % self._test_instance.suite) jacoco_agent_path = os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party', 'jacoco', 'lib', 'jacocoagent.jar') @@ -109,41 +128,58 @@ jvm_args.append( jacoco_args.format(jacoco_agent_path, jacoco_coverage_file)) else: - jvm_args.append('-Djacoco-agent.destfile=%s' % - os.path.join(self._test_instance.coverage_dir, - '%s.exec' % self._test_instance.suite)) + jvm_args.append('-Djacoco-agent.destfile=%s' % jacoco_coverage_file) return jvm_args - #override - def RunTests(self, results): - wrapper_path = os.path.join(constants.GetOutDirectory(), 'bin', 'helper', - self._test_instance.suite) + @property + def _wrapper_path(self): + return os.path.join(constants.GetOutDirectory(), 'bin', 'helper', + self._test_instance.suite) + #override + def GetTestsForListing(self): + with tempfile_ext.NamedTemporaryDirectory() as temp_dir: + cmd = [self._wrapper_path, '--list-tests'] + self._GetFilterArgs() + jvm_args = self._CreateJvmArgsList(for_listing=True) + if jvm_args: + cmd += ['--jvm-args', '"%s"' % ' '.join(jvm_args)] + AddPropertiesJar([cmd], temp_dir, self._test_instance.resource_apk) + lines = subprocess.check_output(cmd, encoding='utf8').splitlines() + + PREFIX = '#TEST# ' + prefix_len = len(PREFIX) + # Filter log messages other than test names (Robolectric logs to stdout). + return sorted(l[prefix_len:] for l in lines if l.startswith(PREFIX)) + + # override + def RunTests(self, results, raw_logs_fh=None): # This avoids searching through the classparth jars for tests classes, # which takes about 1-2 seconds. - # Do not shard when a test filter is present since we do not know at this - # point which tests will be filtered out. - if (self._test_instance.shards == 1 or self._test_instance.test_filter - or self._test_instance.suite in _EXCLUDED_SUITES): + if (self._test_instance.shards == 1 + # TODO(crbug.com/1383650): remove this + or self._test_instance.has_literal_filters or + self._test_instance.suite in _EXCLUDED_SUITES): test_classes = [] shards = 1 else: - test_classes = _GetTestClasses(wrapper_path) + test_classes = _GetTestClasses(self._wrapper_path) shards = ChooseNumOfShards(test_classes, self._test_instance.shards) logging.info('Running tests on %d shard(s).', shards) group_test_list = GroupTestsForShard(shards, test_classes) with tempfile_ext.NamedTemporaryDirectory() as temp_dir: - cmd_list = [[wrapper_path] for _ in range(shards)] + cmd_list = [[self._wrapper_path] for _ in range(shards)] json_result_file_paths = [ os.path.join(temp_dir, 'results%d.json' % i) for i in range(shards) ] jar_args_list = self._CreateJarArgsList(json_result_file_paths, group_test_list, shards) - for i in range(shards): - cmd_list[i].extend(['--jar-args', '"%s"' % ' '.join(jar_args_list[i])]) + if jar_args_list: + for i in range(shards): + cmd_list[i].extend( + ['--jar-args', '"%s"' % ' '.join(jar_args_list[i])]) jvm_args = self._CreateJvmArgsList() if jvm_args: @@ -152,12 +188,21 @@ AddPropertiesJar(cmd_list, temp_dir, self._test_instance.resource_apk) - procs = [ - subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) for cmd in cmd_list - ] - PrintProcessesStdout(procs) + show_logcat = logging.getLogger().isEnabledFor(logging.INFO) + num_omitted_lines = 0 + for line in _RunCommandsAndSerializeOutput(cmd_list): + if raw_logs_fh: + raw_logs_fh.write(line) + if show_logcat or not _LOGCAT_RE.match(line): + sys.stdout.write(line) + else: + num_omitted_lines += 1 + + if num_omitted_lines > 0: + logging.critical('%d log lines omitted.', num_omitted_lines) + sys.stdout.flush() + if raw_logs_fh: + raw_logs_fh.flush() results_list = [] try: @@ -169,15 +214,15 @@ # In the case of a failure in the JUnit or Robolectric test runner # the output json file may never be written. results_list = [ - base_test_result.BaseTestResult( - 'Test Runner Failure', base_test_result.ResultType.UNKNOWN) + base_test_result.BaseTestResult('Test Runner Failure', + base_test_result.ResultType.UNKNOWN) ] test_run_results = base_test_result.TestRunResults() test_run_results.AddResults(results_list) results.append(test_run_results) - #override + # override def TearDown(self): pass @@ -188,7 +233,14 @@ properties_jar_path = os.path.join(temp_dir, 'properties.jar') with zipfile.ZipFile(properties_jar_path, 'w') as z: z.writestr('com/android/tools/test_config.properties', - 'android_resource_apk=%s' % resource_apk) + 'android_resource_apk=%s\n' % resource_apk) + props = [ + 'application = android.app.Application', + 'sdk = 28', + ('shadows = org.chromium.testing.local.' + 'CustomShadowApplicationPackageManager'), + ] + z.writestr('robolectric.properties', '\n'.join(props)) for cmd in cmd_list: cmd.extend(['--classpath', properties_jar_path]) @@ -236,40 +288,122 @@ return test_dict -def PrintProcessesStdout(procs): - """Prints the stdout of all the processes. +def _DumpJavaStacks(pid): + jcmd = os.path.join(constants.JAVA_HOME, 'bin', 'jcmd') + cmd = [jcmd, str(pid), 'Thread.print'] + result = subprocess.run(cmd, + check=False, + stdout=subprocess.PIPE, + encoding='utf8') + if result.returncode: + return 'Failed to dump stacks\n' + result.stdout + return result.stdout - Buffers the stdout of the processes and prints it when finished. + +def _RunCommandsAndSerializeOutput(cmd_list): + """Runs multiple commands in parallel and yields serialized output lines. Args: - procs: A list of subprocesses. + cmd_list: List of commands. Returns: N/A + + Raises: + TimeoutError: If timeout is exceeded. """ - streams = [p.stdout for p in procs] - outputs = collections.defaultdict(list) - first_fd = streams[0].fileno() + num_shards = len(cmd_list) + assert num_shards > 0 + procs = [] + temp_files = [] + for i, cmd in enumerate(cmd_list): + # Shard 0 yields results immediately, the rest write to files. + if i == 0: + temp_files.append(None) # Placeholder. + procs.append( + cmd_helper.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + )) + else: + temp_file = tempfile.TemporaryFile(mode='w+t', encoding='utf-8') + temp_files.append(temp_file) + procs.append(cmd_helper.Popen( + cmd, + stdout=temp_file, + stderr=temp_file, + )) - while streams: - rstreams, _, _ = select.select(streams, [], []) - for stream in rstreams: - line = stream.readline() - if line: - # Print out just one output so user can see work being done rather - # than waiting for it all at the end. - if stream.fileno() == first_fd: - sys.stdout.write(line) - else: - outputs[stream.fileno()].append(line) - else: - streams.remove(stream) # End of stream. + deadline = time.time() + (_SHARD_TIMEOUT / (num_shards // 2 + 1)) - for p in procs: - sys.stdout.write(''.join(outputs[p.stdout.fileno()])) + yield '\n' + yield 'Shard 0 output:\n' + + # The following will be run from a thread to pump Shard 0 results, allowing + # live output while allowing timeout. + def pump_stream_to_queue(f, q): + for line in f: + q.put(line) + q.put(None) + + shard_0_q = queue.Queue() + shard_0_pump = threading.Thread(target=pump_stream_to_queue, + args=(procs[0].stdout, shard_0_q)) + shard_0_pump.start() + + timeout_dumps = {} + + # Print the first process until timeout or completion. + while shard_0_pump.is_alive(): + try: + line = shard_0_q.get(timeout=deadline - time.time()) + if line is None: + break + yield line + except queue.Empty: + if time.time() > deadline: + break + + # Wait for remaining processes to finish. + for i, proc in enumerate(procs): + try: + proc.wait(timeout=deadline - time.time()) + except subprocess.TimeoutExpired: + timeout_dumps[i] = _DumpJavaStacks(proc.pid) + proc.kill() + + # Output any remaining output from a timed-out first shard. + shard_0_pump.join() + while not shard_0_q.empty(): + yield shard_0_q.get() + + for i in range(1, num_shards): + f = temp_files[i] + yield '\n' + yield 'Shard %d output:\n' % i + f.seek(0) + for line in f.readlines(): + yield line + f.close() + + # Output stacks + if timeout_dumps: + yield '\n' + yield ('=' * 80) + '\n' + yield '\nOne or mord shards timed out.\n' + yield ('=' * 80) + '\n' + for i, dump in timeout_dumps.items(): + yield 'Index of timed out shard: %d\n' % i + yield 'Thread dump:\n' + yield dump + yield '\n' + + raise cmd_helper.TimeoutError('Junit shards timed out.') def _GetTestClasses(file_path): - test_jar_paths = subprocess.check_output([file_path, '--print-classpath']) + test_jar_paths = subprocess.check_output([file_path, + '--print-classpath']).decode() test_jar_paths = test_jar_paths.split(':') test_classes = []
diff --git a/build/android/pylib/local/machine/local_machine_junit_test_run_test.py b/build/android/pylib/local/machine/local_machine_junit_test_run_test.py index 2bbe561..d8913b4 100755 --- a/build/android/pylib/local/machine/local_machine_junit_test_run_test.py +++ b/build/android/pylib/local/machine/local_machine_junit_test_run_test.py
@@ -1,11 +1,11 @@ -#!/usr/bin/env vpython -# Copyright 2020 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # pylint: disable=protected-access -from __future__ import absolute_import + import os import unittest @@ -20,17 +20,17 @@ apk = 'resource_apk' cmd_list = [] local_machine_junit_test_run.AddPropertiesJar(cmd_list, temp_dir, apk) - self.assertEquals(cmd_list, []) + self.assertEqual(cmd_list, []) cmd_list = [['test1']] local_machine_junit_test_run.AddPropertiesJar(cmd_list, temp_dir, apk) - self.assertEquals( + self.assertEqual( cmd_list[0], ['test1', '--classpath', os.path.join(temp_dir, 'properties.jar')]) cmd_list = [['test1'], ['test2']] local_machine_junit_test_run.AddPropertiesJar(cmd_list, temp_dir, apk) - self.assertEquals(len(cmd_list[0]), 3) - self.assertEquals( + self.assertEqual(len(cmd_list[0]), 3) + self.assertEqual( cmd_list[1], ['test2', '--classpath', os.path.join(temp_dir, 'properties.jar')]) @@ -43,20 +43,20 @@ test_classes = [1] * 50 shards = local_machine_junit_test_run.ChooseNumOfShards( test_classes, test_shards) - self.assertEquals(1, shards) + self.assertEqual(1, shards) # Tests setting shards. test_shards = 4 shards = local_machine_junit_test_run.ChooseNumOfShards( test_classes, test_shards) - self.assertEquals(4, shards) + self.assertEqual(4, shards) # Tests using min_class per shards. test_classes = [1] * 20 test_shards = 8 shards = local_machine_junit_test_run.ChooseNumOfShards( test_classes, test_shards) - self.assertEquals(2, shards) + self.assertEqual(2, shards) def testGroupTestsForShard(self): test_classes = []
diff --git a/build/android/pylib/monkey/monkey_test_instance.py b/build/android/pylib/monkey/monkey_test_instance.py index 10b1131..d53f5cd 100644 --- a/build/android/pylib/monkey/monkey_test_instance.py +++ b/build/android/pylib/monkey/monkey_test_instance.py
@@ -1,7 +1,8 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import random from pylib import constants @@ -13,7 +14,7 @@ class MonkeyTestInstance(test_instance.TestInstance): def __init__(self, args, _): - super(MonkeyTestInstance, self).__init__() + super().__init__() self._categories = args.categories self._event_count = args.event_count
diff --git a/build/android/pylib/output/__init__.py b/build/android/pylib/output/__init__.py index a22a6ee..b8e1dbd 100644 --- a/build/android/pylib/output/__init__.py +++ b/build/android/pylib/output/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/output/local_output_manager.py b/build/android/pylib/output/local_output_manager.py index 89becd7..74b4b95 100644 --- a/build/android/pylib/output/local_output_manager.py +++ b/build/android/pylib/output/local_output_manager.py
@@ -1,11 +1,15 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import time import os import shutil -import urllib + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote from pylib.base import output_manager @@ -17,7 +21,7 @@ """ def __init__(self, output_dir): - super(LocalOutputManager, self).__init__() + super().__init__() timestamp = time.strftime( '%Y_%m_%dT%H_%M_%S', time.localtime()) self._output_root = os.path.abspath(os.path.join( @@ -32,12 +36,11 @@ class LocalArchivedFile(output_manager.ArchivedFile): def __init__(self, out_filename, out_subdir, datatype, out_root): - super(LocalArchivedFile, self).__init__( - out_filename, out_subdir, datatype) + super().__init__(out_filename, out_subdir, datatype) self._output_path = os.path.join(out_root, out_subdir, out_filename) def _Link(self): - return 'file://%s' % urllib.quote(self._output_path) + return 'file://%s' % quote(self._output_path) def _Archive(self): if not os.path.exists(os.path.dirname(self._output_path)):
diff --git a/build/android/pylib/output/local_output_manager_test.py b/build/android/pylib/output/local_output_manager_test.py index 7954350..d238814 100755 --- a/build/android/pylib/output/local_output_manager_test.py +++ b/build/android/pylib/output/local_output_manager_test.py
@@ -1,5 +1,5 @@ -#! /usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#! /usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/output/noop_output_manager.py b/build/android/pylib/output/noop_output_manager.py index d29a743..acabd30 100644 --- a/build/android/pylib/output/noop_output_manager.py +++ b/build/android/pylib/output/noop_output_manager.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,9 +13,6 @@ class NoopOutputManager(output_manager.OutputManager): - def __init__(self): - super(NoopOutputManager, self).__init__() - #override def _CreateArchivedFile(self, out_filename, out_subdir, datatype): del out_filename, out_subdir, datatype @@ -25,7 +22,7 @@ class NoopArchivedFile(output_manager.ArchivedFile): def __init__(self): - super(NoopArchivedFile, self).__init__(None, None, None) + super().__init__(None, None, None) def Link(self): """NoopArchivedFiles are not retained.""" @@ -36,7 +33,6 @@ def Archive(self): """NoopArchivedFiles are not retained.""" - pass def _Archive(self): pass
diff --git a/build/android/pylib/output/noop_output_manager_test.py b/build/android/pylib/output/noop_output_manager_test.py index 4e470ef..ff4c805 100755 --- a/build/android/pylib/output/noop_output_manager_test.py +++ b/build/android/pylib/output/noop_output_manager_test.py
@@ -1,5 +1,5 @@ -#! /usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#! /usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/output/remote_output_manager.py b/build/android/pylib/output/remote_output_manager.py index 9fdb4bf..bf585bb 100644 --- a/build/android/pylib/output/remote_output_manager.py +++ b/build/android/pylib/output/remote_output_manager.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -22,7 +22,7 @@ Args bucket: Bucket to use when saving to Google Storage. """ - super(RemoteOutputManager, self).__init__() + super().__init__() self._bucket = bucket #override @@ -43,7 +43,7 @@ class LogdogArchivedFile(output_manager.ArchivedFile): def __init__(self, out_filename, out_subdir, datatype): - super(LogdogArchivedFile, self).__init__(out_filename, out_subdir, datatype) + super().__init__(out_filename, out_subdir, datatype) self._stream_name = '%s_%s' % (out_subdir, out_filename) def _Link(self): @@ -57,8 +57,7 @@ class GoogleStorageArchivedFile(output_manager.ArchivedFile): def __init__(self, out_filename, out_subdir, datatype, bucket): - super(GoogleStorageArchivedFile, self).__init__( - out_filename, out_subdir, datatype) + super().__init__(out_filename, out_subdir, datatype) self._bucket = bucket self._upload_path = None self._content_addressed = None
diff --git a/build/android/pylib/output/remote_output_manager_test.py b/build/android/pylib/output/remote_output_manager_test.py index 4c6c081..875451c 100755 --- a/build/android/pylib/output/remote_output_manager_test.py +++ b/build/android/pylib/output/remote_output_manager_test.py
@@ -1,5 +1,5 @@ -#! /usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#! /usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/pexpect.py b/build/android/pylib/pexpect.py index cf59fb0..6ed6451 100644 --- a/build/android/pylib/pexpect.py +++ b/build/android/pylib/pexpect.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from __future__ import absolute_import
diff --git a/build/android/pylib/restart_adbd.sh b/build/android/pylib/restart_adbd.sh index 393b2eb..2016286 100755 --- a/build/android/pylib/restart_adbd.sh +++ b/build/android/pylib/restart_adbd.sh
@@ -1,6 +1,6 @@ #!/system/bin/sh -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/results/__init__.py b/build/android/pylib/results/__init__.py index 4d6aabb..d46d7b4 100644 --- a/build/android/pylib/results/__init__.py +++ b/build/android/pylib/results/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/results/flakiness_dashboard/__init__.py b/build/android/pylib/results/flakiness_dashboard/__init__.py index 4d6aabb..d46d7b4 100644 --- a/build/android/pylib/results/flakiness_dashboard/__init__.py +++ b/build/android/pylib/results/flakiness_dashboard/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/results/flakiness_dashboard/json_results_generator.py b/build/android/pylib/results/flakiness_dashboard/json_results_generator.py index b2e542b..3e753e5 100644 --- a/build/android/pylib/results/flakiness_dashboard/json_results_generator.py +++ b/build/android/pylib/results/flakiness_dashboard/json_results_generator.py
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,7 +13,13 @@ import mimetypes import os import time -import urllib2 +try: + from urllib.request import urlopen, Request + from urllib.error import HTTPError, URLError + from urllib.parse import quote +except ImportError: + from urllib import quote + from urllib2 import urlopen, HTTPError, URLError, Request _log = logging.getLogger(__name__) @@ -44,11 +50,11 @@ def ConvertTrieToFlatPaths(trie, prefix=None): """Flattens the trie of paths, prepending a prefix to each.""" result = {} - for name, data in trie.iteritems(): + for name, data in trie.items(): if prefix: name = prefix + '/' + name - if len(data) and not 'results' in data: + if len(data) != 0 and not 'results' in data: result.update(ConvertTrieToFlatPaths(data, name)) else: result[name] = data @@ -91,11 +97,11 @@ return trie -class TestResult(object): +class TestResult: """A simple class that represents a single test result.""" # Test modifier constants. - (NONE, FAILS, FLAKY, DISABLED) = range(4) + (NONE, FAILS, FLAKY, DISABLED) = list(range(4)) def __init__(self, test, failed=False, elapsed_time=0): self.test_name = test @@ -106,7 +112,7 @@ try: test_name = test.split('.')[1] except IndexError: - _log.warn('Invalid test name: %s.', test) + _log.warning('Invalid test name: %s.', test) if test_name.startswith('FAILS_'): self.modifier = self.FAILS @@ -121,7 +127,7 @@ return self.failed or self.modifier == self.DISABLED -class JSONResultsGeneratorBase(object): +class JSONResultsGeneratorBase: """A JSON results generator for generic tests.""" MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 @@ -195,7 +201,7 @@ self._results_directory = results_file_base_path self._test_results_map = test_results_map - self._test_results = test_results_map.values() + self._test_results = list(test_results_map.values()) self._svn_repositories = svn_repositories if not self._svn_repositories: @@ -217,7 +223,7 @@ WriteJSON(json_object, file_path) def GenerateTimesMSFile(self): - times = TestTimingsTrie(self._test_results_map.values()) + times = TestTimingsTrie(list(self._test_results_map.values())) file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME) WriteJSON(times, file_path) @@ -231,9 +237,10 @@ # If there was an error don't write a results.json # file at all as it would lose all the information on the # bot. - _log.error('Archive directory is inaccessible. Not ' - 'modifying or clobbering the results.json ' - 'file: ' + str(error)) + _log.error( + 'Archive directory is inaccessible. Not ' + 'modifying or clobbering the results.json ' + 'file: %s', error) return None builder_name = self._builder_name @@ -315,7 +322,7 @@ def _GetFailedTestNames(self): """Returns a set of failed test names.""" - return set([r.test_name for r in self._test_results if r.failed]) + return set(r.test_name for r in self._test_results if r.failed) def _GetModifierChar(self, test_name): """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, @@ -326,7 +333,7 @@ return self.__class__.NO_DATA_RESULT test_result = self._test_results_map[test_name] - if test_result.modifier in self.MODIFIER_TO_CHAR.keys(): + if test_result.modifier in list(self.MODIFIER_TO_CHAR.keys()): return self.MODIFIER_TO_CHAR[test_result.modifier] return self.__class__.PASS_RESULT @@ -374,25 +381,21 @@ return {}, None results_file_url = (self.URL_FOR_TEST_LIST_JSON % - (urllib2.quote(self._test_results_server), - urllib2.quote(self._builder_name), - self.RESULTS_FILENAME, - urllib2.quote(self._test_type), - urllib2.quote(self._master_name))) + (quote(self._test_results_server), + quote(self._builder_name), self.RESULTS_FILENAME, + quote(self._test_type), quote(self._master_name))) - # pylint: disable=redefined-variable-type try: # FIXME: We should talk to the network via a Host object. - results_file = urllib2.urlopen(results_file_url) + results_file = urlopen(results_file_url) old_results = results_file.read() - except urllib2.HTTPError as http_error: + except HTTPError as http_error: # A non-4xx status code means the bot is hosed for some reason # and we can't grab the results.json file off of it. if http_error.code < 400 and http_error.code >= 500: error = http_error - except urllib2.URLError as url_error: + except URLError as url_error: error = url_error - # pylint: enable=redefined-variable-type if old_results: # Strip the prefix and suffix so we can get the actual JSON object. @@ -426,7 +429,7 @@ # Create a test modifiers (FAILS, FLAKY etc) summary dictionary. entry = {} - for test_name in self._test_results_map.iterkeys(): + for test_name in self._test_results_map.keys(): result_char = self._GetModifierChar(test_name) entry[result_char] = entry.get(result_char, 0) + 1 @@ -466,7 +469,7 @@ encoded_results: run-length encoded results. An array of arrays, e.g. [[3,'A'],[1,'Q']] encodes AAAQ. """ - if len(encoded_results) and item == encoded_results[0][1]: + if len(encoded_results) != 0 and item == encoded_results[0][1]: num_results = encoded_results[0][0] if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: encoded_results[0][0] = num_results + 1 @@ -517,7 +520,7 @@ this_test[segment] = {} this_test = this_test[segment] - if not len(this_test): + if len(this_test) == 0: self._PopulateResultsAndTimesJSON(this_test) if self.RESULTS in this_test: @@ -543,7 +546,7 @@ # version 3->4 if archive_version == 3: - for results in results_json.values(): + for results in list(results_json.values()): self._ConvertTestsToTrie(results) results_json[self.VERSION_KEY] = self.VERSION @@ -554,7 +557,7 @@ test_results = results[self.TESTS] test_results_trie = {} - for test in test_results.iterkeys(): + for test in test_results.keys(): single_test_result = test_results[test] AddPathToTrie(test, single_test_result, test_results_trie) @@ -620,7 +623,7 @@ return len(results) == 1 and results[0][1] == result_type -class _FileUploader(object): +class _FileUploader: def __init__(self, url, timeout_seconds): self._url = url @@ -629,7 +632,7 @@ def UploadAsMultipartFormData(self, files, attrs): file_objs = [] for filename, path in files: - with file(path, 'rb') as fp: + with open(path, 'rb') as fp: file_objs.append(('file', filename, fp.read())) # FIXME: We should use the same variable names for the formal and actual @@ -642,12 +645,12 @@ end = start + self._timeout_seconds while time.time() < end: try: - request = urllib2.Request(self._url, data, - {'Content-Type': content_type}) - return urllib2.urlopen(request) - except urllib2.HTTPError as e: - _log.warn("Received HTTP status %s loading \"%s\". " - 'Retrying in 10 seconds...', e.code, e.filename) + request = Request(self._url, data, {'Content-Type': content_type}) + return urlopen(request) + except HTTPError as e: + _log.warning( + 'Received HTTP status %s loading "%s". ' + 'Retrying in 10 seconds...', e.code, e.filename) time.sleep(10) @@ -678,7 +681,7 @@ lines.append('--' + BOUNDARY) lines.append('Content-Disposition: form-data; name="%s"' % key) lines.append('') - if isinstance(value, unicode): + if isinstance(value, str): value = value.encode('utf-8') lines.append(value) @@ -688,7 +691,7 @@ 'filename="%s"' % (key, filename)) lines.append('Content-Type: %s' % _GetMIMEType(filename)) lines.append('') - if isinstance(value, unicode): + if isinstance(value, str): value = value.encode('utf-8') lines.append(value)
diff --git a/build/android/pylib/results/flakiness_dashboard/json_results_generator_unittest.py b/build/android/pylib/results/flakiness_dashboard/json_results_generator_unittest.py index d6aee05..b1d8bfd 100644 --- a/build/android/pylib/results/flakiness_dashboard/json_results_generator_unittest.py +++ b/build/android/pylib/results/flakiness_dashboard/json_results_generator_unittest.py
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -47,16 +47,13 @@ def _TestJSONGeneration(self, passed_tests_list, failed_tests_list): tests_set = set(passed_tests_list) | set(failed_tests_list) - DISABLED_tests = set([t for t in tests_set - if t.startswith('DISABLED_')]) - FLAKY_tests = set([t for t in tests_set - if t.startswith('FLAKY_')]) - FAILS_tests = set([t for t in tests_set - if t.startswith('FAILS_')]) + DISABLED_tests = set(t for t in tests_set if t.startswith('DISABLED_')) + FLAKY_tests = set(t for t in tests_set if t.startswith('FLAKY_')) + FAILS_tests = set(t for t in tests_set if t.startswith('FAILS_')) PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests) failed_tests = set(failed_tests_list) - DISABLED_tests - failed_count_map = dict([(t, 1) for t in failed_tests]) + failed_count_map = dict((t, 1) for t in failed_tests) test_timings = {} i = 0 @@ -64,7 +61,7 @@ test_timings[test] = float(self._num_runs * 100 + i) i += 1 - test_results_map = dict() + test_results_map = {} for test in tests_set: test_results_map[test] = json_results_generator.TestResult( test, failed=(test in failed_tests), @@ -76,7 +73,7 @@ None, # don't fetch past json results archive test_results_map) - failed_count_map = dict([(t, 1) for t in failed_tests]) + failed_count_map = dict((t, 1) for t in failed_tests) # Test incremental json results incremental_json = generator.GetJSON() @@ -114,7 +111,7 @@ if tests_set or DISABLED_count: fixable = {} for fixable_items in buildinfo[JRG.FIXABLE]: - for (result_type, count) in fixable_items.iteritems(): + for (result_type, count) in fixable_items.items(): if result_type in fixable: fixable[result_type] = fixable[result_type] + count else: @@ -138,7 +135,7 @@ if failed_count_map: tests = buildinfo[JRG.TESTS] - for test_name in failed_count_map.iterkeys(): + for test_name in failed_count_map.keys(): test = self._FindTestInTrie(test_name, tests) failed = 0
diff --git a/build/android/pylib/results/flakiness_dashboard/results_uploader.py b/build/android/pylib/results/flakiness_dashboard/results_uploader.py index b68a898..e384335 100644 --- a/build/android/pylib/results/flakiness_dashboard/results_uploader.py +++ b/build/android/pylib/results/flakiness_dashboard/results_uploader.py
@@ -1,9 +1,9 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Uploads the results to the flakiness dashboard server.""" -# pylint: disable=E1002,R0201 +# pylint: disable=R0201 import logging import os @@ -25,18 +25,17 @@ """ def __init__(self, builder_name, build_name, build_number, tmp_folder, test_results_map, test_results_server, test_type, master_name): - super(JSONResultsGenerator, self).__init__( - builder_name=builder_name, - build_name=build_name, - build_number=build_number, - results_file_base_path=tmp_folder, - builder_base_url=None, - test_results_map=test_results_map, - svn_repositories=(('webkit', 'third_party/WebKit'), - ('chrome', '.')), - test_results_server=test_results_server, - test_type=test_type, - master_name=master_name) + super().__init__(builder_name=builder_name, + build_name=build_name, + build_number=build_number, + results_file_base_path=tmp_folder, + builder_base_url=None, + test_results_map=test_results_map, + svn_repositories=(('webkit', 'third_party/WebKit'), + ('chrome', '.')), + test_results_server=test_results_server, + test_type=test_type, + master_name=master_name) #override def _GetModifierChar(self, test_name): @@ -61,7 +60,7 @@ if os.path.exists(os.path.join(in_directory, '.git')): return True parent = os.path.dirname(in_directory) - if parent == host_paths.DIR_SOURCE_ROOT or parent == in_directory: + if parent in (host_paths.DIR_SOURCE_ROOT, in_directory): return False return _is_git_directory(parent) @@ -70,8 +69,7 @@ if not os.path.exists(os.path.join(in_directory, '.svn')): if _is_git_directory(in_directory): return repo_utils.GetGitHeadSHA1(in_directory) - else: - return '' + return '' output = cmd_helper.GetCmdOutput(['svn', 'info', '--xml'], cwd=in_directory) try: @@ -82,7 +80,7 @@ return '' -class ResultsUploader(object): +class ResultsUploader: """Handles uploading buildbot tests results to the flakiness dashboard.""" def __init__(self, tests_type): self._build_number = os.environ.get('BUILDBOT_BUILDNUMBER')
diff --git a/build/android/pylib/results/json_results.py b/build/android/pylib/results/json_results.py index 9b3bcb5..c19096a 100644 --- a/build/android/pylib/results/json_results.py +++ b/build/android/pylib/results/json_results.py
@@ -1,7 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import collections import itertools import json @@ -125,7 +126,7 @@ """ tests = {} - counts = {'PASS': 0, 'FAIL': 0} + counts = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'CRASH': 0, 'TIMEOUT': 0} for test_run_result in test_run_results: if isinstance(test_run_result, list): @@ -142,8 +143,16 @@ element['expected'] = 'PASS' - result = 'PASS' if r.GetType( - ) == base_test_result.ResultType.PASS else 'FAIL' + if r.GetType() == base_test_result.ResultType.PASS: + result = 'PASS' + elif r.GetType() == base_test_result.ResultType.SKIP: + result = 'SKIP' + elif r.GetType() == base_test_result.ResultType.CRASH: + result = 'CRASH' + elif r.GetType() == base_test_result.ResultType.TIMEOUT: + result = 'TIMEOUT' + else: + result = 'FAIL' if 'actual' in element: element['actual'] += ' ' + result @@ -220,10 +229,11 @@ results_list = [] testsuite_runs = json_results['per_iteration_data'] for testsuite_run in testsuite_runs: - for test, test_runs in testsuite_run.iteritems(): + for test, test_runs in six.iteritems(testsuite_run): results_list.extend( [base_test_result.BaseTestResult(test, string_as_status(tr['status']), - duration=tr['elapsed_time_ms']) + duration=tr['elapsed_time_ms'], + log=tr.get('output_snippet')) for tr in test_runs]) return results_list
diff --git a/build/android/pylib/results/json_results_test.py b/build/android/pylib/results/json_results_test.py index 6647331..6cf6487 100755 --- a/build/android/pylib/results/json_results_test.py +++ b/build/android/pylib/results/json_results_test.py
@@ -1,10 +1,12 @@ -#!/usr/bin/env vpython -# Copyright 2014 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + import unittest +import six from pylib.base import base_test_result from pylib.results import json_results @@ -19,18 +21,16 @@ all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) - self.assertEquals( - ['test.package.TestName'], - results_dict['all_tests']) - self.assertEquals(1, len(results_dict['per_iteration_data'])) + self.assertEqual(['test.package.TestName'], results_dict['all_tests']) + self.assertEqual(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) - self.assertEquals(1, len(iteration_result['test.package.TestName'])) + self.assertEqual(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('status' in test_iteration_result) - self.assertEquals('SUCCESS', test_iteration_result['status']) + self.assertEqual('SUCCESS', test_iteration_result['status']) def testGenerateResultsDict_skippedResult(self): result = base_test_result.BaseTestResult( @@ -40,18 +40,16 @@ all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) - self.assertEquals( - ['test.package.TestName'], - results_dict['all_tests']) - self.assertEquals(1, len(results_dict['per_iteration_data'])) + self.assertEqual(['test.package.TestName'], results_dict['all_tests']) + self.assertEqual(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) - self.assertEquals(1, len(iteration_result['test.package.TestName'])) + self.assertEqual(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('status' in test_iteration_result) - self.assertEquals('SKIPPED', test_iteration_result['status']) + self.assertEqual('SKIPPED', test_iteration_result['status']) def testGenerateResultsDict_failedResult(self): result = base_test_result.BaseTestResult( @@ -61,18 +59,16 @@ all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) - self.assertEquals( - ['test.package.TestName'], - results_dict['all_tests']) - self.assertEquals(1, len(results_dict['per_iteration_data'])) + self.assertEqual(['test.package.TestName'], results_dict['all_tests']) + self.assertEqual(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) - self.assertEquals(1, len(iteration_result['test.package.TestName'])) + self.assertEqual(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('status' in test_iteration_result) - self.assertEquals('FAILURE', test_iteration_result['status']) + self.assertEqual('FAILURE', test_iteration_result['status']) def testGenerateResultsDict_duration(self): result = base_test_result.BaseTestResult( @@ -82,18 +78,16 @@ all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) - self.assertEquals( - ['test.package.TestName'], - results_dict['all_tests']) - self.assertEquals(1, len(results_dict['per_iteration_data'])) + self.assertEqual(['test.package.TestName'], results_dict['all_tests']) + self.assertEqual(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) - self.assertEquals(1, len(iteration_result['test.package.TestName'])) + self.assertEqual(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('elapsed_time_ms' in test_iteration_result) - self.assertEquals(123, test_iteration_result['elapsed_time_ms']) + self.assertEqual(123, test_iteration_result['elapsed_time_ms']) def testGenerateResultsDict_multipleResults(self): result1 = base_test_result.BaseTestResult( @@ -106,27 +100,26 @@ all_results.AddResult(result2) results_dict = json_results.GenerateResultsDict([all_results]) - self.assertEquals( - ['test.package.TestName1', 'test.package.TestName2'], - results_dict['all_tests']) + self.assertEqual(['test.package.TestName1', 'test.package.TestName2'], + results_dict['all_tests']) self.assertTrue('per_iteration_data' in results_dict) iterations = results_dict['per_iteration_data'] - self.assertEquals(1, len(iterations)) + self.assertEqual(1, len(iterations)) expected_tests = set([ 'test.package.TestName1', 'test.package.TestName2', ]) - for test_name, iteration_result in iterations[0].iteritems(): + for test_name, iteration_result in six.iteritems(iterations[0]): self.assertTrue(test_name in expected_tests) expected_tests.remove(test_name) - self.assertEquals(1, len(iteration_result)) + self.assertEqual(1, len(iteration_result)) test_iteration_result = iteration_result[0] self.assertTrue('status' in test_iteration_result) - self.assertEquals('SUCCESS', test_iteration_result['status']) + self.assertEqual('SUCCESS', test_iteration_result['status']) def testGenerateResultsDict_passOnRetry(self): raw_results = [] @@ -144,28 +137,28 @@ raw_results.append(run_results2) results_dict = json_results.GenerateResultsDict([raw_results]) - self.assertEquals(['test.package.TestName1'], results_dict['all_tests']) + self.assertEqual(['test.package.TestName1'], results_dict['all_tests']) # Check that there's only one iteration. self.assertIn('per_iteration_data', results_dict) iterations = results_dict['per_iteration_data'] - self.assertEquals(1, len(iterations)) + self.assertEqual(1, len(iterations)) # Check that test.package.TestName1 is the only test in the iteration. - self.assertEquals(1, len(iterations[0])) + self.assertEqual(1, len(iterations[0])) self.assertIn('test.package.TestName1', iterations[0]) # Check that there are two results for test.package.TestName1. actual_test_results = iterations[0]['test.package.TestName1'] - self.assertEquals(2, len(actual_test_results)) + self.assertEqual(2, len(actual_test_results)) # Check that the first result is a failure. self.assertIn('status', actual_test_results[0]) - self.assertEquals('FAILURE', actual_test_results[0]['status']) + self.assertEqual('FAILURE', actual_test_results[0]['status']) # Check that the second result is a success. self.assertIn('status', actual_test_results[1]) - self.assertEquals('SUCCESS', actual_test_results[1]['status']) + self.assertEqual('SUCCESS', actual_test_results[1]['status']) def testGenerateResultsDict_globalTags(self): raw_results = [] @@ -173,7 +166,7 @@ results_dict = json_results.GenerateResultsDict( [raw_results], global_tags=global_tags) - self.assertEquals(['UNRELIABLE_RESULTS'], results_dict['global_tags']) + self.assertEqual(['UNRELIABLE_RESULTS'], results_dict['global_tags']) def testGenerateResultsDict_loslessSnippet(self): result = base_test_result.BaseTestResult( @@ -185,22 +178,20 @@ all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) - self.assertEquals( - ['test.package.TestName'], - results_dict['all_tests']) - self.assertEquals(1, len(results_dict['per_iteration_data'])) + self.assertEqual(['test.package.TestName'], results_dict['all_tests']) + self.assertEqual(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) - self.assertEquals(1, len(iteration_result['test.package.TestName'])) + self.assertEqual(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('losless_snippet' in test_iteration_result) self.assertTrue(test_iteration_result['losless_snippet']) self.assertTrue('output_snippet' in test_iteration_result) - self.assertEquals(log, test_iteration_result['output_snippet']) + self.assertEqual(log, test_iteration_result['output_snippet']) self.assertTrue('output_snippet_base64' in test_iteration_result) - self.assertEquals('', test_iteration_result['output_snippet_base64']) + self.assertEqual('', test_iteration_result['output_snippet_base64']) def testGenerateJsonTestResultFormatDict_passedResult(self): result = base_test_result.BaseTestResult('test.package.TestName', @@ -211,18 +202,19 @@ results_dict = json_results.GenerateJsonTestResultFormatDict([all_results], False) - self.assertEquals(1, len(results_dict['tests'])) - self.assertEquals(1, len(results_dict['tests']['test'])) - self.assertEquals(1, len(results_dict['tests']['test']['package'])) - self.assertEquals( + self.assertEqual(1, len(results_dict['tests'])) + self.assertEqual(1, len(results_dict['tests']['test'])) + self.assertEqual(1, len(results_dict['tests']['test']['package'])) + self.assertEqual( 'PASS', results_dict['tests']['test']['package']['TestName']['expected']) - self.assertEquals( + self.assertEqual( 'PASS', results_dict['tests']['test']['package']['TestName']['actual']) - # Note: technically a missing entry counts as zero. - self.assertEquals(1, results_dict['num_failures_by_type']['PASS']) - self.assertEquals(0, results_dict['num_failures_by_type']['FAIL']) + self.assertTrue('FAIL' not in results_dict['num_failures_by_type'] + or results_dict['num_failures_by_type']['FAIL'] == 0) + self.assertIn('PASS', results_dict['num_failures_by_type']) + self.assertEqual(1, results_dict['num_failures_by_type']['PASS']) def testGenerateJsonTestResultFormatDict_failedResult(self): result = base_test_result.BaseTestResult('test.package.TestName', @@ -233,22 +225,50 @@ results_dict = json_results.GenerateJsonTestResultFormatDict([all_results], False) - self.assertEquals(1, len(results_dict['tests'])) - self.assertEquals(1, len(results_dict['tests']['test'])) - self.assertEquals(1, len(results_dict['tests']['test']['package'])) - self.assertEquals( + self.assertEqual(1, len(results_dict['tests'])) + self.assertEqual(1, len(results_dict['tests']['test'])) + self.assertEqual(1, len(results_dict['tests']['test']['package'])) + self.assertEqual( 'PASS', results_dict['tests']['test']['package']['TestName']['expected']) - self.assertEquals( + self.assertEqual( 'FAIL', results_dict['tests']['test']['package']['TestName']['actual']) - self.assertEquals( + self.assertEqual( True, results_dict['tests']['test']['package']['TestName']['is_unexpected']) - self.assertEquals(2, len(results_dict['num_failures_by_type'])) - # Note: technically a missing entry counts as zero. - self.assertEquals(0, results_dict['num_failures_by_type']['PASS']) - self.assertEquals(1, results_dict['num_failures_by_type']['FAIL']) + self.assertTrue('PASS' not in results_dict['num_failures_by_type'] + or results_dict['num_failures_by_type']['PASS'] == 0) + self.assertIn('FAIL', results_dict['num_failures_by_type']) + self.assertEqual(1, results_dict['num_failures_by_type']['FAIL']) + + def testGenerateJsonTestResultFormatDict_skippedResult(self): + result = base_test_result.BaseTestResult('test.package.TestName', + base_test_result.ResultType.SKIP) + + all_results = base_test_result.TestRunResults() + all_results.AddResult(result) + + results_dict = json_results.GenerateJsonTestResultFormatDict([all_results], + False) + self.assertEqual(1, len(results_dict['tests'])) + self.assertEqual(1, len(results_dict['tests']['test'])) + self.assertEqual(1, len(results_dict['tests']['test']['package'])) + self.assertEqual( + 'PASS', + results_dict['tests']['test']['package']['TestName']['expected']) + self.assertEqual( + 'SKIP', results_dict['tests']['test']['package']['TestName']['actual']) + # Should only be set if the test fails. + self.assertNotIn('is_unexpected', + results_dict['tests']['test']['package']['TestName']) + + self.assertTrue('FAIL' not in results_dict['num_failures_by_type'] + or results_dict['num_failures_by_type']['FAIL'] == 0) + self.assertTrue('PASS' not in results_dict['num_failures_by_type'] + or results_dict['num_failures_by_type']['PASS'] == 0) + self.assertIn('SKIP', results_dict['num_failures_by_type']) + self.assertEqual(1, results_dict['num_failures_by_type']['SKIP']) def testGenerateJsonTestResultFormatDict_failedResultWithRetry(self): result_1 = base_test_result.BaseTestResult('test.package.TestName', @@ -266,26 +286,25 @@ results_dict = json_results.GenerateJsonTestResultFormatDict( all_results, False) - self.assertEquals(1, len(results_dict['tests'])) - self.assertEquals(1, len(results_dict['tests']['test'])) - self.assertEquals(1, len(results_dict['tests']['test']['package'])) - self.assertEquals( + self.assertEqual(1, len(results_dict['tests'])) + self.assertEqual(1, len(results_dict['tests']['test'])) + self.assertEqual(1, len(results_dict['tests']['test']['package'])) + self.assertEqual( 'PASS', results_dict['tests']['test']['package']['TestName']['expected']) - self.assertEquals( + self.assertEqual( 'FAIL FAIL', results_dict['tests']['test']['package']['TestName']['actual']) - self.assertEquals( + self.assertEqual( True, results_dict['tests']['test']['package']['TestName']['is_unexpected']) - # Note: technically a missing entry counts as zero. - self.assertEquals(2, len(results_dict['num_failures_by_type'])) - self.assertEquals(0, results_dict['num_failures_by_type']['PASS']) - + self.assertTrue('PASS' not in results_dict['num_failures_by_type'] + or results_dict['num_failures_by_type']['PASS'] == 0) # According to the spec: If a test was run more than once, only the first # invocation's result is included in the totals. - self.assertEquals(1, results_dict['num_failures_by_type']['FAIL']) + self.assertIn('FAIL', results_dict['num_failures_by_type']) + self.assertEqual(1, results_dict['num_failures_by_type']['FAIL']) if __name__ == '__main__':
diff --git a/build/android/pylib/results/presentation/__init__.py b/build/android/pylib/results/presentation/__init__.py index a22a6ee..b8e1dbd 100644 --- a/build/android/pylib/results/presentation/__init__.py +++ b/build/android/pylib/results/presentation/__init__.py
@@ -1,3 +1,3 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/results/presentation/javascript/main_html.js b/build/android/pylib/results/presentation/javascript/main_html.js index 3d94663..e4bf2cc 100644 --- a/build/android/pylib/results/presentation/javascript/main_html.js +++ b/build/android/pylib/results/presentation/javascript/main_html.js
@@ -1,4 +1,4 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. +// Copyright 2017 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/pylib/results/presentation/standard_gtest_merge.py b/build/android/pylib/results/presentation/standard_gtest_merge.py index 58a2936..ab1074e 100755 --- a/build/android/pylib/results/presentation/standard_gtest_merge.py +++ b/build/android/pylib/results/presentation/standard_gtest_merge.py
@@ -1,10 +1,9 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import argparse import json @@ -22,6 +21,9 @@ with open(summary_json) as f: summary = json.load(f) except (IOError, ValueError): + # TODO(crbug.com/1245494):Re-enable this check after the recipe module + # chromium_swarming can run it with py3 + # pylint: disable=raise-missing-from raise Exception('Summary json cannot be loaded.') # Merge all JSON files together. Keep track of missing shards. @@ -43,17 +45,17 @@ # client/swarming.py, which means the state enum is saved in its string # name form, not in the number form. state = result.get('state') - if state == u'BOT_DIED': + if state == 'BOT_DIED': print( 'Shard #%d had a Swarming internal failure' % index, file=sys.stderr) - elif state == u'EXPIRED': + elif state == 'EXPIRED': print('There wasn\'t enough capacity to run your test', file=sys.stderr) - elif state == u'TIMED_OUT': + elif state == 'TIMED_OUT': print('Test runtime exceeded allocated time' 'Either it ran for too long (hard timeout) or it didn\'t produce ' 'I/O for an extended period of time (I/O timeout)', file=sys.stderr) - elif state != u'COMPLETED': + elif state != 'COMPLETED': print('Invalid Swarming task state: %s' % state, file=sys.stderr) json_data, err_msg = load_shard_json(index, result.get('task_id'), @@ -111,7 +113,7 @@ if not matching_json_files: print('shard %s test output missing' % index, file=sys.stderr) return (None, 'shard %s test output was missing' % index) - elif len(matching_json_files) > 1: + if len(matching_json_files) > 1: print('duplicate test output for shard %s' % index, file=sys.stderr) return (None, 'shard %s test output was duplicated' % index) @@ -138,7 +140,7 @@ def merge_list_of_dicts(left, right): """Merges dicts left[0] with right[0], left[1] with right[1], etc.""" output = [] - for i in xrange(max(len(left), len(right))): + for i in range(max(len(left), len(right))): left_dict = left[i] if i < len(left) else {} right_dict = right[i] if i < len(right) else {} merged_dict = left_dict.copy() @@ -151,7 +153,7 @@ output_json, summary_json, jsons_to_merge): output = merge_shard_results(summary_json, jsons_to_merge) - with open(output_json, 'wb') as f: + with open(output_json, 'w') as f: json.dump(output, f) return 0
diff --git a/build/android/pylib/results/presentation/test_results_presentation.py b/build/android/pylib/results/presentation/test_results_presentation.py index 33fae04..9e8b280 100755 --- a/build/android/pylib/results/presentation/test_results_presentation.py +++ b/build/android/pylib/results/presentation/test_results_presentation.py
@@ -1,10 +1,10 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function + import argparse import collections @@ -14,7 +14,12 @@ import tempfile import os import sys -import urllib +try: + from urllib.parse import urlencode + from urllib.request import urlopen +except ImportError: + from urllib import urlencode + from urllib2 import urlopen CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -49,7 +54,7 @@ } -class LinkTarget(object): +class LinkTarget: # Opens the linked document in a new window or tab. NEW_TAB = '_blank' # Opens the linked document in the same frame as it was clicked. @@ -103,20 +108,22 @@ } -def flakiness_dashbord_link(test_name, suite_name): - url_args = urllib.urlencode([ - ('testType', suite_name), - ('tests', test_name)]) - return ('https://test-results.appspot.com/' - 'dashboards/flakiness_dashboard.html#%s' % url_args) +def flakiness_dashbord_link(test_name, suite_name, bucket): + # Assume the bucket will be like "foo-bar-baz", we will take "foo" + # as the test_project. + # Fallback to "chromium" if bucket is not passed, e.g. local_output=True + test_project = bucket.split('-')[0] if bucket else 'chromium' + query = '%s/%s' % (suite_name, test_name) + url_args = urlencode([('t', 'TESTS'), ('q', query), ('tp', test_project)]) + return 'https://ci.chromium.org/ui/search?%s' % url_args -def logs_cell(result, test_name, suite_name): +def logs_cell(result, test_name, suite_name, bucket): """Formats result logs data for processing in jinja template.""" link_list = [] result_link_dict = result.get('links', {}) result_link_dict['flakiness'] = flakiness_dashbord_link( - test_name, suite_name) + test_name, suite_name, bucket) for name, href in sorted(result_link_dict.items()): link_list.append(link( data=name, @@ -124,8 +131,7 @@ target=LinkTarget.NEW_TAB)) if link_list: return links_cell(link_list) - else: - return cell('(no logs)') + return cell('(no logs)') def code_search(test, cs_base_url): @@ -144,7 +150,7 @@ return status -def create_test_table(results_dict, cs_base_url, suite_name): +def create_test_table(results_dict, cs_base_url, suite_name, bucket): """Format test data for injecting into HTML table.""" header_row = [ @@ -156,7 +162,7 @@ ] test_row_blocks = [] - for test_name, test_results in results_dict.iteritems(): + for test_name, test_results in results_dict.items(): test_runs = [] for index, result in enumerate(test_results): if index == 0: @@ -177,7 +183,8 @@ html_class=('center %s' % status_class(result['status']))), cell(data=result['elapsed_time_ms']), # elapsed_time_ms - logs_cell(result, test_name, suite_name), # logs + logs_cell(result, test_name, suite_name, bucket), + # logs pre_cell(data=result['output_snippet'], # output_snippet html_class='left'), ]) @@ -214,31 +221,25 @@ cell(data=0), # elapsed_time_ms ] - suite_row_dict = {} - for test_name, test_results in results_dict.iteritems(): + suite_row_dict = collections.defaultdict(lambda: [ + # Note: |suite_name| will be given in the following for loop. + # It is not assigned yet here. + action_cell('showTestsOfOneSuiteOnlyWithNewState("%s")' % suite_name, + suite_name, 'left'), # suite_name + cell(data=0), # number_success_tests + cell(data=0), # number_fail_tests + cell(data=0), # all_tests + cell(data=0), # elapsed_time_ms + ]) + for test_name, test_results in results_dict.items(): # TODO(mikecase): This logic doesn't work if there are multiple test runs. # That is, if 'per_iteration_data' has multiple entries. # Since we only care about the result of the last test run. result = test_results[-1] - suite_name = (test_name.split('#')[0] if '#' in test_name - else test_name.split('.')[0]) - if suite_name in suite_row_dict: - suite_row = suite_row_dict[suite_name] - else: - suite_row = [ - action_cell( - 'showTestsOfOneSuiteOnlyWithNewState("%s")' % suite_name, - suite_name, - 'left' - ), # suite_name - cell(data=0), # number_success_tests - cell(data=0), # number_fail_tests - cell(data=0), # all_tests - cell(data=0), # elapsed_time_ms - ] - - suite_row_dict[suite_name] = suite_row + suite_name = (test_name.split('#')[0] + if '#' in test_name else test_name.split('.')[0]) + suite_row = suite_row_dict[suite_name] suite_row[ALL_COUNT_INDEX]['data'] += 1 footer_row[ALL_COUNT_INDEX]['data'] += 1 @@ -255,7 +256,7 @@ suite_row[TIME_INDEX]['data'] += result['elapsed_time_ms'] footer_row[TIME_INDEX]['data'] += result['elapsed_time_ms'] - for suite in suite_row_dict.values(): + for suite in list(suite_row_dict.values()): if suite[FAIL_COUNT_INDEX]['data'] > 0: suite[FAIL_COUNT_INDEX]['class'] += ' failure' else: @@ -266,13 +267,12 @@ else: footer_row[FAIL_COUNT_INDEX]['class'] += ' success' - return (header_row, - [[suite_row] for suite_row in suite_row_dict.values()], + return (header_row, [[suite_row] + for suite_row in list(suite_row_dict.values())], footer_row) def feedback_url(result_details_link): - # pylint: disable=redefined-variable-type url_args = [ ('labels', 'Pri-2,Type-Bug,Restrict-View-Google'), ('summary', 'Result Details Feedback:'), @@ -280,8 +280,7 @@ ] if result_details_link: url_args.append(('comment', 'Please check out: %s' % result_details_link)) - url_args = urllib.urlencode(url_args) - # pylint: enable=redefined-variable-type + url_args = urlencode(url_args) return 'https://bugs.chromium.org/p/chromium/issues/entry?%s' % url_args @@ -294,7 +293,7 @@ just a local file. """ test_rows_header, test_rows = create_test_table( - results_dict, cs_base_url, test_name) + results_dict, cs_base_url, test_name, bucket) suite_rows_header, suite_rows, suite_row_footer = create_suite_table( results_dict) @@ -321,17 +320,16 @@ 'feedback_url': feedback_url(None), }) return (html_render, None, None) - else: - dest = google_storage_helper.unique_name( - '%s_%s_%s' % (test_name, builder_name, build_number)) - result_details_link = google_storage_helper.get_url_link( - dest, '%s/html' % bucket) - html_render = main_template.render( # pylint: disable=no-member - { - 'tb_values': [suite_table_values, test_table_values], - 'feedback_url': feedback_url(result_details_link), - }) - return (html_render, dest, result_details_link) + dest = google_storage_helper.unique_name( + '%s_%s_%s' % (test_name, builder_name, build_number)) + result_details_link = google_storage_helper.get_url_link( + dest, '%s/html' % bucket) + html_render = main_template.render( # pylint: disable=no-member + { + 'tb_values': [suite_table_values, test_table_values], + 'feedback_url': feedback_url(result_details_link), + }) + return (html_render, dest, result_details_link) def result_details(json_path, test_name, cs_base_url, bucket=None, @@ -351,7 +349,7 @@ results_dict = collections.defaultdict(list) for testsuite_run in json_object['per_iteration_data']: - for test, test_runs in testsuite_run.iteritems(): + for test, test_runs in testsuite_run.items(): results_dict[test].extend(test_runs) return results_to_html(results_dict, cs_base_url, bucket, test_name, builder_name, build_number, local_output) @@ -378,12 +376,12 @@ ui_screenshots = [] # pylint: disable=too-many-nested-blocks for testsuite_run in json_object['per_iteration_data']: - for _, test_runs in testsuite_run.iteritems(): + for _, test_runs in testsuite_run.items(): for test_run in test_runs: if 'ui screenshot' in test_run['links']: screenshot_link = test_run['links']['ui screenshot'] if screenshot_link.startswith('file:'): - with contextlib.closing(urllib.urlopen(screenshot_link)) as f: + with contextlib.closing(urlopen(screenshot_link)) as f: test_screenshots = json.load(f) else: # Assume anything that isn't a file link is a google storage link @@ -410,7 +408,7 @@ dest = google_storage_helper.unique_name( 'screenshots_%s_%s_%s' % (test_name, builder_name, build_number), suffix='.json') - with tempfile.NamedTemporaryFile(suffix='.json') as temp_file: + with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as temp_file: temp_file.write(screenshot_set) temp_file.flush() return google_storage_helper.upload( @@ -470,7 +468,7 @@ with open(args.output_json, 'w') as f: json.dump({}, f) return - elif len(args.positional) != 0 and args.json_file: + if len(args.positional) != 0 and args.json_file: raise parser.error('Exactly one of args.positional and ' 'args.json_file should be given.') @@ -520,8 +518,7 @@ if ui_screenshot_set_link: ui_catalog_url = 'https://chrome-ui-catalog.appspot.com/' - ui_catalog_query = urllib.urlencode( - {'screenshot_source': ui_screenshot_set_link}) + ui_catalog_query = urlencode({'screenshot_source': ui_screenshot_set_link}) ui_screenshot_link = '%s?%s' % (ui_catalog_url, ui_catalog_query) if args.output_json:
diff --git a/build/android/pylib/results/presentation/test_results_presentation.pydeps b/build/android/pylib/results/presentation/test_results_presentation.pydeps new file mode 100644 index 0000000..031e179 --- /dev/null +++ b/build/android/pylib/results/presentation/test_results_presentation.pydeps
@@ -0,0 +1,46 @@ +# Generated by running: +# build/print_python_deps.py --root build/android/pylib/results/presentation --output build/android/pylib/results/presentation/test_results_presentation.pydeps build/android/pylib/results/presentation/test_results_presentation.py +../../../../../third_party/catapult/devil/devil/__init__.py +../../../../../third_party/catapult/devil/devil/android/__init__.py +../../../../../third_party/catapult/devil/devil/android/constants/__init__.py +../../../../../third_party/catapult/devil/devil/android/constants/chrome.py +../../../../../third_party/catapult/devil/devil/android/sdk/__init__.py +../../../../../third_party/catapult/devil/devil/android/sdk/keyevent.py +../../../../../third_party/catapult/devil/devil/android/sdk/version_codes.py +../../../../../third_party/catapult/devil/devil/base_error.py +../../../../../third_party/catapult/devil/devil/constants/__init__.py +../../../../../third_party/catapult/devil/devil/constants/exit_codes.py +../../../../../third_party/catapult/devil/devil/utils/__init__.py +../../../../../third_party/catapult/devil/devil/utils/cmd_helper.py +../../../../../third_party/jinja2/__init__.py +../../../../../third_party/jinja2/_identifier.py +../../../../../third_party/jinja2/async_utils.py +../../../../../third_party/jinja2/bccache.py +../../../../../third_party/jinja2/compiler.py +../../../../../third_party/jinja2/defaults.py +../../../../../third_party/jinja2/environment.py +../../../../../third_party/jinja2/exceptions.py +../../../../../third_party/jinja2/filters.py +../../../../../third_party/jinja2/idtracking.py +../../../../../third_party/jinja2/lexer.py +../../../../../third_party/jinja2/loaders.py +../../../../../third_party/jinja2/nodes.py +../../../../../third_party/jinja2/optimizer.py +../../../../../third_party/jinja2/parser.py +../../../../../third_party/jinja2/runtime.py +../../../../../third_party/jinja2/tests.py +../../../../../third_party/jinja2/utils.py +../../../../../third_party/jinja2/visitor.py +../../../../../third_party/markupsafe/__init__.py +../../../../../third_party/markupsafe/_compat.py +../../../../../third_party/markupsafe/_native.py +../../__init__.py +../../constants/__init__.py +../../constants/host_paths.py +../../utils/__init__.py +../../utils/decorators.py +../../utils/google_storage_helper.py +../__init__.py +__init__.py +standard_gtest_merge.py +test_results_presentation.py
diff --git a/build/android/pylib/results/report_results.py b/build/android/pylib/results/report_results.py index 56eefac..de19860 100644 --- a/build/android/pylib/results/report_results.py +++ b/build/android/pylib/results/report_results.py
@@ -1,10 +1,9 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Module containing utility functions for reporting results.""" -from __future__ import print_function import logging import os @@ -108,17 +107,17 @@ logging.critical('*' * 80) logging.critical('Summary') logging.critical('*' * 80) - for line in results.GetGtestForm().splitlines(): - color = black_on_white - if 'FAILED' in line: - # Red on white, dim. - color = (logging_utils.BACK.WHITE, logging_utils.FORE.RED, - logging_utils.STYLE.DIM) - elif 'PASSED' in line: - # Green on white, dim. - color = (logging_utils.BACK.WHITE, logging_utils.FORE.GREEN, - logging_utils.STYLE.DIM) - with logging_utils.OverrideColor(logging.CRITICAL, color): + # Assign uniform color, depending on presence of 'FAILED' over lines. + if any('FAILED' in line for line in results.GetGtestForm().splitlines()): + # Red on white, dim. + color = (logging_utils.BACK.WHITE, logging_utils.FORE.RED, + logging_utils.STYLE.DIM) + else: + # Green on white, dim. + color = (logging_utils.BACK.WHITE, logging_utils.FORE.GREEN, + logging_utils.STYLE.DIM) + with logging_utils.OverrideColor(logging.CRITICAL, color): + for line in results.GetGtestForm().splitlines(): logging.critical(line) logging.critical('*' * 80)
diff --git a/build/android/pylib/symbols/apk_lib_dump.py b/build/android/pylib/symbols/apk_lib_dump.py deleted file mode 100755 index ba87026..0000000 --- a/build/android/pylib/symbols/apk_lib_dump.py +++ /dev/null
@@ -1,61 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Dump shared library information from an APK file. - -This script is used to dump which *uncompressed* native shared libraries an -APK contains, as well as their position within the file. This is mostly useful -to diagnose logcat and tombstone symbolization issues when the libraries are -loaded directly from the APK at runtime. - -The default format will print one line per uncompressed shared library with the -following format: - - 0x<start-offset> 0x<end-offset> 0x<file-size> <file-path> - -The --format=python option can be used to dump the same information that is -easy to use in a Python script, e.g. with a line like: - - (0x<start-offset>, 0x<end-offset>, 0x<file-size>, <file-path>), -""" - -from __future__ import print_function - -import argparse -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) - -from pylib.symbols import apk_native_libs - -def main(): - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('apk', help='Input APK file path.') - - parser.add_argument('--format', help='Select output format', - default='default', choices=['default', 'python']) - - args = parser.parse_args() - - apk_reader = apk_native_libs.ApkReader(args.apk) - lib_map = apk_native_libs.ApkNativeLibraries(apk_reader) - for lib_path, file_offset, file_size in lib_map.GetDumpList(): - if args.format == 'python': - print('(0x%08x, 0x%08x, 0x%08x, \'%s\'),' % - (file_offset, file_offset + file_size, file_size, lib_path)) - else: - print('0x%08x 0x%08x 0x%08x %s' % (file_offset, file_offset + file_size, - file_size, lib_path)) - - return 0 - - -if __name__ == '__main__': - sys.exit(main())
diff --git a/build/android/pylib/symbols/apk_native_libs.py b/build/android/pylib/symbols/apk_native_libs.py deleted file mode 100644 index c4af202..0000000 --- a/build/android/pylib/symbols/apk_native_libs.py +++ /dev/null
@@ -1,419 +0,0 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import logging -import os -import re -import struct -import zipfile - -# The default zipfile python module cannot open APKs properly, but this -# fixes it. Note that simply importing this file is sufficient to -# ensure that zip works correctly for all other modules. See: -# http://bugs.python.org/issue14315 -# https://hg.python.org/cpython/rev/6dd5e9556a60#l2.8 -def _PatchZipFile(): - # pylint: disable=protected-access - oldDecodeExtra = zipfile.ZipInfo._decodeExtra - def decodeExtra(self): - try: - oldDecodeExtra(self) - except struct.error: - pass - zipfile.ZipInfo._decodeExtra = decodeExtra -_PatchZipFile() - - -class ApkZipInfo(object): - """Models a single file entry from an ApkReader. - - This is very similar to the zipfile.ZipInfo class. It provides a few - properties describing the entry: - - filename (same as ZipInfo.filename) - - file_size (same as ZipInfo.file_size) - - compress_size (same as ZipInfo.file_size) - - file_offset (note: not provided by ZipInfo) - - And a few useful methods: IsCompressed() and IsElfFile(). - - Entries can be created by using ApkReader() methods. - """ - def __init__(self, zip_file, zip_info): - """Construct instance. Do not call this directly. Use ApkReader methods.""" - self._file = zip_file - self._info = zip_info - self._file_offset = None - - @property - def filename(self): - """Entry's file path within APK.""" - return self._info.filename - - @property - def file_size(self): - """Entry's extracted file size in bytes.""" - return self._info.file_size - - @property - def compress_size(self): - """Entry' s compressed file size in bytes.""" - return self._info.compress_size - - @property - def file_offset(self): - """Entry's starting file offset in the APK.""" - if self._file_offset is None: - self._file_offset = self._ZipFileOffsetFromLocalHeader( - self._file.fp, self._info.header_offset) - return self._file_offset - - def __repr__(self): - """Convert to string for debugging.""" - return 'ApkZipInfo["%s",size=0x%x,compressed=0x%x,offset=0x%x]' % ( - self.filename, self.file_size, self.compress_size, self.file_offset) - - def IsCompressed(self): - """Returns True iff the entry is compressed.""" - return self._info.compress_type != zipfile.ZIP_STORED - - def IsElfFile(self): - """Returns True iff the entry is an ELF file.""" - with self._file.open(self._info, 'r') as f: - return f.read(4) == '\x7fELF' - - @staticmethod - def _ZipFileOffsetFromLocalHeader(fd, local_header_offset): - """Return a file's start offset from its zip archive local header. - - Args: - fd: Input file object. - local_header_offset: Local header offset (from its ZipInfo entry). - Returns: - file start offset. - """ - FILE_NAME_LEN_OFFSET = 26 - FILE_NAME_OFFSET = 30 - fd.seek(local_header_offset + FILE_NAME_LEN_OFFSET) - file_name_len = struct.unpack('H', fd.read(2))[0] - extra_field_len = struct.unpack('H', fd.read(2))[0] - file_offset = (local_header_offset + FILE_NAME_OFFSET + - file_name_len + extra_field_len) - return file_offset - - -class ApkReader(object): - """A convenience class used to read the content of APK files. - - Its design is very similar to the one from zipfile.ZipFile, except - that its returns ApkZipInfo entries which provide a |file_offset| - property that can be used to know where a given file is located inside - the archive. - - It is also easy to mock for unit-testing (see MockApkReader in - apk_utils_unittest.py) without creating any files on disk. - - Usage is the following: - - Create an instance using a with statement (for proper unit-testing). - - Call ListEntries() to list all entries in the archive. This returns - a list of ApkZipInfo entries. - - Or call FindEntry() corresponding to a given path within the archive. - - For example: - with ApkReader(input_apk_path) as reader: - info = reader.FindEntry('lib/armeabi-v7a/libfoo.so') - if info.IsCompressed() or not info.IsElfFile(): - raise Exception('Invalid library path") - - The ApkZipInfo can be used to inspect the entry's metadata, or read its - content with the ReadAll() method. See its documentation for all details. - """ - def __init__(self, apk_path): - """Initialize instance.""" - self._zip_file = zipfile.ZipFile(apk_path, 'r') - self._path = apk_path - - def __enter__(self): - """Python context manager entry.""" - return self - - def __exit__(self, *kwargs): - """Python context manager exit.""" - self.Close() - - @property - def path(self): - """The corresponding input APK path.""" - return self._path - - def Close(self): - """Close the reader (and underlying ZipFile instance).""" - self._zip_file.close() - - def ListEntries(self): - """Return a list of ApkZipInfo entries for this APK.""" - result = [] - for info in self._zip_file.infolist(): - result.append(ApkZipInfo(self._zip_file, info)) - return result - - def FindEntry(self, file_path): - """Return an ApkZipInfo instance for a given archive file path. - - Args: - file_path: zip file path. - Return: - A new ApkZipInfo entry on success. - Raises: - KeyError on failure (entry not found). - """ - info = self._zip_file.getinfo(file_path) - return ApkZipInfo(self._zip_file, info) - - - -class ApkNativeLibraries(object): - """A class for the list of uncompressed shared libraries inside an APK. - - Create a new instance by passing the path to an input APK, then use - the FindLibraryByOffset() method to find the native shared library path - corresponding to a given file offset. - - GetAbiList() and GetLibrariesList() can also be used to inspect - the state of the instance. - """ - def __init__(self, apk_reader): - """Initialize instance. - - Args: - apk_reader: An ApkReader instance corresponding to the input APK. - """ - self._native_libs = [] - for entry in apk_reader.ListEntries(): - # Chromium uses so-called 'placeholder' native shared libraries - # that have a size of 0, and are only used to deal with bugs in - # older Android system releases (they are never loaded and cannot - # appear in stack traces). Ignore these here to avoid generating - # confusing results. - if entry.file_size == 0: - continue - - # Only uncompressed libraries can appear in stack traces. - if entry.IsCompressed(): - continue - - # Only consider files within lib/ and with a filename ending with .so - # at the moment. NOTE: Do not require a 'lib' prefix, since that would - # prevent finding the 'crazy.libXXX.so' libraries used by Chromium. - if (not entry.filename.startswith('lib/') or - not entry.filename.endswith('.so')): - continue - - lib_path = entry.filename - - self._native_libs.append( - (lib_path, entry.file_offset, entry.file_offset + entry.file_size)) - - def IsEmpty(self): - """Return true iff the list is empty.""" - return not bool(self._native_libs) - - def GetLibraries(self): - """Return the list of all library paths in this instance.""" - return sorted([x[0] for x in self._native_libs]) - - def GetDumpList(self): - """Retrieve full library map. - - Returns: - A list of (lib_path, file_offset, file_size) tuples, sorted - in increasing |file_offset| values. - """ - result = [] - for entry in self._native_libs: - lib_path, file_start, file_end = entry - result.append((lib_path, file_start, file_end - file_start)) - - return sorted(result, lambda x, y: cmp(x[1], y[1])) - - def FindLibraryByOffset(self, file_offset): - """Find the native library at a given file offset. - - Args: - file_offset: File offset within the original APK. - Returns: - Returns a (lib_path, lib_offset) tuple on success, or (None, 0) - on failure. Note that lib_path will omit the 'lib/$ABI/' prefix, - lib_offset is the adjustment of file_offset within the library. - """ - for lib_path, start_offset, end_offset in self._native_libs: - if file_offset >= start_offset and file_offset < end_offset: - return (lib_path, file_offset - start_offset) - - return (None, 0) - - -class ApkLibraryPathTranslator(object): - """Translates APK file paths + byte offsets into library path + offset. - - The purpose of this class is to translate a native shared library path - that points to an APK into a new device-specific path that points to a - native shared library, as if it was installed there. E.g.: - - ('/data/data/com.example.app-1/base.apk', 0x123be00) - - would be translated into: - - ('/data/data/com.example.app-1/base.apk!lib/libfoo.so', 0x3be00) - - If the original APK (installed as base.apk) contains an uncompressed shared - library under lib/armeabi-v7a/libfoo.so at offset 0x120000. - - Note that the virtual device path after the ! doesn't necessarily match - the path inside the .apk. This doesn't really matter for the rest of - the symbolization functions since only the file's base name can be used - to find the corresponding file on the host. - - Usage is the following: - - 1/ Create new instance. - - 2/ Call AddHostApk() one or several times to add the host path - of an APK, its package name, and device-installed named. - - 3/ Call TranslatePath() to translate a (path, offset) tuple corresponding - to an on-device APK, into the corresponding virtual device library - path and offset. - """ - - # Depending on the version of the system, a non-system APK might be installed - # on a path that looks like the following: - # - # * /data/..../<package_name>-<number>.apk, where <number> is used to - # distinguish several versions of the APK during package updates. - # - # * /data/..../<package_name>-<suffix>/base.apk, where <suffix> is a - # string of random ASCII characters following the dash after the - # package name. This serves as a way to distinguish the installation - # paths during package update, and randomize its final location - # (to prevent apps from hard-coding the paths to other apps). - # - # Note that the 'base.apk' name comes from the system. - # - # * /data/.../<package_name>-<suffix>/<split_name>.apk, where <suffix> - # is the same as above, and <split_name> is the name of am app bundle - # split APK. - # - # System APKs are installed on paths that look like /system/app/Foo.apk - # but this class ignores them intentionally. - - # Compiler regular expression for the first format above. - _RE_APK_PATH_1 = re.compile( - r'/data/.*/(?P<package_name>[A-Za-z0-9_.]+)-(?P<version>[0-9]+)\.apk') - - # Compiled regular expression for the second and third formats above. - _RE_APK_PATH_2 = re.compile( - r'/data/.*/(?P<package_name>[A-Za-z0-9_.]+)-(?P<suffix>[^/]+)/' + - r'(?P<apk_name>.+\.apk)') - - def __init__(self): - """Initialize instance. Call AddHostApk() to add host apk file paths.""" - self._path_map = {} # Maps (package_name, apk_name) to host-side APK path. - self._libs_map = {} # Maps APK host path to ApkNativeLibrariesMap instance. - - def AddHostApk(self, package_name, native_libs, device_apk_name=None): - """Add a file path to the host APK search list. - - Args: - package_name: Corresponding apk package name. - native_libs: ApkNativeLibraries instance for the corresponding APK. - device_apk_name: Optional expected name of the installed APK on the - device. This is only useful when symbolizing app bundle that run on - Android L+. I.e. it will be ignored in other cases. - """ - if native_libs.IsEmpty(): - logging.debug('Ignoring host APK without any uncompressed native ' + - 'libraries: %s', device_apk_name) - return - - # If the APK name is not provided, use the default of 'base.apk'. This - # will be ignored if we find <package_name>-<number>.apk file paths - # in the input, but will work properly for Android L+, as long as we're - # not using Android app bundles. - device_apk_name = device_apk_name or 'base.apk' - - key = "%s/%s" % (package_name, device_apk_name) - if key in self._libs_map: - raise KeyError('There is already an APK associated with (%s)' % key) - - self._libs_map[key] = native_libs - - @staticmethod - def _MatchApkDeviceInstallPath(apk_path): - """Check whether a given path matches an installed APK device file path. - - Args: - apk_path: Device-specific file path. - Returns: - On success, a (package_name, apk_name) tuple. On failure, (None. None). - """ - m = ApkLibraryPathTranslator._RE_APK_PATH_1.match(apk_path) - if m: - return (m.group('package_name'), 'base.apk') - - m = ApkLibraryPathTranslator._RE_APK_PATH_2.match(apk_path) - if m: - return (m.group('package_name'), m.group('apk_name')) - - return (None, None) - - def TranslatePath(self, apk_path, apk_offset): - """Translate a potential apk file path + offset into library path + offset. - - Args: - apk_path: Library or apk file path on the device (e.g. - '/data/data/com.example.app-XSAHKSJH/base.apk'). - apk_offset: Byte offset within the library or apk. - - Returns: - a new (lib_path, lib_offset) tuple. If |apk_path| points to an APK, - then this function searches inside the corresponding host-side APKs - (added with AddHostApk() above) for the corresponding uncompressed - native shared library at |apk_offset|, if found, this returns a new - device-specific path corresponding to a virtual installation of said - library with an adjusted offset. - - Otherwise, just return the original (apk_path, apk_offset) values. - """ - if not apk_path.endswith('.apk'): - return (apk_path, apk_offset) - - apk_package, apk_name = self._MatchApkDeviceInstallPath(apk_path) - if not apk_package: - return (apk_path, apk_offset) - - key = '%s/%s' % (apk_package, apk_name) - native_libs = self._libs_map.get(key) - if not native_libs: - logging.debug('Unknown %s package', key) - return (apk_path, apk_offset) - - lib_name, new_offset = native_libs.FindLibraryByOffset(apk_offset) - if not lib_name: - logging.debug('Invalid offset in %s.apk package: %d', key, apk_offset) - return (apk_path, apk_offset) - - lib_name = os.path.basename(lib_name) - - # Some libraries are stored with a crazy. prefix inside the APK, this - # is done to prevent the PackageManager from extracting the libraries - # at installation time when running on pre Android M systems, where the - # system linker cannot load libraries directly from APKs. - crazy_prefix = 'crazy.' - if lib_name.startswith(crazy_prefix): - lib_name = lib_name[len(crazy_prefix):] - - # Put this in a fictional lib sub-directory for good measure. - new_path = '%s!lib/%s' % (apk_path, lib_name) - - return (new_path, new_offset)
diff --git a/build/android/pylib/symbols/apk_native_libs_unittest.py b/build/android/pylib/symbols/apk_native_libs_unittest.py deleted file mode 100644 index 416918d..0000000 --- a/build/android/pylib/symbols/apk_native_libs_unittest.py +++ /dev/null
@@ -1,396 +0,0 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import logging -import unittest - -from pylib.symbols import apk_native_libs - -# Mock ELF-like data -MOCK_ELF_DATA = '\x7fELFFFFFFFFFFFFFFFF' - -class MockApkZipInfo(object): - """A mock ApkZipInfo class, returned by MockApkReaderFactory instances.""" - def __init__(self, filename, file_size, compress_size, file_offset, - file_data): - self.filename = filename - self.file_size = file_size - self.compress_size = compress_size - self.file_offset = file_offset - self._data = file_data - - def __repr__(self): - """Convert to string for debugging.""" - return 'MockApkZipInfo["%s",size=%d,compressed=%d,offset=%d]' % ( - self.filename, self.file_size, self.compress_size, self.file_offset) - - def IsCompressed(self): - """Returns True iff the entry is compressed.""" - return self.file_size != self.compress_size - - def IsElfFile(self): - """Returns True iff the entry is an ELF file.""" - if not self._data or len(self._data) < 4: - return False - - return self._data[0:4] == '\x7fELF' - - -class MockApkReader(object): - """A mock ApkReader instance used during unit-testing. - - Do not use directly, but use a MockApkReaderFactory context, as in: - - with MockApkReaderFactory() as mock: - mock.AddTestEntry(file_path, file_size, compress_size, file_data) - ... - - # Actually returns the mock instance. - apk_reader = apk_native_libs.ApkReader('/some/path.apk') - """ - def __init__(self, apk_path='test.apk'): - """Initialize instance.""" - self._entries = [] - self._fake_offset = 0 - self._path = apk_path - - def __enter__(self): - return self - - def __exit__(self, *kwarg): - self.Close() - return - - @property - def path(self): - return self._path - - def AddTestEntry(self, filepath, file_size, compress_size, file_data): - """Add a new entry to the instance for unit-tests. - - Do not call this directly, use the AddTestEntry() method on the parent - MockApkReaderFactory instance. - - Args: - filepath: archive file path. - file_size: uncompressed file size in bytes. - compress_size: compressed size in bytes. - file_data: file data to be checked by IsElfFile() - - Note that file_data can be None, or that its size can be actually - smaller than |compress_size| when used during unit-testing. - """ - self._entries.append(MockApkZipInfo(filepath, file_size, compress_size, - self._fake_offset, file_data)) - self._fake_offset += compress_size - - def Close(self): # pylint: disable=no-self-use - """Close this reader instance.""" - return - - def ListEntries(self): - """Return a list of MockApkZipInfo instances for this input APK.""" - return self._entries - - def FindEntry(self, file_path): - """Find the MockApkZipInfo instance corresponds to a given file path.""" - for entry in self._entries: - if entry.filename == file_path: - return entry - raise KeyError('Could not find mock zip archive member for: ' + file_path) - - -class MockApkReaderTest(unittest.TestCase): - - def testEmpty(self): - with MockApkReader() as reader: - entries = reader.ListEntries() - self.assertTrue(len(entries) == 0) - with self.assertRaises(KeyError): - reader.FindEntry('non-existent-entry.txt') - - def testSingleEntry(self): - with MockApkReader() as reader: - reader.AddTestEntry('some-path/some-file', 20000, 12345, file_data=None) - entries = reader.ListEntries() - self.assertTrue(len(entries) == 1) - entry = entries[0] - self.assertEqual(entry.filename, 'some-path/some-file') - self.assertEqual(entry.file_size, 20000) - self.assertEqual(entry.compress_size, 12345) - self.assertTrue(entry.IsCompressed()) - - entry2 = reader.FindEntry('some-path/some-file') - self.assertEqual(entry, entry2) - - def testMultipleEntries(self): - with MockApkReader() as reader: - _ENTRIES = { - 'foo.txt': (1024, 1024, 'FooFooFoo'), - 'lib/bar/libcode.so': (16000, 3240, 1024, '\x7fELFFFFFFFFFFFF'), - } - for path, props in _ENTRIES.iteritems(): - reader.AddTestEntry(path, props[0], props[1], props[2]) - - entries = reader.ListEntries() - self.assertEqual(len(entries), len(_ENTRIES)) - for path, props in _ENTRIES.iteritems(): - entry = reader.FindEntry(path) - self.assertEqual(entry.filename, path) - self.assertEqual(entry.file_size, props[0]) - self.assertEqual(entry.compress_size, props[1]) - - -class ApkNativeLibrariesTest(unittest.TestCase): - - def setUp(self): - logging.getLogger().setLevel(logging.ERROR) - - def testEmptyApk(self): - with MockApkReader() as reader: - libs_map = apk_native_libs.ApkNativeLibraries(reader) - self.assertTrue(libs_map.IsEmpty()) - self.assertEqual(len(libs_map.GetLibraries()), 0) - lib_path, lib_offset = libs_map.FindLibraryByOffset(0) - self.assertIsNone(lib_path) - self.assertEqual(lib_offset, 0) - - def testSimpleApk(self): - with MockApkReader() as reader: - _MOCK_ENTRIES = [ - # Top-level library should be ignored. - ('libfoo.so', 1000, 1000, MOCK_ELF_DATA, False), - # Library not under lib/ should be ignored. - ('badlib/test-abi/libfoo2.so', 1001, 1001, MOCK_ELF_DATA, False), - # Library under lib/<abi>/ but without .so extension should be ignored. - ('lib/test-abi/libfoo4.so.1', 1003, 1003, MOCK_ELF_DATA, False), - # Library under lib/<abi>/ with .so suffix, but compressed -> ignored. - ('lib/test-abi/libfoo5.so', 1004, 1003, MOCK_ELF_DATA, False), - # First correct library - ('lib/test-abi/libgood1.so', 1005, 1005, MOCK_ELF_DATA, True), - # Second correct library: support sub-directories - ('lib/test-abi/subdir/libgood2.so', 1006, 1006, MOCK_ELF_DATA, True), - # Third correct library, no lib prefix required - ('lib/test-abi/crazy.libgood3.so', 1007, 1007, MOCK_ELF_DATA, True), - ] - file_offsets = [] - prev_offset = 0 - for ent in _MOCK_ENTRIES: - reader.AddTestEntry(ent[0], ent[1], ent[2], ent[3]) - file_offsets.append(prev_offset) - prev_offset += ent[2] - - libs_map = apk_native_libs.ApkNativeLibraries(reader) - self.assertFalse(libs_map.IsEmpty()) - self.assertEqual(libs_map.GetLibraries(), [ - 'lib/test-abi/crazy.libgood3.so', - 'lib/test-abi/libgood1.so', - 'lib/test-abi/subdir/libgood2.so', - ]) - - BIAS = 10 - for mock_ent, file_offset in zip(_MOCK_ENTRIES, file_offsets): - if mock_ent[4]: - lib_path, lib_offset = libs_map.FindLibraryByOffset( - file_offset + BIAS) - self.assertEqual(lib_path, mock_ent[0]) - self.assertEqual(lib_offset, BIAS) - - - def testMultiAbiApk(self): - with MockApkReader() as reader: - _MOCK_ENTRIES = [ - ('lib/abi1/libfoo.so', 1000, 1000, MOCK_ELF_DATA), - ('lib/abi2/libfoo.so', 1000, 1000, MOCK_ELF_DATA), - ] - for ent in _MOCK_ENTRIES: - reader.AddTestEntry(ent[0], ent[1], ent[2], ent[3]) - - libs_map = apk_native_libs.ApkNativeLibraries(reader) - self.assertFalse(libs_map.IsEmpty()) - self.assertEqual(libs_map.GetLibraries(), [ - 'lib/abi1/libfoo.so', 'lib/abi2/libfoo.so']) - - lib1_name, lib1_offset = libs_map.FindLibraryByOffset(10) - self.assertEqual(lib1_name, 'lib/abi1/libfoo.so') - self.assertEqual(lib1_offset, 10) - - lib2_name, lib2_offset = libs_map.FindLibraryByOffset(1000) - self.assertEqual(lib2_name, 'lib/abi2/libfoo.so') - self.assertEqual(lib2_offset, 0) - - -class MockApkNativeLibraries(apk_native_libs.ApkNativeLibraries): - """A mock ApkNativeLibraries instance that can be used as input to - ApkLibraryPathTranslator without creating an ApkReader instance. - - Create a new instance, then call AddTestEntry or AddTestEntries - as many times as necessary, before using it as a regular - ApkNativeLibraries instance. - """ - # pylint: disable=super-init-not-called - def __init__(self): - self._native_libs = [] - - # pylint: enable=super-init-not-called - - def AddTestEntry(self, lib_path, file_offset, file_size): - """Add a new test entry. - - Args: - entry: A tuple of (library-path, file-offset, file-size) values, - (e.g. ('lib/armeabi-v8a/libfoo.so', 0x10000, 0x2000)). - """ - self._native_libs.append((lib_path, file_offset, file_offset + file_size)) - - def AddTestEntries(self, entries): - """Add a list of new test entries. - - Args: - entries: A list of (library-path, file-offset, file-size) values. - """ - for entry in entries: - self.AddTestEntry(entry[0], entry[1], entry[2]) - - -class MockApkNativeLibrariesTest(unittest.TestCase): - - def testEmptyInstance(self): - mock = MockApkNativeLibraries() - self.assertTrue(mock.IsEmpty()) - self.assertEqual(mock.GetLibraries(), []) - self.assertEqual(mock.GetDumpList(), []) - - def testAddTestEntry(self): - mock = MockApkNativeLibraries() - mock.AddTestEntry('lib/armeabi-v7a/libfoo.so', 0x20000, 0x4000) - mock.AddTestEntry('lib/x86/libzoo.so', 0x10000, 0x10000) - mock.AddTestEntry('lib/armeabi-v7a/libbar.so', 0x24000, 0x8000) - self.assertFalse(mock.IsEmpty()) - self.assertEqual(mock.GetLibraries(), ['lib/armeabi-v7a/libbar.so', - 'lib/armeabi-v7a/libfoo.so', - 'lib/x86/libzoo.so']) - self.assertEqual(mock.GetDumpList(), [ - ('lib/x86/libzoo.so', 0x10000, 0x10000), - ('lib/armeabi-v7a/libfoo.so', 0x20000, 0x4000), - ('lib/armeabi-v7a/libbar.so', 0x24000, 0x8000), - ]) - - def testAddTestEntries(self): - mock = MockApkNativeLibraries() - mock.AddTestEntries([ - ('lib/armeabi-v7a/libfoo.so', 0x20000, 0x4000), - ('lib/x86/libzoo.so', 0x10000, 0x10000), - ('lib/armeabi-v7a/libbar.so', 0x24000, 0x8000), - ]) - self.assertFalse(mock.IsEmpty()) - self.assertEqual(mock.GetLibraries(), ['lib/armeabi-v7a/libbar.so', - 'lib/armeabi-v7a/libfoo.so', - 'lib/x86/libzoo.so']) - self.assertEqual(mock.GetDumpList(), [ - ('lib/x86/libzoo.so', 0x10000, 0x10000), - ('lib/armeabi-v7a/libfoo.so', 0x20000, 0x4000), - ('lib/armeabi-v7a/libbar.so', 0x24000, 0x8000), - ]) - - -class ApkLibraryPathTranslatorTest(unittest.TestCase): - - def _CheckUntranslated(self, translator, path, offset): - """Check that a given (path, offset) is not modified by translation.""" - self.assertEqual(translator.TranslatePath(path, offset), (path, offset)) - - - def _CheckTranslated(self, translator, path, offset, new_path, new_offset): - """Check that (path, offset) is translated into (new_path, new_offset).""" - self.assertEqual(translator.TranslatePath(path, offset), - (new_path, new_offset)) - - def testEmptyInstance(self): - translator = apk_native_libs.ApkLibraryPathTranslator() - self._CheckUntranslated( - translator, '/data/data/com.example.app-1/base.apk', 0x123456) - - def testSimpleApk(self): - mock_libs = MockApkNativeLibraries() - mock_libs.AddTestEntries([ - ('lib/test-abi/libfoo.so', 200, 2000), - ('lib/test-abi/libbar.so', 3200, 3000), - ('lib/test-abi/crazy.libzoo.so', 6200, 2000), - ]) - translator = apk_native_libs.ApkLibraryPathTranslator() - translator.AddHostApk('com.example.app', mock_libs) - - # Offset is within the first uncompressed library - self._CheckTranslated( - translator, - '/data/data/com.example.app-9.apk', 757, - '/data/data/com.example.app-9.apk!lib/libfoo.so', 557) - - # Offset is within the second compressed library. - self._CheckUntranslated( - translator, - '/data/data/com.example.app-9/base.apk', 2800) - - # Offset is within the third uncompressed library. - self._CheckTranslated( - translator, - '/data/data/com.example.app-1/base.apk', 3628, - '/data/data/com.example.app-1/base.apk!lib/libbar.so', 428) - - # Offset is within the fourth uncompressed library with crazy. prefix - self._CheckTranslated( - translator, - '/data/data/com.example.app-XX/base.apk', 6500, - '/data/data/com.example.app-XX/base.apk!lib/libzoo.so', 300) - - # Out-of-bounds apk offset. - self._CheckUntranslated( - translator, - '/data/data/com.example.app-1/base.apk', 10000) - - # Invalid package name. - self._CheckUntranslated( - translator, '/data/data/com.example2.app-1/base.apk', 757) - - # Invalid apk name. - self._CheckUntranslated( - translator, '/data/data/com.example.app-2/not-base.apk', 100) - - # Invalid file extensions. - self._CheckUntranslated( - translator, '/data/data/com.example.app-2/base', 100) - - self._CheckUntranslated( - translator, '/data/data/com.example.app-2/base.apk.dex', 100) - - def testBundleApks(self): - mock_libs1 = MockApkNativeLibraries() - mock_libs1.AddTestEntries([ - ('lib/test-abi/libfoo.so', 200, 2000), - ('lib/test-abi/libbbar.so', 3200, 3000), - ]) - mock_libs2 = MockApkNativeLibraries() - mock_libs2.AddTestEntries([ - ('lib/test-abi/libzoo.so', 200, 2000), - ('lib/test-abi/libtool.so', 3000, 4000), - ]) - translator = apk_native_libs.ApkLibraryPathTranslator() - translator.AddHostApk('com.example.app', mock_libs1, 'base-master.apk') - translator.AddHostApk('com.example.app', mock_libs2, 'feature-master.apk') - - self._CheckTranslated( - translator, - '/data/app/com.example.app-XUIYIUW/base-master.apk', 757, - '/data/app/com.example.app-XUIYIUW/base-master.apk!lib/libfoo.so', 557) - - self._CheckTranslated( - translator, - '/data/app/com.example.app-XUIYIUW/feature-master.apk', 3200, - '/data/app/com.example.app-XUIYIUW/feature-master.apk!lib/libtool.so', - 200) - - -if __name__ == '__main__': - unittest.main()
diff --git a/build/android/pylib/symbols/deobfuscator.py b/build/android/pylib/symbols/deobfuscator.py index ffc23b8..7106098 100644 --- a/build/android/pylib/symbols/deobfuscator.py +++ b/build/android/pylib/symbols/deobfuscator.py
@@ -1,175 +1,50 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import logging import os -import subprocess -import threading -import time -import uuid -from devil.utils import reraiser_thread from pylib import constants +from .expensive_line_transformer import ExpensiveLineTransformer +from .expensive_line_transformer import ExpensiveLineTransformerPool + +_MINIMUM_TIMEOUT = 10.0 +_PER_LINE_TIMEOUT = .005 # Should be able to process 200 lines per second. +_PROCESS_START_TIMEOUT = 20.0 +_MAX_RESTARTS = 4 # Should be plenty unless tool is crashing on start-up. +_POOL_SIZE = 4 +_PASSTHROUH_ON_FAILURE = False -_MINIUMUM_TIMEOUT = 3.0 -_PER_LINE_TIMEOUT = .002 # Should be able to process 500 lines per second. -_PROCESS_START_TIMEOUT = 10.0 -_MAX_RESTARTS = 10 # Should be plenty unless tool is crashing on start-up. - - -class Deobfuscator(object): +class Deobfuscator(ExpensiveLineTransformer): def __init__(self, mapping_path): + super().__init__(_PROCESS_START_TIMEOUT, _MINIMUM_TIMEOUT, + _PER_LINE_TIMEOUT) script_path = os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'android', 'stacktrace', 'java_deobfuscate.py') - cmd = [script_path, mapping_path] - # Allow only one thread to call TransformLines() at a time. - self._lock = threading.Lock() - # Ensure that only one thread attempts to kill self._proc in Close(). - self._close_lock = threading.Lock() - self._closed_called = False - # Assign to None so that attribute exists if Popen() throws. - self._proc = None - # Start process eagerly to hide start-up latency. - self._proc_start_time = time.time() - self._proc = subprocess.Popen( - cmd, bufsize=1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - close_fds=True) + self._command = [script_path, mapping_path] + self.start() - def IsClosed(self): - return self._closed_called or self._proc.returncode is not None + @property + def name(self): + return "deobfuscator" - def IsBusy(self): - return self._lock.locked() - - def IsReady(self): - return not self.IsClosed() and not self.IsBusy() - - def TransformLines(self, lines): - """Deobfuscates obfuscated names found in the given lines. - - If anything goes wrong (process crashes, timeout, etc), returns |lines|. - - Args: - lines: A list of strings without trailing newlines. - - Returns: - A list of strings without trailing newlines. - """ - if not lines: - return [] - - # Deobfuscated stacks contain more frames than obfuscated ones when method - # inlining occurs. To account for the extra output lines, keep reading until - # this eof_line token is reached. - eof_line = uuid.uuid4().hex - out_lines = [] - - def deobfuscate_reader(): - while True: - line = self._proc.stdout.readline() - # Return an empty string at EOF (when stdin is closed). - if not line: - break - line = line[:-1] - if line == eof_line: - break - out_lines.append(line) - - if self.IsBusy(): - logging.warning('deobfuscator: Having to wait for Java deobfuscation.') - - # Allow only one thread to operate at a time. - with self._lock: - if self.IsClosed(): - if not self._closed_called: - logging.warning('deobfuscator: Process exited with code=%d.', - self._proc.returncode) - self.Close() - return lines - - # TODO(agrieve): Can probably speed this up by only sending lines through - # that might contain an obfuscated name. - reader_thread = reraiser_thread.ReraiserThread(deobfuscate_reader) - reader_thread.start() - - try: - self._proc.stdin.write('\n'.join(lines)) - self._proc.stdin.write('\n{}\n'.format(eof_line)) - self._proc.stdin.flush() - time_since_proc_start = time.time() - self._proc_start_time - timeout = (max(0, _PROCESS_START_TIMEOUT - time_since_proc_start) + - max(_MINIUMUM_TIMEOUT, len(lines) * _PER_LINE_TIMEOUT)) - reader_thread.join(timeout) - if self.IsClosed(): - logging.warning( - 'deobfuscator: Close() called by another thread during join().') - return lines - if reader_thread.is_alive(): - logging.error('deobfuscator: Timed out.') - self.Close() - return lines - return out_lines - except IOError: - logging.exception('deobfuscator: Exception during java_deobfuscate') - self.Close() - return lines - - def Close(self): - with self._close_lock: - needs_closing = not self.IsClosed() - self._closed_called = True - - if needs_closing: - self._proc.stdin.close() - self._proc.kill() - self._proc.wait() - - def __del__(self): - # self._proc is None when Popen() fails. - if not self._closed_called and self._proc: - logging.error('deobfuscator: Forgot to Close()') - self.Close() + @property + def command(self): + return self._command -class DeobfuscatorPool(object): - # As of Sep 2017, each instance requires about 500MB of RAM, as measured by: - # /usr/bin/time -v build/android/stacktrace/java_deobfuscate.py \ - # out/Release/apks/ChromePublic.apk.mapping - def __init__(self, mapping_path, pool_size=4): - self._mapping_path = mapping_path - self._pool = [Deobfuscator(mapping_path) for _ in xrange(pool_size)] - # Allow only one thread to select from the pool at a time. - self._lock = threading.Lock() - self._num_restarts = 0 +class DeobfuscatorPool(ExpensiveLineTransformerPool): + def __init__(self, mapping_path): + # As of Sep 2017, each instance requires about 500MB of RAM, as measured by: + # /usr/bin/time -v build/android/stacktrace/java_deobfuscate.py \ + # out/Release/apks/ChromePublic.apk.mapping + self.mapping_path = mapping_path + super().__init__(_MAX_RESTARTS, _POOL_SIZE, _PASSTHROUH_ON_FAILURE) - def TransformLines(self, lines): - with self._lock: - assert self._pool, 'TransformLines() called on a closed DeobfuscatorPool.' + @property + def name(self): + return "deobfuscator-pool" - # De-obfuscation is broken. - if self._num_restarts == _MAX_RESTARTS: - raise Exception('Deobfuscation seems broken.') - - # Restart any closed Deobfuscators. - for i, d in enumerate(self._pool): - if d.IsClosed(): - logging.warning('deobfuscator: Restarting closed instance.') - self._pool[i] = Deobfuscator(self._mapping_path) - self._num_restarts += 1 - if self._num_restarts == _MAX_RESTARTS: - logging.warning('deobfuscator: MAX_RESTARTS reached.') - - selected = next((x for x in self._pool if x.IsReady()), self._pool[0]) - # Rotate the order so that next caller will not choose the same one. - self._pool.remove(selected) - self._pool.append(selected) - - return selected.TransformLines(lines) - - def Close(self): - with self._lock: - for d in self._pool: - d.Close() - self._pool = None + def CreateTransformer(self): + return Deobfuscator(self.mapping_path)
diff --git a/build/android/pylib/symbols/elf_symbolizer.py b/build/android/pylib/symbols/elf_symbolizer.py deleted file mode 100644 index 1f2f918..0000000 --- a/build/android/pylib/symbols/elf_symbolizer.py +++ /dev/null
@@ -1,487 +0,0 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import collections -import datetime -import logging -import multiprocessing -import os -import posixpath -import Queue -import re -import subprocess -import sys -import threading -import time - - -# addr2line builds a possibly infinite memory cache that can exhaust -# the computer's memory if allowed to grow for too long. This constant -# controls how many lookups we do before restarting the process. 4000 -# gives near peak performance without extreme memory usage. -ADDR2LINE_RECYCLE_LIMIT = 4000 - - -ELF_MAGIC = '\x7f\x45\x4c\x46' - - -def ContainsElfMagic(file_path): - if os.path.getsize(file_path) < 4: - return False - try: - with open(file_path, 'r') as f: - b = f.read(4) - return b == ELF_MAGIC - except IOError: - return False - - -class ELFSymbolizer(object): - """An uber-fast (multiprocessing, pipelined and asynchronous) ELF symbolizer. - - This class is a frontend for addr2line (part of GNU binutils), designed to - symbolize batches of large numbers of symbols for a given ELF file. It - supports sharding symbolization against many addr2line instances and - pipelining of multiple requests per each instance (in order to hide addr2line - internals and OS pipe latencies). - - The interface exhibited by this class is a very simple asynchronous interface, - which is based on the following three methods: - - SymbolizeAsync(): used to request (enqueue) resolution of a given address. - - The |callback| method: used to communicated back the symbol information. - - Join(): called to conclude the batch to gather the last outstanding results. - In essence, before the Join method returns, this class will have issued as - many callbacks as the number of SymbolizeAsync() calls. In this regard, note - that due to multiprocess sharding, callbacks can be delivered out of order. - - Some background about addr2line: - - it is invoked passing the elf path in the cmdline, piping the addresses in - its stdin and getting results on its stdout. - - it has pretty large response times for the first requests, but it - works very well in streaming mode once it has been warmed up. - - it doesn't scale by itself (on more cores). However, spawning multiple - instances at the same time on the same file is pretty efficient as they - keep hitting the pagecache and become mostly CPU bound. - - it might hang or crash, mostly for OOM. This class deals with both of these - problems. - - Despite the "scary" imports and the multi* words above, (almost) no multi- - threading/processing is involved from the python viewpoint. Concurrency - here is achieved by spawning several addr2line subprocesses and handling their - output pipes asynchronously. Therefore, all the code here (with the exception - of the Queue instance in Addr2Line) should be free from mind-blowing - thread-safety concerns. - - The multiprocess sharding works as follows: - The symbolizer tries to use the lowest number of addr2line instances as - possible (with respect of |max_concurrent_jobs|) and enqueue all the requests - in a single addr2line instance. For few symbols (i.e. dozens) sharding isn't - worth the startup cost. - The multiprocess logic kicks in as soon as the queues for the existing - instances grow. Specifically, once all the existing instances reach the - |max_queue_size| bound, a new addr2line instance is kicked in. - In the case of a very eager producer (i.e. all |max_concurrent_jobs| instances - have a backlog of |max_queue_size|), back-pressure is applied on the caller by - blocking the SymbolizeAsync method. - - This module has been deliberately designed to be dependency free (w.r.t. of - other modules in this project), to allow easy reuse in external projects. - """ - - def __init__(self, elf_file_path, addr2line_path, callback, inlines=False, - max_concurrent_jobs=None, addr2line_timeout=30, max_queue_size=50, - source_root_path=None, strip_base_path=None): - """Args: - elf_file_path: path of the elf file to be symbolized. - addr2line_path: path of the toolchain's addr2line binary. - callback: a callback which will be invoked for each resolved symbol with - the two args (sym_info, callback_arg). The former is an instance of - |ELFSymbolInfo| and contains the symbol information. The latter is an - embedder-provided argument which is passed to SymbolizeAsync(). - inlines: when True, the ELFSymbolInfo will contain also the details about - the outer inlining functions. When False, only the innermost function - will be provided. - max_concurrent_jobs: Max number of addr2line instances spawned. - Parallelize responsibly, addr2line is a memory and I/O monster. - max_queue_size: Max number of outstanding requests per addr2line instance. - addr2line_timeout: Max time (in seconds) to wait for a addr2line response. - After the timeout, the instance will be considered hung and respawned. - source_root_path: In some toolchains only the name of the source file is - is output, without any path information; disambiguation searches - through the source directory specified by |source_root_path| argument - for files whose name matches, adding the full path information to the - output. For example, if the toolchain outputs "unicode.cc" and there - is a file called "unicode.cc" located under |source_root_path|/foo, - the tool will replace "unicode.cc" with - "|source_root_path|/foo/unicode.cc". If there are multiple files with - the same name, disambiguation will fail because the tool cannot - determine which of the files was the source of the symbol. - strip_base_path: Rebases the symbols source paths onto |source_root_path| - (i.e replace |strip_base_path| with |source_root_path). - """ - assert(os.path.isfile(addr2line_path)), 'Cannot find ' + addr2line_path - self.elf_file_path = elf_file_path - self.addr2line_path = addr2line_path - self.callback = callback - self.inlines = inlines - self.max_concurrent_jobs = (max_concurrent_jobs or - min(multiprocessing.cpu_count(), 4)) - self.max_queue_size = max_queue_size - self.addr2line_timeout = addr2line_timeout - self.requests_counter = 0 # For generating monotonic request IDs. - self._a2l_instances = [] # Up to |max_concurrent_jobs| _Addr2Line inst. - - # If necessary, create disambiguation lookup table - self.disambiguate = source_root_path is not None - self.disambiguation_table = {} - self.strip_base_path = strip_base_path - if self.disambiguate: - self.source_root_path = os.path.abspath(source_root_path) - self._CreateDisambiguationTable() - - # Create one addr2line instance. More instances will be created on demand - # (up to |max_concurrent_jobs|) depending on the rate of the requests. - self._CreateNewA2LInstance() - - def SymbolizeAsync(self, addr, callback_arg=None): - """Requests symbolization of a given address. - - This method is not guaranteed to return immediately. It generally does, but - in some scenarios (e.g. all addr2line instances have full queues) it can - block to create back-pressure. - - Args: - addr: address to symbolize. - callback_arg: optional argument which will be passed to the |callback|.""" - assert isinstance(addr, int) - - # Process all the symbols that have been resolved in the meanwhile. - # Essentially, this drains all the addr2line(s) out queues. - for a2l_to_purge in self._a2l_instances: - a2l_to_purge.ProcessAllResolvedSymbolsInQueue() - a2l_to_purge.RecycleIfNecessary() - - # Find the best instance according to this logic: - # 1. Find an existing instance with the shortest queue. - # 2. If all of instances' queues are full, but there is room in the pool, - # (i.e. < |max_concurrent_jobs|) create a new instance. - # 3. If there were already |max_concurrent_jobs| instances and all of them - # had full queues, make back-pressure. - - # 1. - def _SortByQueueSizeAndReqID(a2l): - return (a2l.queue_size, a2l.first_request_id) - a2l = min(self._a2l_instances, key=_SortByQueueSizeAndReqID) - - # 2. - if (a2l.queue_size >= self.max_queue_size and - len(self._a2l_instances) < self.max_concurrent_jobs): - a2l = self._CreateNewA2LInstance() - - # 3. - if a2l.queue_size >= self.max_queue_size: - a2l.WaitForNextSymbolInQueue() - - a2l.EnqueueRequest(addr, callback_arg) - - def WaitForIdle(self): - """Waits for all the outstanding requests to complete.""" - for a2l in self._a2l_instances: - a2l.WaitForIdle() - - def Join(self): - """Waits for all the outstanding requests to complete and terminates.""" - for a2l in self._a2l_instances: - a2l.WaitForIdle() - a2l.Terminate() - - def _CreateNewA2LInstance(self): - assert len(self._a2l_instances) < self.max_concurrent_jobs - a2l = ELFSymbolizer.Addr2Line(self) - self._a2l_instances.append(a2l) - return a2l - - def _CreateDisambiguationTable(self): - """ Non-unique file names will result in None entries""" - start_time = time.time() - logging.info('Collecting information about available source files...') - self.disambiguation_table = {} - - for root, _, filenames in os.walk(self.source_root_path): - for f in filenames: - self.disambiguation_table[f] = os.path.join(root, f) if (f not in - self.disambiguation_table) else None - logging.info('Finished collecting information about ' - 'possible files (took %.1f s).', - (time.time() - start_time)) - - - class Addr2Line(object): - """A python wrapper around an addr2line instance. - - The communication with the addr2line process looks as follows: - [STDIN] [STDOUT] (from addr2line's viewpoint) - > f001111 - > f002222 - < Symbol::Name(foo, bar) for f001111 - < /path/to/source/file.c:line_number - > f003333 - < Symbol::Name2() for f002222 - < /path/to/source/file.c:line_number - < Symbol::Name3() for f003333 - < /path/to/source/file.c:line_number - """ - - SYM_ADDR_RE = re.compile(r'([^:]+):(\?|\d+).*') - - def __init__(self, symbolizer): - self._symbolizer = symbolizer - self._lib_file_name = posixpath.basename(symbolizer.elf_file_path) - - # The request queue (i.e. addresses pushed to addr2line's stdin and not - # yet retrieved on stdout) - self._request_queue = collections.deque() - - # This is essentially len(self._request_queue). It has been optimized to a - # separate field because turned out to be a perf hot-spot. - self.queue_size = 0 - - # Keep track of the number of symbols a process has processed to - # avoid a single process growing too big and using all the memory. - self._processed_symbols_count = 0 - - # Objects required to handle the addr2line subprocess. - self._proc = None # Subprocess.Popen(...) instance. - self._thread = None # Threading.thread instance. - self._out_queue = None # Queue.Queue instance (for buffering a2l stdout). - self._RestartAddr2LineProcess() - - def EnqueueRequest(self, addr, callback_arg): - """Pushes an address to addr2line's stdin (and keeps track of it).""" - self._symbolizer.requests_counter += 1 # For global "age" of requests. - req_idx = self._symbolizer.requests_counter - self._request_queue.append((addr, callback_arg, req_idx)) - self.queue_size += 1 - self._WriteToA2lStdin(addr) - - def WaitForIdle(self): - """Waits until all the pending requests have been symbolized.""" - while self.queue_size > 0: - self.WaitForNextSymbolInQueue() - - def WaitForNextSymbolInQueue(self): - """Waits for the next pending request to be symbolized.""" - if not self.queue_size: - return - - # This outer loop guards against a2l hanging (detecting stdout timeout). - while True: - start_time = datetime.datetime.now() - timeout = datetime.timedelta(seconds=self._symbolizer.addr2line_timeout) - - # The inner loop guards against a2l crashing (checking if it exited). - while datetime.datetime.now() - start_time < timeout: - # poll() returns !None if the process exited. a2l should never exit. - if self._proc.poll(): - logging.warning('addr2line crashed, respawning (lib: %s).', - self._lib_file_name) - self._RestartAddr2LineProcess() - # TODO(primiano): the best thing to do in this case would be - # shrinking the pool size as, very likely, addr2line is crashed - # due to low memory (and the respawned one will die again soon). - - try: - lines = self._out_queue.get(block=True, timeout=0.25) - except Queue.Empty: - # On timeout (1/4 s.) repeat the inner loop and check if either the - # addr2line process did crash or we waited its output for too long. - continue - - # In nominal conditions, we get straight to this point. - self._ProcessSymbolOutput(lines) - return - - # If this point is reached, we waited more than |addr2line_timeout|. - logging.warning('Hung addr2line process, respawning (lib: %s).', - self._lib_file_name) - self._RestartAddr2LineProcess() - - def ProcessAllResolvedSymbolsInQueue(self): - """Consumes all the addr2line output lines produced (without blocking).""" - if not self.queue_size: - return - while True: - try: - lines = self._out_queue.get_nowait() - except Queue.Empty: - break - self._ProcessSymbolOutput(lines) - - def RecycleIfNecessary(self): - """Restarts the process if it has been used for too long. - - A long running addr2line process will consume excessive amounts - of memory without any gain in performance.""" - if self._processed_symbols_count >= ADDR2LINE_RECYCLE_LIMIT: - self._RestartAddr2LineProcess() - - - def Terminate(self): - """Kills the underlying addr2line process. - - The poller |_thread| will terminate as well due to the broken pipe.""" - try: - self._proc.kill() - self._proc.communicate() # Essentially wait() without risking deadlock. - except Exception: # pylint: disable=broad-except - # An exception while terminating? How interesting. - pass - self._proc = None - - def _WriteToA2lStdin(self, addr): - self._proc.stdin.write('%s\n' % hex(addr)) - if self._symbolizer.inlines: - # In the case of inlines we output an extra blank line, which causes - # addr2line to emit a (??,??:0) tuple that we use as a boundary marker. - self._proc.stdin.write('\n') - self._proc.stdin.flush() - - def _ProcessSymbolOutput(self, lines): - """Parses an addr2line symbol output and triggers the client callback.""" - (_, callback_arg, _) = self._request_queue.popleft() - self.queue_size -= 1 - - innermost_sym_info = None - sym_info = None - for (line1, line2) in lines: - prev_sym_info = sym_info - name = line1 if not line1.startswith('?') else None - source_path = None - source_line = None - m = ELFSymbolizer.Addr2Line.SYM_ADDR_RE.match(line2) - if m: - if not m.group(1).startswith('?'): - source_path = m.group(1) - if not m.group(2).startswith('?'): - source_line = int(m.group(2)) - else: - logging.warning('Got invalid symbol path from addr2line: %s', line2) - - # In case disambiguation is on, and needed - was_ambiguous = False - disambiguated = False - if self._symbolizer.disambiguate: - if source_path and not posixpath.isabs(source_path): - path = self._symbolizer.disambiguation_table.get(source_path) - was_ambiguous = True - disambiguated = path is not None - source_path = path if disambiguated else source_path - - # Use absolute paths (so that paths are consistent, as disambiguation - # uses absolute paths) - if source_path and not was_ambiguous: - source_path = os.path.abspath(source_path) - - if source_path and self._symbolizer.strip_base_path: - # Strip the base path - source_path = re.sub('^' + self._symbolizer.strip_base_path, - self._symbolizer.source_root_path or '', source_path) - - sym_info = ELFSymbolInfo(name, source_path, source_line, was_ambiguous, - disambiguated) - if prev_sym_info: - prev_sym_info.inlined_by = sym_info - if not innermost_sym_info: - innermost_sym_info = sym_info - - self._processed_symbols_count += 1 - self._symbolizer.callback(innermost_sym_info, callback_arg) - - def _RestartAddr2LineProcess(self): - if self._proc: - self.Terminate() - - # The only reason of existence of this Queue (and the corresponding - # Thread below) is the lack of a subprocess.stdout.poll_avail_lines(). - # Essentially this is a pipe able to extract a couple of lines atomically. - self._out_queue = Queue.Queue() - - # Start the underlying addr2line process in line buffered mode. - - cmd = [self._symbolizer.addr2line_path, '--functions', '--demangle', - '--exe=' + self._symbolizer.elf_file_path] - if self._symbolizer.inlines: - cmd += ['--inlines'] - self._proc = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE, - stdin=subprocess.PIPE, stderr=sys.stderr, close_fds=True) - - # Start the poller thread, which simply moves atomically the lines read - # from the addr2line's stdout to the |_out_queue|. - self._thread = threading.Thread( - target=ELFSymbolizer.Addr2Line.StdoutReaderThread, - args=(self._proc.stdout, self._out_queue, self._symbolizer.inlines)) - self._thread.daemon = True # Don't prevent early process exit. - self._thread.start() - - self._processed_symbols_count = 0 - - # Replay the pending requests on the new process (only for the case - # of a hung addr2line timing out during the game). - for (addr, _, _) in self._request_queue: - self._WriteToA2lStdin(addr) - - @staticmethod - def StdoutReaderThread(process_pipe, queue, inlines): - """The poller thread fn, which moves the addr2line stdout to the |queue|. - - This is the only piece of code not running on the main thread. It merely - writes to a Queue, which is thread-safe. In the case of inlines, it - detects the ??,??:0 marker and sends the lines atomically, such that the - main thread always receives all the lines corresponding to one symbol in - one shot.""" - try: - lines_for_one_symbol = [] - while True: - line1 = process_pipe.readline().rstrip('\r\n') - line2 = process_pipe.readline().rstrip('\r\n') - if not line1 or not line2: - break - inline_has_more_lines = inlines and (len(lines_for_one_symbol) == 0 or - (line1 != '??' and line2 != '??:0')) - if not inlines or inline_has_more_lines: - lines_for_one_symbol += [(line1, line2)] - if inline_has_more_lines: - continue - queue.put(lines_for_one_symbol) - lines_for_one_symbol = [] - process_pipe.close() - - # Every addr2line processes will die at some point, please die silently. - except (IOError, OSError): - pass - - @property - def first_request_id(self): - """Returns the request_id of the oldest pending request in the queue.""" - return self._request_queue[0][2] if self._request_queue else 0 - - -class ELFSymbolInfo(object): - """The result of the symbolization passed as first arg. of each callback.""" - - def __init__(self, name, source_path, source_line, was_ambiguous=False, - disambiguated=False): - """All the fields here can be None (if addr2line replies with '??').""" - self.name = name - self.source_path = source_path - self.source_line = source_line - # In the case of |inlines|=True, the |inlined_by| points to the outer - # function inlining the current one (and so on, to form a chain). - self.inlined_by = None - self.disambiguated = disambiguated - self.was_ambiguous = was_ambiguous - - def __str__(self): - return '%s [%s:%d]' % ( - self.name or '??', self.source_path or '??', self.source_line or 0)
diff --git a/build/android/pylib/symbols/elf_symbolizer_unittest.py b/build/android/pylib/symbols/elf_symbolizer_unittest.py deleted file mode 100755 index 765b598..0000000 --- a/build/android/pylib/symbols/elf_symbolizer_unittest.py +++ /dev/null
@@ -1,196 +0,0 @@ -#!/usr/bin/env python -# Copyright 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import functools -import logging -import os -import unittest - -from pylib.symbols import elf_symbolizer -from pylib.symbols import mock_addr2line - - -_MOCK_A2L_PATH = os.path.join(os.path.dirname(mock_addr2line.__file__), - 'mock_addr2line') -_INCOMPLETE_MOCK_ADDR = 1024 * 1024 -_UNKNOWN_MOCK_ADDR = 2 * 1024 * 1024 -_INLINE_MOCK_ADDR = 3 * 1024 * 1024 - - -class ELFSymbolizerTest(unittest.TestCase): - def setUp(self): - self._callback = functools.partial( - ELFSymbolizerTest._SymbolizeCallback, self) - self._resolved_addresses = set() - # Mute warnings, we expect them due to the crash/hang tests. - logging.getLogger().setLevel(logging.ERROR) - - def testParallelism1(self): - self._RunTest(max_concurrent_jobs=1, num_symbols=100) - - def testParallelism4(self): - self._RunTest(max_concurrent_jobs=4, num_symbols=100) - - def testParallelism8(self): - self._RunTest(max_concurrent_jobs=8, num_symbols=100) - - def testCrash(self): - os.environ['MOCK_A2L_CRASH_EVERY'] = '99' - self._RunTest(max_concurrent_jobs=1, num_symbols=100) - os.environ['MOCK_A2L_CRASH_EVERY'] = '0' - - def testHang(self): - os.environ['MOCK_A2L_HANG_EVERY'] = '99' - self._RunTest(max_concurrent_jobs=1, num_symbols=100) - os.environ['MOCK_A2L_HANG_EVERY'] = '0' - - def testInlines(self): - """Stimulate the inline processing logic.""" - symbolizer = elf_symbolizer.ELFSymbolizer( - elf_file_path='/path/doesnt/matter/mock_lib1.so', - addr2line_path=_MOCK_A2L_PATH, - callback=self._callback, - inlines=True, - max_concurrent_jobs=4) - - for addr in xrange(1000): - exp_inline = False - exp_unknown = False - - # First 100 addresses with inlines. - if addr < 100: - addr += _INLINE_MOCK_ADDR - exp_inline = True - - # Followed by 100 without inlines. - elif addr < 200: - pass - - # Followed by 100 interleaved inlines and not inlines. - elif addr < 300: - if addr & 1: - addr += _INLINE_MOCK_ADDR - exp_inline = True - - # Followed by 100 interleaved inlines and unknonwn. - elif addr < 400: - if addr & 1: - addr += _INLINE_MOCK_ADDR - exp_inline = True - else: - addr += _UNKNOWN_MOCK_ADDR - exp_unknown = True - - exp_name = 'mock_sym_for_addr_%d' % addr if not exp_unknown else None - exp_source_path = 'mock_src/mock_lib1.so.c' if not exp_unknown else None - exp_source_line = addr if not exp_unknown else None - cb_arg = (addr, exp_name, exp_source_path, exp_source_line, exp_inline) - symbolizer.SymbolizeAsync(addr, cb_arg) - - symbolizer.Join() - - def testIncompleteSyminfo(self): - """Stimulate the symbol-not-resolved logic.""" - symbolizer = elf_symbolizer.ELFSymbolizer( - elf_file_path='/path/doesnt/matter/mock_lib1.so', - addr2line_path=_MOCK_A2L_PATH, - callback=self._callback, - max_concurrent_jobs=1) - - # Test symbols with valid name but incomplete path. - addr = _INCOMPLETE_MOCK_ADDR - exp_name = 'mock_sym_for_addr_%d' % addr - exp_source_path = None - exp_source_line = None - cb_arg = (addr, exp_name, exp_source_path, exp_source_line, False) - symbolizer.SymbolizeAsync(addr, cb_arg) - - # Test symbols with no name or sym info. - addr = _UNKNOWN_MOCK_ADDR - exp_name = None - exp_source_path = None - exp_source_line = None - cb_arg = (addr, exp_name, exp_source_path, exp_source_line, False) - symbolizer.SymbolizeAsync(addr, cb_arg) - - symbolizer.Join() - - def testWaitForIdle(self): - symbolizer = elf_symbolizer.ELFSymbolizer( - elf_file_path='/path/doesnt/matter/mock_lib1.so', - addr2line_path=_MOCK_A2L_PATH, - callback=self._callback, - max_concurrent_jobs=1) - - # Test symbols with valid name but incomplete path. - addr = _INCOMPLETE_MOCK_ADDR - exp_name = 'mock_sym_for_addr_%d' % addr - exp_source_path = None - exp_source_line = None - cb_arg = (addr, exp_name, exp_source_path, exp_source_line, False) - symbolizer.SymbolizeAsync(addr, cb_arg) - symbolizer.WaitForIdle() - - # Test symbols with no name or sym info. - addr = _UNKNOWN_MOCK_ADDR - exp_name = None - exp_source_path = None - exp_source_line = None - cb_arg = (addr, exp_name, exp_source_path, exp_source_line, False) - symbolizer.SymbolizeAsync(addr, cb_arg) - symbolizer.Join() - - def _RunTest(self, max_concurrent_jobs, num_symbols): - symbolizer = elf_symbolizer.ELFSymbolizer( - elf_file_path='/path/doesnt/matter/mock_lib1.so', - addr2line_path=_MOCK_A2L_PATH, - callback=self._callback, - max_concurrent_jobs=max_concurrent_jobs, - addr2line_timeout=0.5) - - for addr in xrange(num_symbols): - exp_name = 'mock_sym_for_addr_%d' % addr - exp_source_path = 'mock_src/mock_lib1.so.c' - exp_source_line = addr - cb_arg = (addr, exp_name, exp_source_path, exp_source_line, False) - symbolizer.SymbolizeAsync(addr, cb_arg) - - symbolizer.Join() - - # Check that all the expected callbacks have been received. - for addr in xrange(num_symbols): - self.assertIn(addr, self._resolved_addresses) - self._resolved_addresses.remove(addr) - - # Check for unexpected callbacks. - self.assertEqual(len(self._resolved_addresses), 0) - - def _SymbolizeCallback(self, sym_info, cb_arg): - self.assertTrue(isinstance(sym_info, elf_symbolizer.ELFSymbolInfo)) - self.assertTrue(isinstance(cb_arg, tuple)) - self.assertEqual(len(cb_arg), 5) - - # Unpack expectations from the callback extra argument. - (addr, exp_name, exp_source_path, exp_source_line, exp_inlines) = cb_arg - if exp_name is None: - self.assertIsNone(sym_info.name) - else: - self.assertTrue(sym_info.name.startswith(exp_name)) - self.assertEqual(sym_info.source_path, exp_source_path) - self.assertEqual(sym_info.source_line, exp_source_line) - - if exp_inlines: - self.assertEqual(sym_info.name, exp_name + '_inner') - self.assertEqual(sym_info.inlined_by.name, exp_name + '_middle') - self.assertEqual(sym_info.inlined_by.inlined_by.name, - exp_name + '_outer') - - # Check against duplicate callbacks. - self.assertNotIn(addr, self._resolved_addresses) - self._resolved_addresses.add(addr) - - -if __name__ == '__main__': - unittest.main()
diff --git a/build/android/pylib/symbols/expensive_line_transformer.py b/build/android/pylib/symbols/expensive_line_transformer.py new file mode 100644 index 0000000..08cbe52 --- /dev/null +++ b/build/android/pylib/symbols/expensive_line_transformer.py
@@ -0,0 +1,233 @@ +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from abc import ABC, abstractmethod +import logging +import subprocess +import threading +import time +import uuid + +from devil.utils import reraiser_thread + + +class ExpensiveLineTransformer(ABC): + def __init__(self, process_start_timeout, minimum_timeout, per_line_timeout): + self._process_start_timeout = process_start_timeout + self._minimum_timeout = minimum_timeout + self._per_line_timeout = per_line_timeout + self._started = False + # Allow only one thread to call TransformLines() at a time. + self._lock = threading.Lock() + # Ensure that only one thread attempts to kill self._proc in Close(). + self._close_lock = threading.Lock() + self._closed_called = False + # Assign to None so that attribute exists if Popen() throws. + self._proc = None + # Start process eagerly to hide start-up latency. + self._proc_start_time = None + + def start(self): + # delay the start of the process, to allow the initialization of the + # descendant classes first. + if self._started: + logging.error('%s: Trying to start an already started command', self.name) + return + + # Start process eagerly to hide start-up latency. + self._proc_start_time = time.time() + + if not self.command: + logging.error('%s: No command available', self.name) + return + + self._proc = subprocess.Popen(self.command, + bufsize=1, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True, + close_fds=True) + self._started = True + + def IsClosed(self): + return (not self._started or self._closed_called + or self._proc.returncode is not None) + + def IsBusy(self): + return self._lock.locked() + + def IsReady(self): + return self._started and not self.IsClosed() and not self.IsBusy() + + def TransformLines(self, lines): + """Symbolizes names found in the given lines. + + If anything goes wrong (process crashes, timeout, etc), returns |lines|. + + Args: + lines: A list of strings without trailing newlines. + + Returns: + A list of strings without trailing newlines. + """ + if not lines: + return [] + + # symbolized output contain more lines than the input, as the symbolized + # stacktraces will be added. To account for the extra output lines, keep + # reading until this eof_line token is reached. Using a format that will + # be considered a "useful line" without modifying its output by + # third_party/android_platform/development/scripts/stack_core.py + eof_line = self.getEofLine() + out_lines = [] + + def _reader(): + while True: + line = self._proc.stdout.readline() + # Return an empty string at EOF (when stdin is closed). + if not line: + break + line = line[:-1] + if line == eof_line: + break + out_lines.append(line) + + if self.IsBusy(): + logging.warning('%s: Having to wait for transformation.', self.name) + + # Allow only one thread to operate at a time. + with self._lock: + if self.IsClosed(): + if self._started and not self._closed_called: + logging.warning('%s: Process exited with code=%d.', self.name, + self._proc.returncode) + self.Close() + return lines + + reader_thread = reraiser_thread.ReraiserThread(_reader) + reader_thread.start() + + try: + self._proc.stdin.write('\n'.join(lines)) + self._proc.stdin.write('\n{}\n'.format(eof_line)) + self._proc.stdin.flush() + time_since_proc_start = time.time() - self._proc_start_time + timeout = (max(0, self._process_start_timeout - time_since_proc_start) + + max(self._minimum_timeout, + len(lines) * self._per_line_timeout)) + reader_thread.join(timeout) + if self.IsClosed(): + logging.warning('%s: Close() called by another thread during join().', + self.name) + return lines + if reader_thread.is_alive(): + logging.error('%s: Timed out after %f seconds with input:', self.name, + timeout) + for l in lines: + logging.error(l) + logging.error(eof_line) + logging.error('%s: End of timed out input.', self.name) + logging.error('%s: Timed out output was:', self.name) + for l in out_lines: + logging.error(l) + logging.error('%s: End of timed out output.', self.name) + self.Close() + return lines + return out_lines + except IOError: + logging.exception('%s: Exception during transformation', self.name) + self.Close() + return lines + + def Close(self): + with self._close_lock: + needs_closing = not self.IsClosed() + self._closed_called = True + + if needs_closing: + self._proc.stdin.close() + self._proc.kill() + self._proc.wait() + + def __del__(self): + # self._proc is None when Popen() fails. + if not self._closed_called and self._proc: + logging.error('%s: Forgot to Close()', self.name) + self.Close() + + @property + @abstractmethod + def name(self): + ... + + @property + @abstractmethod + def command(self): + ... + + @staticmethod + def getEofLine(): + # Use a format that will be considered a "useful line" without modifying its + # output by third_party/android_platform/development/scripts/stack_core.py + return "Generic useful log header: \'{}\'".format(uuid.uuid4().hex) + + +class ExpensiveLineTransformerPool(ABC): + def __init__(self, max_restarts, pool_size, passthrough_on_failure): + self._max_restarts = max_restarts + self._pool = [self.CreateTransformer() for _ in range(pool_size)] + self._passthrough_on_failure = passthrough_on_failure + # Allow only one thread to select from the pool at a time. + self._lock = threading.Lock() + self._num_restarts = 0 + + def __enter__(self): + pass + + def __exit__(self, *args): + self.Close() + + def TransformLines(self, lines): + with self._lock: + assert self._pool, 'TransformLines() called on a closed Pool.' + + # transformation is broken. + if self._num_restarts == self._max_restarts: + if self._passthrough_on_failure: + return lines + raise Exception('%s is broken.' % self.name) + + # Restart any closed transformer. + for i, d in enumerate(self._pool): + if d.IsClosed(): + logging.warning('%s: Restarting closed instance.', self.name) + self._pool[i] = self.CreateTransformer() + self._num_restarts += 1 + if self._num_restarts == self._max_restarts: + logging.warning('%s: MAX_RESTARTS reached.', self.name) + if self._passthrough_on_failure: + return lines + raise Exception('%s is broken.' % self.name) + + selected = next((x for x in self._pool if x.IsReady()), self._pool[0]) + # Rotate the order so that next caller will not choose the same one. + self._pool.remove(selected) + self._pool.append(selected) + + return selected.TransformLines(lines) + + def Close(self): + with self._lock: + for d in self._pool: + d.Close() + self._pool = None + + @abstractmethod + def CreateTransformer(self): + ... + + @property + @abstractmethod + def name(self): + ...
diff --git a/build/android/pylib/symbols/mock_addr2line/mock_addr2line b/build/android/pylib/symbols/mock_addr2line/mock_addr2line index 8b2a723..431f387 100755 --- a/build/android/pylib/symbols/mock_addr2line/mock_addr2line +++ b/build/android/pylib/symbols/mock_addr2line/mock_addr2line
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2014 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,7 +9,6 @@ original address (so it is easy to double-check consistency in unittests). """ -from __future__ import print_function import optparse import os
diff --git a/build/android/pylib/symbols/stack_symbolizer.py b/build/android/pylib/symbols/stack_symbolizer.py index 4173741..e3203bf 100644 --- a/build/android/pylib/symbols/stack_symbolizer.py +++ b/build/android/pylib/symbols/stack_symbolizer.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,10 +10,18 @@ from devil.utils import cmd_helper from pylib import constants +from pylib.constants import host_paths +from .expensive_line_transformer import ExpensiveLineTransformer +from .expensive_line_transformer import ExpensiveLineTransformerPool -_STACK_TOOL = os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', - 'third_party', 'android_platform', 'development', - 'scripts', 'stack') +_STACK_TOOL = os.path.join(host_paths.ANDROID_PLATFORM_DEVELOPMENT_SCRIPTS_PATH, + 'stack') +_MINIMUM_TIMEOUT = 10.0 +_PER_LINE_TIMEOUT = .005 # Should be able to process 200 lines per second. +_PROCESS_START_TIMEOUT = 20.0 +_MAX_RESTARTS = 4 # Should be plenty unless tool is crashing on start-up. +_POOL_SIZE = 1 +_PASSTHROUH_ON_FAILURE = True ABI_REG = re.compile('ABI: \'(.+?)\'') @@ -27,7 +35,7 @@ raise RuntimeError('Unknown device ABI: %s' % device_abi) -class Symbolizer(object): +class Symbolizer: """A helper class to symbolize stack.""" def __init__(self, apk_under_test=None): @@ -72,7 +80,7 @@ constants.GetOutDirectory(), '--more-info'] env = dict(os.environ) env['PYTHONDONTWRITEBYTECODE'] = '1' - with tempfile.NamedTemporaryFile() as f: + with tempfile.NamedTemporaryFile(mode='w') as f: f.write('\n'.join(data_to_symbolize)) f.flush() start = time.time() @@ -84,3 +92,46 @@ if not include_stack and 'Stack Data:' in line: break yield line + + +class PassThroughSymbolizer(ExpensiveLineTransformer): + def __init__(self, device_abi): + self._command = None + super().__init__(_PROCESS_START_TIMEOUT, _MINIMUM_TIMEOUT, + _PER_LINE_TIMEOUT) + if not os.path.exists(_STACK_TOOL): + logging.warning('%s: %s missing. Unable to resolve native stack traces.', + PassThroughSymbolizer.name, _STACK_TOOL) + return + arch = _DeviceAbiToArch(device_abi) + if not arch: + logging.warning('%s: No device_abi can be found.', + PassThroughSymbolizer.name) + return + self._command = [ + _STACK_TOOL, '--arch', arch, '--output-directory', + constants.GetOutDirectory(), '--more-info', '--pass-through', '--flush', + '--quiet', '-' + ] + self.start() + + @property + def name(self): + return "symbolizer" + + @property + def command(self): + return self._command + + +class PassThroughSymbolizerPool(ExpensiveLineTransformerPool): + def __init__(self, device_abi): + self._device_abi = device_abi + super().__init__(_MAX_RESTARTS, _POOL_SIZE, _PASSTHROUH_ON_FAILURE) + + def CreateTransformer(self): + return PassThroughSymbolizer(self._device_abi) + + @property + def name(self): + return "symbolizer-pool"
diff --git a/build/android/pylib/symbols/symbol_utils.py b/build/android/pylib/symbols/symbol_utils.py deleted file mode 100644 index dea3c63..0000000 --- a/build/android/pylib/symbols/symbol_utils.py +++ /dev/null
@@ -1,814 +0,0 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -from __future__ import print_function - -import bisect -import collections -import logging -import os -import re - -from pylib.constants import host_paths -from pylib.symbols import elf_symbolizer - - -def _AndroidAbiToCpuArch(android_abi): - """Return the Chromium CPU architecture name for a given Android ABI.""" - _ARCH_MAP = { - 'armeabi': 'arm', - 'armeabi-v7a': 'arm', - 'arm64-v8a': 'arm64', - 'x86_64': 'x64', - } - return _ARCH_MAP.get(android_abi, android_abi) - - -def _HexAddressRegexpFor(android_abi): - """Return a regexp matching hexadecimal addresses for a given Android ABI.""" - if android_abi in ['x86_64', 'arm64-v8a', 'mips64']: - width = 16 - else: - width = 8 - return '[0-9a-f]{%d}' % width - - -class HostLibraryFinder(object): - """Translate device library path to matching host unstripped library path. - - Usage is the following: - 1) Create instance. - 2) Call AddSearchDir() once or more times to add host directory path to - look for unstripped native libraries. - 3) Call Find(device_libpath) repeatedly to translate a device-specific - library path into the corresponding host path to the unstripped - version. - """ - def __init__(self): - """Initialize instance.""" - self._search_dirs = [] - self._lib_map = {} # Map of library name to host file paths. - - def AddSearchDir(self, lib_dir): - """Add a directory to the search path for host native shared libraries. - - Args: - lib_dir: host path containing native libraries. - """ - if not os.path.exists(lib_dir): - logging.warning('Ignoring missing host library directory: %s', lib_dir) - return - if not os.path.isdir(lib_dir): - logging.warning('Ignoring invalid host library directory: %s', lib_dir) - return - self._search_dirs.append(lib_dir) - self._lib_map = {} # Reset the map. - - def Find(self, device_libpath): - """Find the host file path matching a specific device library path. - - Args: - device_libpath: device-specific file path to library or executable. - Returns: - host file path to the unstripped version of the library, or None. - """ - host_lib_path = None - lib_name = os.path.basename(device_libpath) - host_lib_path = self._lib_map.get(lib_name) - if not host_lib_path: - for search_dir in self._search_dirs: - lib_path = os.path.join(search_dir, lib_name) - if os.path.exists(lib_path): - host_lib_path = lib_path - break - - if not host_lib_path: - logging.debug('Could not find host library for: %s', lib_name) - self._lib_map[lib_name] = host_lib_path - - return host_lib_path - - - -class SymbolResolver(object): - """A base class for objets that can symbolize library (path, offset) - pairs into symbol information strings. Usage is the following: - - 1) Create new instance (by calling the constructor of a derived - class, since this is only the base one). - - 2) Call SetAndroidAbi() before any call to FindSymbolInfo() in order - to set the Android CPU ABI used for symbolization. - - 3) Before the first call to FindSymbolInfo(), one can call - AddLibraryOffset(), or AddLibraryOffsets() to record a set of offsets - that you will want to symbolize later through FindSymbolInfo(). Doing - so allows some SymbolResolver derived classes to work faster (e.g. the - one that invokes the 'addr2line' program, since the latter works faster - if the offsets provided as inputs are sorted in increasing order). - - 3) Call FindSymbolInfo(path, offset) to return the corresponding - symbol information string, or None if this doesn't correspond - to anything the instance can handle. - - Note that whether the path is specific to the device or to the - host depends on the derived class implementation. - """ - def __init__(self): - self._android_abi = None - self._lib_offsets_map = collections.defaultdict(set) - - def SetAndroidAbi(self, android_abi): - """Set the Android ABI value for this instance. - - Calling this function before FindSymbolInfo() is required by some - derived class implementations. - - Args: - android_abi: Native Android CPU ABI name (e.g. 'armeabi-v7a'). - Raises: - Exception if the ABI was already set with a different value. - """ - if self._android_abi and self._android_abi != android_abi: - raise Exception('Cannot reset Android ABI to new value %s, already set ' - 'to %s' % (android_abi, self._android_abi)) - - self._android_abi = android_abi - - def AddLibraryOffset(self, lib_path, offset): - """Associate a single offset to a given device library. - - This must be called before FindSymbolInfo(), otherwise its input arguments - will be ignored. - - Args: - lib_path: A library path. - offset: An integer offset within the corresponding library that will be - symbolized by future calls to FindSymbolInfo. - """ - self._lib_offsets_map[lib_path].add(offset) - - def AddLibraryOffsets(self, lib_path, lib_offsets): - """Associate a set of wanted offsets to a given device library. - - This must be called before FindSymbolInfo(), otherwise its input arguments - will be ignored. - - Args: - lib_path: A library path. - lib_offsets: An iterable of integer offsets within the corresponding - library that will be symbolized by future calls to FindSymbolInfo. - """ - self._lib_offsets_map[lib_path].update(lib_offsets) - - # pylint: disable=unused-argument,no-self-use - def FindSymbolInfo(self, lib_path, lib_offset): - """Symbolize a device library path and offset. - - Args: - lib_path: Library path (device or host specific, depending on the - derived class implementation). - lib_offset: Integer offset within the library. - Returns: - Corresponding symbol information string, or None. - """ - # The base implementation cannot symbolize anything. - return None - # pylint: enable=unused-argument,no-self-use - - -class ElfSymbolResolver(SymbolResolver): - """A SymbolResolver that can symbolize host path + offset values using - an elf_symbolizer.ELFSymbolizer instance. - """ - def __init__(self, addr2line_path_for_tests=None): - super(ElfSymbolResolver, self).__init__() - self._addr2line_path = addr2line_path_for_tests - - # Used to cache one ELFSymbolizer instance per library path. - self._elf_symbolizer_cache = {} - - # Used to cache FindSymbolInfo() results. Maps host library paths - # to (offset -> symbol info string) dictionaries. - self._symbol_info_cache = collections.defaultdict(dict) - self._allow_symbolizer = True - - def _CreateSymbolizerFor(self, host_path): - """Create the ELFSymbolizer instance associated with a given lib path.""" - addr2line_path = self._addr2line_path - if not addr2line_path: - if not self._android_abi: - raise Exception( - 'Android CPU ABI must be set before calling FindSymbolInfo!') - - cpu_arch = _AndroidAbiToCpuArch(self._android_abi) - self._addr2line_path = host_paths.ToolPath('addr2line', cpu_arch) - - return elf_symbolizer.ELFSymbolizer( - elf_file_path=host_path, addr2line_path=self._addr2line_path, - callback=ElfSymbolResolver._Callback, inlines=True) - - def DisallowSymbolizerForTesting(self): - """Disallow FindSymbolInfo() from using a symbolizer. - - This is used during unit-testing to ensure that the offsets that were - recorded via AddLibraryOffset()/AddLibraryOffsets() are properly - symbolized, but not anything else. - """ - self._allow_symbolizer = False - - def FindSymbolInfo(self, host_path, offset): - """Override SymbolResolver.FindSymbolInfo. - - Args: - host_path: Host-specific path to the native shared library. - offset: Integer offset within the native library. - Returns: - A symbol info string, or None. - """ - offset_map = self._symbol_info_cache[host_path] - symbol_info = offset_map.get(offset) - if symbol_info: - return symbol_info - - # Create symbolizer on demand. - symbolizer = self._elf_symbolizer_cache.get(host_path) - if not symbolizer: - symbolizer = self._CreateSymbolizerFor(host_path) - self._elf_symbolizer_cache[host_path] = symbolizer - - # If there are pre-recorded offsets for this path, symbolize them now. - offsets = self._lib_offsets_map.get(host_path) - if offsets: - offset_map = {} - for pre_offset in offsets: - symbolizer.SymbolizeAsync( - pre_offset, callback_arg=(offset_map, pre_offset)) - symbolizer.WaitForIdle() - self._symbol_info_cache[host_path] = offset_map - - symbol_info = offset_map.get(offset) - if symbol_info: - return symbol_info - - if not self._allow_symbolizer: - return None - - # Symbolize single offset. Slower if addresses are not provided in - # increasing order to addr2line. - symbolizer.SymbolizeAsync(offset, - callback_arg=(offset_map, offset)) - symbolizer.WaitForIdle() - return offset_map.get(offset) - - @staticmethod - def _Callback(sym_info, callback_arg): - offset_map, offset = callback_arg - offset_map[offset] = str(sym_info) - - -class DeviceSymbolResolver(SymbolResolver): - """A SymbolResolver instance that accepts device-specific path. - - Usage is the following: - 1) Create new instance, passing a parent SymbolResolver instance that - accepts host-specific paths, and a HostLibraryFinder instance. - - 2) Optional: call AddApkOffsets() to add offsets from within an APK - that contains uncompressed native shared libraries. - - 3) Use it as any SymbolResolver instance. - """ - def __init__(self, host_resolver, host_lib_finder): - """Initialize instance. - - Args: - host_resolver: A parent SymbolResolver instance that will be used - to resolve symbols from host library paths. - host_lib_finder: A HostLibraryFinder instance used to locate - unstripped libraries on the host. - """ - super(DeviceSymbolResolver, self).__init__() - self._host_lib_finder = host_lib_finder - self._bad_device_lib_paths = set() - self._host_resolver = host_resolver - - def SetAndroidAbi(self, android_abi): - super(DeviceSymbolResolver, self).SetAndroidAbi(android_abi) - self._host_resolver.SetAndroidAbi(android_abi) - - def AddLibraryOffsets(self, device_lib_path, lib_offsets): - """Associate a set of wanted offsets to a given device library. - - This must be called before FindSymbolInfo(), otherwise its input arguments - will be ignored. - - Args: - device_lib_path: A device-specific library path. - lib_offsets: An iterable of integer offsets within the corresponding - library that will be symbolized by future calls to FindSymbolInfo. - want to symbolize. - """ - if device_lib_path in self._bad_device_lib_paths: - return - - host_lib_path = self._host_lib_finder.Find(device_lib_path) - if not host_lib_path: - # NOTE: self._bad_device_lib_paths is only used to only print this - # warning once per bad library. - logging.warning('Could not find host library matching device path: %s', - device_lib_path) - self._bad_device_lib_paths.add(device_lib_path) - return - - self._host_resolver.AddLibraryOffsets(host_lib_path, lib_offsets) - - def AddApkOffsets(self, device_apk_path, apk_offsets, apk_translator): - """Associate a set of wanted offsets to a given device APK path. - - This converts the APK-relative offsets into offsets relative to the - uncompressed libraries it contains, then calls AddLibraryOffsets() - for each one of the libraries. - - Must be called before FindSymbolInfo() as well, otherwise input arguments - will be ignored. - - Args: - device_apk_path: Device-specific APK path. - apk_offsets: Iterable of offsets within the APK file. - apk_translator: An ApkLibraryPathTranslator instance used to extract - library paths from the APK. - """ - libraries_map = collections.defaultdict(set) - for offset in apk_offsets: - lib_path, lib_offset = apk_translator.TranslatePath(device_apk_path, - offset) - libraries_map[lib_path].add(lib_offset) - - for lib_path, lib_offsets in libraries_map.iteritems(): - self.AddLibraryOffsets(lib_path, lib_offsets) - - def FindSymbolInfo(self, device_path, offset): - """Overrides SymbolResolver.FindSymbolInfo. - - Args: - device_path: Device-specific library path (e.g. - '/data/app/com.example.app-1/lib/x86/libfoo.so') - offset: Offset in device library path. - Returns: - Corresponding symbol information string, or None. - """ - host_path = self._host_lib_finder.Find(device_path) - if not host_path: - return None - - return self._host_resolver.FindSymbolInfo(host_path, offset) - - -class MemoryMap(object): - """Models the memory map of a given process. Usage is: - - 1) Create new instance, passing the Android ABI. - - 2) Call TranslateLine() whenever you want to detect and translate any - memory map input line. - - 3) Otherwise, it is possible to parse the whole memory map input with - ParseLines(), then call FindSectionForAddress() repeatedly in order - to translate a memory address into the corresponding mapping and - file information tuple (e.g. to symbolize stack entries). - """ - - # A named tuple describing interesting memory map line items. - # Fields: - # addr_start: Mapping start address in memory. - # file_offset: Corresponding file offset. - # file_size: Corresponding mapping size in bytes. - # file_path: Input file path. - # match: Corresponding regular expression match object. - LineTuple = collections.namedtuple('MemoryMapLineTuple', - 'addr_start,file_offset,file_size,' - 'file_path, match') - - # A name tuple describing a memory map section. - # Fields: - # address: Memory address. - # size: Size in bytes in memory - # offset: Starting file offset. - # path: Input file path. - SectionTuple = collections.namedtuple('MemoryMapSection', - 'address,size,offset,path') - - def __init__(self, android_abi): - """Initializes instance. - - Args: - android_abi: Android CPU ABI name (e.g. 'armeabi-v7a') - """ - hex_addr = _HexAddressRegexpFor(android_abi) - - # pylint: disable=line-too-long - # A regular expression used to match memory map entries which look like: - # b278c000-b2790fff r-- 4fda000 5000 /data/app/com.google.android.apps.chrome-2/base.apk - # pylint: enable=line-too-long - self._re_map_section = re.compile( - r'\s*(?P<addr_start>' + hex_addr + r')-(?P<addr_end>' + hex_addr + ')' + - r'\s+' + - r'(?P<perm>...)\s+' + - r'(?P<file_offset>[0-9a-f]+)\s+' + - r'(?P<file_size>[0-9a-f]+)\s*' + - r'(?P<file_path>[^ \t]+)?') - - self._addr_map = [] # Sorted list of (address, size, path, offset) tuples. - self._sorted_addresses = [] # Sorted list of address fields in _addr_map. - self._in_section = False - - def TranslateLine(self, line, apk_path_translator): - """Try to translate a memory map input line, if detected. - - This only takes care of converting mapped APK file path and offsets - into a corresponding uncompressed native library file path + new offsets, - e.g. '..... <offset> <size> /data/.../base.apk' gets - translated into '.... <new-offset> <size> /data/.../base.apk!lib/libfoo.so' - - This function should always work, even if ParseLines() was not called - previously. - - Args: - line: Input memory map / tombstone line. - apk_translator: An ApkLibraryPathTranslator instance, used to map - APK offsets into uncompressed native libraries + new offsets. - Returns: - Translated memory map line, if relevant, or unchanged input line - otherwise. - """ - t = self._ParseLine(line.rstrip()) - if not t: - return line - - new_path, new_offset = apk_path_translator.TranslatePath( - t.file_path, t.file_offset) - - if new_path == t.file_path: - return line - - pos = t.match.start('file_path') - return '%s%s (offset 0x%x)%s' % (line[0:pos], new_path, new_offset, - line[t.match.end('file_path'):]) - - def ParseLines(self, input_lines, in_section=False): - """Parse a list of input lines and extract the APK memory map out of it. - - Args: - input_lines: list, or iterable, of input lines. - in_section: Optional. If true, considers that the input lines are - already part of the memory map. Otherwise, wait until the start of - the section appears in the input before trying to record data. - Returns: - True iff APK-related memory map entries were found. False otherwise. - """ - addr_list = [] # list of (address, size, file_path, file_offset) tuples. - self._in_section = in_section - for line in input_lines: - t = self._ParseLine(line.rstrip()) - if not t: - continue - - addr_list.append(t) - - self._addr_map = sorted(addr_list, - lambda x, y: cmp(x.addr_start, y.addr_start)) - self._sorted_addresses = [e.addr_start for e in self._addr_map] - return bool(self._addr_map) - - def _ParseLine(self, line): - """Used internally to recognized memory map input lines. - - Args: - line: Input logcat or tomstone line. - Returns: - A LineTuple instance on success, or None on failure. - """ - if not self._in_section: - self._in_section = line.startswith('memory map:') - return None - - m = self._re_map_section.match(line) - if not m: - self._in_section = False # End of memory map section - return None - - # Only accept .apk and .so files that are not from the system partitions. - file_path = m.group('file_path') - if not file_path: - return None - - if file_path.startswith('/system') or file_path.startswith('/vendor'): - return None - - if not (file_path.endswith('.apk') or file_path.endswith('.so')): - return None - - addr_start = int(m.group('addr_start'), 16) - file_offset = int(m.group('file_offset'), 16) - file_size = int(m.group('file_size'), 16) - - return self.LineTuple(addr_start, file_offset, file_size, file_path, m) - - def Dump(self): - """Print memory map for debugging.""" - print('MEMORY MAP [') - for t in self._addr_map: - print('[%08x-%08x %08x %08x %s]' % - (t.addr_start, t.addr_start + t.file_size, t.file_size, - t.file_offset, t.file_path)) - print('] MEMORY MAP') - - def FindSectionForAddress(self, addr): - """Find the map section corresponding to a specific memory address. - - Call this method only after using ParseLines() was called to extract - relevant information from the memory map. - - Args: - addr: Memory address - Returns: - A SectionTuple instance on success, or None on failure. - """ - pos = bisect.bisect_right(self._sorted_addresses, addr) - if pos > 0: - # All values in [0,pos) are <= addr, just ensure that the last - # one contains the address as well. - entry = self._addr_map[pos - 1] - if entry.addr_start + entry.file_size > addr: - return self.SectionTuple(entry.addr_start, entry.file_size, - entry.file_offset, entry.file_path) - return None - - -class BacktraceTranslator(object): - """Translates backtrace-related lines in a tombstone or crash report. - - Usage is the following: - 1) Create new instance with appropriate arguments. - 2) If the tombstone / logcat input is available, one can call - FindLibraryOffsets() in order to detect which library offsets - will need to be symbolized during a future parse. Doing so helps - speed up the ELF symbolizer. - 3) For each tombstone/logcat input line, call TranslateLine() to - try to detect and symbolize backtrace lines. - """ - - # A named tuple for relevant input backtrace lines. - # Fields: - # rel_pc: Instruction pointer, relative to offset in library start. - # location: Library or APK file path. - # offset: Load base of executable code in library or apk file path. - # match: The corresponding regular expression match object. - # Note: - # The actual instruction pointer always matches the position at - # |offset + rel_pc| in |location|. - LineTuple = collections.namedtuple('BacktraceLineTuple', - 'rel_pc,location,offset,match') - - def __init__(self, android_abi, apk_translator): - """Initialize instance. - - Args: - android_abi: Android CPU ABI name (e.g. 'armeabi-v7a'). - apk_translator: ApkLibraryPathTranslator instance used to convert - mapped APK file offsets into uncompressed library file paths with - new offsets. - """ - hex_addr = _HexAddressRegexpFor(android_abi) - - # A regular expression used to match backtrace lines. - self._re_backtrace = re.compile( - r'.*#(?P<frame>[0-9]{2})\s+' + - r'(..)\s+' + - r'(?P<rel_pc>' + hex_addr + r')\s+' + - r'(?P<location>[^ \t]+)' + - r'(\s+\(offset 0x(?P<offset>[0-9a-f]+)\))?') - - # In certain cases, offset will be provided as <location>+0x<offset> - # instead of <location> (offset 0x<offset>). This is a regexp to detect - # this. - self._re_location_offset = re.compile( - r'.*\+0x(?P<offset>[0-9a-f]+)$') - - self._apk_translator = apk_translator - self._in_section = False - - def _ParseLine(self, line): - """Used internally to detect and decompose backtrace input lines. - - Args: - line: input tombstone line. - Returns: - A LineTuple instance on success, None on failure. - """ - if not self._in_section: - self._in_section = line.startswith('backtrace:') - return None - - line = line.rstrip() - m = self._re_backtrace.match(line) - if not m: - self._in_section = False - return None - - location = m.group('location') - offset = m.group('offset') - if not offset: - m2 = self._re_location_offset.match(location) - if m2: - offset = m2.group('offset') - location = location[0:m2.start('offset') - 3] - - if not offset: - return None - - offset = int(offset, 16) - rel_pc = int(m.group('rel_pc'), 16) - - # Two cases to consider here: - # - # * If this is a library file directly mapped in memory, then |rel_pc| - # if the direct offset within the library, and doesn't need any kind - # of adjustement. - # - # * If this is a library mapped directly from an .apk file, then - # |rel_pc| is the offset in the APK, and |offset| happens to be the - # load base of the corresponding library. - # - if location.endswith('.so'): - # For a native library directly mapped from the file system, - return self.LineTuple(rel_pc, location, offset, m) - - if location.endswith('.apk'): - # For a native library inside an memory-mapped APK file, - new_location, new_offset = self._apk_translator.TranslatePath( - location, offset) - - return self.LineTuple(rel_pc, new_location, new_offset, m) - - # Ignore anything else (e.g. .oat or .odex files). - return None - - def FindLibraryOffsets(self, input_lines, in_section=False): - """Parse a tombstone's backtrace section and find all library offsets in it. - - Args: - input_lines: List or iterables of intput tombstone lines. - in_section: Optional. If True, considers that the stack section has - already started. - Returns: - A dictionary mapping device library paths to sets of offsets within - then. - """ - self._in_section = in_section - result = collections.defaultdict(set) - for line in input_lines: - t = self._ParseLine(line) - if not t: - continue - - result[t.location].add(t.offset + t.rel_pc) - return result - - def TranslateLine(self, line, symbol_resolver): - """Symbolize backtrace line if recognized. - - Args: - line: input backtrace line. - symbol_resolver: symbol resolver instance to use. This method will - call its FindSymbolInfo(device_lib_path, lib_offset) method to - convert offsets into symbol informations strings. - Returns: - Translated line (unchanged if not recognized as a back trace). - """ - t = self._ParseLine(line) - if not t: - return line - - symbol_info = symbol_resolver.FindSymbolInfo(t.location, - t.offset + t.rel_pc) - if not symbol_info: - symbol_info = 'offset 0x%x' % t.offset - - pos = t.match.start('location') - pos2 = t.match.end('offset') + 1 - if pos2 <= 0: - pos2 = t.match.end('location') - return '%s%s (%s)%s' % (line[:pos], t.location, symbol_info, line[pos2:]) - - -class StackTranslator(object): - """Translates stack-related lines in a tombstone or crash report.""" - - # A named tuple describing relevant stack input lines. - # Fields: - # address: Address as it appears in the stack. - # lib_path: Library path where |address| is mapped. - # lib_offset: Library load base offset. for |lib_path|. - # match: Corresponding regular expression match object. - LineTuple = collections.namedtuple('StackLineTuple', - 'address, lib_path, lib_offset, match') - - def __init__(self, android_abi, memory_map, apk_translator): - """Initialize instance.""" - hex_addr = _HexAddressRegexpFor(android_abi) - - # pylint: disable=line-too-long - # A regular expression used to recognize stack entries like: - # - # #05 bf89a180 bf89a1e4 [stack] - # bf89a1c8 a0c01c51 /data/app/com.google.android.apps.chrome-2/base.apk - # bf89a080 00000000 - # ........ ........ - # pylint: enable=line-too-long - self._re_stack_line = re.compile( - r'\s+(?P<frame_number>#[0-9]+)?\s*' + - r'(?P<stack_addr>' + hex_addr + r')\s+' + - r'(?P<stack_value>' + hex_addr + r')' + - r'(\s+(?P<location>[^ \t]+))?') - - self._re_stack_abbrev = re.compile(r'\s+[.]+\s+[.]+') - - self._memory_map = memory_map - self._apk_translator = apk_translator - self._in_section = False - - def _ParseLine(self, line): - """Check a given input line for a relevant _re_stack_line match. - - Args: - line: input tombstone line. - Returns: - A LineTuple instance on success, None on failure. - """ - line = line.rstrip() - if not self._in_section: - self._in_section = line.startswith('stack:') - return None - - m = self._re_stack_line.match(line) - if not m: - if not self._re_stack_abbrev.match(line): - self._in_section = False - return None - - location = m.group('location') - if not location: - return None - - if not location.endswith('.apk') and not location.endswith('.so'): - return None - - addr = int(m.group('stack_value'), 16) - t = self._memory_map.FindSectionForAddress(addr) - if t is None: - return None - - lib_path = t.path - lib_offset = t.offset + (addr - t.address) - - if lib_path.endswith('.apk'): - lib_path, lib_offset = self._apk_translator.TranslatePath( - lib_path, lib_offset) - - return self.LineTuple(addr, lib_path, lib_offset, m) - - def FindLibraryOffsets(self, input_lines, in_section=False): - """Parse a tombstone's stack section and find all library offsets in it. - - Args: - input_lines: List or iterables of intput tombstone lines. - in_section: Optional. If True, considers that the stack section has - already started. - Returns: - A dictionary mapping device library paths to sets of offsets within - then. - """ - result = collections.defaultdict(set) - self._in_section = in_section - for line in input_lines: - t = self._ParseLine(line) - if t: - result[t.lib_path].add(t.lib_offset) - return result - - def TranslateLine(self, line, symbol_resolver=None): - """Try to translate a line of the stack dump.""" - t = self._ParseLine(line) - if not t: - return line - - symbol_info = symbol_resolver.FindSymbolInfo(t.lib_path, t.lib_offset) - if not symbol_info: - return line - - pos = t.match.start('location') - pos2 = t.match.end('location') - return '%s%s (%s)%s' % (line[:pos], t.lib_path, symbol_info, line[pos2:])
diff --git a/build/android/pylib/symbols/symbol_utils_unittest.py b/build/android/pylib/symbols/symbol_utils_unittest.py deleted file mode 100644 index ed87f9e..0000000 --- a/build/android/pylib/symbols/symbol_utils_unittest.py +++ /dev/null
@@ -1,942 +0,0 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import collections -import contextlib -import logging -import os -import re -import shutil -import tempfile -import unittest - -from pylib.symbols import apk_native_libs_unittest -from pylib.symbols import mock_addr2line -from pylib.symbols import symbol_utils - -_MOCK_ELF_DATA = apk_native_libs_unittest.MOCK_ELF_DATA - -_MOCK_A2L_PATH = os.path.join(os.path.dirname(mock_addr2line.__file__), - 'mock_addr2line') - - -# pylint: disable=line-too-long - -# list of (start_offset, end_offset, size, libpath) tuples corresponding -# to the content of base.apk. This was taken from an x86 ChromeModern.apk -# component build. -_TEST_APK_LIBS = [ - (0x01331000, 0x013696bc, 0x000386bc, 'libaccessibility.cr.so'), - (0x0136a000, 0x013779c4, 0x0000d9c4, 'libanimation.cr.so'), - (0x01378000, 0x0137f7e8, 0x000077e8, 'libapdu.cr.so'), - (0x01380000, 0x0155ccc8, 0x001dccc8, 'libbase.cr.so'), - (0x0155d000, 0x015ab98c, 0x0004e98c, 'libbase_i18n.cr.so'), - (0x015ac000, 0x015dff4c, 0x00033f4c, 'libbindings.cr.so'), - (0x015e0000, 0x015f5a54, 0x00015a54, 'libbindings_base.cr.so'), - (0x0160e000, 0x01731960, 0x00123960, 'libblink_common.cr.so'), - (0x01732000, 0x0174ce54, 0x0001ae54, 'libblink_controller.cr.so'), - (0x0174d000, 0x0318c528, 0x01a3f528, 'libblink_core.cr.so'), - (0x0318d000, 0x03191700, 0x00004700, 'libblink_mojom_broadcastchannel_bindings_shared.cr.so'), - (0x03192000, 0x03cd7918, 0x00b45918, 'libblink_modules.cr.so'), - (0x03cd8000, 0x03d137d0, 0x0003b7d0, 'libblink_mojo_bindings_shared.cr.so'), - (0x03d14000, 0x03d2670c, 0x0001270c, 'libblink_offscreen_canvas_mojo_bindings_shared.cr.so'), - (0x03d27000, 0x046c7054, 0x009a0054, 'libblink_platform.cr.so'), - (0x046c8000, 0x0473fbfc, 0x00077bfc, 'libbluetooth.cr.so'), - (0x04740000, 0x04878f40, 0x00138f40, 'libboringssl.cr.so'), - (0x04879000, 0x0498466c, 0x0010b66c, 'libc++_shared.so'), - (0x04985000, 0x0498d93c, 0x0000893c, 'libcaptive_portal.cr.so'), - (0x0498e000, 0x049947cc, 0x000067cc, 'libcapture_base.cr.so'), - (0x04995000, 0x04b39f18, 0x001a4f18, 'libcapture_lib.cr.so'), - (0x04b3a000, 0x04b488ec, 0x0000e8ec, 'libcbor.cr.so'), - (0x04b49000, 0x04e9ea5c, 0x00355a5c, 'libcc.cr.so'), - (0x04e9f000, 0x04ed6404, 0x00037404, 'libcc_animation.cr.so'), - (0x04ed7000, 0x04ef5ab4, 0x0001eab4, 'libcc_base.cr.so'), - (0x04ef6000, 0x04fd9364, 0x000e3364, 'libcc_blink.cr.so'), - (0x04fda000, 0x04fe2758, 0x00008758, 'libcc_debug.cr.so'), - (0x04fe3000, 0x0500ae0c, 0x00027e0c, 'libcc_ipc.cr.so'), - (0x0500b000, 0x05078f38, 0x0006df38, 'libcc_paint.cr.so'), - (0x05079000, 0x0507e734, 0x00005734, 'libcdm_manager.cr.so'), - (0x0507f000, 0x06f4d744, 0x01ece744, 'libchrome.cr.so'), - (0x06f54000, 0x06feb830, 0x00097830, 'libchromium_sqlite3.cr.so'), - (0x06fec000, 0x0706f554, 0x00083554, 'libclient.cr.so'), - (0x07070000, 0x0708da60, 0x0001da60, 'libcloud_policy_proto_generated_compile.cr.so'), - (0x0708e000, 0x07121f28, 0x00093f28, 'libcodec.cr.so'), - (0x07122000, 0x07134ab8, 0x00012ab8, 'libcolor_space.cr.so'), - (0x07135000, 0x07138614, 0x00003614, 'libcommon.cr.so'), - (0x07139000, 0x0717c938, 0x00043938, 'libcompositor.cr.so'), - (0x0717d000, 0x0923d78c, 0x020c078c, 'libcontent.cr.so'), - (0x0923e000, 0x092ae87c, 0x0007087c, 'libcontent_common_mojo_bindings_shared.cr.so'), - (0x092af000, 0x092be718, 0x0000f718, 'libcontent_public_common_mojo_bindings_shared.cr.so'), - (0x092bf000, 0x092d9a20, 0x0001aa20, 'libcrash_key.cr.so'), - (0x092da000, 0x092eda58, 0x00013a58, 'libcrcrypto.cr.so'), - (0x092ee000, 0x092f16e0, 0x000036e0, 'libdevice_base.cr.so'), - (0x092f2000, 0x092fe8d8, 0x0000c8d8, 'libdevice_event_log.cr.so'), - (0x092ff000, 0x093026a4, 0x000036a4, 'libdevice_features.cr.so'), - (0x09303000, 0x093f1220, 0x000ee220, 'libdevice_gamepad.cr.so'), - (0x093f2000, 0x09437f54, 0x00045f54, 'libdevice_vr_mojo_bindings.cr.so'), - (0x09438000, 0x0954c168, 0x00114168, 'libdevice_vr_mojo_bindings_blink.cr.so'), - (0x0954d000, 0x0955d720, 0x00010720, 'libdevice_vr_mojo_bindings_shared.cr.so'), - (0x0955e000, 0x0956b9c0, 0x0000d9c0, 'libdevices.cr.so'), - (0x0956c000, 0x0957cae8, 0x00010ae8, 'libdiscardable_memory_client.cr.so'), - (0x0957d000, 0x09588854, 0x0000b854, 'libdiscardable_memory_common.cr.so'), - (0x09589000, 0x0959cbb4, 0x00013bb4, 'libdiscardable_memory_service.cr.so'), - (0x0959d000, 0x095b6b90, 0x00019b90, 'libdisplay.cr.so'), - (0x095b7000, 0x095be930, 0x00007930, 'libdisplay_types.cr.so'), - (0x095bf000, 0x095c46c4, 0x000056c4, 'libdisplay_util.cr.so'), - (0x095c5000, 0x095f54a4, 0x000304a4, 'libdomain_reliability.cr.so'), - (0x095f6000, 0x0966fe08, 0x00079e08, 'libembedder.cr.so'), - (0x09670000, 0x096735f8, 0x000035f8, 'libembedder_switches.cr.so'), - (0x09674000, 0x096a3460, 0x0002f460, 'libevents.cr.so'), - (0x096a4000, 0x096b6d40, 0x00012d40, 'libevents_base.cr.so'), - (0x096b7000, 0x0981a778, 0x00163778, 'libffmpeg.cr.so'), - (0x0981b000, 0x09945c94, 0x0012ac94, 'libfido.cr.so'), - (0x09946000, 0x09a330dc, 0x000ed0dc, 'libfingerprint.cr.so'), - (0x09a34000, 0x09b53170, 0x0011f170, 'libfreetype_harfbuzz.cr.so'), - (0x09b54000, 0x09bc5c5c, 0x00071c5c, 'libgcm.cr.so'), - (0x09bc6000, 0x09cc8584, 0x00102584, 'libgeolocation.cr.so'), - (0x09cc9000, 0x09cdc8d4, 0x000138d4, 'libgeometry.cr.so'), - (0x09cdd000, 0x09cec8b4, 0x0000f8b4, 'libgeometry_skia.cr.so'), - (0x09ced000, 0x09d10e14, 0x00023e14, 'libgesture_detection.cr.so'), - (0x09d11000, 0x09d7595c, 0x0006495c, 'libgfx.cr.so'), - (0x09d76000, 0x09d7d7cc, 0x000077cc, 'libgfx_ipc.cr.so'), - (0x09d7e000, 0x09d82708, 0x00004708, 'libgfx_ipc_buffer_types.cr.so'), - (0x09d83000, 0x09d89748, 0x00006748, 'libgfx_ipc_color.cr.so'), - (0x09d8a000, 0x09d8f6f4, 0x000056f4, 'libgfx_ipc_geometry.cr.so'), - (0x09d90000, 0x09d94754, 0x00004754, 'libgfx_ipc_skia.cr.so'), - (0x09d95000, 0x09d9869c, 0x0000369c, 'libgfx_switches.cr.so'), - (0x09d99000, 0x09dba0ac, 0x000210ac, 'libgin.cr.so'), - (0x09dbb000, 0x09e0a8cc, 0x0004f8cc, 'libgl_in_process_context.cr.so'), - (0x09e0b000, 0x09e17a18, 0x0000ca18, 'libgl_init.cr.so'), - (0x09e18000, 0x09ee34e4, 0x000cb4e4, 'libgl_wrapper.cr.so'), - (0x09ee4000, 0x0a1a2e00, 0x002bee00, 'libgles2.cr.so'), - (0x0a1a3000, 0x0a24556c, 0x000a256c, 'libgles2_implementation.cr.so'), - (0x0a246000, 0x0a267038, 0x00021038, 'libgles2_utils.cr.so'), - (0x0a268000, 0x0a3288e4, 0x000c08e4, 'libgpu.cr.so'), - (0x0a329000, 0x0a3627ec, 0x000397ec, 'libgpu_ipc_service.cr.so'), - (0x0a363000, 0x0a388a18, 0x00025a18, 'libgpu_util.cr.so'), - (0x0a389000, 0x0a506d8c, 0x0017dd8c, 'libhost.cr.so'), - (0x0a507000, 0x0a6f0ec0, 0x001e9ec0, 'libicui18n.cr.so'), - (0x0a6f1000, 0x0a83b4c8, 0x0014a4c8, 'libicuuc.cr.so'), - (0x0a83c000, 0x0a8416e4, 0x000056e4, 'libinterfaces_shared.cr.so'), - (0x0a842000, 0x0a87e2a0, 0x0003c2a0, 'libipc.cr.so'), - (0x0a87f000, 0x0a88c98c, 0x0000d98c, 'libipc_mojom.cr.so'), - (0x0a88d000, 0x0a8926e4, 0x000056e4, 'libipc_mojom_shared.cr.so'), - (0x0a893000, 0x0a8a1e18, 0x0000ee18, 'libkeyed_service_content.cr.so'), - (0x0a8a2000, 0x0a8b4a30, 0x00012a30, 'libkeyed_service_core.cr.so'), - (0x0a8b5000, 0x0a930a80, 0x0007ba80, 'libleveldatabase.cr.so'), - (0x0a931000, 0x0a9b3908, 0x00082908, 'libmanager.cr.so'), - (0x0a9b4000, 0x0aea9bb4, 0x004f5bb4, 'libmedia.cr.so'), - (0x0aeaa000, 0x0b08cb88, 0x001e2b88, 'libmedia_blink.cr.so'), - (0x0b08d000, 0x0b0a4728, 0x00017728, 'libmedia_devices_mojo_bindings_shared.cr.so'), - (0x0b0a5000, 0x0b1943ec, 0x000ef3ec, 'libmedia_gpu.cr.so'), - (0x0b195000, 0x0b2d07d4, 0x0013b7d4, 'libmedia_mojo_services.cr.so'), - (0x0b2d1000, 0x0b2d4760, 0x00003760, 'libmessage_center.cr.so'), - (0x0b2d5000, 0x0b2e0938, 0x0000b938, 'libmessage_support.cr.so'), - (0x0b2e1000, 0x0b2f3ad0, 0x00012ad0, 'libmetrics_cpp.cr.so'), - (0x0b2f4000, 0x0b313bb8, 0x0001fbb8, 'libmidi.cr.so'), - (0x0b314000, 0x0b31b848, 0x00007848, 'libmojo_base_lib.cr.so'), - (0x0b31c000, 0x0b3329f8, 0x000169f8, 'libmojo_base_mojom.cr.so'), - (0x0b333000, 0x0b34b98c, 0x0001898c, 'libmojo_base_mojom_blink.cr.so'), - (0x0b34c000, 0x0b354700, 0x00008700, 'libmojo_base_mojom_shared.cr.so'), - (0x0b355000, 0x0b3608b0, 0x0000b8b0, 'libmojo_base_shared_typemap_traits.cr.so'), - (0x0b361000, 0x0b3ad454, 0x0004c454, 'libmojo_edk.cr.so'), - (0x0b3ae000, 0x0b3c4a20, 0x00016a20, 'libmojo_edk_ports.cr.so'), - (0x0b3c5000, 0x0b3d38a0, 0x0000e8a0, 'libmojo_mojom_bindings.cr.so'), - (0x0b3d4000, 0x0b3da6e8, 0x000066e8, 'libmojo_mojom_bindings_shared.cr.so'), - (0x0b3db000, 0x0b3e27f0, 0x000077f0, 'libmojo_public_system.cr.so'), - (0x0b3e3000, 0x0b3fa9fc, 0x000179fc, 'libmojo_public_system_cpp.cr.so'), - (0x0b3fb000, 0x0b407728, 0x0000c728, 'libmojom_core_shared.cr.so'), - (0x0b408000, 0x0b421744, 0x00019744, 'libmojom_platform_shared.cr.so'), - (0x0b422000, 0x0b43451c, 0x0001251c, 'libnative_theme.cr.so'), - (0x0b435000, 0x0baaa1bc, 0x006751bc, 'libnet.cr.so'), - (0x0bac4000, 0x0bb74670, 0x000b0670, 'libnetwork_cpp.cr.so'), - (0x0bb75000, 0x0bbaee8c, 0x00039e8c, 'libnetwork_cpp_base.cr.so'), - (0x0bbaf000, 0x0bd21844, 0x00172844, 'libnetwork_service.cr.so'), - (0x0bd22000, 0x0bd256e4, 0x000036e4, 'libnetwork_session_configurator.cr.so'), - (0x0bd26000, 0x0bd33734, 0x0000d734, 'libonc.cr.so'), - (0x0bd34000, 0x0bd9ce18, 0x00068e18, 'libperfetto.cr.so'), - (0x0bd9d000, 0x0bda4854, 0x00007854, 'libplatform.cr.so'), - (0x0bda5000, 0x0bec5ce4, 0x00120ce4, 'libpolicy_component.cr.so'), - (0x0bec6000, 0x0bf5ab58, 0x00094b58, 'libpolicy_proto.cr.so'), - (0x0bf5b000, 0x0bf86fbc, 0x0002bfbc, 'libprefs.cr.so'), - (0x0bf87000, 0x0bfa5d74, 0x0001ed74, 'libprinting.cr.so'), - (0x0bfa6000, 0x0bfe0e80, 0x0003ae80, 'libprotobuf_lite.cr.so'), - (0x0bfe1000, 0x0bff0a18, 0x0000fa18, 'libproxy_config.cr.so'), - (0x0bff1000, 0x0c0f6654, 0x00105654, 'libpublic.cr.so'), - (0x0c0f7000, 0x0c0fa6a4, 0x000036a4, 'librange.cr.so'), - (0x0c0fb000, 0x0c118058, 0x0001d058, 'libraster.cr.so'), - (0x0c119000, 0x0c133d00, 0x0001ad00, 'libresource_coordinator_cpp.cr.so'), - (0x0c134000, 0x0c1396a0, 0x000056a0, 'libresource_coordinator_cpp_base.cr.so'), - (0x0c13a000, 0x0c1973b8, 0x0005d3b8, 'libresource_coordinator_public_mojom.cr.so'), - (0x0c198000, 0x0c2033e8, 0x0006b3e8, 'libresource_coordinator_public_mojom_blink.cr.so'), - (0x0c204000, 0x0c219744, 0x00015744, 'libresource_coordinator_public_mojom_shared.cr.so'), - (0x0c21a000, 0x0c21e700, 0x00004700, 'libsandbox.cr.so'), - (0x0c21f000, 0x0c22f96c, 0x0001096c, 'libsandbox_services.cr.so'), - (0x0c230000, 0x0c249d58, 0x00019d58, 'libseccomp_bpf.cr.so'), - (0x0c24a000, 0x0c24e714, 0x00004714, 'libseccomp_starter_android.cr.so'), - (0x0c24f000, 0x0c4ae9f0, 0x0025f9f0, 'libservice.cr.so'), - (0x0c4af000, 0x0c4c3ae4, 0x00014ae4, 'libservice_manager_cpp.cr.so'), - (0x0c4c4000, 0x0c4cb708, 0x00007708, 'libservice_manager_cpp_types.cr.so'), - (0x0c4cc000, 0x0c4fbe30, 0x0002fe30, 'libservice_manager_mojom.cr.so'), - (0x0c4fc000, 0x0c532e78, 0x00036e78, 'libservice_manager_mojom_blink.cr.so'), - (0x0c533000, 0x0c53669c, 0x0000369c, 'libservice_manager_mojom_constants.cr.so'), - (0x0c537000, 0x0c53e85c, 0x0000785c, 'libservice_manager_mojom_constants_blink.cr.so'), - (0x0c53f000, 0x0c542668, 0x00003668, 'libservice_manager_mojom_constants_shared.cr.so'), - (0x0c543000, 0x0c54d700, 0x0000a700, 'libservice_manager_mojom_shared.cr.so'), - (0x0c54e000, 0x0c8fc6ec, 0x003ae6ec, 'libsessions.cr.so'), - (0x0c8fd000, 0x0c90a924, 0x0000d924, 'libshared_memory_support.cr.so'), - (0x0c90b000, 0x0c9148ec, 0x000098ec, 'libshell_dialogs.cr.so'), - (0x0c915000, 0x0cf8de70, 0x00678e70, 'libskia.cr.so'), - (0x0cf8e000, 0x0cf978bc, 0x000098bc, 'libsnapshot.cr.so'), - (0x0cf98000, 0x0cfb7d9c, 0x0001fd9c, 'libsql.cr.so'), - (0x0cfb8000, 0x0cfbe744, 0x00006744, 'libstartup_tracing.cr.so'), - (0x0cfbf000, 0x0d19b4e4, 0x001dc4e4, 'libstorage_browser.cr.so'), - (0x0d19c000, 0x0d2a773c, 0x0010b73c, 'libstorage_common.cr.so'), - (0x0d2a8000, 0x0d2ac6fc, 0x000046fc, 'libsurface.cr.so'), - (0x0d2ad000, 0x0d2baa98, 0x0000da98, 'libtracing.cr.so'), - (0x0d2bb000, 0x0d2f36b0, 0x000386b0, 'libtracing_cpp.cr.so'), - (0x0d2f4000, 0x0d326e70, 0x00032e70, 'libtracing_mojom.cr.so'), - (0x0d327000, 0x0d33270c, 0x0000b70c, 'libtracing_mojom_shared.cr.so'), - (0x0d333000, 0x0d46d804, 0x0013a804, 'libui_android.cr.so'), - (0x0d46e000, 0x0d4cb3f8, 0x0005d3f8, 'libui_base.cr.so'), - (0x0d4cc000, 0x0d4dbc40, 0x0000fc40, 'libui_base_ime.cr.so'), - (0x0d4dc000, 0x0d4e58d4, 0x000098d4, 'libui_data_pack.cr.so'), - (0x0d4e6000, 0x0d51d1e0, 0x000371e0, 'libui_devtools.cr.so'), - (0x0d51e000, 0x0d52b984, 0x0000d984, 'libui_message_center_cpp.cr.so'), - (0x0d52c000, 0x0d539a48, 0x0000da48, 'libui_touch_selection.cr.so'), - (0x0d53a000, 0x0d55bc60, 0x00021c60, 'liburl.cr.so'), - (0x0d55c000, 0x0d55f6b4, 0x000036b4, 'liburl_ipc.cr.so'), - (0x0d560000, 0x0d5af110, 0x0004f110, 'liburl_matcher.cr.so'), - (0x0d5b0000, 0x0d5e2fac, 0x00032fac, 'libuser_manager.cr.so'), - (0x0d5e3000, 0x0d5e66e4, 0x000036e4, 'libuser_prefs.cr.so'), - (0x0d5e7000, 0x0e3e1cc8, 0x00dfacc8, 'libv8.cr.so'), - (0x0e3e2000, 0x0e400ae0, 0x0001eae0, 'libv8_libbase.cr.so'), - (0x0e401000, 0x0e4d91d4, 0x000d81d4, 'libviz_common.cr.so'), - (0x0e4da000, 0x0e4df7e4, 0x000057e4, 'libviz_resource_format.cr.so'), - (0x0e4e0000, 0x0e5b7120, 0x000d7120, 'libweb_dialogs.cr.so'), - (0x0e5b8000, 0x0e5c7a18, 0x0000fa18, 'libwebdata_common.cr.so'), - (0x0e5c8000, 0x0e61bfe4, 0x00053fe4, 'libwtf.cr.so'), -] - - -# A small memory map fragment extracted from a tombstone for a process that -# had loaded the APK corresponding to _TEST_APK_LIBS above. -_TEST_MEMORY_MAP = r'''memory map: -12c00000-12ccafff rw- 0 cb000 /dev/ashmem/dalvik-main space (deleted) -12ccb000-130cafff rw- cb000 400000 /dev/ashmem/dalvik-main space (deleted) -130cb000-32bfffff --- 4cb000 1fb35000 /dev/ashmem/dalvik-main space (deleted) -32c00000-32c00fff rw- 0 1000 /dev/ashmem/dalvik-main space 1 (deleted) -32c01000-52bfffff --- 1000 1ffff000 /dev/ashmem/dalvik-main space 1 (deleted) -6f3b8000-6fd90fff rw- 0 9d9000 /data/dalvik-cache/x86/system@framework@boot.art -6fd91000-71c42fff r-- 0 1eb2000 /data/dalvik-cache/x86/system@framework@boot.oat -71c43000-7393efff r-x 1eb2000 1cfc000 /data/dalvik-cache/x86/system@framework@boot.oat (load base 0x71c43000) -7393f000-7393ffff rw- 3bae000 1000 /data/dalvik-cache/x86/system@framework@boot.oat -73940000-73a1bfff rw- 0 dc000 /dev/ashmem/dalvik-zygote space (deleted) -73a1c000-73a1cfff rw- 0 1000 /dev/ashmem/dalvik-non moving space (deleted) -73a1d000-73a2dfff rw- 1000 11000 /dev/ashmem/dalvik-non moving space (deleted) -73a2e000-77540fff --- 12000 3b13000 /dev/ashmem/dalvik-non moving space (deleted) -77541000-7793ffff rw- 3b25000 3ff000 /dev/ashmem/dalvik-non moving space (deleted) -923aa000-92538fff r-- 8a9000 18f000 /data/app/com.example.app-2/base.apk -92539000-9255bfff r-- 0 23000 /data/data/com.example.app/app_data/paks/es.pak@162db1c6689 -9255c000-92593fff r-- 213000 38000 /data/app/com.example.app-2/base.apk -92594000-925c0fff r-- 87d000 2d000 /data/app/com.example.app-2/base.apk -925c1000-927d3fff r-- a37000 213000 /data/app/com.example.app-2/base.apk -927d4000-92e07fff r-- 24a000 634000 /data/app/com.example.app-2/base.apk -92e08000-92e37fff r-- a931000 30000 /data/app/com.example.app-2/base.apk -92e38000-92e86fff r-x a961000 4f000 /data/app/com.example.app-2/base.apk -92e87000-92e8afff rw- a9b0000 4000 /data/app/com.example.app-2/base.apk -92e8b000-92e8bfff rw- 0 1000 -92e8c000-92e9dfff r-- d5b0000 12000 /data/app/com.example.app-2/base.apk -92e9e000-92ebcfff r-x d5c2000 1f000 /data/app/com.example.app-2/base.apk -92ebd000-92ebefff rw- d5e1000 2000 /data/app/com.example.app-2/base.apk -92ebf000-92ebffff rw- 0 1000 -''' - -# list of (address, size, path, offset) tuples that must appear in -# _TEST_MEMORY_MAP. Not all sections need to be listed. -_TEST_MEMORY_MAP_SECTIONS = [ - (0x923aa000, 0x18f000, '/data/app/com.example.app-2/base.apk', 0x8a9000), - (0x9255c000, 0x038000, '/data/app/com.example.app-2/base.apk', 0x213000), - (0x92594000, 0x02d000, '/data/app/com.example.app-2/base.apk', 0x87d000), - (0x925c1000, 0x213000, '/data/app/com.example.app-2/base.apk', 0xa37000), -] - -_EXPECTED_TEST_MEMORY_MAP = r'''memory map: -12c00000-12ccafff rw- 0 cb000 /dev/ashmem/dalvik-main space (deleted) -12ccb000-130cafff rw- cb000 400000 /dev/ashmem/dalvik-main space (deleted) -130cb000-32bfffff --- 4cb000 1fb35000 /dev/ashmem/dalvik-main space (deleted) -32c00000-32c00fff rw- 0 1000 /dev/ashmem/dalvik-main space 1 (deleted) -32c01000-52bfffff --- 1000 1ffff000 /dev/ashmem/dalvik-main space 1 (deleted) -6f3b8000-6fd90fff rw- 0 9d9000 /data/dalvik-cache/x86/system@framework@boot.art -6fd91000-71c42fff r-- 0 1eb2000 /data/dalvik-cache/x86/system@framework@boot.oat -71c43000-7393efff r-x 1eb2000 1cfc000 /data/dalvik-cache/x86/system@framework@boot.oat (load base 0x71c43000) -7393f000-7393ffff rw- 3bae000 1000 /data/dalvik-cache/x86/system@framework@boot.oat -73940000-73a1bfff rw- 0 dc000 /dev/ashmem/dalvik-zygote space (deleted) -73a1c000-73a1cfff rw- 0 1000 /dev/ashmem/dalvik-non moving space (deleted) -73a1d000-73a2dfff rw- 1000 11000 /dev/ashmem/dalvik-non moving space (deleted) -73a2e000-77540fff --- 12000 3b13000 /dev/ashmem/dalvik-non moving space (deleted) -77541000-7793ffff rw- 3b25000 3ff000 /dev/ashmem/dalvik-non moving space (deleted) -923aa000-92538fff r-- 8a9000 18f000 /data/app/com.example.app-2/base.apk -92539000-9255bfff r-- 0 23000 /data/data/com.example.app/app_data/paks/es.pak@162db1c6689 -9255c000-92593fff r-- 213000 38000 /data/app/com.example.app-2/base.apk -92594000-925c0fff r-- 87d000 2d000 /data/app/com.example.app-2/base.apk -925c1000-927d3fff r-- a37000 213000 /data/app/com.example.app-2/base.apk -927d4000-92e07fff r-- 24a000 634000 /data/app/com.example.app-2/base.apk -92e08000-92e37fff r-- a931000 30000 /data/app/com.example.app-2/base.apk!lib/libmanager.cr.so (offset 0x0) -92e38000-92e86fff r-x a961000 4f000 /data/app/com.example.app-2/base.apk!lib/libmanager.cr.so (offset 0x30000) -92e87000-92e8afff rw- a9b0000 4000 /data/app/com.example.app-2/base.apk!lib/libmanager.cr.so (offset 0x7f000) -92e8b000-92e8bfff rw- 0 1000 -92e8c000-92e9dfff r-- d5b0000 12000 /data/app/com.example.app-2/base.apk!lib/libuser_manager.cr.so (offset 0x0) -92e9e000-92ebcfff r-x d5c2000 1f000 /data/app/com.example.app-2/base.apk!lib/libuser_manager.cr.so (offset 0x12000) -92ebd000-92ebefff rw- d5e1000 2000 /data/app/com.example.app-2/base.apk!lib/libuser_manager.cr.so (offset 0x31000) -92ebf000-92ebffff rw- 0 1000 -''' - -# Example stack section, taken from the same tombstone that _TEST_MEMORY_MAP -# was extracted from. -_TEST_STACK = r'''stack: - bf89a070 b7439468 /system/lib/libc.so - bf89a074 bf89a1e4 [stack] - bf89a078 932d4000 /data/app/com.example.app-2/base.apk - bf89a07c b73bfbc9 /system/lib/libc.so (pthread_mutex_lock+65) - bf89a080 00000000 - bf89a084 4000671c /dev/ashmem/dalvik-main space 1 (deleted) - bf89a088 932d1d86 /data/app/com.example.app-2/base.apk - bf89a08c b743671c /system/lib/libc.so - bf89a090 b77f8c00 /system/bin/linker - bf89a094 b743cc90 - bf89a098 932d1d4a /data/app/com.example.app-2/base.apk - bf89a09c b73bf271 /system/lib/libc.so (__pthread_internal_find(long)+65) - bf89a0a0 b743cc90 - bf89a0a4 bf89a0b0 [stack] - bf89a0a8 bf89a0b8 [stack] - bf89a0ac 00000008 - ........ ........ - #00 bf89a0b0 00000006 - bf89a0b4 00000002 - bf89a0b8 b743671c /system/lib/libc.so - bf89a0bc b73bf5d9 /system/lib/libc.so (pthread_kill+71) - #01 bf89a0c0 00006937 - bf89a0c4 00006937 - bf89a0c8 00000006 - bf89a0cc b77fd3a9 /system/bin/app_process32 (sigprocmask+141) - bf89a0d0 00000002 - bf89a0d4 bf89a0ec [stack] - bf89a0d8 00000000 - bf89a0dc b743671c /system/lib/libc.so - bf89a0e0 bf89a12c [stack] - bf89a0e4 bf89a1e4 [stack] - bf89a0e8 932d1d4a /data/app/com.example.app-2/base.apk - bf89a0ec b7365206 /system/lib/libc.so (raise+37) - #02 bf89a0f0 b77f8c00 /system/bin/linker - bf89a0f4 00000006 - bf89a0f8 b7439468 /system/lib/libc.so - bf89a0fc b743671c /system/lib/libc.so - bf89a100 bf89a12c [stack] - bf89a104 b743671c /system/lib/libc.so - bf89a108 bf89a12c [stack] - bf89a10c b735e9e5 /system/lib/libc.so (abort+81) - #03 bf89a110 00000006 - bf89a114 bf89a12c [stack] - bf89a118 00000000 - bf89a11c b55a3d3b /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::DefaultLogHandler(google::protobuf::LogLevel, char const*, int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)+99) - bf89a120 b7439468 /system/lib/libc.so - bf89a124 b55ba38d /system/lib/libprotobuf-cpp-lite.so - bf89a128 b55ba408 /system/lib/libprotobuf-cpp-lite.so - bf89a12c ffffffdf - bf89a130 0000003d - bf89a134 adfedf00 [anon:libc_malloc] - bf89a138 bf89a158 [stack] - #04 bf89a13c a0cee7f0 /data/app/com.example.app-2/base.apk - bf89a140 b55c1cb0 /system/lib/libprotobuf-cpp-lite.so - bf89a144 bf89a1e4 [stack] -''' - -# Expected value of _TEST_STACK after translation of addresses in the APK -# into offsets into libraries. -_EXPECTED_STACK = r'''stack: - bf89a070 b7439468 /system/lib/libc.so - bf89a074 bf89a1e4 [stack] - bf89a078 932d4000 /data/app/com.example.app-2/base.apk - bf89a07c b73bfbc9 /system/lib/libc.so (pthread_mutex_lock+65) - bf89a080 00000000 - bf89a084 4000671c /dev/ashmem/dalvik-main space 1 (deleted) - bf89a088 932d1d86 /data/app/com.example.app-2/base.apk - bf89a08c b743671c /system/lib/libc.so - bf89a090 b77f8c00 /system/bin/linker - bf89a094 b743cc90 - bf89a098 932d1d4a /data/app/com.example.app-2/base.apk - bf89a09c b73bf271 /system/lib/libc.so (__pthread_internal_find(long)+65) - bf89a0a0 b743cc90 - bf89a0a4 bf89a0b0 [stack] - bf89a0a8 bf89a0b8 [stack] - bf89a0ac 00000008 - ........ ........ - #00 bf89a0b0 00000006 - bf89a0b4 00000002 - bf89a0b8 b743671c /system/lib/libc.so - bf89a0bc b73bf5d9 /system/lib/libc.so (pthread_kill+71) - #01 bf89a0c0 00006937 - bf89a0c4 00006937 - bf89a0c8 00000006 - bf89a0cc b77fd3a9 /system/bin/app_process32 (sigprocmask+141) - bf89a0d0 00000002 - bf89a0d4 bf89a0ec [stack] - bf89a0d8 00000000 - bf89a0dc b743671c /system/lib/libc.so - bf89a0e0 bf89a12c [stack] - bf89a0e4 bf89a1e4 [stack] - bf89a0e8 932d1d4a /data/app/com.example.app-2/base.apk - bf89a0ec b7365206 /system/lib/libc.so (raise+37) - #02 bf89a0f0 b77f8c00 /system/bin/linker - bf89a0f4 00000006 - bf89a0f8 b7439468 /system/lib/libc.so - bf89a0fc b743671c /system/lib/libc.so - bf89a100 bf89a12c [stack] - bf89a104 b743671c /system/lib/libc.so - bf89a108 bf89a12c [stack] - bf89a10c b735e9e5 /system/lib/libc.so (abort+81) - #03 bf89a110 00000006 - bf89a114 bf89a12c [stack] - bf89a118 00000000 - bf89a11c b55a3d3b /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::DefaultLogHandler(google::protobuf::LogLevel, char const*, int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)+99) - bf89a120 b7439468 /system/lib/libc.so - bf89a124 b55ba38d /system/lib/libprotobuf-cpp-lite.so - bf89a128 b55ba408 /system/lib/libprotobuf-cpp-lite.so - bf89a12c ffffffdf - bf89a130 0000003d - bf89a134 adfedf00 [anon:libc_malloc] - bf89a138 bf89a158 [stack] - #04 bf89a13c a0cee7f0 /data/app/com.example.app-2/base.apk - bf89a140 b55c1cb0 /system/lib/libprotobuf-cpp-lite.so - bf89a144 bf89a1e4 [stack] -''' - -_TEST_BACKTRACE = r'''backtrace: - #00 pc 00084126 /system/lib/libc.so (tgkill+22) - #01 pc 000815d8 /system/lib/libc.so (pthread_kill+70) - #02 pc 00027205 /system/lib/libc.so (raise+36) - #03 pc 000209e4 /system/lib/libc.so (abort+80) - #04 pc 0000cf73 /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::LogMessage::Finish()+117) - #05 pc 0000cf8e /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::LogFinisher::operator=(google::protobuf::internal::LogMessage&)+26) - #06 pc 0000d27f /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::VerifyVersion(int, int, char const*)+574) - #07 pc 007cd236 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #08 pc 000111a9 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0xbfc2000) - #09 pc 00013228 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0xbfc2000) - #10 pc 000131de /data/app/com.google.android.apps.chrome-2/base.apk (offset 0xbfc2000) - #11 pc 007cd2d8 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #12 pc 007cd956 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #13 pc 007c2d4a /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #14 pc 009fc9f1 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #15 pc 009fc8ea /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #16 pc 00561c63 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #17 pc 0106fbdb /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #18 pc 004d7371 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #19 pc 004d8159 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #20 pc 004d7b96 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #21 pc 004da4b6 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #22 pc 005ab66c /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x7daa000) - #23 pc 005afca2 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x7daa000) - #24 pc 0000cae8 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x598d000) - #25 pc 00ce864f /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x7daa000) - #26 pc 00ce8dfa /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x7daa000) - #27 pc 00ce74c6 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x7daa000) - #28 pc 00004616 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x961e000) - #29 pc 00ce8215 /data/app/com.google.android.apps.chrome-2/base.apk (offset 0x7daa000) - #30 pc 0013d8c7 /system/lib/libart.so (art_quick_generic_jni_trampoline+71) - #31 pc 00137c52 /system/lib/libart.so (art_quick_invoke_static_stub+418) - #32 pc 00143651 /system/lib/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+353) - #33 pc 005e06ae /system/lib/libart.so (artInterpreterToCompiledCodeBridge+190) - #34 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #35 pc 0032cfc0 /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)0, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+160) - #36 pc 000fc703 /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+29891) - #37 pc 00300af7 /system/lib/libart.so (artInterpreterToInterpreterBridge+188) - #38 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #39 pc 0032cfc0 /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)0, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+160) - #40 pc 000fc703 /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+29891) - #41 pc 00300af7 /system/lib/libart.so (artInterpreterToInterpreterBridge+188) - #42 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #43 pc 0032ebf9 /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)2, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+297) - #44 pc 000fc955 /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+30485) - #45 pc 00300af7 /system/lib/libart.so (artInterpreterToInterpreterBridge+188) - #46 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #47 pc 0033090c /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)4, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+636) - #48 pc 000fc67f /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+29759) - #49 pc 00300700 /system/lib/libart.so (art::interpreter::EnterInterpreterFromEntryPoint(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame*)+128) - #50 pc 00667c73 /system/lib/libart.so (artQuickToInterpreterBridge+808) - #51 pc 0013d98d /system/lib/libart.so (art_quick_to_interpreter_bridge+77) - #52 pc 7264bc5b /data/dalvik-cache/x86/system@framework@boot.oat (offset 0x1eb2000) -''' - -_EXPECTED_BACKTRACE = r'''backtrace: - #00 pc 00084126 /system/lib/libc.so (tgkill+22) - #01 pc 000815d8 /system/lib/libc.so (pthread_kill+70) - #02 pc 00027205 /system/lib/libc.so (raise+36) - #03 pc 000209e4 /system/lib/libc.so (abort+80) - #04 pc 0000cf73 /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::LogMessage::Finish()+117) - #05 pc 0000cf8e /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::LogFinisher::operator=(google::protobuf::internal::LogMessage&)+26) - #06 pc 0000d27f /system/lib/libprotobuf-cpp-lite.so (google::protobuf::internal::VerifyVersion(int, int, char const*)+574) - #07 pc 007cd236 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #08 pc 000111a9 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libprotobuf_lite.cr.so (offset 0x1c000) - #09 pc 00013228 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libprotobuf_lite.cr.so (offset 0x1c000) - #10 pc 000131de /data/app/com.google.android.apps.chrome-2/base.apk!lib/libprotobuf_lite.cr.so (offset 0x1c000) - #11 pc 007cd2d8 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #12 pc 007cd956 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #13 pc 007c2d4a /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #14 pc 009fc9f1 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #15 pc 009fc8ea /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #16 pc 00561c63 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #17 pc 0106fbdb /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #18 pc 004d7371 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #19 pc 004d8159 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #20 pc 004d7b96 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #21 pc 004da4b6 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #22 pc 005ab66c /data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so (offset 0xc2d000) - #23 pc 005afca2 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so (offset 0xc2d000) - #24 pc 0000cae8 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so (offset 0x90e000) - #25 pc 00ce864f /data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so (offset 0xc2d000) - #26 pc 00ce8dfa /data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so (offset 0xc2d000) - #27 pc 00ce74c6 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so (offset 0xc2d000) - #28 pc 00004616 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libembedder.cr.so (offset 0x28000) - #29 pc 00ce8215 /data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so (offset 0xc2d000) - #30 pc 0013d8c7 /system/lib/libart.so (art_quick_generic_jni_trampoline+71) - #31 pc 00137c52 /system/lib/libart.so (art_quick_invoke_static_stub+418) - #32 pc 00143651 /system/lib/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+353) - #33 pc 005e06ae /system/lib/libart.so (artInterpreterToCompiledCodeBridge+190) - #34 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #35 pc 0032cfc0 /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)0, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+160) - #36 pc 000fc703 /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+29891) - #37 pc 00300af7 /system/lib/libart.so (artInterpreterToInterpreterBridge+188) - #38 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #39 pc 0032cfc0 /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)0, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+160) - #40 pc 000fc703 /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+29891) - #41 pc 00300af7 /system/lib/libart.so (artInterpreterToInterpreterBridge+188) - #42 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #43 pc 0032ebf9 /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)2, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+297) - #44 pc 000fc955 /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+30485) - #45 pc 00300af7 /system/lib/libart.so (artInterpreterToInterpreterBridge+188) - #46 pc 00328b5d /system/lib/libart.so (bool art::interpreter::DoCall<false, false>(art::ArtMethod*, art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+445) - #47 pc 0033090c /system/lib/libart.so (bool art::interpreter::DoInvoke<(art::InvokeType)4, false, false>(art::Thread*, art::ShadowFrame&, art::Instruction const*, unsigned short, art::JValue*)+636) - #48 pc 000fc67f /system/lib/libart.so (art::JValue art::interpreter::ExecuteGotoImpl<false, false>(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame&, art::JValue)+29759) - #49 pc 00300700 /system/lib/libart.so (art::interpreter::EnterInterpreterFromEntryPoint(art::Thread*, art::DexFile::CodeItem const*, art::ShadowFrame*)+128) - #50 pc 00667c73 /system/lib/libart.so (artQuickToInterpreterBridge+808) - #51 pc 0013d98d /system/lib/libart.so (art_quick_to_interpreter_bridge+77) - #52 pc 7264bc5b /data/dalvik-cache/x86/system@framework@boot.oat (offset 0x1eb2000) -''' - -_EXPECTED_BACKTRACE_OFFSETS_MAP = { - '/data/app/com.google.android.apps.chrome-2/base.apk!lib/libprotobuf_lite.cr.so': - set([ - 0x1c000 + 0x111a9, - 0x1c000 + 0x13228, - 0x1c000 + 0x131de, - ]), - - '/data/app/com.google.android.apps.chrome-2/base.apk!lib/libchrome.cr.so': - set([ - 0x90e000 + 0x7cd236, - 0x90e000 + 0x7cd2d8, - 0x90e000 + 0x7cd956, - 0x90e000 + 0x7c2d4a, - 0x90e000 + 0x9fc9f1, - 0x90e000 + 0x9fc8ea, - 0x90e000 + 0x561c63, - 0x90e000 + 0x106fbdb, - 0x90e000 + 0x4d7371, - 0x90e000 + 0x4d8159, - 0x90e000 + 0x4d7b96, - 0x90e000 + 0x4da4b6, - 0x90e000 + 0xcae8, - ]), - '/data/app/com.google.android.apps.chrome-2/base.apk!lib/libcontent.cr.so': - set([ - 0xc2d000 + 0x5ab66c, - 0xc2d000 + 0x5afca2, - 0xc2d000 + 0xce864f, - 0xc2d000 + 0xce8dfa, - 0xc2d000 + 0xce74c6, - 0xc2d000 + 0xce8215, - ]), - '/data/app/com.google.android.apps.chrome-2/base.apk!lib/libembedder.cr.so': - set([ - 0x28000 + 0x4616, - ]) -} - -# pylint: enable=line-too-long - -_ONE_MB = 1024 * 1024 -_TEST_SYMBOL_DATA = { - # Regular symbols - 0: 'mock_sym_for_addr_0 [mock_src/libmock1.so.c:0]', - 0x1000: 'mock_sym_for_addr_4096 [mock_src/libmock1.so.c:4096]', - - # Symbols without source file path. - _ONE_MB: 'mock_sym_for_addr_1048576 [??:0]', - _ONE_MB + 0x8234: 'mock_sym_for_addr_1081908 [??:0]', - - # Unknown symbol. - 2 * _ONE_MB: '?? [??:0]', - - # Inlined symbol. - 3 * _ONE_MB: - 'mock_sym_for_addr_3145728_inner [mock_src/libmock1.so.c:3145728]', -} - -@contextlib.contextmanager -def _TempDir(): - dirname = tempfile.mkdtemp() - try: - yield dirname - finally: - shutil.rmtree(dirname) - - -def _TouchFile(path): - # Create parent directories. - try: - os.makedirs(os.path.dirname(path)) - except OSError: - pass - with open(path, 'a'): - os.utime(path, None) - -class MockApkTranslator(object): - """A mock ApkLibraryPathTranslator object used for testing.""" - - # Regex that matches the content of APK native library map files generated - # with apk_lib_dump.py. - _RE_MAP_FILE = re.compile( - r'0x(?P<file_start>[0-9a-f]+)\s+' + - r'0x(?P<file_end>[0-9a-f]+)\s+' + - r'0x(?P<file_size>[0-9a-f]+)\s+' + - r'0x(?P<lib_path>[0-9a-f]+)\s+') - - def __init__(self, test_apk_libs=None): - """Initialize instance. - - Args: - test_apk_libs: Optional list of (file_start, file_end, size, lib_path) - tuples, like _TEST_APK_LIBS for example. This will be used to - implement TranslatePath(). - """ - self._apk_libs = [] - if test_apk_libs: - self._AddLibEntries(test_apk_libs) - - def _AddLibEntries(self, entries): - self._apk_libs = sorted(self._apk_libs + entries, - lambda x, y: cmp(x[0], y[0])) - - def ReadMapFile(self, file_path): - """Read an .apk.native-libs file that was produced with apk_lib_dump.py. - - Args: - file_path: input path to .apk.native-libs file. Its format is - essentially: 0x<start> 0x<end> 0x<size> <library-path> - """ - new_libs = [] - with open(file_path) as f: - for line in f.readlines(): - m = MockApkTranslator._RE_MAP_FILE.match(line) - if m: - file_start = int(m.group('file_start'), 16) - file_end = int(m.group('file_end'), 16) - file_size = int(m.group('file_size'), 16) - lib_path = m.group('lib_path') - # Sanity check - if file_start + file_size != file_end: - logging.warning('%s: Inconsistent (start, end, size) values ' - '(0x%x, 0x%x, 0x%x)', - file_path, file_start, file_end, file_size) - else: - new_libs.append((file_start, file_end, file_size, lib_path)) - - self._AddLibEntries(new_libs) - - def TranslatePath(self, lib_path, lib_offset): - """Translate an APK file path + offset into a library path + offset.""" - min_pos = 0 - max_pos = len(self._apk_libs) - while min_pos < max_pos: - mid_pos = (min_pos + max_pos) / 2 - mid_entry = self._apk_libs[mid_pos] - mid_offset = mid_entry[0] - mid_size = mid_entry[2] - if lib_offset < mid_offset: - max_pos = mid_pos - elif lib_offset >= mid_offset + mid_size: - min_pos = mid_pos + 1 - else: - # Found it - new_path = '%s!lib/%s' % (lib_path, mid_entry[3]) - new_offset = lib_offset - mid_offset - return (new_path, new_offset) - - return lib_path, lib_offset - - -class HostLibraryFinderTest(unittest.TestCase): - - def testEmpty(self): - finder = symbol_utils.HostLibraryFinder() - self.assertIsNone(finder.Find('/data/data/com.example.app-1/lib/libfoo.so')) - self.assertIsNone( - finder.Find('/data/data/com.example.app-1/base.apk!lib/libfoo.so')) - - - def testSimpleDirectory(self): - finder = symbol_utils.HostLibraryFinder() - with _TempDir() as tmp_dir: - host_libfoo_path = os.path.join(tmp_dir, 'libfoo.so') - host_libbar_path = os.path.join(tmp_dir, 'libbar.so') - _TouchFile(host_libfoo_path) - _TouchFile(host_libbar_path) - - finder.AddSearchDir(tmp_dir) - - # Regular library path (extracted at installation by the PackageManager). - # Note that the extraction path has changed between Android releases, - # i.e. it can be /data/app/, /data/data/ or /data/app-lib/ depending - # on the system. - self.assertEqual( - host_libfoo_path, - finder.Find('/data/app-lib/com.example.app-1/lib/libfoo.so')) - - # Verify that the path doesn't really matter - self.assertEqual( - host_libfoo_path, - finder.Find('/whatever/what.apk!lib/libfoo.so')) - - self.assertEqual( - host_libbar_path, - finder.Find('/data/data/com.example.app-1/lib/libbar.so')) - - self.assertIsNone( - finder.Find('/data/data/com.example.app-1/lib/libunknown.so')) - - - def testMultipleDirectories(self): - with _TempDir() as tmp_dir: - # Create the following files: - # <tmp_dir>/aaa/ - # libfoo.so - # <tmp_dir>/bbb/ - # libbar.so - # libfoo.so (this one should never be seen because 'aaa' - # shall be first in the search path list). - # - aaa_dir = os.path.join(tmp_dir, 'aaa') - bbb_dir = os.path.join(tmp_dir, 'bbb') - os.makedirs(aaa_dir) - os.makedirs(bbb_dir) - - host_libfoo_path = os.path.join(aaa_dir, 'libfoo.so') - host_libbar_path = os.path.join(bbb_dir, 'libbar.so') - host_libfoo2_path = os.path.join(bbb_dir, 'libfoo.so') - - _TouchFile(host_libfoo_path) - _TouchFile(host_libbar_path) - _TouchFile(host_libfoo2_path) - - finder = symbol_utils.HostLibraryFinder() - finder.AddSearchDir(aaa_dir) - finder.AddSearchDir(bbb_dir) - - self.assertEqual( - host_libfoo_path, - finder.Find('/data/data/com.example.app-1/lib/libfoo.so')) - - self.assertEqual( - host_libfoo_path, - finder.Find('/data/whatever/base.apk!lib/libfoo.so')) - - self.assertEqual( - host_libbar_path, - finder.Find('/data/data/com.example.app-1/lib/libbar.so')) - - self.assertIsNone( - finder.Find('/data/data/com.example.app-1/lib/libunknown.so')) - - -class ElfSymbolResolverTest(unittest.TestCase): - - def testCreation(self): - resolver = symbol_utils.ElfSymbolResolver( - addr2line_path_for_tests=_MOCK_A2L_PATH) - self.assertTrue(resolver) - - def testWithSimpleOffsets(self): - resolver = symbol_utils.ElfSymbolResolver( - addr2line_path_for_tests=_MOCK_A2L_PATH) - resolver.SetAndroidAbi('ignored-abi') - - for addr, expected_sym in _TEST_SYMBOL_DATA.iteritems(): - self.assertEqual(resolver.FindSymbolInfo('/some/path/libmock1.so', addr), - expected_sym) - - def testWithPreResolvedSymbols(self): - resolver = symbol_utils.ElfSymbolResolver( - addr2line_path_for_tests=_MOCK_A2L_PATH) - resolver.SetAndroidAbi('ignored-abi') - resolver.AddLibraryOffsets('/some/path/libmock1.so', - _TEST_SYMBOL_DATA.keys()) - - resolver.DisallowSymbolizerForTesting() - - for addr, expected_sym in _TEST_SYMBOL_DATA.iteritems(): - sym_info = resolver.FindSymbolInfo('/some/path/libmock1.so', addr) - self.assertIsNotNone(sym_info, 'None symbol info for addr %x' % addr) - self.assertEqual( - sym_info, expected_sym, - 'Invalid symbol info for addr %x [%s] expected [%s]' % ( - addr, sym_info, expected_sym)) - - -class MemoryMapTest(unittest.TestCase): - - def testCreation(self): - mem_map = symbol_utils.MemoryMap('test-abi32') - self.assertIsNone(mem_map.FindSectionForAddress(0)) - - def testParseLines(self): - mem_map = symbol_utils.MemoryMap('test-abi32') - mem_map.ParseLines(_TEST_MEMORY_MAP.splitlines()) - for exp_addr, exp_size, exp_path, exp_offset in _TEST_MEMORY_MAP_SECTIONS: - text = '(addr:%x, size:%x, path:%s, offset=%x)' % ( - exp_addr, exp_size, exp_path, exp_offset) - - t = mem_map.FindSectionForAddress(exp_addr) - self.assertTrue(t, 'Could not find %s' % text) - self.assertEqual(t.address, exp_addr) - self.assertEqual(t.size, exp_size) - self.assertEqual(t.offset, exp_offset) - self.assertEqual(t.path, exp_path) - - def testTranslateLine(self): - android_abi = 'test-abi' - apk_translator = MockApkTranslator(_TEST_APK_LIBS) - mem_map = symbol_utils.MemoryMap(android_abi) - for line, expected_line in zip(_TEST_MEMORY_MAP.splitlines(), - _EXPECTED_TEST_MEMORY_MAP.splitlines()): - self.assertEqual(mem_map.TranslateLine(line, apk_translator), - expected_line) - -class StackTranslatorTest(unittest.TestCase): - - def testSimpleStack(self): - android_abi = 'test-abi32' - mem_map = symbol_utils.MemoryMap(android_abi) - mem_map.ParseLines(_TEST_MEMORY_MAP) - apk_translator = MockApkTranslator(_TEST_APK_LIBS) - stack_translator = symbol_utils.StackTranslator(android_abi, mem_map, - apk_translator) - input_stack = _TEST_STACK.splitlines() - expected_stack = _EXPECTED_STACK.splitlines() - self.assertEqual(len(input_stack), len(expected_stack)) - for stack_line, expected_line in zip(input_stack, expected_stack): - new_line = stack_translator.TranslateLine(stack_line) - self.assertEqual(new_line, expected_line) - - -class MockSymbolResolver(symbol_utils.SymbolResolver): - - # A regex matching a symbol definition as it appears in a test symbol file. - # Format is: <hex-offset> <whitespace> <symbol-string> - _RE_SYMBOL_DEFINITION = re.compile( - r'(?P<offset>[0-9a-f]+)\s+(?P<symbol>.*)') - - def __init__(self): - super(MockSymbolResolver, self).__init__() - self._map = collections.defaultdict(dict) - - def AddTestLibrarySymbols(self, lib_name, offsets_map): - """Add a new test entry for a given library name. - - Args: - lib_name: Library name (e.g. 'libfoo.so') - offsets_map: A mapping from offsets to symbol info strings. - """ - self._map[lib_name] = offsets_map - - def ReadTestFile(self, file_path, lib_name): - """Read a single test symbol file, matching a given library. - - Args: - file_path: Input file path. - lib_name: Library name these symbols correspond to (e.g. 'libfoo.so') - """ - with open(file_path) as f: - for line in f.readlines(): - line = line.rstrip() - m = MockSymbolResolver._RE_SYMBOL_DEFINITION.match(line) - if m: - offset = int(m.group('offset')) - symbol = m.group('symbol') - self._map[lib_name][offset] = symbol - - def ReadTestFilesInDir(self, dir_path, file_suffix): - """Read all symbol test files in a given directory. - - Args: - dir_path: Directory path. - file_suffix: File suffix used to detect test symbol files. - """ - for filename in os.listdir(dir_path): - if filename.endswith(file_suffix): - lib_name = filename[:-len(file_suffix)] - self.ReadTestFile(os.path.join(dir_path, filename), lib_name) - - def FindSymbolInfo(self, device_path, device_offset): - """Implement SymbolResolver.FindSymbolInfo.""" - lib_name = os.path.basename(device_path) - offsets = self._map.get(lib_name) - if not offsets: - return None - - return offsets.get(device_offset) - - -class BacktraceTranslatorTest(unittest.TestCase): - - def testEmpty(self): - android_abi = 'test-abi' - apk_translator = MockApkTranslator() - backtrace_translator = symbol_utils.BacktraceTranslator(android_abi, - apk_translator) - self.assertTrue(backtrace_translator) - - def testFindLibraryOffsets(self): - android_abi = 'test-abi' - apk_translator = MockApkTranslator(_TEST_APK_LIBS) - backtrace_translator = symbol_utils.BacktraceTranslator(android_abi, - apk_translator) - input_backtrace = _EXPECTED_BACKTRACE.splitlines() - expected_lib_offsets_map = _EXPECTED_BACKTRACE_OFFSETS_MAP - offset_map = backtrace_translator.FindLibraryOffsets(input_backtrace) - for lib_path, offsets in offset_map.iteritems(): - self.assertTrue(lib_path in expected_lib_offsets_map, - '%s is not in expected library-offsets map!' % lib_path) - sorted_offsets = sorted(offsets) - sorted_expected_offsets = sorted(expected_lib_offsets_map[lib_path]) - self.assertEqual(sorted_offsets, sorted_expected_offsets, - '%s has invalid offsets %s expected %s' % ( - lib_path, sorted_offsets, sorted_expected_offsets)) - - def testTranslateLine(self): - android_abi = 'test-abi' - apk_translator = MockApkTranslator(_TEST_APK_LIBS) - backtrace_translator = symbol_utils.BacktraceTranslator(android_abi, - apk_translator) - input_backtrace = _TEST_BACKTRACE.splitlines() - expected_backtrace = _EXPECTED_BACKTRACE.splitlines() - self.assertEqual(len(input_backtrace), len(expected_backtrace)) - for trace_line, expected_line in zip(input_backtrace, expected_backtrace): - line = backtrace_translator.TranslateLine(trace_line, - MockSymbolResolver()) - self.assertEqual(line, expected_line) - - -if __name__ == '__main__': - unittest.main()
diff --git a/build/android/pylib/utils/app_bundle_utils.py b/build/android/pylib/utils/app_bundle_utils.py index b2e9927..9a52d85 100644 --- a/build/android/pylib/utils/app_bundle_utils.py +++ b/build/android/pylib/utils/app_bundle_utils.py
@@ -1,13 +1,15 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import json import logging import os +import pathlib import re +import shutil import sys -import tempfile +import zipfile sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'gyp')) @@ -16,21 +18,24 @@ from util import resource_utils import bundletool -# List of valid modes for GenerateBundleApks() -BUILD_APKS_MODES = ('default', 'universal', 'system', 'system_compressed') +# "system_apks" is "default", but with locale list and compressed dex. +_SYSTEM_MODES = ('system', 'system_apks') +BUILD_APKS_MODES = _SYSTEM_MODES + ('default', 'universal') OPTIMIZE_FOR_OPTIONS = ('ABI', 'SCREEN_DENSITY', 'LANGUAGE', 'TEXTURE_COMPRESSION_FORMAT') -_SYSTEM_MODES = ('system_compressed', 'system') _ALL_ABIS = ['armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'] +def _BundleMinSdkVersion(bundle_path): + manifest_data = bundletool.RunBundleTool( + ['dump', 'manifest', '--bundle', bundle_path]) + return int(re.search(r'minSdkVersion.*?(\d+)', manifest_data).group(1)) + + def _CreateDeviceSpec(bundle_path, sdk_version, locales): if not sdk_version: - manifest_data = bundletool.RunBundleTool( - ['dump', 'manifest', '--bundle', bundle_path]) - sdk_version = int( - re.search(r'minSdkVersion.*?(\d+)', manifest_data).group(1)) + sdk_version = _BundleMinSdkVersion(bundle_path) # Setting sdkVersion=minSdkVersion prevents multiple per-minSdkVersion .apk # files from being created within the .apks file. @@ -42,6 +47,20 @@ } +def _FixBundleDexCompressionGlob(src_bundle, dst_bundle): + # Modifies the BundleConfig.pb of the given .aab to add "classes*.dex" to the + # "uncompressedGlob" list. + with zipfile.ZipFile(src_bundle) as src, \ + zipfile.ZipFile(dst_bundle, 'w') as dst: + for info in src.infolist(): + data = src.read(info) + if info.filename == 'BundleConfig.pb': + # A classesX.dex entry is added by create_app_bundle.py so that we can + # modify it here in order to have it take effect. b/176198991 + data = data.replace(b'classesX.dex', b'classes*.dex') + dst.writestr(info, data) + + def GenerateBundleApks(bundle_path, bundle_apks_path, aapt2_path, @@ -49,6 +68,7 @@ keystore_password, keystore_alias, mode=None, + local_testing=False, minimal=False, minimal_sdk_version=None, check_for_noop=True, @@ -97,23 +117,38 @@ def rebuild(): logging.info('Building %s', bundle_apks_path) - with tempfile.NamedTemporaryFile(suffix='.apks') as tmp_apks_file: + with build_utils.TempDir() as tmp_dir: + tmp_apks_file = os.path.join(tmp_dir, 'output.apks') cmd_args = [ 'build-apks', '--aapt2=%s' % aapt2_path, - '--output=%s' % tmp_apks_file.name, - '--bundle=%s' % bundle_path, + '--output=%s' % tmp_apks_file, '--ks=%s' % keystore_path, '--ks-pass=pass:%s' % keystore_password, '--ks-key-alias=%s' % keystore_alias, '--overwrite', ] + input_bundle_path = bundle_path + # Work around bundletool not respecting uncompressDexFiles setting. + # b/176198991 + if mode not in _SYSTEM_MODES and _BundleMinSdkVersion(bundle_path) >= 27: + input_bundle_path = os.path.join(tmp_dir, 'system.aab') + _FixBundleDexCompressionGlob(bundle_path, input_bundle_path) + + cmd_args += ['--bundle=%s' % input_bundle_path] + + if local_testing: + cmd_args += ['--local-testing'] if mode is not None: if mode not in BUILD_APKS_MODES: raise Exception('Invalid mode parameter %s (should be in %s)' % (mode, BUILD_APKS_MODES)) - cmd_args += ['--mode=' + mode] + if mode != 'system_apks': + cmd_args += ['--mode=' + mode] + else: + # Specify --optimize-for to prevent language splits being created. + cmd_args += ['--optimize-for=device_tier'] if optimize_for: if optimize_for not in OPTIMIZE_FOR_OPTIONS: @@ -122,32 +157,27 @@ (mode, OPTIMIZE_FOR_OPTIONS)) cmd_args += ['--optimize-for=' + optimize_for] - with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as spec_file: - if device_spec: - json.dump(device_spec, spec_file) - spec_file.flush() - cmd_args += ['--device-spec=' + spec_file.name] - bundletool.RunBundleTool(cmd_args) + if device_spec: + data = json.dumps(device_spec) + logging.debug('Device Spec: %s', data) + spec_file = pathlib.Path(tmp_dir) / 'device.json' + spec_file.write_text(data) + cmd_args += ['--device-spec=' + str(spec_file)] - # Make the resulting .apks file hermetic. - with build_utils.TempDir() as temp_dir, \ - build_utils.AtomicOutput(bundle_apks_path, only_if_changed=False) as f: - files = build_utils.ExtractAll(tmp_apks_file.name, temp_dir) - build_utils.DoZip(files, f, base_dir=temp_dir) + bundletool.RunBundleTool(cmd_args) + + shutil.move(tmp_apks_file, bundle_apks_path) if check_for_noop: - # NOTE: BUNDLETOOL_JAR_PATH is added to input_strings, rather than - # input_paths, to speed up MD5 computations by about 400ms (the .jar file - # contains thousands of class files which are checked independently, - # resulting in an .md5.stamp of more than 60000 lines!). - input_paths = [bundle_path, aapt2_path, keystore_path] + input_paths = [ + bundle_path, + bundletool.BUNDLETOOL_JAR_PATH, + aapt2_path, + keystore_path, + ] input_strings = [ keystore_password, keystore_alias, - bundletool.BUNDLETOOL_JAR_PATH, - # NOTE: BUNDLETOOL_VERSION is already part of BUNDLETOOL_JAR_PATH, but - # it's simpler to assume that this may not be the case in the future. - bundletool.BUNDLETOOL_VERSION, device_spec, ] if mode is not None:
diff --git a/build/android/pylib/utils/argparse_utils.py b/build/android/pylib/utils/argparse_utils.py index 06544a2..698be78 100644 --- a/build/android/pylib/utils/argparse_utils.py +++ b/build/android/pylib/utils/argparse_utils.py
@@ -1,8 +1,8 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function + import argparse @@ -23,7 +23,7 @@ help='What this helps with') ''' # Derived from argparse._HelpAction from - # https://github.com/python/cpython/blob/master/Lib/argparse.py + # https://github.com/python/cpython/blob/main/Lib/argparse.py # pylint: disable=redefined-builtin # (complains about 'help' being redefined) @@ -33,11 +33,11 @@ default=argparse.SUPPRESS, custom_help_text=None, help=None): - super(CustomHelpAction, self).__init__(option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) + super().__init__(option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) if not custom_help_text: raise ValueError('custom_help_text is required')
diff --git a/build/android/pylib/utils/chrome_proxy_utils.py b/build/android/pylib/utils/chrome_proxy_utils.py index 149d0b9..14960f4 100644 --- a/build/android/pylib/utils/chrome_proxy_utils.py +++ b/build/android/pylib/utils/chrome_proxy_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Utilities for setting up and tear down WPR and TsProxy service.""" @@ -18,7 +18,7 @@ DEFAULT_UPLOAD_BANDWIDTH_KBPS = 72000 -class WPRServer(object): +class WPRServer: """Utils to set up a webpagereplay_go_server instance.""" def __init__(self): @@ -88,7 +88,7 @@ return self._archive_path -class ChromeProxySession(object): +class ChromeProxySession: """Utils to help set up a Chrome Proxy.""" def __init__(self, device_proxy_port=DEFAULT_DEVICE_PORT):
diff --git a/build/android/pylib/utils/chrome_proxy_utils_test.py b/build/android/pylib/utils/chrome_proxy_utils_test.py index b38b268..2b89812 100755 --- a/build/android/pylib/utils/chrome_proxy_utils_test.py +++ b/build/android/pylib/utils/chrome_proxy_utils_test.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2020 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for chrome_proxy_utils.""" @@ -91,7 +91,7 @@ wpr_mock.assert_called_once_with() ts_proxy_mock.assert_called_once_with() self.assertFalse(chrome_proxy.wpr_replay_mode) - self.assertEquals(chrome_proxy.wpr_archive_path, os.path.abspath(__file__)) + self.assertEqual(chrome_proxy.wpr_archive_path, os.path.abspath(__file__)) def test_SetWPRRecordMode(self): chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) @@ -108,7 +108,7 @@ def test_SetWPRArchivePath(self): chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) chrome_proxy._wpr_server._archive_path = 'abc' - self.assertEquals(chrome_proxy.wpr_archive_path, 'abc') + self.assertEqual(chrome_proxy.wpr_archive_path, 'abc') def test_UseDefaultDeviceProxyPort(self): chrome_proxy = chrome_proxy_utils.ChromeProxySession() @@ -117,7 +117,7 @@ 'PhrPvGIaAMmd29hj8BCZOq096yj7uMpRNHpn5PDxI6I=', '--proxy-server=socks5://localhost:1080' ] - self.assertEquals(chrome_proxy.device_proxy_port, 1080) + self.assertEqual(chrome_proxy.device_proxy_port, 1080) self.assertListEqual(chrome_proxy.GetFlags(), expected_flags) def test_UseNewDeviceProxyPort(self): @@ -127,7 +127,7 @@ 'PhrPvGIaAMmd29hj8BCZOq096yj7uMpRNHpn5PDxI6I=', '--proxy-server=socks5://localhost:1' ] - self.assertEquals(chrome_proxy.device_proxy_port, 1) + self.assertEqual(chrome_proxy.device_proxy_port, 1) self.assertListEqual(chrome_proxy.GetFlags(), expected_flags)
diff --git a/build/android/pylib/utils/decorators.py b/build/android/pylib/utils/decorators.py index 8eec1d1..0cef420 100644 --- a/build/android/pylib/utils/decorators.py +++ b/build/android/pylib/utils/decorators.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/pylib/utils/decorators_test.py b/build/android/pylib/utils/decorators_test.py index 73a9f0d..f8d9075 100755 --- a/build/android/pylib/utils/decorators_test.py +++ b/build/android/pylib/utils/decorators_test.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -35,8 +35,8 @@ def doesNotRaiseException(): return 999 - self.assertEquals(raiseException(), 111) - self.assertEquals(doesNotRaiseException(), 999) + self.assertEqual(raiseException(), 111) + self.assertEqual(doesNotRaiseException(), 999) class MemoizeDecoratorTest(unittest.TestCase): @@ -79,13 +79,13 @@ return notMemoized.count notMemoized.count = 0 - self.assertEquals(memoized(), 1) - self.assertEquals(memoized(), 1) - self.assertEquals(memoized(), 1) + self.assertEqual(memoized(), 1) + self.assertEqual(memoized(), 1) + self.assertEqual(memoized(), 1) - self.assertEquals(notMemoized(), 1) - self.assertEquals(notMemoized(), 2) - self.assertEquals(notMemoized(), 3) + self.assertEqual(notMemoized(), 1) + self.assertEqual(notMemoized(), 2) + self.assertEqual(notMemoized(), 3) def testFunctionMemoizedBasedOnArgs(self): """Tests that |Memoize| caches results based on args and kwargs.""" @@ -94,10 +94,10 @@ def returnValueBasedOnArgsKwargs(a, k=0): return a + k - self.assertEquals(returnValueBasedOnArgsKwargs(1, 1), 2) - self.assertEquals(returnValueBasedOnArgsKwargs(1, 2), 3) - self.assertEquals(returnValueBasedOnArgsKwargs(2, 1), 3) - self.assertEquals(returnValueBasedOnArgsKwargs(3, 3), 6) + self.assertEqual(returnValueBasedOnArgsKwargs(1, 1), 2) + self.assertEqual(returnValueBasedOnArgsKwargs(1, 2), 3) + self.assertEqual(returnValueBasedOnArgsKwargs(2, 1), 3) + self.assertEqual(returnValueBasedOnArgsKwargs(3, 3), 6) if __name__ == '__main__':
diff --git a/build/android/pylib/utils/device_dependencies.py b/build/android/pylib/utils/device_dependencies.py index 9cb5bd8..5f3f1ed 100644 --- a/build/android/pylib/utils/device_dependencies.py +++ b/build/android/pylib/utils/device_dependencies.py
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,16 +7,19 @@ from pylib import constants - _EXCLUSIONS = [ - re.compile(r'.*OWNERS'), # Should never be included. + # Misc files that exist to document directories + re.compile(r'.*METADATA'), + re.compile(r'.*OWNERS'), + re.compile(r'.*\.md'), re.compile(r'.*\.crx'), # Chrome extension zip files. - re.compile(os.path.join('.*', - r'\.git.*')), # Any '.git*' directories/files. + re.compile(r'.*/\.git.*'), # Any '.git*' directories/files. re.compile(r'.*\.so'), # Libraries packed into .apk. re.compile(r'.*Mojo.*manifest\.json'), # Some source_set()s pull these in. re.compile(r'.*\.py'), # Some test_support targets include python deps. re.compile(r'.*\.apk'), # Should be installed separately. + re.compile(r'.*\.jar'), # Never need java intermediates. + re.compile(r'.*\.crx'), # Used by download_from_google_storage. re.compile(r'.*lib.java/.*'), # Never need java intermediates. # Test filter files: @@ -30,21 +33,27 @@ # v8's blobs and icu data get packaged into APKs. re.compile(r'.*snapshot_blob.*\.bin'), - re.compile(r'.*icudtl.bin'), + re.compile(r'.*icudtl\.bin'), # Scripts that are needed by swarming, but not on devices: re.compile(r'.*llvm-symbolizer'), - re.compile(r'.*md5sum_bin'), - re.compile(os.path.join('.*', 'development', 'scripts', 'stack')), + re.compile(r'.*md5sum_(?:bin|dist)'), + re.compile(r'.*/development/scripts/stack'), + re.compile(r'.*/build/android/pylib/symbols'), + re.compile(r'.*/build/android/stacktrace'), # Required for java deobfuscation on the host: re.compile(r'.*build/android/stacktrace/.*'), re.compile(r'.*third_party/jdk/.*'), re.compile(r'.*third_party/proguard/.*'), + # Our tests don't need these. + re.compile(r'.*/devtools-frontend/src/front_end/.*'), + # Build artifacts: re.compile(r'.*\.stamp'), - re.compile(r'.*.pak\.info'), + re.compile(r'.*\.pak\.info'), + re.compile(r'.*\.build_config.json'), re.compile(r'.*\.incremental\.json'), ]
diff --git a/build/android/pylib/utils/device_dependencies_test.py b/build/android/pylib/utils/device_dependencies_test.py index b2da5a7..2ff937e 100755 --- a/build/android/pylib/utils/device_dependencies_test.py +++ b/build/android/pylib/utils/device_dependencies_test.py
@@ -1,5 +1,5 @@ -#! /usr/bin/env vpython -# Copyright 2016 The Chromium Authors. All rights reserved. +#! /usr/bin/env vpython3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,40 +16,36 @@ test_path = os.path.join(constants.DIR_SOURCE_ROOT, 'foo', 'bar', 'baz.txt') output_directory = os.path.join( constants.DIR_SOURCE_ROOT, 'out-foo', 'Release') - self.assertEquals( - [None, 'foo', 'bar', 'baz.txt'], - device_dependencies.DevicePathComponentsFor( - test_path, output_directory)) + self.assertEqual([None, 'foo', 'bar', 'baz.txt'], + device_dependencies.DevicePathComponentsFor( + test_path, output_directory)) def testOutputDirectoryFile(self): test_path = os.path.join(constants.DIR_SOURCE_ROOT, 'out-foo', 'Release', 'icudtl.dat') output_directory = os.path.join( constants.DIR_SOURCE_ROOT, 'out-foo', 'Release') - self.assertEquals( - [None, 'icudtl.dat'], - device_dependencies.DevicePathComponentsFor( - test_path, output_directory)) + self.assertEqual([None, 'icudtl.dat'], + device_dependencies.DevicePathComponentsFor( + test_path, output_directory)) def testOutputDirectorySubdirFile(self): test_path = os.path.join(constants.DIR_SOURCE_ROOT, 'out-foo', 'Release', 'test_dir', 'icudtl.dat') output_directory = os.path.join( constants.DIR_SOURCE_ROOT, 'out-foo', 'Release') - self.assertEquals( - [None, 'test_dir', 'icudtl.dat'], - device_dependencies.DevicePathComponentsFor( - test_path, output_directory)) + self.assertEqual([None, 'test_dir', 'icudtl.dat'], + device_dependencies.DevicePathComponentsFor( + test_path, output_directory)) def testOutputDirectoryPakFile(self): test_path = os.path.join(constants.DIR_SOURCE_ROOT, 'out-foo', 'Release', 'foo.pak') output_directory = os.path.join( constants.DIR_SOURCE_ROOT, 'out-foo', 'Release') - self.assertEquals( - [None, 'paks', 'foo.pak'], - device_dependencies.DevicePathComponentsFor( - test_path, output_directory)) + self.assertEqual([None, 'paks', 'foo.pak'], + device_dependencies.DevicePathComponentsFor( + test_path, output_directory)) if __name__ == '__main__':
diff --git a/build/android/pylib/utils/dexdump.py b/build/android/pylib/utils/dexdump.py index f81ac60..0913aad 100644 --- a/build/android/pylib/utils/dexdump.py +++ b/build/android/pylib/utils/dexdump.py
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,6 +8,8 @@ import sys import tempfile from xml.etree import ElementTree +from collections import namedtuple +from typing import Dict from devil.utils import cmd_helper from pylib import constants @@ -18,6 +20,27 @@ DEXDUMP_PATH = os.path.join(constants.ANDROID_SDK_TOOLS, 'dexdump') +# Annotations dict format: +# { +# 'empty-annotation-class-name': None, +# 'annotation-class-name': { +# 'fieldA': 'primitive-value', +# 'fieldB': [ 'array-item-1', 'array-item-2', ... ], +# 'fieldC': { # CURRENTLY UNSUPPORTED. +# /* Object value */ +# 'field': 'primitive-value', +# 'field': [ 'array-item-1', 'array-item-2', ... ], +# 'field': { /* Object value */ } +# } +# } +# } +Annotations = namedtuple('Annotations', + ['classAnnotations', 'methodsAnnotations']) + +# Finds each space-separated "foo=..." (where ... can contain spaces). +_ANNOTATION_VALUE_MATCHER = re.compile(r'\w+=.*?(?:$|(?= \w+=))') + + def Dump(apk_path): """Dumps class and method information from a APK into a dict via dexdump. @@ -29,7 +52,10 @@ <package_name>: { 'classes': { <class_name>: { - 'methods': [<method_1>, <method_2>] + 'methods': [<method_1>, <method_2>], + 'superclass': <string>, + 'is_abstract': <boolean>, + 'annotations': <Annotations> } } } @@ -42,7 +68,7 @@ dexfile_dir, pattern='*classes*.dex'): output_xml = cmd_helper.GetCmdOutput( - [DEXDUMP_PATH, '-l', 'xml', dex_file]) + [DEXDUMP_PATH, '-a', '-j', '-l', 'xml', dex_file]) # Dexdump doesn't escape its XML output very well; decode it as utf-8 with # invalid sequences replaced, then remove forbidden characters and # re-encode it (as etree expects a byte string as input so it can figure @@ -50,20 +76,142 @@ BAD_XML_CHARS = re.compile( u'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f-\x84\x86-\x9f' + u'\ud800-\udfff\ufdd0-\ufddf\ufffe-\uffff]') - if sys.version_info[0] < 3: - decoded_xml = output_xml.decode('utf-8', 'replace') - clean_xml = BAD_XML_CHARS.sub(u'\ufffd', decoded_xml) - else: - # Line duplicated to avoid pylint redefined-variable-type error. - clean_xml = BAD_XML_CHARS.sub(u'\ufffd', output_xml) + + # Line duplicated to avoid pylint redefined-variable-type error. + clean_xml = BAD_XML_CHARS.sub(u'\ufffd', output_xml) + + # Constructors are referenced as "<init>" in our annotations + # which will result in in the ElementTree failing to parse + # our xml as it won't find a closing tag for this + clean_xml = clean_xml.replace('<init>', 'constructor') + + annotations = _ParseAnnotations(clean_xml) + parsed_dex_files.append( - _ParseRootNode(ElementTree.fromstring(clean_xml.encode('utf-8')))) + _ParseRootNode(ElementTree.fromstring(clean_xml.encode('utf-8')), + annotations)) return parsed_dex_files finally: shutil.rmtree(dexfile_dir) -def _ParseRootNode(root): +def _ParseAnnotationValues(values_str): + if not values_str: + return None + ret = {} + for key_value in _ANNOTATION_VALUE_MATCHER.findall(values_str): + key, value_str = key_value.split('=', 1) + # TODO: support for dicts if ever needed. + if value_str.startswith('{ ') and value_str.endswith(' }'): + value = value_str[2:-2].split() + else: + value = value_str + ret[key] = value + return ret + + +def _ParseAnnotations(dexRaw: str) -> Dict[int, Annotations]: + """ Parse XML strings and return a list of Annotations mapped to + classes by index. + + Annotations are written to the dex dump as human readable blocks of text + The only prescription is that they appear before the class in our xml file + They are not required to be nested within the package as our classes + It is simpler to parse for all the annotations and then associate them + back to the + classes + + Example: + Class #12 annotations: + Annotations on class + VISIBILITY_RUNTIME Ldalvik/annotation/EnclosingClass; value=... + Annotations on method #512 'example' + VISIBILITY_SYSTEM Ldalvik/annotation/Signature; value=... + VISIBILITY_RUNTIME Landroidx/test/filters/SmallTest; + VISIBILITY_RUNTIME Lorg/chromium/base/test/util/Feature; value={ Cronet } + VISIBILITY_RUNTIME LFoo; key1={ A B } key2=4104 key3=null + """ + + # We want to find the lines matching the annotations header pattern + # Eg: Class #12 annotations -> true + annotationsBlockMatcher = re.compile(u'^Class #.*annotations:$') + # We want to retrieve the index of the class + # Eg: Class #12 annotations -> 12 + classIndexMatcher = re.compile(u'(?<=#)[0-9]*') + # We want to retrieve the method name from between the quotes + # of the annotations line + # Eg: Annotations on method #512 'example' -> example + methodMatcher = re.compile(u"(?<=')[^']*") + # We want to match everything after the last slash until before the semi colon + # Eg: Ldalvik/annotation/Signature; -> Signature + annotationMatcher = re.compile(u'([^/]+); ?(.*)?') + + annotations = {} + currentAnnotationsForClass = None + currentAnnotationsBlock: Dict[str, None] = None + + # This loop does four things + # 1. It looks for a line telling us we are describing annotations for + # a new class + # 2. It looks for a line telling us if the annotations we find will be + # for the class or for any of it's methods; we will keep reference to + # this + # 3. It adds the annotations to whatever we are holding reference to + # 4. It looks for a line to see if we should start looking for a + # new class again + for line in dexRaw.splitlines(): + if currentAnnotationsForClass is None: + # Step 1 + # We keep searching until we find an annotation descriptor + # This lets us know that we are storing annotations for a new class + if annotationsBlockMatcher.match(line): + currentClassIndex = int(classIndexMatcher.findall(line)[0]) + currentAnnotationsForClass = Annotations(classAnnotations={}, + methodsAnnotations={}) + annotations[currentClassIndex] = currentAnnotationsForClass + else: + # Step 2 + # If we find a descriptor indicating we are tracking annotations + # for the class or it's methods, we'll keep a reference of this + # block for when we start finding annotation references + if line.startswith(u'Annotations on class'): + currentAnnotationsBlock = currentAnnotationsForClass.classAnnotations + elif line.startswith(u'Annotations on method'): + method = methodMatcher.findall(line)[0] + currentAnnotationsBlock = {} + currentAnnotationsForClass.methodsAnnotations[ + method] = currentAnnotationsBlock + + # If we match against any other type of annotations + # we will ignore them + elif line.startswith(u'Annotations on'): + currentAnnotationsBlock = None + + # Step 3 + # We are only adding runtime annotations as those are the types + # that will affect if we should run tests or not (where this is + # being used) + elif currentAnnotationsBlock is not None and line.strip().startswith( + 'VISIBILITY_RUNTIME'): + annotationName, annotationValuesStr = annotationMatcher.findall(line)[0] + annotationValues = _ParseAnnotationValues(annotationValuesStr) + + # Our instrumentation tests expect a mapping of "Annotation: Value" + # We aren't using the value for anything and this would increase + # the complexity of this parser so just mapping these to None + currentAnnotationsBlock.update({annotationName: annotationValues}) + + # Step 4 + # Empty lines indicate that the annotation descriptions are complete + # and we should look for new classes + elif not line.strip(): + currentAnnotationsForClass = None + currentAnnotationsBlock = None + + return annotations + + +def _ParseRootNode(root, annotations: Dict[int, Annotations]): """Parses the XML output of dexdump. This output is in the following format. This is a subset of the information contained within dexdump output. @@ -86,10 +234,17 @@ </api> """ results = {} + + # Annotations are referenced by the class order + # To match them, we need to keep track of the class number and + # match it to the appropriate annotation at that stage + classCount = 0 + for child in root: if child.tag == 'package': package_name = child.attrib['name'] - parsed_node = _ParsePackageNode(child) + parsed_node, classCount = _ParsePackageNode(child, classCount, + annotations) if package_name in results: results[package_name]['classes'].update(parsed_node['classes']) else: @@ -97,40 +252,62 @@ return results -def _ParsePackageNode(package_node): +def _ParsePackageNode(package_node, classCount: int, + annotations: Dict[int, Annotations]): """Parses a <package> node from the dexdump xml output. Returns: - A dict in the format: - { + A tuple in the format: + (classes: { 'classes': { <class_1>: { - 'methods': [<method_1>, <method_2>] + 'methods': [<method_1>, <method_2>], + 'superclass': <string>, + 'is_abstract': <boolean>, + 'annotations': <Annotations or None> }, <class_2>: { - 'methods': [<method_1>, <method_2>] + 'methods': [<method_1>, <method_2>], + 'superclass': <string>, + 'is_abstract': <boolean>, + 'annotations': <Annotations or None> }, } - } + }, classCount: number) """ classes = {} for child in package_node: if child.tag == 'class': - classes[child.attrib['name']] = _ParseClassNode(child) - return {'classes': classes} + classes[child.attrib['name']] = _ParseClassNode(child, classCount, + annotations) + classCount += 1 + return ({'classes': classes}, classCount) -def _ParseClassNode(class_node): +def _ParseClassNode(class_node, classIndex: int, + annotations: Dict[int, Annotations]): """Parses a <class> node from the dexdump xml output. Returns: A dict in the format: { - 'methods': [<method_1>, <method_2>] + 'methods': [<method_1>, <method_2>], + 'superclass': <string>, + 'is_abstract': <boolean> } """ methods = [] for child in class_node: - if child.tag == 'method': + if child.tag == 'method' and child.attrib['visibility'] == 'public': methods.append(child.attrib['name']) - return {'methods': methods, 'superclass': class_node.attrib['extends']} + return { + 'methods': + methods, + 'superclass': + class_node.attrib['extends'], + 'is_abstract': + class_node.attrib.get('abstract') == 'true', + 'annotations': + annotations.get(classIndex, + Annotations(classAnnotations={}, methodsAnnotations={})) + }
diff --git a/build/android/pylib/utils/dexdump_test.py b/build/android/pylib/utils/dexdump_test.py index 3197853..2b7c728 100755 --- a/build/android/pylib/utils/dexdump_test.py +++ b/build/android/pylib/utils/dexdump_test.py
@@ -1,5 +1,5 @@ -#! /usr/bin/env vpython -# Copyright 2016 The Chromium Authors. All rights reserved. +#! /usr/bin/env vpython3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,131 +10,197 @@ # pylint: disable=protected-access +emptyAnnotations = dexdump.Annotations(classAnnotations={}, + methodsAnnotations={}) + class DexdumpXMLParseTest(unittest.TestCase): - def testParseRootXmlNode(self): + def testParseAnnotations(self): example_xml_string = ( - '<api>' - '<package name="com.foo.bar1">' - '<class' - ' name="Class1"' - ' extends="java.lang.Object"' - ' abstract="false"' - ' static="false"' - ' final="true"' - ' visibility="public">' - '<method' - ' name="class1Method1"' - ' return="java.lang.String"' - ' abstract="false"' - ' native="false"' - ' synchronized="false"' - ' static="false"' - ' final="false"' - ' visibility="public">' - '</method>' - '<method' - ' name="class1Method2"' - ' return="viod"' - ' abstract="false"' - ' native="false"' - ' synchronized="false"' - ' static="false"' - ' final="false"' - ' visibility="public">' - '</method>' - '</class>' - '<class' - ' name="Class2"' - ' extends="java.lang.Object"' - ' abstract="false"' - ' static="false"' - ' final="true"' - ' visibility="public">' - '<method' - ' name="class2Method1"' - ' return="java.lang.String"' - ' abstract="false"' - ' native="false"' - ' synchronized="false"' - ' static="false"' - ' final="false"' - ' visibility="public">' - '</method>' - '</class>' - '</package>' - '<package name="com.foo.bar2">' - '</package>' - '<package name="com.foo.bar3">' - '</package>' - '</api>') + '<package name="com.foo.bar">\n' + 'Class #1 annotations:\n' + 'Annotations on class\n' + ' VISIBILITY_RUNTIME Ldalvik/annotation/AppModeFull; value=Alpha\n' + 'Annotations on method #512 \'example\'\n' + ' VISIBILITY_SYSTEM Ldalvik/annotation/Signature; value=Bravo\n' + ' VISIBILITY_RUNTIME Ldalvik/annotation/Test;\n' + ' VISIBILITY_RUNTIME Ldalvik/annotation/Test2; value=Charlie\n' + ' VISIBILITY_RUNTIME Ldalvik/annotation/Test3; A=B x B={ C D }\n' + ' VISIBILITY_RUNTIME Ldalvik/annotation/Test4; A=B x B={ C D } C=D\n' + '<class name="Class1" extends="java.lang.Object">\n' + '</class>\n' + '<class name="Class2" extends="java.lang.Object">\n' + '</class>\n' + '</package>\n') - actual = dexdump._ParseRootNode( - ElementTree.fromstring(example_xml_string)) + actual = dexdump._ParseAnnotations(example_xml_string) expected = { - 'com.foo.bar1' : { - 'classes': { - 'Class1': { - 'methods': ['class1Method1', 'class1Method2'], - 'superclass': 'java.lang.Object', - }, - 'Class2': { - 'methods': ['class2Method1'], - 'superclass': 'java.lang.Object', - } - }, - }, - 'com.foo.bar2' : {'classes': {}}, - 'com.foo.bar3' : {'classes': {}}, + 1: + dexdump.Annotations( + classAnnotations={'AppModeFull': { + 'value': 'Alpha' + }}, + methodsAnnotations={ + 'example': { + 'Test': None, + 'Test2': { + 'value': 'Charlie' + }, + 'Test3': { + 'A': 'B x', + 'B': ['C', 'D'] + }, + 'Test4': { + 'A': 'B x', + 'B': ['C', 'D'], + 'C': 'D' + }, + } + }, + ) } - self.assertEquals(expected, actual) + + self.assertEqual(expected, actual) + + def testParseRootXmlNode(self): + example_xml_string = ('<api>' + '<package name="com.foo.bar1">' + '<class' + ' name="Class1"' + ' extends="java.lang.Object"' + ' abstract="false"' + ' static="false"' + ' final="true"' + ' visibility="public">' + '<method' + ' name="class1Method1"' + ' return="java.lang.String"' + ' abstract="false"' + ' native="false"' + ' synchronized="false"' + ' static="false"' + ' final="false"' + ' visibility="public">' + '</method>' + '<method' + ' name="class1Method2"' + ' return="viod"' + ' abstract="false"' + ' native="false"' + ' synchronized="false"' + ' static="false"' + ' final="false"' + ' visibility="public">' + '</method>' + '</class>' + '<class' + ' name="Class2"' + ' extends="java.lang.Object"' + ' abstract="true"' + ' static="false"' + ' final="true"' + ' visibility="public">' + '<method' + ' name="class2Method1"' + ' return="java.lang.String"' + ' abstract="false"' + ' native="false"' + ' synchronized="false"' + ' static="false"' + ' final="false"' + ' visibility="public">' + '</method>' + '</class>' + '</package>' + '<package name="com.foo.bar2">' + '</package>' + '<package name="com.foo.bar3">' + '</package>' + '</api>') + + actual = dexdump._ParseRootNode(ElementTree.fromstring(example_xml_string), + {}) + + expected = { + 'com.foo.bar1': { + 'classes': { + 'Class1': { + 'methods': ['class1Method1', 'class1Method2'], + 'superclass': 'java.lang.Object', + 'is_abstract': False, + 'annotations': emptyAnnotations, + }, + 'Class2': { + 'methods': ['class2Method1'], + 'superclass': 'java.lang.Object', + 'is_abstract': True, + 'annotations': emptyAnnotations, + } + }, + }, + 'com.foo.bar2': { + 'classes': {} + }, + 'com.foo.bar3': { + 'classes': {} + }, + } + self.assertEqual(expected, actual) def testParsePackageNode(self): example_xml_string = ( '<package name="com.foo.bar">' '<class name="Class1" extends="java.lang.Object">' '</class>' - '<class name="Class2" extends="java.lang.Object">' + '<class name="Class2" extends="java.lang.Object" abstract="true">' '</class>' '</package>') - actual = dexdump._ParsePackageNode( - ElementTree.fromstring(example_xml_string)) + (actual, classCount) = dexdump._ParsePackageNode( + ElementTree.fromstring(example_xml_string), 0, {}) expected = { - 'classes': { - 'Class1': { - 'methods': [], - 'superclass': 'java.lang.Object', + 'classes': { + 'Class1': { + 'methods': [], + 'superclass': 'java.lang.Object', + 'is_abstract': False, + 'annotations': emptyAnnotations, + }, + 'Class2': { + 'methods': [], + 'superclass': 'java.lang.Object', + 'is_abstract': True, + 'annotations': emptyAnnotations, + }, }, - 'Class2': { - 'methods': [], - 'superclass': 'java.lang.Object', - }, - }, } - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) + self.assertEqual(classCount, 2) def testParseClassNode(self): - example_xml_string = ( - '<class name="Class1" extends="java.lang.Object">' - '<method name="method1">' - '</method>' - '<method name="method2">' - '</method>' - '</class>') + example_xml_string = ('<class name="Class1" extends="java.lang.Object">' + '<method name="method1" visibility="public">' + '</method>' + '<method name="method2" visibility="public">' + '</method>' + '<method name="method3" visibility="private">' + '</method>' + '</class>') - actual = dexdump._ParseClassNode( - ElementTree.fromstring(example_xml_string)) + actual = dexdump._ParseClassNode(ElementTree.fromstring(example_xml_string), + 0, {}) expected = { - 'methods': ['method1', 'method2'], - 'superclass': 'java.lang.Object', + 'methods': ['method1', 'method2'], + 'superclass': 'java.lang.Object', + 'is_abstract': False, + 'annotations': emptyAnnotations, } - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) if __name__ == '__main__':
diff --git a/build/android/pylib/utils/gold_utils.py b/build/android/pylib/utils/gold_utils.py index 0b79a6d..9dc9fe3 100644 --- a/build/android/pylib/utils/gold_utils.py +++ b/build/android/pylib/utils/gold_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """//build/android implementations of //testing/skia_gold_common. @@ -74,5 +74,5 @@ class AndroidSkiaGoldProperties(skia_gold_properties.SkiaGoldProperties): @staticmethod - def _GetGitOriginMasterHeadSha1(): - return repo_utils.GetGitOriginMasterHeadSHA1(host_paths.DIR_SOURCE_ROOT) + def _GetGitOriginMainHeadSha1(): + return repo_utils.GetGitOriginMainHeadSHA1(host_paths.DIR_SOURCE_ROOT)
diff --git a/build/android/pylib/utils/gold_utils_test.py b/build/android/pylib/utils/gold_utils_test.py index 2d3cc5c..8a9f8a3 100755 --- a/build/android/pylib/utils/gold_utils_test.py +++ b/build/android/pylib/utils/gold_utils_test.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2020 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for gold_utils.""" @@ -66,14 +66,14 @@ class AndroidSkiaGoldSessionDiffLinksTest(fake_filesystem_unittest.TestCase): - class FakeArchivedFile(object): + class FakeArchivedFile: def __init__(self, path): self.name = path def Link(self): return 'file://' + self.name - class FakeOutputManager(object): + class FakeOutputManager: def __init__(self): self.output_dir = tempfile.mkdtemp()
diff --git a/build/android/pylib/utils/google_storage_helper.py b/build/android/pylib/utils/google_storage_helper.py index d184810..27af709 100644 --- a/build/android/pylib/utils/google_storage_helper.py +++ b/build/android/pylib/utils/google_storage_helper.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,7 +13,10 @@ import os import sys import time -import urlparse +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse from pylib.constants import host_paths from pylib.utils import decorators @@ -22,9 +25,8 @@ sys.path.append(host_paths.DEVIL_PATH) from devil.utils import cmd_helper -_GSUTIL_PATH = os.path.join( - host_paths.DIR_SOURCE_ROOT, 'third_party', 'catapult', - 'third_party', 'gsutil', 'gsutil.py') +_GSUTIL_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party', + 'catapult', 'third_party', 'gsutil', 'gsutil') _PUBLIC_URL = 'https://storage.googleapis.com/%s/' _AUTHENTICATED_URL = 'https://storage.cloud.google.com/%s/' @@ -67,7 +69,7 @@ def read_from_link(link): # Note that urlparse returns the path with an initial '/', so we only need to # add one more after the 'gs;' - gs_path = 'gs:/%s' % urlparse.urlparse(link).path + gs_path = 'gs:/%s' % urlparse(link).path cmd = [_GSUTIL_PATH, '-q', 'cat', gs_path] return cmd_helper.GetCmdOutput(cmd)
diff --git a/build/android/pylib/utils/instrumentation_tracing.py b/build/android/pylib/utils/instrumentation_tracing.py index f1d03a0..3c9304e 100644 --- a/build/android/pylib/utils/instrumentation_tracing.py +++ b/build/android/pylib/utils/instrumentation_tracing.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -28,7 +28,8 @@ # Modules to exclude by default (to avoid problems like infinite loops) DEFAULT_EXCLUDE = [r'py_trace_event\..*'] -class _TraceArguments(object): + +class _TraceArguments: def __init__(self): """Wraps a dictionary to ensure safe evaluation of repr().""" self._arguments = {} @@ -75,7 +76,7 @@ if module_name in included: includes = True elif to_include: - includes = any([pattern.match(module_name) for pattern in to_include]) + includes = any(pattern.match(module_name) for pattern in to_include) else: includes = True @@ -161,6 +162,7 @@ if event == "return": trace_event.trace_end(function_name) return None + return None return traceFunction
diff --git a/build/android/pylib/utils/local_utils.py b/build/android/pylib/utils/local_utils.py index 027cca3..a7d39d6 100644 --- a/build/android/pylib/utils/local_utils.py +++ b/build/android/pylib/utils/local_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Utilities for determining if a test is being run locally or not."""
diff --git a/build/android/pylib/utils/logdog_helper.py b/build/android/pylib/utils/logdog_helper.py index 68a7ba5..e1562f5 100644 --- a/build/android/pylib/utils/logdog_helper.py +++ b/build/android/pylib/utils/logdog_helper.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -11,9 +11,11 @@ from pylib import constants from pylib.utils import decorators -sys.path.insert(0, os.path.abspath(os.path.join( - constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client'))) -from libs.logdog import bootstrap # pylint: disable=import-error +sys.path.insert( + 0, + os.path.abspath( + os.path.join(constants.DIR_SOURCE_ROOT, 'third_party', 'logdog'))) +from logdog import bootstrap # pylint: disable=import-error @decorators.NoRaiseException(default_return_value='',
diff --git a/build/android/pylib/utils/logging_utils.py b/build/android/pylib/utils/logging_utils.py index 9c4eae3..fdb0fa6 100644 --- a/build/android/pylib/utils/logging_utils.py +++ b/build/android/pylib/utils/logging_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -24,6 +24,7 @@ # pylint: disable=no-member color_map = { logging.DEBUG: (FORE.CYAN), + logging.INFO: (), # Use default style. logging.WARNING: (FORE.YELLOW), logging.ERROR: (FORE.RED), logging.CRITICAL: (BACK.RED), @@ -31,7 +32,7 @@ def __init__(self, wrapped_formatter=None): """Wraps a |logging.Formatter| and adds color.""" - super(_ColorFormatter, self).__init__(self) + super().__init__() self._wrapped_formatter = wrapped_formatter or logging.Formatter() #override @@ -63,24 +64,27 @@ """ def __init__(self, force_color=False): - super(ColorStreamHandler, self).__init__() + super().__init__() self.force_color = force_color self.setFormatter(logging.Formatter()) @property def is_tty(self): - isatty = getattr(self.stream, 'isatty', None) - return isatty and isatty() + try: + isatty = getattr(self.stream, 'isatty') + except AttributeError: + return False + return isatty() #override - def setFormatter(self, formatter): + def setFormatter(self, fmt): if self.force_color or self.is_tty: - formatter = _ColorFormatter(formatter) - super(ColorStreamHandler, self).setFormatter(formatter) + fmt = _ColorFormatter(fmt) + super().setFormatter(fmt) @staticmethod def MakeDefault(force_color=False): - """ + """ Replaces the default logging handlers with a coloring handler. To use a colorizing handler at the same time as others, either register them after this call, or add the ColorStreamHandler on the logger using @@ -89,9 +93,9 @@ Args: force_color: Set to True to bypass the tty check and always colorize. """ - # If the existing handlers aren't removed, messages are duplicated - logging.getLogger().handlers = [] - logging.getLogger().addHandler(ColorStreamHandler(force_color)) + # If the existing handlers aren't removed, messages are duplicated + logging.getLogger().handlers = [] + logging.getLogger().addHandler(ColorStreamHandler(force_color)) @contextlib.contextmanager @@ -110,7 +114,7 @@ try: yield finally: - for formatter, prev_color in prev_colors.iteritems(): + for formatter, prev_color in prev_colors.items(): formatter.color_map[level] = prev_color
diff --git a/build/android/pylib/utils/maven_downloader.py b/build/android/pylib/utils/maven_downloader.py index 1dc1542..fd9d973 100755 --- a/build/android/pylib/utils/maven_downloader.py +++ b/build/android/pylib/utils/maven_downloader.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -23,7 +23,7 @@ raise -class MavenDownloader(object): +class MavenDownloader: ''' Downloads and installs the requested artifacts from the Google Maven repo. The artifacts are expected to be specified in the format @@ -71,7 +71,7 @@ return self._debug -class _SingleArtifactDownloader(object): +class _SingleArtifactDownloader: '''Handles downloading and installing a single Maven artifact.''' _POM_FILE_TYPE = 'pom' @@ -121,8 +121,8 @@ if ret_code != 0: raise Exception('Command "{}" failed'.format(' '.join(cmd))) except OSError as e: - if e.errno == os.errno.ENOENT: - raise Exception('mvn command not found. Please install Maven.') + if e.errno == errno.ENOENT: + raise Exception('mvn command not found. Please install Maven.') from e raise return os.path.join(os.path.join(*group_id.split('.')),
diff --git a/build/android/pylib/utils/proguard.py b/build/android/pylib/utils/proguard.py deleted file mode 100644 index 9d5bae2..0000000 --- a/build/android/pylib/utils/proguard.py +++ /dev/null
@@ -1,285 +0,0 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os -import re -import tempfile - -from devil.utils import cmd_helper -from pylib import constants - - -_PROGUARD_CLASS_RE = re.compile(r'\s*?- Program class:\s*([\S]+)$') -_PROGUARD_SUPERCLASS_RE = re.compile(r'\s*? Superclass:\s*([\S]+)$') -_PROGUARD_SECTION_RE = re.compile( - r'^(Interfaces|Constant Pool|Fields|Methods|Class file attributes) ' - r'\(count = \d+\):$') -_PROGUARD_METHOD_RE = re.compile(r'\s*?- Method:\s*(\S*)[(].*$') -_PROGUARD_ANNOTATION_RE = re.compile(r'^(\s*?)- Annotation \[L(\S*);\]:$') -_ELEMENT_PRIMITIVE = 0 -_ELEMENT_ARRAY = 1 -_ELEMENT_ANNOTATION = 2 -_PROGUARD_ELEMENT_RES = [ - (_ELEMENT_PRIMITIVE, - re.compile(r'^(\s*?)- Constant element value \[(\S*) .*\]$')), - (_ELEMENT_ARRAY, - re.compile(r'^(\s*?)- Array element value \[(\S*)\]:$')), - (_ELEMENT_ANNOTATION, - re.compile(r'^(\s*?)- Annotation element value \[(\S*)\]:$')) -] -_PROGUARD_INDENT_WIDTH = 2 -_PROGUARD_ANNOTATION_VALUE_RE = re.compile(r'^(\s*?)- \S+? \[(.*)\]$') - - -def _GetProguardPath(): - return os.path.join(constants.DIR_SOURCE_ROOT, 'third_party', 'proguard', - 'lib', 'proguard603.jar') - - -def Dump(jar_path): - """Dumps class and method information from a JAR into a dict via proguard. - - Args: - jar_path: An absolute path to the JAR file to dump. - Returns: - A dict in the following format: - { - 'classes': [ - { - 'class': '', - 'superclass': '', - 'annotations': {/* dict -- see below */}, - 'methods': [ - { - 'method': '', - 'annotations': {/* dict -- see below */}, - }, - ... - ], - }, - ... - ], - } - - Annotations dict format: - { - 'empty-annotation-class-name': None, - 'annotation-class-name': { - 'field': 'primitive-value', - 'field': [ 'array-item-1', 'array-item-2', ... ], - 'field': { - /* Object value */ - 'field': 'primitive-value', - 'field': [ 'array-item-1', 'array-item-2', ... ], - 'field': { /* Object value */ } - } - } - } - - Note that for top-level annotations their class names are used for - identification, whereas for any nested annotations the corresponding - field names are used. - - One drawback of this approach is that an array containing empty - annotation classes will be represented as an array of 'None' values, - thus it will not be possible to find out annotation class names. - On the other hand, storing both annotation class name and the field name - would produce a very complex JSON. - """ - - with tempfile.NamedTemporaryFile() as proguard_output: - cmd_helper.GetCmdStatusAndOutput([ - 'java', - '-jar', _GetProguardPath(), - '-injars', jar_path, - '-dontshrink', '-dontoptimize', '-dontobfuscate', '-dontpreverify', - '-dump', proguard_output.name]) - return Parse(proguard_output) - -class _AnnotationElement(object): - def __init__(self, name, ftype, depth): - self.ref = None - self.name = name - self.ftype = ftype - self.depth = depth - -class _ParseState(object): - _INITIAL_VALUES = (lambda: None, list, dict) - # Empty annotations are represented as 'None', not as an empty dictionary. - _LAZY_INITIAL_VALUES = (lambda: None, list, lambda: None) - - def __init__(self): - self._class_result = None - self._method_result = None - self._parse_annotations = False - self._annotation_stack = [] - - def ResetPerSection(self, section_name): - self.InitMethod(None) - self._parse_annotations = ( - section_name in ['Class file attributes', 'Methods']) - - def ParseAnnotations(self): - return self._parse_annotations - - def CreateAndInitClass(self, class_name): - self.InitMethod(None) - self._class_result = { - 'class': class_name, - 'superclass': '', - 'annotations': {}, - 'methods': [], - } - return self._class_result - - def HasCurrentClass(self): - return bool(self._class_result) - - def SetSuperClass(self, superclass): - assert self.HasCurrentClass() - self._class_result['superclass'] = superclass - - def InitMethod(self, method_name): - self._annotation_stack = [] - if method_name: - self._method_result = { - 'method': method_name, - 'annotations': {}, - } - self._class_result['methods'].append(self._method_result) - else: - self._method_result = None - - def InitAnnotation(self, annotation, depth): - if not self._annotation_stack: - # Add a fake parent element comprising 'annotations' dictionary, - # so we can work uniformly with both top-level and nested annotations. - annotations = _AnnotationElement( - '<<<top level>>>', _ELEMENT_ANNOTATION, depth - 1) - if self._method_result: - annotations.ref = self._method_result['annotations'] - else: - annotations.ref = self._class_result['annotations'] - self._annotation_stack = [annotations] - self._BacktrackAnnotationStack(depth) - if not self.HasCurrentAnnotation(): - self._annotation_stack.append( - _AnnotationElement(annotation, _ELEMENT_ANNOTATION, depth)) - self._CreateAnnotationPlaceHolder(self._LAZY_INITIAL_VALUES) - - def HasCurrentAnnotation(self): - return len(self._annotation_stack) > 1 - - def InitAnnotationField(self, field, field_type, depth): - self._BacktrackAnnotationStack(depth) - # Create the parent representation, if needed. E.g. annotations - # are represented with `None`, not with `{}` until they receive the first - # field. - self._CreateAnnotationPlaceHolder(self._INITIAL_VALUES) - if self._annotation_stack[-1].ftype == _ELEMENT_ARRAY: - # Nested arrays are not allowed in annotations. - assert not field_type == _ELEMENT_ARRAY - # Use array index instead of bogus field name. - field = len(self._annotation_stack[-1].ref) - self._annotation_stack.append(_AnnotationElement(field, field_type, depth)) - self._CreateAnnotationPlaceHolder(self._LAZY_INITIAL_VALUES) - - def UpdateCurrentAnnotationFieldValue(self, value, depth): - self._BacktrackAnnotationStack(depth) - self._InitOrUpdateCurrentField(value) - - def _CreateAnnotationPlaceHolder(self, constructors): - assert self.HasCurrentAnnotation() - field = self._annotation_stack[-1] - if field.ref is None: - field.ref = constructors[field.ftype]() - self._InitOrUpdateCurrentField(field.ref) - - def _BacktrackAnnotationStack(self, depth): - stack = self._annotation_stack - while len(stack) > 0 and stack[-1].depth >= depth: - stack.pop() - - def _InitOrUpdateCurrentField(self, value): - assert self.HasCurrentAnnotation() - parent = self._annotation_stack[-2] - assert not parent.ref is None - # There can be no nested constant element values. - assert parent.ftype in [_ELEMENT_ARRAY, _ELEMENT_ANNOTATION] - field = self._annotation_stack[-1] - if isinstance(value, str) and not field.ftype == _ELEMENT_PRIMITIVE: - # The value comes from the output parser via - # UpdateCurrentAnnotationFieldValue, and should be a value of a constant - # element. If it isn't, just skip it. - return - if parent.ftype == _ELEMENT_ARRAY and field.name >= len(parent.ref): - parent.ref.append(value) - else: - parent.ref[field.name] = value - - -def _GetDepth(prefix): - return len(prefix) // _PROGUARD_INDENT_WIDTH - -def Parse(proguard_output): - results = { - 'classes': [], - } - - state = _ParseState() - - for line in proguard_output: - line = line.strip('\r\n') - - m = _PROGUARD_CLASS_RE.match(line) - if m: - results['classes'].append( - state.CreateAndInitClass(m.group(1).replace('/', '.'))) - continue - - if not state.HasCurrentClass(): - continue - - m = _PROGUARD_SUPERCLASS_RE.match(line) - if m: - state.SetSuperClass(m.group(1).replace('/', '.')) - continue - - m = _PROGUARD_SECTION_RE.match(line) - if m: - state.ResetPerSection(m.group(1)) - continue - - m = _PROGUARD_METHOD_RE.match(line) - if m: - state.InitMethod(m.group(1)) - continue - - if not state.ParseAnnotations(): - continue - - m = _PROGUARD_ANNOTATION_RE.match(line) - if m: - # Ignore the annotation package. - state.InitAnnotation(m.group(2).split('/')[-1], _GetDepth(m.group(1))) - continue - - if state.HasCurrentAnnotation(): - m = None - for (element_type, element_re) in _PROGUARD_ELEMENT_RES: - m = element_re.match(line) - if m: - state.InitAnnotationField( - m.group(2), element_type, _GetDepth(m.group(1))) - break - if m: - continue - m = _PROGUARD_ANNOTATION_VALUE_RE.match(line) - if m: - state.UpdateCurrentAnnotationFieldValue( - m.group(2), _GetDepth(m.group(1))) - else: - state.InitMethod(None) - - return results
diff --git a/build/android/pylib/utils/proguard_test.py b/build/android/pylib/utils/proguard_test.py deleted file mode 100755 index b11c299..0000000 --- a/build/android/pylib/utils/proguard_test.py +++ /dev/null
@@ -1,495 +0,0 @@ -#! /usr/bin/env vpython -# Copyright 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import unittest - -from pylib.utils import proguard - -class TestParse(unittest.TestCase): - - def setUp(self): - self.maxDiff = None - - def testClass(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - ' Superclass: java/lang/Object']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': 'java.lang.Object', - 'annotations': {}, - 'methods': [] - } - ] - } - self.assertEquals(expected, actual) - - def testMethod(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Methods (count = 1):', - '- Method: <init>()V']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': {}, - 'methods': [ - { - 'method': '<init>', - 'annotations': {} - } - ] - } - ] - } - self.assertEquals(expected, actual) - - def testClassAnnotation(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Class file attributes (count = 3):', - ' - Annotation [Lorg/example/Annotation;]:', - ' - Annotation [Lorg/example/AnnotationWithValue;]:', - ' - Constant element value [attr \'13\']', - ' - Utf8 [val]', - ' - Annotation [Lorg/example/AnnotationWithTwoValues;]:', - ' - Constant element value [attr1 \'13\']', - ' - Utf8 [val1]', - ' - Constant element value [attr2 \'13\']', - ' - Utf8 [val2]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': { - 'Annotation': None, - 'AnnotationWithValue': {'attr': 'val'}, - 'AnnotationWithTwoValues': {'attr1': 'val1', 'attr2': 'val2'} - }, - 'methods': [] - } - ] - } - self.assertEquals(expected, actual) - - def testClassAnnotationWithArrays(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Class file attributes (count = 3):', - ' - Annotation [Lorg/example/AnnotationWithEmptyArray;]:', - ' - Array element value [arrayAttr]:', - ' - Annotation [Lorg/example/AnnotationWithOneElemArray;]:', - ' - Array element value [arrayAttr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val]', - ' - Annotation [Lorg/example/AnnotationWithTwoElemArray;]:', - ' - Array element value [arrayAttr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val1]', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val2]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': { - 'AnnotationWithEmptyArray': {'arrayAttr': []}, - 'AnnotationWithOneElemArray': {'arrayAttr': ['val']}, - 'AnnotationWithTwoElemArray': {'arrayAttr': ['val1', 'val2']} - }, - 'methods': [] - } - ] - } - self.assertEquals(expected, actual) - - def testNestedClassAnnotations(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Class file attributes (count = 1):', - ' - Annotation [Lorg/example/OuterAnnotation;]:', - ' - Constant element value [outerAttr \'13\']', - ' - Utf8 [outerVal]', - ' - Array element value [outerArr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [outerArrVal1]', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [outerArrVal2]', - ' - Annotation element value [emptyAnn]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:', - ' - Annotation element value [ann]:', - ' - Annotation [Lorg/example/InnerAnnotation;]:', - ' - Constant element value [innerAttr \'13\']', - ' - Utf8 [innerVal]', - ' - Array element value [innerArr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [innerArrVal1]', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [innerArrVal2]', - ' - Annotation element value [emptyInnerAnn]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': { - 'OuterAnnotation': { - 'outerAttr': 'outerVal', - 'outerArr': ['outerArrVal1', 'outerArrVal2'], - 'emptyAnn': None, - 'ann': { - 'innerAttr': 'innerVal', - 'innerArr': ['innerArrVal1', 'innerArrVal2'], - 'emptyInnerAnn': None - } - } - }, - 'methods': [] - } - ] - } - self.assertEquals(expected, actual) - - def testClassArraysOfAnnotations(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Class file attributes (count = 1):', - ' - Annotation [Lorg/example/OuterAnnotation;]:', - ' - Array element value [arrayWithEmptyAnnotations]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:', - ' - Array element value [outerArray]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/InnerAnnotation;]:', - ' - Constant element value [innerAttr \'115\']', - ' - Utf8 [innerVal]', - ' - Array element value [arguments]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/InnerAnnotation$Argument;]:', - ' - Constant element value [arg1Attr \'115\']', - ' - Utf8 [arg1Val]', - ' - Array element value [arg1Array]:', - ' - Constant element value [(default) \'73\']', - ' - Integer [11]', - ' - Constant element value [(default) \'73\']', - ' - Integer [12]', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/InnerAnnotation$Argument;]:', - ' - Constant element value [arg2Attr \'115\']', - ' - Utf8 [arg2Val]', - ' - Array element value [arg2Array]:', - ' - Constant element value [(default) \'73\']', - ' - Integer [21]', - ' - Constant element value [(default) \'73\']', - ' - Integer [22]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': { - 'OuterAnnotation': { - 'arrayWithEmptyAnnotations': [None, None], - 'outerArray': [ - { - 'innerAttr': 'innerVal', - 'arguments': [ - {'arg1Attr': 'arg1Val', 'arg1Array': ['11', '12']}, - {'arg2Attr': 'arg2Val', 'arg2Array': ['21', '22']} - ] - } - ] - } - }, - 'methods': [] - } - ] - } - self.assertEquals(expected, actual) - - def testReadFullClassFileAttributes(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Class file attributes (count = 3):', - ' - Source file attribute:', - ' - Utf8 [Class.java]', - ' - Runtime visible annotations attribute:', - ' - Annotation [Lorg/example/IntValueAnnotation;]:', - ' - Constant element value [value \'73\']', - ' - Integer [19]', - ' - Inner classes attribute (count = 1)', - ' - InnerClassesInfo:', - ' Access flags: 0x9 = public static', - ' - Class [org/example/Class1]', - ' - Class [org/example/Class2]', - ' - Utf8 [OnPageFinishedHelper]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': { - 'IntValueAnnotation': { - 'value': '19', - } - }, - 'methods': [] - } - ] - } - self.assertEquals(expected, actual) - - def testMethodAnnotation(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Methods (count = 1):', - '- Method: Test()V', - ' - Annotation [Lorg/example/Annotation;]:', - ' - Annotation [Lorg/example/AnnotationWithValue;]:', - ' - Constant element value [attr \'13\']', - ' - Utf8 [val]', - ' - Annotation [Lorg/example/AnnotationWithTwoValues;]:', - ' - Constant element value [attr1 \'13\']', - ' - Utf8 [val1]', - ' - Constant element value [attr2 \'13\']', - ' - Utf8 [val2]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': {}, - 'methods': [ - { - 'method': 'Test', - 'annotations': { - 'Annotation': None, - 'AnnotationWithValue': {'attr': 'val'}, - 'AnnotationWithTwoValues': {'attr1': 'val1', 'attr2': 'val2'} - }, - } - ] - } - ] - } - self.assertEquals(expected, actual) - - def testMethodAnnotationWithArrays(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Methods (count = 1):', - '- Method: Test()V', - ' - Annotation [Lorg/example/AnnotationWithEmptyArray;]:', - ' - Array element value [arrayAttr]:', - ' - Annotation [Lorg/example/AnnotationWithOneElemArray;]:', - ' - Array element value [arrayAttr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val]', - ' - Annotation [Lorg/example/AnnotationWithTwoElemArray;]:', - ' - Array element value [arrayAttr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val1]', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val2]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': {}, - 'methods': [ - { - 'method': 'Test', - 'annotations': { - 'AnnotationWithEmptyArray': {'arrayAttr': []}, - 'AnnotationWithOneElemArray': {'arrayAttr': ['val']}, - 'AnnotationWithTwoElemArray': {'arrayAttr': ['val1', 'val2']} - }, - } - ] - } - ] - } - self.assertEquals(expected, actual) - - def testMethodAnnotationWithPrimitivesAndArrays(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Methods (count = 1):', - '- Method: Test()V', - ' - Annotation [Lorg/example/AnnotationPrimitiveThenArray;]:', - ' - Constant element value [attr \'13\']', - ' - Utf8 [val]', - ' - Array element value [arrayAttr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val]', - ' - Annotation [Lorg/example/AnnotationArrayThenPrimitive;]:', - ' - Array element value [arrayAttr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val]', - ' - Constant element value [attr \'13\']', - ' - Utf8 [val]', - ' - Annotation [Lorg/example/AnnotationTwoArrays;]:', - ' - Array element value [arrayAttr1]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val1]', - ' - Array element value [arrayAttr2]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [val2]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': {}, - 'methods': [ - { - 'method': 'Test', - 'annotations': { - 'AnnotationPrimitiveThenArray': {'attr': 'val', - 'arrayAttr': ['val']}, - 'AnnotationArrayThenPrimitive': {'arrayAttr': ['val'], - 'attr': 'val'}, - 'AnnotationTwoArrays': {'arrayAttr1': ['val1'], - 'arrayAttr2': ['val2']} - }, - } - ] - } - ] - } - self.assertEquals(expected, actual) - - def testNestedMethodAnnotations(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Methods (count = 1):', - '- Method: Test()V', - ' - Annotation [Lorg/example/OuterAnnotation;]:', - ' - Constant element value [outerAttr \'13\']', - ' - Utf8 [outerVal]', - ' - Array element value [outerArr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [outerArrVal1]', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [outerArrVal2]', - ' - Annotation element value [emptyAnn]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:', - ' - Annotation element value [ann]:', - ' - Annotation [Lorg/example/InnerAnnotation;]:', - ' - Constant element value [innerAttr \'13\']', - ' - Utf8 [innerVal]', - ' - Array element value [innerArr]:', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [innerArrVal1]', - ' - Constant element value [(default) \'13\']', - ' - Utf8 [innerArrVal2]', - ' - Annotation element value [emptyInnerAnn]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': {}, - 'methods': [ - { - 'method': 'Test', - 'annotations': { - 'OuterAnnotation': { - 'outerAttr': 'outerVal', - 'outerArr': ['outerArrVal1', 'outerArrVal2'], - 'emptyAnn': None, - 'ann': { - 'innerAttr': 'innerVal', - 'innerArr': ['innerArrVal1', 'innerArrVal2'], - 'emptyInnerAnn': None - } - } - }, - } - ] - } - ] - } - self.assertEquals(expected, actual) - - def testMethodArraysOfAnnotations(self): - actual = proguard.Parse( - ['- Program class: org/example/Test', - 'Methods (count = 1):', - '- Method: Test()V', - ' - Annotation [Lorg/example/OuterAnnotation;]:', - ' - Array element value [arrayWithEmptyAnnotations]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/EmptyAnnotation;]:', - ' - Array element value [outerArray]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/InnerAnnotation;]:', - ' - Constant element value [innerAttr \'115\']', - ' - Utf8 [innerVal]', - ' - Array element value [arguments]:', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/InnerAnnotation$Argument;]:', - ' - Constant element value [arg1Attr \'115\']', - ' - Utf8 [arg1Val]', - ' - Array element value [arg1Array]:', - ' - Constant element value [(default) \'73\']', - ' - Integer [11]', - ' - Constant element value [(default) \'73\']', - ' - Integer [12]', - ' - Annotation element value [(default)]:', - ' - Annotation [Lorg/example/InnerAnnotation$Argument;]:', - ' - Constant element value [arg2Attr \'115\']', - ' - Utf8 [arg2Val]', - ' - Array element value [arg2Array]:', - ' - Constant element value [(default) \'73\']', - ' - Integer [21]', - ' - Constant element value [(default) \'73\']', - ' - Integer [22]']) - expected = { - 'classes': [ - { - 'class': 'org.example.Test', - 'superclass': '', - 'annotations': {}, - 'methods': [ - { - 'method': 'Test', - 'annotations': { - 'OuterAnnotation': { - 'arrayWithEmptyAnnotations': [None, None], - 'outerArray': [ - { - 'innerAttr': 'innerVal', - 'arguments': [ - {'arg1Attr': 'arg1Val', 'arg1Array': ['11', '12']}, - {'arg2Attr': 'arg2Val', 'arg2Array': ['21', '22']} - ] - } - ] - } - } - } - ] - } - ] - } - self.assertEquals(expected, actual) - - -if __name__ == '__main__': - unittest.main()
diff --git a/build/android/pylib/utils/repo_utils.py b/build/android/pylib/utils/repo_utils.py index f9d300a..4e1b7a5 100644 --- a/build/android/pylib/utils/repo_utils.py +++ b/build/android/pylib/utils/repo_utils.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -20,3 +20,9 @@ command_line = ['git', 'rev-parse', 'origin/master'] output = cmd_helper.GetCmdOutput(command_line, cwd=in_directory) return output.strip() + + +def GetGitOriginMainHeadSHA1(in_directory): + command_line = ['git', 'rev-parse', 'origin/main'] + output = cmd_helper.GetCmdOutput(command_line, cwd=in_directory) + return output.strip()
diff --git a/build/android/pylib/utils/shared_preference_utils.py b/build/android/pylib/utils/shared_preference_utils.py index ae0d31b..93324c6 100644 --- a/build/android/pylib/utils/shared_preference_utils.py +++ b/build/android/pylib/utils/shared_preference_utils.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,12 +19,19 @@ strings. """ if isinstance(data, dict): - return {UnicodeToStr(key): UnicodeToStr(value) - for key, value in data.iteritems()} - elif isinstance(data, list): + return { + UnicodeToStr(key): UnicodeToStr(value) + for key, value in data.items() + } + if isinstance(data, list): return [UnicodeToStr(element) for element in data] - elif isinstance(data, unicode): - return data.encode('utf-8') + try: + # Python-2 compatibility. + if isinstance(data, unicode): + return data.encode('utf-8') + except NameError: + # Strings are already unicode in python3. + pass return data @@ -80,16 +87,30 @@ shared_pref.Remove(key) except KeyError: logging.warning("Attempted to remove non-existent key %s", key) - for key, value in setting.get('set', {}).iteritems(): - if isinstance(value, bool): + for key, value in setting.get('set', {}).items(): + is_set = False + if not is_set and isinstance(value, bool): shared_pref.SetBoolean(key, value) - elif isinstance(value, basestring): - shared_pref.SetString(key, value) - elif isinstance(value, long) or isinstance(value, int): - shared_pref.SetLong(key, value) - elif isinstance(value, list): + is_set = True + try: + # Python-2 compatibility. + if not is_set and isinstance(value, basestring): + shared_pref.SetString(key, value) + is_set = True + if not is_set and isinstance(value, (long, int)): + shared_pref.SetLong(key, value) + is_set = True + except NameError: + if not is_set and isinstance(value, str): + shared_pref.SetString(key, value) + is_set = True + if not is_set and isinstance(value, int): + shared_pref.SetLong(key, value) + is_set = True + if not is_set and isinstance(value, list): shared_pref.SetStringSet(key, value) - else: + is_set = True + if not is_set: raise ValueError("Given invalid value type %s for key %s" % ( str(type(value)), key)) shared_pref.Commit()
diff --git a/build/android/pylib/utils/simpleperf.py b/build/android/pylib/utils/simpleperf.py index b3ba00e..f096093 100644 --- a/build/android/pylib/utils/simpleperf.py +++ b/build/android/pylib/utils/simpleperf.py
@@ -1,8 +1,9 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import contextlib +import logging import os import shutil import subprocess @@ -10,7 +11,7 @@ import tempfile from devil import devil_env -from devil.android import device_signal +from devil.android import device_signal, device_errors from devil.android.sdk import version_codes from pylib import constants @@ -108,6 +109,7 @@ return 'main' if thread_name.startswith('RenderThread'): return 'render' + raise ValueError('got no matching thread_name') def _GetSpecifiedTID(device, pid, thread_specifier): @@ -156,8 +158,8 @@ @contextlib.contextmanager def RunSimpleperf(device, device_simpleperf_path, package_name, - process_specifier, thread_specifier, profiler_args, - host_out_path): + process_specifier, thread_specifier, events, + profiler_args, host_out_path): pid = _GetSpecifiedPID(device, package_name, process_specifier) tid = _GetSpecifiedTID(device, pid, thread_specifier) if pid is None and tid is None: @@ -167,16 +169,34 @@ profiler_args = list(profiler_args) if profiler_args and profiler_args[0] == 'record': profiler_args.pop(0) + profiler_args.extend(('-e', events)) if '--call-graph' not in profiler_args and '-g' not in profiler_args: profiler_args.append('-g') if '-f' not in profiler_args: profiler_args.extend(('-f', '1000')) + device_out_path = '/data/local/tmp/perf.data' + should_remove_device_out_path = True if '-o' in profiler_args: device_out_path = profiler_args[profiler_args.index('-o') + 1] + should_remove_device_out_path = False else: profiler_args.extend(('-o', device_out_path)) + # Remove the default output to avoid confusion if simpleperf opts not + # to update the file. + file_exists = True + try: + device.adb.Shell('readlink -e ' + device_out_path) + except device_errors.AdbCommandFailedError: + file_exists = False + if file_exists: + logging.warning('%s output file already exists on device', device_out_path) + if not should_remove_device_out_path: + raise RuntimeError('Specified output file \'{}\' already exists, not ' + 'continuing'.format(device_out_path)) + device.adb.Shell('rm -f ' + device_out_path) + if tid: profiler_args.extend(('-t', str(tid))) else: @@ -195,7 +215,18 @@ quiet=True) if completed: adb_shell_simpleperf_process.wait() - device.PullFile(device_out_path, host_out_path) + ret = adb_shell_simpleperf_process.returncode + if ret == 0: + # Successfully gathered a profile + device.PullFile(device_out_path, host_out_path) + else: + logging.warning( + 'simpleperf exited unusually, expected exit 0, got %d', ret + ) + stdout, stderr = adb_shell_simpleperf_process.communicate() + logging.info('stdout: \'%s\', stderr: \'%s\'', stdout, stderr) + raise RuntimeError('simpleperf exited with unexpected code {} ' + '(run with -vv for full stdout/stderr)'.format(ret)) def ConvertSimpleperfToPprof(simpleperf_out_path, build_directory, @@ -216,8 +247,10 @@ report_path = os.path.join(script_dir, 'report.py') report_cmd = [sys.executable, report_path, '-i', simpleperf_out_path] device_lib_path = None - for line in subprocess.check_output( - report_cmd, stderr=subprocess.STDOUT).splitlines(): + output = subprocess.check_output(report_cmd, stderr=subprocess.STDOUT) + if isinstance(output, bytes): + output = output.decode() + for line in output.splitlines(): fields = line.split() if len(fields) < 5: continue
diff --git a/build/android/pylib/utils/test_filter.py b/build/android/pylib/utils/test_filter.py index 6db6243..c532f32 100644 --- a/build/android/pylib/utils/test_filter.py +++ b/build/android/pylib/utils/test_filter.py
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,9 +9,6 @@ _CMDLINE_NAME_SEGMENT_RE = re.compile( r' with(?:out)? \{[^\}]*\}') -class ConflictingPositiveFiltersException(Exception): - """Raised when both filter file and filter argument have positive filters.""" - def ParseFilterFile(input_lines): """Converts test filter file contents to positive and negative pattern lists. @@ -20,7 +17,7 @@ syntax that |input_lines| are expected to follow. See - https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-a-subset-of-the-tests + https://github.com/google/googletest/blob/main/docs/advanced.md#running-a-subset-of-the-tests for description of the syntax that --gtest_filter argument should follow. Args: @@ -50,20 +47,26 @@ '--gtest-filter-file', # New argument. '--test-launcher-filter-file', - dest='test_filter_file', + action='append', + dest='test_filter_files', help='Path to file that contains googletest-style filter strings. ' 'See also //testing/buildbot/filters/README.md.') filter_group = parser.add_mutually_exclusive_group() - filter_group.add_argument( - '-f', '--test-filter', '--gtest_filter', '--gtest-filter', - dest='test_filter', - help='googletest-style filter string.', - default=os.environ.get('GTEST_FILTER')) + filter_group.add_argument('-f', + '--test-filter', + '--gtest_filter', + '--gtest-filter', + dest='test_filters', + action='append', + help='googletest-style filter string.', + default=os.environ.get('GTEST_FILTER')) filter_group.add_argument( '--isolated-script-test-filter', + action='append', + dest='isolated_script_test_filters', help='isolated script filter string. ' - 'Like gtest filter strings, but with :: separators instead of :') + 'Like gtest filter strings, but with :: separators instead of :') def AppendPatternsToFilter(test_filter, positive_patterns=None, @@ -107,35 +110,36 @@ return bool(len(test_filter) > 0 and test_filter[0] != '-') -def InitializeFilterFromArgs(args): +def InitializeFiltersFromArgs(args): """Returns a filter string from the command-line option values. Args: args: an argparse.Namespace instance resulting from a using parser to which the filter options above were added. - - Raises: - ConflictingPositiveFiltersException if both filter file and command line - specify positive filters. """ - test_filter = '' - if args.isolated_script_test_filter: - args.test_filter = args.isolated_script_test_filter.replace('::', ':') - if args.test_filter: - test_filter = _CMDLINE_NAME_SEGMENT_RE.sub( - '', args.test_filter.replace('#', '.')) + test_filters = [] + if args.isolated_script_test_filters: + args.test_filters = [ + isolated_script_test_filter.replace('::', ':') + for isolated_script_test_filter in args.isolated_script_test_filters + ] + if args.test_filters: + for filt in args.test_filters: + test_filters.append( + _CMDLINE_NAME_SEGMENT_RE.sub('', filt.replace('#', '.'))) - if args.test_filter_file: - for test_filter_file in args.test_filter_file.split(';'): + if not args.test_filter_files: + return test_filters + + # At this point it's potentially several files, in a list and ; separated + for test_filter_files in args.test_filter_files: + # At this point it's potentially several files, ; separated + for test_filter_file in test_filter_files.split(';'): + # At this point it's individual files with open(test_filter_file, 'r') as f: - positive_file_patterns, negative_file_patterns = ParseFilterFile(f) - if positive_file_patterns and HasPositivePatterns(test_filter): - raise ConflictingPositiveFiltersException( - 'Cannot specify positive pattern in both filter file and ' + - 'filter command line argument') - test_filter = AppendPatternsToFilter( - test_filter, - positive_patterns=positive_file_patterns, - negative_patterns=negative_file_patterns) + positive_patterns, negative_patterns = ParseFilterFile(f) + filter_string = AppendPatternsToFilter('', positive_patterns, + negative_patterns) + test_filters.append(filter_string) - return test_filter + return test_filters
diff --git a/build/android/pylib/utils/test_filter_test.py b/build/android/pylib/utils/test_filter_test.py index 1ae5a7e..fa07182 100755 --- a/build/android/pylib/utils/test_filter_test.py +++ b/build/android/pylib/utils/test_filter_test.py
@@ -1,9 +1,10 @@ -#!/usr/bin/env vpython -# Copyright 2018 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse +import os import sys import tempfile import unittest @@ -22,7 +23,7 @@ ] actual = test_filter.ParseFilterFile(input_lines) expected = ['positive1', 'positive2', 'positive3'], [] - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def testParseFilterFile_onlyPositive(self): input_lines = [ @@ -31,7 +32,7 @@ ] actual = test_filter.ParseFilterFile(input_lines) expected = ['positive1', 'positive2'], [] - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def testParseFilterFile_onlyNegative(self): input_lines = [ @@ -40,7 +41,7 @@ ] actual = test_filter.ParseFilterFile(input_lines) expected = [], ['negative1', 'negative2'] - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def testParseFilterFile_positiveAndNegative(self): input_lines = [ @@ -51,7 +52,7 @@ ] actual = test_filter.ParseFilterFile(input_lines) expected = ['positive1', 'positive2'], ['negative1', 'negative2'] - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) class InitializeFilterFromArgsTest(unittest.TestCase): @@ -62,9 +63,9 @@ args = parser.parse_args([ '--test-filter', 'FooTest.testFoo:BarTest.testBar']) - expected = 'FooTest.testFoo:BarTest.testBar' - actual = test_filter.InitializeFilterFromArgs(args) - self.assertEquals(actual, expected) + expected = ['FooTest.testFoo:BarTest.testBar'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) def testInitializeJavaStyleFilter(self): parser = argparse.ArgumentParser() @@ -72,9 +73,9 @@ args = parser.parse_args([ '--test-filter', 'FooTest#testFoo:BarTest#testBar']) - expected = 'FooTest.testFoo:BarTest.testBar' - actual = test_filter.InitializeFilterFromArgs(args) - self.assertEquals(actual, expected) + expected = ['FooTest.testFoo:BarTest.testBar'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) def testInitializeBasicIsolatedScript(self): parser = argparse.ArgumentParser() @@ -82,28 +83,32 @@ args = parser.parse_args([ '--isolated-script-test-filter', 'FooTest.testFoo::BarTest.testBar']) - expected = 'FooTest.testFoo:BarTest.testBar' - actual = test_filter.InitializeFilterFromArgs(args) - self.assertEquals(actual, expected) + expected = ['FooTest.testFoo:BarTest.testBar'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) + @unittest.skipIf(os.name == "nt", "Opening NamedTemporaryFile by name " + "doesn't work in Windows.") def testFilterArgWithPositiveFilterInFilterFile(self): parser = argparse.ArgumentParser() test_filter.AddFilterOptions(parser) - with tempfile.NamedTemporaryFile() as tmp_file: + with tempfile.NamedTemporaryFile(mode='w') as tmp_file: tmp_file.write('positive1\npositive2\n-negative2\n-negative3\n') tmp_file.seek(0) args = parser.parse_args([ '--test-filter=-negative1', '--test-launcher-filter-file', tmp_file.name]) - expected = 'positive1:positive2-negative1:negative2:negative3' - actual = test_filter.InitializeFilterFromArgs(args) - self.assertEquals(actual, expected) + expected = ['-negative1', 'positive1:positive2-negative2:negative3'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) + @unittest.skipIf(os.name == "nt", "Opening NamedTemporaryFile by name " + "doesn't work in Windows.") def testFilterFileWithPositiveFilterInFilterArg(self): parser = argparse.ArgumentParser() test_filter.AddFilterOptions(parser) - with tempfile.NamedTemporaryFile() as tmp_file: + with tempfile.NamedTemporaryFile(mode='w') as tmp_file: tmp_file.write('-negative2\n-negative3\n') tmp_file.seek(0) args = parser.parse_args([ @@ -111,89 +116,103 @@ 'positive1:positive2-negative1', '--test-launcher-filter-file', tmp_file.name]) - expected = 'positive1:positive2-negative1:negative2:negative3' - actual = test_filter.InitializeFilterFromArgs(args) - self.assertEquals(actual, expected) + expected = ['positive1:positive2-negative1', '-negative2:negative3'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) + @unittest.skipIf(os.name == "nt", "Opening NamedTemporaryFile by name " + "doesn't work in Windows.") def testPositiveFilterInBothFileAndArg(self): parser = argparse.ArgumentParser() test_filter.AddFilterOptions(parser) - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write('positive1\n') + with tempfile.NamedTemporaryFile(mode='w') as tmp_file: + tmp_file.write('positive2-negative2\n') tmp_file.seek(0) args = parser.parse_args([ - '--test-filter', - 'positive2', - '--test-launcher-filter-file', - tmp_file.name]) - with self.assertRaises(test_filter.ConflictingPositiveFiltersException): - test_filter.InitializeFilterFromArgs(args) + '--test-filter', 'positive1-negative1', '--test-launcher-filter-file', + tmp_file.name + ]) + expected = ['positive1-negative1', 'positive2-negative2'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) + @unittest.skipIf(os.name == "nt", "Opening NamedTemporaryFile by name " + "doesn't work in Windows.") def testFilterArgWithFilterFileAllNegative(self): parser = argparse.ArgumentParser() test_filter.AddFilterOptions(parser) - with tempfile.NamedTemporaryFile() as tmp_file: + with tempfile.NamedTemporaryFile(mode='w') as tmp_file: tmp_file.write('-negative3\n-negative4\n') tmp_file.seek(0) args = parser.parse_args([ '--test-filter=-negative1:negative2', '--test-launcher-filter-file', tmp_file.name]) - expected = '-negative1:negative2:negative3:negative4' - actual = test_filter.InitializeFilterFromArgs(args) - self.assertEquals(actual, expected) + expected = ['-negative1:negative2', '-negative3:negative4'] + actual = test_filter.InitializeFiltersFromArgs(args) + self.assertEqual(actual, expected) class AppendPatternsToFilter(unittest.TestCase): def testAllEmpty(self): expected = '' actual = test_filter.AppendPatternsToFilter('', [], []) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendOnlyPositiveToEmptyFilter(self): expected = 'positive' actual = test_filter.AppendPatternsToFilter('', ['positive']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendOnlyNegativeToEmptyFilter(self): expected = '-negative' actual = test_filter.AppendPatternsToFilter('', negative_patterns=['negative']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendToEmptyFilter(self): expected = 'positive-negative' actual = test_filter.AppendPatternsToFilter('', ['positive'], ['negative']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendToPositiveOnlyFilter(self): expected = 'positive1:positive2-negative' actual = test_filter.AppendPatternsToFilter('positive1', ['positive2'], ['negative']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendToNegativeOnlyFilter(self): expected = 'positive-negative1:negative2' actual = test_filter.AppendPatternsToFilter('-negative1', ['positive'], ['negative2']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendPositiveToFilter(self): expected = 'positive1:positive2-negative1' actual = test_filter.AppendPatternsToFilter('positive1-negative1', ['positive2']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendNegativeToFilter(self): expected = 'positive1-negative1:negative2' actual = test_filter.AppendPatternsToFilter('positive1-negative1', negative_patterns=['negative2']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendBothToFilter(self): expected = 'positive1:positive2-negative1:negative2' actual = test_filter.AppendPatternsToFilter('positive1-negative1', positive_patterns=['positive2'], negative_patterns=['negative2']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendMultipleToFilter(self): expected = 'positive1:positive2:positive3-negative1:negative2:negative3' actual = test_filter.AppendPatternsToFilter('positive1-negative1', ['positive2', 'positive3'], ['negative2', 'negative3']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testRepeatedAppendToFilter(self): expected = 'positive1:positive2:positive3-negative1:negative2:negative3' filter_string = test_filter.AppendPatternsToFilter('positive1-negative1', @@ -201,32 +220,36 @@ ['negative2']) actual = test_filter.AppendPatternsToFilter(filter_string, ['positive3'], ['negative3']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testAppendHashSeparatedPatternsToFilter(self): expected = 'positive.test1:positive.test2-negative.test1:negative.test2' actual = test_filter.AppendPatternsToFilter('positive#test1-negative#test1', ['positive#test2'], ['negative#test2']) - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) class HasPositivePatterns(unittest.TestCase): def testEmpty(self): expected = False actual = test_filter.HasPositivePatterns('') - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testHasOnlyPositive(self): expected = True actual = test_filter.HasPositivePatterns('positive') - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testHasOnlyNegative(self): expected = False actual = test_filter.HasPositivePatterns('-negative') - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) + def testHasBoth(self): expected = True actual = test_filter.HasPositivePatterns('positive-negative') - self.assertEquals(actual, expected) + self.assertEqual(actual, expected) if __name__ == '__main__':
diff --git a/build/android/pylib/utils/time_profile.py b/build/android/pylib/utils/time_profile.py index 094799c..54b96c2 100644 --- a/build/android/pylib/utils/time_profile.py +++ b/build/android/pylib/utils/time_profile.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,7 +6,7 @@ import time -class TimeProfile(object): +class TimeProfile: """Class for simple profiling of action, with logging of cost.""" def __init__(self, description='operation'):
diff --git a/build/android/pylib/utils/xvfb.py b/build/android/pylib/utils/xvfb.py index cb9d50e..6ab24af 100644 --- a/build/android/pylib/utils/xvfb.py +++ b/build/android/pylib/utils/xvfb.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -16,7 +16,7 @@ return sys.platform.startswith('linux') -class Xvfb(object): +class Xvfb: """Class to start and stop Xvfb if relevant. Nop if not Linux.""" def __init__(self):
diff --git a/build/android/pylib/valgrind_tools.py b/build/android/pylib/valgrind_tools.py index 4689dc3..8c00705 100644 --- a/build/android/pylib/valgrind_tools.py +++ b/build/android/pylib/valgrind_tools.py
@@ -1,10 +1,11 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # pylint: disable=R0201 -from __future__ import print_function + + import logging import sys @@ -34,7 +35,7 @@ EXTRA_OPTIONS = 'strict_memcmp=0,use_sigaltstack=1' def __init__(self, device): - super(AddressSanitizerTool, self).__init__() + super().__init__() self._device = device @classmethod @@ -91,10 +92,10 @@ ctor = TOOL_REGISTRY.get(tool_name) if ctor: return ctor(device) - else: - print('Unknown tool %s, available tools: %s' % (tool_name, ', '.join( - sorted(TOOL_REGISTRY.keys())))) - sys.exit(1) + print('Unknown tool %s, available tools: %s' % + (tool_name, ', '.join(sorted(TOOL_REGISTRY.keys())))) + sys.exit(1) + def PushFilesForTool(tool_name, device): """Pushes the files required for |tool_name| to |device|.
diff --git a/build/android/resource_sizes.gni b/build/android/resource_sizes.gni index 2c91749..c599bbb 100644 --- a/build/android/resource_sizes.gni +++ b/build/android/resource_sizes.gni
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -32,10 +32,7 @@ "@WrappedPath(.)", ] - data = [ - "//.vpython", - "//.vpython3", - ] + data = [] if (defined(invoker.trichrome_chrome_path)) { data += [ invoker.trichrome_chrome_path, @@ -78,18 +75,15 @@ # relative to $root_build_dir. The resulting JSON file is written to # "$root_build_dir/config/${invoker.name}_size_config.json". # -# Variables: -# name: The name of the path to the generated size config JSON file. -# mapping_files: List of mapping files. -# to_resource_sizes_py: Scope containing data to pass to resource_sizes.py, -# processed by generate_commit_size_analysis.py. -# supersize_input_file: Main input for SuperSize. +# Refer to tools/binary_size/generate_commit_size_analysis.py for JSON schema. +# template("android_size_bot_config") { _full_target_name = get_label_info(target_name, "label_no_toolchain") _out_json = { _HEADER = "Written by build target '${_full_target_name}'" forward_variables_from(invoker, [ + "archive_files", "mapping_files", "to_resource_sizes_py", "supersize_input_file",
diff --git a/build/android/resource_sizes.py b/build/android/resource_sizes.py index c592970..05ee86c 100755 --- a/build/android/resource_sizes.py +++ b/build/android/resource_sizes.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright (c) 2011 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2011 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,7 +8,6 @@ More information at //docs/speed/binary_size/metrics.md. """ -from __future__ import print_function import argparse import collections @@ -33,11 +32,11 @@ from pylib.constants import host_paths _AAPT_PATH = lazy.WeakConstant(lambda: build_tools.GetPath('aapt')) -_BUILD_UTILS_PATH = os.path.join( - host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'gyp') - -with host_paths.SysPath(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build')): - import gn_helpers # pylint: disable=import-error +_ANDROID_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', + 'android', 'gyp') +_BUILD_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', 'util') +_READOBJ_PATH = os.path.join(constants.ANDROID_NDK_ROOT, 'toolchains', 'llvm', + 'prebuilt', 'linux-x86_64', 'bin', 'llvm-readobj') with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): import perf_tests_results_helper # pylint: disable=import-error @@ -45,12 +44,12 @@ with host_paths.SysPath(host_paths.TRACING_PATH): from tracing.value import convert_chart_json # pylint: disable=import-error -with host_paths.SysPath(_BUILD_UTILS_PATH, 0): +with host_paths.SysPath(_ANDROID_UTILS_PATH, 0): from util import build_utils # pylint: disable=import-error - from util import zipalign # pylint: disable=import-error - -zipalign.ApplyZipFileZipAlignFix() +with host_paths.SysPath(_BUILD_UTILS_PATH, 0): + from lib.results import result_sink # pylint: disable=import-error + from lib.results import result_types # pylint: disable=import-error # Captures an entire config from aapt output. _AAPT_CONFIG_PATTERN = r'config %s:(.*?)config [a-zA-Z-]+:' @@ -89,7 +88,7 @@ } -class _AccumulatingReporter(object): +class _AccumulatingReporter: def __init__(self): self._combined_metrics = collections.defaultdict(int) @@ -98,26 +97,25 @@ def DumpReports(self, report_func): for (graph_title, trace_title, - units), value in sorted(self._combined_metrics.iteritems()): + units), value in sorted(self._combined_metrics.items()): report_func(graph_title, trace_title, value, units) class _ChartJsonReporter(_AccumulatingReporter): def __init__(self, chartjson): - super(_ChartJsonReporter, self).__init__() + super().__init__() self._chartjson = chartjson self.trace_title_prefix = '' def __call__(self, graph_title, trace_title, value, units): - super(_ChartJsonReporter, self).__call__(graph_title, trace_title, value, - units) + super().__call__(graph_title, trace_title, value, units) perf_tests_results_helper.ReportPerfResult( self._chartjson, graph_title, self.trace_title_prefix + trace_title, value, units) def SynthesizeTotals(self, unique_method_count): - for tup, value in sorted(self._combined_metrics.iteritems()): + for tup, value in sorted(self._combined_metrics.items()): graph_title, trace_title, units = tup if trace_title == 'unique methods': value = unique_method_count @@ -167,35 +165,34 @@ return start_of_central_directory - end_of_last_file -def _RunReadelf(so_path, options, tool_prefix=''): - return cmd_helper.GetCmdOutput( - [tool_prefix + 'readelf'] + options + [so_path]) +def _RunReadobj(so_path, options): + return cmd_helper.GetCmdOutput([_READOBJ_PATH, '--elf-output-style=GNU'] + + options + [so_path]) -def _ExtractLibSectionSizesFromApk(apk_path, lib_path, tool_prefix): +def _ExtractLibSectionSizesFromApk(apk_path, lib_path): with Unzip(apk_path, filename=lib_path) as extracted_lib_path: grouped_section_sizes = collections.defaultdict(int) no_bits_section_sizes, section_sizes = _CreateSectionNameSizeMap( - extracted_lib_path, tool_prefix) - for group_name, section_names in _READELF_SIZES_METRICS.iteritems(): + extracted_lib_path) + for group_name, section_names in _READELF_SIZES_METRICS.items(): for section_name in section_names: if section_name in section_sizes: grouped_section_sizes[group_name] += section_sizes.pop(section_name) # Consider all NOBITS sections as .bss. - grouped_section_sizes['bss'] = sum( - v for v in no_bits_section_sizes.itervalues()) + grouped_section_sizes['bss'] = sum(no_bits_section_sizes.values()) # Group any unknown section headers into the "other" group. - for section_header, section_size in section_sizes.iteritems(): + for section_header, section_size in section_sizes.items(): sys.stderr.write('Unknown elf section header: %s\n' % section_header) grouped_section_sizes['other'] += section_size return grouped_section_sizes -def _CreateSectionNameSizeMap(so_path, tool_prefix): - stdout = _RunReadelf(so_path, ['-S', '--wide'], tool_prefix) +def _CreateSectionNameSizeMap(so_path): + stdout = _RunReadobj(so_path, ['-S', '--wide']) section_sizes = {} no_bits_section_sizes = {} # Matches [ 2] .hash HASH 00000000006681f0 0001f0 003154 04 A 3 0 8 @@ -212,18 +209,19 @@ output = cmd_helper.GetCmdOutput([ _AAPT_PATH.read(), 'd', 'xmltree', apk_path, 'AndroidManifest.xml']) - def parse_attr(name): + def parse_attr(namespace, name): # android:extractNativeLibs(0x010104ea)=(type 0x12)0x0 # android:extractNativeLibs(0x010104ea)=(type 0x12)0xffffffff # dist:onDemand=(type 0x12)0xffffffff - m = re.search(name + r'(?:\(.*?\))?=\(type .*?\)(\w+)', output) + m = re.search( + f'(?:{namespace}:)?{name}' + r'(?:\(.*?\))?=\(type .*?\)(\w+)', output) return m and int(m.group(1), 16) - skip_extract_lib = bool(parse_attr('android:extractNativeLibs')) - sdk_version = parse_attr('android:minSdkVersion') - is_feature_split = parse_attr('android:isFeatureSplit') + skip_extract_lib = bool(parse_attr('android', 'extractNativeLibs')) + sdk_version = parse_attr('android', 'minSdkVersion') + is_feature_split = parse_attr('android', 'isFeatureSplit') # Can use <dist:on-demand>, or <module dist:onDemand="true">. - on_demand = parse_attr('dist:onDemand') or 'dist:on-demand' in output + on_demand = parse_attr('dist', 'onDemand') or 'on-demand' in output on_demand = bool(on_demand and is_feature_split) return sdk_version, skip_extract_lib, on_demand @@ -265,7 +263,7 @@ config_count = num_translations - 2 size = 0 - for res_id, string_val in en_strings.iteritems(): + for res_id, string_val in en_strings.items(): if string_val == fr_strings[res_id]: string_size = len(string_val) # 7 bytes is the per-entry overhead (not specific to any string). See @@ -294,7 +292,7 @@ return output -class _FileGroup(object): +class _FileGroup: """Represents a category that apk files can fall into.""" def __init__(self, name): @@ -342,7 +340,6 @@ report_func, dex_stats_collector, out_dir, - tool_prefix, apks_path=None, split_name=None): """Analyse APK to determine size contributions of different file classes. @@ -400,8 +397,13 @@ is_webview = 'WebView' in orig_filename is_monochrome = 'Monochrome' in orig_filename is_library = 'Library' in orig_filename + is_trichrome = 'TrichromeChrome' in orig_filename + # WebView is always a shared APK since other apps load it. + # Library is always shared since it's used by chrome and webview + # Chrome is always shared since renderers can't access dex otherwise + # (see DexFixer). is_shared_apk = sdk_version >= 24 and (is_monochrome or is_webview - or is_library) + or is_library or is_trichrome) # Dex decompression overhead varies by Android version. if sdk_version < 21: # JellyBean & KitKat @@ -426,8 +428,14 @@ should_extract_lib = not skip_extract_lib and basename.startswith('lib') native_code.AddZipInfo( member, extracted_multiplier=int(should_extract_lib)) - elif filename.endswith('.dex'): - java_code.AddZipInfo(member, extracted_multiplier=dex_multiplier) + elif filename.startswith('classes') and filename.endswith('.dex'): + # Android P+, uncompressed dex does not need to be extracted. + compressed = member.compress_type != zipfile.ZIP_STORED + multiplier = dex_multiplier + if not compressed and sdk_version >= 28: + multiplier -= 1 + + java_code.AddZipInfo(member, extracted_multiplier=multiplier) elif re.search(_RE_NON_LANGUAGE_PAK, filename): native_resources_no_translations.AddZipInfo(member) elif filename.endswith('.pak') or filename.endswith('.lpak'): @@ -492,9 +500,15 @@ report_func('Uncompressed', group.name + ' size', uncompressed_size, 'bytes') - if group is java_code and is_shared_apk: + if group is java_code: # Updates are compiled using quicken, but system image uses speed-profile. - extracted_size = int(uncompressed_size * speed_profile_dex_multiplier) + multiplier = speed_profile_dex_multiplier + + # Android P+, uncompressed dex does not need to be extracted. + compressed = uncompressed_size != actual_size + if not compressed and sdk_version >= 28: + multiplier -= 1 + extracted_size = int(uncompressed_size * multiplier) total_install_size_android_go += extracted_size report_func('InstallBreakdownGo', group.name + ' size', actual_size + extracted_size, 'bytes') @@ -512,9 +526,8 @@ report_func('InstallSize', 'APK size', total_apk_size, 'bytes') report_func('InstallSize', 'Estimated installed size', int(total_install_size), 'bytes') - if is_shared_apk: - report_func('InstallSize', 'Estimated installed size (Android Go)', - int(total_install_size_android_go), 'bytes') + report_func('InstallSize', 'Estimated installed size (Android Go)', + int(total_install_size_android_go), 'bytes') transfer_size = _CalculateCompressedSize(apk_path) report_func('TransferSize', 'Transfer size (deflate)', transfer_size, 'bytes') @@ -529,10 +542,9 @@ main_lib_info = native_code.FindLargest() native_code_unaligned_size = 0 for lib_info in native_code.AllEntries(): - section_sizes = _ExtractLibSectionSizesFromApk(apk_path, lib_info.filename, - tool_prefix) - native_code_unaligned_size += sum( - v for k, v in section_sizes.iteritems() if k != 'bss') + section_sizes = _ExtractLibSectionSizesFromApk(apk_path, lib_info.filename) + native_code_unaligned_size += sum(v for k, v in section_sizes.items() + if k != 'bss') # Size of main .so vs remaining. if lib_info == main_lib_info: main_lib_size = lib_info.file_size @@ -540,7 +552,7 @@ secondary_size = native_code.ComputeUncompressedSize() - main_lib_size report_func('Specifics', 'other lib size', secondary_size, 'bytes') - for metric_name, size in section_sizes.iteritems(): + for metric_name, size in section_sizes.items(): report_func('MainLibInfo', metric_name, size, 'bytes') # Main metric that we want to monitor for jumps. @@ -634,7 +646,7 @@ compressor = zlib.compressobj() total_size = 0 with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(CHUNK_SIZE), ''): + for chunk in iter(lambda: f.read(CHUNK_SIZE), b''): total_size += len(compressor.compress(chunk)) total_size += len(compressor.flush()) return total_size @@ -652,7 +664,7 @@ yield unzipped_files[0] -def _ConfigOutDirAndToolsPrefix(out_dir): +def _ConfigOutDir(out_dir): if out_dir: constants.SetOutputDirectory(out_dir) else: @@ -661,10 +673,8 @@ constants.CheckOutputDirectory() out_dir = constants.GetOutDirectory() except Exception: # pylint: disable=broad-except - return out_dir, '' - build_vars = gn_helpers.ReadBuildVars(out_dir) - tool_prefix = os.path.join(out_dir, build_vars['android_tool_prefix']) - return out_dir, tool_prefix + pass + return out_dir def _IterSplits(namelist): @@ -686,16 +696,15 @@ temp_file.flush() -def _AnalyzeApkOrApks(report_func, apk_path, args): +def _AnalyzeApkOrApks(report_func, apk_path, out_dir): # Create DexStatsCollector here to track unique methods across base & chrome # modules. dex_stats_collector = method_count.DexStatsCollector() - out_dir, tool_prefix = _ConfigOutDirAndToolsPrefix(args.out_dir) if apk_path.endswith('.apk'): sdk_version, _, _ = _ParseManifestAttributes(apk_path) _AnalyzeInternal(apk_path, sdk_version, report_func, dex_stats_collector, - out_dir, tool_prefix) + out_dir) elif apk_path.endswith('.apks'): with tempfile.NamedTemporaryFile(suffix='.apk') as f: with zipfile.ZipFile(apk_path) as z: @@ -726,7 +735,6 @@ inner_report_func, inner_dex_stats_collector, out_dir, - tool_prefix, apks_path=apk_path, split_name=split_name) report_func('DFM_' + split_name, 'Size with hindi', size, 'bytes') @@ -772,13 +780,14 @@ for prefix, path in specs: if path: reporter.trace_title_prefix = prefix - child_dex_stats_collector = _AnalyzeApkOrApks(reporter, path, args) + child_dex_stats_collector = _AnalyzeApkOrApks(reporter, path, + args.out_dir) dex_stats_collector.MergeFrom(prefix, child_dex_stats_collector) if any(path for _, path in specs): reporter.SynthesizeTotals(dex_stats_collector.GetUniqueMethodCount()) else: - _AnalyzeApkOrApks(reporter, args.input, args) + _AnalyzeApkOrApks(reporter, args.input, args.out_dir) if chartjson: _DumpChartJson(args, chartjson) @@ -813,11 +822,12 @@ histogram_path = os.path.join(args.output_dir, 'perf_results.json') logging.critical('Dumping histograms to %s', histogram_path) - with open(histogram_path, 'w') as json_file: + with open(histogram_path, 'wb') as json_file: json_file.write(histogram_result.stdout) def main(): + build_utils.InitLogging('RESOURCE_SIZES_DEBUG') argparser = argparse.ArgumentParser(description='Print APK size metrics.') argparser.add_argument( '--min-pak-resource-size', @@ -875,12 +885,14 @@ '--trichrome-library', help='Path to Trichrome Library .apk') args = argparser.parse_args() + args.out_dir = _ConfigOutDir(args.out_dir) devil_chromium.Initialize(output_directory=args.out_dir) # TODO(bsheedy): Remove this once uses of --chartjson have been removed. if args.chartjson: args.output_format = 'chartjson' + result_sink_client = result_sink.TryInitClient() isolated_script_output = {'valid': False, 'failures': []} test_name = 'resource_sizes (%s)' % os.path.basename(args.input) @@ -904,6 +916,13 @@ json.dump(isolated_script_output, output_file) with open(args.isolated_script_test_output, 'w') as output_file: json.dump(isolated_script_output, output_file) + if result_sink_client: + status = result_types.PASS + if not isolated_script_output['valid']: + status = result_types.UNKNOWN + elif isolated_script_output['failures']: + status = result_types.FAIL + result_sink_client.Post(test_name, status, None, None, None) if __name__ == '__main__':
diff --git a/build/android/resource_sizes.pydeps b/build/android/resource_sizes.pydeps index d956f5b..86db3ff 100644 --- a/build/android/resource_sizes.pydeps +++ b/build/android/resource_sizes.pydeps
@@ -43,12 +43,15 @@ ../../third_party/catapult/tracing/tracing/value/convert_chart_json.py ../../third_party/catapult/tracing/tracing_project.py ../gn_helpers.py +../util/lib/__init__.py ../util/lib/common/perf_result_data_type.py ../util/lib/common/perf_tests_results_helper.py +../util/lib/results/__init__.py +../util/lib/results/result_sink.py +../util/lib/results/result_types.py devil_chromium.py gyp/util/__init__.py gyp/util/build_utils.py -gyp/util/zipalign.py method_count.py pylib/__init__.py pylib/constants/__init__.py
diff --git a/build/android/screenshot.py b/build/android/screenshot.py index 523d859..6366e85 100755 --- a/build/android/screenshot.py +++ b/build/android/screenshot.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2015 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/stacktrace/BUILD.gn b/build/android/stacktrace/BUILD.gn index ce13a15..0501a96 100644 --- a/build/android/stacktrace/BUILD.gn +++ b/build/android/stacktrace/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,10 +9,7 @@ # Avoid using java_prebuilt() to ensure all uses go through the checked-in # wrapper script. - input_jars_paths = [ - "//third_party/proguard/lib/proguard603.jar", - "//third_party/proguard/lib/retrace603.jar", - ] + input_jars_paths = [ "//third_party/r8/lib/r8.jar" ] } # Use the checked-in copy of the wrapper script & .jar rather than the built @@ -20,9 +17,8 @@ group("java_deobfuscate") { data = [ "java_deobfuscate.py", - "java_deobfuscate.jar", - "//third_party/proguard/lib/proguard603.jar", - "//third_party/proguard/lib/retrace603.jar", + "java_deobfuscate_java.jar", + "//third_party/r8/lib/r8.jar", ] deps = [ "//third_party/jdk:java_data" ] }
diff --git a/build/android/stacktrace/README.md b/build/android/stacktrace/README.md index 58ea94b..528af22 100644 --- a/build/android/stacktrace/README.md +++ b/build/android/stacktrace/README.md
@@ -14,8 +14,8 @@ ## Update Instructions: - ninja -C out/Release java_deobfuscate - cp out/Release/lib.java/build/android/stacktrace/java_deobfuscate.jar build/android/stacktrace + ninja -C out/Release java_deobfuscate_java + cp out/Release/lib.java/build/android/stacktrace/java_deobfuscate_java.jar build/android/stacktrace # stackwalker.py
diff --git a/build/android/stacktrace/crashpad_stackwalker.py b/build/android/stacktrace/crashpad_stackwalker.py index 9616a54..9703b7c 100755 --- a/build/android/stacktrace/crashpad_stackwalker.py +++ b/build/android/stacktrace/crashpad_stackwalker.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/stacktrace/java/org/chromium/build/FlushingReTrace.java b/build/android/stacktrace/java/org/chromium/build/FlushingReTrace.java index baa9313..3e27197 100644 --- a/build/android/stacktrace/java/org/chromium/build/FlushingReTrace.java +++ b/build/android/stacktrace/java/org/chromium/build/FlushingReTrace.java
@@ -1,18 +1,21 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. +// Copyright 2017 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. package org.chromium.build; +import com.android.tools.r8.DiagnosticsHandler; +import com.android.tools.r8.retrace.ProguardMappingSupplier; +import com.android.tools.r8.retrace.Retrace; +import com.android.tools.r8.retrace.RetraceCommand; +import com.android.tools.r8.retrace.StackTraceSupplier; + import java.io.BufferedReader; -import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.LineNumberReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; - -import proguard.retrace.ReTrace; +import java.util.Collections; +import java.util.List; /** * A wrapper around ReTrace that: @@ -40,6 +43,9 @@ // Normal stack trace lines look like: // \tat org.chromium.chrome.browser.tab.Tab.handleJavaCrash(Tab.java:682) + "(?:.*?(?::|\\bat)\\s+%c\\.%m\\s*\\(\\s*%s(?:\\s*:\\s*%l\\s*)?\\))|" + // Stack trace from crbug.com/1300215 looks like: + // 0xffffffff (chromium-TrichromeChromeGoogle.aab-canary-490400033: 70) ii2.p + + "(?:.*?\\(\\s*%s(?:\\s*:\\s*%l\\s*)?\\)\\s*%c\\.%m)|" // E.g.: Caused by: java.lang.NullPointerException: Attempt to read from field 'int bLA' // on a null object reference + "(?:.*java\\.lang\\.NullPointerException.*[\"']%t\\s*%c\\.(?:%f|%m\\(%a\\))[\"'].*)|" @@ -93,24 +99,48 @@ usage(); } - File mappingFile = new File(args[0]); try { - LineNumberReader reader = new LineNumberReader( - new BufferedReader(new InputStreamReader(System.in, "UTF-8"))); + ProguardMappingSupplier mappingSupplier = + ProguardMappingSupplier.builder() + .setProguardMapProducer(() -> new FileInputStream(args[0])) + .build(); + // Force earger parsing of .mapping file (~10 second operation). It otherwise would + // not happen until the first line of input is received. + // https://crbug.com/1351023 + mappingSupplier.createRetracer(new DiagnosticsHandler() {}); - // Enabling autoFlush is the main difference from ReTrace.main(). - boolean autoFlush = true; - PrintWriter writer = - new PrintWriter(new OutputStreamWriter(System.out, "UTF-8"), autoFlush); + // This whole command was given to us by the R8 team in b/234758957. + RetraceCommand retraceCommand = + RetraceCommand.builder() + .setMappingSupplier(mappingSupplier) + .setRetracedStackTraceConsumer( + retraced -> retraced.forEach(System.out::println)) + .setRegularExpression(LINE_PARSE_REGEX) + .setStackTrace(new StackTraceSupplier() { + final BufferedReader mReader = new BufferedReader( + new InputStreamReader(System.in, "UTF-8")); - boolean verbose = false; - new ReTrace(LINE_PARSE_REGEX, verbose, mappingFile).retrace(reader, writer); + @Override + public List<String> get() { + try { + String line = mReader.readLine(); + if (line == null) { + return null; + } + return Collections.singletonList(line); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + }) + .build(); + Retrace.run(retraceCommand); } catch (IOException ex) { // Print a verbose stack trace. ex.printStackTrace(); System.exit(1); } - System.exit(0); } }
diff --git a/build/android/stacktrace/java_deobfuscate.jar b/build/android/stacktrace/java_deobfuscate.jar deleted file mode 100644 index 36a1b70..0000000 --- a/build/android/stacktrace/java_deobfuscate.jar +++ /dev/null Binary files differ
diff --git a/build/android/stacktrace/java_deobfuscate.py b/build/android/stacktrace/java_deobfuscate.py index 8c231ec..fa872d9 100755 --- a/build/android/stacktrace/java_deobfuscate.py +++ b/build/android/stacktrace/java_deobfuscate.py
@@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Wrapper script for java_deobfuscate. @@ -14,15 +14,11 @@ DIR_SOURCE_ROOT = os.path.normpath( os.path.join(os.path.dirname(__file__), '../../../')) - def main(): classpath = [ os.path.join(DIR_SOURCE_ROOT, 'build', 'android', 'stacktrace', - 'java_deobfuscate.jar'), - os.path.join(DIR_SOURCE_ROOT, 'third_party', 'proguard', 'lib', - 'proguard603.jar'), - os.path.join(DIR_SOURCE_ROOT, 'third_party', 'proguard', 'lib', - 'retrace603.jar'), + 'java_deobfuscate_java.jar'), + os.path.join(DIR_SOURCE_ROOT, 'third_party', 'r8', 'lib', 'r8.jar') ] java_path = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current', 'bin', 'java') @@ -32,6 +28,7 @@ 'org.chromium.build.FlushingReTrace' ] cmd.extend(sys.argv[1:]) + os.execvp(cmd[0], cmd)
diff --git a/build/android/stacktrace/java_deobfuscate_java.jar b/build/android/stacktrace/java_deobfuscate_java.jar new file mode 100644 index 0000000..8f31b76 --- /dev/null +++ b/build/android/stacktrace/java_deobfuscate_java.jar Binary files differ
diff --git a/build/android/stacktrace/java_deobfuscate_test.py b/build/android/stacktrace/java_deobfuscate_test.py index 1bf81c9..de236ef 100755 --- a/build/android/stacktrace/java_deobfuscate_test.py +++ b/build/android/stacktrace/java_deobfuscate_test.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Tests for java_deobfuscate.""" @@ -52,6 +52,7 @@ 'Caused by: FOO: Error message', '\tat FOO.bar(PG:1)', '\t at\t FOO.bar\t (\t PG:\t 1\t )', + '0xfff \t( \tPG:\t 1 \t)\tFOO.bar', ('Unable to start activity ComponentInfo{garbage.in/here.test}:' ' java.lang.NullPointerException: Attempt to invoke interface method' ' \'void FOO.bar(int,android.os.Bundle)\' on a null object reference'), @@ -83,6 +84,7 @@ '\tat this.was.Deobfuscated.someMethod(Deobfuscated.java:65)', ('\t at\t this.was.Deobfuscated.someMethod\t ' '(\t Deobfuscated.java:\t 65\t )'), + '0xfff \t( \tDeobfuscated.java:\t 65 \t)\tthis.was.Deobfuscated.someMethod', ('Unable to start activity ComponentInfo{garbage.in/here.test}:' ' java.lang.NullPointerException: Attempt to invoke interface method' ' \'void this.was.Deobfuscated.someMethod(int,android.os.Bundle)\' on a' @@ -101,12 +103,12 @@ class JavaDeobfuscateTest(unittest.TestCase): def __init__(self, *args, **kwargs): - super(JavaDeobfuscateTest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._map_file = None def setUp(self): self._map_file = tempfile.NamedTemporaryFile() - self._map_file.write(TEST_MAP) + self._map_file.write(TEST_MAP.encode('utf-8')) self._map_file.flush() def tearDown(self): @@ -124,8 +126,8 @@ cmd = [_JAVA_DEOBFUSCATE_PATH, self._map_file.name] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - proc_output, _ = proc.communicate(''.join(input_lines)) - actual_output_lines = proc_output.splitlines(True) + proc_output, _ = proc.communicate(''.join(input_lines).encode()) + actual_output_lines = proc_output.decode().splitlines(True) for actual, expected in zip(actual_output_lines, expected_output_lines): self.assertTrue( actual == expected or actual.replace('bar', 'someMethod') == expected,
diff --git a/build/android/stacktrace/stackwalker.py b/build/android/stacktrace/stackwalker.py index 4f2782f..ad60e99 100755 --- a/build/android/stacktrace/stackwalker.py +++ b/build/android/stacktrace/stackwalker.py
@@ -1,10 +1,9 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import argparse import os
diff --git a/build/android/test/BUILD.gn b/build/android/test/BUILD.gn index d5f8609..e9bbbce 100644 --- a/build/android/test/BUILD.gn +++ b/build/android/test/BUILD.gn
@@ -1,30 +1,83 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/android/android_nocompile.gni") +import("missing_symbol_test.gni") import("nocompile_gn/nocompile_sources.gni") -if (enable_java_templates) { - android_nocompile_test_suite("android_lint_test") { - # Depend on lint Python script so that the action is re-run whenever the lint script is - # modified. - pydeps = [ "//build/android/gyp/lint.pydeps" ] - tests = [ - { - target = "nocompile_gn:default_locale_lint_test" - nocompile_sources = - rebase_path(default_locale_lint_test_nocompile_sources, - "", - "nocompile_gn") - expected_compile_output_regex = "Warning:.*DefaultLocale" - }, - { - target = "nocompile_gn:new_api_lint_test" - nocompile_sources = - rebase_path(new_api_lint_test_nocompile_sources, "", "nocompile_gn") - expected_compile_output_regex = "Error:.*NewApi" - }, - ] - } +group("android_nocompile_tests") { + testonly = true + + # No-compile tests use an output directory dedicated to no-compile tests. + # All test suites use targets in nocompile_gn/BUILD.gn in order to share the + # same target output directory and avoid running 'gn gen' for each + # android_nocompile_test_suite(). + deps = [ + ":android_lint_tests", + ":android_lookup_dep_tests", + ] +} + +android_nocompile_test_suite("android_lint_tests") { + # Depend on lint script so that the action is re-run whenever the script is modified. + pydeps = [ "//build/android/gyp/lint.pydeps" ] + + tests = [ + { + target = "nocompile_gn:default_locale_lint_test" + nocompile_sources = + rebase_path(default_locale_lint_test_nocompile_sources, + "", + "nocompile_gn") + expected_compile_output_regex = "Warning:.*DefaultLocale" + }, + { + target = "nocompile_gn:new_api_lint_test" + nocompile_sources = + rebase_path(new_api_lint_test_nocompile_sources, "", "nocompile_gn") + expected_compile_output_regex = "Error:.*NewApi" + }, + ] +} + +android_nocompile_test_suite("android_lookup_dep_tests") { + sources = + [ rebase_path(missing_symbol_generated_importer_template_nocompile_source, + "", + "nocompile_gn") ] + + tests = [ + { + target = "nocompile_gn:import_child_missing_symbol_test_java" + nocompile_sources = + rebase_path(import_child_missing_symbol_test_nocompile_sources, + "", + "nocompile_gn") + expected_compile_output_regex = "Hint: Try adding the following to //build/android/test/nocompile_gn:import_child_missing_symbol_test_java\n *\"//build/android/test/nocompile_gn:sub_b_java\"" + }, + { + target = "nocompile_gn:import_parent_missing_symbol_test_java" + nocompile_sources = [] + expected_compile_output_regex = "Hint: Try adding the following to //build/android/test/nocompile_gn:import_parent_missing_symbol_test_java\n *\"//build/android/test/nocompile_gn:b_java\"" + }, + { + target = "nocompile_gn:import_turbine_missing_symbol_test_java" + nocompile_sources = + rebase_path(import_turbine_missing_symbol_test_nocompile_sources, + "", + "nocompile_gn") + expected_compile_output_regex = "Hint: Try adding the following to //build/android/test/nocompile_gn:import_turbine_missing_symbol_test_java\n *\"//build/android/test/nocompile_gn:b_java\"" + }, + { + target = "nocompile_gn:prebuilt_missing_symbol_test_java" + nocompile_sources = [] + expected_compile_output_regex = "Hint: Try adding the following to //build/android/test/nocompile_gn:prebuilt_missing_symbol_test_java\n *\"//build/android/test/nocompile_gn:c_prebuilt_java\"" + }, + { + target = "nocompile_gn:cpp_template_missing_symbol_test_java" + nocompile_sources = [] + expected_compile_output_regex = "Hint: Try adding the following to //build/android/test/nocompile_gn:cpp_template_missing_symbol_test_java\n *\"//build/android/test/nocompile_gn:d_java\"" + }, + ] }
diff --git a/build/android/test/incremental_javac_gn/BUILD.gn b/build/android/test/incremental_javac_gn/BUILD.gn new file mode 100644 index 0000000..9411497 --- /dev/null +++ b/build/android/test/incremental_javac_gn/BUILD.gn
@@ -0,0 +1,98 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") + +declare_args() { + incremental_javac_test_toggle_gn = false +} + +all_test_sources = [ + "../../java/test/NoSignatureChangeIncrementalJavacTestHelper.template", + "../../java/test/NoSignatureChangeIncrementalJavacTestHelper2.java", +] + +template("incremental_javac_prebuilt") { + _out_jar = "${target_gen_dir}/${target_name}.jar" + + action(target_name) { + script = "incremental_javac_test_android_library.py" + forward_variables_from(invoker, + [ + "sources", + "testonly", + ]) + deps = [ invoker.toggle_gn_target ] + + inputs = [] + if (defined(invoker.pydeps)) { + foreach(_pydeps_file, invoker.pydeps) { + _pydeps_file_lines = [] + _pydeps_file_lines = read_file(_pydeps_file, "list lines") + _pydeps_entries = [] + _pydeps_entries = filter_exclude(_pydeps_file_lines, [ "#*" ]) + _pydeps_file_dir = get_path_info(_pydeps_file, "dir") + inputs += rebase_path(_pydeps_entries, ".", _pydeps_file_dir) + } + } + + outputs = [ _out_jar ] + + args = [ + "--target-name", + get_label_info("${invoker.toggle_gn_target}", "label_no_toolchain"), + "--gn-args-path", + "args.gn", + "--out-dir", + rebase_path("${target_out_dir}/${target_name}/incremental_javac_out", + root_build_dir), + "--out-jar", + rebase_path(_out_jar, root_build_dir), + ] + } +} + +# Use jinja_template() instead of java_cpp_template() because incremental builds +# are not done when non-.java files change. +jinja_template("changing_javagen") { + input = "../../java/test/NoSignatureChangeIncrementalJavacTestHelper.template" + assert(filter_include(all_test_sources, [ input ]) != []) + output = + "${target_gen_dir}/test/NoSignatureChangeIncrementalJavacTestHelper.java" + if (incremental_javac_test_toggle_gn) { + variables = [ "foo_return_value=foo2" ] + } else { + variables = [ "foo_return_value=foo" ] + } +} + +android_library("changing_java") { + testonly = true + + # Should not be re-compiled during incremental build. + sources = + [ "../../java/test/NoSignatureChangeIncrementalJavacTestHelper2.java" ] + assert(filter_include(all_test_sources, sources) != []) + + # Should be recompiled during incremental build. + sources += get_target_outputs(":changing_javagen") + deps = [ ":changing_javagen" ] +} + +# Compiles :changing_java with and without |incremental_javac_test_toggle_gn|. +incremental_javac_prebuilt("no_signature_change_prebuilt_generator") { + testonly = true + sources = all_test_sources + toggle_gn_target = ":changing_java" + pydeps = [ "//build/android/gyp/compile_java.pydeps" ] +} + +android_java_prebuilt("no_signature_change_prebuilt_java") { + testonly = true + _generator_outputs = + get_target_outputs(":no_signature_change_prebuilt_generator") + jar_paths = filter_include(_generator_outputs, [ "*.jar" ]) + jar_path = jar_paths[0] + deps = [ ":no_signature_change_prebuilt_generator" ] +}
diff --git a/build/android/test/incremental_javac_gn/incremental_javac_test_android_library.py b/build/android/test/incremental_javac_gn/incremental_javac_test_android_library.py new file mode 100755 index 0000000..6407450 --- /dev/null +++ b/build/android/test/incremental_javac_gn/incremental_javac_test_android_library.py
@@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Compiles twice: With incremental_javac_test_toggle_gn=[false, true] + +The purpose of compiling the target twice is to test that builds generated by +the incremental build code path are valid. +""" + +import argparse +import os +import pathlib +import subprocess +import shutil + +_CHROMIUM_SRC = pathlib.Path(__file__).resolve().parents[4].resolve() +_NINJA_PATH = _CHROMIUM_SRC / 'third_party' / 'ninja' / 'ninja' + +# Relative to _CHROMIUM_SRC +_GN_SRC_REL_PATH = 'buildtools/linux64/gn' + +_USING_PARTIAL_JAVAC_MSG = 'Using partial javac optimization' + + +def _raise_command_exception(args, returncode, output): + """Raises an exception whose message describes a command failure. + + Args: + args: shell command-line (as passed to subprocess.Popen()) + returncode: status code. + output: command output. + Raises: + a new Exception. + """ + message = ('Command failed with status {}: {}\n' + 'Output:-----------------------------------------\n{}\n' + '------------------------------------------------\n').format( + returncode, args, output) + raise Exception(message) + + +def _run_command(args, check_returncode=True, cwd=None, env=None): + """Runs shell command. Raises exception if command fails.""" + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=cwd, + env=env, + universal_newlines=True) + pout, _ = p.communicate() + if check_returncode and p.returncode != 0: + _raise_command_exception(args, p.returncode, pout) + return pout + + +def _copy_and_append_gn_args(src_args_path, dest_args_path, extra_args): + """Copies args.gn. + + Args: + src_args_path: args.gn file to copy. + dest_args_path: Copy file destination. + extra_args: Text to append to args.gn after copy. + """ + with open(src_args_path) as f: + initial_args_str = f.read() + + with open(dest_args_path, 'w') as f: + f.write(initial_args_str) + f.write('\n') + + # Write |extra_args| after |initial_args_str| so that |extra_args| + # overwrites |initial_args_str| in the case of duplicate entries. + f.write('\n'.join(extra_args)) + + +def _run_gn(args, check_returncode=True): + _run_command([_GN_SRC_REL_PATH] + args, + check_returncode=check_returncode, + cwd=_CHROMIUM_SRC) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--target-name', + required=True, + help='name of target to build with and without ' + + 'incremental_javac_test_toggle_gn=true') + parser.add_argument('--gn-args-path', + required=True, + help='Path to args.gn file to copy args from.') + parser.add_argument('--out-dir', + required=True, + help='Path to output directory to use for compilation.') + parser.add_argument('--out-jar', + required=True, + help='Path where output jar should be stored.') + options = parser.parse_args() + + options.out_dir = pathlib.Path(options.out_dir).resolve() + + options.out_dir.mkdir(parents=True, exist_ok=True) + + # Clear the output directory so that first compile is not an incremental + # build. + # This will make the test fail in the scenario that: + # - The output directory contains a previous build generated by this script. + # - Incremental builds are broken and are a no-op. + _run_gn(['clean', options.out_dir.relative_to(_CHROMIUM_SRC)], + check_returncode=False) + + out_gn_args_path = options.out_dir / 'args.gn' + extra_gn_args = [ + 'treat_warnings_as_errors = true', + # GOMA does not work with non-standard output directories. + 'use_goma = false', + ] + _copy_and_append_gn_args( + options.gn_args_path, out_gn_args_path, + extra_gn_args + ['incremental_javac_test_toggle_gn = false']) + + _run_gn([ + '--root-target=' + options.target_name, 'gen', + options.out_dir.relative_to(_CHROMIUM_SRC) + ]) + + ninja_env = os.environ.copy() + ninja_env['JAVAC_DEBUG'] = '1' + + # Strip leading '//' + gn_path = options.target_name[2:] + ninja_args = [_NINJA_PATH, '-C', options.out_dir, gn_path] + ninja_output = _run_command(ninja_args, env=ninja_env) + if _USING_PARTIAL_JAVAC_MSG in ninja_output: + raise Exception('Incorrectly using partial javac for clean compile.') + + _copy_and_append_gn_args( + options.gn_args_path, out_gn_args_path, + extra_gn_args + ['incremental_javac_test_toggle_gn = true']) + ninja_output = _run_command(ninja_args, env=ninja_env) + if _USING_PARTIAL_JAVAC_MSG not in ninja_output: + raise Exception('Not using partial javac for incremental compile.') + + expected_output_path = '{}/obj/{}.javac.jar'.format(options.out_dir, + gn_path.replace(':', '/')) + if not os.path.exists(expected_output_path): + raise Exception('{} not created.'.format(expected_output_path)) + + shutil.copyfile(expected_output_path, options.out_jar) + + +if __name__ == '__main__': + main()
diff --git a/build/android/test/missing_symbol_test.gni b/build/android/test/missing_symbol_test.gni new file mode 100644 index 0000000..3cc4741 --- /dev/null +++ b/build/android/test/missing_symbol_test.gni
@@ -0,0 +1,57 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/android_nocompile.gni") +import("//build/config/android/rules.gni") + +missing_symbol_generated_importer_template_nocompile_source = + "//build/android/java/test/missing_symbol/Importer.template" + +template("missing_symbol_test") { + # Not named "_java" to prevent target from being considered a classpath dep. + _helper_target_name = string_replace("${target_name}__helper", "java", "") + + group(_helper_target_name) { + # Make group() depend on dependencies that |target_name| cannot find so that + # the missing symbol resolver can find and suggest the missing GN dep. + deps = invoker.deps + } + + android_library(target_name) { + sources = [ "//tools/android/errorprone_plugin/test/src/org/chromium/tools/errorprone/plugin/Empty.java" ] + not_needed(invoker, + [ + "sources", + "importer_srcjar_deps", + ]) + if (enable_android_nocompile_tests) { + if (defined(invoker.sources)) { + sources += invoker.sources + } + if (defined(invoker.importer_srcjar_deps)) { + srcjar_deps = invoker.importer_srcjar_deps + } + } + + deps = [ ":${_helper_target_name}" ] + } +} + +# missing_symbol_test() template wrapper which generates importer class. +template("missing_symbol_generated_importer_test") { + _importer_generator_target = "${target_name}__importer_javagen" + java_cpp_template(_importer_generator_target) { + sources = [ missing_symbol_generated_importer_template_nocompile_source ] + defines = [ + "_IMPORTER_PACKAGE=${invoker.importer_package}", + "_IMPORTEE_PACKAGE=${invoker.imported_package}", + "_IMPORTEE_CLASS_NAME=${invoker.imported_class_name}", + ] + } + + missing_symbol_test(target_name) { + importer_srcjar_deps = [ ":${_importer_generator_target}" ] + forward_variables_from(invoker, [ "deps" ]) + } +}
diff --git a/build/android/test/nocompile_gn/BUILD.gn b/build/android/test/nocompile_gn/BUILD.gn index d3262fe..406bd8c 100644 --- a/build/android/test/nocompile_gn/BUILD.gn +++ b/build/android/test/nocompile_gn/BUILD.gn
@@ -1,7 +1,8 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/android/test/missing_symbol_test.gni") import("//build/config/android/android_nocompile.gni") import("//build/config/android/rules.gni") import("nocompile_sources.gni") @@ -31,7 +32,7 @@ deps = [ "${_apk_target}__java" ] build_config_dep = "$_apk_target$build_config_target_suffix" build_config = get_label_info(_apk_target, "target_gen_dir") + "/" + - get_label_info(_apk_target, "name") + ".build_config" + get_label_info(_apk_target, "name") + ".build_config.json" if (enable_android_nocompile_tests) { skip_build_server = true } @@ -45,3 +46,56 @@ lint_test("new_api_lint_test") { sources = new_api_lint_test_nocompile_sources } + +missing_symbol_generated_importer_test( + "import_parent_missing_symbol_test_java") { + importer_package = "test.missing_symbol.child_missing" + imported_package = "test.missing_symbol" + imported_class_name = "B" + deps = [ ":b_java" ] +} + +missing_symbol_test("import_child_missing_symbol_test_java") { + sources = import_child_missing_symbol_test_nocompile_sources + deps = [ ":sub_b_java" ] +} + +missing_symbol_test("import_turbine_missing_symbol_test_java") { + sources = import_turbine_missing_symbol_test_nocompile_sources + deps = [ ":b_java" ] +} + +missing_symbol_generated_importer_test("prebuilt_missing_symbol_test_java") { + importer_package = "test.missing_symbol.prebuilt_missing" + imported_package = "test.missing_symbol" + imported_class_name = "C" + deps = [ ":c_prebuilt_java" ] +} + +missing_symbol_generated_importer_test( + "cpp_template_missing_symbol_test_java") { + importer_package = "test.missing_symbol.cpp_template_missing" + imported_package = "test.missing_symbol" + imported_class_name = "D" + deps = [ ":d_java" ] +} + +android_library("b_java") { + sources = [ "../../java/test/missing_symbol/B.java" ] +} + +android_library("sub_b_java") { + sources = [ "../../java/test/missing_symbol/sub/SubB.java" ] +} + +android_java_prebuilt("c_prebuilt_java") { + jar_path = "../../java/test/missing_symbol/c.jar" +} + +android_library("d_java") { + srcjar_deps = [ ":d_template_javagen" ] +} + +java_cpp_template("d_template_javagen") { + sources = [ "../../java/test/missing_symbol/D.template" ] +}
diff --git a/build/android/test/nocompile_gn/nocompile_sources.gni b/build/android/test/nocompile_gn/nocompile_sources.gni index 8fc049e..36cd915 100644 --- a/build/android/test/nocompile_gn/nocompile_sources.gni +++ b/build/android/test/nocompile_gn/nocompile_sources.gni
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,3 +6,9 @@ [ "../../java/test/DefaultLocaleLintTest.java" ] new_api_lint_test_nocompile_sources = [ "../../java/test/NewApiLintTest.java" ] + +import_child_missing_symbol_test_nocompile_sources = + [ "../../java/test/missing_symbol/ImportsSubB.java" ] + +import_turbine_missing_symbol_test_nocompile_sources = + [ "../../java/test/missing_symbol/sub/BInMethodSignature.java" ]
diff --git a/build/android/test_runner.py b/build/android/test_runner.py index 84010c3..34b8deb 100755 --- a/build/android/test_runner.py +++ b/build/android/test_runner.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,10 +10,12 @@ import argparse import collections import contextlib +import io import itertools import logging import os import re +import shlex import shutil import signal import sys @@ -27,8 +29,6 @@ # See http://crbug.com/724524 and https://bugs.python.org/issue7980. import _strptime # pylint: disable=unused-import -# pylint: disable=redefined-builtin -from six.moves import range # Needed for python 3 compatibility. # pylint: disable=ungrouped-imports from pylib.constants import host_paths @@ -44,7 +44,6 @@ from pylib.base import environment_factory from pylib.base import output_manager from pylib.base import output_manager_factory -from pylib.base import result_sink from pylib.base import test_instance_factory from pylib.base import test_run_factory from pylib.results import json_results @@ -57,9 +56,13 @@ from py_utils import contextlib_ext +from lib.results import result_sink # pylint: disable=import-error + _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join( host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json')) +_RERUN_FAILED_TESTS_FILE = 'rerun_failed_tests.filter' + def _RealPath(arg): if arg.startswith('//'): @@ -179,6 +182,15 @@ help='Whether to archive test output locally and generate ' 'a local results detail page.') + parser.add_argument('--list-tests', + action='store_true', + help='List available tests and exit.') + + parser.add_argument('--wrapper-script-args', + help='A string of args that were passed to the wrapper ' + 'script. This should probably not be edited by a ' + 'user as it is passed by the wrapper itself.') + class FastLocalDevAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): namespace.enable_concurrent_adb = True @@ -187,6 +199,7 @@ namespace.local_output = True namespace.num_retries = 0 namespace.skip_clear_data = True + namespace.use_persistent_shell = True parser.add_argument( '--fast-local-dev', @@ -195,7 +208,7 @@ action=FastLocalDevAction, help='Alias for: --num-retries=0 --enable-device-cache ' '--enable-concurrent-adb --skip-clear-data ' - '--extract-test-list-from-filter --local-output') + '--extract-test-list-from-filter --use-persistent-shell --local-output') # TODO(jbudorick): Remove this once downstream bots have switched to # api.test_results. @@ -224,6 +237,18 @@ dest='repeat', type=int, default=0, help='Number of times to repeat the specified set of tests.') + # Not useful for junit tests. + parser.add_argument( + '--use-persistent-shell', + action='store_true', + help='Uses a persistent shell connection for the adb connection.') + + parser.add_argument('--disable-test-server', + action='store_true', + help='Disables SpawnedTestServer which doesn' + 't work with remote adb. ' + 'WARNING: Will break tests which require the server.') + # This is currently only implemented for gtests and instrumentation tests. parser.add_argument( '--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests', @@ -243,12 +268,10 @@ def ProcessCommonOptions(args): """Processes and handles all common options.""" run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False) - # pylint: disable=redefined-variable-type if args.verbose_count > 0: handler = logging_utils.ColorStreamHandler() else: handler = logging.StreamHandler(sys.stdout) - # pylint: enable=redefined-variable-type handler.setFormatter(run_tests_helper.CustomFormatter()) logging.getLogger().addHandler(handler) @@ -337,6 +360,12 @@ action='store_true', default=False, help='Enable graphical window display on the emulator.') + parser.add_argument( + '--emulator-debug-tags', + help='Comma-separated list of debug tags. This can be used to enable or ' + 'disable debug messages from specific parts of the emulator, e.g. ' + 'init,snapshot. See "emulator -help-debug-tags" ' + 'for a full list of tags.') def AddGTestOptions(parser): @@ -354,10 +383,6 @@ help='Host directory to which app data files will be' ' saved. Used with --app-data-file.') parser.add_argument( - '--delete-stale-data', - dest='delete_stale_data', action='store_true', - help='Delete stale test data on the device.') - parser.add_argument( '--enable-xml-result-parsing', action='store_true', help=argparse.SUPPRESS) parser.add_argument( @@ -414,6 +439,12 @@ '--coverage-dir', type=os.path.realpath, help='Directory in which to place all generated coverage files.') + parser.add_argument( + '--use-existing-test-data', + action='store_true', + help='Do not push new files to the device, instead using existing APK ' + 'and test data. Only use when running the same test for multiple ' + 'iterations.') def AddInstrumentationTestOptions(parser): @@ -421,12 +452,32 @@ parser = parser.add_argument_group('instrumentation arguments') + parser.add_argument('--additional-apex', + action='append', + dest='additional_apexs', + default=[], + type=_RealPath, + help='Additional apex that must be installed on ' + 'the device when the tests are run') parser.add_argument( '--additional-apk', action='append', dest='additional_apks', default=[], type=_RealPath, help='Additional apk that must be installed on ' 'the device when the tests are run') + parser.add_argument('--forced-queryable-additional-apk', + action='append', + dest='forced_queryable_additional_apks', + default=[], + type=_RealPath, + help='Configures an additional-apk to be forced ' + 'to be queryable by other APKs.') + parser.add_argument('--instant-additional-apk', + action='append', + dest='instant_additional_apks', + default=[], + type=_RealPath, + help='Configures an additional-apk to be an instant APK') parser.add_argument( '-A', '--annotation', dest='annotation_str', @@ -440,6 +491,11 @@ '--apk-under-test', help='Path or name of the apk under test.') parser.add_argument( + '--store-data-in-app-directory', + action='store_true', + help='Store test data in the application\'s data directory. By default ' + 'the test data is stored in the external storage folder.') + parser.add_argument( '--module', action='append', dest='modules', @@ -463,41 +519,28 @@ help='Directory in which to place all generated ' 'Jacoco coverage files.') parser.add_argument( - '--delete-stale-data', - action='store_true', dest='delete_stale_data', - help='Delete stale test data on the device.') - parser.add_argument( '--disable-dalvik-asserts', dest='set_asserts', action='store_false', default=True, help='Removes the dalvik.vm.enableassertions property') parser.add_argument( - '--enable-java-deobfuscation', - action='store_true', - help='Deobfuscate java stack traces in test output and logcat.') + '--proguard-mapping-path', + help='.mapping file to use to Deobfuscate java stack traces in test ' + 'output and logcat.') parser.add_argument( '-E', '--exclude-annotation', dest='exclude_annotation_str', help='Comma-separated list of annotations. Exclude tests with these ' 'annotations.') - def package_replacement(arg): - split_arg = arg.split(',') - if len(split_arg) != 2: - raise argparse.ArgumentError( - arg, - 'Expected two comma-separated strings for --replace-system-package, ' - 'received %d' % len(split_arg)) - PackageReplacement = collections.namedtuple('PackageReplacement', - ['package', 'replacement_apk']) - return PackageReplacement(package=split_arg[0], - replacement_apk=_RealPath(split_arg[1])) + parser.add_argument( + '--enable-breakpad-dump', + action='store_true', + help='Stores any breakpad dumps till the end of the test.') parser.add_argument( '--replace-system-package', - type=package_replacement, default=None, - help='Specifies a system package to replace with a given APK for the ' - 'duration of the test. Given as a comma-separated pair of strings, ' - 'the first element being the package and the second the path to the ' - 'replacement APK. Only supports replacing one package. Example: ' - '--replace-system-package com.example.app,path/to/some.apk') + type=_RealPath, + default=None, + help='Use this apk to temporarily replace a system package with the same ' + 'package name.') parser.add_argument( '--remove-system-package', default=[], @@ -507,7 +550,11 @@ 'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. ' 'Unlike --replace-system-package, the app will not be restored after ' 'tests are finished.') - + parser.add_argument( + '--use-voice-interaction-service', + help='This can be used to update the voice interaction service to be a ' + 'custom one. This is useful for mocking assistants. eg: ' + 'android.assist.service/.MainInteractionService') parser.add_argument( '--use-webview-provider', type=_RealPath, default=None, @@ -516,6 +563,20 @@ "on Nougat the provider can't be determined and so " 'the system will choose the default provider.') parser.add_argument( + '--run-setup-command', + default=[], + action='append', + dest='run_setup_commands', + help='This can be used to run a custom shell command on the device as a ' + 'setup step') + parser.add_argument( + '--run-teardown-command', + default=[], + action='append', + dest='run_teardown_commands', + help='This can be used to run a custom shell command on the device as a ' + 'teardown step') + parser.add_argument( '--runtime-deps-path', dest='runtime_deps_path', type=os.path.realpath, help='Runtime data dependency file from GN.') @@ -554,13 +615,27 @@ required=True, help='Path or name of the apk containing the tests.') parser.add_argument( - '--test-jar', - help='Path of jar containing test java files.') + '--test-apk-as-instant', + action='store_true', + help='Install the test apk as an instant app. ' + 'Instant apps run in a more restrictive execution environment.') + parser.add_argument( + '--test-launcher-batch-limit', + dest='test_launcher_batch_limit', + type=int, + help=('Not actually used for instrumentation tests, but can be used as ' + 'a proxy for determining if the current run is a retry without ' + 'patch.')) parser.add_argument( '--timeout-scale', type=float, help='Factor by which timeouts should be scaled.') parser.add_argument( + '--is-unit-test', + action='store_true', + help=('Specify the test suite as composed of unit tests, blocking ' + 'certain operations.')) + parser.add_argument( '-w', '--wait-for-java-debugger', action='store_true', help='Wait for java debugger to attach before running any application ' 'code. Also disables test timeouts and sets retries=0.') @@ -572,6 +647,12 @@ help='If true, WPR server runs in record mode.' 'otherwise, runs in replay mode.') + parser.add_argument( + '--approve-app-links', + help='Force enables Digital Asset Link verification for the provided ' + 'package and domain, example usage: --approve-app-links ' + 'com.android.package:www.example.com') + # These arguments are suppressed from the help text because they should # only ever be specified by an intermediate script. parser.add_argument( @@ -591,6 +672,10 @@ help='A non-default code review system to pass to pass to Gold, if ' 'applicable') parser.add_argument( + '--continuous-integration-system', + help='A non-default continuous integration system to pass to Gold, if ' + 'applicable') + parser.add_argument( '--git-revision', help='The git commit currently being tested.') parser.add_argument( '--gerrit-issue', @@ -682,6 +767,8 @@ parser.add_argument( '--robolectric-runtime-deps-dir', help='Path to runtime deps for Robolectric.') + parser.add_argument('--native-libs-dir', + help='Path to search for native libraries.') parser.add_argument( '--resource-apk', required=True, @@ -810,8 +897,44 @@ if command == 'python': return _RunPythonTests(args) - else: - raise Exception('Unknown test type.') + raise Exception('Unknown test type.') + + +def _SinkTestResult(test_result, test_file_name, result_sink_client): + """Upload test result to result_sink. + + Args: + test_result: A BaseTestResult object + test_file_name: A string representing the file location of the test + result_sink_client: A ResultSinkClient object + + Returns: + N/A + """ + # Some tests put in non utf-8 char as part of the test + # which breaks uploads, so need to decode and re-encode. + log_decoded = test_result.GetLog() + if isinstance(log_decoded, bytes): + log_decoded = log_decoded.decode('utf-8', 'replace') + html_artifact = '' + https_artifacts = [] + for link_name, link_url in sorted(test_result.GetLinks().items()): + if link_url.startswith('https:'): + https_artifacts.append('<li><a target="_blank" href=%s>%s</a></li>' % + (link_url, link_name)) + else: + logging.info('Skipping non-https link %r (%s) for test %s.', link_name, + link_url, test_result.GetName()) + if https_artifacts: + html_artifact += '<ul>%s</ul>' % '\n'.join(https_artifacts) + result_sink_client.Post(test_result.GetNameForResultSink(), + test_result.GetType(), + test_result.GetDuration(), + log_decoded.encode('utf-8'), + test_file_name, + variant=test_result.GetVariantForResultSink(), + failure_reason=test_result.GetFailureReason(), + html_artifact=html_artifact) _SUPPORTED_IN_PLATFORM_MODE = [ @@ -923,12 +1046,7 @@ match = re.search(r'^(.+\..+)#', r.GetName()) test_file_name = test_class_to_file_name_dict.get( match.group(1)) if match else None - # Some tests put in non utf-8 char as part of the test - # which breaks uploads, so need to decode and re-encode. - result_sink_client.Post( - r.GetName(), r.GetType(), r.GetDuration(), - r.GetLog().decode('utf-8', 'replace').encode('utf-8'), - test_file_name) + _SinkTestResult(r, test_file_name, result_sink_client) @contextlib.contextmanager def upload_logcats_file(): @@ -953,6 +1071,9 @@ upload_logcats_file(), 'upload_logcats_file' in args and args.upload_logcats_file) + save_detailed_results = (args.local_output or not local_utils.IsOnSwarming() + ) and not args.isolated_script_test_output + ### Set up test objects. out_manager = output_manager_factory.CreateOutputManager(args) @@ -964,8 +1085,24 @@ contexts_to_notify_on_sigterm.append(env) contexts_to_notify_on_sigterm.append(test_run) + if args.list_tests: + try: + with out_manager, env, test_instance, test_run: + test_names = test_run.GetTestsForListing() + print('There are {} tests:'.format(len(test_names))) + for n in test_names: + print(n) + return 0 + except NotImplementedError: + sys.stderr.write('Test does not support --list-tests (type={}).\n'.format( + args.command)) + return 1 + ### Run. with out_manager, json_finalizer(): + # |raw_logs_fh| is only used by Robolectric tests. + raw_logs_fh = io.StringIO() if save_detailed_results else None + with json_writer(), logcats_uploader, env, test_instance, test_run: repetitions = (range(args.repeat + @@ -981,7 +1118,7 @@ raw_results = [] all_raw_results.append(raw_results) - test_run.RunTests(raw_results) + test_run.RunTests(raw_results, raw_logs_fh=raw_logs_fh) if not raw_results: all_raw_results.pop() continue @@ -1002,6 +1139,12 @@ annotation=getattr(args, 'annotations', None), flakiness_server=getattr(args, 'flakiness_dashboard_server', None)) + + failed_tests = (iteration_results.GetNotPass() - + iteration_results.GetSkip()) + if failed_tests: + _LogRerunStatement(failed_tests, args.wrapper_script_args) + if args.break_on_failure and not iteration_results.DidRunPass(): break @@ -1030,8 +1173,17 @@ str(tot_tests), str(iteration_count)) - if (args.local_output or not local_utils.IsOnSwarming() - ) and not args.isolated_script_test_output: + if save_detailed_results: + assert raw_logs_fh + raw_logs_fh.seek(0) + raw_logs = raw_logs_fh.read() + if raw_logs: + with out_manager.ArchivedTempfile( + 'raw_logs.txt', 'raw_logs', + output_manager.Datatype.TEXT) as raw_logs_file: + raw_logs_file.write(raw_logs) + logging.critical('RAW LOGS: %s', raw_logs_file.Link()) + with out_manager.ArchivedTempfile( 'test_results_presentation.html', 'test_results_presentation', @@ -1041,7 +1193,7 @@ test_name=args.command, cs_base_url='http://cs.chromium.org', local_output=True) - results_detail_file.write(result_html_string.encode('utf-8')) + results_detail_file.write(result_html_string) results_detail_file.flush() logging.critical('TEST RESULTS: %s', results_detail_file.Link()) @@ -1059,6 +1211,66 @@ else constants.ERROR_EXIT_CODE) +def _LogRerunStatement(failed_tests, wrapper_arg_str): + """Logs a message that can rerun the failed tests. + + Logs a copy/pasteable message that filters tests so just the failing tests + are run. + + Args: + failed_tests: A set of test results that did not pass. + wrapper_arg_str: A string of args that were passed to the called wrapper + script. + """ + rerun_arg_list = [] + try: + constants.CheckOutputDirectory() + # constants.CheckOutputDirectory throws bare exceptions. + except: # pylint: disable=bare-except + logging.exception('Output directory not found. Unable to generate failing ' + 'test filter file.') + return + + output_directory = constants.GetOutDirectory() + if not os.path.exists(output_directory): + logging.error('Output directory not found. Unable to generate failing ' + 'test filter file.') + return + + test_filter_file = os.path.join(os.path.relpath(output_directory), + _RERUN_FAILED_TESTS_FILE) + arg_list = shlex.split(wrapper_arg_str) if wrapper_arg_str else sys.argv + index = 0 + while index < len(arg_list): + arg = arg_list[index] + # Skip adding the filter=<file> and/or the filter arg as we're replacing + # it with the new filter arg. + # This covers --test-filter=, --test-launcher-filter-file=, --gtest-filter=, + # --test-filter *Foobar.baz, -f *foobar, --package-filter <package>, + # --runner-filter <runner>. + if 'filter' in arg or arg == '-f': + index += 1 if '=' in arg else 2 + continue + + rerun_arg_list.append(arg) + index += 1 + + failed_test_list = [str(t) for t in failed_tests] + with open(test_filter_file, 'w') as fp: + for t in failed_test_list: + # Test result names can have # in them that don't match when applied as + # a test name filter. + fp.write('%s\n' % t.replace('#', '.')) + + rerun_arg_list.append('--test-launcher-filter-file=%s' % test_filter_file) + msg = """ + %d Test(s) failed. + Rerun failed tests with copy and pastable command: + %s + """ + logging.critical(msg, len(failed_tests), shlex.join(rerun_arg_list)) + + def DumpThreadStacks(_signal, _frame): for thread in threading.enumerate(): reraiser_thread.LogThreadStack(thread)
diff --git a/build/android/test_runner.pydeps b/build/android/test_runner.pydeps index 660f8f8..5c1cd13 100644 --- a/build/android/test_runner.pydeps +++ b/build/android/test_runner.pydeps
@@ -67,6 +67,7 @@ ../../third_party/catapult/devil/devil/android/sdk/shared_prefs.py ../../third_party/catapult/devil/devil/android/sdk/split_select.py ../../third_party/catapult/devil/devil/android/sdk/version_codes.py +../../third_party/catapult/devil/devil/android/settings.py ../../third_party/catapult/devil/devil/android/tools/__init__.py ../../third_party/catapult/devil/devil/android/tools/device_recovery.py ../../third_party/catapult/devil/devil/android/tools/device_status.py @@ -95,6 +96,7 @@ ../../third_party/catapult/devil/devil/utils/timeout_retry.py ../../third_party/catapult/devil/devil/utils/watchdog_timer.py ../../third_party/catapult/devil/devil/utils/zip_utils.py +../../third_party/catapult/third_party/six/six.py ../../third_party/colorama/src/colorama/__init__.py ../../third_party/colorama/src/colorama/ansi.py ../../third_party/colorama/src/colorama/ansitowin32.py @@ -102,7 +104,8 @@ ../../third_party/colorama/src/colorama/win32.py ../../third_party/colorama/src/colorama/winterm.py ../../third_party/jinja2/__init__.py -../../third_party/jinja2/_compat.py +../../third_party/jinja2/_identifier.py +../../third_party/jinja2/async_utils.py ../../third_party/jinja2/bccache.py ../../third_party/jinja2/compiler.py ../../third_party/jinja2/defaults.py @@ -119,30 +122,33 @@ ../../third_party/jinja2/tests.py ../../third_party/jinja2/utils.py ../../third_party/jinja2/visitor.py +../../third_party/logdog/logdog/__init__.py +../../third_party/logdog/logdog/bootstrap.py +../../third_party/logdog/logdog/stream.py +../../third_party/logdog/logdog/streamname.py +../../third_party/logdog/logdog/varint.py ../../third_party/markupsafe/__init__.py ../../third_party/markupsafe/_compat.py ../../third_party/markupsafe/_native.py -../../tools/swarming_client/libs/__init__.py -../../tools/swarming_client/libs/logdog/__init__.py -../../tools/swarming_client/libs/logdog/bootstrap.py -../../tools/swarming_client/libs/logdog/stream.py -../../tools/swarming_client/libs/logdog/streamname.py -../../tools/swarming_client/libs/logdog/varint.py +../action_helpers.py ../gn_helpers.py ../print_python_deps.py ../skia_gold_common/__init__.py ../skia_gold_common/skia_gold_properties.py ../skia_gold_common/skia_gold_session.py ../skia_gold_common/skia_gold_session_manager.py +../util/lib/__init__.py ../util/lib/common/chrome_test_server_spawner.py ../util/lib/common/unittest_util.py -convert_dex_profile.py +../util/lib/results/__init__.py +../util/lib/results/result_sink.py +../util/lib/results/result_types.py +../zip_helpers.py devil_chromium.py gyp/dex.py gyp/util/__init__.py gyp/util/build_utils.py gyp/util/md5_check.py -gyp/util/zipalign.py incremental_install/__init__.py incremental_install/installer.py pylib/__init__.py @@ -152,7 +158,6 @@ pylib/base/environment_factory.py pylib/base/output_manager.py pylib/base/output_manager_factory.py -pylib/base/result_sink.py pylib/base/test_collection.py pylib/base/test_exception.py pylib/base/test_instance.py @@ -204,6 +209,7 @@ pylib/results/report_results.py pylib/symbols/__init__.py pylib/symbols/deobfuscator.py +pylib/symbols/expensive_line_transformer.py pylib/symbols/stack_symbolizer.py pylib/utils/__init__.py pylib/utils/chrome_proxy_utils.py @@ -216,7 +222,6 @@ pylib/utils/local_utils.py pylib/utils/logdog_helper.py pylib/utils/logging_utils.py -pylib/utils/proguard.py pylib/utils/repo_utils.py pylib/utils/shared_preference_utils.py pylib/utils/test_filter.py
diff --git a/build/android/test_wrapper/logdog_wrapper.py b/build/android/test_wrapper/logdog_wrapper.py index 782d5d8..5620657 100755 --- a/build/android/test_wrapper/logdog_wrapper.py +++ b/build/android/test_wrapper/logdog_wrapper.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2016 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,12 +7,15 @@ import argparse import contextlib +import json import logging import os import signal import subprocess import sys +import six + _SRC_PATH = os.path.abspath(os.path.join( os.path.dirname(__file__), '..', '..', '..')) sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil')) @@ -23,17 +26,17 @@ from devil.utils import timeout_retry from py_utils import tempfile_ext -PROJECT = 'chromium' OUTPUT = 'logdog' COORDINATOR_HOST = 'luci-logdog.appspot.com' -SERVICE_ACCOUNT_JSON = ('/creds/service_accounts' - '/service-account-luci-logdog-publisher.json') LOGDOG_TERMINATION_TIMEOUT = 30 def CommandParser(): # Parses the command line arguments being passed in - parser = argparse.ArgumentParser() + if six.PY3: + parser = argparse.ArgumentParser(allow_abbrev=False) + else: + parser = argparse.ArgumentParser() wrapped = parser.add_mutually_exclusive_group() wrapped.add_argument( '--target', @@ -71,6 +74,28 @@ str(popen.pid)) +def GetProjectFromLuciContext(): + """Return the "project" from LUCI_CONTEXT. + + LUCI_CONTEXT contains a section "realm.name" whose value follows the format + "<project>:<realm>". This method parses and return the "project" part. + + Fallback to "chromium" if realm name is None + """ + project = 'chromium' + ctx_path = os.environ.get('LUCI_CONTEXT') + if ctx_path: + try: + with open(ctx_path) as f: + luci_ctx = json.load(f) + realm_name = luci_ctx.get('realm', {}).get('name') + if realm_name: + project = realm_name.split(':')[0] + except (OSError, IOError, ValueError): + pass + return project + + def main(): parser = CommandParser() args, extra_cmd_args = parser.parse_known_args(sys.argv[1:]) @@ -99,18 +124,18 @@ 'butler.sock') prefix = os.path.join('android', 'swarming', 'logcats', os.environ.get('SWARMING_TASK_ID')) + project = GetProjectFromLuciContext() logdog_cmd = [ args.logdog_bin_cmd, - '-project', PROJECT, + '-project', project, '-output', OUTPUT, '-prefix', prefix, - '--service-account-json', SERVICE_ACCOUNT_JSON, '-coordinator-host', COORDINATOR_HOST, 'serve', '-streamserver-uri', streamserver_uri] test_env.update({ - 'LOGDOG_STREAM_PROJECT': PROJECT, + 'LOGDOG_STREAM_PROJECT': project, 'LOGDOG_STREAM_PREFIX': prefix, 'LOGDOG_STREAM_SERVER_PATH': streamserver_uri, 'LOGDOG_COORDINATOR_HOST': COORDINATOR_HOST,
diff --git a/build/android/tests/symbolize/Makefile b/build/android/tests/symbolize/Makefile index 4fc53da..82c9ea5 100644 --- a/build/android/tests/symbolize/Makefile +++ b/build/android/tests/symbolize/Makefile
@@ -1,4 +1,4 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/tests/symbolize/a.cc b/build/android/tests/symbolize/a.cc index f0c7ca4..6744118 100644 --- a/build/android/tests/symbolize/a.cc +++ b/build/android/tests/symbolize/a.cc
@@ -1,4 +1,4 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2013 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/tests/symbolize/b.cc b/build/android/tests/symbolize/b.cc index db87520..9279977 100644 --- a/build/android/tests/symbolize/b.cc +++ b/build/android/tests/symbolize/b.cc
@@ -1,4 +1,4 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2013 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/android/tombstones.py b/build/android/tombstones.py index 082e7c1..430b284 100755 --- a/build/android/tombstones.py +++ b/build/android/tombstones.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # @@ -156,7 +156,7 @@ return ret # Sort the tombstones in date order, descending - all_tombstones.sort(cmp=lambda a, b: cmp(b[1], a[1])) + all_tombstones.sort(key=lambda a: a[1], reverse=True) # Only resolve the most recent unless --all-tombstones given. tombstones = all_tombstones if resolve_all_tombstones else [all_tombstones[0]]
diff --git a/build/android/unused_resources/BUILD.gn b/build/android/unused_resources/BUILD.gn new file mode 100644 index 0000000..8eb5623 --- /dev/null +++ b/build/android/unused_resources/BUILD.gn
@@ -0,0 +1,19 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") + +java_binary("unused_resources") { + sources = [ "//build/android/unused_resources/UnusedResources.java" ] + main_class = "build.android.unused_resources.UnusedResources" + deps = [ + "//third_party/android_deps:com_android_tools_common_java", + "//third_party/android_deps:com_android_tools_layoutlib_layoutlib_api_java", + "//third_party/android_deps:com_android_tools_sdk_common_java", + "//third_party/android_deps:com_google_guava_guava_java", + "//third_party/kotlin_stdlib:kotlin_stdlib_java", + "//third_party/r8:r8_java", + ] + wrapper_script_name = "helper/unused_resources" +}
diff --git a/build/android/unused_resources/UnusedResources.java b/build/android/unused_resources/UnusedResources.java new file mode 100644 index 0000000..079fa96 --- /dev/null +++ b/build/android/unused_resources/UnusedResources.java
@@ -0,0 +1,619 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Modifications are owned by the Chromium Authors. +// Copyright 2021 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package build.android.unused_resources; + +import static com.android.ide.common.symbols.SymbolIo.readFromAapt; +import static com.android.utils.SdkUtils.endsWithIgnoreCase; +import static com.google.common.base.Charsets.UTF_8; + +import com.android.ide.common.resources.usage.ResourceUsageModel; +import com.android.ide.common.resources.usage.ResourceUsageModel.Resource; +import com.android.ide.common.symbols.Symbol; +import com.android.ide.common.symbols.SymbolTable; +import com.android.resources.ResourceFolderType; +import com.android.resources.ResourceType; +import com.android.tools.r8.CompilationFailedException; +import com.android.tools.r8.ProgramResource; +import com.android.tools.r8.ProgramResourceProvider; +import com.android.tools.r8.ResourceShrinker; +import com.android.tools.r8.ResourceShrinker.Command; +import com.android.tools.r8.ResourceShrinker.ReferenceChecker; +import com.android.tools.r8.origin.PathOrigin; +import com.android.utils.XmlUtils; +import com.google.common.base.Charsets; +import com.google.common.collect.Maps; +import com.google.common.io.ByteStreams; +import com.google.common.io.Closeables; +import com.google.common.io.Files; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import javax.xml.parsers.ParserConfigurationException; + +/** + Copied with modifications from gradle core source + https://cs.android.com/search?q=f:build-system.*ResourceUsageAnalyzer.java + + Modifications are mostly to: + - Remove unused code paths to reduce complexity. + - Reduce dependencies unless absolutely required. +*/ + +public class UnusedResources { + private static final String ANDROID_RES = "android_res/"; + private static final String DOT_DEX = ".dex"; + private static final String DOT_CLASS = ".class"; + private static final String DOT_XML = ".xml"; + private static final String DOT_JAR = ".jar"; + private static final String FN_RESOURCE_TEXT = "R.txt"; + + /* A source of resource classes to track, can be either a folder or a jar */ + private final Iterable<File> mRTxtFiles; + private final File mProguardMapping; + /** These can be class or dex files. */ + private final Iterable<File> mClasses; + private final Iterable<File> mManifests; + private final Iterable<File> mResourceDirs; + + private final File mReportFile; + private final StringWriter mDebugOutput; + private final PrintWriter mDebugPrinter; + + /** The computed set of unused resources */ + private List<Resource> mUnused; + + /** + * Map from resource class owners (VM format class) to corresponding resource entries. + * This lets us map back from code references (obfuscated class and possibly obfuscated field + * reference) back to the corresponding resource type and name. + */ + private Map<String, Pair<ResourceType, Map<String, String>>> mResourceObfuscation = + Maps.newHashMapWithExpectedSize(30); + + /** Obfuscated name of android/support/v7/widget/SuggestionsAdapter.java */ + private String mSuggestionsAdapter; + + /** Obfuscated name of android/support/v7/internal/widget/ResourcesWrapper.java */ + private String mResourcesWrapper; + + /* A Pair class because java does not come with batteries included. */ + private static class Pair<U, V> { + private U mFirst; + private V mSecond; + + Pair(U first, V second) { + this.mFirst = first; + this.mSecond = second; + } + + public U getFirst() { + return mFirst; + } + + public V getSecond() { + return mSecond; + } + } + + public UnusedResources(Iterable<File> rTxtFiles, Iterable<File> classes, + Iterable<File> manifests, File mapping, Iterable<File> resources, File reportFile) { + mRTxtFiles = rTxtFiles; + mProguardMapping = mapping; + mClasses = classes; + mManifests = manifests; + mResourceDirs = resources; + + mReportFile = reportFile; + if (reportFile != null) { + mDebugOutput = new StringWriter(8 * 1024); + mDebugPrinter = new PrintWriter(mDebugOutput); + } else { + mDebugOutput = null; + mDebugPrinter = null; + } + } + + public void close() { + if (mDebugOutput != null) { + String output = mDebugOutput.toString(); + + if (mReportFile != null) { + File dir = mReportFile.getParentFile(); + if (dir != null) { + if ((dir.exists() || dir.mkdir()) && dir.canWrite()) { + try { + Files.asCharSink(mReportFile, Charsets.UTF_8).write(output); + } catch (IOException ignore) { + } + } + } + } + } + } + + public void analyze() throws IOException, ParserConfigurationException, SAXException { + gatherResourceValues(mRTxtFiles); + recordMapping(mProguardMapping); + + for (File jarOrDir : mClasses) { + recordClassUsages(jarOrDir); + } + recordManifestUsages(mManifests); + recordResources(mResourceDirs); + dumpReferences(); + mModel.processToolsAttributes(); + mUnused = mModel.findUnused(); + } + + public void emitConfig(Path destination) throws IOException { + File destinationFile = destination.toFile(); + if (!destinationFile.exists()) { + destinationFile.getParentFile().mkdirs(); + boolean success = destinationFile.createNewFile(); + if (!success) { + throw new IOException("Could not create " + destination); + } + } + StringBuilder sb = new StringBuilder(); + Collections.sort(mUnused); + for (Resource resource : mUnused) { + sb.append(resource.type + "/" + resource.name + "#remove\n"); + } + Files.asCharSink(destinationFile, UTF_8).write(sb.toString()); + } + + private void dumpReferences() { + if (mDebugPrinter != null) { + mDebugPrinter.print(mModel.dumpReferences()); + } + } + + private void recordResources(Iterable<File> resources) + throws IOException, SAXException, ParserConfigurationException { + for (File resDir : resources) { + File[] resourceFolders = resDir.listFiles(); + assert resourceFolders != null : "Invalid resource directory " + resDir; + for (File folder : resourceFolders) { + ResourceFolderType folderType = ResourceFolderType.getFolderType(folder.getName()); + if (folderType != null) { + recordResources(folderType, folder); + } + } + } + } + + private void recordResources(ResourceFolderType folderType, File folder) + throws ParserConfigurationException, SAXException, IOException { + File[] files = folder.listFiles(); + if (files != null) { + for (File file : files) { + String path = file.getPath(); + mModel.file = file; + try { + boolean isXml = endsWithIgnoreCase(path, DOT_XML); + if (isXml) { + String xml = Files.toString(file, UTF_8); + Document document = XmlUtils.parseDocument(xml, true); + mModel.visitXmlDocument(file, folderType, document); + } else { + mModel.visitBinaryResource(folderType, file); + } + } finally { + mModel.file = null; + } + } + } + } + + void recordMapping(File mapping) throws IOException { + if (mapping == null || !mapping.exists()) { + return; + } + final String arrowString = " -> "; + final String resourceString = ".R$"; + Map<String, String> nameMap = null; + for (String line : Files.readLines(mapping, UTF_8)) { + // Ignore R8's mapping comments. + if (line.startsWith("#")) { + continue; + } + if (line.startsWith(" ") || line.startsWith("\t")) { + if (nameMap != null) { + // We're processing the members of a resource class: record names into the map + int n = line.length(); + int i = 0; + for (; i < n; i++) { + if (!Character.isWhitespace(line.charAt(i))) { + break; + } + } + if (i < n && line.startsWith("int", i)) { // int or int[] + int start = line.indexOf(' ', i + 3) + 1; + int arrow = line.indexOf(arrowString); + if (start > 0 && arrow != -1) { + int end = line.indexOf(' ', start + 1); + if (end != -1) { + String oldName = line.substring(start, end); + String newName = + line.substring(arrow + arrowString.length()).trim(); + if (!newName.equals(oldName)) { + nameMap.put(newName, oldName); + } + } + } + } + } + continue; + } else { + nameMap = null; + } + int index = line.indexOf(resourceString); + if (index == -1) { + // Record obfuscated names of a few known appcompat usages of + // Resources#getIdentifier that are unlikely to be used for general + // resource name reflection + if (line.startsWith("android.support.v7.widget.SuggestionsAdapter ")) { + mSuggestionsAdapter = + line.substring(line.indexOf(arrowString) + arrowString.length(), + line.indexOf(':') != -1 ? line.indexOf(':') : line.length()) + .trim() + .replace('.', '/') + + DOT_CLASS; + } else if (line.startsWith("android.support.v7.internal.widget.ResourcesWrapper ") + || line.startsWith("android.support.v7.widget.ResourcesWrapper ") + || (mResourcesWrapper == null // Recently wrapper moved + && line.startsWith( + "android.support.v7.widget.TintContextWrapper$TintResources "))) { + mResourcesWrapper = + line.substring(line.indexOf(arrowString) + arrowString.length(), + line.indexOf(':') != -1 ? line.indexOf(':') : line.length()) + .trim() + .replace('.', '/') + + DOT_CLASS; + } + continue; + } + int arrow = line.indexOf(arrowString, index + 3); + if (arrow == -1) { + continue; + } + String typeName = line.substring(index + resourceString.length(), arrow); + ResourceType type = ResourceType.fromClassName(typeName); + if (type == null) { + continue; + } + int end = line.indexOf(':', arrow + arrowString.length()); + if (end == -1) { + end = line.length(); + } + String target = line.substring(arrow + arrowString.length(), end).trim(); + String ownerName = target.replace('.', '/'); + + nameMap = Maps.newHashMap(); + Pair<ResourceType, Map<String, String>> pair = new Pair(type, nameMap); + mResourceObfuscation.put(ownerName, pair); + // For fast lookup in isResourceClass + mResourceObfuscation.put(ownerName + DOT_CLASS, pair); + } + } + + private void recordManifestUsages(File manifest) + throws IOException, ParserConfigurationException, SAXException { + String xml = Files.toString(manifest, UTF_8); + Document document = XmlUtils.parseDocument(xml, true); + mModel.visitXmlDocument(manifest, null, document); + } + + private void recordManifestUsages(Iterable<File> manifests) + throws IOException, ParserConfigurationException, SAXException { + for (File manifest : manifests) { + recordManifestUsages(manifest); + } + } + + private void recordClassUsages(File file) throws IOException { + assert file.isFile(); + if (file.getPath().endsWith(DOT_DEX)) { + byte[] bytes = Files.toByteArray(file); + recordClassUsages(file, file.getName(), bytes); + } else if (file.getPath().endsWith(DOT_JAR)) { + ZipInputStream zis = null; + try { + FileInputStream fis = new FileInputStream(file); + try { + zis = new ZipInputStream(fis); + ZipEntry entry = zis.getNextEntry(); + while (entry != null) { + String name = entry.getName(); + if (name.endsWith(DOT_DEX)) { + byte[] bytes = ByteStreams.toByteArray(zis); + if (bytes != null) { + recordClassUsages(file, name, bytes); + } + } + + entry = zis.getNextEntry(); + } + } finally { + Closeables.close(fis, true); + } + } finally { + Closeables.close(zis, true); + } + } + } + + private String stringifyResource(Resource resource) { + return String.format("%s:%s:0x%08x", resource.type, resource.name, resource.value); + } + + private void recordClassUsages(File file, String name, byte[] bytes) { + assert name.endsWith(DOT_DEX); + ReferenceChecker callback = new ReferenceChecker() { + @Override + public boolean shouldProcess(String internalName) { + // We do not need to ignore R subclasses since R8 now removes + // unused resource id fields in R subclasses thus their + // remaining presence means real usage. + return true; + } + + @Override + public void referencedInt(int value) { + UnusedResources.this.referencedInt("dex", value, file, name); + } + + @Override + public void referencedString(String value) { + // do nothing. + } + + @Override + public void referencedStaticField(String internalName, String fieldName) { + Resource resource = getResourceFromCode(internalName, fieldName); + if (resource != null) { + ResourceUsageModel.markReachable(resource); + if (mDebugPrinter != null) { + mDebugPrinter.println("Marking " + stringifyResource(resource) + + " reachable: referenced from dex" + + " in " + file + ":" + name + " (static field access " + + internalName + "." + fieldName + ")"); + } + } + } + + @Override + public void referencedMethod( + String internalName, String methodName, String methodDescriptor) { + // Do nothing. + } + }; + ProgramResource resource = ProgramResource.fromBytes( + new PathOrigin(file.toPath()), ProgramResource.Kind.DEX, bytes, null); + ProgramResourceProvider provider = () -> Arrays.asList(resource); + try { + Command command = + (new ResourceShrinker.Builder()).addProgramResourceProvider(provider).build(); + ResourceShrinker.run(command, callback); + } catch (CompilationFailedException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + + /** Returns whether the given class file name points to an aapt-generated compiled R class. */ + boolean isResourceClass(String name) { + if (mResourceObfuscation.containsKey(name)) { + return true; + } + int index = name.lastIndexOf('/'); + if (index != -1 && name.startsWith("R$", index + 1) && name.endsWith(DOT_CLASS)) { + String typeName = name.substring(index + 3, name.length() - DOT_CLASS.length()); + return ResourceType.fromClassName(typeName) != null; + } + return false; + } + + Resource getResourceFromCode(String owner, String name) { + Pair<ResourceType, Map<String, String>> pair = mResourceObfuscation.get(owner); + if (pair != null) { + ResourceType type = pair.getFirst(); + Map<String, String> nameMap = pair.getSecond(); + String renamedField = nameMap.get(name); + if (renamedField != null) { + name = renamedField; + } + return mModel.getResource(type, name); + } + if (isValidResourceType(owner)) { + ResourceType type = + ResourceType.fromClassName(owner.substring(owner.lastIndexOf('$') + 1)); + if (type != null) { + return mModel.getResource(type, name); + } + } + return null; + } + + private Boolean isValidResourceType(String candidateString) { + return candidateString.contains("/") + && candidateString.substring(candidateString.lastIndexOf('/') + 1).contains("$"); + } + + private void gatherResourceValues(Iterable<File> rTxts) throws IOException { + for (File rTxt : rTxts) { + assert rTxt.isFile(); + assert rTxt.getName().endsWith(FN_RESOURCE_TEXT); + addResourcesFromRTxtFile(rTxt); + } + } + + private void addResourcesFromRTxtFile(File file) { + try { + SymbolTable st = readFromAapt(file, null); + for (Symbol symbol : st.getSymbols().values()) { + String symbolValue = symbol.getValue(); + if (symbol.getResourceType() == ResourceType.STYLEABLE) { + if (symbolValue.trim().startsWith("{")) { + // Only add the styleable parent, styleable children are not yet supported. + mModel.addResource(symbol.getResourceType(), symbol.getName(), null); + } + } else { + if (mDebugPrinter != null) { + mDebugPrinter.println("Extracted R.txt resource: " + + symbol.getResourceType() + ":" + symbol.getName() + ":" + + String.format( + "0x%08x", Integer.parseInt(symbolValue.substring(2), 16))); + } + mModel.addResource(symbol.getResourceType(), symbol.getName(), symbolValue); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + ResourceUsageModel getModel() { + return mModel; + } + + private void referencedInt(String context, int value, File file, String currentClass) { + Resource resource = mModel.getResource(value); + if (ResourceUsageModel.markReachable(resource) && mDebugPrinter != null) { + mDebugPrinter.println("Marking " + stringifyResource(resource) + + " reachable: referenced from " + context + " in " + file + ":" + + currentClass); + } + } + + private final ResourceShrinkerUsageModel mModel = new ResourceShrinkerUsageModel(); + + private class ResourceShrinkerUsageModel extends ResourceUsageModel { + public File file; + + /** + * Whether we should ignore tools attribute resource references. + * <p> + * For example, for resource shrinking we want to ignore tools attributes, + * whereas for resource refactoring on the source code we do not. + * + * @return whether tools attributes should be ignored + */ + @Override + protected boolean ignoreToolsAttributes() { + return true; + } + + @Override + protected void onRootResourcesFound(List<Resource> roots) { + if (mDebugPrinter != null) { + mDebugPrinter.println("\nThe root reachable resources are:"); + for (Resource root : roots) { + mDebugPrinter.println(" " + stringifyResource(root) + ","); + } + } + } + + @Override + protected Resource declareResource(ResourceType type, String name, Node node) { + Resource resource = super.declareResource(type, name, node); + resource.addLocation(file); + return resource; + } + + @Override + protected void referencedString(String string) { + // Do nothing + } + } + + public static void main(String[] args) throws Exception { + List<File> rTxtFiles = null; // R.txt files + List<File> classes = null; // Dex/jar w dex + List<File> manifests = null; // manifests + File mapping = null; // mapping + List<File> resources = null; // resources dirs + File log = null; // output log for debugging + Path configPath = null; // output config + for (int i = 0; i < args.length; i += 2) { + switch (args[i]) { + case "--rtxts": + rTxtFiles = Arrays.stream(args[i + 1].split(":")) + .map(s -> new File(s)) + .collect(Collectors.toList()); + break; + case "--dexes": + classes = Arrays.stream(args[i + 1].split(":")) + .map(s -> new File(s)) + .collect(Collectors.toList()); + break; + case "--manifests": + manifests = Arrays.stream(args[i + 1].split(":")) + .map(s -> new File(s)) + .collect(Collectors.toList()); + break; + case "--mapping": + mapping = new File(args[i + 1]); + break; + case "--resourceDirs": + resources = Arrays.stream(args[i + 1].split(":")) + .map(s -> new File(s)) + .collect(Collectors.toList()); + break; + case "--log": + log = new File(args[i + 1]); + break; + case "--outputConfig": + configPath = Paths.get(args[i + 1]); + break; + default: + throw new IllegalArgumentException(args[i] + " is not a valid arg."); + } + } + UnusedResources unusedResources = + new UnusedResources(rTxtFiles, classes, manifests, mapping, resources, log); + unusedResources.analyze(); + unusedResources.close(); + unusedResources.emitConfig(configPath); + } +}
diff --git a/build/android/update_deps/update_third_party_deps.py b/build/android/update_deps/update_third_party_deps.py index 3a869c4..50c0e22 100755 --- a/build/android/update_deps/update_third_party_deps.py +++ b/build/android/update_deps/update_third_party_deps.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2016 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/android/update_verification.py b/build/android/update_verification.py index 3d478f4..55a403e 100755 --- a/build/android/update_verification.py +++ b/build/android/update_verification.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env vpython +#!/usr/bin/env vpython3 # -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -28,6 +28,8 @@ import logging import sys +# import raw_input when converted to python3 +from six.moves import input # pylint: disable=redefined-builtin import devil_chromium from devil.android import apk_helper @@ -36,10 +38,11 @@ from devil.android import device_utils from devil.utils import run_tests_helper + def CreateAppData(device, old_apk, app_data, package_name): device.Install(old_apk) - raw_input('Set the application state. Once ready, press enter and ' - 'select "Backup my data" on the device.') + input('Set the application state. Once ready, press enter and ' + 'select "Backup my data" on the device.') device.adb.Backup(app_data, packages=[package_name]) logging.critical('Application data saved to %s', app_data) @@ -47,8 +50,8 @@ device.Install(old_apk) device.adb.Restore(app_data) # Restore command is not synchronous - raw_input('Select "Restore my data" on the device. Then press enter to ' - 'continue.') + input('Select "Restore my data" on the device. Then press enter to ' + 'continue.') if not device.IsApplicationInstalled(package_name): raise Exception('Expected package %s to already be installed. ' 'Package name might have changed!' % package_name)
diff --git a/build/android/video_recorder.py b/build/android/video_recorder.py index 6c54e7a..3938779 100755 --- a/build/android/video_recorder.py +++ b/build/android/video_recorder.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env vpython -# Copyright 2015 The Chromium Authors. All rights reserved. +#!/usr/bin/env vpython3 +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/apple/apple_info_plist.gni b/build/apple/apple_info_plist.gni index fe51773..bf66dbd 100644 --- a/build/apple/apple_info_plist.gni +++ b/build/apple/apple_info_plist.gni
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/apple/compile_entitlements.gni b/build/apple/compile_entitlements.gni index 006d5ac..1f84a11 100644 --- a/build/apple/compile_entitlements.gni +++ b/build/apple/compile_entitlements.gni
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/apple/compile_plist.gni b/build/apple/compile_plist.gni index 90485b6..df8de0c 100644 --- a/build/apple/compile_plist.gni +++ b/build/apple/compile_plist.gni
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/apple/convert_plist.gni b/build/apple/convert_plist.gni index a1134d9..740bfc7 100644 --- a/build/apple/convert_plist.gni +++ b/build/apple/convert_plist.gni
@@ -1,4 +1,4 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/apple/plist_util.py b/build/apple/plist_util.py index 54cf461..016a06a 100644 --- a/build/apple/plist_util.py +++ b/build/apple/plist_util.py
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,11 +12,6 @@ import tempfile import shlex -if sys.version_info.major < 3: - basestring_compat = basestring -else: - basestring_compat = str - # Xcode substitutes variables like ${PRODUCT_NAME} or $(PRODUCT_NAME) when # compiling Info.plist. It also supports supports modifiers like :identifier # or :rfc1034identifier. SUBSTITUTION_REGEXP_LIST is a list of regular @@ -89,46 +84,30 @@ return {k: Interpolate(v, substitutions) for k, v in value.items()} if isinstance(value, list): return [Interpolate(v, substitutions) for v in value] - if isinstance(value, basestring_compat): + if isinstance(value, str): return InterpolateString(value, substitutions) return value def LoadPList(path): """Loads Plist at |path| and returns it as a dictionary.""" - if sys.version_info.major == 2: - fd, name = tempfile.mkstemp() - try: - subprocess.check_call(['plutil', '-convert', 'xml1', '-o', name, path]) - with os.fdopen(fd, 'rb') as f: - return plistlib.readPlist(f) - finally: - os.unlink(name) - else: - with open(path, 'rb') as f: - return plistlib.load(f) + with open(path, 'rb') as f: + return plistlib.load(f) def SavePList(path, format, data): """Saves |data| as a Plist to |path| in the specified |format|.""" - # The below does not replace the destination file but update it in place, - # so if more than one hardlink points to destination all of them will be - # modified. This is not what is expected, so delete destination file if - # it does exist. - if os.path.exists(path): + # The open() call does not replace the destination file but updates it + # in place, so if more than one hardlink points to destination all of them + # will be modified. This is not what is expected, so delete destination file + # if it does exist. + try: os.unlink(path) - if sys.version_info.major == 2: - fd, name = tempfile.mkstemp() - try: - with os.fdopen(fd, 'wb') as f: - plistlib.writePlist(data, f) - subprocess.check_call(['plutil', '-convert', format, '-o', path, name]) - finally: - os.unlink(name) - else: - with open(path, 'wb') as f: - plist_format = {'binary1': plistlib.FMT_BINARY, 'xml1': plistlib.FMT_XML} - plistlib.dump(data, f, fmt=plist_format[format]) + except FileNotFoundError: + pass + with open(path, 'wb') as f: + plist_format = {'binary1': plistlib.FMT_BINARY, 'xml1': plistlib.FMT_XML} + plistlib.dump(data, f, fmt=plist_format[format]) def MergePList(plist1, plist2): @@ -243,10 +222,6 @@ def Main(): - # Cache this codec so that plistlib can find it. See - # https://crbug.com/1005190#c2 for more details. - codecs.lookup('utf-8') - parser = argparse.ArgumentParser(description='manipulate plist files') subparsers = parser.add_subparsers() @@ -258,8 +233,4 @@ if __name__ == '__main__': - # TODO(https://crbug.com/941669): Temporary workaround until all scripts use - # python3 by default. - if sys.version_info[0] < 3: - os.execvp('python3', ['python3'] + sys.argv) sys.exit(Main())
diff --git a/build/apple/tweak_info_plist.gni b/build/apple/tweak_info_plist.gni index 33f22ca..347c5d5 100644 --- a/build/apple/tweak_info_plist.gni +++ b/build/apple/tweak_info_plist.gni
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/apple/tweak_info_plist.py b/build/apple/tweak_info_plist.py index 76f64dc..8aa28b0 100755 --- a/build/apple/tweak_info_plist.py +++ b/build/apple/tweak_info_plist.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Copyright 2012 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -20,7 +20,6 @@ # by the time the app target is done, the info.plist is correct. # -from __future__ import print_function import optparse import os @@ -229,6 +228,16 @@ _RemoveKeys(plist, 'GTMUserAgentID', 'GTMUserAgentVersion') +def _AddPrivilegedHelperId(plist, privileged_helper_id): + plist['SMPrivilegedExecutables'] = { + privileged_helper_id: 'identifier ' + privileged_helper_id + } + + +def _RemovePrivilegedHelperId(plist): + _RemoveKeys(plist, 'SMPrivilegedExecutables') + + def Main(argv): parser = optparse.OptionParser('%prog [options]') parser.add_option('--plist', @@ -289,9 +298,6 @@ type='int', default=False, help='Add GTM metadata [1 or 0]') - # TODO(crbug.com/1140474): Remove once iOS 14.2 reaches mass adoption. - parser.add_option('--lock-to-version', - help='Set CFBundleVersion to given value + @MAJOR@@PATH@') parser.add_option( '--version-overrides', action='append', @@ -309,6 +315,12 @@ type='string', default=None, help='The version string [major.minor.build.patch]') + parser.add_option('--privileged_helper_id', + dest='privileged_helper_id', + action='store', + type='string', + default=None, + help='The id of the privileged helper executable.') (options, args) = parser.parse_args(argv) if len(args) > 0: @@ -359,25 +371,10 @@ 'CFBundleVersion': '@BUILD@.@PATCH@', } else: - # TODO(crbug.com/1140474): Remove once iOS 14.2 reaches mass adoption. - if options.lock_to_version: - # Pull in the PATCH number and format it to 3 digits. - VERSION_TOOL = os.path.join(TOP, 'build/util/version.py') - VERSION_FILE = os.path.join(TOP, 'chrome/VERSION') - (stdout, - retval) = _GetOutput([VERSION_TOOL, '-f', VERSION_FILE, '-t', '@PATCH@']) - if retval != 0: - return 2 - patch = '{:03d}'.format(int(stdout)) - version_format_for_key = { - 'CFBundleShortVersionString': '@MAJOR@.@BUILD@.@PATCH@', - 'CFBundleVersion': options.lock_to_version + '.@MAJOR@' + patch - } - else: - version_format_for_key = { - 'CFBundleShortVersionString': '@MAJOR@.@BUILD@.@PATCH@', - 'CFBundleVersion': '@MAJOR@.@MINOR@.@BUILD@.@PATCH@' - } + version_format_for_key = { + 'CFBundleShortVersionString': '@MAJOR@.@BUILD@.@PATCH@', + 'CFBundleVersion': '@MAJOR@.@MINOR@.@BUILD@.@PATCH@' + } if options.use_breakpad: version_format_for_key['BreakpadVersion'] = \ @@ -423,6 +420,12 @@ else: _RemoveGTMKeys(plist) + # Add SMPrivilegedExecutables keys. + if options.privileged_helper_id: + _AddPrivilegedHelperId(plist, options.privileged_helper_id) + else: + _RemovePrivilegedHelperId(plist) + output_path = options.plist_path if options.plist_output is not None: output_path = options.plist_output
diff --git a/build/apple/write_pkg_info.py b/build/apple/write_pkg_info.py index 8d07cdb..2f59c2f 100644 --- a/build/apple/write_pkg_info.py +++ b/build/apple/write_pkg_info.py
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -24,8 +24,10 @@ args = parser.parse_args() # Remove the output if it exists already. - if os.path.exists(args.output): + try: os.unlink(args.output) + except FileNotFoundError: + pass plist = plist_util.LoadPList(args.plist) package_type = plist['CFBundlePackageType'] @@ -47,8 +49,4 @@ if __name__ == '__main__': - # TODO(https://crbug.com/941669): Temporary workaround until all scripts use - # python3 by default. - if sys.version_info[0] < 3: - os.execvp('python3', ['python3'] + sys.argv) sys.exit(Main())
diff --git a/build/apple/xcrun.py b/build/apple/xcrun.py index 71bf50c..011dd47 100755 --- a/build/apple/xcrun.py +++ b/build/apple/xcrun.py
@@ -1,5 +1,5 @@ -#!/usr/bin/python3 -# Copyright 2020 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """
diff --git a/build/args/README.txt b/build/args/README.txt index 825bf64..b82fb04 100644 --- a/build/args/README.txt +++ b/build/args/README.txt
@@ -1,10 +1,6 @@ This directory is here to hold .gni files that contain sets of GN build arguments for given configurations. -(Currently this directory is empty because we removed the only thing here, but -this has come up several times so I'm confident we'll need this again. If this -directory is still empty by 2017, feel free to delete it. --Brett) - Some projects or bots may have build configurations with specific combinations of flags. Rather than making a new global flag for your specific project and adding it all over the build to each arg it should affect, you can add a .gni
diff --git a/build/args/chromeos/README.md b/build/args/chromeos/README.md index e02e185..2842252 100644 --- a/build/args/chromeos/README.md +++ b/build/args/chromeos/README.md
@@ -1,4 +1,6 @@ -This directory is used to store GN arg mapping for Chrome OS boards. +This directory is used to store GN arg mapping for Chrome OS boards. The values +of the args are determined by processing the [chromeos-chrome ebuild] for a +given board and a given ChromeOS version (stored in the [CHROMEOS_LKGM] file). Files in this directory are populated by running `gclient sync` with specific arguments set in the .gclient file. Specifically: @@ -50,3 +52,6 @@ TODO(bpastene): Make 'cros_boards' a first class citizen in gclient and replace it with 'target_boards' instead. + +[chromeos-chrome ebuild]: https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay/+/HEAD/chromeos-base/chromeos-chrome/chromeos-chrome-9999.ebuild +[CHROMEOS_LKGM]: https://chromium.googlesource.com/chromium/src/+/HEAD/chromeos/CHROMEOS_LKGM
diff --git a/build/args/headless.gn b/build/args/headless.gn index 9b8392c..8834eb1 100644 --- a/build/args/headless.gn +++ b/build/args/headless.gn
@@ -11,12 +11,14 @@ ozone_auto_platforms = false ozone_platform = "headless" ozone_platform_headless = true +angle_enable_vulkan = true +angle_enable_swiftshader = true # Embed resource.pak into binary to simplify deployment. headless_use_embedded_resources = true -# Expose headless bindings for freetype library bundled with Chromium. -headless_fontconfig_utils = true +# Disable headless commands support. +headless_enable_commands = false # Don't use Prefs component, disabling access to Local State prefs. headless_use_prefs = false @@ -39,6 +41,7 @@ enable_print_preview = false enable_remoting = false use_alsa = false +use_bluez = false use_cups = false use_dbus = false use_gio = false @@ -51,6 +54,3 @@ use_glib = false use_gtk = false use_pangocairo = false - -# TODO(1096425): Remove this once use_x11 goes away. -use_x11 = false
diff --git a/build/build-ctags.sh b/build/build-ctags.sh index 61e017e..d7756a2 100755 --- a/build/build-ctags.sh +++ b/build/build-ctags.sh
@@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/build_config.h b/build/build_config.h index 9d3f852..798b494 100644 --- a/build/build_config.h +++ b/build/build_config.h
@@ -1,34 +1,50 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Copyright 2012 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// This file adds defines about the platform we're currently building on. +// This file doesn't belong to any GN target by design for faster build and +// less developer overhead. + +// This file adds build flags about the OS we're currently building on. They are +// defined directly in this file instead of via a `buildflag_header` target in a +// GN file for faster build. They are defined using the corresponding OS defines +// (e.g. OS_WIN) which are also defined in this file (except for OS_CHROMEOS, +// which is set by the build system). These defines are deprecated and should +// NOT be used directly. For example: +// Please Use: #if BUILDFLAG(IS_WIN) +// Deprecated: #if defined(OS_WIN) // // Operating System: -// OS_AIX / OS_ANDROID / OS_ASMJS / OS_FREEBSD / OS_FUCHSIA / OS_IOS / -// OS_LINUX / OS_MAC / OS_NACL (SFI or NONSFI) / OS_NETBSD / OS_OPENBSD / -// OS_QNX / OS_SOLARIS / OS_WIN +// IS_AIX / IS_ANDROID / IS_ASMJS / IS_CHROMEOS / IS_FREEBSD / IS_FUCHSIA / +// IS_IOS / IS_IOS_MACCATALYST / IS_LINUX / IS_MAC / IS_NACL / IS_NETBSD / +// IS_OPENBSD / IS_QNX / IS_SOLARIS / IS_WIN // Operating System family: -// OS_APPLE: IOS or MAC -// OS_BSD: FREEBSD or NETBSD or OPENBSD -// OS_POSIX: AIX or ANDROID or ASMJS or CHROMEOS or FREEBSD or IOS or LINUX +// IS_APPLE: IOS or MAC or IOS_MACCATALYST +// IS_BSD: FREEBSD or NETBSD or OPENBSD +// IS_POSIX: AIX or ANDROID or ASMJS or CHROMEOS or FREEBSD or IOS or LINUX // or MAC or NACL or NETBSD or OPENBSD or QNX or SOLARIS + +// This file also adds defines specific to the platform, architecture etc. // -// /!\ Note: OS_CHROMEOS is set by the build system, not this file +// Platform: +// IS_OZONE // // Compiler: // COMPILER_MSVC / COMPILER_GCC // // Processor: -// ARCH_CPU_ARM64 / ARCH_CPU_ARMEL / ARCH_CPU_MIPS / ARCH_CPU_MIPS64 / -// ARCH_CPU_MIPS64EL / ARCH_CPU_MIPSEL / ARCH_CPU_PPC64 / ARCH_CPU_S390 / -// ARCH_CPU_S390X / ARCH_CPU_X86 / ARCH_CPU_X86_64 +// ARCH_CPU_ARM64 / ARCH_CPU_ARMEL / ARCH_CPU_LOONG32 / ARCH_CPU_LOONG64 / +// ARCH_CPU_MIPS / ARCH_CPU_MIPS64 / ARCH_CPU_MIPS64EL / ARCH_CPU_MIPSEL / +// ARCH_CPU_PPC64 / ARCH_CPU_S390 / ARCH_CPU_S390X / ARCH_CPU_X86 / +// ARCH_CPU_X86_64 / ARCH_CPU_RISCV64 // Processor family: // ARCH_CPU_ARM_FAMILY: ARMEL or ARM64 +// ARCH_CPU_LOONG_FAMILY: LOONG32 or LOONG64 // ARCH_CPU_MIPS_FAMILY: MIPS64EL or MIPSEL or MIPS64 or MIPS // ARCH_CPU_PPC64_FAMILY: PPC64 // ARCH_CPU_S390_FAMILY: S390 or S390X // ARCH_CPU_X86_FAMILY: X86 or X86_64 +// ARCH_CPU_RISCV_FAMILY: Riscv64 // Processor features: // ARCH_CPU_31_BITS / ARCH_CPU_32_BITS / ARCH_CPU_64_BITS // ARCH_CPU_BIG_ENDIAN / ARCH_CPU_LITTLE_ENDIAN @@ -36,20 +52,14 @@ #ifndef BUILD_BUILD_CONFIG_H_ #define BUILD_BUILD_CONFIG_H_ +#include "build/buildflag.h" // IWYU pragma: export + // A set of macros to use for platform detection. #if defined(STARBOARD) // noop #elif defined(__native_client__) // __native_client__ must be first, so that other OS_ defines are not set. #define OS_NACL 1 -// OS_NACL comes in two sandboxing technology flavors, SFI or Non-SFI. -// PNaCl toolchain defines __native_client_nonsfi__ macro in Non-SFI build -// mode, while it does not in SFI build mode. -#if defined(__native_client_nonsfi__) -#define OS_NACL_NONSFI -#else -#define OS_NACL_SFI -#endif #elif defined(ANDROID) #define OS_ANDROID 1 #elif defined(__APPLE__) @@ -59,6 +69,11 @@ #include <TargetConditionals.h> #if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE #define OS_IOS 1 +// Catalyst is the technology that allows running iOS apps on macOS. These +// builds are both OS_IOS and OS_IOS_MACCATALYST. +#if defined(TARGET_OS_MACCATALYST) && TARGET_OS_MACCATALYST +#define OS_IOS_MACCATALYST +#endif // defined(TARGET_OS_MACCATALYST) && TARGET_OS_MACCATALYST #else #define OS_MAC 1 #endif // defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE @@ -69,7 +84,7 @@ #define OS_LINUX 1 #endif // !defined(OS_CHROMEOS) // Include a system header to pull in features.h for glibc/uclibc macros. -#include <unistd.h> +#include <assert.h> #if defined(__GLIBC__) && !defined(__UCLIBC__) // We really are using glibc, not uClibc pretending to be glibc. #define LIBC_GLIBC 1 @@ -92,11 +107,13 @@ #define OS_AIX 1 #elif defined(__asmjs__) || defined(__wasm__) #define OS_ASMJS 1 +#elif defined(__MVS__) +#define OS_ZOS 1 #else #error Please add support for your platform in build/build_config.h #endif // NOTE: Adding a new port? Please follow -// https://chromium.googlesource.com/chromium/src/+/master/docs/new_port_policy.md +// https://chromium.googlesource.com/chromium/src/+/main/docs/new_port_policy.md #if defined(OS_MAC) || defined(OS_IOS) #define OS_APPLE 1 @@ -114,10 +131,131 @@ defined(OS_FREEBSD) || defined(OS_IOS) || defined(OS_LINUX) || \ defined(OS_CHROMEOS) || defined(OS_MAC) || defined(OS_NACL) || \ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_QNX) || \ - defined(OS_SOLARIS) + defined(OS_SOLARIS) || defined(OS_ZOS) #define OS_POSIX 1 #endif +// OS build flags +#if defined(OS_AIX) +#define BUILDFLAG_INTERNAL_IS_AIX() (1) +#else +#define BUILDFLAG_INTERNAL_IS_AIX() (0) +#endif + +#if defined(OS_ANDROID) +#define BUILDFLAG_INTERNAL_IS_ANDROID() (1) +#else +#define BUILDFLAG_INTERNAL_IS_ANDROID() (0) +#endif + +#if defined(OS_APPLE) +#define BUILDFLAG_INTERNAL_IS_APPLE() (1) +#else +#define BUILDFLAG_INTERNAL_IS_APPLE() (0) +#endif + +#if defined(OS_ASMJS) +#define BUILDFLAG_INTERNAL_IS_ASMJS() (1) +#else +#define BUILDFLAG_INTERNAL_IS_ASMJS() (0) +#endif + +#if defined(OS_BSD) +#define BUILDFLAG_INTERNAL_IS_BSD() (1) +#else +#define BUILDFLAG_INTERNAL_IS_BSD() (0) +#endif + +#if defined(OS_CHROMEOS) +#define BUILDFLAG_INTERNAL_IS_CHROMEOS() (1) +#else +#define BUILDFLAG_INTERNAL_IS_CHROMEOS() (0) +#endif + +#if defined(OS_FREEBSD) +#define BUILDFLAG_INTERNAL_IS_FREEBSD() (1) +#else +#define BUILDFLAG_INTERNAL_IS_FREEBSD() (0) +#endif + +#if defined(OS_FUCHSIA) +#define BUILDFLAG_INTERNAL_IS_FUCHSIA() (1) +#else +#define BUILDFLAG_INTERNAL_IS_FUCHSIA() (0) +#endif + +#if defined(OS_IOS) +#define BUILDFLAG_INTERNAL_IS_IOS() (1) +#else +#define BUILDFLAG_INTERNAL_IS_IOS() (0) +#endif + +#if defined(OS_IOS_MACCATALYST) +#define BUILDFLAG_INTERNAL_IS_IOS_MACCATALYST() (1) +#else +#define BUILDFLAG_INTERNAL_IS_IOS_MACCATALYST() (0) +#endif + +#if defined(OS_LINUX) +#define BUILDFLAG_INTERNAL_IS_LINUX() (1) +#else +#define BUILDFLAG_INTERNAL_IS_LINUX() (0) +#endif + +#if defined(OS_MAC) +#define BUILDFLAG_INTERNAL_IS_MAC() (1) +#else +#define BUILDFLAG_INTERNAL_IS_MAC() (0) +#endif + +#if defined(OS_NACL) +#define BUILDFLAG_INTERNAL_IS_NACL() (1) +#else +#define BUILDFLAG_INTERNAL_IS_NACL() (0) +#endif + +#if defined(OS_NETBSD) +#define BUILDFLAG_INTERNAL_IS_NETBSD() (1) +#else +#define BUILDFLAG_INTERNAL_IS_NETBSD() (0) +#endif + +#if defined(OS_OPENBSD) +#define BUILDFLAG_INTERNAL_IS_OPENBSD() (1) +#else +#define BUILDFLAG_INTERNAL_IS_OPENBSD() (0) +#endif + +#if defined(OS_POSIX) +#define BUILDFLAG_INTERNAL_IS_POSIX() (1) +#else +#define BUILDFLAG_INTERNAL_IS_POSIX() (0) +#endif + +#if defined(OS_QNX) +#define BUILDFLAG_INTERNAL_IS_QNX() (1) +#else +#define BUILDFLAG_INTERNAL_IS_QNX() (0) +#endif + +#if defined(OS_SOLARIS) +#define BUILDFLAG_INTERNAL_IS_SOLARIS() (1) +#else +#define BUILDFLAG_INTERNAL_IS_SOLARIS() (0) +#endif + +#if defined(OS_WIN) +#define BUILDFLAG_INTERNAL_IS_WIN() (1) +#else +#define BUILDFLAG_INTERNAL_IS_WIN() (0) +#endif + +#if defined(USE_OZONE) +#define BUILDFLAG_INTERNAL_IS_OZONE() (1) +#else +#define BUILDFLAG_INTERNAL_IS_OZONE() (0) +#endif + // Compiler detection. Note: clang masquerades as GCC on POSIX and as MSVC on // Windows. #if defined(__GNUC__) @@ -225,6 +363,21 @@ #define ARCH_CPU_32_BITS 1 #define ARCH_CPU_BIG_ENDIAN 1 #endif +#elif defined(__loongarch32) +#define ARCH_CPU_LOONG_FAMILY 1 +#define ARCH_CPU_LOONG32 1 +#define ARCH_CPU_32_BITS 1 +#define ARCH_CPU_LITTLE_ENDIAN 1 +#elif defined(__loongarch64) +#define ARCH_CPU_LOONG_FAMILY 1 +#define ARCH_CPU_LOONG64 1 +#define ARCH_CPU_64_BITS 1 +#define ARCH_CPU_LITTLE_ENDIAN 1 +#elif defined(__riscv) && (__riscv_xlen == 64) +#define ARCH_CPU_RISCV_FAMILY 1 +#define ARCH_CPU_RISCV64 1 +#define ARCH_CPU_64_BITS 1 +#define ARCH_CPU_LITTLE_ENDIAN 1 #else #error Please add support for your architecture in build/build_config.h #endif
diff --git a/build/buildflag.h b/build/buildflag.h index 5776a75..6346979 100644 --- a/build/buildflag.h +++ b/build/buildflag.h
@@ -1,4 +1,4 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. +// Copyright 2015 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/buildflag_header.gni b/build/buildflag_header.gni index 821c4ef..f7b42f7 100644 --- a/build/buildflag_header.gni +++ b/build/buildflag_header.gni
@@ -1,4 +1,4 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/check_gn_headers.py b/build/check_gn_headers.py index 9bdbba8..6bfb878 100755 --- a/build/check_gn_headers.py +++ b/build/check_gn_headers.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,8 +9,6 @@ dependency generated by the compiler, and report if they don't exist in GN. """ -from __future__ import print_function - import argparse import json import os @@ -30,7 +28,10 @@ """Return all the header files from ninja_deps""" def NinjaSource(): - cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-t', 'deps'] + cmd = [ + os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja'), '-C', out_dir, + '-t', 'deps' + ] # A negative bufsize means to use the system default, which usually # means fully buffered. popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1) @@ -112,7 +113,7 @@ """Parse GN output and get the header files""" all_headers = set() - for _target, properties in gn['targets'].iteritems(): + for _target, properties in gn['targets'].items(): sources = properties.get('sources', []) public = properties.get('public', []) # Exclude '"public": "*"'. @@ -294,7 +295,7 @@ print(' ', cc) print('\nMissing headers sorted by number of affected object files:') - count = {k: len(v) for (k, v) in d.iteritems()} + count = {k: len(v) for (k, v) in d.items()} for f in sorted(count, key=count.get, reverse=True): if f in missing: print(count[f], f)
diff --git a/build/check_gn_headers_unittest.py b/build/check_gn_headers_unittest.py index 20c3b13..954d95b 100755 --- a/build/check_gn_headers_unittest.py +++ b/build/check_gn_headers_unittest.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -71,7 +71,7 @@ 'dir3/path/b.h': ['obj/c.o'], 'c3.hh': ['obj/c.o'], } - self.assertEquals(headers, expected) + self.assertEqual(headers, expected) def testGn(self): headers = check_gn_headers.ParseGNProjectJSON(gn_input, @@ -83,7 +83,7 @@ 'base/p.h', 'out/Release/gen/a.h', ]) - self.assertEquals(headers, expected) + self.assertEqual(headers, expected) def testWhitelist(self): output = check_gn_headers.ParseWhiteList(whitelist) @@ -93,7 +93,7 @@ 'dir/white-both.c', 'a/b/c', ]) - self.assertEquals(output, expected) + self.assertEqual(output, expected) if __name__ == '__main__':
diff --git a/build/check_gn_headers_whitelist.txt b/build/check_gn_headers_whitelist.txt index 2acf1b7..dfefd7d 100644 --- a/build/check_gn_headers_whitelist.txt +++ b/build/check_gn_headers_whitelist.txt
@@ -3,12 +3,11 @@ ash/accelerators/accelerator_table.h ash/ash_export.h +ash/constants/ash_switches.h ash/metrics/task_switch_metrics_recorder.h ash/metrics/task_switch_source.h -ash/metrics/user_metrics_action.h ash/metrics/user_metrics_recorder.h ash/public/cpp/ash_public_export.h -ash/public/cpp/ash_switches.h ash/public/cpp/shelf_types.h ash/session/session_observer.h ash/shell.h @@ -17,24 +16,20 @@ cc/input/browser_controls_state.h cc/input/event_listener_properties.h cc/input/scrollbar.h -cc/input/scroller_size_metrics.h cc/layers/performance_properties.h chrome/browser/android/android_theme_resources.h chrome/browser/android/resource_id.h -chrome/browser/ash/certificate_provider/certificate_info.h -chrome/browser/ash/certificate_provider/certificate_provider.h -chrome/browser/ash/certificate_provider/certificate_provider_service.h -chrome/browser/ash/certificate_provider/certificate_provider_service_factory.h -chrome/browser/ash/certificate_provider/certificate_requests.h -chrome/browser/ash/certificate_provider/pin_dialog_manager.h -chrome/browser/ash/certificate_provider/sign_requests.h -chrome/browser/ash/certificate_provider/thread_safe_certificate_map.h chrome/browser/ash/login/signin/oauth2_login_manager.h -chrome/browser/ash/login/signin/oauth2_login_verifier.h chrome/browser/ash/login/signin/oauth2_token_fetcher.h -chrome/browser/ash/notifications/request_pin_view.h chrome/browser/ash/profiles/profile_helper.h chrome/browser/ash/settings/cros_settings.h +chrome/browser/certificate_provider/certificate_provider.h +chrome/browser/certificate_provider/certificate_provider_service.h +chrome/browser/certificate_provider/certificate_provider_service_factory.h +chrome/browser/certificate_provider/certificate_requests.h +chrome/browser/certificate_provider/pin_dialog_manager.h +chrome/browser/certificate_provider/sign_requests.h +chrome/browser/certificate_provider/thread_safe_certificate_map.h chrome/browser/component_updater/component_installer_errors.h chrome/browser/download/download_file_icon_extractor.h chrome/browser/extensions/api/networking_cast_private/chrome_networking_cast_private_delegate.h @@ -46,7 +41,7 @@ chrome/browser/media_galleries/media_file_system_context.h chrome/browser/notifications/displayed_notifications_dispatch_callback.h chrome/browser/ui/app_icon_loader_delegate.h -chrome/browser/ui/app_list/app_list_syncable_service_factory.h +chrome/browser/ash/app_list/app_list_syncable_service_factory.h chrome/browser/ui/ash/ash_util.h chrome/browser/ui/ash/multi_user/multi_user_util.h chrome/browser/ui/network_profile_bubble.h @@ -58,17 +53,12 @@ chrome/install_static/install_util.h chrome/install_static/test/scoped_install_details.h chrome/installer/util/google_update_settings.h -components/browser_watcher/features.h -components/browser_watcher/stability_paths.h -components/cast_certificate/cast_crl_root_ca_cert_der-inc.h components/cdm/browser/cdm_message_filter_android.h components/device_event_log/device_event_log_export.h components/login/login_export.h +components/media_router/common/providers/cast/certificate/cast_crl_root_ca_cert_der-inc.h components/nacl/browser/nacl_browser_delegate.h components/nacl/renderer/ppb_nacl_private.h -components/omnibox/browser/autocomplete_i18n.h -components/omnibox/browser/autocomplete_provider_client.h -components/omnibox/browser/autocomplete_provider_listener.h components/policy/core/browser/configuration_policy_handler_parameters.h components/policy/proto/policy_proto_export.h components/rlz/rlz_tracker_delegate.h @@ -117,7 +107,6 @@ gpu/config/gpu_lists_version.h gpu/gles2_conform_support/gtf/gtf_stubs.h gpu/gpu_export.h -headless/lib/headless_macros.h ipc/ipc_channel_proxy_unittest_messages.h ipc/ipc_message_null_macros.h media/audio/audio_logging.h @@ -220,27 +209,6 @@ third_party/snappy/linux/config.h third_party/speech-dispatcher/libspeechd.h third_party/sqlite/sqlite3.h -third_party/tcmalloc/chromium/src/addressmap-inl.h -third_party/tcmalloc/chromium/src/base/basictypes.h -third_party/tcmalloc/chromium/src/base/dynamic_annotations.h -third_party/tcmalloc/chromium/src/base/googleinit.h -third_party/tcmalloc/chromium/src/base/linux_syscall_support.h -third_party/tcmalloc/chromium/src/base/spinlock_linux-inl.h -third_party/tcmalloc/chromium/src/base/stl_allocator.h -third_party/tcmalloc/chromium/src/base/thread_annotations.h -third_party/tcmalloc/chromium/src/base/thread_lister.h -third_party/tcmalloc/chromium/src/gperftools/malloc_extension_c.h -third_party/tcmalloc/chromium/src/gperftools/malloc_hook_c.h -third_party/tcmalloc/chromium/src/gperftools/tcmalloc.h -third_party/tcmalloc/chromium/src/heap-profile-stats.h -third_party/tcmalloc/chromium/src/libc_override.h -third_party/tcmalloc/chromium/src/malloc_hook_mmap_linux.h -third_party/tcmalloc/chromium/src/packed-cache-inl.h -third_party/tcmalloc/chromium/src/page_heap_allocator.h -third_party/tcmalloc/chromium/src/pagemap.h -third_party/tcmalloc/chromium/src/stacktrace_x86-inl.h -third_party/tcmalloc/chromium/src/system-alloc.h -third_party/tcmalloc/chromium/src/tcmalloc_guard.h third_party/wayland/include/config.h third_party/wayland/include/src/wayland-version.h third_party/woff2/src/port.h
diff --git a/build/check_return_value.py b/build/check_return_value.py index 9caa15f..2337e96 100755 --- a/build/check_return_value.py +++ b/build/check_return_value.py
@@ -1,12 +1,11 @@ -#!/usr/bin/env python -# Copyright 2014 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """This program wraps an arbitrary command and prints "1" if the command ran successfully.""" -from __future__ import print_function import os import subprocess
diff --git a/build/chromeos/.style.yapf b/build/chromeos/.style.yapf index de0c6a7..fdd0723 100644 --- a/build/chromeos/.style.yapf +++ b/build/chromeos/.style.yapf
@@ -1,2 +1,2 @@ [style] -based_on_style = chromium +based_on_style = yapf
diff --git a/build/chromeos/PRESUBMIT.py b/build/chromeos/PRESUBMIT.py index 312faf0..b9734e6 100644 --- a/build/chromeos/PRESUBMIT.py +++ b/build/chromeos/PRESUBMIT.py
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Presubmit script for build/chromeos/. @@ -8,13 +8,25 @@ """ +USE_PYTHON3 = True + + def CommonChecks(input_api, output_api): results = [] - results += input_api.canned_checks.RunPylint( - input_api, output_api, pylintrc='pylintrc') - tests = input_api.canned_checks.GetUnitTestsInDirectory( - input_api, output_api, '.', [r'^.+_test\.py$'], run_on_python3=True) - results += input_api.RunTests(tests) + # These tests don't run on Windows and give verbose and cryptic failure + # messages. Linting the code on a platform where it will not run is also not + # valuable and gives spurious errors. + if input_api.sys.platform != 'win32': + results += input_api.canned_checks.RunPylint( + input_api, output_api, pylintrc='pylintrc', version='2.6') + tests = input_api.canned_checks.GetUnitTestsInDirectory( + input_api, + output_api, + '.', [r'^.+_test\.py$'], + run_on_python2=False, + run_on_python3=True, + skip_shebang_check=True) + results += input_api.RunTests(tests) return results
diff --git a/build/chromeos/generate_skylab_deps.py b/build/chromeos/generate_skylab_deps.py new file mode 100755 index 0000000..a929245 --- /dev/null +++ b/build/chromeos/generate_skylab_deps.py
@@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import os +import re +import sys + +# The basic shell script for client test run in Skylab. The arguments listed +# here will be fed by autotest at the run time. +# +# * test-launcher-summary-output: the path for the json result. It will be +# assigned by autotest, who will upload it to GCS upon test completion. +# * test-launcher-shard-index: the index for this test run. +# * test-launcher-total-shards: the total test shards. +# * test_args: arbitrary runtime arguments configured in test_suites.pyl, +# attached after '--'. +BASIC_SHELL_SCRIPT = """ +#!/bin/sh + +while [[ $# -gt 0 ]]; do + case "$1" in + --test-launcher-summary-output) + summary_output=$2 + shift 2 + ;; + + --test-launcher-shard-index) + shard_index=$2 + shift 2 + ;; + + --test-launcher-total-shards) + total_shards=$2 + shift 2 + ;; + + --) + test_args=$2 + break + ;; + + *) + break + ;; + esac +done + +if [ ! -d $(dirname $summary_output) ] ; then + mkdir -p $(dirname $summary_output) +fi + +cd `dirname $0` && cd .. +""" + + +def build_test_script(args): + # Build the shell script that will be used on the device to invoke the test. + # Stored here as a list of lines. + device_test_script_contents = BASIC_SHELL_SCRIPT.split('\n') + + test_invocation = ('LD_LIBRARY_PATH=./ ./%s ' + ' --test-launcher-summary-output=$summary_output' + ' --test-launcher-shard-index=$shard_index' + ' --test-launcher-total-shards=$total_shards' + ' $test_args' % args.test_exe) + + device_test_script_contents.append(test_invocation) + with open(args.output, 'w') as w: + w.write('\n'.join(device_test_script_contents)) + os.chmod(args.output, 0o755) + + +def build_filter_file(args): + # TODO(b/227381644): This expression is hard to follow and should be + # simplified. This would require a change on the cros infra side as well + tast_expr_dict = {} + default_disabled_tests = [] + if args.disabled_tests is not None: + default_disabled_tests = [ + '!"name:{0}"'.format(test) for test in args.disabled_tests + ] + + default_enabled_test_term = '' + if args.enabled_tests is not None: + default_enabled_test_term = (' || ').join( + ['"name:{0}"'.format(test) for test in args.enabled_tests]) + + # Generate the default expression to be used when there is no known key + tast_expr = args.tast_expr if args.tast_expr else "" + + if default_disabled_tests: + default_disabled_term = " && ".join(default_disabled_tests) + tast_expr = "{0} && {1}".format(tast_expr, default_disabled_term) if \ + tast_expr else default_disabled_term + + if default_enabled_test_term: + tast_expr = "{0} && ({1})".format( + tast_expr, + default_enabled_test_term) if tast_expr else default_enabled_test_term + + tast_expr_dict['default'] = "({0})".format(tast_expr) + + # Generate an expression for each collection in the gni file + if args.tast_control is not None: + with open(args.tast_control, 'r') as tast_control_file: + gni = tast_control_file.read() + filter_lists = re.findall(r'(.*) = \[([^\]]*)\]', gni) + for filter_list in filter_lists: + tast_expr = args.tast_expr if args.tast_expr else "" + + milestone_disabled_tests = { + '!"name:{0}"'.format(test) + for test in re.findall(r'"([^"]+)"', filter_list[1]) + } + + milestone_disabled_tests.update(default_disabled_tests) + + if milestone_disabled_tests: + tast_expr = "{0} && {1}".format( + tast_expr, " && ".join(milestone_disabled_tests) + ) if tast_expr else " && ".join(milestone_disabled_tests) + + if default_enabled_test_term: + tast_expr = "{0} && ({1})".format( + tast_expr, default_enabled_test_term + ) if tast_expr else default_enabled_test_term + + if tast_expr: + tast_expr_dict[filter_list[0]] = "({0})".format(tast_expr) + + if len(tast_expr_dict) > 0: + with open(args.output, "w") as file: + json.dump(tast_expr_dict, file, indent=2) + os.chmod(args.output, 0o644) + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest='command') + + script_gen_parser = subparsers.add_parser('generate-runner') + script_gen_parser.add_argument( + '--test-exe', + type=str, + required=True, + help='Path to test executable to run inside the device.') + script_gen_parser.add_argument('--verbose', '-v', action='store_true') + script_gen_parser.add_argument( + '--output', + required=True, + type=str, + help='Path to create the runner script.') + script_gen_parser.set_defaults(func=build_test_script) + + filter_gen_parser = subparsers.add_parser('generate-filter') + filter_gen_parser.add_argument( + '--tast-expr', + type=str, + required=False, + help='Tast expression to determine tests to run. This creates the ' + 'initial set of tests that can be further filtered.') + filter_gen_parser.add_argument( + '--enabled-tests', + type=str, + required=False, + action='append', + help='Name of tests to allow to test (unnamed tests will not run).') + filter_gen_parser.add_argument( + '--disabled-tests', + type=str, + required=False, + action='append', + help='Names of tests to disable from running') + filter_gen_parser.add_argument( + '--tast-control', + type=str, + required=False, + help='Filename for the tast_control file containing version skew ' + 'test filters to generate.') + filter_gen_parser.add_argument( + '--output', + required=True, + type=str, + help='Path to create the plain text filter file.') + filter_gen_parser.set_defaults(func=build_filter_file) + + args = parser.parse_args() + + if (args.command == "generate-filter" and args.disabled_tests is None and + args.enabled_tests is None and args.tast_expr is None): + parser.error( + '--disabled-tests, --enabled-tests, or --tast-expr must be provided ' + 'to generate-filter') + + args.func(args) + + return 0 + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/build/chromeos/generate_skylab_deps_test.py b/build/chromeos/generate_skylab_deps_test.py new file mode 100755 index 0000000..9a30825 --- /dev/null +++ b/build/chromeos/generate_skylab_deps_test.py
@@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import unittest +from unittest import mock + +import generate_skylab_deps + +TAST_CONTROL = ''' +# Ignore comments +tast_disabled_tests_from_chrome_all = [ + "example.all.test1", +] +tast_disabled_tests_from_chrome_m100 = [ + "example.m100.test1", +] +tast_disabled_tests_from_lacros_all = [] +''' + +TAST_EXPR = '"group:mainline" && "dep:chrome" && !informational' + +REQUIRED_ARGS = ['script', 'generate-filter', '--output', 'output.filter'] + + +class GenerateSkylabDepsTest(unittest.TestCase): + + def testTastExpr(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + ['--tast-expr', TAST_EXPR] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + self.assertEqual(filter_dict['default'], '(%s)' % TAST_EXPR) + + def testTastExprAndDisableTests(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + [ + '--tast-expr', TAST_EXPR, '--disabled-tests', 'disabled.test1', + '--disabled-tests', 'disabled.test2' + ] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + self.assertEqual( + filter_dict['default'], + '(%s && !"name:disabled.test1" && !"name:disabled.test2")' % + TAST_EXPR) + + def testEnableTests(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + [ + '--enabled-tests', 'enabled.test1', '--enabled-tests', 'enabled.test2' + ] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + self.assertEqual(filter_dict['default'], + '("name:enabled.test1" || "name:enabled.test2")') + + def testTastControlWithTastExpr(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + [ + '--tast-expr', + TAST_EXPR, + '--tast-control', + 'mocked_input', + ] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + self.assertEqual(filter_dict['default'], '(%s)' % TAST_EXPR) + self.assertEqual(filter_dict['tast_disabled_tests_from_chrome_m100'], + '(%s && !"name:example.m100.test1")' % TAST_EXPR) + + def testTastControlWithTastExprAndDisabledTests(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + [ + '--tast-expr', TAST_EXPR, '--tast-control', 'mocked_input', + '--disabled-tests', 'disabled.test1', '--disabled-tests', + 'disabled.test2' + ] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + self.assertEqual( + filter_dict['default'], + '("group:mainline" && "dep:chrome" && !informational && !'\ + '"name:disabled.test1" && !"name:disabled.test2")' + ) + + # The list from a set is indeterminent + self.assertIn('"group:mainline" && "dep:chrome" && !informational', + filter_dict['tast_disabled_tests_from_chrome_m100']) + self.assertIn('&& !"name:disabled.test1"', + filter_dict['tast_disabled_tests_from_chrome_m100']) + self.assertIn('&& !"name:disabled.test2"', + filter_dict['tast_disabled_tests_from_chrome_m100']) + self.assertIn('&& !"name:example.m100.test1"', + filter_dict['tast_disabled_tests_from_chrome_m100']) + + def testTastControlWithTastExprAndEnabledTests(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + [ + '--tast-expr', TAST_EXPR, '--tast-control', 'mocked_input', + '--enabled-tests', 'enabled.test1', '--enabled-tests', 'enabled.test2' + ] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + self.assertEqual( + filter_dict['default'], + '("group:mainline" && "dep:chrome" && !informational && '\ + '("name:enabled.test1" || "name:enabled.test2"))' + ) + self.assertEqual( + filter_dict['tast_disabled_tests_from_chrome_m100'], + '("group:mainline" && "dep:chrome" && !informational && '\ + '!"name:example.m100.test1" && ("name:enabled.test1" '\ + '|| "name:enabled.test2"))' + ) + + def testTastControlWithEnabledTests(self): + file_mock = mock.mock_open(read_data=TAST_CONTROL) + args = REQUIRED_ARGS + [ + '--tast-control', + 'mocked_input', + '--enabled-tests', + 'enabled.test1', + '--enabled-tests', + 'enabled.test2', + ] + + with mock.patch('sys.argv', args),\ + mock.patch('builtins.open', file_mock),\ + mock.patch('os.chmod'),\ + mock.patch("json.dump", mock.MagicMock()) as dump: + generate_skylab_deps.main() + filter_dict = dump.call_args[0][0] + # Should not include 'all' collection from TAST_CONTROL since that would + # need to be passed in the --disabled-tests to be included + self.assertEqual(filter_dict['default'], + '("name:enabled.test1" || "name:enabled.test2")') + self.assertEqual( + filter_dict['tast_disabled_tests_from_chrome_m100'], + '(!"name:example.m100.test1" && '\ + '("name:enabled.test1" || "name:enabled.test2"))' + ) + + +if __name__ == '__main__': + unittest.main()
diff --git a/build/chromeos/test_runner.py b/build/chromeos/test_runner.py index c669e35..14c31e1 100755 --- a/build/chromeos/test_runner.py +++ b/build/chromeos/test_runner.py
@@ -1,6 +1,6 @@ #!/usr/bin/env vpython3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,20 +9,19 @@ import json import logging import os -import pipes import re import shutil import signal import socket import sys import tempfile +import six # The following non-std imports are fetched via vpython. See the list at # //.vpython import dateutil.parser # pylint: disable=import-error import jsonlines # pylint: disable=import-error import psutil # pylint: disable=import-error -import six CHROMIUM_SRC_PATH = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) @@ -31,13 +30,15 @@ # output json ourselves. sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android')) from pylib.base import base_test_result # pylint: disable=import-error -from pylib.base import result_sink # pylint: disable=import-error from pylib.results import json_results # pylint: disable=import-error -if six.PY2: - import subprocess32 as subprocess # pylint: disable=import-error -else: - import subprocess # pylint: disable=import-error,wrong-import-order +sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util')) +# TODO(crbug.com/1421441): Re-enable the 'no-name-in-module' check. +from lib.results import result_sink # pylint: disable=import-error,no-name-in-module + +assert not six.PY2, 'Py2 not supported for this file.' + +import subprocess # pylint: disable=import-error,wrong-import-order DEFAULT_CROS_CACHE = os.path.abspath( os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache')) @@ -55,6 +56,7 @@ LAB_DUT_HOSTNAME = 'variable_chromeos_device_hostname' SYSTEM_LOG_LOCATIONS = [ + '/home/chronos/crash/', '/var/log/chrome/', '/var/log/messages', '/var/log/ui/', @@ -67,7 +69,7 @@ pass -class RemoteTest(object): +class RemoteTest: # This is a basic shell script that can be appended to in order to invoke the # test on the device. @@ -130,20 +132,6 @@ if args.public_image: self._test_cmd += ['--public-image'] - # This environment variable is set for tests that have been instrumented - # for code coverage. Its incoming value is expected to be a location - # inside a subdirectory of result_dir above. This is converted to an - # absolute path that the vm is able to write to, and passed in the - # --results-src flag to cros_run_vm_test for copying out of the vm before - # its termination. - self._llvm_profile_var = None - if os.environ.get('LLVM_PROFILE_FILE'): - _, llvm_profile_file = os.path.split(os.environ['LLVM_PROFILE_FILE']) - self._llvm_profile_var = '/tmp/profraw/%s' % llvm_profile_file - - # This should make the vm test runner exfil the profiling data. - self._test_cmd += ['--results-src', '/tmp/profraw'] - self._test_env = setup_env() @property @@ -163,7 +151,7 @@ os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH), ] logging.info('Running the following command on the device:') - logging.info('\n' + '\n'.join(script_contents)) + logging.info('\n%s', '\n'.join(script_contents)) fd, tmp_path = tempfile.mkstemp(suffix='.sh', dir=self._path_to_outdir) os.fchmod(fd, 0o755) with os.fdopen(fd, 'w') as f: @@ -174,7 +162,7 @@ # Traps SIGTERM and kills all child processes of cros_run_test when it's # caught. This will allow us to capture logs from the device if a test hangs # and gets timeout-killed by swarming. See also: - # https://chromium.googlesource.com/infra/luci/luci-py/+/master/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance + # https://chromium.googlesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance test_proc = None def _kill_child_procs(trapped_signal, _): @@ -216,28 +204,14 @@ if test_proc.returncode == 0: break - ret = self.post_run(test_proc.returncode) + self.post_run(test_proc.returncode) # Allow post_run to override test proc return code. (Useful when the host # side Tast bin returns 0 even for failed tests.) - if ret is not None: - return ret return test_proc.returncode - def post_run(self, return_code): + def post_run(self, _): if self._on_device_script: os.remove(self._on_device_script) - # Create a simple json results file for a test run. The results will contain - # only one test (suite_name), and will either be a PASS or FAIL depending on - # return_code. - if self._test_launcher_summary_output: - result = ( - base_test_result.ResultType.FAIL - if return_code else base_test_result.ResultType.PASS) - suite_result = base_test_result.BaseTestResult(self.suite_name, result) - run_results = base_test_result.TestRunResults() - run_results.AddResult(suite_result) - with open(self._test_launcher_summary_output, 'w') as f: - json.dump(json_results.GenerateResultsDict([run_results]), f) @staticmethod def get_artifacts(path): @@ -252,7 +226,15 @@ for dirpath, _, filenames in os.walk(path): for f in filenames: artifact_path = os.path.join(dirpath, f) - artifacts[os.path.relpath(artifact_path, path)] = { + artifact_id = os.path.relpath(artifact_path, path) + # Some artifacts will have non-Latin characters in the filename, eg: + # 'ui_tree_Chinese Pinyin-ä½ å¥½.txt'. ResultDB's API rejects such + # characters as an artifact ID, so force the file name down into ascii. + # For more info, see: + # https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/proto/v1/artifact.proto;drc=3bff13b8037ca76ec19f9810033d914af7ec67cb;l=46 + artifact_id = artifact_id.encode('ascii', 'replace').decode() + artifact_id = artifact_id.replace('\\', '?') + artifacts[artifact_id] = { 'filePath': artifact_path, } return artifacts @@ -261,10 +243,11 @@ class TastTest(RemoteTest): def __init__(self, args, unknown_args): - super(TastTest, self).__init__(args, unknown_args) + super().__init__(args, unknown_args) self._suite_name = args.suite_name self._tast_vars = args.tast_vars + self._tast_retries = args.tast_retries self._tests = args.tests # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it # here and parse it later to integrate it into Tast executions. @@ -272,14 +255,9 @@ self._attr_expr = args.attr_expr self._should_strip = args.strip_chrome self._deploy_lacros = args.deploy_lacros + self._deploy_chrome = args.deploy_chrome - if self._deploy_lacros and self._should_strip: - raise TestFormatError( - '--strip-chrome is only applicable to ash-chrome because ' - 'lacros-chrome deployment uses --nostrip by default, so it cannot ' - 'be specificed with --deploy-lacros.') - - if not self._llvm_profile_var and not self._logs_dir: + if not self._logs_dir: # The host-side Tast bin returns 0 when tests fail, so we need to capture # and parse its json results to reliably determine if tests fail. raise TestFormatError( @@ -312,101 +290,72 @@ ] # Lacros deployment mounts itself by default. - self._test_cmd.extend([ - '--deploy-lacros', '--lacros-launcher-script', - LACROS_LAUNCHER_SCRIPT_PATH - ] if self._deploy_lacros else ['--deploy', '--mount']) + if self._deploy_lacros: + self._test_cmd.extend([ + '--deploy-lacros', '--lacros-launcher-script', + LACROS_LAUNCHER_SCRIPT_PATH + ]) + if self._deploy_chrome: + self._test_cmd.extend(['--deploy', '--mount']) + else: + self._test_cmd.extend(['--deploy', '--mount']) self._test_cmd += [ '--build-dir', os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH) ] + self._additional_args - # Coverage tests require some special pre-test setup, so use an - # on_device_script in that case. For all other tests, use cros_run_test's - # built-in '--tast' option. This gives us much better results reporting. - if self._llvm_profile_var: - # Build the shell script that will be used on the device to invoke the - # test. - device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] - device_test_script_contents += [ - 'echo "LLVM_PROFILE_FILE=%s" >> /etc/chrome_dev.conf' % - (self._llvm_profile_var) - ] - - local_test_runner_cmd = ['local_test_runner', '-waituntilready'] - if self._use_vm: - # If we're running tests in VMs, tell the test runner to skip tests that - # aren't compatible. - local_test_runner_cmd.append('-extrauseflags=tast_vm') - if self._attr_expr: - local_test_runner_cmd.append(pipes.quote(self._attr_expr)) - else: - local_test_runner_cmd.extend(self._tests) - device_test_script_contents.append(' '.join(local_test_runner_cmd)) - - self._on_device_script = self.write_test_script_to_disk( - device_test_script_contents) - + # Capture tast's results in the logs dir as well. + if self._logs_dir: self._test_cmd += [ - '--files', - os.path.relpath(self._on_device_script), '--', - './' + os.path.relpath(self._on_device_script, self._path_to_outdir) + '--results-dir', + self._logs_dir, ] + self._test_cmd += [ + '--tast-total-shards=%d' % self._test_launcher_total_shards, + '--tast-shard-index=%d' % self._test_launcher_shard_index, + ] + # If we're using a test filter, replace the contents of the Tast + # conditional with a long list of "name:test" expressions, one for each + # test in the filter. + if self._gtest_style_filter: + if self._attr_expr or self._tests: + logging.warning( + 'Presence of --gtest_filter will cause the specified Tast expr' + ' or test list to be ignored.') + names = [] + for test in self._gtest_style_filter.split(':'): + names.append('"name:%s"' % test) + self._attr_expr = '(' + ' || '.join(names) + ')' + + if self._attr_expr: + # Don't use pipes.quote() here. Something funky happens with the arg + # as it gets passed down from cros_run_test to tast. (Tast picks up the + # escaping single quotes and complains that the attribute expression + # "must be within parentheses".) + self._test_cmd.append('--tast=%s' % self._attr_expr) else: - # Capture tast's results in the logs dir as well. - if self._logs_dir: - self._test_cmd += [ - '--results-dir', - self._logs_dir, - ] - self._test_cmd += [ - '--tast-total-shards=%d' % self._test_launcher_total_shards, - '--tast-shard-index=%d' % self._test_launcher_shard_index, - ] - # If we're using a test filter, replace the contents of the Tast - # conditional with a long list of "name:test" expressions, one for each - # test in the filter. - if self._gtest_style_filter: - if self._attr_expr or self._tests: - logging.warning( - 'Presence of --gtest_filter will cause the specified Tast expr' - ' or test list to be ignored.') - names = [] - for test in self._gtest_style_filter.split(':'): - names.append('"name:%s"' % test) - self._attr_expr = '(' + ' || '.join(names) + ')' + self._test_cmd.append('--tast') + self._test_cmd.extend(self._tests) - if self._attr_expr: - # Don't use pipes.quote() here. Something funky happens with the arg - # as it gets passed down from cros_run_test to tast. (Tast picks up the - # escaping single quotes and complains that the attribute expression - # "must be within parentheses".) - self._test_cmd.append('--tast=%s' % self._attr_expr) - else: - self._test_cmd.append('--tast') - self._test_cmd.extend(self._tests) + for v in self._tast_vars or []: + self._test_cmd.extend(['--tast-var', v]) - for v in self._tast_vars or []: - self._test_cmd.extend(['--tast-var', v]) + if self._tast_retries: + self._test_cmd.append('--tast-retries=%d' % self._tast_retries) - # Mounting ash-chrome gives it enough disk space to not need stripping, - # but only for one not instrumented with code coverage. - # Lacros uses --nostrip by default, so there is no need to specify. - if not self._deploy_lacros and not self._should_strip: - self._test_cmd.append('--nostrip') + # Mounting ash-chrome gives it enough disk space to not need stripping, + # but only for one not instrumented with code coverage. + # Lacros uses --nostrip by default, so there is no need to specify. + if not self._deploy_lacros and not self._should_strip: + self._test_cmd.append('--nostrip') def post_run(self, return_code): - # If we don't need to parse the host-side Tast tool's results, fall back to - # the parent method's default behavior. - if self._llvm_profile_var: - return super(TastTest, self).post_run(return_code) - tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') if not os.path.exists(tast_results_path): logging.error( 'Tast results not found at %s. Falling back to generic result ' 'reporting.', tast_results_path) - return super(TastTest, self).post_run(return_code) + return super().post_run(return_code) # See the link below for the format of the results: # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult @@ -420,22 +369,26 @@ # Use dateutil to parse the timestamps since datetime can't handle # nanosecond precision. duration = dateutil.parser.parse(end) - dateutil.parser.parse(start) - duration_ms = duration.total_seconds() * 1000 + # If the duration is negative, Tast has likely reported an incorrect + # duration. See https://issuetracker.google.com/issues/187973541. Round + # up to 0 in that case to avoid confusing RDB. + duration_ms = max(duration.total_seconds() * 1000, 0) if bool(test['skipReason']): result = base_test_result.ResultType.SKIP elif errors: result = base_test_result.ResultType.FAIL else: result = base_test_result.ResultType.PASS + primary_error_message = None error_log = '' if errors: # See the link below for the format of these errors: - # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/tast/testing#Error + # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go + primary_error_message = errors[0]['reason'] for err in errors: error_log += err['stack'] + '\n' - error_log += ( - "\nIf you're unsure why this test failed, consult the steps " - 'outlined in\n%s\n' % TAST_DEBUG_DOC) + debug_link = ("If you're unsure why this test failed, consult the steps " + 'outlined <a href="%s">here</a>.' % TAST_DEBUG_DOC) base_result = base_test_result.BaseTestResult( test['name'], result, duration=duration_ms, log=error_log) suite_results.AddResult(base_result) @@ -446,8 +399,15 @@ # inside as an RDB 'artifact'. (This could include system logs, screen # shots, etc.) artifacts = self.get_artifacts(test['outDir']) - self._rdb_client.Post(test['name'], result, duration_ms, error_log, - artifacts) + self._rdb_client.Post( + test['name'], + result, + duration_ms, + error_log, + None, + artifacts=artifacts, + failure_reason=primary_error_message, + html_artifact=debug_link) if self._rdb_client and self._logs_dir: # Attach artifacts from the device that don't apply to a single test. @@ -463,7 +423,7 @@ if not suite_results.DidRunPass(): return 1 - elif return_code: + if return_code: logging.warning( 'No failed tests found, but exit code of %d was returned from ' 'cros_run_test.', return_code) @@ -519,7 +479,7 @@ ] def __init__(self, args, unknown_args): - super(GTestTest, self).__init__(args, unknown_args) + super().__init__(args, unknown_args) self._test_exe = args.test_exe self._runtime_deps_path = args.runtime_deps_path @@ -579,31 +539,26 @@ # Build the shell script that will be used on the device to invoke the test. # Stored here as a list of lines. device_test_script_contents = self.BASIC_SHELL_SCRIPT[:] - if self._llvm_profile_var: - device_test_script_contents += [ - 'export LLVM_PROFILE_FILE=%s' % self._llvm_profile_var, - ] - for var_name, var_val in self._env_vars: device_test_script_contents += ['export %s=%s' % (var_name, var_val)] if self._vpython_dir: vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, - 'vpython') + 'vpython3') cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir, - 'bin', 'python') + 'bin', 'python3') if not os.path.exists(vpython_path) or not os.path.exists(cpython_path): raise TestFormatError( - '--vpython-dir must point to a dir with both infra/python/cpython ' - 'and infra/tools/luci/vpython installed.') + '--vpython-dir must point to a dir with both ' + 'infra/3pp/tools/cpython3 and infra/tools/luci/vpython installed.') vpython_spec_path = os.path.relpath( - os.path.join(CHROMIUM_SRC_PATH, '.vpython'), self._path_to_outdir) + os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir) # Initialize the vpython cache. This can take 10-20s, and some tests # can't afford to wait that long on the first invocation. device_test_script_contents.extend([ 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' % (self._vpython_dir, self._vpython_dir), - 'vpython -vpython-spec %s -vpython-tool install' % + 'vpython3 -vpython-spec %s -vpython-tool install' % (vpython_spec_path), ]) @@ -618,7 +573,7 @@ if self._trace_dir: device_test_script_contents.extend([ 'rm -rf %s' % device_trace_dir, - 'su chronos -c -- "mkdir -p %s"' % device_trace_dir, + 'sudo -E -u chronos -- /bin/bash -c "mkdir -p %s"' % device_trace_dir, ]) test_invocation += ' --trace-dir=%s' % device_trace_dir @@ -632,7 +587,8 @@ # The UI service on the device owns the chronos user session, so shutting # it down as chronos kills the entire execution of the test. So we'll have # to run as root up until the test invocation. - test_invocation = 'su chronos -c -- "%s"' % test_invocation + test_invocation = ( + 'sudo -E -u chronos -- /bin/bash -c "%s"' % test_invocation) # And we'll need to chown everything since cros_run_test's "--as-chronos" # option normally does that for us. device_test_script_contents.append('chown -R chronos: ../..') @@ -658,9 +614,6 @@ os.path.abspath( os.path.join(self._path_to_outdir, self._vpython_dir)), CHROMIUM_SRC_PATH)) - # TODO(bpastene): Add the vpython spec to the test's runtime deps instead - # of handling it here. - runtime_files.append('.vpython') for f in runtime_files: self._test_cmd.extend(['--files', f]) @@ -690,13 +643,17 @@ if self._on_device_script: os.remove(self._on_device_script) + if self._test_launcher_summary_output and self._rdb_client: + logging.error('Native ResultDB integration is not supported for GTests. ' + 'Upload results via result_adapter instead. ' + 'See crbug.com/1330441.') + def device_test(args, unknown_args): # cros_run_test has trouble with relative paths that go up directories, # so cd to src/, which should be the root of all data deps. os.chdir(CHROMIUM_SRC_PATH) - # pylint: disable=redefined-variable-type # TODO: Remove the above when depot_tool's pylint is updated to include the # fix to https://github.com/PyCQA/pylint/issues/710. if args.test_type == 'tast': @@ -714,7 +671,7 @@ def host_cmd(args, cmd_args): if not cmd_args: raise TestFormatError('Must specify command to run on the host.') - elif args.deploy_chrome and not args.path_to_outdir: + if args.deploy_chrome and not args.path_to_outdir: raise TestFormatError( '--path-to-outdir must be specified if --deploy-chrome is passed.') @@ -752,11 +709,22 @@ test_env = setup_env() if args.deploy_chrome or args.deploy_lacros: - # Mounting ash-chrome gives it enough disk space to not need stripping. - cros_run_test_cmd.extend([ - '--deploy-lacros', '--lacros-launcher-script', - LACROS_LAUNCHER_SCRIPT_PATH - ] if args.deploy_lacros else ['--deploy', '--mount', '--nostrip']) + if args.deploy_lacros: + cros_run_test_cmd.extend([ + '--deploy-lacros', '--lacros-launcher-script', + LACROS_LAUNCHER_SCRIPT_PATH + ]) + if args.deploy_chrome: + # Mounting ash-chrome gives it enough disk space to not need stripping + # most of the time. + cros_run_test_cmd.extend(['--deploy', '--mount']) + else: + # Mounting ash-chrome gives it enough disk space to not need stripping + # most of the time. + cros_run_test_cmd.extend(['--deploy', '--mount']) + + if not args.strip_chrome: + cros_run_test_cmd.append('--nostrip') cros_run_test_cmd += [ '--build-dir', @@ -799,6 +767,13 @@ parser.add_argument( '--board', type=str, required=True, help='Type of CrOS device.') parser.add_argument( + '--deploy-chrome', + action='store_true', + help='Will deploy a locally built ash-chrome binary to the device ' + 'before running the host-cmd.') + parser.add_argument( + '--deploy-lacros', action='store_true', help='Deploy a lacros-chrome.') + parser.add_argument( '--cros-cache', type=str, default=DEFAULT_CROS_CACHE, @@ -847,6 +822,10 @@ '--public-image', action='store_true', help='Will flash a public "full" image to the device.') + parser.add_argument( + '--magic-vm-cache', + help='Path to the magic CrOS VM cache dir. See the comment above ' + '"magic_cros_vm_cache" in mixins.pyl for more info.') vm_or_device_group = parser.add_mutually_exclusive_group() vm_or_device_group.add_argument( @@ -871,14 +850,10 @@ 'will be 127.0.0.1:9222.') host_cmd_parser.set_defaults(func=host_cmd) host_cmd_parser.add_argument( - '--deploy-chrome', + '--strip-chrome', action='store_true', - help='Will deploy a locally built ash-chrome binary to the device before ' - 'running the host-cmd.') - host_cmd_parser.add_argument( - '--deploy-lacros', - action='store_true', - help='Deploy a lacros-chrome instead of ash-chrome.') + help='Strips symbols from ash-chrome or lacros-chrome before deploying ' + ' to the device.') gtest_parser = subparsers.add_parser( 'gtest', help='Runs a device-side gtest.') @@ -919,7 +894,7 @@ tast_test_parser = subparsers.add_parser( 'tast', help='Runs a device-side set of Tast tests. For more details, see: ' - 'https://chromium.googlesource.com/chromiumos/platform/tast/+/master/docs/running_tests.md' + 'https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/running_tests.md' ) tast_test_parser.set_defaults(func=device_test) tast_test_parser.add_argument( @@ -943,21 +918,22 @@ action='store_true', help='Strips symbols from ash-chrome before deploying to the device.') tast_test_parser.add_argument( - '--deploy-lacros', - action='store_true', - help='Deploy a lacros-chrome instead of ash-chrome.') - tast_test_parser.add_argument( '--tast-var', action='append', dest='tast_vars', help='Runtime variables for Tast tests, and the format are expected to ' 'be "key=value" pairs.') tast_test_parser.add_argument( + '--tast-retries', + type=int, + dest='tast_retries', + help='Number of retries for failed Tast tests on the same DUT.') + tast_test_parser.add_argument( '--test', '-t', action='append', dest='tests', - help='A Tast test to run in the device (eg: "ui.ChromeLogin").') + help='A Tast test to run in the device (eg: "login.Chrome").') tast_test_parser.add_argument( '--gtest_filter', type=str, @@ -966,20 +942,14 @@ 'cmd-line API, this will overwrite the value(s) of "--test" above.') add_common_args(gtest_parser, tast_test_parser, host_cmd_parser) - - args = sys.argv[1:] - unknown_args = [] - # If a '--' is present in the args, treat everything to the right of it as - # args to the test and everything to the left as args to this test runner. - # Otherwise treat all known args as args to this test runner and all unknown - # args as test args. - if '--' in args: - unknown_args = args[args.index('--') + 1:] - args = args[0:args.index('--')] - if unknown_args: - args = parser.parse_args(args=args) - else: - args, unknown_args = parser.parse_known_args() + args, unknown_args = parser.parse_known_args() + # Re-add N-1 -v/--verbose flags to the args we'll pass to whatever we are + # running. The assumption is that only one verbosity incrase would be meant + # for this script since it's a boolean value instead of increasing verbosity + # with more instances. + verbose_flags = [a for a in sys.argv if a in ('-v', '--verbose')] + if verbose_flags: + unknown_args += verbose_flags[1:] logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN) @@ -1001,6 +971,18 @@ LAB_DUT_HOSTNAME) return 1 + if args.flash and args.public_image: + # The flashing tools depend on being unauthenticated with GS when flashing + # public images, so make sure the env var GS uses to locate its creds is + # unset in that case. + os.environ.pop('BOTO_CONFIG', None) + + if args.magic_vm_cache: + full_vm_cache_path = os.path.join(CHROMIUM_SRC_PATH, args.magic_vm_cache) + if os.path.exists(full_vm_cache_path): + with open(os.path.join(full_vm_cache_path, 'swarming.txt'), 'w') as f: + f.write('non-empty file to make swarming persist this cache') + return args.func(args, unknown_args)
diff --git a/build/chromeos/test_runner_test.py b/build/chromeos/test_runner_test.py index 15d1b1f..c61c7a4 100755 --- a/build/chromeos/test_runner_test.py +++ b/build/chromeos/test_runner_test.py
@@ -1,5 +1,5 @@ #!/usr/bin/env vpython3 -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -9,17 +9,17 @@ import sys import tempfile import unittest +import six # The following non-std imports are fetched via vpython. See the list at # //.vpython import mock # pylint: disable=import-error from parameterized import parameterized # pylint: disable=import-error -import six import test_runner _TAST_TEST_RESULTS_JSON = { - "name": "ui.ChromeLogin", + "name": "login.Chrome", "errors": None, "start": "2020-01-01T15:41:30.799228462-08:00", "end": "2020-01-01T15:41:53.318914698-08:00", @@ -47,7 +47,7 @@ if six.PY3: self.assertSetEqual(set(list1), set(list2)) else: - self.assertItemsEqual(list1, list2) + self.assertCountEqual(list1, list2) class TastTests(TestRunnerTest): @@ -101,7 +101,7 @@ args = self.get_common_tast_args(False) + [ '--attr-expr=( "group:mainline" && "dep:chrome" && !informational)', - '--gtest_filter=ui.ChromeLogin:ui.WindowControl', + '--gtest_filter=login.Chrome:ui.WindowControl', ] with mock.patch.object(sys, 'argv', args),\ mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: @@ -111,7 +111,7 @@ # The gtest filter should cause the Tast expr to be replaced with a list # of the tests in the filter. expected_cmd = self.get_common_tast_expectations(False) + [ - '--tast=("name:ui.ChromeLogin" || "name:ui.WindowControl")' + '--tast=("name:login.Chrome" || "name:ui.WindowControl")' ] self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) @@ -179,7 +179,7 @@ json.dump(_TAST_TEST_RESULTS_JSON, f) args = self.get_common_tast_args(use_vm) + [ - '-t=ui.ChromeLogin', + '-t=login.Chrome', '--tast-var=key=value', ] with mock.patch.object(sys, 'argv', args),\ @@ -187,7 +187,30 @@ mock_popen.return_value.returncode = 0 test_runner.main() expected_cmd = self.get_common_tast_expectations(use_vm) + [ - '--tast', 'ui.ChromeLogin', '--tast-var', 'key=value' + '--tast', 'login.Chrome', '--tast-var', 'key=value' + ] + + self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) + + @parameterized.expand([ + [True], + [False], + ]) + def test_tast_retries(self, use_vm): + """Tests running a tast tests with retries.""" + with open(os.path.join(self._tmp_dir, 'streamed_results.jsonl'), 'w') as f: + json.dump(_TAST_TEST_RESULTS_JSON, f) + + args = self.get_common_tast_args(use_vm) + [ + '-t=login.Chrome', + '--tast-retries=1', + ] + with mock.patch.object(sys, 'argv', args),\ + mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: + mock_popen.return_value.returncode = 0 + test_runner.main() + expected_cmd = self.get_common_tast_expectations(use_vm) + [ + '--tast', 'login.Chrome', '--tast-retries=1' ] self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) @@ -202,7 +225,7 @@ json.dump(_TAST_TEST_RESULTS_JSON, f) args = self.get_common_tast_args(use_vm) + [ - '-t=ui.ChromeLogin', + '-t=login.Chrome', ] with mock.patch.object(sys, 'argv', args),\ mock.patch.object(test_runner.subprocess, 'Popen') as mock_popen: @@ -210,7 +233,7 @@ test_runner.main() expected_cmd = self.get_common_tast_expectations(use_vm) + [ - '--tast', 'ui.ChromeLogin' + '--tast', 'login.Chrome' ] self.safeAssertItemsEqual(expected_cmd, mock_popen.call_args[0][0]) @@ -281,10 +304,10 @@ gtest.build_test_command() # Create the two expected tools, and the test should be ready to run. - with open(os.path.join(args.vpython_dir, 'vpython'), 'w'): + with open(os.path.join(args.vpython_dir, 'vpython3'), 'w'): pass # Just touch the file. os.mkdir(os.path.join(args.vpython_dir, 'bin')) - with open(os.path.join(args.vpython_dir, 'bin', 'python'), 'w'): + with open(os.path.join(args.vpython_dir, 'bin', 'python3'), 'w'): pass gtest = test_runner.GTestTest(args, None) gtest.build_test_command() @@ -293,10 +316,12 @@ class HostCmdTests(TestRunnerTest): @parameterized.expand([ - [True], - [False], + [True, False, True], + [False, True, True], + [True, True, False], + [False, True, False], ]) - def test_host_cmd(self, is_lacros): + def test_host_cmd(self, is_lacros, is_ash, strip_chrome): args = [ 'script_name', 'host-cmd', @@ -307,8 +332,10 @@ ] if is_lacros: args += ['--deploy-lacros'] - else: + if is_ash: args += ['--deploy-chrome'] + if strip_chrome: + args += ['--strip-chrome'] args += [ '--', 'fake_cmd', @@ -337,8 +364,10 @@ '--lacros-launcher-script', test_runner.LACROS_LAUNCHER_SCRIPT_PATH, ] - else: - expected_cmd += ['--mount', '--nostrip', '--deploy'] + if is_ash: + expected_cmd += ['--mount', '--deploy'] + if not strip_chrome: + expected_cmd += ['--nostrip'] expected_cmd += [ '--',
diff --git a/build/cipd/cipd.gni b/build/cipd/cipd.gni index e7795c1..852adef 100644 --- a/build/cipd/cipd.gni +++ b/build/cipd/cipd.gni
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -123,7 +123,7 @@ } action(target_name) { script = "//build/cipd/cipd_from_file.py" - inputs = [ "//build/cipd/cipd_from_file.py" ] + inputs = [ invoker.files_file ] args = [ "--description=" + invoker.description, "--buildtype=" + invoker.buildtype,
diff --git a/build/cipd/cipd_from_file.py b/build/cipd/cipd_from_file.py index 0f08f69..979b2b5 100755 --- a/build/cipd/cipd_from_file.py +++ b/build/cipd/cipd_from_file.py
@@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2021 The Chromium Authors. All rights reserved. +# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Script to generate yaml file based on FILES.cfg."""
diff --git a/build/cipd/clobber_cipd_root.py b/build/cipd/clobber_cipd_root.py deleted file mode 100755 index 5d36c72..0000000 --- a/build/cipd/clobber_cipd_root.py +++ /dev/null
@@ -1,33 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Clobbers a CIPD root.""" - -import argparse -import os -import shutil -import sys - - -def main(): - parser = argparse.ArgumentParser( - description='Clobbers the CIPD root in the given directory.') - - parser.add_argument( - '--root', - required=True, - help='Root directory for dependency.') - args = parser.parse_args() - - cipd_root_dir = os.path.join(args.root, '.cipd') - if os.path.exists(cipd_root_dir): - shutil.rmtree(cipd_root_dir) - - return 0 - - -if __name__ == '__main__': - sys.exit(main())
diff --git a/build/clobber.py b/build/clobber.py index 1de3212..e886737 100755 --- a/build/clobber.py +++ b/build/clobber.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2015 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -22,36 +22,43 @@ On error, returns the empty string.""" result = "" with open(build_ninja_file, 'r') as f: - # Read until the third blank line. The first thing GN writes to the file - # is "ninja_required_version = x.y.z", then the "rule gn" and the third - # is the section for "build build.ninja", separated by blank lines. - num_blank_lines = 0 - while num_blank_lines < 3: - line = f.readline() - if len(line) == 0: - return '' # Unexpected EOF. + # Reads until the first empty line after the "build build.ninja:" target. + # We assume everything before it necessary as well (eg the + # "ninja_required_version" line). + found_build_dot_ninja_target = False + for line in f.readlines(): result += line - if line[0] == '\n': - num_blank_lines = num_blank_lines + 1 - return result + if line.startswith('build build.ninja:'): + found_build_dot_ninja_target = True + if found_build_dot_ninja_target and line[0] == '\n': + return result + return '' # We got to EOF and didn't find what we were looking for. -def delete_dir(build_dir): - if os.path.islink(build_dir): - return +def _rmtree(d): # For unknown reasons (anti-virus?) rmtree of Chromium build directories # often fails on Windows. if sys.platform.startswith('win'): - subprocess.check_call(['rmdir', '/s', '/q', build_dir], shell=True) + subprocess.check_call(['rmdir', '/s', '/q', d], shell=True) else: - shutil.rmtree(build_dir) + shutil.rmtree(d) + + +def _clean_dir(build_dir): + # Remove files/sub directories individually instead of recreating the build + # dir because it fails when the build dir is symlinked or mounted. + for e in os.scandir(build_dir): + if e.is_dir(): + _rmtree(e.path) + else: + os.remove(e.path) def delete_build_dir(build_dir): # GN writes a build.ninja.d file. Note that not all GN builds have args.gn. build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d') if not os.path.exists(build_ninja_d_file): - delete_dir(build_dir) + _clean_dir(build_dir) return # GN builds aren't automatically regenerated when you sync. To avoid @@ -68,15 +75,16 @@ except IOError: args_contents = '' - e = None + exception_during_rm = None try: - # delete_dir and os.mkdir() may fail, such as when chrome.exe is running, + # _clean_dir() may fail, such as when chrome.exe is running, # and we still want to restore args.gn/build.ninja/build.ninja.d, so catch # the exception and rethrow it later. - delete_dir(build_dir) - os.mkdir(build_dir) + # We manually rm files inside the build dir rather than using "gn clean/gen" + # since we may not have run all necessary DEPS hooks yet at this point. + _clean_dir(build_dir) except Exception as e: - pass + exception_during_rm = e # Put back the args file (if any). if args_contents != '': @@ -105,9 +113,10 @@ with open(build_ninja_d_file, 'w') as f: f.write('build.ninja: nonexistant_file.gn\n') - if e: + if exception_during_rm: # Rethrow the exception we caught earlier. - raise e + raise exception_during_rm + def clobber(out_dir): """Clobber contents of build directory.
diff --git a/build/clobber_unittest.py b/build/clobber_unittest.py new file mode 100755 index 0000000..d38c447 --- /dev/null +++ b/build/clobber_unittest.py
@@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import pathlib +import shutil +import sys +import tempfile +import textwrap +import unittest +from unittest import mock + +import clobber + + +class TestExtractBuildCommand(unittest.TestCase): + def setUp(self): + self.build_ninja_file, self.build_ninja_path = tempfile.mkstemp(text=True) + + def tearDown(self): + os.close(self.build_ninja_file) + os.remove(self.build_ninja_path) + + def test_normal_extraction(self): + build_ninja_file_contents = textwrap.dedent(""" + ninja_required_version = 1.7.2 + + rule gn + command = ../../buildtools/gn --root=../.. -q --regeneration gen . + pool = console + description = Regenerating ninja files + + build build.ninja.stamp: gn + generator = 1 + depfile = build.ninja.d + + build build.ninja: phony build.ninja.stamp + generator = 1 + + pool build_toolchain_action_pool + depth = 72 + + pool build_toolchain_link_pool + depth = 23 + + subninja toolchain.ninja + subninja clang_newlib_x64/toolchain.ninja + subninja glibc_x64/toolchain.ninja + subninja irt_x64/toolchain.ninja + subninja nacl_bootstrap_x64/toolchain.ninja + subninja newlib_pnacl/toolchain.ninja + + build blink_python_tests: phony obj/blink_python_tests.stamp + build blink_tests: phony obj/blink_tests.stamp + + default all + """) # Based off of a standard linux build dir. + with open(self.build_ninja_path, 'w') as f: + f.write(build_ninja_file_contents) + + expected_build_ninja_file_contents = textwrap.dedent(""" + ninja_required_version = 1.7.2 + + rule gn + command = ../../buildtools/gn --root=../.. -q --regeneration gen . + pool = console + description = Regenerating ninja files + + build build.ninja.stamp: gn + generator = 1 + depfile = build.ninja.d + + build build.ninja: phony build.ninja.stamp + generator = 1 + + """) + + self.assertEqual(clobber.extract_gn_build_commands(self.build_ninja_path), + expected_build_ninja_file_contents) + + def test_unexpected_format(self): + # No "build build.ninja:" line should make it return an empty string. + build_ninja_file_contents = textwrap.dedent(""" + ninja_required_version = 1.7.2 + + rule gn + command = ../../buildtools/gn --root=../.. -q --regeneration gen . + pool = console + description = Regenerating ninja files + + subninja toolchain.ninja + + build blink_python_tests: phony obj/blink_python_tests.stamp + build blink_tests: phony obj/blink_tests.stamp + + """) + with open(self.build_ninja_path, 'w') as f: + f.write(build_ninja_file_contents) + + self.assertEqual(clobber.extract_gn_build_commands(self.build_ninja_path), + '') + + +class TestDelete(unittest.TestCase): + def setUp(self): + self.build_dir = tempfile.mkdtemp() + + pathlib.Path(os.path.join(self.build_dir, 'build.ninja')).touch() + pathlib.Path(os.path.join(self.build_dir, 'build.ninja.d')).touch() + + def tearDown(self): + shutil.rmtree(self.build_dir) + + def test_delete_build_dir_full(self): + # Create a dummy file in the build dir and ensure it gets removed. + dummy_file = os.path.join(self.build_dir, 'dummy') + pathlib.Path(dummy_file).touch() + + clobber.delete_build_dir(self.build_dir) + + self.assertFalse(os.path.exists(dummy_file)) + + def test_delete_build_dir_fail(self): + # Make delete_dir() throw to ensure it's handled gracefully. + + with mock.patch('clobber._clean_dir', side_effect=OSError): + with self.assertRaises(OSError): + clobber.delete_build_dir(self.build_dir) + + @unittest.skipIf(sys.platform == 'win32', 'Symlinks are not allowed on Windows by default') + def test_delete_build_dir_link(self): + with tempfile.TemporaryDirectory() as tmpdir: + # create a symlink. + build_dir = os.path.join(tmpdir, 'link') + os.symlink(self.build_dir, build_dir) + + # create a dummy file. + dummy_file = os.path.join(build_dir, 'dummy') + pathlib.Path(dummy_file).touch() + clobber.delete_build_dir(build_dir) + + self.assertFalse(os.path.exists(dummy_file)) + + +if __name__ == '__main__': + unittest.main()
diff --git a/build/compiled_action.gni b/build/compiled_action.gni index 7e25a0b..6a632bd 100644 --- a/build/compiled_action.gni +++ b/build/compiled_action.gni
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/compute_build_timestamp.py b/build/compute_build_timestamp.py index ceb507b..befe844 100755 --- a/build/compute_build_timestamp.py +++ b/build/compute_build_timestamp.py
@@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright 2018 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Returns a timestamp that approximates the build date. @@ -27,7 +27,6 @@ # the symbol server, so rarely changing timestamps can cause conflicts there # as well. We only upload symbols for official builds to the symbol server. -from __future__ import print_function import argparse import calendar @@ -116,9 +115,21 @@ # builds are typically added to symbol servers and Windows symbol servers # use the link timestamp as the prime differentiator, but for unofficial # builds we do lots of quantization to avoid churn. - if args.build_type != 'official': + offset = 0 + if args.build_type == 'official': + if os.name == 'nt': + version_path = os.path.join(THIS_DIR, os.pardir, 'chrome', 'VERSION') + with open(version_path) as f: + patch_line = f.readlines()[3].strip() + # Use the patch number as an offset to the build date so that multiple + # versions with different patch numbers built from the same source code + # will get different build_date values. This is critical for Windows + # symbol servers, to avoid collisions. + assert patch_line.startswith('PATCH=') + offset = int(patch_line[6:]) + else: build_date = GetUnofficialBuildDate(build_date) - print(int(calendar.timegm(build_date.utctimetuple()))) + print(offset + int(calendar.timegm(build_date.utctimetuple()))) return 0
diff --git a/build/config/BUILD.gn b/build/config/BUILD.gn index a78ddc5..2106261 100644 --- a/build/config/BUILD.gn +++ b/build/config/BUILD.gn
@@ -1,11 +1,10 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/c++/c++.gni") import("//build/config/chrome_build.gni") import("//build/config/chromecast_build.gni") -import("//build/config/crypto.gni") import("//build/config/dcheck_always_on.gni") import("//build/config/features.gni") @@ -49,9 +48,6 @@ defines = [] if (dcheck_always_on) { defines += [ "DCHECK_ALWAYS_ON=1" ] - if (dcheck_is_configurable) { - defines += [ "DCHECK_IS_CONFIGURABLE" ] - } } if (use_udev) { # TODO(brettw) should probably be "=1". @@ -63,10 +59,10 @@ if (use_glib) { defines += [ "USE_GLIB=1" ] } - if (use_nss_certs) { - defines += [ "USE_NSS_CERTS=1" ] - } if (use_ozone && !is_android) { + # Chrome code should check BUILDFLAG(IS_OZONE) instead of + # defined(USE_OZONE). + # # Note that some Chrome OS builds unconditionally set |use_ozone| to true, # but they also build some targets with the Android toolchain. This ensures # that Android targets still build with USE_OZONE=0 in such cases. @@ -75,9 +71,6 @@ # setting use_ozone globally. defines += [ "USE_OZONE=1" ] } - if (use_x11) { - defines += [ "USE_X11=1" ] - } if (is_asan || is_hwasan || is_lsan || is_tsan || is_msan) { defines += [ "MEMORY_TOOL_REPLACES_ALLOCATOR" ] } @@ -96,7 +89,7 @@ if (is_msan) { defines += [ "MEMORY_SANITIZER" ] } - if (is_ubsan || is_ubsan_null || is_ubsan_vptr || is_ubsan_security) { + if (is_ubsan || is_ubsan_vptr || is_ubsan_security) { defines += [ "UNDEFINED_SANITIZER" ] } if (is_official_build) { @@ -238,6 +231,9 @@ visibility = [ ":executable_deps", ":loadable_module_deps", + ":rust_bin_deps", + ":rust_cdylib_deps", + ":rust_dylib_deps", ":shared_library_deps", ] @@ -264,6 +260,9 @@ if (is_fuchsia) { public_deps += [ "//third_party/fuchsia-sdk/sdk/build/config:runtime_library_group" ] + if (is_asan) { + public_deps += [ "//build/config/fuchsia:asan_runtime_library" ] + } } } @@ -273,16 +272,44 @@ if (export_libcxxabi_from_executables) { public_deps += [ "//buildtools/third_party/libc++abi" ] } + public_configs = [ "//build/config/sanitizers:link_executable" ] +} + +# Only the rust_bin template in BUILDCONFIG.gn should reference this. +group("rust_bin_deps") { + public_deps = [ ":common_deps" ] + if (export_libcxxabi_from_executables) { + public_deps += [ "//buildtools/third_party/libc++abi" ] + } + public_configs = [ "//build/config/sanitizers:link_executable" ] } # Only the loadable_module template in BUILDCONFIG.gn should reference this. group("loadable_module_deps") { public_deps = [ ":common_deps" ] + + public_configs = [ "//build/config/sanitizers:link_shared_library" ] } # Only the shared_library template in BUILDCONFIG.gn should reference this. group("shared_library_deps") { public_deps = [ ":common_deps" ] + + public_configs = [ "//build/config/sanitizers:link_shared_library" ] +} + +# Only the rust_dylib template in BUILDCONFIG.gn should reference this. +group("rust_dylib_deps") { + public_deps = [ ":common_deps" ] + + public_configs = [ "//build/config/sanitizers:link_shared_library" ] +} + +# Only the rust_cdylib template in BUILDCONFIG.gn should reference this. +group("rust_cdylib_deps") { + public_deps = [ ":common_deps" ] + + public_configs = [ "//build/config/sanitizers:link_shared_library" ] } # Executable configs ----------------------------------------------------------- @@ -301,6 +328,7 @@ if (is_win) { configs += _windows_linker_configs + configs += [ "//build/config/win:exe_flags" ] } else if (is_mac) { configs += [ "//build/config/mac:mac_dynamic_flags" ] } else if (is_ios) { @@ -310,10 +338,8 @@ ] } else if (is_linux || is_chromeos || is_android || current_os == "aix") { configs += [ "//build/config/gcc:executable_config" ] - if (is_chromecast) { + if (is_castos || is_cast_android) { configs += [ "//build/config/chromecast:executable_config" ] - } else if (is_fuchsia) { - configs += [ "//build/config/fuchsia:executable_config" ] } } @@ -325,7 +351,6 @@ if (use_locally_built_instrumented_libraries) { configs += [ "//third_party/instrumented_libraries:locally_built_ldflags" ] } - configs += [ "//build/config/sanitizers:link_executable" ] } # Shared library configs ------------------------------------------------------- @@ -343,7 +368,7 @@ "//build/config/ios:ios_dynamic_flags", "//build/config/ios:ios_shared_library_flags", ] - } else if (is_chromecast) { + } else if (is_castos || is_cast_android) { configs += [ "//build/config/chromecast:shared_library_config" ] } else if (is_linux || is_chromeos || current_os == "aix") { configs += [ "//build/config/gcc:shared_library_config" ] @@ -357,7 +382,6 @@ if (use_locally_built_instrumented_libraries) { configs += [ "//third_party/instrumented_libraries:locally_built_ldflags" ] } - configs += [ "//build/config/sanitizers:link_shared_library" ] } # Add this config to your target to enable precompiled headers. @@ -388,3 +412,10 @@ } } } + +# Add this config to link steps in order to compress debug sections. This is +# especially useful on 32-bit architectures in order to keep file sizes under +# 4gb. +config("compress_debug_sections") { + ldflags = [ "-gz" ] +}
diff --git a/build/config/BUILDCONFIG.gn b/build/config/BUILDCONFIG.gn index 0ef73ab..3365142 100644 --- a/build/config/BUILDCONFIG.gn +++ b/build/config/BUILDCONFIG.gn
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,7 +6,7 @@ # WHAT IS THIS FILE? # ============================================================================= # -# This is the master GN build configuration. This file is loaded after the +# This is the main GN build configuration. This file is loaded after the # build args (args.gn) for the build directory and after the toplevel ".gn" # file (which points to this file as the build configuration). # @@ -128,6 +128,11 @@ # release (!is_debug). This might be better expressed as a tri-state # (debug, release, official) but for historical reasons there are two # separate flags. + # + # IMPORTANT NOTE: (!is_debug) is *not* sufficient to get satisfying + # performance. In particular, DCHECK()s are still enabled for release builds, + # which can halve overall performance, and do increase memory usage. Always + # set "is_official_build" to true for any build intended to ship to end-users. is_official_build = false # Set to true when compiling with the Clang compiler. @@ -145,6 +150,11 @@ # every toolchain can pass through the "global" value via toolchain_args(). host_toolchain = "" + # Do not set this directly. + # It should be set only by //build/toolchains/android:robolectric_x64. + # True when compiling native code for use with robolectric_binary(). + is_robolectric = false + # DON'T ADD MORE FLAGS HERE. Read the comment above. } @@ -159,11 +169,13 @@ # When false, components will be linked statically. # # For more information see - # https://chromium.googlesource.com/chromium/src/+/master/docs/component_build.md + # https://chromium.googlesource.com/chromium/src/+/main/docs/component_build.md is_component_build = is_debug && current_os != "ios" } assert(!(is_debug && is_official_build), "Can't do official debug builds") +assert(!(current_os == "ios" && is_component_build), + "Can't use component build on iOS") # ============================================================================== # TOOLCHAIN SETUP @@ -211,6 +223,8 @@ } } else if (host_os == "aix") { host_toolchain = "//build/toolchain/aix:$host_cpu" + } else if (host_os == "zos") { + host_toolchain = "//build/toolchain/zos:$host_cpu" } else { assert(false, "Unsupported host_os: $host_os") } @@ -219,8 +233,7 @@ _default_toolchain = "" if (target_os == "android") { - assert(host_os == "linux" || host_os == "mac", - "Android builds are only supported on Linux and Mac hosts.") + assert(host_os == "linux", "Android builds are only supported on Linux.") _default_toolchain = "//build/toolchain/android:android_clang_$target_cpu" } else if (target_os == "chromeos" || target_os == "linux") { # See comments in build/toolchain/cros/BUILD.gn about board compiles. @@ -253,6 +266,8 @@ _default_toolchain = "//build/toolchain/win:uwp_$target_cpu" } else if (target_os == "aix") { _default_toolchain = "//build/toolchain/aix:$target_cpu" +} else if (target_os == "zos") { + _default_toolchain = "//build/toolchain/zos:$target_cpu" } else { assert(false, "Unsupported target_os: $target_os") } @@ -318,21 +333,22 @@ "//build/config/compiler:afdo", "//build/config/compiler:afdo_optimize_size", "//build/config/compiler:cet_shadow_stack", + "//build/config/compiler:chromium_code", "//build/config/compiler:compiler", "//build/config/compiler:compiler_arm_fpu", "//build/config/compiler:compiler_arm_thumb", - "//build/config/compiler:chromium_code", "//build/config/compiler:default_include_dirs", + "//build/config/compiler:default_init_stack_vars", "//build/config/compiler:default_optimization", "//build/config/compiler:default_stack_frames", "//build/config/compiler:default_symbols", "//build/config/compiler:export_dynamic", "//build/config/compiler:no_exceptions", "//build/config/compiler:no_rtti", + "//build/config/compiler:no_unresolved_symbols", "//build/config/compiler:runtime_library", "//build/config/compiler:thin_archive", "//build/config/compiler:thinlto_optimize_default", - "//build/config/compiler:default_init_stack_vars", "//build/config/compiler/pgo:default_pgo_flags", "//build/config/coverage:default_coverage", "//build/config/sanitizers:default_sanitizer_flags", @@ -382,10 +398,18 @@ # Static libraries and source sets use only the compiler ones. set_defaults("static_library") { configs = default_compiler_configs + + # For Rust, a static library involves linking in all dependencies, and it + # performs LTO. But since we will perform LTO in the C++ linker which + # consumes the library, we defer LTO from Rust into the linker. + configs += [ "//build/config/compiler:rust_defer_lto_to_linker" ] } set_defaults("source_set") { configs = default_compiler_configs } +set_defaults("rust_library") { + configs = default_compiler_configs +} # Compute the set of configs common to all linked targets (shared libraries, # loadable modules, executables) to avoid duplication below. @@ -399,8 +423,8 @@ # that shouldn't use the windows subsystem. "//build/config/win:console", ] -} else if (is_mac) { - _linker_configs = [ "//build/config/mac:strip_all" ] +} else if (is_apple) { + _linker_configs = [ "//build/config/apple:strip_all" ] } else { _linker_configs = [] } @@ -450,6 +474,22 @@ } } +default_rust_proc_macro_configs = + default_shared_library_configs + [ "//build/rust:proc_macro_extern" ] + + # Rust proc macros don't support (Thin)LTO, so always remove it. + [ + "//build/config/compiler:thinlto_optimize_default", + "//build/config/compiler:thinlto_optimize_max", + ] - + [ + "//build/config/compiler:thinlto_optimize_default", + "//build/config/compiler:thinlto_optimize_max", + ] + +set_defaults("rust_proc_macro") { + configs = default_rust_proc_macro_configs +} + # A helper for forwarding testonly and visibility. # Forwarding "*" does not include variables from outer scopes (to avoid copying # all globals into each template invocation), so it will not pick up @@ -474,6 +514,9 @@ "executable", "loadable_module", "shared_library", + "rust_bin", + "rust_dylib", + "rust_cdylib", ]) { template(_target_type) { # Alias "target_name" because it is clobbered by forward_variables_from(). @@ -499,7 +542,7 @@ # On Android, write shared library output file to metadata. We will use # this information to, for instance, collect all shared libraries that # should be packaged into an APK. - if (!defined(invoker.metadata) && is_android && + if (!defined(invoker.metadata) && (is_android || is_robolectric) && (_target_type == "shared_library" || _target_type == "loadable_module")) { _output_name = _target_name @@ -550,6 +593,16 @@ template("component") { if (is_component_build) { _component_mode = "shared_library" + + # Generate a unique output_name for a shared library if not set by invoker. + if (!defined(invoker.output_name)) { + _output_name = get_label_info(":$target_name", "label_no_toolchain") + _output_name = + string_replace(_output_name, "$target_name:$target_name", target_name) + _output_name = string_replace(_output_name, "//", "") + _output_name = string_replace(_output_name, "/", "_") + _output_name = string_replace(_output_name, ":", "_") + } } else if (defined(invoker.static_component_type)) { assert(invoker.static_component_type == "static_library" || invoker.static_component_type == "source_set") @@ -562,19 +615,139 @@ _component_mode = "static_library" } target(_component_mode, target_name) { + if (defined(_output_name)) { + output_name = _output_name + } forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) forward_variables_from(invoker, "*", TESTONLY_AND_VISIBILITY) } } # Component defaults +# Set a variable since we also want to make this available +# to mixed_component.gni +if (is_component_build) { + default_component_configs = default_shared_library_configs + if (is_android) { + default_component_configs -= + [ "//build/config/android:hide_all_but_jni_onload" ] + } +} else { + default_component_configs = default_compiler_configs +} + set_defaults("component") { - if (is_component_build) { - configs = default_shared_library_configs - if (is_android) { - configs -= [ "//build/config/android:hide_all_but_jni_onload" ] + configs = default_component_configs +} + +# ============================================================================= +# ACTION OVERRIDE +# ============================================================================= +# +# We override gn action() to support remote execution using rewrapper. The +# invoker should set allow_remote to true if remote execution is desired. +# +# As remote execution requires inputs to be made more explicit than is normally +# expected with gn, you may find that setting allow_remote to true will result +# in many missing file errors. In most cases, this should be resolved by +# explicitly declaring these inputs/sources. +# +# However, it may be impractical to determine these inputs in gn. For such +# cases, the invoker can specify a custom input processor, which are currently +# defined and implemented in //build/util/action_remote.py. The appropriate +# value should be set using the custom_processor arg. + +# Variables needed by rbe.gni aren't available at the top of this file. +import("//build/toolchain/rbe.gni") + +# TODO(b/253987456): Add action_foreach support. +foreach(_target_type, [ "action" ]) { + template(_target_type) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + forward_variables_from(invoker, [ "allow_remote" ]) + action("${target_name}") { + forward_variables_from(invoker, + [ + "args", + "assert_no_deps", + "check_includes", + "configs", + "data_deps", + "data", + "depfile", + "deps", + "metadata", + "outputs", + "pool", + "script", + "public_configs", + "public_deps", + "response_file_contents", + "sources", + "write_runtime_deps", + ]) + allow_remote = false + if (defined(invoker.allow_remote)) { + allow_remote = invoker.allow_remote + } + + # If remote execution is desired, only run remotely when use_remoteexec + # is enabled, and the environment is not nacl. + # TODO(b/259381924): Investigate enabling in nacl config. + if (allow_remote && use_remoteexec && !is_nacl) { + pool = "//build/toolchain:remote_action_pool($default_toolchain)" + script = "//build/util/action_remote.py" + inputs = [ invoker.script ] + + re_inputs = [ rebase_path(invoker.script, rbe_exec_root) ] + if (defined(invoker.inputs)) { + foreach(input, invoker.inputs) { + re_inputs += [ rebase_path(input, rbe_exec_root) ] + inputs += [ input ] + } + } + if (defined(invoker.sources)) { + foreach(source, invoker.sources) { + re_inputs += [ rebase_path(source, rbe_exec_root) ] + } + } + + re_outputs = [] + if (defined(invoker.outputs)) { + foreach(output, invoker.outputs) { + re_outputs += [ rebase_path(output, rbe_exec_root) ] + } + } + + # Write input/output lists to files as these can grow extremely large. + re_inputs_file = "$target_gen_dir/${target_name}__remote_inputs.rsp" + write_file(re_inputs_file, re_inputs) + inputs += [ re_inputs_file ] + re_outputs_file = "$target_gen_dir/${target_name}__remote_outputs.rsp" + write_file(re_outputs_file, re_outputs) + + args = [] + args += [ "$rbe_bin_dir/rewrapper" ] + if (defined(invoker.custom_processor)) { + args += [ "--custom_processor=" + invoker.custom_processor ] + } + + args += [ + "--cfg=$rbe_py_cfg_file", + "--exec_root=$rbe_exec_root", + "--input_list_paths=" + rebase_path(re_inputs_file, root_build_dir), + "--output_list_paths=" + rebase_path(re_outputs_file, root_build_dir), + "python3", + rebase_path(invoker.script, root_build_dir), + ] + + if (defined(invoker.args)) { + args += invoker.args + } + } else { + forward_variables_from(invoker, [ "inputs" ]) + not_needed(invoker, [ "custom_processor" ]) + } } - } else { - configs = default_compiler_configs } }
diff --git a/build/config/OWNERS b/build/config/OWNERS index eeb6706..580fa2e 100644 --- a/build/config/OWNERS +++ b/build/config/OWNERS
@@ -1,5 +1,4 @@ -dpranke@google.com -scottmg@chromium.org - per-file ozone.gni=file://ui/ozone/OWNERS per-file ozone_extra.gni=file://ui/ozone/OWNERS +per-file rust.gni=file://build/rust/OWNERS +per-file chromecast_build.gni=file://build/config/chromecast/OWNERS
diff --git a/build/config/aix/BUILD.gn b/build/config/aix/BUILD.gn index 6c8749a..6e55c83 100644 --- a/build/config/aix/BUILD.gn +++ b/build/config/aix/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -28,6 +28,7 @@ "-maix64", "-fdata-sections", "-ffunction-sections", + "-fno-extern-tls-init", "-O3", # "-Werror" @@ -46,4 +47,15 @@ "-maix64", "-Wl,-bbigtoc", ] + + if (is_component_build) { + cflags += [ "-fpic" ] + ldflags += [ + "-Wl,-brtl", + + # -bnoipath so that only names of .so objects are stored in loader + # section, excluding leading "./" + "-Wl,-bnoipath", + ] + } }
diff --git a/build/config/android/BUILD.gn b/build/config/android/BUILD.gn index 8eed45e..63b37e0 100644 --- a/build/config/android/BUILD.gn +++ b/build/config/android/BUILD.gn
@@ -1,8 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("//build/config/android/config.gni") +import("//build/config/android/rules.gni") import("//build/config/c++/c++.gni") import("//build/config/compiler/compiler.gni") import("//build/config/sanitizers/sanitizers.gni") @@ -33,32 +33,34 @@ "ANDROID_NDK_VERSION_ROLL=${android_ndk_version}_1", ] - if (current_cpu == "mips64el") { - cflags += [ - # Have to force IAS for mips64. - "-fintegrated-as", - ] - } - ldflags = [ - # Don't allow visible symbols from libgcc or libc++ to be - # re-exported. - "-Wl,--exclude-libs=libgcc.a", - # Don't allow visible symbols from libraries that contain # assembly code with symbols that aren't hidden properly. # http://crbug.com/448386 "-Wl,--exclude-libs=libvpx_assembly_arm.a", ] - # TODO(crbug.com/1184398): Move to compiler-rt when we are ready. - ldflags += [ "--rtlib=libgcc" ] if (current_cpu == "arm64") { - # For outline atomics on AArch64 (can't pass this unconditionally - # due to unused flag warning on other targets). - cflags += [ "--rtlib=libgcc" ] + # Reduce the page size from 65536 in order to reduce binary size slightly + # by shrinking the alignment gap between segments. This also causes all + # segments to be mapped adjacently, which breakpad relies on. + ldflags += [ "-Wl,-z,max-page-size=4096" ] } + if (current_cpu == "arm64") { + if (arm_control_flow_integrity == "standard") { + cflags += [ "-mbranch-protection=standard" ] + rustflags = [ "-Zbranch-protection=bti" ] + } else if (arm_control_flow_integrity == "pac") { + cflags += [ "-mbranch-protection=pac-ret" ] + rustflags = [ "-Zbranch-protection=pac-ret" ] + } + } + + # Instead of using an unwind lib from the toolchain, + # buildtools/third_party/libunwind will be built and used directly. + ldflags += [ "--unwindlib=none" ] + # $compile_api_level corresponds to the API level used for the sysroot path # calculation in //build/config/android/config.gni if (android_64bit_target_cpu) { @@ -80,11 +82,8 @@ # that is Android-only. Please see that target for advice on what should go in # :runtime_library vs. :compiler. config("runtime_library") { - # Let the linker find libgcc.a. - ldflags = [ "--gcc-toolchain=" + - rebase_path(android_toolchain_root, root_build_dir) ] - libs = [] + ldflags = [] # On 64-bit platforms, the only symbols provided by libandroid_support.a are # strto{d,f,l,ul}_l. These symbols are not used by our libc++, and newer NDKs @@ -94,24 +93,9 @@ libs += [ "android_support" ] } - # arm builds of libc++ starting in NDK r12 depend on unwind. - if (current_cpu == "arm") { - libs += [ "unwind" ] - } - if (current_cpu == "arm" && arm_version == 6) { libs += [ "atomic" ] } - - if (current_cpu == "mipsel") { - libs += [ "atomic" ] - } - - # TODO(jdduke) Re-enable on mips after resolving linking - # issues with libc++ (crbug.com/456380). - if (current_cpu != "mipsel" && current_cpu != "mips64el") { - ldflags += [ "-Wl,--warn-shared-textrel" ] - } } config("hide_all_but_jni_onload") { @@ -130,6 +114,27 @@ ldflags = [ "-Wl,--pack-dyn-relocs=android" ] } +config("lld_relr_relocations") { + # RELR supported API 30+, but supported 28+ with --use-android-relr-tags. + # https://android.googlesource.com/platform/bionic/+/master/android-changes-for-ndk-developers.md#relative-relocations-relr + ldflags = [ "-Wl,--pack-dyn-relocs=relr,--use-android-relr-tags" ] +} + +config("lld_branch_target_hardening") { + # Config opts a shared library into BTI linker hardening. This + # is an opt-in config (rather than default-enabled) to avoid + # interfering with the V8 CFI bots (crbug.com/1334614). + if (current_cpu == "arm64") { + if (arm_control_flow_integrity == "standard") { + # Linking objects without GNU_PROPERTY_AARCH64_FEATURE_1_BTI + # in their .gnu.note section implicitly results in the final + # binary losing Branch Target Identification (BTI) support. + # Issue a warning if this happens. + ldflags = [ "-Wl,-z,force-bti" ] + } + } +} + # Used for instrumented build to generate the orderfile. config("default_orderfile_instrumentation") { if (use_order_profiling) { @@ -142,12 +147,22 @@ } } +config("jni_include_dir") { + include_dirs = [ jni_headers_dir ] +} + if (current_toolchain == default_toolchain) { pool("goma_javac_pool") { # Override action_pool when goma is enabled for javac. depth = 10000 } + # nocompile tests share output directory to avoid them all needing to rebuild + # things. But this also means they can't run in parallel. + pool("nocompile_pool") { + depth = 1 + } + # When defined, this pool should be used instead of link_pool for command # that need 1-2GB of RAM. https://crbug.com/1078460 if (defined(java_cmd_pool_size)) {
diff --git a/build/config/android/DIR_METADATA b/build/config/android/DIR_METADATA new file mode 100644 index 0000000..cdc2d6f --- /dev/null +++ b/build/config/android/DIR_METADATA
@@ -0,0 +1 @@ +mixins: "//build/android/COMMON_METADATA"
diff --git a/build/config/android/abi.gni b/build/config/android/abi.gni index 53e5701..e044ac6 100644 --- a/build/config/android/abi.gni +++ b/build/config/android/abi.gni
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -29,6 +29,11 @@ # Build additional browser splits with HWASAN instrumentation enabled. build_hwasan_splits = false + + # *For CQ puposes only* Leads to non-working APKs. + # Forces all APKs/bundles to be 64-bit only to improve build speed in the CQ + # (no need to also build 32-bit library). + skip_secondary_abi_for_cq = false } assert(!devtools_instrumentation_dumping || use_order_profiling, @@ -64,11 +69,17 @@ # Place holder for mips64 support, not tested. android_abi_target = "mips64el-linux-android" +} else if (current_cpu == "riscv64") { + android_app_abi = "riscv64" + + # Place holder for riscv64 support, not tested. + android_abi_target = "riscv64-linux-android" } else { assert(false, "Unknown Android ABI: " + current_cpu) } -if (target_cpu == "arm64" || target_cpu == "x64" || target_cpu == "mips64el") { +if (target_cpu == "arm64" || target_cpu == "x64" || target_cpu == "mips64el" || + target_cpu == "riscv64") { android_64bit_target_cpu = true } else if (target_cpu == "arm" || target_cpu == "x86" || target_cpu == "mipsel") {
diff --git a/build/config/android/android_nocompile.gni b/build/config/android/android_nocompile.gni index a99bad3..0b3f517 100644 --- a/build/config/android/android_nocompile.gni +++ b/build/config/android/android_nocompile.gni
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -34,10 +34,12 @@ action(target_name) { testonly = true script = "//build/android/gyp/nocompile_test.py" + pool = "//build/config/android:nocompile_pool" _tests = invoker.tests _test0 = _tests[0] _test0_dir = get_label_info(_test0["target"], "dir") + _test0_target_out_dir = get_label_info(_test0["target"], "target_out_dir") foreach(_test_config, _tests) { assert( _test0_dir == get_label_info(_test_config["target"], "dir"), @@ -49,17 +51,28 @@ deps += invoker.deps } - inputs = [] - if (defined(invoker.pydeps)) { - foreach(_pydeps_file, invoker.pydeps) { - _pydeps_file_lines = read_file(_pydeps_file, "list lines") - _pydeps_entries = filter_exclude(_pydeps_file_lines, [ "#*" ]) - _pydeps_file_dir = get_path_info(_pydeps_file, "dir") - inputs += rebase_path(_pydeps_entries, ".", _pydeps_file_dir) - } + sources = [] + if (defined(invoker.sources)) { + sources += invoker.sources } - sources = [] + # Depend on compile_java Python scripts so that the action is re-run whenever the script is + # modified. + _pydeps = [ "//build/android/gyp/compile_java.pydeps" ] + if (defined(invoker.pydeps)) { + _pydeps += invoker.pydeps + } + + inputs = [] + foreach(_pydeps_file, _pydeps) { + _pydeps_file_lines = [] + _pydeps_file_lines = read_file(_pydeps_file, "list lines") + _pydeps_entries = [] + _pydeps_entries = filter_exclude(_pydeps_file_lines, [ "#*" ]) + _pydeps_file_dir = get_path_info(_pydeps_file, "dir") + inputs += rebase_path(_pydeps_entries, ".", _pydeps_file_dir) + } + _json_test_configs = [] foreach(_test_config, _tests) { _test = _test_config["target"] @@ -78,13 +91,18 @@ _config_path = "$target_gen_dir/${target_name}.nocompile_config" write_file(_config_path, _json_test_configs, "json") + # Compute output directory for no-compile tests based on the directory containing test + # targets instead of based on the test suite target name. This avoids calling 'gn gen' for each + # android_nocompile_test_suite() for test suites whose tests are declared in the same BUILD.gn + # file. + _out_dir = "${_test0_target_out_dir}/nocompile_out" + _stamp_path = "${target_gen_dir}/${target_name}.stamp" args = [ "--gn-args-path", "args.gn", "--out-dir", - rebase_path("${target_out_dir}/${target_name}/nocompile_out", - root_build_dir), + rebase_path(_out_dir, root_build_dir), "--test-configs-path", rebase_path(_config_path, root_build_dir), "--stamp",
diff --git a/build/config/android/build_vars.gni b/build/config/android/build_vars.gni index a47607d..27866a7 100644 --- a/build/config/android/build_vars.gni +++ b/build/config/android/build_vars.gni
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -17,7 +17,9 @@ android_sdk_root = rebase_path(android_sdk_root, root_build_dir) android_sdk_version = android_sdk_version android_tool_prefix = rebase_path(android_tool_prefix, root_build_dir) + default_min_sdk_version = default_min_sdk_version final_android_sdk = final_android_sdk + public_android_sdk_version = public_android_sdk_version if (defined(android_secondary_abi_cpu)) { android_secondary_abi_toolchain =
diff --git a/build/config/android/channel.gni b/build/config/android/channel.gni index 6348bb9..0f8d453 100644 --- a/build/config/android/channel.gni +++ b/build/config/android/channel.gni
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/android/config.gni b/build/config/android/config.gni index 8ffe591..5f48367 100644 --- a/build/config/android/config.gni +++ b/build/config/android/config.gni
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,6 +8,7 @@ # toolchain, this GNI file may be read and processed from within Chrome OS # toolchains. Checking |is_android| here would therefore be too restrictive. if (is_android || is_chromeos) { + import("//build/config/android/channel.gni") import("//build/config/chromecast_build.gni") import("//build/config/dcheck_always_on.gni") import("//build_overrides/build.gni") @@ -30,45 +31,50 @@ } } - has_chrome_android_internal = - exec_script("//build/dir_exists.py", - [ rebase_path("//clank", root_build_dir) ], - "string") == "True" - # We are using a separate declare_args block for only this argument so that # we can decide if we have to pull in definitions from the internal config # early. declare_args() { # Enables using the internal Chrome for Android repository. The default - # value depends on whether the repository is available, and if it's not but - # this argument is manually set to True, the generation will fail. - # The main purpose of this argument is to avoid having to maintain 2 - # repositories to support both public only and internal builds. - enable_chrome_android_internal = has_chrome_android_internal + # is set from gclient vars, with target_os needed for chromeos. + # Can be set to false to disable all internal android things. + enable_chrome_android_internal = + build_with_chromium && checkout_src_internal && target_os == "android" # The default to use for android:minSdkVersion for targets that do # not explicitly set it. default_min_sdk_version = 24 - # [WIP] Allows devs to achieve much faster edit-build-install cycles. - # Currently only works for ChromeModern apks due to incremental install. - # This needs to be in a separate declare_args as it determines some of the - # args in the main declare_args block below. - android_fast_local_dev = false - } - - declare_args() { - # Android API level for 32 bits platforms - android32_ndk_api_level = default_min_sdk_version - - # Android API level for 64 bits platforms - if (default_min_sdk_version < 24) { - android64_ndk_api_level = 24 + # Static analysis can be either "on" or "off" or "build_server". This + # controls how android lint, error-prone, bytecode checks are run. This + # needs to be in a separate declare_args as it determines some of the args + # in the main declare_args block below. + # "on" is the default. + # "off" avoids running any static analysis. This is the default for + # official builds to reduce build times. Failures in static analysis + # would have been caught by other bots. + # "build_server" ensures that fast_local_dev_server.py is running and + # offloads analysis tasks to it to be run after the build completes. + # This is the recommended setting for local development. + if (is_official_build) { + android_static_analysis = "off" } else { - android64_ndk_api_level = default_min_sdk_version + android_static_analysis = "on" } + + # Build incremental targets whenever possible. + # See //build/android/incremental_install/README.md for more details. + incremental_install = false } + # Avoid typos when setting android_static_analysis in args.gn. + assert(android_static_analysis == "on" || android_static_analysis == "off" || + android_static_analysis == "build_server") + + # This configuration has not bot coverage and has broken multiple times. + # Warn against it. + assert(!(enable_chrome_android_internal && skip_secondary_abi_for_cq)) + if (enable_chrome_android_internal) { import("//clank/config.gni") } else { @@ -81,18 +87,28 @@ if (!defined(default_android_ndk_root)) { default_android_ndk_root = "//third_party/android_ndk" - default_android_ndk_version = "r25" - default_android_ndk_major_version = 25 + default_android_ndk_version = "r23" + default_android_ndk_major_version = 23 } else { assert(defined(default_android_ndk_version)) assert(defined(default_android_ndk_major_version)) } public_android_sdk_root = "//third_party/android_sdk/public" - if (android_sdk_release == "r") { + public_android_sdk_build_tools = + "${public_android_sdk_root}/build-tools/33.0.0" + public_android_sdk_version = "33" + if (android_sdk_release == "t") { default_android_sdk_root = public_android_sdk_root - default_android_sdk_version = "30" - default_android_sdk_build_tools_version = "30.0.1" + default_android_sdk_version = public_android_sdk_version + default_android_sdk_build_tools_version = "33.0.0" + public_android_sdk = true + } + + if (android_sdk_release == "tprivacysandbox") { + default_android_sdk_root = public_android_sdk_root + default_android_sdk_version = "TiramisuPrivacySandbox" + default_android_sdk_build_tools_version = "33.0.0" public_android_sdk = true } @@ -105,7 +121,7 @@ # Purposefully repeated so that downstream can change # default_android_sdk_root without changing lint version. default_lint_android_sdk_root = public_android_sdk_root - default_lint_android_sdk_version = 30 + default_lint_android_sdk_version = 33 } if (!defined(default_extras_android_sdk_root)) { @@ -124,31 +140,7 @@ # google_play_services_package contains the path where individual client # targets (e.g. google_play_services_base_java) are located. if (!defined(google_play_services_package)) { - if (is_chromecast && chromecast_branding != "public") { - google_play_services_package = "//chromecast/internal/android/prebuilt/google-play-services-first-party" - } else { - google_play_services_package = "//third_party/android_deps" - } - } - - if (!defined(dagger_java_target)) { - dagger_java_target = - "//third_party/android_deps:com_google_dagger_dagger_java" - } - - if (!defined(dagger_annotation_processor_target)) { - dagger_annotation_processor_target = - "//third_party/android_deps:com_google_dagger_dagger_compiler_java" - } - - if (!defined(guava_android_target)) { - guava_android_target = - "//third_party/android_deps:com_google_guava_guava_android_java" - } - - if (!defined(material_design_target)) { - material_design_target = - "//third_party/android_deps:com_google_android_material_material_java" + google_play_services_package = "//third_party/android_deps" } if (!defined(android_protoc_bin)) { @@ -171,6 +163,17 @@ android_ndk_version = default_android_ndk_version android_ndk_major_version = default_android_ndk_major_version + # Android API level for 32 bits platforms + android32_ndk_api_level = default_min_sdk_version + + # Android API level for 64 bits platforms + android64_ndk_api_level = default_min_sdk_version + + if (default_min_sdk_version < 21) { + # Android did not support 64 bit before API 21. + android64_ndk_api_level = 21 + } + android_sdk_root = default_android_sdk_root android_sdk_version = default_android_sdk_version android_sdk_build_tools_version = default_android_sdk_build_tools_version @@ -204,30 +207,20 @@ # Java debug on Android. Having this on enables multidexing, and turning it # off will enable proguard. - is_java_debug = is_debug + is_java_debug = is_debug || incremental_install # Mark APKs as android:debuggable="true". debuggable_apks = !is_official_build # Set to false to disable the Errorprone compiler. - # Defaults to false for official builds to reduce build times. - # Static analysis failures should have been already caught by normal bots. - # Disabled when fast_local_dev is turned on. - use_errorprone_java_compiler = !is_official_build && !android_fast_local_dev - - # Build incremental targets whenever possible. - # See //build/android/incremental_install/README.md for more details. - incremental_install = android_fast_local_dev + use_errorprone_java_compiler = android_static_analysis != "off" # When true, updates all android_aar_prebuilt() .info files during gn gen. # Refer to android_aar_prebuilt() for more details. update_android_aar_prebuilts = false - # Turns off android lint. Useful for prototyping or for faster local builds. - # Defaults to true for official builds to reduce build times. - # Static analysis failures should have been already caught by normal bots. - # Disabled when fast_local_dev is turned on. - disable_android_lint = is_official_build || android_fast_local_dev + # Turns off android lint. + disable_android_lint = android_static_analysis == "off" # Location of aapt2 used for app bundles. For now, a more recent version # than the one distributed with the Android SDK is required. @@ -247,8 +240,23 @@ # support mapping these names. enable_arsc_obfuscation = true + # Controls whether |strip_unused_resources| is respected. Useful when trying + # to analyze APKs using tools that do not support missing resources from + # resources.arsc. + enable_unused_resource_stripping = true + + # Controls whether |baseline_profile_path| is respected. Useful to disable + # baseline profiles. + # Currently disabled while bundletool does not support baseline profiles in + # non-base splits. + enable_baseline_profiles = false + # The target to use as the system WebView implementation. - system_webview_apk_target = "//android_webview:system_webview_apk" + if (android_64bit_target_cpu && skip_secondary_abi_for_cq) { + system_webview_apk_target = "//android_webview:system_webview_64_apk" + } else { + system_webview_apk_target = "//android_webview:system_webview_apk" + } # Where to write failed expectations for bots to read. expectations_failure_dir = "$root_build_dir/failed_expectations" @@ -264,7 +272,7 @@ } # Whether java assertions and Preconditions checks are enabled. - enable_java_asserts = is_java_debug || dcheck_always_on + enable_java_asserts = dcheck_always_on || !is_official_build # Reduce build time by using d8 incremental build. enable_incremental_d8 = true @@ -272,15 +280,19 @@ # Use hashed symbol names to reduce JNI symbol overhead. use_hashed_jni_names = !is_java_debug - # Desugar lambdas and interfaces methods using Desugar.jar rather than - # D8/R8. D8/R8 will still be used for backported method desugaring. - enable_bazel_desugar = true + # Enables JNI multiplexing to reduce JNI native methods overhead. + allow_jni_multiplexing = false - # Enables Java library desugaring. - # This will cause an extra classes.dex file to appear in every apk. - enable_jdk_library_desugaring = true + # Enables trace event injection on Android views with bytecode rewriting. + # This adds an additional step on android_app_bundle_module targets that + # adds trace events to some methods in android.view.View subclasses. + enable_trace_event_bytecode_rewriting = + !is_java_debug && android_channel != "stable" } + assert(!incremental_install || is_java_debug, + "incremental_install=true && is_java_debug=false is not supported.") + # Host stuff ----------------------------------------------------------------- # Defines the name the Android build gives to the current host CPU @@ -322,32 +334,33 @@ # like the toolchain roots. if (current_cpu == "x86") { android_prebuilt_arch = "android-x86" - _binary_prefix = "i686-linux-android" } else if (current_cpu == "arm") { android_prebuilt_arch = "android-arm" - _binary_prefix = "arm-linux-androideabi" } else if (current_cpu == "mipsel") { android_prebuilt_arch = "android-mips" - _binary_prefix = "mipsel-linux-android" } else if (current_cpu == "x64") { android_prebuilt_arch = "android-x86_64" - _binary_prefix = "x86_64-linux-android" } else if (current_cpu == "arm64") { android_prebuilt_arch = "android-arm64" - _binary_prefix = "aarch64-linux-android" } else if (current_cpu == "mips64el") { android_prebuilt_arch = "android-mips64" - _binary_prefix = "mips64el-linux-android" + } else if (current_cpu == "riscv64") { + # Place holder for riscv64 support, not tested. + android_prebuilt_arch = "android-riscv64" } else { assert(false, "Need android libgcc support for your target arch.") } android_toolchain_root = "$android_ndk_root/toolchains/llvm/prebuilt/${android_host_os}-${android_host_arch}" - android_tool_prefix = "$android_toolchain_root/bin/$_binary_prefix-" - android_readelf = "${android_tool_prefix}readelf" + android_ndk_library_path = "$android_toolchain_root/lib64" + android_tool_prefix = "$android_toolchain_root/bin/llvm-" + android_readelf = "${android_tool_prefix}readobj" android_objcopy = "${android_tool_prefix}objcopy" android_gdbserver = "$android_ndk_root/prebuilt/$android_prebuilt_arch/gdbserver/gdbserver" android_sdk_tools_bundle_aapt2 = "${android_sdk_tools_bundle_aapt2_dir}/aapt2" + + # Toolchain used to create native libraries for robolectric_binary() targets. + robolectric_toolchain = "//build/toolchain/android:robolectric_$host_cpu" }
diff --git a/build/config/android/copy_ex.gni b/build/config/android/copy_ex.gni index d3705dd..8e70c30 100644 --- a/build/config/android/copy_ex.gni +++ b/build/config/android/copy_ex.gni
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. #
diff --git a/build/config/android/create_unwind_table.gni b/build/config/android/create_unwind_table.gni new file mode 100644 index 0000000..92b7427 --- /dev/null +++ b/build/config/android/create_unwind_table.gni
@@ -0,0 +1,50 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") + +unwind_table_asset_v2_filename = "unwind_cfi_32_v2" + +_dump_syms_target = "//third_party/breakpad:dump_syms($host_toolchain)" +_dump_syms = get_label_info(_dump_syms_target, "root_out_dir") + "/dump_syms" +_readobj_path = "$clang_base_path/bin/llvm-readobj" + +template("unwind_table_v2") { + action(target_name) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + _output_path = + "$target_out_dir/$target_name/$unwind_table_asset_v2_filename" + + # Strip the "lib" prefix, if present. Add and then remove a space because + # our ownly tool is "replace all". + _library_name = get_label_info(invoker.library_target, "name") + _library_name = + string_replace(string_replace(" $_library_name", " $shlib_prefix", ""), + " ", + "") + _library_path = "$root_out_dir/lib.unstripped/$shlib_prefix$_library_name$shlib_extension" + + script = "//build/android/gyp/create_unwind_table.py" + outputs = [ _output_path ] + inputs = [ + _dump_syms, + _library_path, + ] + deps = [ + _dump_syms_target, + invoker.library_target, + ] + + args = [ + "--input_path", + rebase_path(_library_path, root_build_dir), + "--output_path", + rebase_path(_output_path, root_build_dir), + "--dump_syms_path", + rebase_path(_dump_syms, root_build_dir), + "--readobj_path", + rebase_path(_readobj_path, root_build_dir), + ] + } +}
diff --git a/build/config/android/extract_unwind_tables.gni b/build/config/android/extract_unwind_tables.gni index 5444c5b..d4daa6a 100644 --- a/build/config/android/extract_unwind_tables.gni +++ b/build/config/android/extract_unwind_tables.gni
@@ -1,44 +1,47 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/android/rules.gni") -template("unwind_table_asset") { - # Note: This file name is used in multiple monochrome build scripts. - _asset_path = "${target_gen_dir}/${target_name}/unwind_cfi_32" - _unwind_action = "${target_name}__extract" +unwind_table_asset_v1_filename = "unwind_cfi_32" - action(_unwind_action) { +_dump_syms_target = "//third_party/breakpad:dump_syms($host_toolchain)" +_dump_syms = get_label_info(_dump_syms_target, "root_out_dir") + "/dump_syms" + +template("unwind_table_v1") { + action(target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + _output_path = + "$target_out_dir/$target_name/$unwind_table_asset_v1_filename" - _root_dir = "$root_out_dir" - if (defined(android_secondary_abi_cpu)) { - _root_dir = get_label_info(":foo($android_secondary_abi_toolchain)", - "root_out_dir") - } + # Strip the "lib" prefix, if present. Add and then remove a space because + # our ownly tool is "replace all". + _library_name = get_label_info(invoker.library_target, "name") + _library_name = + string_replace(string_replace(" $_library_name", " $shlib_prefix", ""), + " ", + "") + _library_path = "$root_out_dir/lib.unstripped/$shlib_prefix$_library_name$shlib_extension" script = "//build/android/gyp/extract_unwind_tables.py" - outputs = [ _asset_path ] - inputs = [ "${_root_dir}/lib.unstripped/$shlib_prefix${invoker.library_target}$shlib_extension" ] + outputs = [ _output_path ] + inputs = [ + _dump_syms, + _library_path, + ] + deps = [ + _dump_syms_target, + invoker.library_target, + ] args = [ "--input_path", - rebase_path( - "${_root_dir}/lib.unstripped/$shlib_prefix${invoker.library_target}$shlib_extension", - root_build_dir), + rebase_path(_library_path, root_build_dir), "--output_path", - rebase_path(_asset_path, root_build_dir), + rebase_path(_output_path, root_build_dir), "--dump_syms_path", - rebase_path("$root_out_dir/dump_syms", root_build_dir), + rebase_path(_dump_syms, root_build_dir), ] - deps = invoker.deps - deps += [ "//third_party/breakpad:dump_syms" ] - } - android_assets(target_name) { - forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - sources = [ _asset_path ] - disable_compression = true - deps = [ ":$_unwind_action" ] } }
diff --git a/build/config/android/internal_rules.gni b/build/config/android/internal_rules.gni index 75ab855..427fa0d 100644 --- a/build/config/android/internal_rules.gni +++ b/build/config/android/internal_rules.gni
@@ -1,11 +1,11 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # Do not add any imports to non-//build directories here. # Some projects (e.g. V8) do not have non-build directories DEPS'ed in. +import("//build/config/android/channel.gni") import("//build/config/android/config.gni") -import("//build/config/android/copy_ex.gni") import("//build/config/compiler/compiler.gni") import("//build/config/compute_inputs_for_analyze.gni") import("//build/config/coverage/coverage.gni") @@ -15,11 +15,20 @@ import("//build/toolchain/kythe.gni") import("//build/util/generate_wrapper.gni") import("//build_overrides/build.gni") -if (is_starboardized_toolchain) { +if (is_starboardized_toolchain || current_toolchain == default_toolchain) { import("//build/toolchain/concurrent_links.gni") } assert(is_android) +default_android_sdk_dep = "//third_party/android_sdk:android_sdk_java" +_jacoco_dep = "//third_party/jacoco:jacocoagent_java" +_jacoco_host_jar = + "$root_build_dir/lib.java/third_party/jacoco/jacocoagent_java.jar" +_robolectric_libs_dir = + rebase_path( + get_label_info("//:foo($robolectric_toolchain)", "root_out_dir"), + root_build_dir) + # The following _java_*_types variables capture all the existing target types. # If a new type is introduced, please add it to one of these categories, # preferring the more specific resource/library types. @@ -43,102 +52,64 @@ "dist_jar", "java_annotation_processor", "java_binary", - "junit_binary", + "robolectric_binary", ] # All _java_resource_types targets must conform to these patterns. -_java_resource_patterns = [ - "*:*_assets", +java_resource_patterns = [ + "*_assets", + "*_grd", + "*_java_strings", + "*locale_paks", + "*_resources", + "*strings_java", "*android*:assets", "*:*_apk_*resources", "*android*:resources", - "*:*_resources", - "*:*_grd", - "*:*locale_paks", - "*:*_java_strings", - "*:*strings_java", ] # All _java_library_types targets must conform to these patterns. This includes # all non-leaf targets that use java_library_impl. -_java_library_patterns = [ - "*:*_java", - "*:*_javalib", - "*:*_java_*", # e.g. chrome_java_test_support +java_library_patterns = [ + "*_java", + "*_javalib", + "*javatests", + "*_bundle_module", + "*:*_java_*", # E.g. chrome_java_test_support "*:java", + "*/java", # to allow filtering without expanding labels //a/java -> + # //a/java:java "*:junit", + "*/junit", "*:junit_*", "*:*_junit_*", - "*:*javatests", - "*:*_bundle_module", - # TODO(agrieve): Rename targets below to match above patterns. + # TODO(agrieve): Rename to glue_java + "//android_webview/glue", "//android_webview/glue:glue", ] -# These identify all non-leaf targets that have .build_config files. This is the +# These identify all non-leaf targets that have .build_config.json files. This is the # set of patterns that other targets can use to filter out java targets. -java_target_patterns = _java_library_patterns + _java_resource_patterns +java_target_patterns = java_library_patterns + java_resource_patterns _r8_path = "//third_party/r8/lib/r8.jar" -_custom_d8_path = "//third_party/r8/custom_d8.jar" -_desugar_jdk_libs_json = "//third_party/r8/desugar_jdk_libs.json" -_desugar_jdk_libs_jar = "//third_party/android_deps/libs/com_android_tools_desugar_jdk_libs/desugar_jdk_libs-1.1.1.jar" -_desugar_jdk_libs_configuration_jar = "//third_party/android_deps/libs/com_android_tools_desugar_jdk_libs_configuration/desugar_jdk_libs_configuration-1.1.1.jar" -_desugar_runtime_jar = "$root_build_dir/obj/third_party/bazel/desugar/Desugar_runtime.processed.jar" -_dexdump_path = "$android_sdk_build_tools/dexdump" -_dexlayout_path = "//third_party/android_build_tools/art/dexlayout" -_profman_path = "//third_party/android_build_tools/art/profman" -_art_lib_file_names = [ - "libartbase.so", - "libart-compiler.so", - "libart-dexlayout.so", - "libart-disassembler.so", - "libart-gtest.so", - "libart.so", - "libbacktrace.so", - "libbase.so", - "libcrypto-host.so", - "libc++.so", - "libcutils.so", - "libdexfile.so", - "libexpat-host.so", - "libicui18n-host.so", - "libicuuc-host.so", - "libjavacore.so", - "libjavacrypto.so", - "liblog.so", - "liblz4.so", - "liblzma.so", - "libnativebridge.so", - "libnativehelper.so", - "libnativeloader.so", - "libopenjdkjvm.so", - "libopenjdkjvmti.so", - "libopenjdk.so", - "libprofile.so", - "libsigchain.so", - "libssl-host.so", - "libunwindstack.so", - "libvixl-arm64.so", - "libvixl-arm.so", - "libvixld-arm64.so", - "libvixld-arm.so", - "libz-host.so", - "libziparchive.so", - "slicer.so", -] -_default_art_libs = [] -foreach(lib, _art_lib_file_names) { - _default_art_libs += [ "//third_party/android_build_tools/art/lib/$lib" ] -} +# This duplication is intentional, so we avoid updating the r8.jar used by +# dexing unless necessary, since each update invalidates all incremental dexing +# and unnecessarily slows down all bots. +_d8_path = "//third_party/r8/d8/lib/r8.jar" +_custom_d8_path = "//third_party/r8/custom_d8.jar" +_default_lint_jar_path = "//third_party/android_build_tools/lint/lint.jar" +_custom_lint_jar_path = "//third_party/android_build_tools/lint/custom_lint.jar" +_manifest_merger_jar_path = + "//third_party/android_build_tools/manifest_merger/manifest-merger.jar" # Put the bug number in the target name so that false-positives have a hint in # the error message about why non-existent dependencies are there. build_config_target_suffix = "__build_config_crbug_908819" -# Write the target's .build_config file. This is a json file that contains a +# Write the target's .build_config.json file. This is a json file that contains a # dictionary of information about how to build this target (things that # require knowledge about this target's dependencies and cannot be calculated # at gn-time). There is a special syntax to add a value in that dictionary to @@ -153,16 +124,15 @@ _target_label = get_label_info(":${_parent_invoker.target_name}", "label_no_toolchain") - # Ensure targets match naming patterns so that __assetres, __header, __impl - # targets work properly. Those generated targets allow for effective deps - # filtering. + # Ensure targets match naming patterns so that __assetres, __header, __host, + # and __validate targets work properly. if (filter_exclude([ _type ], _java_resource_types) == []) { - if (filter_exclude([ _target_label ], _java_resource_patterns) != []) { + if (filter_exclude([ _target_label ], java_resource_patterns) != []) { assert(false, "Invalid java resource target name: $_target_label") } } else if (filter_exclude([ _type ], _java_library_types) == []) { - if (filter_exclude([ _target_label ], _java_library_patterns) != [] || - filter_exclude([ _target_label ], _java_resource_patterns) == []) { + if (filter_exclude([ _target_label ], java_library_patterns) != [] || + filter_exclude([ _target_label ], java_resource_patterns) == []) { assert(false, "Invalid java library target name: $_target_label") } } else if (_type == "group") { @@ -196,28 +166,45 @@ outputs = [ invoker.build_config ] _deps_configs = [] - _public_deps_configs = [] if (defined(invoker.possible_config_deps)) { foreach(_possible_dep, invoker.possible_config_deps) { _dep_label = get_label_info(_possible_dep, "label_no_toolchain") if (filter_exclude([ _dep_label ], java_target_patterns) == []) { - # Put the bug number in the target name so that false-positives - # have a hint in the error message about non-existent dependencies. deps += [ "$_dep_label$build_config_target_suffix" ] _dep_gen_dir = get_label_info(_possible_dep, "target_gen_dir") _dep_name = get_label_info(_possible_dep, "name") - _dep_config = "$_dep_gen_dir/$_dep_name.build_config" + _dep_config = "$_dep_gen_dir/$_dep_name.build_config.json" _deps_configs += [ _dep_config ] - if (defined(invoker.possible_config_public_deps)) { - if (filter_include([ _possible_dep ], - invoker.possible_config_public_deps) != []) { - _public_deps_configs += [ _dep_config ] - } - } } } } + _public_deps_configs = [] + if (defined(invoker.possible_config_public_deps)) { + foreach(_possible_dep, invoker.possible_config_public_deps) { + _dep_label = get_label_info(_possible_dep, "label_no_toolchain") + + # E.g. Adding an action that generates a .java file that is then + # consumed by a subsequent java_library() target would not work + # because the libraries depend only on the nested targets of one + # another. It is simplest to just ban non-java public_deps. + assert(filter_exclude([ _dep_label ], java_target_patterns) == [], + "Only java_library targets can be used as public_deps. " + + "Found:\n${_dep_label}\non Target:\n" + + get_label_info(":$target_name", "label_no_toolchain")) + + # Put the bug number in the target name so that false-positives + # have a hint in the error message about non-existent dependencies. + deps += [ "$_dep_label$build_config_target_suffix" ] + _dep_gen_dir = get_label_info(_possible_dep, "target_gen_dir") + _dep_name = get_label_info(_possible_dep, "name") + _dep_config = "$_dep_gen_dir/$_dep_name.build_config.json" + + _public_deps_configs += [ _dep_config ] + } + } + inputs += _deps_configs + inputs += _public_deps_configs _rebased_deps_configs = rebase_path(_deps_configs, root_build_dir) _rebased_public_deps_configs = rebase_path(_public_deps_configs, root_build_dir) @@ -234,9 +221,8 @@ _target_label, ] - if (defined(invoker.ignore_dependency_public_deps) && - invoker.ignore_dependency_public_deps) { - args += [ "--ignore-dependency-public-deps" ] + if (defined(invoker.preferred_dep) && invoker.preferred_dep) { + args += [ "--preferred-dep" ] } if (defined(invoker.aar_path)) { @@ -275,6 +261,12 @@ rebase_path(invoker.ijar_path, root_build_dir), ] } + if (defined(invoker.kotlinc_jar_path)) { + args += [ + "--kotlinc-jar-path", + rebase_path(invoker.kotlinc_jar_path, root_build_dir), + ] + } if (defined(invoker.java_resources_jar)) { args += [ "--java-resources-jar-path", @@ -284,15 +276,16 @@ if (defined(invoker.annotation_processor_deps) && invoker.annotation_processor_deps != []) { _processor_configs = [] - foreach(_processor_dep, invoker.annotation_processor_deps) { - _dep_label = get_label_info(_processor_dep, "label_no_toolchain") - _dep_gen_dir = get_label_info(_processor_dep, "target_gen_dir") - _dep_name = get_label_info(_processor_dep, "name") + foreach(_dep_label, invoker.annotation_processor_deps) { deps += [ "$_dep_label$build_config_target_suffix" ] - _processor_configs += [ "$_dep_gen_dir/$_dep_name.build_config" ] + _dep_gen_dir = get_label_info(_dep_label, "target_gen_dir") + _dep_name = get_label_info(_dep_label, "name") + _dep_config = "$_dep_gen_dir/$_dep_name.build_config.json" + _processor_configs += [ _dep_config ] } _rebased_processor_configs = rebase_path(_processor_configs, root_build_dir) + inputs += _processor_configs args += [ "--annotation-processor-configs=$_rebased_processor_configs" ] } @@ -324,17 +317,20 @@ invoker.bypass_platform_checks) { args += [ "--bypass-platform-checks" ] } + if (defined(invoker.is_robolectric) && invoker.is_robolectric) { + args += [ "--is-robolectric" ] + } if (defined(invoker.apk_under_test)) { - deps += [ "${invoker.apk_under_test}$build_config_target_suffix" ] - apk_under_test_gen_dir = - get_label_info(invoker.apk_under_test, "target_gen_dir") - apk_under_test_name = get_label_info(invoker.apk_under_test, "name") - apk_under_test_config = - "$apk_under_test_gen_dir/$apk_under_test_name.build_config" + _dep_label = invoker.apk_under_test + _dep_gen_dir = get_label_info(_dep_label, "target_gen_dir") + _dep_name = get_label_info(_dep_label, "name") + _dep_config = "$_dep_gen_dir/$_dep_name.build_config.json" + inputs += [ _dep_config ] + deps += [ "$_dep_label$build_config_target_suffix" ] args += [ "--tested-apk-config", - rebase_path(apk_under_test_config, root_build_dir), + rebase_path(_dep_config, root_build_dir), ] } @@ -360,6 +356,12 @@ args += [ "--treat-as-locale-paks" ] } + if (defined(invoker.merged_android_manifest)) { + args += [ + "--merged-android-manifest", + rebase_path(invoker.merged_android_manifest, root_build_dir), + ] + } if (defined(invoker.android_manifest)) { inputs += [ invoker.android_manifest ] args += [ @@ -396,10 +398,6 @@ rebase_path(invoker.res_size_info_path, root_build_dir), ] } - if (defined(invoker.resource_dirs)) { - resource_dirs = rebase_path(invoker.resource_dirs, root_build_dir) - args += [ "--resource-dirs=$resource_dirs" ] - } if (defined(invoker.res_sources_path)) { _res_sources_path = rebase_path(invoker.res_sources_path, root_build_dir) args += [ "--res-sources-path=$_res_sources_path" ] @@ -436,10 +434,6 @@ ] } - if (defined(invoker.is_base_module) && invoker.is_base_module) { - args += [ "--is-base-module" ] - } - if (defined(invoker.loadable_modules)) { _rebased_loadable_modules = rebase_path(invoker.loadable_modules, root_build_dir) @@ -474,19 +468,10 @@ args += [ "--secondary-native-lib-placeholders=${invoker.secondary_native_lib_placeholders}" ] } - if (defined(invoker.uncompress_shared_libraries) && - invoker.uncompress_shared_libraries) { - args += [ "--uncompress-shared-libraries" ] - } - if (defined(invoker.library_always_compress)) { args += [ "--library-always-compress=${invoker.library_always_compress}" ] } - if (defined(invoker.library_renames)) { - args += [ "--library-renames=${invoker.library_renames}" ] - } - if (defined(invoker.apk_path)) { # TODO(tiborg): Remove APK path from build config and use # install_artifacts from metadata instead. @@ -504,10 +489,10 @@ } } - if (defined(invoker.java_sources_file)) { + if (defined(invoker.target_sources_file)) { args += [ - "--java-sources-file", - rebase_path(invoker.java_sources_file, root_build_dir), + "--target-sources-file", + rebase_path(invoker.target_sources_file, root_build_dir), ] } if (defined(invoker.srcjar)) { @@ -550,20 +535,6 @@ rebase_path(invoker.proguard_configs, root_build_dir) args += [ "--proguard-configs=$_rebased_proguard_configs" ] } - if (defined(invoker.static_library_dependent_targets)) { - _dependent_configs = [] - foreach(_dep, invoker.static_library_dependent_targets) { - _dep_name = _dep.name - _dep_label = get_label_info(_dep_name, "label_no_toolchain") - deps += [ "$_dep_label$build_config_target_suffix" ] - _dep_gen_dir = get_label_info(_dep_name, "target_gen_dir") - _dep_name = get_label_info(_dep_name, "name") - _config = - rebase_path("$_dep_gen_dir/$_dep_name.build_config", root_build_dir) - _dependent_configs += [ _config ] - } - args += [ "--static-library-dependent-configs=$_dependent_configs" ] - } if (defined(invoker.gradle_treat_as_prebuilt) && invoker.gradle_treat_as_prebuilt) { args += [ "--gradle-treat-as-prebuilt" ] @@ -575,24 +546,76 @@ ] } if (defined(invoker.base_module_target)) { - _base_label = - get_label_info(invoker.base_module_target, "label_no_toolchain") - _dep_gen_dir = get_label_info(_base_label, "target_gen_dir") - _dep_name = get_label_info(_base_label, "name") - deps += [ "$_base_label$build_config_target_suffix" ] - _base_module_build_config = "$_dep_gen_dir/$_dep_name.build_config" - inputs += [ _base_module_build_config ] + _dep_label = invoker.base_module_target + _dep_gen_dir = get_label_info(_dep_label, "target_gen_dir") + _dep_name = get_label_info(_dep_label, "name") + _dep_config = "$_dep_gen_dir/$_dep_name.build_config.json" + deps += [ "$_dep_label$build_config_target_suffix" ] + inputs += [ _dep_config ] args += [ "--base-module-build-config", - rebase_path(_base_module_build_config, root_build_dir), + rebase_path(_dep_config, root_build_dir), ] } + if (defined(invoker.parent_module_target)) { + _dep_label = invoker.parent_module_target + _dep_gen_dir = get_label_info(_dep_label, "target_gen_dir") + _dep_name = get_label_info(_dep_label, "name") + _dep_config = "$_dep_gen_dir/$_dep_name.build_config.json" + deps += [ "$_dep_label$build_config_target_suffix" ] + inputs += [ _dep_config ] + args += [ + "--parent-module-build-config", + rebase_path(_dep_config, root_build_dir), + ] + } + if (defined(invoker.module_name)) { + args += [ + "--module-name", + invoker.module_name, + ] + } + if (defined(invoker.modules)) { + foreach(_module, invoker.modules) { + if (defined(_module.uses_split)) { + args += [ "--uses-split=${_module.name}:${_module.uses_split}" ] + } + } + } if (defined(invoker.module_build_configs)) { inputs += invoker.module_build_configs _rebased_configs = rebase_path(invoker.module_build_configs, root_build_dir) args += [ "--module-build-configs=$_rebased_configs" ] } + if (defined(invoker.add_view_trace_events) && + invoker.add_view_trace_events) { + # Adding trace events involves rewriting bytecode and generating a new set + # of jar files. In order to avoid conflicts between bundles we save the + # new jars in a bundle specific gen/ directory. The build config for the + # bundle, and each one of its modules need a path to a bundle specific + # gen/ directory in order to generate a list of rewritten jar paths. + # We use the base module's target_gen_dir because non-base modules and the + # app bundle targets have a reference to it (base_module_target). + if (_type == "android_app_bundle") { + _trace_events_target_name = + get_label_info(_parent_invoker.base_module_target, "name") + } else if (defined(invoker.base_module_target)) { + _trace_events_target_name = + get_label_info(invoker.base_module_target, "name") + } else { + _grandparent_invoker = _parent_invoker.invoker + _trace_events_target_name = _grandparent_invoker.target_name + } + + # FIXME: This should likely be using the base module's target_out_dir + # rather than the current target's. + args += [ + "--trace-events-jar-dir", + rebase_path("$target_out_dir/$_trace_events_target_name", + root_build_dir), + ] + } if (defined(invoker.version_name)) { args += [ "--version-name", @@ -648,22 +671,6 @@ template("generate_r_java") { action_with_pydeps(target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "deps" ]) - if (!defined(deps)) { - deps = [] - } - if (defined(invoker.possible_resource_deps)) { - foreach(_dep, invoker.possible_resource_deps) { - _target_label = get_label_info(_dep, "label_no_toolchain") - if (filter_exclude([ _target_label ], _java_library_patterns) == [] && - filter_exclude([ _target_label ], _java_resource_patterns) != []) { - # Depend on the java libraries' transitive __assetres target instead. - # This is required to ensure depending on java_groups works. - deps += [ "${_target_label}__assetres" ] - } else { - deps += [ _dep ] - } - } - } depfile = "$target_gen_dir/${invoker.target_name}.d" inputs = [ invoker.build_config ] outputs = [ invoker.srcjar_path ] @@ -686,6 +693,7 @@ testonly = true _test_name = invoker.test_name _test_type = invoker.test_type + _is_unit_test = defined(invoker.is_unit_test) && invoker.is_unit_test _incremental_apk = defined(invoker.incremental_apk) && invoker.incremental_apk _runtime_deps = @@ -719,7 +727,8 @@ if (defined(invoker.apk_under_test)) { _install_artifacts_json = "${target_gen_dir}/${target_name}.install_artifacts" - generated_file("${target_name}__install_artifacts") { + _install_artifacts_target_name = "${target_name}__install_artifacts" + generated_file(_install_artifacts_target_name) { deps = [ invoker.apk_under_test ] output_conversion = "json" outputs = [ _install_artifacts_json ] @@ -730,12 +739,18 @@ } generate_android_wrapper(target_name) { + forward_variables_from(invoker, + [ + "assert_no_deps", + "public_deps", + "visibility", + ]) wrapper_script = "$root_build_dir/bin/run_${_test_name}" executable = "//testing/test_env.py" - if (defined(android_test_runner_script)) { - _runner_script = android_test_runner_script + if (defined(invoker.android_test_runner_script)) { + _runner_script = invoker.android_test_runner_script } else { _runner_script = "//build/android/test_runner.py" } @@ -745,9 +760,12 @@ deps = invoker.deps } data_deps = [ - "//build/android:test_runner_py", + "//build/android:test_runner_core_py", "//testing:test_scripts_shared", ] + if (_test_type != "junit") { + data_deps += [ "//build/android:test_runner_device_support" ] + } if (defined(invoker.data_deps)) { data_deps += invoker.data_deps } @@ -761,8 +779,13 @@ _test_type, "--output-directory", "@WrappedPath(.)", + "--wrapper-script-args", ] + if (_is_unit_test) { + executable_args += [ "--is-unit-test" ] + } + if (_runtime_deps) { deps += [ ":$_runtime_deps_target" ] data += [ _runtime_deps_file ] @@ -781,7 +804,7 @@ deps += [ "${invoker.apk_target}$build_config_target_suffix" ] _apk_build_config = get_label_info(invoker.apk_target, "target_gen_dir") + "/" + - get_label_info(invoker.apk_target, "name") + ".build_config" + get_label_info(invoker.apk_target, "name") + ".build_config.json" _rebased_apk_build_config = rebase_path(_apk_build_config, root_build_dir) not_needed([ "_rebased_apk_build_config" ]) } else if (_test_type == "gtest") { @@ -818,24 +841,22 @@ if (_incremental_apk) { _test_apk = "@WrappedPath(@FileArg($_rebased_apk_build_config:deps_info:incremental_apk_path))" } - _rebased_test_jar = rebase_path(invoker.test_jar, root_build_dir) executable_args += [ "--test-apk", _test_apk, - "--test-jar", - "@WrappedPath(${_rebased_test_jar})", ] if (defined(invoker.apk_under_test)) { if (_incremental_apk) { deps += [ "${invoker.apk_under_test}$build_config_target_suffix" ] _apk_under_test_build_config = get_label_info(invoker.apk_under_test, "target_gen_dir") + "/" + - get_label_info(invoker.apk_under_test, "name") + ".build_config" + get_label_info(invoker.apk_under_test, "name") + + ".build_config.json" _rebased_apk_under_test_build_config = rebase_path(_apk_under_test_build_config, root_build_dir) _apk_under_test = "@WrappedPath(@FileArg($_rebased_apk_under_test_build_config:deps_info:incremental_apk_path))" } else { - deps += [ ":${target_name}__install_artifacts" ] + deps += [ ":${_install_artifacts_target_name}" ] _rebased_install_artifacts_json = rebase_path(_install_artifacts_json, root_build_dir) _apk_under_test = @@ -851,16 +872,25 @@ _build_config = get_label_info(invoker.use_webview_provider, "target_gen_dir") + "/" + get_label_info(invoker.use_webview_provider, "name") + - ".build_config" + ".build_config.json" _rebased_build_config = rebase_path(_build_config, root_build_dir) executable_args += [ "--use-webview-provider", "@WrappedPath(@FileArg($_rebased_build_config:deps_info:apk_path))", ] } - if (defined(invoker.proguard_enabled) && invoker.proguard_enabled && - !_incremental_apk) { - executable_args += [ "--enable-java-deobfuscation" ] + if (defined(invoker.proguard_mapping_path)) { + if (_incremental_apk) { + not_needed(invoker, [ "proguard_mapping_path" ]) + } else { + data += [ invoker.proguard_mapping_path ] + _rebased_mapping_path = + rebase_path(invoker.proguard_mapping_path, root_build_dir) + executable_args += [ + "--proguard-mapping-path", + "@WrappedPath($_rebased_mapping_path)", + ] + } } if (use_jacoco_coverage) { # Set a default coverage output directory (can be overridden by user @@ -878,15 +908,17 @@ executable_args += [ "--test-suite", invoker.test_suite, + "--native-libs-dir", + "@WrappedPath($_robolectric_libs_dir)", ] + # Test runner uses this generated wrapper script. + data += [ "$root_build_dir/bin/helper/${invoker.test_suite}" ] + deps += [ ":${invoker.test_suite}$build_config_target_suffix" ] - _junit_binary_build_config = - "${target_gen_dir}/${invoker.test_suite}.build_config" _rebased_robolectric_runtime_deps_dir = - rebase_path("$root_build_dir/lib.java/third_party/robolectric", - root_build_dir) + rebase_path("//third_party/robolectric/lib", root_build_dir) _rebased_resource_apk = rebase_path(invoker.resource_apk, root_build_dir) executable_args += [ "--resource-apk", @@ -916,8 +948,9 @@ if (defined(invoker.additional_apks)) { foreach(additional_apk, invoker.additional_apks) { deps += [ "$additional_apk$build_config_target_suffix" ] - _build_config = get_label_info(additional_apk, "target_gen_dir") + "/" + - get_label_info(additional_apk, "name") + ".build_config" + _build_config = + get_label_info(additional_apk, "target_gen_dir") + "/" + + get_label_info(additional_apk, "name") + ".build_config.json" _rebased_build_config = rebase_path(_build_config, root_build_dir) executable_args += [ "--additional-apk", @@ -982,9 +1015,6 @@ template("android_lint") { action_with_pydeps(target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - if (!defined(deps)) { - deps = [] - } # https://crbug.com/1098752 Fix for bot OOM (https://crbug.com/1098333). if (defined(java_cmd_pool_size)) { @@ -996,22 +1026,23 @@ # Lint requires generated sources and generated resources from the build. # Turbine __header targets depend on all generated sources, and the # __assetres targets depend on all generated resources. + deps = [] if (defined(invoker.deps)) { - foreach(_dep, invoker.deps) { - _target_label = get_label_info(_dep, "label_no_toolchain") - if (filter_exclude([ _target_label ], _java_library_patterns) == [] && - filter_exclude([ _target_label ], _java_resource_patterns) != - []) { - deps += [ - "${_target_label}__assetres", - "${_target_label}__header", - ] - } else { - # Keep non-java deps as they may generate files used only by lint. - # e.g. generated suppressions.xml files. - deps += [ _dep ] - } + _lib_deps = + filter_exclude(filter_include(invoker.deps, java_library_patterns), + java_resource_patterns) + foreach(_lib_dep, _lib_deps) { + # Expand //foo/java -> //foo/java:java + _lib_dep = get_label_info(_lib_dep, "label_no_toolchain") + deps += [ + "${_lib_dep}__assetres", + "${_lib_dep}__header", + ] } + + # Keep non-java deps as they may generate files used only by lint. + # e.g. generated suppressions.xml files. + deps += filter_exclude(invoker.deps, _lib_deps) } if (defined(invoker.min_sdk_version)) { @@ -1020,7 +1051,12 @@ _min_sdk_version = default_min_sdk_version } - _lint_binary_path = "$lint_android_sdk_root/cmdline-tools/latest/bin/lint" + if (defined(invoker.lint_jar_path)) { + _lint_jar_path = invoker.lint_jar_path + } else { + _lint_jar_path = _default_lint_jar_path + } + _cache_dir = "$root_build_dir/android_lint_cache" # Save generated xml files in a consistent location for debugging. @@ -1030,17 +1066,20 @@ script = "//build/android/gyp/lint.py" depfile = "$target_gen_dir/$target_name.d" inputs = [ - _lint_binary_path, + _lint_jar_path, + _custom_lint_jar_path, _backported_methods, ] args = [ "--target-name", - get_label_info(target_name, "label_no_toolchain"), + get_label_info(":${target_name}", "label_no_toolchain"), "--depfile", rebase_path(depfile, root_build_dir), - "--lint-binary-path", - rebase_path(_lint_binary_path, root_build_dir), + "--lint-jar-path", + rebase_path(_lint_jar_path, root_build_dir), + "--custom-lint-jar-path", + rebase_path(_custom_lint_jar_path, root_build_dir), "--cache-dir", rebase_path(_cache_dir, root_build_dir), "--lint-gen-dir", @@ -1056,6 +1095,8 @@ if (defined(invoker.skip_build_server) && invoker.skip_build_server) { # Nocompile tests need lint to fail through ninja. args += [ "--skip-build-server" ] + } else if (android_static_analysis == "build_server") { + args += [ "--use-build-server" ] } if (defined(invoker.lint_suppressions_file)) { @@ -1117,7 +1158,7 @@ # Lint requires all source and all resource files to be passed in the # same invocation for checks like UnusedResources. - "--java-sources=@FileArg($_rebased_build_config:deps_info:lint_java_sources)", + "--sources=@FileArg($_rebased_build_config:deps_info:lint_sources)", "--aars=@FileArg($_rebased_build_config:deps_info:lint_aars)", "--srcjars=@FileArg($_rebased_build_config:deps_info:lint_srcjars)", "--resource-sources=@FileArg($_rebased_build_config:deps_info:lint_resource_sources)", @@ -1137,12 +1178,7 @@ } template("proguard") { - forward_variables_from(invoker, - TESTONLY_AND_VISIBILITY + [ - "data", - "data_deps", - "public_deps", - ]) + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) _script = "//build/android/gyp/proguard.py" _deps = invoker.deps @@ -1159,13 +1195,25 @@ _mapping_path = "${invoker.output_path}.mapping" } - _enable_jdk_library_desugaring = enable_jdk_library_desugaring - if (defined(invoker.supports_jdk_library_desugaring) && - !invoker.supports_jdk_library_desugaring) { - _enable_jdk_library_desugaring = false + _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) + + # This is generally the apk name, and serves to identify the mapping + # file that would be required to deobfuscate a stacktrace. + _mapping_basename = get_path_info(_mapping_path, "name") + _version_code = "@FileArg($_rebased_build_config:deps_info:version_code)" + _package_name = "@FileArg($_rebased_build_config:deps_info:package_name)" + if (defined(invoker.package_name)) { + _package_name = invoker.package_name + } + if (defined(invoker.version_code)) { + _version_code = invoker.version_code } - _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) + # The Mapping ID is parsed to when uploading mapping files. + # See: https://crbug.com/1417308 + _source_file_template = + "chromium-$_mapping_basename-$android_channel-$_version_code" + _args = [ "--mapping-output", rebase_path(_mapping_path, root_build_dir), @@ -1175,33 +1223,17 @@ "@FileArg($_rebased_build_config:android:sdk_jars)", "--r8-path", rebase_path(_r8_path, root_build_dir), + "--package-name=$_package_name", + "--source-file", + _source_file_template, ] if (treat_warnings_as_errors) { _args += [ "--warnings-as-errors" ] } - if (defined(invoker.desugar_jars_paths)) { - _rebased_desugar_jars_paths = - rebase_path(invoker.desugar_jars_paths, root_build_dir) - args += [ "--classpath=${_rebased_desugar_jars_paths}" ] - } if ((!defined(invoker.proguard_enable_obfuscation) || invoker.proguard_enable_obfuscation) && enable_proguard_obfuscation) { - _proguard_sourcefile_suffix = "" - if (defined(invoker.proguard_sourcefile_suffix)) { - _proguard_sourcefile_suffix = "-${invoker.proguard_sourcefile_suffix}" - } - - # This is generally the apk name, and serves to identify the mapping - # file that would be required to deobfuscate a stacktrace. - _mapping_id = get_path_info(_mapping_path, "name") - _args += [ - "--enable-obfuscation", - "--sourcefile", - "chromium-${_mapping_id}${_proguard_sourcefile_suffix}", - ] - } else if (defined(invoker.proguard_sourcefile_suffix)) { - not_needed(invoker, [ "proguard_sourcefile_suffix" ]) + _args += [ "--enable-obfuscation" ] } if (defined(invoker.modules)) { @@ -1211,8 +1243,17 @@ _args += [ "--feature-name=${_feature_module.name}", "--dex-dest=@FileArg($_rebased_module_build_config:final_dex:path)", - "--feature-jars=@FileArg($_rebased_module_build_config:deps_info:device_classpath)", ] + + # The bundle's build config has the correct classpaths - the individual + # modules' build configs may double-use some jars. + if (defined(invoker.add_view_trace_events) && + invoker.add_view_trace_events) { + _args += [ "--feature-jars=@FileArg($_rebased_build_config:modules:${_feature_module.name}:trace_event_rewritten_device_classpath)" ] + } else { + _args += [ "--feature-jars=@FileArg($_rebased_build_config:modules:${_feature_module.name}:device_classpath)" ] + } + if (defined(_feature_module.uses_split)) { _args += [ "--uses-split=${_feature_module.name}:${_feature_module.uses_split}" ] } @@ -1236,55 +1277,24 @@ } _outputs += [ _mapping_path ] - if (defined(invoker.disable_r8_outlining) && invoker.disable_r8_outlining) { - _args += [ "--disable-outlining" ] - } - if (defined(invoker.enable_proguard_checks) && !invoker.enable_proguard_checks) { _args += [ "--disable-checks" ] } - if (defined(invoker.is_static_library) && invoker.is_static_library) { - _args += [ - "--extra-mapping-output-paths", - "@FileArg($_rebased_build_config:deps_info:static_library_proguard_mapping_output_paths)", - ] - } - - if (_enable_jdk_library_desugaring) { - _args += [ - "--desugar-jdk-libs-json", - rebase_path(_desugar_jdk_libs_json, root_build_dir), - ] - _inputs += [ _desugar_jdk_libs_json ] - - _args += [ - "--desugar-jdk-libs-jar", - rebase_path(_desugar_jdk_libs_jar, root_build_dir), - "--desugar-jdk-libs-configuration-jar", - rebase_path(_desugar_jdk_libs_configuration_jar, root_build_dir), - ] - _inputs += [ - _desugar_jdk_libs_jar, - _desugar_jdk_libs_configuration_jar, - ] - - _desugared_library_keep_rule_output_path = - "$target_gen_dir/$target_name.desugared_library_keep_rules.flags" - _args += [ - "--desugared-library-keep-rule-output", - rebase_path(_desugared_library_keep_rule_output_path, root_build_dir), - ] - } _ignore_desugar_missing_deps = defined(invoker.ignore_desugar_missing_deps) && invoker.ignore_desugar_missing_deps - if (!_ignore_desugar_missing_deps && !enable_bazel_desugar) { + if (!_ignore_desugar_missing_deps) { _args += [ "--show-desugar-default-interface-warnings" ] } - if (enable_java_asserts) { + if (defined(invoker.custom_assertion_handler)) { + _args += [ + "--assertion-handler", + invoker.custom_assertion_handler, + ] + } else if (enable_java_asserts) { # The default for generating dex file format is # --force-disable-assertions. _args += [ "--force-enable-assertions" ] @@ -1341,6 +1351,12 @@ _deps += [ ":$_expectations_target" ] } action_with_pydeps(target_name) { + forward_variables_from(invoker, + [ + "data", + "data_deps", + "public_deps", + ]) script = _script deps = _deps inputs = _inputs @@ -1364,7 +1380,7 @@ # # Variables # main_class: The class containing the program entry point. - # build_config: Path to .build_config for the jar (contains classpath). + # build_config: Path to .build_config.json for the jar (contains classpath). # script_name: Name of the script to generate. # wrapper_script_args: List of extra arguments to pass to the executable. # tiered_stop_at_level_one: Whether to pass --tiered-stop-at-level-one @@ -1376,6 +1392,11 @@ _main_class = invoker.main_class _build_config = invoker.build_config _script_name = invoker.script_name + if (defined(invoker.max_heap_size)) { + _max_heap_size = invoker.max_heap_size + } else { + _max_heap_size = "1G" + } script = "//build/android/gyp/create_java_binary_script.py" inputs = [ _build_config ] @@ -1387,28 +1408,37 @@ rebase_path(_java_script, root_build_dir), "--main-class", _main_class, - ] - args += [ "--classpath=@FileArg($_rebased_build_config:deps_info:host_classpath)", + "--max-heap-size=$_max_heap_size", ] + data = [] if (use_jacoco_coverage) { args += [ "--classpath", - rebase_path("//third_party/jacoco/lib/jacocoagent.jar", - root_build_dir), + rebase_path(_jacoco_host_jar, root_build_dir), ] - } - if (use_jacoco_coverage || !treat_warnings_as_errors) { - args += [ "--noverify" ] + data += [ _jacoco_host_jar ] } if (defined(invoker.tiered_stop_at_level_one) && invoker.tiered_stop_at_level_one) { args += [ "--tiered-stop-at-level-one" ] } + if (defined(invoker.extra_classpath_jars)) { + _rebased_extra_classpath_jars = + rebase_path(invoker.extra_classpath_jars, root_build_dir) + args += [ "--classpath=${_rebased_extra_classpath_jars}" ] + data += invoker.extra_classpath_jars + } if (defined(invoker.wrapper_script_args)) { args += [ "--" ] + invoker.wrapper_script_args } + if (defined(invoker.use_jdk_11) && invoker.use_jdk_11) { + args += [ "--use-jdk-11" ] + deps += [ "//third_party/jdk11:java_data" ] + } else { + deps += [ "//third_party/jdk:java_data" ] + } } } @@ -1428,7 +1458,7 @@ !defined(invoker.enable_multidex) || invoker.enable_multidex _enable_main_dex_list = _enable_multidex && _min_sdk_version < 21 _enable_desugar = !defined(invoker.enable_desugar) || invoker.enable_desugar - _desugar_needs_classpath = _enable_desugar && !enable_bazel_desugar + _desugar_needs_classpath = _enable_desugar # It's not safe to dex merge with libraries dex'ed at higher api versions. assert(!_is_dex_merging || _min_sdk_version >= default_min_sdk_version) @@ -1452,6 +1482,10 @@ assert(!(defined(invoker.apply_mapping) && !_proguard_enabled), "apply_mapping can only be specified if proguard is enabled.") + if (defined(invoker.custom_assertion_handler)) { + assert(_proguard_enabled, + "Proguard is required to support the custom assertion handler.") + } if (_enable_main_dex_list) { _main_dex_rules = "//build/android/main_dex_classes.flags" @@ -1467,23 +1501,23 @@ proguard(_proguard_target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ + "add_view_trace_events", "build_config", + "custom_assertion_handler", "data", "data_deps", "deps", - "desugar_jars_paths", - "disable_r8_outlining", "enable_proguard_checks", "expected_proguard_config", "expected_proguard_config_base", "ignore_desugar_missing_deps", - "is_static_library", "modules", + "package_name", "proguard_enable_obfuscation", "proguard_mapping_path", "proguard_sourcefile_suffix", - "supports_jdk_library_desugaring", "top_target_name", + "version_code", ]) inputs = [] if (defined(invoker.inputs)) { @@ -1499,17 +1533,12 @@ ] if (defined(invoker.has_apk_under_test) && invoker.has_apk_under_test) { args += [ "--input-paths=@FileArg($_rebased_build_config:deps_info:device_classpath_extended)" ] + } else if (defined(invoker.add_view_trace_events) && + invoker.add_view_trace_events && defined(invoker.modules)) { + args += [ "--input-paths=@FileArg($_rebased_build_config:deps_info:trace_event_rewritten_device_classpath)" ] } else { args += [ "--input-paths=@FileArg($_rebased_build_config:deps_info:device_classpath)" ] } - if (enable_bazel_desugar) { - deps += [ "//third_party/bazel/desugar:desugar_runtime_java" ] - inputs += [ _desugar_runtime_jar ] - args += [ - "--input-paths", - rebase_path(_desugar_runtime_jar, root_build_dir), - ] - } if (defined(invoker.proguard_args)) { args += invoker.proguard_args } @@ -1544,18 +1573,14 @@ } } else { # !_proguard_enabled _is_library = defined(invoker.is_library) && invoker.is_library + assert(!(defined(invoker.input_classes_filearg) && _is_library)) + assert(_is_library == defined(invoker.unprocessed_jar_path)) _input_class_jars = [] if (defined(invoker.input_class_jars)) { _input_class_jars = invoker.input_class_jars } _deps = invoker.deps - if (!_is_library && enable_bazel_desugar) { - # It would be more efficient to use the pre-dex'ed copy of the runtime, - # but it's easier to add it in this way. - _deps += [ "//third_party/bazel/desugar:desugar_runtime_java" ] - _input_class_jars += [ _desugar_runtime_jar ] - } if (_input_class_jars != []) { _rebased_input_class_jars = rebase_path(_input_class_jars, root_build_dir) @@ -1572,9 +1597,12 @@ depfile = "$target_gen_dir/$target_name.d" outputs = [ invoker.output ] inputs = [ - _r8_path, + _d8_path, _custom_d8_path, ] + if (defined(invoker.inputs)) { + inputs += invoker.inputs + } if (!_is_library) { # http://crbug.com/725224. Fix for bots running out of memory. @@ -1592,7 +1620,7 @@ rebase_path(outputs[0], root_build_dir), "--min-api=$_min_sdk_version", "--r8-jar-path", - rebase_path(_r8_path, root_build_dir), + rebase_path(_d8_path, root_build_dir), "--custom-d8-jar-path", rebase_path(_custom_d8_path, root_build_dir), @@ -1640,32 +1668,16 @@ if (defined(invoker.input_classes_filearg)) { inputs += [ invoker.build_config ] args += [ "--class-inputs-filearg=${invoker.input_classes_filearg}" ] + + # Required for the same reason as unprocessed_jar_path is added to + # classpath (see note below). + args += [ "--classpath=${invoker.input_classes_filearg}" ] } if (_input_class_jars != []) { inputs += _input_class_jars args += [ "--class-inputs=${_rebased_input_class_jars}" ] } - if (defined(invoker.dexlayout_profile)) { - args += [ - "--dexlayout-profile", - rebase_path(invoker.dexlayout_profile, root_build_dir), - "--dexlayout-path", - rebase_path(_dexlayout_path, root_build_dir), - "--profman-path", - rebase_path(_profman_path, root_build_dir), - "--dexdump-path", - rebase_path(_dexdump_path, root_build_dir), - ] - inputs += [ - _dexlayout_path, - _profman_path, - _dexdump_path, - invoker.dexlayout_profile, - ] - inputs += _default_art_libs - } - # Never compile intemediates with --release in order to: # 1) not require recompiles when toggling is_java_debug, # 2) allow incremental_install=1 to still have local variable @@ -1677,52 +1689,46 @@ if (_enable_desugar) { args += [ "--desugar" ] - # Passing the flag for dex merging causes invalid dex files to be created. - if (enable_jdk_library_desugaring && !_is_dex_merging) { - inputs += [ _desugar_jdk_libs_json ] - args += [ - "--desugar-jdk-libs-json", - rebase_path(_desugar_jdk_libs_json, root_build_dir), - ] - } _ignore_desugar_missing_deps = defined(invoker.ignore_desugar_missing_deps) && invoker.ignore_desugar_missing_deps - if (!_ignore_desugar_missing_deps && !enable_bazel_desugar) { + if (!_ignore_desugar_missing_deps) { args += [ "--show-desugar-default-interface-warnings" ] } } if (_desugar_needs_classpath) { + # Cannot use header jar for the active jar, because it does not + # contain anonymous classes. https://crbug.com/1342018#c5 + # Cannot use processed .jar here because it might have classes + # filtered out via jar_excluded_patterns. + # Must come first in classpath in order to take precedence over + # deps that defined the same classes (via jar_excluded_patterns). + if (defined(invoker.unprocessed_jar_path)) { + args += [ + "--classpath", + rebase_path(invoker.unprocessed_jar_path, root_build_dir), + + # Pass the full classpath to find new dependencies that are not in + # the .desugardeps file. + "--classpath=@FileArg($_rebased_build_config:deps_info:javac_full_interface_classpath)", + ] + inputs += [ invoker.unprocessed_jar_path ] + } _desugar_dependencies_path = "$target_gen_dir/$target_name.desugardeps" args += [ "--desugar-dependencies", rebase_path(_desugar_dependencies_path, root_build_dir), "--bootclasspath=@FileArg($_rebased_build_config:android:sdk_jars)", - - # Pass the full classpath to find new dependencies that are not in - # the .desugardeps file. - "--classpath=@FileArg($_rebased_build_config:deps_info:javac_full_interface_classpath)", ] - if (defined(invoker.desugar_jars_paths)) { - _rebased_desugar_jars_paths = - rebase_path(invoker.desugar_jars_paths, root_build_dir) - args += [ "--classpath=${_rebased_desugar_jars_paths}" ] - } - if (defined(invoker.final_ijar_path)) { - # Need to include the input .interface.jar on the classpath in order to make - # jar_excluded_patterns classes visible to desugar. - args += [ - "--classpath", - rebase_path(invoker.final_ijar_path, root_build_dir), - ] - inputs += [ invoker.final_ijar_path ] - } - } else { - not_needed(invoker, [ "desugar_jars_paths" ]) } - if (enable_java_asserts) { + if (defined(invoker.custom_assertion_handler)) { + args += [ + "--assertion-handler", + invoker.custom_assertion_handler, + ] + } else if (enable_java_asserts) { # The default for generating dex file format is # --force-disable-assertions. args += [ "--force-enable-assertions" ] @@ -1731,38 +1737,6 @@ } } - # Variables - # output: Path to output ".l8.dex". - # min_sdk_version: The minimum Android SDK version this target supports. - template("dex_jdk_libs") { - action_with_pydeps(target_name) { - script = "//build/android/gyp/dex_jdk_libs.py" - inputs = [ - _r8_path, - _desugar_jdk_libs_json, - _desugar_jdk_libs_jar, - _desugar_jdk_libs_configuration_jar, - ] - outputs = [ invoker.output ] - args = [ - "--r8-path", - rebase_path(_r8_path, root_build_dir), - "--desugar-jdk-libs-json", - rebase_path(_desugar_jdk_libs_json, root_build_dir), - "--desugar-jdk-libs-jar", - rebase_path(_desugar_jdk_libs_jar, root_build_dir), - "--desugar-jdk-libs-configuration-jar", - rebase_path(_desugar_jdk_libs_configuration_jar, root_build_dir), - "--output", - rebase_path(invoker.output, root_build_dir), - "--min-api=${invoker.min_sdk_version}", - ] - if (treat_warnings_as_errors) { - args += [ "--warnings-as-errors" ] - } - } - } - template("jacoco_instr") { action_with_pydeps(target_name) { forward_variables_from(invoker, @@ -1777,7 +1751,7 @@ _jacococli_jar = "//third_party/jacoco/lib/jacococli.jar" script = "//build/android/gyp/jacoco_instr.py" - inputs = invoker.java_files + [ + inputs = invoker.source_files + [ _jacococli_jar, invoker.input_jar_path, ] @@ -1792,8 +1766,8 @@ rebase_path(invoker.output_jar_path, root_build_dir), "--sources-json-file", rebase_path(_sources_json_file, root_build_dir), - "--java-sources-file", - rebase_path(invoker.java_sources_file, root_build_dir), + "--target-sources-file", + rebase_path(invoker.target_sources_file, root_build_dir), "--jacococli-jar", rebase_path(_jacococli_jar, root_build_dir), ] @@ -1809,7 +1783,12 @@ template("filter_jar") { action_with_pydeps(target_name) { script = "//build/android/gyp/filter_zip.py" - forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "deps" ]) + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "deps", + "data", + "data_deps", + ]) inputs = [ invoker.input_jar ] if (defined(invoker.inputs)) { inputs += invoker.inputs @@ -1824,8 +1803,6 @@ if (defined(invoker.jar_included_patterns)) { _jar_included_patterns = invoker.jar_included_patterns } - _strip_resource_classes = defined(invoker.strip_resource_classes) && - invoker.strip_resource_classes args = [ "--input", rebase_path(invoker.input_jar, root_build_dir), @@ -1834,176 +1811,59 @@ "--exclude-globs=${_jar_excluded_patterns}", "--include-globs=${_jar_included_patterns}", ] - if (_strip_resource_classes) { - inputs += [ invoker.build_config ] - _rebased_build_config = - rebase_path(invoker.build_config, root_build_dir) - args += [ "--strip-resource-classes-for=@FileArg($_rebased_build_config:javac:resource_packages)" ] - } } } - template("process_java_prebuilt") { + template("process_java_library") { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) - not_needed([ "_rebased_build_config" ]) - not_needed(invoker, [ "build_config_dep" ]) - - _deps = invoker.jar_deps _previous_output_jar = invoker.input_jar_path - # Create the .jar in lib.java for use by java_binary. - if (defined(invoker.host_jar_path)) { - if (defined(invoker.jacoco_instrument) && invoker.jacoco_instrument) { - _filter_jar_target_name = "${target_name}_host__filter_jar" - _filter_jar_output_jar = "$target_out_dir/$target_name.host_filter.jar" - } else { - _filter_jar_target_name = "${target_name}_host" - _filter_jar_output_jar = invoker.host_jar_path - } - filter_jar(_filter_jar_target_name) { - forward_variables_from(invoker, - [ - "jar_excluded_patterns", - "jar_included_patterns", - "strip_resource_classes", - ]) - deps = _deps - input_jar = _previous_output_jar - output_jar = _filter_jar_output_jar - inputs = [] - if (defined(strip_resource_classes) && strip_resource_classes) { - inputs += [ invoker.build_config ] - deps += [ invoker.build_config_dep ] - args += [ "--strip-resource-classes-for=@FileArg($_rebased_build_config:javac:resource_packages)" ] - } - if (defined(invoker.inputs)) { - inputs += invoker.inputs - deps += invoker.input_deps - } - } - - if (defined(invoker.jacoco_instrument) && invoker.jacoco_instrument) { - # Jacoco must run after desugar (or else desugar sometimes fails). - # It must run after filtering to avoid the same (filtered) class mapping - # to multiple .jar files. - # We run offline code coverage processing here rather than with a - # javaagent as the desired coverage data was not being generated. - # See crbug.com/1097815. - jacoco_instr("${target_name}_host") { - deps = [ ":$_filter_jar_target_name" ] + invoker.jar_deps - forward_variables_from(invoker, - [ - "java_files", - "java_sources_file", - ]) - - input_jar_path = _filter_jar_output_jar - output_jar_path = invoker.host_jar_path - } - } + if (invoker.jacoco_instrument) { + _filter_jar_target_name = "${target_name}__filter_jar" + _filter_jar_output_jar = "$target_out_dir/$target_name.filter.jar" + } else { + _filter_jar_target_name = target_name + _filter_jar_output_jar = invoker.output_jar_path } - if (defined(invoker.device_jar_path)) { - if (invoker.enable_desugar) { - _desugar_target = "${target_name}_device__desugar" - _desugar_output_jar = "$target_out_dir/$target_name.desugar.jar" + filter_jar(_filter_jar_target_name) { + forward_variables_from(invoker, + [ + "data", + "data_deps", + "jar_excluded_patterns", + "jar_included_patterns", + ]) + deps = invoker.deps + input_jar = _previous_output_jar + output_jar = _filter_jar_output_jar + } - action_with_pydeps(_desugar_target) { - script = "//build/android/gyp/desugar.py" - deps = _deps + invoker.classpath_deps - depfile = "$target_gen_dir/$target_name.d" - _desugar_jar = "//third_party/bazel/desugar/Desugar.jar" - - inputs = [ - invoker.build_config, - _previous_output_jar, - _desugar_jar, - ] - outputs = [ _desugar_output_jar ] - args = [ - "--desugar-jar", - rebase_path(_desugar_jar, root_build_dir), - "--input-jar", - rebase_path(_previous_output_jar, root_build_dir), - "--output-jar", - rebase_path(_desugar_output_jar, root_build_dir), - - # Temporarily using java_full_interface_classpath until classpath validation of targets - # is implemented, see http://crbug.com/885273 - "--classpath=@FileArg($_rebased_build_config:deps_info:javac_full_interface_classpath)", - "--bootclasspath=@FileArg($_rebased_build_config:android:sdk_interface_jars)", - "--depfile", - rebase_path(depfile, root_build_dir), - ] - if (defined(invoker.desugar_jars_paths)) { - _rebased_desugar_jars_paths = - rebase_path(invoker.desugar_jars_paths, root_build_dir) - args += [ "--classpath=${_rebased_desugar_jars_paths}" ] - } - if (treat_warnings_as_errors) { - args += [ "--warnings-as-errors" ] - } - } - - _deps = [] - _deps = [ ":$_desugar_target" ] - _previous_output_jar = _desugar_output_jar - } - - if (invoker.jacoco_instrument) { - _filter_jar_target_name = "${target_name}_device__filter_jar" - _filter_jar_output_jar = - "$target_out_dir/$target_name.device_filter.jar" - } else { - _filter_jar_target_name = "${target_name}_device" - _filter_jar_output_jar = invoker.device_jar_path - } - filter_jar(_filter_jar_target_name) { + if (invoker.jacoco_instrument) { + # Jacoco must run after desugar (or else desugar sometimes fails). + # It must run after filtering to avoid the same (filtered) class mapping + # to multiple .jar files. + # We run offline code coverage processing here rather than with a + # javaagent as the desired coverage data was not being generated. + # See crbug.com/1097815. + jacoco_instr(target_name) { + deps = [ ":$_filter_jar_target_name" ] + invoker.deps forward_variables_from(invoker, [ - "jar_excluded_patterns", - "jar_included_patterns", - "strip_resource_classes", + "source_files", + "target_sources_file", ]) - deps = _deps - input_jar = _previous_output_jar - output_jar = _filter_jar_output_jar - inputs = [] - if (defined(strip_resource_classes) && strip_resource_classes) { - inputs += [ invoker.build_config ] - deps += [ invoker.build_config_dep ] - args += [ "--strip-resource-classes-for=@FileArg($_rebased_build_config:javac:resource_packages)" ] - } - if (!defined(invoker.host_jar_path) && defined(invoker.inputs)) { - inputs += invoker.inputs - deps += invoker.input_deps - } - } - if (invoker.jacoco_instrument) { - # Jacoco must run after desugar (or else desugar sometimes fails). - # It must run after filtering to avoid the same (filtered) class mapping - # to multiple .jar files. - jacoco_instr("${target_name}_device") { - deps = [ ":$_filter_jar_target_name" ] + invoker.jar_deps - forward_variables_from(invoker, - [ - "java_files", - "java_sources_file", - ]) - - input_jar_path = _filter_jar_output_jar - output_jar_path = invoker.device_jar_path - } + input_jar_path = _filter_jar_output_jar + output_jar_path = invoker.output_jar_path } } } template("bytecode_processor") { action_with_pydeps(target_name) { - forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "data_deps" ]) _bytecode_checker_script = "$root_build_dir/bin/helper/bytecode_processor" script = "//build/android/gyp/bytecode_processor.py" inputs = [ @@ -2018,7 +1878,7 @@ _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) args = [ "--target-name", - get_label_info(target_name, "label_no_toolchain"), + get_label_info(":${target_name}", "label_no_toolchain"), "--script", rebase_path(_bytecode_checker_script, root_build_dir), "--gn-target=${invoker.target_label}", @@ -2030,7 +1890,10 @@ "--full-classpath-jars=@FileArg($_rebased_build_config:deps_info:javac_full_classpath)", "--full-classpath-gn-targets=@FileArg($_rebased_build_config:deps_info:javac_full_classpath_targets)", ] - if (invoker.requires_android) { + if (android_static_analysis == "build_server") { + args += [ "--use-build-server" ] + } + if (invoker.include_android_sdk) { args += [ "--sdk-classpath-jars=@FileArg($_rebased_build_config:android:sdk_jars)" ] } if (invoker.is_prebuilt) { @@ -2056,6 +1919,7 @@ inputs = [ invoker.build_config, invoker.input_manifest, + _manifest_merger_jar_path, ] outputs = [ invoker.output_manifest ] @@ -2064,9 +1928,8 @@ args = [ "--depfile", rebase_path(depfile, root_build_dir), - "--android-sdk-cmdline-tools", - rebase_path("${public_android_sdk_root}/cmdline-tools/latest", - root_build_dir), + "--manifest-merger-jar", + rebase_path(_manifest_merger_jar_path, root_build_dir), "--root-manifest", rebase_path(invoker.input_manifest, root_build_dir), "--output", @@ -2097,12 +1960,7 @@ # Input variables: # deps: Specifies the input dependencies for this target. # - # build_config: Path to the .build_config file corresponding to the target. - # - # resource_dirs (Deprecated): - # ** This is deprecated, please specify files using |sources| parameter ** - # List of directories containing Android resources, layout should be - # similar to what aapt -S <dir> expects. + # build_config: Path to the .build_config.json file corresponding to the target. # # sources: # List of input resource files. @@ -2132,6 +1990,7 @@ forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "deps", + "public_deps", "sources", ]) script = "//build/android/gyp/prepare_resources.py" @@ -2170,11 +2029,15 @@ if (defined(invoker.strip_drawables) && invoker.strip_drawables) { args += [ "--strip-drawables" ] } + if (defined(invoker.allow_missing_resources) && + invoker.allow_missing_resources) { + args += [ "--allow-missing-resources" ] + } } } # A template that is used to compile all resources needed by a binary - # (e.g. an android_apk or a junit_binary) into an intermediate .ar_ + # (e.g. an android_apk or a robolectric_binary) into an intermediate .ar_ # archive. It can also generate an associated .srcjar that contains the # final R.java sources for all resource packages the binary depends on. # @@ -2183,9 +2046,9 @@ # # deps: Specifies the input dependencies for this target. # - # build_config: Path to the .build_config file corresponding to the target. + # build_config: Path to the .build_config.json file corresponding to the target. # - # build_config_dep: Dep target to generate the .build_config file. + # build_config_dep: Dep target to generate the .build_config.json file. # # android_manifest: Path to root manifest for the binary. # @@ -2219,10 +2082,6 @@ # resources to put in the final output, even if aapt_locale_allowlist # is defined to a smaller subset. # - # support_zh_hk: (optional) - # If true, support zh-HK in Chrome on Android by using the resources - # from zh-TW. See https://crbug.com/780847. - # # aapt_locale_allowlist: (optional) # Restrict compiled locale-dependent resources to a specific allowlist. # NOTE: This is a list of Chromium locale names, not Android ones. @@ -2238,8 +2097,6 @@ # # resource_values_filter_rules: (optional) # - # no_xml_namespaces: (optional) - # # png_to_webp: (optional) # If true, convert all PNG resources (except 9-patch files) to WebP. # @@ -2260,29 +2117,15 @@ # Use resource IDs provided by another APK target when compiling resources # (via. "aapt2 link --stable-ids") # - # short_resource_paths: (optional) - # Rename the paths within a the apk to be randomly generated short - # strings to reduce binary size. - # - # strip_resource_names: (optional) - # Strip resource names from the resources table of the apk. # # Output variables: # arsc_output: Path to output .ap_ file (optional). # # proto_output: Path to output .proto.ap_ file (optional). # - # optimized_arsc_output: Path to optimized .ap_ file (optional). - # - # optimized_proto_output: Path to optimized .proto.ap_ file (optional). - # # r_text_out_path: (optional): # Path for the corresponding generated R.txt file. # - # resources_path_map_out_path: (optional): - # Path for the generated map between original resource paths and - # shortend resource paths. - # # proguard_file: (optional) # Path to proguard configuration file for this apk target. # @@ -2291,30 +2134,17 @@ template("compile_resources") { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - _deps = [ - invoker.android_sdk_dep, - invoker.build_config_dep, - ] + _deps = invoker.deps + [ + invoker.android_sdk_dep, + invoker.build_config_dep, + ] if (defined(invoker.android_manifest_dep)) { _deps += [ invoker.android_manifest_dep ] } - foreach(_dep, invoker.deps) { - _target_label = get_label_info(_dep, "label_no_toolchain") - if (filter_exclude([ _target_label ], _java_library_patterns) == [] && - filter_exclude([ _target_label ], _java_resource_patterns) != []) { - # Depend on the java libraries' transitive __assetres target instead. - _deps += [ "${_target_label}__assetres" ] - } else { - _deps += [ _dep ] - } - } if (defined(invoker.arsc_output)) { _arsc_output = invoker.arsc_output } - if (defined(invoker.optimized_arsc_output)) { - _optimized_arsc_output = invoker.optimized_arsc_output - } _final_srcjar_path = "${target_gen_dir}/${target_name}.srcjar" _script = "//build/android/gyp/compile_resources.py" @@ -2332,7 +2162,6 @@ rebase_path(android_sdk_tools_bundle_aapt2, root_build_dir), "--dependencies-res-zips=@FileArg($_rebased_build_config:deps_info:dependency_zips)", "--extra-res-packages=@FileArg($_rebased_build_config:deps_info:extra_package_names)", - "--extra-main-r-text-files=@FileArg($_rebased_build_config:deps_info:extra_main_r_text_files)", "--min-sdk-version=${invoker.min_sdk_version}", "--target-sdk-version=${invoker.target_sdk_version}", "--webp-cache-dir=obj/android-webp-cache", @@ -2346,9 +2175,6 @@ "--srcjar-out", rebase_path(_final_srcjar_path, root_build_dir), ] - if (defined(invoker.no_xml_namespaces) && invoker.no_xml_namespaces) { - _args += [ "--no-xml-namespaces" ] - } if (defined(invoker.version_code)) { _args += [ "--version-code", @@ -2382,36 +2208,6 @@ rebase_path(invoker.size_info_path, root_build_dir), ] } - if (defined(_optimized_arsc_output)) { - _outputs += [ _optimized_arsc_output ] - _args += [ - "--optimized-arsc-path", - rebase_path(_optimized_arsc_output, root_build_dir), - ] - } - if (defined(invoker.optimized_proto_output)) { - _outputs += [ invoker.optimized_proto_output ] - _args += [ - "--optimized-proto-path", - rebase_path(invoker.optimized_proto_output, root_build_dir), - ] - } - if (defined(invoker.resources_config_paths)) { - _inputs += invoker.resources_config_paths - _rebased_resource_configs = - rebase_path(invoker.resources_config_paths, root_build_dir) - _args += [ "--resources-config-paths=${_rebased_resource_configs}" ] - } - if (defined(invoker.short_resource_paths) && invoker.short_resource_paths) { - _args += [ "--short-resource-paths" ] - if (defined(invoker.resources_path_map_out_path)) { - _outputs += [ invoker.resources_path_map_out_path ] - _args += [ - "--resources-path-map-out-path", - rebase_path(invoker.resources_path_map_out_path, root_build_dir), - ] - } - } if (defined(invoker.r_java_root_package_name)) { _args += [ @@ -2420,10 +2216,6 @@ ] } - if (defined(invoker.strip_resource_names) && invoker.strip_resource_names) { - _args += [ "--strip-resource-names" ] - } - # Useful to have android:debuggable in the manifest even for Release # builds. Just omit it for officai if (debuggable_apks) { @@ -2535,11 +2327,8 @@ [ "--values-filter-rules=${invoker.resource_values_filter_rules}" ] } - if (defined(invoker.support_zh_hk) && invoker.support_zh_hk) { - _args += [ "--support-zh-hk" ] - } - if (defined(invoker.include_resource)) { + _inputs += [ invoker.include_resource ] _rebased_include_resources = rebase_path(invoker.include_resource, root_build_dir) _args += [ "--include-resources=$_rebased_include_resources" ] @@ -2624,19 +2413,21 @@ ] inputs += [ invoker.expected_android_manifest_base ] } + if (defined(invoker.expected_android_manifest_version_code_offset)) { + args += [ + "--verification-version-code-offset", + invoker.expected_android_manifest_version_code_offset, + ] + } + if (defined(invoker.expected_android_manifest_library_version_offset)) { + args += [ + "--verification-library-version-offset", + invoker.expected_android_manifest_library_version_offset, + ] + } if (fail_on_android_expectations) { args += [ "--fail-on-expectations" ] } - if (defined(invoker.extra_verification_manifest)) { - inputs += [ invoker.extra_verification_manifest ] - args += [ - "--extra-verification-manifest", - rebase_path(invoker.extra_verification_manifest, root_build_dir), - ] - if (defined(invoker.extra_verification_manifest_dep)) { - deps += [ invoker.extra_verification_manifest_dep ] - } - } } _deps += [ ":$_expectations_target" ] } @@ -2654,35 +2445,122 @@ } } + # A template that is used to optimize compiled resources using aapt2 optimize. + # + # proto_input_path: + # Path to input compiled .proto.ap_ file. + # + # short_resource_paths: (optional) + # Rename the paths within a the apk to be randomly generated short + # strings to reduce binary size. + # + # strip_resource_names: (optional) + # Strip resource names from the resources table of the apk. + # + # resources_configs_paths: (optional) + # List of resource configs to use for optimization. + # + # optimized_proto_output: + # Path to output optimized .proto.ap_ file. + # + # resources_path_map_out_path: (optional): + # Path for the generated map between original resource paths and + # shortened resource paths. + template("optimize_resources") { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + action_with_pydeps(target_name) { + forward_variables_from(invoker, [ "deps" ]) + script = "//build/android/gyp/optimize_resources.py" + outputs = [ invoker.optimized_proto_output ] + inputs = [ + android_sdk_tools_bundle_aapt2, + invoker.r_text_path, + invoker.proto_input_path, + ] + args = [ + "--aapt2-path", + rebase_path(android_sdk_tools_bundle_aapt2, root_build_dir), + "--r-text-in", + rebase_path(invoker.r_text_path, root_build_dir), + "--proto-path", + rebase_path(invoker.proto_input_path, root_build_dir), + "--optimized-proto-path", + rebase_path(invoker.optimized_proto_output, root_build_dir), + ] + + if (defined(invoker.resources_config_paths)) { + inputs += invoker.resources_config_paths + _rebased_resource_configs = + rebase_path(invoker.resources_config_paths, root_build_dir) + args += [ "--resources-config-paths=${_rebased_resource_configs}" ] + } + + if (defined(invoker.short_resource_paths) && + invoker.short_resource_paths) { + args += [ "--short-resource-paths" ] + if (defined(invoker.resources_path_map_out_path)) { + outputs += [ invoker.resources_path_map_out_path ] + args += [ + "--resources-path-map-out-path", + rebase_path(invoker.resources_path_map_out_path, root_build_dir), + ] + } + } + + if (defined(invoker.strip_resource_names) && + invoker.strip_resource_names) { + args += [ "--strip-resource-names" ] + } + } + } + + # A template that is used to find unused resources. template("unused_resources") { - _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) - _shrinker_dep = "//build/android/gyp/resources_shrinker:resources_shrinker" - _shrinker_script = "$root_build_dir/bin/helper/resources_shrinker" action_with_pydeps(target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "deps" ]) - script = "//build/android/gyp/resources_shrinker/shrinker.py" - inputs = [ - invoker.build_config, - invoker.proguard_mapping_path, - _shrinker_script, + script = "//build/android/gyp/unused_resources.py" + depfile = "$target_gen_dir/${target_name}.d" + _unused_resources_script = "$root_build_dir/bin/helper/unused_resources" + inputs = [ _unused_resources_script ] + outputs = [ + invoker.output_config, + invoker.output_r_txt, ] - outputs = [ invoker.output_config ] if (!defined(deps)) { deps = [] } - deps += [ _shrinker_dep ] + deps += [ "//build/android/unused_resources:unused_resources" ] + _rebased_module_build_config = + rebase_path(invoker.build_config, root_build_dir) args = [ "--script", - rebase_path(_shrinker_script, root_build_dir), - "--dependencies-res-zips=@FileArg($_rebased_build_config:deps_info:dependency_zips)", - "--proguard-mapping", - rebase_path(invoker.proguard_mapping_path, root_build_dir), - "--r-text=@FileArg($_rebased_build_config:deps_info:r_text_path)", - "--dex=@FileArg($_rebased_build_config:final_dex:path)", - "--android-manifest=@FileArg($_rebased_build_config:deps_info:android_manifest)", + rebase_path(_unused_resources_script, root_build_dir), "--output-config", rebase_path(invoker.output_config, root_build_dir), + "--r-text-in=@FileArg($_rebased_module_build_config:deps_info:r_text_path)", + "--r-text-out", + rebase_path(invoker.output_r_txt, root_build_dir), + "--dependencies-res-zips=@FileArg($_rebased_module_build_config:deps_info:dependency_zips)", + "--depfile", + rebase_path(depfile, root_build_dir), ] + + if (defined(invoker.proguard_mapping_path)) { + inputs += [ invoker.proguard_mapping_path ] + args += [ + "--proguard-mapping", + rebase_path(invoker.proguard_mapping_path, root_build_dir), + ] + } + + foreach(_build_config, invoker.all_module_build_configs) { + inputs += [ _build_config ] + _rebased_build_config = rebase_path(_build_config, root_build_dir) + args += [ + "--dexes=@FileArg($_rebased_build_config:final_dex:path)", + "--android-manifests=@FileArg($_rebased_build_config:deps_info:merged_android_manifest)", + ] + } } } @@ -2749,12 +2627,49 @@ } } + template("create_binary_profile") { + action_with_pydeps(target_name) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + forward_variables_from(invoker, [ "deps" ]) + script = "//build/android/gyp/binary_baseline_profile.py" + depfile = "$target_gen_dir/$target_name.d" + outputs = [ + invoker.binary_baseline_profile_path, + invoker.binary_baseline_profile_metadata_path, + ] + _profgen_path = "$android_sdk_root/cmdline-tools/latest/bin/profgen" + _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) + inputs = [ + invoker.build_config, + invoker.proguard_mapping_path, + invoker.input_profile_path, + _profgen_path, + ] + args = [ + "--profgen", + rebase_path(_profgen_path, root_build_dir), + "--output-profile", + rebase_path(invoker.binary_baseline_profile_path, root_build_dir), + "--output-metadata", + rebase_path(invoker.binary_baseline_profile_metadata_path, + root_build_dir), + "--dex=@FileArg($_rebased_build_config:final_dex:path)", + "--proguard-mapping", + rebase_path(invoker.proguard_mapping_path, root_build_dir), + "--input-profile-path", + rebase_path(invoker.input_profile_path, root_build_dir), + "--depfile", + rebase_path(depfile, root_build_dir), + ] + } + } + # Creates a signed and aligned .apk. # # Variables # apk_name: (optional) APK name (without .apk suffix). If provided, will # be used to generate .info files later used by the supersize tool. - # assets_build_config: Path to android_apk .build_config containing merged + # assets_build_config: Path to android_apk .build_config.json containing merged # asset information. # deps: Specifies the dependencies of this target. # dex_path: Path to classes.dex file to include (optional). @@ -2762,7 +2677,6 @@ # and assets is consistent with the given expectation file. # expected_libs_and_assets_base: Treat expected_libs_and_assets as a diff # with this file as the base. - # jdk_libs_dex: Path to classes.dex for desugar_jdk_libs. # packaged_resources_path: Path to .ap_ to use. # output_apk_path: Output path for the generated .apk. # min_sdk_version: The minimum Android SDK version this target supports. @@ -2779,10 +2693,10 @@ # keystore_path: Path to keystore to use for signing. # keystore_name: Key alias to use. # keystore_password: Keystore password. - # uncompress_shared_libraries: (optional, default false) Whether to store - # native libraries inside the APK uncompressed and page-aligned. template("package_apk") { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "public_deps" ]) + _is_robolectric_apk = + defined(invoker.is_robolectric_apk) && invoker.is_robolectric_apk _deps = invoker.deps _native_lib_placeholders = [] if (defined(invoker.native_lib_placeholders)) { @@ -2795,16 +2709,8 @@ } _script = "//build/android/gyp/apkbuilder.py" - _apksigner = "$android_sdk_build_tools/lib/apksigner.jar" - _zipalign = "$android_sdk_build_tools/zipalign" - _inputs = [ - invoker.build_config, - invoker.keystore_path, - invoker.packaged_resources_path, - _apksigner, - _zipalign, - ] + _inputs = [ invoker.packaged_resources_path ] _outputs = [ invoker.output_apk_path ] _data = [ invoker.output_apk_path ] @@ -2813,45 +2719,75 @@ rebase_path(invoker.packaged_resources_path, root_build_dir) _rebased_packaged_apk_path = rebase_path(invoker.output_apk_path, root_build_dir) - _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) _args = [ "--resource-apk=$_rebased_compiled_resources_path", "--output-apk=$_rebased_packaged_apk_path", - "--assets=@FileArg($_rebased_build_config:assets)", - "--uncompressed-assets=@FileArg($_rebased_build_config:uncompressed_assets)", - "--apksigner-jar", - rebase_path(_apksigner, root_build_dir), - "--zipalign-path", - rebase_path(_zipalign, root_build_dir), - "--key-path", - rebase_path(invoker.keystore_path, root_build_dir), - "--key-name", - invoker.keystore_name, - "--key-passwd", - invoker.keystore_password, "--min-sdk-version=${invoker.min_sdk_version}", - - # TODO(mlopatkin) We are relying on the fact that build_config is an APK - # build_config. - "--java-resources=@FileArg($_rebased_build_config:java_resources_jars)", ] - if (is_official_build) { - _args += [ "--best-compression" ] + + # system_image_stub_apk does not use a build_config.json. + if (defined(invoker.build_config)) { + _inputs += [ invoker.build_config ] + _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) + _args += [ + "--assets=@FileArg($_rebased_build_config:assets)", + "--uncompressed-assets=@FileArg($_rebased_build_config:uncompressed_assets)", + ] + if (!_is_robolectric_apk) { + _args += [ "--java-resources=@FileArg($_rebased_build_config:java_resources_jars)" ] + } } - if (defined(invoker.uncompress_dex) && invoker.uncompress_dex) { + if (defined(invoker.extra_assets)) { + _args += [ "--assets=${invoker.extra_assets}" ] + } + if (!_is_robolectric_apk) { + _apksigner = "$android_sdk_build_tools/lib/apksigner.jar" + _zipalign = "$android_sdk_build_tools/zipalign" + _keystore_path = android_keystore_path + _keystore_name = android_keystore_name + _keystore_password = android_keystore_password + + if (defined(invoker.keystore_path)) { + _keystore_path = invoker.keystore_path + _keystore_name = invoker.keystore_name + _keystore_password = invoker.keystore_password + } + + _inputs += [ + _apksigner, + _zipalign, + _keystore_path, + ] + _args += [ + "--apksigner-jar", + rebase_path(_apksigner, root_build_dir), + "--zipalign-path", + rebase_path(_zipalign, root_build_dir), + "--key-path", + rebase_path(_keystore_path, root_build_dir), + "--key-name", + _keystore_name, + "--key-passwd", + _keystore_password, + ] + if (is_official_build) { + _args += [ "--best-compression" ] + } + } + if (defined(invoker.uncompress_dex)) { + _uncompress_dex = invoker.uncompress_dex + } else { + # Uncompressed dex support started on Android P. + _uncompress_dex = invoker.min_sdk_version >= 28 + } + + if (_uncompress_dex) { _args += [ "--uncompress-dex" ] } - if (defined(invoker.uncompress_shared_libraries) && - invoker.uncompress_shared_libraries) { - _args += [ "--uncompress-shared-libraries=True" ] - } if (defined(invoker.library_always_compress)) { _args += [ "--library-always-compress=${invoker.library_always_compress}" ] } - if (defined(invoker.library_renames)) { - _args += [ "--library-renames=${invoker.library_renames}" ] - } if (defined(invoker.dex_path)) { _inputs += [ invoker.dex_path ] _args += [ @@ -2859,13 +2795,6 @@ rebase_path(invoker.dex_path, root_build_dir), ] } - if (defined(invoker.jdk_libs_dex)) { - _inputs += [ invoker.jdk_libs_dex ] - _args += [ - "--jdk-libs-dex-file", - rebase_path(invoker.jdk_libs_dex, root_build_dir), - ] - } if ((defined(invoker.loadable_modules) && invoker.loadable_modules != []) || defined(invoker.native_libs_filearg) || _native_lib_placeholders != []) { @@ -2913,10 +2842,10 @@ _failure_file = "$expectations_failure_dir/" + string_replace(invoker.expected_libs_and_assets, "/", "_") - inputs = [ - invoker.build_config, - invoker.expected_libs_and_assets, - ] + inputs = [ invoker.expected_libs_and_assets ] + if (defined(invoker.build_config)) { + inputs += [ invoker.build_config ] + } deps = [ invoker.build_config_dep ] outputs = [ _actual_file, @@ -2960,30 +2889,27 @@ } # Compile Java source files into a .jar file, potentially using an - # annotation processor, and/or the errorprone compiler. + # annotation processor, and/or the errorprone compiler. Also includes Kotlin + # source files in the resulting info file. # # Note that the only way to specify custom annotation processors is # by using build_config to point to a file that corresponds to a java-related # target that includes javac:processor_classes entries (i.e. there is no # variable here that can be used for this purpose). # - # Note also the peculiar use of java_files / java_sources_file. The content - # of the java_files list and the java_sources_file file must match exactly. - # This rule uses java_files only to list the inputs to the action that - # calls compile_java.py, but will pass the list of Java source files - # with the '@${java_sources_file}" command-line syntax. Not a problem in - # practice since this is only called from java_library_impl() that sets up - # the variables properly. + # Note also the peculiar use of source_files / target_sources_file. The content + # of the source_files list and the source files in target_sources_file file must + # match exactly. # # Variables: # main_target_name: Used when extracting srcjars for codesearch. - # java_files: Optional list of Java source file paths. + # source_files: Optional list of Java and Kotlin source file paths. # srcjar_deps: Optional list of .srcjar dependencies (not file paths). # The corresponding source files they contain will be compiled too. - # java_sources_file: Optional path to file containing list of Java source - # file paths. This must always be provided if java_files is not empty - # and must match it exactly. - # build_config: Path to the .build_config file of the corresponding + # target_sources_file: Optional path to file containing list of source file + # paths. This must always be provided if java_files is not empty and the + # .java files in it must match the list of java_files exactly. + # build_config: Path to the .build_config.json file of the corresponding # java_library_impl() target. The following entries will be used by this # template: javac:srcjars, deps_info:javac_full_classpath, # deps_info:javac_full_interface_classpath, javac:processor_classpath, @@ -3027,7 +2953,7 @@ _srcjar_deps = [] if (defined(invoker.srcjar_deps)) { - _srcjar_deps += invoker.srcjar_deps + _srcjar_deps = invoker.srcjar_deps } _java_srcjars = [] @@ -3038,11 +2964,8 @@ } # generated_jar_path is an output when use_turbine and an input otherwise. - if (!invoker.use_turbine && defined(invoker.generated_jar_path)) { - _annotation_processing = false + if (!invoker.use_turbine) { _java_srcjars += [ invoker.generated_jar_path ] - } else { - _annotation_processing = true } _javac_args = [] @@ -3059,7 +2982,7 @@ if (target_name == "chrome_java__header") { # Regression test for: https://crbug.com/1154302 - assert_no_deps = [ "//base:base_java__impl" ] + assert_no_deps = [ "//base:base_java__compile_java" ] } depfile = "$target_gen_dir/$target_name.d" @@ -3072,9 +2995,9 @@ if (!invoker.enable_errorprone && !invoker.use_turbine) { outputs += [ invoker.output_jar_path + ".info" ] } - inputs = invoker.java_files + _java_srcjars + [ _build_config ] - if (invoker.java_files != []) { - inputs += [ invoker.java_sources_file ] + inputs = invoker.source_files + _java_srcjars + [ _build_config ] + if (invoker.source_files != []) { + inputs += [ invoker.target_sources_file ] } _rebased_build_config = rebase_path(_build_config, root_build_dir) @@ -3090,8 +3013,15 @@ "--generated-dir=$_rebased_generated_dir", "--jar-path=$_rebased_output_jar_path", "--java-srcjars=$_rebased_java_srcjars", + "--target-name", + get_label_info(":${target_name}", "label_no_toolchain"), ] + # SDK jar must be first on classpath. + if (invoker.include_android_sdk) { + args += [ "--classpath=@FileArg($_rebased_build_config:android:sdk_interface_jars)" ] + } + if (defined(invoker.header_jar_path)) { inputs += [ invoker.header_jar_path ] args += [ @@ -3103,6 +3033,16 @@ args += [ "--classpath=$_header_jar_classpath" ] } + if (defined(invoker.kotlin_jar_path)) { + inputs += [ invoker.kotlin_jar_path ] + _rebased_kotlin_jar_path = + rebase_path(invoker.kotlin_jar_path, root_build_dir) + args += [ + "--kotlin-jar-path=$_rebased_kotlin_jar_path", + "--classpath=$_rebased_kotlin_jar_path", + ] + } + if (invoker.use_turbine) { # Prefer direct deps for turbine as much as possible. args += [ "--classpath=@FileArg($_rebased_build_config:javac:interface_classpath)" ] @@ -3110,7 +3050,7 @@ args += [ "--classpath=@FileArg($_rebased_build_config:deps_info:javac_full_interface_classpath)" ] } - if (_annotation_processing) { + if (invoker.use_turbine) { args += [ "--processorpath=@FileArg($_rebased_build_config:javac:processor_classpath)", "--processors=@FileArg($_rebased_build_config:javac:processor_classes)", @@ -3129,10 +3069,6 @@ ] } - # Currently turbine does not support JDK11. - if (invoker.supports_android || invoker.use_turbine) { - args += [ "--java-version=1.8" ] - } if (use_java_goma) { args += [ "--gomacc-path=$goma_dir/gomacc" ] @@ -3145,9 +3081,6 @@ if (enable_kythe_annotations && !invoker.enable_errorprone) { args += [ "--enable-kythe-annotations" ] } - if (invoker.requires_android) { - args += [ "--bootclasspath=@FileArg($_rebased_build_config:android:sdk_interface_jars)" ] - } if (_chromium_code) { args += [ "--chromium-code=1" ] if (treat_warnings_as_errors) { @@ -3165,10 +3098,9 @@ _dep_gen_dir = get_label_info(_errorprone_dep, "target_gen_dir") _dep_name = get_label_info(_errorprone_dep, "name") _rebased_errorprone_buildconfig = - rebase_path("$_dep_gen_dir/$_dep_name.build_config", root_build_dir) + rebase_path("$_dep_gen_dir/$_dep_name.build_config.json", + root_build_dir) args += [ - "--target-name", - get_label_info(target_name, "label_no_toolchain"), "--processorpath=@FileArg($_rebased_errorprone_buildconfig:deps_info:host_classpath)", "--enable-errorprone", ] @@ -3176,6 +3108,8 @@ if (defined(invoker.skip_build_server) && invoker.skip_build_server) { # Nocompile tests need lint to fail through ninja. args += [ "--skip-build-server" ] + } else if (android_static_analysis == "build_server") { + args += [ "--use-build-server" ] } foreach(e, _processor_args) { @@ -3189,8 +3123,9 @@ [ "--additional-jar-file=" + rebase_path(file_tuple[0], root_build_dir) + ":" + file_tuple[1] ] } - if (invoker.java_files != []) { - args += [ "@" + rebase_path(invoker.java_sources_file, root_build_dir) ] + if (invoker.source_files != []) { + args += + [ "@" + rebase_path(invoker.target_sources_file, root_build_dir) ] } foreach(e, _javac_args) { args += [ "--javac-arg=" + e ] @@ -3198,26 +3133,92 @@ } } - template("java_lib_group") { - forward_variables_from(invoker, [ "testonly" ]) - _group_name = invoker.group_name - not_needed([ "_group_name" ]) - group(target_name) { + # Compile Kotlin source files into .class files and store them in a .jar. + # This explicitly does not run annotation processing on the Kotlin files. + # Java files and srcjars are also passed to kotlinc for reference, although + # no .class files will be generated for any Java files. A subsequent call to + # javac will be required to actually compile Java files into .class files. + # + # This action also creates a "header" .jar file for the Kotlin source files. + # It is similar to using turbine to create headers for Java files, but since + # turbine does not support Kotlin files, this is done via a plugin for + # kotlinc instead, at the same time as compilation (whereas turbine is run as + # a separate action before javac compilation). + template("compile_kt") { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + + _build_config = invoker.build_config + _chromium_code = invoker.chromium_code + + _srcjar_deps = [] + if (defined(invoker.srcjar_deps)) { + _srcjar_deps = invoker.srcjar_deps + } + + _java_srcjars = [] + foreach(dep, _srcjar_deps) { + _dep_gen_dir = get_label_info(dep, "target_gen_dir") + _dep_name = get_label_info(dep, "name") + _java_srcjars += [ "$_dep_gen_dir/$_dep_name.srcjar" ] + } + + action_with_pydeps(target_name) { + script = "//build/android/gyp/compile_kt.py" + depfile = "$target_gen_dir/$target_name.d" + deps = _srcjar_deps if (defined(invoker.deps)) { - deps = [] - foreach(_dep, invoker.deps) { - _target_label = get_label_info(_dep, "label_no_toolchain") - if (filter_exclude([ _target_label ], _java_library_patterns) == [] && - filter_exclude([ _target_label ], _java_resource_patterns) != - []) { - # This is a java library dep, so replace it. - deps += [ "${_target_label}__${_group_name}" ] - } else { - # Transitive java group targets should also include direct deps. - deps += [ _dep ] - } + deps += invoker.deps + } + + outputs = [ + invoker.output_jar_path, + invoker.output_interface_jar_path, + ] + inputs = invoker.source_files + _java_srcjars + [ + _build_config, + invoker.target_sources_file, + ] + + _rebased_build_config = rebase_path(_build_config, root_build_dir) + _rebased_output_jar_path = + rebase_path(invoker.output_jar_path, root_build_dir) + _rebased_output_interface_jar_path = + rebase_path(invoker.output_interface_jar_path, root_build_dir) + _rebased_java_srcjars = rebase_path(_java_srcjars, root_build_dir) + _rebased_depfile = rebase_path(depfile, root_build_dir) + _rebased_generated_dir = rebase_path( + "$target_gen_dir/${invoker.main_target_name}/generated_java", + root_build_dir) + args = [ + "--depfile=$_rebased_depfile", + "--generated-dir=$_rebased_generated_dir", + "--jar-path=$_rebased_output_jar_path", + "--interface-jar-path=$_rebased_output_interface_jar_path", + "--java-srcjars=$_rebased_java_srcjars", + ] + + # SDK jar must be first on classpath. + if (invoker.include_android_sdk) { + args += [ "--classpath=@FileArg($_rebased_build_config:android:sdk_interface_jars)" ] + } + + args += [ "--classpath=@FileArg($_rebased_build_config:deps_info:javac_full_interface_classpath)" ] + + if (use_java_goma) { + args += [ "--gomacc-path=$goma_dir/gomacc" ] + + # Override the default action_pool when goma is enabled. + pool = "//build/config/android:goma_javac_pool" + } + + if (_chromium_code) { + args += [ "--chromium-code" ] + if (treat_warnings_as_errors) { + args += [ "--warnings-as-errors" ] } } + + args += [ "@" + rebase_path(invoker.target_sources_file, root_build_dir) ] } } @@ -3272,7 +3273,7 @@ # # Variables: # type: type of Java target, valid values: 'java_library', 'java_binary', - # 'junit_binary', 'java_annotation_processor', and 'android_apk' + # 'robolectric_binary', 'java_annotation_processor', and 'android_apk' # main_target_name: optional. If provided, overrides target_name when # creating sub-targets (e.g. "${main_target_name}__dex") and # some output files (e.g. "${main_target_name}.sources"). Only used @@ -3280,21 +3281,21 @@ # be the name of the main APK target. # supports_android: Optional. True if target can run on Android. # requires_android: Optional. True if target can only run on Android. - # java_files: Optional list of Java source file paths for this target. + # source_files: Optional list of Java source file paths for this target. # javac_args: Optional list of extra arguments to pass to javac. # errorprone_args: Optional list of extra arguments to pass to. # srcjar_deps: Optional list of .srcjar targets (not file paths). The Java # source files they contain will also be compiled for this target. - # java_sources_file: Optional path to a file which will be written with - # the content of java_files. If not provided, the file will be written + # target_sources_file: Optional path to a file which will be written with + # the content of source_files. If not provided, the file will be written # under $target_gen_dir/$main_target_name.sources. Ignored if - # java_files is empty. If not + # sources_files is empty. If not # jar_path: Optional path to a prebuilt .jar file for this target. # Mutually exclusive with java_files and srcjar_deps. # output_name: Optional output name for the final jar path. Used to # determine the name of the final jar. Default is to use the same # name as jar_path, if provided, or main_target_name. - # main_class: Main Java class name for 'java_binary', 'junit_binary' and + # main_class: Main Java class name for 'java_binary', 'robolectric_binary' and # 'java_annotation_processor' target types. Should not be set for other # ones. # deps: Dependencies for this target. @@ -3319,18 +3320,12 @@ # input_jars_paths: Optional list of additional .jar file paths, which will # be added to the compile-time classpath when building this target (but # not to the runtime classpath). - # desugar_jars_paths: Optional list of additional .jar file paths, which will - # be added to the desugar classpath when building this target (but not to - # any other classpath). This is only used to break dependency cycles. # gradle_treat_as_prebuilt: Cause generate_gradle.py to reference this # library via its built .jar rather than including its .java sources. # proguard_enabled: Optional. True to enable ProGuard obfuscation. # proguard_configs: Optional list of additional proguard config file paths. - # bypass_platform_checks: Optional. If True, platform checks will not - # be performed. They are used to verify that every target with - # requires_android only depends on targets that, at least supports_android. - # Similarly, if a target has !supports_android, then it cannot depend on - # any other target that has requires_android. + # is_robolectric: Optional. If True, this is a host side android test binary + # which is allowed to depend on other android targets. # include_java_resources: Optional. If True, include Java (not Android) # resources into final .jar file. # jar_excluded_patterns: Optional list of .class file patterns to exclude @@ -3366,8 +3361,6 @@ # be stored in the APK. # secondary_abi_loadable_modules: Optional list of native libraries for # secondary ABI. - # uncompress_shared_libraries: Optional. True to store native shared - # libraries uncompressed and page-aligned. # proto_resources_path: The path of an zip archive containing the APK's # resources compiled to the protocol buffer format (instead of regular # binary xml + resources.arsc). @@ -3379,7 +3372,7 @@ # list of string resources to keep in the base split APK for any bundle # that uses this target. # - # For 'java_binary' and 'junit_binary' targets only. Ignored by others: + # For 'java_binary' and 'robolectric_binary' targets only. Ignored by others: # # wrapper_script_name: Optional name for the generated wrapper script. # Default is main target name. @@ -3392,24 +3385,28 @@ forward_variables_from(invoker, [ "testonly" ]) _is_prebuilt = defined(invoker.jar_path) - _is_annotation_processor = invoker.type == "java_annotation_processor" - _is_java_binary = - invoker.type == "java_binary" || invoker.type == "junit_binary" + _type = invoker.type + _is_annotation_processor = _type == "java_annotation_processor" + _is_java_binary = _type == "java_binary" || _type == "robolectric_binary" + _is_library = _type == "java_library" _supports_android = defined(invoker.supports_android) && invoker.supports_android _requires_android = defined(invoker.requires_android) && invoker.requires_android + _supports_host = !_requires_android + if (_is_java_binary || _is_annotation_processor) { + assert(!_requires_android && !_supports_android) + } + + _bypass_platform_checks = defined(invoker.bypass_platform_checks) && + invoker.bypass_platform_checks + _is_robolectric = defined(invoker.is_robolectric) && invoker.is_robolectric _invoker_deps = [] if (defined(invoker.deps)) { _invoker_deps += invoker.deps } if (defined(invoker.public_deps)) { - foreach(_public_dep, invoker.public_deps) { - if (filter_include([ _public_dep ], _invoker_deps) != []) { - assert(false, "'public_deps' and 'deps' overlap: $_public_dep") - } - } _invoker_deps += invoker.public_deps } @@ -3418,19 +3415,16 @@ _main_target_name = invoker.main_target_name } - if (defined(invoker.resources_package)) { - _resources_package = invoker.resources_package + _source_files = [] + if (defined(invoker.sources)) { + _source_files = invoker.sources } - _java_files = [] - if (defined(invoker.sources)) { - _java_files = invoker.sources - } _srcjar_deps = [] if (defined(invoker.srcjar_deps)) { _srcjar_deps = invoker.srcjar_deps } - _has_sources = _java_files != [] || _srcjar_deps != [] + _has_sources = _source_files != [] || _srcjar_deps != [] if (_is_prebuilt) { assert(!_has_sources) @@ -3441,14 +3435,13 @@ } if (_is_java_binary) { - assert(defined(invoker.main_class), - "${invoker.type}() must set main_class") + assert(defined(invoker.main_class), "${_type}() must set main_class") } else if (_is_annotation_processor) { assert(defined(invoker.main_class), "java_annotation_processor() must set main_class") } else { assert(!defined(invoker.main_class), - "main_class cannot be used for target of type ${invoker.type}") + "main_class cannot be used for target of type ${_type}") } if (defined(invoker.chromium_code)) { @@ -3458,16 +3451,16 @@ _chromium_code = filter_exclude([ get_label_info(":$_main_target_name", "dir") ], [ "*\bthird_party\b*" ]) != [] - if (!_chromium_code && !_is_prebuilt && _java_files != []) { + if (!_chromium_code && !_is_prebuilt && _source_files != []) { # Unless third_party code has an org.chromium file in it. _chromium_code = - filter_exclude(_java_files, [ "*\bchromium\b*" ]) != _java_files + filter_exclude(_source_files, [ "*\bchromium\b*" ]) != _source_files } } # Define build_config_deps which will be a list of targets required to # build the _build_config. - _build_config = "$target_gen_dir/$_main_target_name.build_config" + _build_config = "$target_gen_dir/$_main_target_name.build_config.json" _build_config_target_name = "${_main_target_name}$build_config_target_suffix" @@ -3477,16 +3470,25 @@ if (_is_prebuilt || _has_sources) { if (defined(invoker.output_name)) { _output_name = invoker.output_name - } else if (_is_prebuilt) { - _output_name = get_path_info(invoker.jar_path, "name") } else { _output_name = _main_target_name } - _build_host_jar = _is_java_binary || _is_annotation_processor || - invoker.type == "java_library" - _build_device_jar = - invoker.type != "system_java_library" && _supports_android + _build_host_jar = + _is_java_binary || _is_annotation_processor || _type == "java_library" + _build_device_jar = _type != "system_java_library" && _supports_android + + _jacoco_instrument = + use_jacoco_coverage && _chromium_code && _source_files != [] && + _build_device_jar && (!defined(invoker.testonly) || !invoker.testonly) + if (defined(invoker.jacoco_never_instrument)) { + _jacoco_instrument = + !invoker.jacoco_never_instrument && _jacoco_instrument + } + if (_jacoco_instrument) { + _invoker_deps += [ _jacoco_dep ] + } + if (_build_host_jar) { # Jar files can be needed at runtime (by Robolectric tests or java binaries), # so do not put them under obj/. @@ -3497,16 +3499,27 @@ "$root_out_dir/lib.java$_target_dir_name/$_output_name.jar" } if (_build_device_jar) { - _device_processed_jar_path = - "$target_out_dir/$_output_name.processed.jar" _dex_path = "$target_out_dir/$_main_target_name.dex.jar" _enable_desugar = !defined(invoker.enable_desugar) || invoker.enable_desugar + + # Build speed optimization: Skip "process device" step if the step + # would be just a copy and avoid the copy. + _process_device_jar = + defined(invoker.bytecode_rewriter_target) || _jacoco_instrument || + defined(invoker.jar_excluded_patterns) || + defined(invoker.jar_included_patterns) + if (!_process_device_jar && _is_prebuilt) { + _device_processed_jar_path = invoker.jar_path + } else { + _device_processed_jar_path = + "$target_out_dir/$_output_name.processed.jar" + } } # For static libraries, the javac jar output is created at the intermediate # path so that it can be processed by another target and moved to the final - # spot that the .build_config knows about. Technically this should be done + # spot that the .build_config.json knows about. Technically this should be done # for the ijar as well, but this is only used for APK targets where # the ijar path isn't actually used. if (_has_sources) { @@ -3516,7 +3529,11 @@ } if (_has_sources) { - _javac_jar_path = "$target_out_dir/$_main_target_name.javac.jar" + if (_build_device_jar && !_process_device_jar) { + _javac_jar_path = _device_processed_jar_path + } else { + _javac_jar_path = "$target_out_dir/$_main_target_name.javac.jar" + } _generated_jar_path = "$target_gen_dir/$_main_target_name.generated.srcjar" } @@ -3528,87 +3545,89 @@ } } + _java_assetres_deps = filter_include(_invoker_deps, java_resource_patterns) + + # Cannot use minus operator because it does not work when the operand has + # repeated entries. + _invoker_deps_minus_assetres = + filter_exclude(_invoker_deps, _java_assetres_deps) + _lib_deps = + filter_include(_invoker_deps_minus_assetres, java_library_patterns) + _non_java_deps = filter_exclude(_invoker_deps_minus_assetres, _lib_deps) + + _java_header_deps = [] # Turbine / ijar + + # It would be more ideal to split this into __host and __javac, but we + # combine the two concepts to save on a group() target. + _java_host_deps = [] # Processed host .jar + javac .jar. + _java_validate_deps = [] # Bytecode checker & errorprone. + + foreach(_lib_dep, _lib_deps) { + # Expand //foo/java -> //foo/java:java + _lib_dep = get_label_info(_lib_dep, "label_no_toolchain") + _java_assetres_deps += [ "${_lib_dep}__assetres" ] + _java_header_deps += [ "${_lib_dep}__header" ] + _java_host_deps += [ "${_lib_dep}__host" ] + _java_validate_deps += [ "${_lib_dep}__validate" ] + } + + # APK and base module targets are special because: + # 1) They do not follow java target naming scheme (since they are not + # generally deps, there is no need for them to). + # 2) They do not bother to define a __host target. + # Since __host is used as an indirect dep for the compile_java artifacts, + # add the __compile_java target directly for them. + if (defined(invoker.apk_under_test)) { + _java_assetres_deps += [ "${invoker.apk_under_test}__java__assetres" ] + _java_header_deps += [ "${invoker.apk_under_test}__java__header" ] + _java_validate_deps += [ "${invoker.apk_under_test}__java__validate" ] + _java_host_deps += [ "${invoker.apk_under_test}__compile_java" ] + } + if (defined(invoker.base_module_target)) { + _java_assetres_deps += [ "${invoker.base_module_target}__java__assetres" ] + _java_header_deps += [ "${invoker.base_module_target}__java__header" ] + _java_validate_deps += [ "${invoker.base_module_target}__java__validate" ] + _java_host_deps += [ "${invoker.base_module_target}__compile_java" ] + } + + not_needed([ "_non_java_deps" ]) + if (_is_prebuilt || _has_sources) { - _java_res_deps = [] - _java_header_deps = [] - _java_impl_deps = [] - _non_java_deps = [] - foreach(_dep, _invoker_deps) { - _target_label = get_label_info(_dep, "label_no_toolchain") - if (filter_exclude([ _target_label ], _java_resource_patterns) == []) { - _java_res_deps += [ _dep ] - } else if (filter_exclude([ _target_label ], _java_library_patterns) == - []) { - # This is a java library dep, so it has header and impl targets. - _java_header_deps += [ "${_target_label}__header" ] - _java_impl_deps += [ "${_target_label}__impl" ] - } else { - _non_java_deps += [ _dep ] - } - } + # Classpath deps are used for header and dex targets, they do not need + # __assetres deps. + # _non_java_deps are needed for input_jars_paths that are generated. + _header_classpath_deps = + _java_header_deps + _non_java_deps + [ ":$_build_config_target_name" ] - # Don't need to depend on the apk-under-test to be packaged. - if (defined(invoker.apk_under_test)) { - _java_header_deps += [ "${invoker.apk_under_test}__java__header" ] - _java_impl_deps += [ "${invoker.apk_under_test}__java__impl" ] - } - - # These deps cannot be passed via invoker.deps since bundle_module targets - # have bundle_module.build_config without the __java suffix, so they are - # special and cannot be passed as regular deps to write_build_config. - if (defined(invoker.base_module_target)) { - _java_header_deps += [ "${invoker.base_module_target}__java__header" ] - _java_impl_deps += [ "${invoker.base_module_target}__java__impl" ] - } - - _extra_java_deps = [] - _jacoco_instrument = - use_jacoco_coverage && _chromium_code && _java_files != [] && - _build_device_jar && (!defined(invoker.testonly) || !invoker.testonly) - if (defined(invoker.jacoco_never_instrument)) { - _jacoco_instrument = - !invoker.jacoco_never_instrument && _jacoco_instrument - } - if (_jacoco_instrument) { - _extra_java_deps += [ "//third_party/jacoco:jacocoagent_java" ] - } + _javac_classpath_deps = + _java_host_deps + _non_java_deps + [ ":$_build_config_target_name" ] _include_android_sdk = _build_device_jar if (defined(invoker.include_android_sdk)) { _include_android_sdk = invoker.include_android_sdk } if (_include_android_sdk) { - _sdk_java_dep = "//third_party/android_sdk:android_sdk_java" if (defined(invoker.alternative_android_sdk_dep)) { - _sdk_java_dep = invoker.alternative_android_sdk_dep + _android_sdk_dep = invoker.alternative_android_sdk_dep + } else { + _android_sdk_dep = default_android_sdk_dep } - # This is an android_system_java_prebuilt target, so no headers. - _extra_java_deps += [ _sdk_java_dep ] + _header_classpath_deps += [ "${_android_sdk_dep}__header" ] + _javac_classpath_deps += [ "${_android_sdk_dep}" ] } - - # Classpath deps is used for header and dex targets, they do not need - # resource deps. - _classpath_deps = _java_header_deps + _non_java_deps + _extra_java_deps + - [ ":$_build_config_target_name" ] - - _full_classpath_deps = - _java_impl_deps + _java_res_deps + _non_java_deps + _extra_java_deps + - [ ":$_build_config_target_name" ] } # Often needed, but too hard to figure out when ahead of time. not_needed([ - "_classpath_deps", - "_full_classpath_deps", + "_header_classpath_deps", + "_javac_classpath_deps", ]) - if (_java_files != []) { - _java_sources_file = "$target_gen_dir/$_main_target_name.sources" - if (defined(invoker.java_sources_file)) { - _java_sources_file = invoker.java_sources_file - } - write_file(_java_sources_file, rebase_path(_java_files, root_build_dir)) + if (_source_files != []) { + _target_sources_file = "$target_gen_dir/$_main_target_name.sources" + write_file(_target_sources_file, + rebase_path(_source_files, root_build_dir)) } write_build_config(_build_config_target_name) { @@ -3619,21 +3638,28 @@ "base_allowlist_rtxt_path", "gradle_treat_as_prebuilt", "input_jars_paths", + "preferred_dep", "low_classpath_priority", "main_class", + "mergeable_android_manifests", + "module_name", + "parent_module_target", "proguard_configs", "proguard_enabled", "proguard_mapping_path", "public_target_label", "r_text_path", "type", + "version_code", + "version_name", ]) - if (type == "android_apk" || type == "android_app_bundle_module") { + if (_type == "android_apk" || _type == "android_app_bundle_module") { forward_variables_from( invoker, [ "android_manifest", "android_manifest_dep", + "merged_android_manifest", "final_dex_path", "loadable_modules", "native_lib_placeholders", @@ -3642,13 +3668,10 @@ "secondary_abi_shared_libraries_runtime_deps_file", "secondary_native_lib_placeholders", "shared_libraries_runtime_deps_file", - "static_library_dependent_targets", - "uncompress_shared_libraries", "library_always_compress", - "library_renames", ]) } - if (type == "android_apk") { + if (_type == "android_apk") { forward_variables_from(invoker, [ "apk_path", @@ -3657,15 +3680,13 @@ "incremental_install_json_path", ]) } - if (type == "android_app_bundle_module") { + if (_type == "android_app_bundle_module") { forward_variables_from(invoker, [ + "add_view_trace_events", "base_module_target", - "is_base_module", "module_pathmap_path", "proto_resources_path", - "version_name", - "version_code", ]) } chromium_code = _chromium_code @@ -3674,25 +3695,30 @@ # Specifically avoid passing in invoker.base_module_target as one of the # possible_config_deps. - possible_config_deps = _invoker_deps - if (defined(_extra_java_deps)) { - possible_config_deps += _extra_java_deps + possible_config_deps = [] + if (defined(invoker.deps)) { + possible_config_deps = invoker.deps + } + if (defined(invoker.public_deps)) { + possible_config_public_deps = invoker.public_deps } if (defined(apk_under_test)) { possible_config_deps += [ apk_under_test ] } - - if (defined(invoker.public_deps)) { - possible_config_public_deps = invoker.public_deps + if (defined(_jacoco_instrument) && _jacoco_instrument) { + possible_config_deps += [ _jacoco_dep ] + } + if (defined(_android_sdk_dep)) { + possible_config_deps += [ _android_sdk_dep ] } supports_android = _supports_android requires_android = _requires_android - bypass_platform_checks = defined(invoker.bypass_platform_checks) && - invoker.bypass_platform_checks + is_robolectric = _is_robolectric + bypass_platform_checks = _bypass_platform_checks - if (defined(_resources_package)) { - custom_package = _resources_package + if (defined(invoker.resources_package)) { + custom_package = invoker.resources_package } if (_is_prebuilt || _has_sources) { ijar_path = _final_ijar_path @@ -3705,8 +3731,8 @@ device_jar_path = _device_processed_jar_path dex_path = _dex_path } - if (_java_files != []) { - java_sources_file = _java_sources_file + if (_source_files != []) { + target_sources_file = _target_sources_file } bundled_srcjars = [] @@ -3731,9 +3757,10 @@ _header_target_name = "${target_name}__header" } - _public_deps = [] - _analysis_public_deps = [] if (_has_sources) { + _kt_files = filter_include(_source_files, [ "*.kt" ]) + _java_files = filter_exclude(_source_files, [ "*.kt" ]) + if (defined(invoker.enable_errorprone)) { _enable_errorprone = invoker.enable_errorprone } else { @@ -3741,31 +3768,56 @@ _java_files != [] && _chromium_code && use_errorprone_java_compiler } - _type = invoker.type + if (defined(invoker.resources_package) && _type == "java_library") { + # TODO(crbug.com/1296632): remove _bypass_platform_checks from the list + # once all robolectric targets have migrated to robolectric_library. + assert(_requires_android || _bypass_platform_checks || _is_robolectric, + "Setting resources_package applicable only for " + + "android_library(), or robolectric_library(). " + + "Target=$target_name") - _uses_fake_rjava = _type == "java_library" && _requires_android - - if (_uses_fake_rjava && defined(_resources_package)) { - # has _resources at the end so it looks like a resources pattern, since - # it does act like one (and other resources patterns need to depend on - # this before they can read its output R.txt). - _fake_rjava_target = "${target_name}__rjava_resources" - _possible_resource_deps = _invoker_deps + # Serves double purpose: Generating R.java, as well as being the + #__assetres target (instead of using a separate group). + _fake_rjava_target = "${target_name}__assetres" generate_r_java(_fake_rjava_target) { - deps = [ ":$_build_config_target_name" ] - if (defined(_possible_resource_deps)) { - possible_resource_deps = _possible_resource_deps - } + deps = [ ":$_build_config_target_name" ] + _java_assetres_deps + + _non_java_deps build_config = _build_config # Filepath has to be exactly this because compile_java looks for the # srcjar of srcjar_deps at this location $gen_dir/$target_name.srcjar srcjar_path = "$target_gen_dir/$target_name.srcjar" - package = _resources_package + package = invoker.resources_package } _srcjar_deps += [ ":$_fake_rjava_target" ] } + if (_kt_files != []) { + _kt_allowlist = [ + "android/java/src/org/chromium/chrome/browser/tabmodel/AsyncTabParamsManagerImpl.kt", + "webengine_shell_apk/src/org/chromium/webengine/shell/*.kt", + ] + assert(filter_exclude(_kt_files, _kt_allowlist) == [], + "Only a files in the allowlist can be included for now. Feel " + + "free to remove this assert when experimenting locally.") + _compile_kt_target_name = "${_main_target_name}__compile_kt" + _kotlinc_jar_path = "$target_out_dir/$_output_name.kotlinc.jar" + _kotlin_interface_jar_path = + "$target_out_dir/$_output_name.kt-jvm-abi.jar" + compile_kt(_compile_kt_target_name) { + deps = _header_classpath_deps + output_jar_path = _kotlinc_jar_path + output_interface_jar_path = _kotlin_interface_jar_path + main_target_name = _main_target_name + build_config = _build_config + srcjar_deps = _srcjar_deps + source_files = _source_files + target_sources_file = _target_sources_file + chromium_code = _chromium_code + include_android_sdk = _is_robolectric || _requires_android + } + } + template("compile_java_helper") { _enable_errorprone = defined(invoker.enable_errorprone) && invoker.enable_errorprone @@ -3779,11 +3831,22 @@ # Filtering out generated files resulted in no files left. group(target_name) { not_needed(invoker, "*") + deps = _header_classpath_deps } } else { compile_java(target_name) { - forward_variables_from(invoker, "*", TESTONLY_AND_VISIBILITY) + forward_variables_from(invoker, + "*", + TESTONLY_AND_VISIBILITY + [ "deps" ]) + deps = _header_classpath_deps + if (defined(invoker.deps)) { + deps += invoker.deps + } output_jar_path = invoker.output_jar_path + if (defined(invoker.kotlin_jar_path)) { + deps += [ ":$_compile_kt_target_name" ] + kotlin_jar_path = invoker.kotlin_jar_path + } enable_errorprone = _enable_errorprone use_turbine = defined(invoker.use_turbine) && invoker.use_turbine @@ -3791,22 +3854,17 @@ build_config = _build_config if (_enable_errorprone) { - java_files = _filtered_java_files + source_files = _filtered_java_files } else { - java_files = _java_files + source_files = _source_files srcjar_deps = _srcjar_deps } - if (java_files != []) { - java_sources_file = _java_sources_file + if (source_files != []) { + target_sources_file = _target_sources_file } chromium_code = _chromium_code - supports_android = _supports_android - requires_android = _requires_android - if (!defined(deps)) { - deps = [] - } - deps += _classpath_deps + include_android_sdk = _is_robolectric || _requires_android } } } @@ -3829,8 +3887,10 @@ output_jar_path = _final_ijar_path generated_jar_path = _generated_jar_path deps = _annotation_processor_deps + if (_kt_files != []) { + kotlin_jar_path = _kotlin_interface_jar_path + } } - _public_deps += [ ":$_header_target_name" ] _compile_java_target = "${_main_target_name}__compile_java" compile_java_helper(_compile_java_target) { @@ -3839,6 +3899,9 @@ deps = [ ":$_header_target_name" ] header_jar_path = _final_ijar_path generated_jar_path = _generated_jar_path + if (_kt_files != []) { + kotlin_jar_path = _kotlinc_jar_path + } } if (_enable_errorprone) { _compile_java_errorprone_target = "${_main_target_name}__errorprone" @@ -3852,18 +3915,23 @@ javac_args += invoker.errorprone_args } deps = [ ":$_header_target_name" ] + if (_kt_files != []) { + kotlin_jar_path = _kotlinc_jar_path + } header_jar_path = _final_ijar_path generated_jar_path = _generated_jar_path output_jar_path = "$target_out_dir/$target_name.errorprone.stamp" } - _analysis_public_deps += [ ":$_compile_java_errorprone_target" ] + _java_validate_deps += [ ":$_compile_java_errorprone_target" ] } } # _has_sources if (_is_prebuilt || _build_device_jar || _build_host_jar) { - _unprocessed_jar_deps = [] if (_has_sources) { - _unprocessed_jar_deps += [ ":$_compile_java_target" ] + _unprocessed_jar_deps = [ ":$_compile_java_target" ] + } else { + # jars might be generated by a dep. + _unprocessed_jar_deps = _non_java_deps } } @@ -3899,7 +3967,7 @@ "--output-jar", rebase_path(_rewritten_jar, root_build_dir), ] - deps = _unprocessed_jar_deps + _full_classpath_deps + + deps = _unprocessed_jar_deps + _javac_classpath_deps + [ invoker.bytecode_rewriter_target ] } @@ -3916,120 +3984,147 @@ input_jar = _unprocessed_jar_path output_jar = _final_ijar_path - # Normally ijar does not require any deps, but: - # 1 - Some jars are bytecode rewritten by _unprocessed_jar_deps. - # 2 - Other jars need to be unzipped by _non_java_deps. - # 3 - It is expected that depending on a header target implies depending - # on its transitive header target deps via _java_header_deps. - deps = _unprocessed_jar_deps + _non_java_deps + _java_header_deps + # ijar needs only _unprocessed_jar_deps, but this also needs to export + # __header target from deps. + deps = _unprocessed_jar_deps + _java_header_deps } - _public_deps += [ ":$_header_target_name" ] } if (_build_host_jar || _build_device_jar) { - _process_prebuilt_target_name = "${target_name}__process" - process_java_prebuilt(_process_prebuilt_target_name) { - forward_variables_from(invoker, - [ - "jar_excluded_patterns", - "jar_included_patterns", - ]) - build_config = _build_config - build_config_dep = ":$_build_config_target_name" - input_jar_path = _unprocessed_jar_path - jar_deps = _unprocessed_jar_deps + _full_classpath_deps - if (_build_host_jar) { - host_jar_path = _host_processed_jar_path - } - if (_build_device_jar) { - device_jar_path = _device_processed_jar_path - jacoco_instrument = _jacoco_instrument - if (_jacoco_instrument) { - java_files = _java_files - java_sources_file = _java_sources_file - } - enable_desugar = _enable_desugar && enable_bazel_desugar - if (enable_desugar) { - classpath_deps = _classpath_deps - forward_variables_from(invoker, [ "desugar_jars_paths" ]) - } - } - - # proguard_configs listed on java_library targets need to be marked - # as inputs to at least one action so that "gn analyze" will know - # about them. Although ijar doesn't use them, it's a convenient spot - # to list them. - # https://crbug.com/827197 - if (defined(invoker.proguard_configs)) { - inputs = invoker.proguard_configs - input_deps = _non_java_deps + _srcjar_deps # For the aapt-generated - # proguard rules. - } - } - if (_build_host_jar) { - _public_deps += [ ":${_process_prebuilt_target_name}_host" ] - } - if (_build_device_jar) { - _public_deps += [ ":${_process_prebuilt_target_name}_device" ] - } - - _enable_bytecode_checks = !defined(invoker.enable_bytecode_checks) || - invoker.enable_bytecode_checks + _enable_bytecode_checks = + (!defined(invoker.enable_bytecode_checks) || + invoker.enable_bytecode_checks) && android_static_analysis != "off" if (_enable_bytecode_checks) { - _bytecode_checks_target = "${target_name}__validate_classpath" - bytecode_processor(_bytecode_checks_target) { + _validate_target_name = "${target_name}__validate" + bytecode_processor(_validate_target_name) { forward_variables_from(invoker, [ "missing_classes_allowlist" ]) - deps = _unprocessed_jar_deps + _full_classpath_deps + + deps = _unprocessed_jar_deps + _javac_classpath_deps + [ ":$_build_config_target_name" ] - requires_android = _requires_android + data_deps = _java_validate_deps + if (defined(_compile_java_errorprone_target)) { + data_deps += [ ":$_compile_java_errorprone_target" ] + } + + include_android_sdk = _requires_android || _is_robolectric target_label = get_label_info(":${invoker.target_name}", "label_no_toolchain") input_jar = _unprocessed_jar_path build_config = _build_config is_prebuilt = _is_prebuilt } - _analysis_public_deps += [ ":$_bytecode_checks_target" ] + } else { + not_needed(invoker, [ "missing_classes_allowlist" ]) } - } - if (_build_device_jar) { - dex("${target_name}__dex") { - forward_variables_from(invoker, - [ - "desugar_jars_paths", - "proguard_enable_obfuscation", - ]) - input_class_jars = [ _device_processed_jar_path ] - enable_desugar = _enable_desugar - ignore_desugar_missing_deps = !_enable_bytecode_checks + if (_build_host_jar) { + _process_host_jar_target_name = "${target_name}__host" + process_java_library(_process_host_jar_target_name) { + forward_variables_from(invoker, + [ + "jar_excluded_patterns", + "jar_included_patterns", + ]) - # There's no value in per-class dexing prebuilts since they never - # change just one class at a time. - disable_incremental = _is_prebuilt - output = _dex_path - deps = [ ":${_process_prebuilt_target_name}_device" ] + # Robolectric tests require these to be on swarming. + data = [ _host_processed_jar_path ] + input_jar_path = _unprocessed_jar_path + deps = _unprocessed_jar_deps + _javac_classpath_deps + output_jar_path = _host_processed_jar_path + jacoco_instrument = _jacoco_instrument + if (_jacoco_instrument) { + source_files = _source_files + target_sources_file = _target_sources_file + } - if (enable_desugar && !enable_bazel_desugar) { - # Desugaring with D8 requires full classpath. - build_config = _build_config - final_ijar_path = _final_ijar_path - deps += _classpath_deps + [ ":$_header_target_name" ] + # _java_host_deps isn't necessary for process_java_library(), but is + # necessary so that this target can be used to depend on transitive + # __device targets without the need to create a separate group() + # target. This trade-off works because process_java_library is fast. + deps += _java_host_deps + + # Add runtime_deps here since robolectric_binary does not depend on top-level group. + if (defined(invoker.data)) { + data += invoker.data + } + if (defined(invoker.data_deps)) { + data_deps = invoker.data_deps + } + } + } + + if (_build_device_jar) { + if (_process_device_jar) { + _process_device_jar_target_name = "${target_name}__process_device" + process_java_library(_process_device_jar_target_name) { + forward_variables_from(invoker, + [ + "jar_excluded_patterns", + "jar_included_patterns", + ]) + input_jar_path = _unprocessed_jar_path + + deps = _unprocessed_jar_deps + _javac_classpath_deps + output_jar_path = _device_processed_jar_path + jacoco_instrument = _jacoco_instrument + if (_jacoco_instrument) { + source_files = _source_files + target_sources_file = _target_sources_file + } + } + _process_device_jar_deps = [ ":${_process_device_jar_target_name}" ] + } else { + assert(_unprocessed_jar_path == _device_processed_jar_path) + _process_device_jar_deps = _unprocessed_jar_deps } - enable_multidex = false - is_library = true + _dex_target_name = "${target_name}__dex" + dex(_dex_target_name) { + forward_variables_from(invoker, [ "proguard_enable_obfuscation" ]) + input_class_jars = [ _device_processed_jar_path ] + enable_desugar = _enable_desugar + ignore_desugar_missing_deps = !_enable_bytecode_checks + + # There's no value in per-class dexing prebuilts since they never + # change just one class at a time. + disable_incremental = _is_prebuilt + output = _dex_path + deps = _process_device_jar_deps + + if (enable_desugar) { + # Desugaring with D8 requires full classpath. + build_config = _build_config + unprocessed_jar_path = _unprocessed_jar_path + deps += _header_classpath_deps + _unprocessed_jar_deps + } + + enable_multidex = false + is_library = true + + # proguard_configs listed on java_library targets need to be marked + # as inputs to at least one target so that "gn analyze" will know + # about them. Although this target doesn't use them, it's a convenient spot + # to list them. + # https://crbug.com/827197 + if (compute_inputs_for_analyze && defined(invoker.proguard_configs)) { + inputs = invoker.proguard_configs + + # For the aapt-generated proguard rules. + deps += _non_java_deps + _srcjar_deps + } + } } - _public_deps += [ ":${target_name}__dex" ] } if (_is_java_binary) { # Targets might use the generated script while building, so make it a dep # rather than a data_dep. - java_binary_script("${target_name}__java_binary_script") { + _java_binary_script_target_name = "${target_name}__java_binary_script" + java_binary_script(_java_binary_script_target_name) { forward_variables_from(invoker, [ "tiered_stop_at_level_one", "main_class", + "max_heap_size", "wrapper_script_args", ]) build_config = _build_config @@ -4038,46 +4133,116 @@ script_name = invoker.wrapper_script_name } deps = [ ":$_build_config_target_name" ] - } - _public_deps += [ ":${target_name}__java_binary_script" ] - } + if (_is_robolectric) { + # For robolectric tests, we also add the normal sdk jar to the + # classpath since whenever we start using a new Android SDK, + # robolectric doesn't support it, and they often take a few months to + # support it. This causes issues when mocking classes that reference + # new SDK classes, so providing our normal SDK will allow these + # classes to resolve. For an example, see crbug.com/1350963. + extra_classpath_jars = [ android_sdk_jar ] - # The __impl target contains all non-analysis steps for this template. - # Having this separated out from the main target (which contains analysis - # steps) allows analysis steps for this target to be run concurrently with - # the non-analysis steps of other targets that depend on this one. - group("${target_name}__impl") { - public_deps = _public_deps - } - - java_lib_group("${target_name}__assetres") { - deps = _invoker_deps - group_name = "assetres" - - if (defined(_fake_rjava_target)) { - deps += [ ":$_fake_rjava_target" ] + # Mockito bug with JDK17 requires us to use JDK11 until we find a fix + # for crbug.com/1409661. + use_jdk_11 = true + } } } + if (!defined(_validate_target_name)) { + _validate_target_name = "${target_name}__validate" + + # Allow other targets to depend on this __validate one. + group(_validate_target_name) { + deps = _java_validate_deps + } + } + + if (_supports_host && !defined(_process_host_jar_target_name)) { + group("${target_name}__host") { + deps = _java_host_deps + } + } + + # robolectric_library can depend on java_library, so java_library must + # define __assetres. + if ((_is_library || _supports_android || _is_robolectric) && + !defined(_fake_rjava_target)) { + group("${target_name}__assetres") { + if (_supports_android || _is_robolectric) { + deps = _java_assetres_deps + } + } + } + + # The top-level group is used: + # 1) To allow building the target explicitly via ninja, + # 2) To trigger all analysis deps, + # 3) By custom action() targets that want to use artifacts as inputs. group(target_name) { forward_variables_from(invoker, [ "assert_no_deps", "data", "data_deps", - "deps", - "public_deps", "visibility", ]) - if (!defined(public_deps)) { - public_deps = [] - } - public_deps += [ ":${target_name}__impl" ] - if (defined(_analysis_public_deps)) { - if (!defined(data_deps)) { - data_deps = [] + if (_requires_android || (_supports_android && _is_library)) { + # For non-robolectric targets, depend on other java target's top-level + # groups so that the __dex step gets depended on. + forward_variables_from(invoker, + [ + "deps", + "public_deps", + ]) + if (!defined(deps)) { + deps = [] } - data_deps += _analysis_public_deps + if (!defined(public_deps)) { + public_deps = [] + } + } else { + # For robolectric targets, depend only on non-java deps and the specific + # subtargets below, which will not include __dex. + deps = _non_java_deps + public_deps = [] + if (defined(invoker.public_deps)) { + public_deps += + filter_exclude(invoker.public_deps, java_target_patterns) + } + } + if (defined(_jacoco_instrument) && _jacoco_instrument) { + deps += [ _jacoco_dep ] + } + if (defined(invoker.apk_under_test)) { + deps += [ invoker.apk_under_test ] + } + if (defined(_process_device_jar_target_name)) { + public_deps += [ ":$_process_device_jar_target_name" ] + } + if (defined(_dex_target_name)) { + public_deps += [ ":$_dex_target_name" ] + } + if (_supports_android && _is_library) { + # Robolectric targets define __assetres, but there's no need to build it + # by default. + public_deps += [ ":${target_name}__assetres" ] + } + if (_supports_host) { + # android_* targets define __host, but there's no need to build it by + # default. + public_deps += [ ":${target_name}__host" ] + } + if (_is_java_binary) { + public_deps += [ ":$_java_binary_script_target_name" ] + } + if (!defined(data_deps)) { + data_deps = [] + } + if (defined(_validate_target_name)) { + data_deps += [ ":$_validate_target_name" ] + } else { + data_deps += _java_validate_deps } } } @@ -4118,8 +4283,6 @@ _rebased_build_config = rebase_path(invoker.build_config, root_build_dir) _rebased_native_libraries_config = rebase_path(invoker.native_libraries_config, root_build_dir) - _proguard_enabled = - defined(invoker.proguard_enabled) && invoker.proguard_enabled forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) _deps = invoker.deps @@ -4130,7 +4293,7 @@ # by apkbuild.py --format=bundle-module. This means not using # apksigner and zipalign as well, nor the keystore. Other # dependencies like extra native libraries are all pulled from the - # .build_config through @FileArg() references (see below) and + # .build_config.json through @FileArg() references (see below) and # will be listed in the generated depfile instead. _inputs = [ invoker.build_config, @@ -4154,10 +4317,7 @@ ":native:secondary_native_library_placeholders)", "--android-abi=$android_app_abi", "--min-sdk-version=${invoker.min_sdk_version}", - "--uncompress-shared-libraries=@FileArg(" + - "$_rebased_build_config:native:uncompress_shared_libraries)", "--library-always-compress=@FileArg($_rebased_build_config:native:library_always_compress)", - "--library-renames=@FileArg($_rebased_build_config:native:library_renames)", ] if (defined(android_app_secondary_abi)) { _rebased_secondary_abi_native_libraries_config = @@ -4176,6 +4336,9 @@ if (defined(invoker.uncompress_dex) && invoker.uncompress_dex) { _args += [ "--uncompress-dex" ] } + if (defined(invoker.extra_assets)) { + _args += [ "--assets=${invoker.extra_assets}" ] + } # Use either provided dex path or build config path based on type of module. if (defined(invoker.dex_path)) { @@ -4186,20 +4349,6 @@ _args += [ "--dex-file=@FileArg($_rebased_build_config:final_dex:path)" ] } - # The library is imported via proguard when proguard is enabled. - if (!_proguard_enabled && enable_jdk_library_desugaring && - invoker.module_name == "base") { - _all_jdk_libs = "//build/android:all_jdk_libs" - _deps += [ _all_jdk_libs ] - _jdk_libs_dex = - get_label_info(_all_jdk_libs, "target_out_dir") + "/all_jdk_libs.l8.dex" - _inputs += [ _jdk_libs_dex ] - _args += [ - "--jdk-libs-dex-file", - rebase_path(_jdk_libs_dex, root_build_dir), - ] - } - if (treat_warnings_as_errors) { _args += [ "--warnings-as-errors" ] } @@ -4264,59 +4413,6 @@ } } -# Splits input dex file(s) based on given feature jars into seperate dex files -# for each feature. -# -# Variables: -# proguard_mapping: Path to input proguard mapping produced by synchronized -# proguarding. -# input_dex_zip: Path to zipped dex files to split. -# all_modules: Path to list of all modules. Each Module must have -# build_config, name, and build_config_target properties. -# feature_jars_args: Optional list of args to be passed to dexsplitter.py. -# If used should include the jars owned by each feature (in the same order -# as all_modules). Allows invoker to pull the list of jars from a different -# .build_config than the module's .build_config. -template("dexsplitter") { - action_with_pydeps(target_name) { - forward_variables_from(invoker, [ "deps" ]) - script = "//build/android/gyp/dexsplitter.py" - _stamp = "${target_gen_dir}/${target_name}.stamp" - outputs = [ _stamp ] - - depfile = "${target_gen_dir}/${target_name}.d" - args = [ - "--stamp", - rebase_path(_stamp, root_build_dir), - "--depfile", - rebase_path(depfile, root_build_dir), - "--r8-path", - rebase_path(_r8_path, root_build_dir), - "--input-dex-zip", - rebase_path(invoker.input_dex_zip, root_build_dir), - "--proguard-mapping-file", - rebase_path(invoker.proguard_mapping, root_build_dir), - ] - - foreach(_feature_module, invoker.all_modules) { - _rebased_module_build_config = - rebase_path(_feature_module.build_config, root_build_dir) - args += [ - "--feature-name", - _feature_module.name, - "--dex-dest=@FileArg($_rebased_module_build_config:final_dex:path)", - ] - if (!defined(invoker.feature_jars_args)) { - args += [ "--feature-jars=@FileArg($_rebased_module_build_config:deps_info:device_classpath)" ] - } - deps += [ _feature_module.build_config_target ] - } - if (defined(invoker.feature_jars_args)) { - args += invoker.feature_jars_args - } - } -} - # Allots native libraries depended on by feature modules to the module the # libraries should be packaged into. The packaging module may be different from # the dependee module in case a library is depended on by multiple modules. In
diff --git a/build/config/android/linker_version_script.gni b/build/config/android/linker_version_script.gni index 96d8b66..864233c 100644 --- a/build/config/android/linker_version_script.gni +++ b/build/config/android/linker_version_script.gni
@@ -1,7 +1,8 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/config/android/config.gni") import("//build/config/python.gni") # Generate a custom linker version script that can later be used with @@ -16,13 +17,17 @@ # template("generate_linker_version_script") { action_with_pydeps(target_name) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) script = "//build/android/gyp/generate_linker_version_script.py" outputs = [ invoker.linker_script ] inputs = [] args = [ "--output=" + rebase_path(invoker.linker_script, root_build_dir) ] - if (defined(invoker.export_java_symbols) && invoker.export_java_symbols) { - args += [ "--export-java-symbols" ] + if (defined(invoker.testonly) && invoker.testonly) { + args += [ "--export-fortesting-java-symbols" ] + } + if (allow_jni_multiplexing) { + args += [ "--jni-multiplexing" ] } if (defined(invoker.export_feature_registrations) &&
diff --git a/build/config/android/rules.gni b/build/config/android/rules.gni index e52396b..a3eccbf 100644 --- a/build/config/android/rules.gni +++ b/build/config/android/rules.gni
@@ -1,24 +1,29 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # Do not add any imports to non-//build directories here. # Some projects (e.g. V8) do not have non-build directories DEPS'ed in. - -import("//build/config/android/channel.gni") import("//build/config/android/config.gni") -import("//build/config/android/internal_rules.gni") +import("//build/config/android/copy_ex.gni") import("//build/config/clang/clang.gni") import("//build/config/compiler/compiler.gni") import("//build/config/coverage/coverage.gni") import("//build/config/python.gni") +import("//build/config/rts.gni") +import("//build/config/sanitizers/sanitizers.gni") import("//build/config/zip.gni") import("//build/toolchain/toolchain.gni") +assert(is_android || is_robolectric) -assert(is_android) - -declare_args() { - enable_jni_tracing = false +# Use a dedicated include dir so that files can #include headers from other +# toolchains without affecting non-JNI #includes. +if (target_os == "android") { + jni_headers_dir = "$root_build_dir/gen/jni_headers" +} else { + # Chrome OS builds cannot share gen/ directories because is_android=false + # within default_toolchain. + jni_headers_dir = "$root_gen_dir/jni_headers" } if (target_cpu == "arm") { @@ -34,6 +39,9 @@ _sanitizer_runtimes = [ "$clang_base_path/lib/clang/$clang_version/lib/linux/libclang_rt.ubsan_standalone-$_sanitizer_arch-android.so" ] } +_BUNDLETOOL_JAR_PATH = + "//third_party/android_build_tools/bundletool/bundletool.jar" + # Creates a dist directory for a native executable. # # Running a native executable on a device requires all the shared library @@ -120,11 +128,15 @@ } if (enable_java_templates) { - import("//build/config/sanitizers/sanitizers.gni") + if (is_android) { + import("//build/config/android/internal_rules.gni") + } # JNI target implementation. See generate_jni or generate_jar_jni for usage. template("generate_jni_impl") { - _jni_output_dir = "${target_gen_dir}/${target_name}" + _prev_jni_output_dir = "$target_gen_dir/$target_name" + _subdir = rebase_path(target_gen_dir, root_gen_dir) + _jni_output_dir = "$jni_headers_dir/$_subdir/$target_name" if (defined(invoker.jni_generator_include)) { _jni_generator_include = invoker.jni_generator_include _jni_generator_include_deps = [] @@ -156,37 +168,58 @@ public_deps = [] } public_deps += _jni_generator_include_deps + inputs = [] args = [ "--ptr_type=long", + + # TODO(agrieve): --prev_output_dir used only to make incremental builds + # work. Remove --prev_output_dir at some point after 2022. + "--prev_output_dir", + rebase_path(_prev_jni_output_dir, root_build_dir), + "--output_dir", + rebase_path(_jni_output_dir, root_build_dir), "--includes", rebase_path(_jni_generator_include, _jni_output_dir), ] if (defined(invoker.classes)) { - if (defined(invoker.jar_file)) { - _jar_file = invoker.jar_file + if (is_robolectric) { + not_needed(invoker, [ "jar_file" ]) } else { - _jar_file = android_sdk_jar + if (defined(invoker.jar_file)) { + _jar_file = invoker.jar_file + } else { + _jar_file = android_sdk_jar + } + inputs += [ _jar_file ] + args += [ + "--jar_file", + rebase_path(_jar_file, root_build_dir), + ] } - inputs += [ _jar_file ] - args += [ - "--jar_file", - rebase_path(_jar_file, root_build_dir), - ] _input_args = invoker.classes _input_names = invoker.classes if (defined(invoker.always_mangle) && invoker.always_mangle) { args += [ "--always_mangle" ] } + if (defined(invoker.unchecked_exceptions) && + invoker.unchecked_exceptions) { + args += [ "--unchecked_exceptions" ] + } } else { assert(defined(invoker.sources)) inputs += invoker.sources _input_args = rebase_path(invoker.sources, root_build_dir) _input_names = invoker.sources - if (use_hashed_jni_names) { + if (!is_robolectric && use_hashed_jni_names) { args += [ "--use_proxy_hash" ] } + + if (!is_robolectric && defined(invoker.enable_jni_multiplexing) && + invoker.enable_jni_multiplexing) { + args += [ "--enable_jni_multiplexing" ] + } if (defined(invoker.namespace)) { args += [ "-n ${invoker.namespace}" ] } @@ -197,17 +230,16 @@ outputs = [] foreach(_name, _input_names) { - _name_part = get_path_info(_name, "name") - outputs += [ "${_jni_output_dir}/${_name_part}_jni.h" ] - } + _name = get_path_info(_name, "name") + "_jni.h" + outputs += [ "$_jni_output_dir/$_name" ] - # Avoid passing GN lists because not all webrtc embedders use //build. - foreach(_output, outputs) { + # Avoid passing GN lists because not all webrtc embedders use //build. args += [ - "--output_file", - rebase_path(_output, root_build_dir), + "--output_name", + _name, ] } + foreach(_input, _input_args) { args += [ "--input_file=$_input" ] } @@ -215,8 +247,35 @@ if (enable_profiling) { args += [ "--enable_profiling" ] } - if (enable_jni_tracing) { - args += [ "--enable_tracing" ] + if (current_toolchain != default_toolchain && target_os == "android") { + # Rather than regenerating .h files in secondary toolchains, re-use the + # ones from the primary toolchain by depending on it and adding the + # root gen directory to the include paths. + # https://crbug.com/1369398 + inputs = [] + outputs = [] + _stamp = "$target_gen_dir/$target_name.stamp" + outputs = [ _stamp ] + + # Since we used to generate the .h files rather than delegate, the + # script will delete all .h files it finds in --prev_output_dir. + # TODO(agrieve): --prev_output_dir used only to make incremental builds + # work. Convert to group() target at some point after 2022. + args += [ + "--stamp", + rebase_path(_stamp, root_build_dir), + ] + deps = [] + public_deps = [] + public_deps = [ ":$target_name($default_toolchain)" ] + public_configs = + [ "//build/config/android:jni_include_dir($default_toolchain)" ] + } else { + public_configs = [ "//build/config/android:jni_include_dir" ] + if (defined(visibility)) { + # Allow dependency on ourselves from secondary toolchain. + visibility += [ ":$target_name" ] + } } } } @@ -264,6 +323,8 @@ # android.jar # always_mangle: Mangle all generated method names. By default, the script # only mangles methods that cause ambiguity due to method overload. + # unchecked_exceptions: Don't CHECK() for exceptions in generated stubs. + # This behaves as if every method had @CalledByNativeUnchecked. # deps, public_deps: As normal # # Example @@ -281,7 +342,10 @@ forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) } } +} # enable_java_templates +# non-robolectric things +if (enable_java_templates && is_android) { # Declare a jni registration target. # # This target generates a srcjar containing a copy of GEN_JNI.java, which has @@ -296,9 +360,10 @@ # about the format of the header file. # # Variables - # targets: List of .build_config supported targets to provide java sources. - # header_output: Path to the generated .h file (optional). - # sources_exclusions: List of .java files that should be skipped. (optional) + # targets: List of .build_config.json supported targets to provide java sources. + # manual_jni_registration: Manually do JNI registration - required for feature + # splits which provide their own native library. (optional) + # file_exclusions: List of .java files that should be skipped. (optional) # namespace: Registration functions will be wrapped into this. (optional) # require_native_mocks: Enforce that any native calls using # org.chromium.base.annotations.NativeMethods must have a mock set @@ -313,8 +378,8 @@ # Example # generate_jni_registration("chrome_jni_registration") { # targets = [ ":chrome_public_apk" ] - # header_output = "$target_gen_dir/$target_name.h" - # sources_exclusions = [ + # manual_jni_registration = false + # file_exclusions = [ # "//path/to/Exception.java", # ] # } @@ -340,50 +405,84 @@ _build_config = get_label_info("${_target}($default_toolchain)", "target_gen_dir") + "/" + get_label_info("${_target}($default_toolchain)", "name") + - ".build_config" + ".build_config.json" _rebased_build_config = rebase_path(_build_config, root_build_dir) inputs += [ _build_config ] if (defined(invoker.no_transitive_deps) && invoker.no_transitive_deps) { - args += [ "--sources-files=@FileArg($_rebased_build_config:deps_info:java_sources_file)" ] + args += [ "--sources-files=@FileArg($_rebased_build_config:deps_info:target_sources_file)" ] } else { args += [ # This is a list of .sources files. - "--sources-files=@FileArg($_rebased_build_config:deps_info:jni:all_source)", + "--sources-files=@FileArg($_rebased_build_config:deps_info:jni_all_source)", ] } } + if (defined(invoker.include_testonly)) { + _include_testonly = invoker.include_testonly + } else { + _include_testonly = defined(testonly) && testonly + } + if (_include_testonly) { + args += [ "--include-test-only" ] + } if (use_hashed_jni_names) { - args += [ "--use_proxy_hash" ] + args += [ "--use-proxy-hash" ] } if (defined(invoker.enable_native_mocks) && invoker.enable_native_mocks) { - args += [ "--enable_proxy_mocks" ] + args += [ "--enable-proxy-mocks" ] if (defined(invoker.require_native_mocks) && invoker.require_native_mocks) { - args += [ "--require_mocks" ] + args += [ "--require-mocks" ] } } - if (defined(invoker.header_output)) { - outputs += [ invoker.header_output ] - args += [ - "--header-path", - rebase_path(invoker.header_output, root_build_dir), - ] + _manual_jni_registration = defined(invoker.manual_jni_registration) && + invoker.manual_jni_registration + _enable_jni_multiplexing = defined(invoker.enable_jni_multiplexing) && + invoker.enable_jni_multiplexing + if (_manual_jni_registration) { + args += [ "--manual-jni-registration" ] + } + if (_enable_jni_multiplexing) { + args += [ "--enable-jni-multiplexing" ] } - if (defined(invoker.sources_exclusions)) { - _rebase_sources_exclusions = - rebase_path(invoker.sources_exclusions, root_build_dir) - args += [ "--sources-exclusions=$_rebase_sources_exclusions" ] + if ((!defined(invoker.prevent_header_output) || + !invoker.prevent_header_output) && + (_manual_jni_registration || _enable_jni_multiplexing)) { + assert(current_toolchain == default_toolchain, + "We do not need >1 toolchain copies of the same header.") + + _subdir = rebase_path(target_gen_dir, root_gen_dir) + _jni_header_output = + "$jni_headers_dir/$_subdir/${target_name}_generated.h" + outputs += [ _jni_header_output ] + args += [ + "--header-path", + rebase_path(_jni_header_output, root_build_dir), + ] + + # This gives targets depending on this registration access to our generated header. + public_configs = [ "//build/config/android:jni_include_dir" ] + } + + if (defined(invoker.file_exclusions)) { + _rebase_file_exclusions = + rebase_path(invoker.file_exclusions, root_build_dir) + args += [ "--file-exclusions=$_rebase_file_exclusions" ] } if (defined(invoker.namespace)) { args += [ "--namespace=${invoker.namespace}" ] } + + if (defined(invoker.module_name)) { + args += [ "--module-name=${invoker.module_name}" ] + } } } @@ -585,8 +684,8 @@ # foo_features.cc: # # // A feature. - # const base::Feature kSomeFeature{"SomeFeature", - # base::FEATURE_DISABLED_BY_DEFAULT}; + # BASE_FEATURE(kSomeFeature, "SomeFeature", + # base::FEATURE_DISABLED_BY_DEFAULT); # # FooFeatures.java.tmpl # @@ -609,7 +708,11 @@ # my.java.package. template("java_cpp_features") { action_with_pydeps(target_name) { - forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "sources" ]) + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "deps", + "sources", + ]) # The sources aren't compiled so don't check their dependencies. check_includes = false @@ -687,8 +790,6 @@ # version_number: (Optional) String of expected version of 'main' native # library. # enable_chromium_linker: (Optional) Whether to use the Chromium linker. - # load_library_from_apk: (Optional) Whether libraries should be loaded from - # the APK without uncompressing. # use_final_fields: True to use final fields. When false, all other # variables must not be set. template("write_native_libraries_java") { @@ -716,14 +817,16 @@ if (invoker.use_final_fields) { # Write native_libraries_list_file via depfile rather than specifyin it # as a dep in order allow R8 to run in parallel with native compilation. - depfile = "$target_gen_dir/$target_name.d" - args += [ - "--final", - "--depfile", - rebase_path(depfile, root_build_dir), - "--native-libraries-list", - rebase_path(invoker.native_libraries_list_file, root_build_dir), - ] + args += [ "--final" ] + if (defined(invoker.native_libraries_list_file)) { + depfile = "$target_gen_dir/$target_name.d" + args += [ + "--native-libraries-list", + rebase_path(invoker.native_libraries_list_file, root_build_dir), + "--depfile", + rebase_path(depfile, root_build_dir), + ] + } if (defined(invoker.main_component_library)) { args += [ "--main-component-library", @@ -734,13 +837,6 @@ invoker.enable_chromium_linker) { args += [ "--enable-chromium-linker" ] } - if (defined(invoker.load_library_from_apk) && - invoker.load_library_from_apk) { - args += [ "--load-library-from-apk" ] - } - if (defined(invoker.use_modern_linker) && invoker.use_modern_linker) { - args += [ "--use-modern-linker" ] - } } } } @@ -787,7 +883,7 @@ # template("android_generated_resources") { forward_variables_from(invoker, [ "testonly" ]) - _build_config = "$target_gen_dir/${target_name}.build_config" + _build_config = "$target_gen_dir/${target_name}.build_config.json" _rtxt_out_path = "$target_gen_dir/${target_name}.R.txt" write_build_config("$target_name$build_config_target_suffix") { forward_variables_from(invoker, [ "resource_overlay" ]) @@ -953,6 +1049,8 @@ # merged into apks that directly or indirectly depend on this target. # android_manifest_dep: Target that generates AndroidManifest (if applicable) # custom_package: java package for generated .java files. + # allow_missing_resources: Do not fail if a resource exists in a directory + # but is not listed in sources. # shared_resources: If true make a resource package that can be loaded by a # different application at runtime to access the package's resources. # resource_overlay: Whether the resources in 'sources' should override @@ -991,15 +1089,11 @@ not_needed(invoker, [ "v14_skip" ]) } - assert(!defined(invoker.resource_dirs) || defined(invoker.sources), - "resource_dirs in android_resources is deprecated. Please use " + - "sources=[] and list resource files instead. Details: " + - "https://crbug.com/1026378") _res_sources_path = "$target_gen_dir/${invoker.target_name}.res.sources" _resources_zip = "$target_out_dir/$target_name.resources.zip" _r_text_out_path = _base_path + "_R.txt" - _build_config = _base_path + ".build_config" + _build_config = _base_path + ".build_config.json" _build_config_target_name = "$target_name$build_config_target_suffix" _deps = [] @@ -1008,9 +1102,9 @@ } if (defined(invoker.alternative_android_sdk_dep)) { - _deps += [ invoker.alternative_android_sdk_dep ] + _android_sdk_dep = invoker.alternative_android_sdk_dep } else { - _deps += [ "//third_party/android_sdk:android_sdk_java" ] + _android_sdk_dep = default_android_sdk_dep } _resource_files = [] @@ -1049,7 +1143,7 @@ ]) r_text = _r_text_out_path - possible_config_deps = _deps + possible_config_deps = _deps + [ _android_sdk_dep ] # Always merge manifests from resources. # * Might want to change this at some point for consistency and clarity, @@ -1062,10 +1156,30 @@ prepare_resources(target_name) { forward_variables_from(invoker, [ + "allow_missing_resources", + "public_deps", "strip_drawables", "visibility", ]) - deps = _deps + _lib_deps = filter_exclude(filter_include(_deps, java_library_patterns), + java_resource_patterns) + if (defined(public_deps)) { + # Since java library targets depend directly on sub-targets rather than + # top-level targets, public_deps are not properly propagated, at least + # in terms of the "did you depend on the target that generates your + # inputs" GN check. + assert(filter_include(public_deps, java_target_patterns) == [], + "Java targets should use deps, not public_deps. " + + "target=${target_name}, public_deps=${public_deps}") + } + + # Depend on non-library deps and on __assetres subtargets of library deps. + deps = filter_exclude(_deps, _lib_deps) + [ _android_sdk_dep ] + foreach(_lib_dep, _lib_deps) { + # Expand //foo/java -> //foo/java:java + _lib_dep = get_label_info(_lib_dep, "label_no_toolchain") + deps += [ "${_lib_dep}__assetres" ] + } res_sources_path = _res_sources_path sources = _resource_files @@ -1121,9 +1235,17 @@ template("android_assets") { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - _build_config = "$target_gen_dir/$target_name.build_config" + _build_config = "$target_gen_dir/$target_name.build_config.json" _build_config_target_name = "$target_name$build_config_target_suffix" + _sources = [] + if (defined(invoker.sources)) { + _sources = invoker.sources + } + _renaming_sources = [] + if (defined(invoker.renaming_sources)) { + _renaming_sources = invoker.renaming_sources + } write_build_config(_build_config_target_name) { type = "android_assets" build_config = _build_config @@ -1138,13 +1260,13 @@ possible_config_deps = invoker.deps } - if (defined(invoker.sources)) { - asset_sources = invoker.sources + if (_sources != []) { + asset_sources = _sources } - if (defined(invoker.renaming_sources)) { + if (_renaming_sources != []) { assert(defined(invoker.renaming_destinations)) _source_count = 0 - foreach(_, invoker.renaming_sources) { + foreach(_, _renaming_sources) { _source_count += 1 } _dest_count = 0 @@ -1154,14 +1276,36 @@ assert( _source_count == _dest_count, "android_assets() renaming_sources.length != renaming_destinations.length") - asset_renaming_sources = invoker.renaming_sources + asset_renaming_sources = _renaming_sources asset_renaming_destinations = invoker.renaming_destinations } } - group(target_name) { - forward_variables_from(invoker, [ "deps" ]) - public_deps = [ ":$_build_config_target_name" ] + # Use an action in order to mark sources as "inputs" to a GN target so that + # GN will fail if the appropriate deps do not exist, and so that "gn refs" + # will know about the sources. We do not add these inputs & deps to the + # __build_config target because we want building .build_config.json files + # to be fast (and because write_build_config.py does not need the files to + # exist). + _all_sources = _sources + _renaming_sources + if (_all_sources != []) { + action(target_name) { + forward_variables_from(invoker, [ "deps" ]) + public_deps = [ ":$_build_config_target_name" ] + + script = "//build/android/gyp/validate_inputs.py" + inputs = _all_sources + outputs = [ "$target_gen_dir/$target_name.stamp" ] + args = [ + "--stamp", + rebase_path(outputs[0], root_build_dir), + ] + rebase_path(_all_sources, root_build_dir) + } + } else { + group(target_name) { + forward_variables_from(invoker, [ "deps" ]) + public_deps = [ ":$_build_config_target_name" ] + } } } @@ -1174,32 +1318,56 @@ # } # } template("java_group") { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) _build_config_vars = [ "input_jars_paths", + "preferred_dep", "mergeable_android_manifests", "proguard_configs", + "requires_android", ] - forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + _invoker_deps = [] + if (defined(invoker.deps)) { + _invoker_deps += invoker.deps + } + if (defined(invoker.public_deps)) { + _invoker_deps += invoker.public_deps + } write_build_config("$target_name$build_config_target_suffix") { forward_variables_from(invoker, _build_config_vars) type = "group" - build_config = "$target_gen_dir/${invoker.target_name}.build_config" + build_config = "$target_gen_dir/${invoker.target_name}.build_config.json" supports_android = true - if (defined(invoker.deps)) { - possible_config_deps = invoker.deps - } + possible_config_deps = _invoker_deps + } + + _assetres_deps = filter_include(_invoker_deps, java_resource_patterns) + _invoker_deps_minus_assetres = filter_exclude(_invoker_deps, _assetres_deps) + _lib_deps = + filter_include(_invoker_deps_minus_assetres, java_library_patterns) + + _expanded_lib_deps = [] + foreach(_lib_dep, _lib_deps) { + _expanded_lib_deps += [ get_label_info(_lib_dep, "label_no_toolchain") ] } foreach(_group_name, [ - "header", - "impl", "assetres", + "header", + "host", + "validate", ]) { - java_lib_group("${target_name}__${_group_name}") { - forward_variables_from(invoker, [ "deps" ]) - group_name = _group_name + group("${target_name}__$_group_name") { + deps = [] + foreach(_lib_dep, _expanded_lib_deps) { + deps += [ "${_lib_dep}__${_group_name}" ] + } + if (_group_name == "assetres") { + deps += _assetres_deps + } } } + group(target_name) { forward_variables_from(invoker, "*", @@ -1240,10 +1408,6 @@ forward_variables_from(invoker, "*", TESTONLY_AND_VISIBILITY) forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) type = "java_binary" - if (!defined(data_deps)) { - data_deps = [] - } - data_deps += [ "//third_party/jdk:java_data" ] } } @@ -1279,90 +1443,165 @@ } } - # Declare a Junit executable target + # Declare a Robolectric host side test binary. # - # This target creates an executable from java code for running as a junit test - # suite. The executable will be in the output folder's /bin/ directory. + # This target creates an executable from java code for running as a + # Robolectric test suite. The executable will be in the output folder's /bin/ + # directory. # # Supports all variables of java_binary(). # # Example - # junit_binary("foo") { + # robolectric_binary("foo") { # sources = [ "org/chromium/foo/FooTest.java" ] # deps = [ ":bar_java" ] # } - template("junit_binary") { + template("robolectric_binary") { testonly = true - _java_binary_target_name = "${target_name}__java_binary" - _test_runner_target_name = "${target_name}__test_runner_script" _main_class = "org.chromium.testing.local.JunitTestMain" - - _build_config = "$target_gen_dir/$target_name.build_config" + _build_config = "$target_gen_dir/$target_name.build_config.json" _build_config_target_name = "$target_name$build_config_target_suffix" - _deps = [ + _java_binary_target_name = "${target_name}__java_binary" + + _invoker_deps = [ "//testing/android/junit:junit_test_support", "//third_party/android_deps:robolectric_all_java", "//third_party/junit", "//third_party/mockito:mockito_java", ] if (defined(invoker.deps)) { - _deps += invoker.deps + _invoker_deps += invoker.deps } + _non_java_deps = filter_exclude(_invoker_deps, java_target_patterns) + _java_assetres_deps = [ ":${_java_binary_target_name}__assetres" ] + if (defined(invoker.alternative_android_sdk_dep)) { _android_sdk_dep = invoker.alternative_android_sdk_dep } else { - _android_sdk_dep = "//third_party/android_sdk:android_sdk_java" + _android_sdk_dep = default_android_sdk_dep } - # a package name or a manifest is required to have resources. This is + # A package name or a manifest is required to have resources. This is # added so that junit tests that do not care about the package name can # still use resources without having to explicitly set one. if (defined(invoker.package_name)) { _package_name = invoker.package_name } else if (!defined(invoker.android_manifest)) { - _package_name = "org.chromium.test" + _package_name = "no.manifest.configured" } - _resource_arsc_output = "${target_gen_dir}/${target_name}.ap_" - _compile_resources_target = "${target_name}__compile_resources" - compile_resources(_compile_resources_target) { - forward_variables_from(invoker, [ "android_manifest" ]) - deps = _deps + _merge_manifest_target_name = "${target_name}__merge_manifests" + _android_manifest = + "$target_gen_dir/$target_name.AndroidManifest.merged.xml" + + merge_manifests(_merge_manifest_target_name) { + if (defined(invoker.android_manifest)) { + input_manifest = invoker.android_manifest + } else { + input_manifest = "//build/android/AndroidManifest.xml" + } + + if (defined(_package_name)) { + manifest_package = _package_name + } + output_manifest = _android_manifest + build_config = _build_config + min_sdk_version = default_min_sdk_version + target_sdk_version = android_sdk_version + deps = _non_java_deps + _java_assetres_deps + + [ ":$_build_config_target_name" ] + if (defined(invoker.android_manifest_dep)) { + deps += [ invoker.android_manifest_dep ] + } + } + + _resource_arsc_output = "${target_out_dir}/${target_name}.ap_" + _compile_resources_target_name = "${target_name}__compile_resources" + compile_resources(_compile_resources_target_name) { + deps = _non_java_deps + _java_assetres_deps + + [ ":$_merge_manifest_target_name" ] android_sdk_dep = _android_sdk_dep build_config_dep = ":$_build_config_target_name" build_config = _build_config if (defined(_package_name)) { rename_manifest_package = _package_name } - if (!defined(android_manifest)) { - android_manifest = "//build/android/AndroidManifest.xml" - } + android_manifest = _android_manifest arsc_output = _resource_arsc_output min_sdk_version = default_min_sdk_version target_sdk_version = android_sdk_version } - _jni_srcjar_target = "${target_name}__final_jni" - _outer_target_name = target_name - generate_jni_registration(_jni_srcjar_target) { - enable_native_mocks = true - require_native_mocks = true - targets = [ ":$_outer_target_name" ] + # apkbuilder step needed only to add android assets to the .ap_ file. + _apkbuilder_output = "${target_out_dir}/${target_name}.robo.ap_" + _apkbuilder_target_name = "${target_name}__apkbuilder" + package_apk("$_apkbuilder_target_name") { + build_config = _build_config + min_sdk_version = default_min_sdk_version + deps = _java_assetres_deps + [ + ":$_build_config_target_name", + ":$_compile_resources_target_name", + ] + + is_robolectric_apk = true + packaged_resources_path = _resource_arsc_output + output_apk_path = _apkbuilder_output + } + + # Some may want to disable this to remove dependency on //base + # (JNI generator is in //base). + _generate_final_jni = + !defined(invoker.generate_final_jni) || invoker.generate_final_jni + if (_generate_final_jni) { + _jni_srcjar_target_name = "${target_name}__final_jni" + _outer_target_name = target_name + generate_jni_registration(_jni_srcjar_target_name) { + enable_native_mocks = true + require_native_mocks = !defined(invoker.shared_libraries) + targets = [ ":$_outer_target_name" ] + } + + if (defined(invoker.shared_libraries)) { + foreach(_dep, invoker.shared_libraries) { + assert( + string_replace(_dep, robolectric_toolchain, "") != _dep, + "$target_name has shared_libraries with incorrect toolchain. " + + "Should contain (\$robolectric_toolchain) suffix: $_dep") + } + + # Write shared library output files of all dependencies to a file. Those + # will be the shared libraries packaged into the APK. + _shared_library_list_file = "$target_gen_dir/$target_name.native_libs" + generated_file("${target_name}__shared_library_list") { + deps = invoker.shared_libraries + outputs = [ _shared_library_list_file ] + data_keys = [ "shared_libraries" ] + walk_keys = [ "shared_libraries_barrier" ] + rebase = root_build_dir + } + } + _native_libraries_target_name = "${target_name}__native_libraries" + write_native_libraries_java(_native_libraries_target_name) { + enable_chromium_linker = false + use_final_fields = true + if (defined(_shared_library_list_file)) { + native_libraries_list_file = _shared_library_list_file + } + } } java_library_impl(_java_binary_target_name) { - forward_variables_from(invoker, "*", TESTONLY_AND_VISIBILITY + [ "deps" ]) - type = "junit_binary" + forward_variables_from(invoker, + "*", + TESTONLY_AND_VISIBILITY + [ + "deps", + "shared_libraries", + ]) + type = "robolectric_binary" main_target_name = invoker.target_name - # Include the android SDK jar(s) for resource processing. - include_android_sdk = true - - # Robolectric can handle deps that set !supports_android as well those - # that set requires_android. - bypass_platform_checks = true - deps = _deps + deps = _invoker_deps testonly = true main_class = _main_class wrapper_script_name = "helper/$main_target_name" @@ -1372,37 +1611,51 @@ # 66%, which makes sharding more effective. tiered_stop_at_level_one = true + is_robolectric = true + include_android_sdk = true + alternative_android_sdk_dep = + "//third_party/robolectric:robolectric_test_sdk_java" + if (!defined(srcjar_deps)) { srcjar_deps = [] } srcjar_deps += [ - ":$_compile_resources_target", - ":$_jni_srcjar_target", - - # This dep is required for any targets that depend on //base:base_java. - "//build/android:build_config_gen", + ":$_compile_resources_target_name", + "//build/android:build_config_for_testing_gen", ] + if (_generate_final_jni) { + srcjar_deps += [ + ":$_jni_srcjar_target_name", + ":$_native_libraries_target_name", + ] + } } - test_runner_script(_test_runner_target_name) { - test_name = invoker.target_name - test_suite = invoker.target_name - test_type = "junit" - ignore_all_data_deps = true - resource_apk = _resource_arsc_output - } - - group(target_name) { + test_runner_script(target_name) { forward_variables_from(invoker, [ "assert_no_deps", "visibility", ]) - public_deps = [ + test_name = invoker.target_name + test_suite = invoker.target_name + test_type = "junit" + ignore_all_data_deps = true + resource_apk = _apkbuilder_output + deps = [ + ":$_apkbuilder_target_name", ":$_build_config_target_name", - ":$_java_binary_target_name", - ":$_test_runner_target_name", + ":${_java_binary_target_name}__host", + ":${_java_binary_target_name}__java_binary_script", + ":${_java_binary_target_name}__validate", + "//third_party/robolectric:robolectric_runtime_jars", ] + if (defined(invoker.shared_libraries)) { + data_deps = invoker.shared_libraries + } + + # Add non-libary deps, since the __host target does not depend on them. + deps += filter_exclude(_invoker_deps, java_library_patterns) } } @@ -1516,8 +1769,6 @@ # # Variables: # output: Path to the output jar. - # override_build_config: Use a pre-existing .build_config. Must be of type - # "apk". # use_interface_jars: Use all dependent interface .jars rather than # implementation .jars. # use_unprocessed_jars: Use unprocessed / undesugared .jars. @@ -1534,8 +1785,6 @@ # TODO(crbug.com/1042017): Remove. not_needed(invoker, [ "no_build_hooks" ]) forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - _supports_android = - !defined(invoker.supports_android) || invoker.supports_android _use_interface_jars = defined(invoker.use_interface_jars) && invoker.use_interface_jars _use_unprocessed_jars = @@ -1547,31 +1796,23 @@ _jar_target_name = target_name - _deps = [] - if (defined(invoker.deps)) { - _deps = invoker.deps - } - if (_supports_android) { - _deps += [ "//third_party/android_sdk:android_sdk_java" ] - } - - if (defined(invoker.override_build_config)) { - _build_config = invoker.override_build_config + if (defined(invoker.build_config)) { + _build_config = invoker.build_config + _build_config_dep = invoker.build_config_dep } else { - _build_config = "$target_gen_dir/$target_name.build_config" + _build_config = "$target_gen_dir/$target_name.build_config.json" _build_config_target_name = "$target_name$build_config_target_suffix" + _build_config_dep = ":$_build_config_target_name" write_build_config(_build_config_target_name) { type = "dist_jar" - supports_android = _supports_android + supports_android = + !defined(invoker.supports_android) || invoker.supports_android requires_android = defined(invoker.requires_android) && invoker.requires_android - possible_config_deps = _deps - ignore_dependency_public_deps = _direct_deps_only + possible_config_deps = invoker.deps build_config = _build_config } - - _deps += [ ":$_build_config_target_name" ] } _rebased_build_config = rebase_path(_build_config, root_build_dir) @@ -1579,7 +1820,22 @@ forward_variables_from(invoker, [ "data" ]) script = "//build/android/gyp/zip.py" depfile = "$target_gen_dir/$target_name.d" - deps = _deps + deps = [ _build_config_dep ] + + if (_use_interface_jars) { + _lib_deps = + filter_exclude(filter_include(invoker.deps, java_library_patterns), + java_resource_patterns) + _other_deps = filter_exclude(invoker.deps, _lib_deps) + foreach(_lib_dep, _lib_deps) { + # Expand //foo/java -> //foo/java:java + _lib_dep = get_label_info(_lib_dep, "label_no_toolchain") + deps += [ "${_lib_dep}__header" ] + } + deps += _other_deps + } else { + deps += invoker.deps + } inputs = [ _build_config ] @@ -1614,6 +1870,7 @@ args += [ "--input-zips=@FileArg($_rebased_build_config:deps_info:device_classpath)" ] } } + _excludes = [] if (defined(invoker.jar_excluded_patterns)) { _excludes += invoker.jar_excluded_patterns @@ -1622,6 +1879,9 @@ # Turbine adds files like: META-INF/TRANSITIVE/.../Foo.class # These confuse proguard: https://crbug.com/1081443 _excludes += [ "META-INF/*" ] + } else { + # Manifest files will never be correct when merging jars. + _excludes += [ "META-INF/*.MF" ] } if (_excludes != []) { args += [ "--input-zips-excluded-globs=$_excludes" ] @@ -1636,6 +1896,8 @@ # proguard_enabled: Whether to enable R8. # proguard_configs: List of proguard configs. # proguard_enable_obfuscation: Whether to enable obfuscation (default=true). + # package_name: Used in the Proguard map ID. + # version_code: Used in the Proguard map ID. # # Example # dist_dex("lib_fatjar") { @@ -1643,12 +1905,12 @@ # output = "$root_build_dir/MyLibrary.jar" # } template("dist_dex") { - _deps = [ "//third_party/android_sdk:android_sdk_java" ] + _deps = [ default_android_sdk_dep ] if (defined(invoker.deps)) { _deps += invoker.deps } - _build_config = "$target_gen_dir/$target_name.build_config" + _build_config = "$target_gen_dir/$target_name.build_config.json" _build_config_target_name = "$target_name$build_config_target_suffix" write_build_config(_build_config_target_name) { @@ -1664,19 +1926,19 @@ build_config = _build_config } - _deps += [ ":$_build_config_target_name" ] - dex(target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "data", "data_deps", + "package_name", "proguard_configs", "proguard_enabled", "proguard_enable_obfuscation", "min_sdk_version", + "version_code", ]) - deps = _deps + deps = [ ":$_build_config_target_name" ] + _deps build_config = _build_config enable_multidex = false output = invoker.output @@ -1686,13 +1948,10 @@ # per-target dex steps are emitted here since this is using jar files # rather than dex files. ignore_desugar_missing_deps = true - - # When trying to build a stand-alone .dex, don't add in jdk_libs_dex. - supports_jdk_library_desugaring = false } else { _rebased_build_config = rebase_path(_build_config, root_build_dir) input_dex_filearg = - "@FileArg(${_rebased_build_config}:final_dex:all_dex_files)" + "@FileArg(${_rebased_build_config}:deps_info:all_dex_files)" } } } @@ -1717,9 +1976,11 @@ # proguard_configs: List of proguard configs (optional). # android_manifest: Path to AndroidManifest.xml (optional). # native_libraries: list of native libraries (optional). - # direct_deps_only: Do not recurse on deps. (optional, defaults false). - # jar_excluded_patterns (optional): List of globs for paths to exclude. - # jar_included_patterns (optional): List of globs for paths to include. + # direct_deps_only: Do not recurse on deps (optional, defaults false). + # jar_excluded_patterns: List of globs for paths to exclude (optional). + # jar_included_patterns: List of globs for paths to include (optional). + # generate_final_jni: If defined an true, generate the final + # `GEN_JNI.java` and include it in the output `.aar` (optional) # # Example # dist_aar("my_aar") { @@ -1729,15 +1990,34 @@ template("dist_aar") { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - _deps = [] - if (defined(invoker.deps)) { - _deps = invoker.deps - } - _direct_deps_only = defined(invoker.direct_deps_only) && invoker.direct_deps_only - _build_config = "$target_gen_dir/$target_name.build_config" + _deps = [] + + _generate_final_jni = + defined(invoker.generate_final_jni) && invoker.generate_final_jni + if (_generate_final_jni) { + _outer_target_name = target_name + _jni_srcjar_target = "${target_name}__final_jni" + generate_jni_registration(_jni_srcjar_target) { + targets = [ ":$_outer_target_name" ] + } + _jni_java_target = "${target_name}__final_jni_java" + java_library_impl(_jni_java_target) { + type = "java_library" + supports_android = true + requires_android = true + srcjar_deps = [ ":$_jni_srcjar_target" ] + } + _deps += [ ":$_jni_java_target" ] + } + + if (defined(invoker.deps)) { + _deps += invoker.deps + } + + _build_config = "$target_gen_dir/$target_name.build_config.json" _build_config_target_name = "$target_name$build_config_target_suffix" write_build_config(_build_config_target_name) { @@ -1746,7 +2026,6 @@ possible_config_deps = _deps supports_android = true requires_android = true - ignore_dependency_public_deps = _direct_deps_only build_config = _build_config } @@ -1755,7 +2034,11 @@ _rebased_build_config = rebase_path(_build_config, root_build_dir) action_with_pydeps(target_name) { - forward_variables_from(invoker, [ "data" ]) + forward_variables_from(invoker, + [ + "data", + "assert_no_deps", + ]) depfile = "$target_gen_dir/$target_name.d" deps = _deps script = "//build/android/gyp/dist_aar.py" @@ -1783,8 +2066,11 @@ if (_direct_deps_only) { args += [ "--jars=@FileArg($_rebased_build_config:javac:classpath)" ] } else { - args += [ "--jars=@FileArg($_rebased_build_config:deps_info:javac_full_classpath)" ] + args += [ + "--jars=@FileArg($_rebased_build_config:deps_info:device_classpath)", + ] } + if (defined(invoker.android_manifest)) { args += [ "--android-manifest", @@ -1823,14 +2109,8 @@ # Supports all variables of java_library(), plus: # deps: In addition to defining java deps, this can also include # android_assets() and android_resources() targets. - # alternative_android_sdk_ijar: if set, the given android_sdk_ijar file - # replaces the default android_sdk_ijar. - # alternative_android_sdk_ijar_dep: the target that generates - # alternative_android_sdk_ijar, must be set if alternative_android_sdk_ijar - # is used. - # alternative_android_sdk_jar: actual jar corresponding to - # alternative_android_sdk_ijar, must be set if alternative_android_sdk_ijar - # is used. + # alternative_android_sdk_dep: android_system_java_prebuilt target to use + # in place of the default android.jar. # # Example # android_library("foo_java") { @@ -1865,11 +2145,67 @@ "*/R\$*.class", "*/Manifest.class", "*/Manifest\$*.class", - "*/GEN_JNI.class", + "*/*GEN_JNI.class", ] } } + # Declare an Android robolectric library target + # + # This target creates an Android library containing java code and Android + # resources. + # + # Supports all variables of java_library(), plus: + # deps: In addition to defining java deps, this can also include + # android_assets() and android_resources() targets. + # + # Example + # robolectric_library("foo_junit") { + # sources = [ + # "android/org/chromium/foo/FooTest.java", + # "android/org/chromium/foo/FooTestUtils.java", + # "android/org/chromium/foo/FooMock.java", + # ] + # deps = [ + # "//base:base_junit_test_support" + # ] + # srcjar_deps = [ + # ":foo_generated_enum" + # ] + # jar_excluded_patterns = [ + # "*/FooService.class", "org/chromium/FooService\$*.class" + # ] + # } + template("robolectric_library") { + java_library(target_name) { + forward_variables_from(invoker, "*", TESTONLY_AND_VISIBILITY) + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + + testonly = true + + is_robolectric = true + include_android_sdk = true + alternative_android_sdk_dep = + "//third_party/robolectric:robolectric_test_sdk_java" + + if (!defined(jar_excluded_patterns)) { + jar_excluded_patterns = [] + } + jar_excluded_patterns += [ + "*/R.class", + "*/R\$*.class", + "*/Manifest.class", + "*/Manifest\$*.class", + "*/*GEN_JNI.class", + ] + + if (!defined(deps)) { + deps = [] + } + deps += [ "//third_party/android_deps:robolectric_all_java" ] + } + } + # Declare an Android library target for a prebuilt jar # # This target creates an Android library containing java code and Android @@ -1922,8 +2258,9 @@ defines = [] # Set these even when !use_final_fields so that they have correct default - # values within junit_binary(), which ignores jar_excluded_patterns. - if (enable_java_asserts) { + # values within robolectric_binary(), which ignores jar_excluded_patterns. + if ((defined(invoker.assertions_implicitly_enabled) && + invoker.assertions_implicitly_enabled) || enable_java_asserts) { defines += [ "_ENABLE_ASSERTS" ] } if (use_cfi_diag || is_ubsan || is_ubsan_security || is_ubsan_vptr) { @@ -1934,10 +2271,6 @@ defines += [ "_IS_CHROME_BRANDED" ] } - if (is_chromecast && chromecast_branding == "internal") { - defines += [ "_IS_CHROMECAST_BRANDING_INTERNAL" ] - } - if (defined(invoker.bundles_supported) && invoker.bundles_supported) { defines += [ "_BUNDLES_SUPPORTED" ] } @@ -1967,6 +2300,10 @@ ] } } + + if (defined(testonly) && testonly) { + defines += [ "_IS_FOR_TEST" ] + } } } @@ -1980,14 +2317,12 @@ # is_bundle_module: Whether or not this target is part of a bundle build. # java_package: Java package for the generated class. # use_chromium_linker: - # use_modern_linker: template("generate_product_config_srcjar") { java_cpp_template(target_name) { defines = [] _use_final = defined(invoker.build_config) || - defined(invoker.use_chromium_linker) || - defined(invoker.use_modern_linker) || defined(invoker.is_bundle) + defined(invoker.use_chromium_linker) || defined(invoker.is_bundle) if (_use_final) { defines += [ "USE_FINAL" ] } @@ -1997,12 +2332,9 @@ _use_chromium_linker = defined(invoker.use_chromium_linker) && invoker.use_chromium_linker - _use_modern_linker = - defined(invoker.use_modern_linker) && invoker.use_modern_linker _is_bundle = defined(invoker.is_bundle_module) && invoker.is_bundle_module defines += [ "USE_CHROMIUM_LINKER_VALUE=$_use_chromium_linker", - "USE_MODERN_LINKER_VALUE=$_use_modern_linker", "IS_BUNDLE_VALUE=$_is_bundle", ] if (defined(invoker.build_config)) { @@ -2029,8 +2361,7 @@ # * dependencies of this .so are not automatically included # * ".cr.so" is never added # * they are not side-loaded when incremental_install=true. - # * load_library_from_apk, use_chromium_linker, - # and enable_relocation_packing do not apply + # * use_chromium_linker, and enable_relocation_packing do not apply # Use this instead of shared_libraries when you are going to load the library # conditionally, and only when shared_libraries doesn't work for you. # secondary_abi_loadable_modules: This is the loadable_modules analog to @@ -2050,10 +2381,11 @@ # is true when building with Chromium for non-test APKs. # generate_final_jni: If defined and false, skip generating the # GEN_JNI srcjar. - # jni_registration_header: If specified, causes the - # ${target_name}__final_jni target to additionally output a - # header file to this path for use with manual JNI registration. - # jni_sources_exclusions: List of source path to exclude from the + # generate_native_libraries_java: If defined, whether NativeLibraries.java is + # generated is solely controlled by this flag. Otherwise, the default behavior + # is NativeLibraries.java will only be generated for the base module/apk when + # its `shared_libraries` is not empty. + # jni_file_exclusions: List of source path to exclude from the # final_jni step. # aapt_locale_allowlist: If set, all locales not in this list will be # stripped from resources.arsc. @@ -2074,12 +2406,12 @@ # shared_resources_allowlist_target: Optional name of a target specifying # an input R.txt file that lists the resources that can be exported # by the APK when shared_resources or app_as_shared_lib is defined. - # uncompress_shared_libraries: True if shared libraries should be stored - # uncompressed in the APK. Must be unset or true if load_library_from_apk - # is set to true. # uncompress_dex: Store final .dex files uncompressed in the apk. + # omit_dex: If true, do not build or include classes.dex. # strip_resource_names: True if resource names should be stripped from the # resources.arsc file in the apk or module. + # strip_unused_resources: True if unused resources should be stripped from + # the apk or module. # short_resource_paths: True if resource paths should be shortened in the # apk or module. # resources_config_paths: List of paths to the aapt2 optimize config files @@ -2093,15 +2425,8 @@ # dependent resource targets which override another target set # overlay_resources=true. This check is on for non-test targets and # cannot be disabled. - # static_library_dependent_targets: A list of scopes describing targets that - # use this target as a static library. Common Java code from the targets - # listed in static_library_dependent_targets will be moved into this - # target. Scope members are name and is_resource_ids_provider. # static_library_provider: Specifies a single target that this target will # use as a static library APK. - # static_library_synchronized_proguard: When proguard is enabled, the - # static_library_provider target will provide the dex file(s) for this - # target. # min_sdk_version: The minimum Android SDK version this target supports. # Optional, default $default_min_sdk_version. # target_sdk_version: The target Android SDK version for this target. @@ -2118,7 +2443,6 @@ # ProductConfig.java file will be generated for each package. # enable_proguard_checks: Turns on -checkdiscard directives and missing # symbols check in the proguard step (default=true). - # disable_r8_outlining: Turn off outlining during the proguard step. # annotation_processor_deps: List of java_annotation_processor targets to # use when compiling the sources given to this target (optional). # processor_args_javac: List of args to pass to annotation processors when @@ -2138,10 +2462,11 @@ # with this file as the base. template("android_apk_or_module") { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) - assert(defined(invoker.android_manifest)) + _template_name = target_name _base_path = "$target_out_dir/$target_name/$target_name" - _build_config = "$target_gen_dir/$target_name.build_config" + _build_config = "$target_gen_dir/$target_name.build_config.json" _build_config_target = "$target_name$build_config_target_suffix" + _java_target_name = "${_template_name}__java" _min_sdk_version = default_min_sdk_version _target_sdk_version = android_sdk_version @@ -2152,8 +2477,6 @@ _target_sdk_version = invoker.target_sdk_version } - _template_name = target_name - _is_bundle_module = defined(invoker.is_bundle_module) && invoker.is_bundle_module if (_is_bundle_module) { @@ -2161,6 +2484,7 @@ defined(invoker.is_base_module) && invoker.is_base_module } + _omit_dex = defined(invoker.omit_dex) && invoker.omit_dex _enable_multidex = !defined(invoker.enable_multidex) || invoker.enable_multidex @@ -2169,17 +2493,6 @@ _final_rtxt_path = "${_final_apk_path}.R.txt" } - _short_resource_paths = - defined(invoker.short_resource_paths) && invoker.short_resource_paths && - enable_arsc_obfuscation - _strip_resource_names = - defined(invoker.strip_resource_names) && invoker.strip_resource_names && - enable_arsc_obfuscation - _optimize_resources = _strip_resource_names || _short_resource_paths - - if (!_is_bundle_module && _short_resource_paths) { - _final_pathmap_path = "${_final_apk_path}.pathmap.txt" - } _res_size_info_path = "$target_out_dir/$target_name.ap_.info" if (!_is_bundle_module) { _final_apk_path_no_ext_list = @@ -2196,20 +2509,12 @@ if (_is_bundle_module) { # Path to the intermediate proto-format resources zip file. _proto_resources_path = "$target_out_dir/$target_name.proto.ap_" - if (_optimize_resources) { - _optimized_proto_resources_path = - "$target_out_dir/$target_name.optimized.proto.ap_" - } } else { # resource_sizes.py needs to be able to find the unpacked resources.arsc # file based on apk name to compute normatlized size. _resource_sizes_arsc_path = "$root_out_dir/arsc/" + rebase_path(_final_apk_path_no_ext, root_build_dir) + ".ap_" - if (_optimize_resources) { - _optimized_arsc_resources_path = - "$target_out_dir/$target_name.optimized.ap_" - } } if (defined(invoker.version_code)) { @@ -2232,45 +2537,23 @@ _version_name = android_override_version_name } - _deps = [] if (defined(invoker.deps)) { - _deps = invoker.deps + _invoker_deps = invoker.deps + } else { + _invoker_deps = [] } + _non_java_deps = filter_exclude(_invoker_deps, java_target_patterns) + _java_assetres_deps = [ ":${_java_target_name}__assetres" ] _srcjar_deps = [] if (defined(invoker.srcjar_deps)) { _srcjar_deps = invoker.srcjar_deps } - _android_root_manifest_deps = [] - if (defined(invoker.android_manifest_dep)) { - _android_root_manifest_deps = [ invoker.android_manifest_dep ] - } - _android_root_manifest = invoker.android_manifest - _use_chromium_linker = defined(invoker.use_chromium_linker) && invoker.use_chromium_linker - _use_modern_linker = - defined(invoker.use_modern_linker) && invoker.use_modern_linker - _load_library_from_apk = - defined(invoker.load_library_from_apk) && invoker.load_library_from_apk - - not_needed([ - "_use_chromium_linker", - "_use_modern_linker", - ]) - assert(!_load_library_from_apk || _use_chromium_linker, - "load_library_from_apk requires use_chromium_linker") - - # Make sure that uncompress_shared_libraries is set to true if - # load_library_from_apk is true. - if (defined(invoker.uncompress_shared_libraries)) { - _uncompress_shared_libraries = invoker.uncompress_shared_libraries - assert(!_load_library_from_apk || _uncompress_shared_libraries) - } else { - _uncompress_shared_libraries = _load_library_from_apk - } + not_needed([ "_use_chromium_linker" ]) # The dependency that makes the chromium linker, if any is needed. _native_libs_deps = [] @@ -2323,12 +2606,13 @@ _rebased_build_config = rebase_path(_build_config, root_build_dir) assert(_rebased_build_config != "") # Mark as used. - _generate_buildconfig_java = !defined(invoker.apk_under_test) + _generate_buildconfig_java = !defined(invoker.apk_under_test) && !_omit_dex if (defined(invoker.generate_buildconfig_java)) { _generate_buildconfig_java = invoker.generate_buildconfig_java } - _generate_productconfig_java = defined(invoker.product_config_java_packages) + _generate_productconfig_java = + defined(invoker.product_config_java_packages) && !_omit_dex # JNI generation usually goes hand-in-hand with buildconfig generation. _generate_final_jni = _generate_buildconfig_java @@ -2343,44 +2627,16 @@ _proguard_mapping_path = "$_final_apk_path.mapping" } - # TODO(http://crbug.com/901465): Move shared Java code to static libraries - # when !_proguard_enabled too. - _is_static_library_provider = - defined(invoker.static_library_dependent_targets) && _proguard_enabled - if (_is_static_library_provider) { - _static_library_sync_dex_path = "$_base_path.synchronized.r8dex.jar" - _resource_ids_provider_deps = [] - foreach(_target, invoker.static_library_dependent_targets) { - if (_target.is_resource_ids_provider) { - assert(_resource_ids_provider_deps == [], - "Can only have 1 resource_ids_provider_dep") - _resource_ids_provider_deps += [ _target.name ] - } - } - _resource_ids_provider_dep = _resource_ids_provider_deps[0] - } else if (defined(invoker.resource_ids_provider_dep)) { + if (defined(invoker.resource_ids_provider_dep)) { _resource_ids_provider_dep = invoker.resource_ids_provider_dep } - if (_is_static_library_provider) { - _shared_resources_allowlist_target = _resource_ids_provider_dep - } else if (defined(invoker.shared_resources_allowlist_target)) { + if (defined(invoker.shared_resources_allowlist_target)) { _shared_resources_allowlist_target = invoker.shared_resources_allowlist_target } _uses_static_library = defined(invoker.static_library_provider) - _uses_static_library_synchronized_proguard = - defined(invoker.static_library_synchronized_proguard) && - invoker.static_library_synchronized_proguard - - if (_uses_static_library_synchronized_proguard) { - assert(_uses_static_library) - - # These will be provided by the static library APK. - _generate_buildconfig_java = false - _generate_final_jni = false - } # TODO(crbug.com/864142): Allow incremental installs of bundle modules. _incremental_apk = !_is_bundle_module && @@ -2392,18 +2648,18 @@ _incremental_apk_path = "${_final_apk_path_no_ext}_incremental.apk" } - if (!_incremental_apk) { + if (!_incremental_apk && !_omit_dex) { # Bundle modules don't build the dex here, but need to write this path - # to their .build_config file. + # to their .build_config.json file only when proguarding. if (_proguard_enabled) { _final_dex_path = "$_base_path.r8dex.jar" - } else { + } else if (!_is_bundle_module) { _final_dex_path = "$_base_path.mergeddex.jar" } } _android_manifest = - "$target_gen_dir/${_template_name}_manifest/AndroidManifest.xml" + "$target_gen_dir/${_template_name}/AndroidManifest.merged.xml" _merge_manifest_target = "${_template_name}__merge_manifests" merge_manifests(_merge_manifest_target) { forward_variables_from(invoker, @@ -2411,15 +2667,21 @@ "manifest_package", "max_sdk_version", ]) - input_manifest = _android_root_manifest + input_manifest = invoker.android_manifest output_manifest = _android_manifest build_config = _build_config min_sdk_version = _min_sdk_version target_sdk_version = _target_sdk_version - deps = _android_root_manifest_deps + [ ":$_build_config_target" ] + + # Depend on android_resources() targets that use generated files + # in mergeable_android_manifests (such as android_aar_prebuilt). + deps = _java_assetres_deps + [ ":$_build_config_target" ] + if (defined(invoker.android_manifest_dep)) { + deps += [ invoker.android_manifest_dep ] + } } - _final_deps = [] + _final_deps = [ ":$_java_target_name" ] _enable_main_dex_list = _enable_multidex && _min_sdk_version < 21 if (_enable_main_dex_list) { @@ -2428,16 +2690,10 @@ } _generated_proguard_config = "$_base_path.resources.proguard.txt" - if (_generate_buildconfig_java && - defined(invoker.product_version_resources_dep)) { - # Needs to be added as a .build_config dep to pick up resources. - _deps += [ invoker.product_version_resources_dep ] - } - if (defined(invoker.alternative_android_sdk_dep)) { _android_sdk_dep = invoker.alternative_android_sdk_dep } else { - _android_sdk_dep = "//third_party/android_sdk:android_sdk_java" + _android_sdk_dep = default_android_sdk_dep } if (defined(_shared_resources_allowlist_target)) { @@ -2452,9 +2708,25 @@ "${_shared_resources_allowlist_target}__compile_resources" } - if (_short_resource_paths) { - _resources_path_map_out_path = - "${target_gen_dir}/${_template_name}_resources_path_map.txt" + if (_incremental_apk) { + _incremental_android_manifest = + "$target_gen_dir/${_template_name}/AndroidManifest.incremental.xml" + _incremental_manifest_target_name = "${target_name}__incremental_manifest" + action_with_pydeps(_incremental_manifest_target_name) { + deps = [ ":$_merge_manifest_target" ] + script = + "//build/android/incremental_install/generate_android_manifest.py" + inputs = [ _android_manifest ] + outputs = [ _incremental_android_manifest ] + + args = [ + "--disable-isolated-processes", + "--src-manifest", + rebase_path(_android_manifest, root_build_dir), + "--dst-manifest", + rebase_path(_incremental_android_manifest, root_build_dir), + ] + } } _compile_resources_target = "${_template_name}__compile_resources" @@ -2463,33 +2735,28 @@ _compile_resources_emit_ids_out = "${target_gen_dir}/${_compile_resources_target}.resource_ids" compile_resources(_compile_resources_target) { - forward_variables_from(invoker, - [ - "aapt_locale_allowlist", - "app_as_shared_lib", - "enforce_resource_overlays_in_tests", - "expected_android_manifest", - "expected_android_manifest_base", - "extra_verification_manifest", - "extra_verification_manifest_dep", - "manifest_package", - "max_sdk_version", - "no_xml_namespaces", - "package_id", - "package_name", - "png_to_webp", - "r_java_root_package_name", - "resource_exclusion_exceptions", - "resource_exclusion_regex", - "resource_values_filter_rules", - "resources_config_paths", - "shared_resources", - "shared_resources_allowlist_locales", - "support_zh_hk", - "uses_split", - ]) - short_resource_paths = _short_resource_paths - strip_resource_names = _strip_resource_names + forward_variables_from( + invoker, + [ + "aapt_locale_allowlist", + "app_as_shared_lib", + "enforce_resource_overlays_in_tests", + "expected_android_manifest", + "expected_android_manifest_base", + "expected_android_manifest_library_version_offset", + "expected_android_manifest_version_code_offset", + "manifest_package", + "max_sdk_version", + "package_id", + "png_to_webp", + "r_java_root_package_name", + "resource_exclusion_exceptions", + "resource_exclusion_regex", + "resource_values_filter_rules", + "shared_resources", + "shared_resources_allowlist_locales", + "uses_split", + ]) android_manifest = _android_manifest android_manifest_dep = ":$_merge_manifest_target" version_code = _version_code @@ -2505,6 +2772,10 @@ resource_ids_provider_dep = _resource_ids_provider_dep } + if (defined(invoker.module_name)) { + package_name = invoker.module_name + } + if (defined(invoker.post_process_package_resources_script)) { post_process_script = invoker.post_process_package_resources_script } @@ -2515,21 +2786,15 @@ if (_enable_main_dex_list) { proguard_file_main_dex = _generated_proguard_main_dex_config } - if (_short_resource_paths) { - resources_path_map_out_path = _resources_path_map_out_path - } build_config = _build_config build_config_dep = ":$_build_config_target" android_sdk_dep = _android_sdk_dep - deps = _deps + deps = _java_assetres_deps + _non_java_deps - # The static library uses the R.txt files generated by the - # static_library_dependent_targets when generating the final R.java file. - if (_is_static_library_provider) { - foreach(_dep, invoker.static_library_dependent_targets) { - deps += [ "${_dep.name}__compile_resources" ] - } + if (_incremental_apk) { + android_manifest = _incremental_android_manifest + android_manifest_dep = ":$_incremental_manifest_target_name" } if (defined(invoker.apk_under_test)) { @@ -2545,27 +2810,16 @@ assert(!defined(resource_ids_provider_dep)) resource_ids_provider_dep = invoker.apk_under_test - include_resource = - get_label_info(invoker.apk_under_test, "target_out_dir") + "/" + - get_label_info(invoker.apk_under_test, "name") + ".ap_" _link_against = invoker.apk_under_test } if (_is_bundle_module) { is_bundle_module = true proto_output = _proto_resources_path - if (_optimize_resources) { - optimized_proto_output = _optimized_proto_resources_path - } if (defined(invoker.base_module_target)) { - include_resource = - get_label_info(invoker.base_module_target, "target_out_dir") + - "/" + get_label_info(invoker.base_module_target, "name") + ".ap_" _link_against = invoker.base_module_target } - } else if (_optimize_resources) { - optimized_arsc_output = _optimized_arsc_resources_path } if (defined(_link_against)) { @@ -2588,16 +2842,63 @@ } _srcjar_deps += [ ":$_compile_resources_target" ] - if (defined(_resource_sizes_arsc_path)) { - _copy_arsc_target = "${_template_name}__copy_arsc" - copy(_copy_arsc_target) { - deps = [ ":$_compile_resources_target" ] + # We don't ship apks anymore, only optimize bundle builds + if (_is_bundle_module) { + _short_resource_paths = + defined(invoker.short_resource_paths) && + invoker.short_resource_paths && enable_arsc_obfuscation + _strip_resource_names = + defined(invoker.strip_resource_names) && + invoker.strip_resource_names && enable_arsc_obfuscation + _strip_unused_resources = + defined(invoker.strip_unused_resources) && + invoker.strip_unused_resources && enable_unused_resource_stripping + _optimize_resources = _strip_resource_names || _short_resource_paths || + _strip_unused_resources + } - # resource_sizes.py doesn't care if it gets the optimized .arsc. - sources = [ _arsc_resources_path ] - outputs = [ _resource_sizes_arsc_path ] + if (_is_bundle_module && _optimize_resources) { + _optimized_proto_resources_path = + "$target_out_dir/$target_name.optimized.proto.ap_" + if (_short_resource_paths) { + _resources_path_map_out_path = + "${target_gen_dir}/${_template_name}_resources_path_map.txt" } - _final_deps += [ ":$_copy_arsc_target" ] + _optimize_resources_target = "${_template_name}__optimize_resources" + optimize_resources(_optimize_resources_target) { + deps = _non_java_deps + [ ":$_compile_resources_target" ] + short_resource_paths = _short_resource_paths + strip_resource_names = _strip_resource_names + if (_short_resource_paths) { + resources_path_map_out_path = _resources_path_map_out_path + } + r_text_path = _compile_resources_rtxt_out + proto_input_path = _proto_resources_path + optimized_proto_output = _optimized_proto_resources_path + if (_strip_unused_resources) { + # These need to be kept in sync with the target names + output paths + # in the android_app_bundle template. + _unused_resources_target = "${_template_name}__unused_resources" + _unused_resources_config_path = + "$target_gen_dir/${_template_name}_unused_resources.config" + resources_config_paths = [ _unused_resources_config_path ] + deps += [ ":$_unused_resources_target" ] + } else { + resources_config_paths = [] + } + if (defined(invoker.resources_config_paths)) { + resources_config_paths += invoker.resources_config_paths + } + } + + if (_strip_unused_resources) { + # Copy the unused resources config to the final bundle output dir. + _copy_unused_resources_target = + "${_template_name}__copy_unused_resources" + _final_deps += [ ":$_copy_unused_resources_target" ] + } + } else { + not_needed(invoker, [ "resources_config_paths" ]) } if (!_is_bundle_module) { @@ -2614,26 +2915,27 @@ outputs = [ _final_rtxt_path ] } _final_deps += [ ":$_copy_rtxt_target" ] - - if (_short_resource_paths) { - # Do the same for path map - _copy_pathmap_target = "${_template_name}__copy_pathmap" - copy(_copy_pathmap_target) { - deps = [ ":$_compile_resources_target" ] - sources = [ _resources_path_map_out_path ] - outputs = [ _final_pathmap_path ] - - # The monochrome_public_apk_checker test needs pathmap when run on swarming. - data = [ _final_pathmap_path ] - } - _final_deps += [ ":$_copy_pathmap_target" ] - } } - _generate_native_libraries_java = - (!_is_bundle_module || _is_base_module) && - (_native_libs_deps != [] || _secondary_abi_native_libs_deps != []) && - !_uses_static_library_synchronized_proguard + if (defined(_resource_sizes_arsc_path)) { + _copy_arsc_target = "${_template_name}__copy_arsc" + copy(_copy_arsc_target) { + deps = [ ":$_compile_resources_target" ] + + # resource_sizes.py doesn't care if it gets the optimized .arsc. + sources = [ _arsc_resources_path ] + outputs = [ _resource_sizes_arsc_path ] + } + _final_deps += [ ":$_copy_arsc_target" ] + } + + if (defined(invoker.generate_native_libraries_java)) { + _generate_native_libraries_java = invoker.generate_native_libraries_java + } else { + _generate_native_libraries_java = + (!_is_bundle_module || _is_base_module) && !_omit_dex && + !defined(invoker.apk_under_test) + } if (_generate_native_libraries_java) { write_native_libraries_java("${_template_name}__native_libraries") { forward_variables_from(invoker, [ "main_component_library" ]) @@ -2641,14 +2943,22 @@ # Do not add a dep on the generated_file target in order to avoid having # to build the native libraries before this target. The dependency is # instead captured via a depfile. - if (_native_libs_deps != []) { + if (_uses_static_library) { + _prefix = get_label_info(invoker.static_library_provider, + "target_gen_dir") + "/" + + get_label_info(invoker.static_library_provider, "name") + if (defined(invoker.static_library_provider_use_secondary_abi) && + invoker.static_library_provider_use_secondary_abi) { + native_libraries_list_file = "${_prefix}.secondary_abi_native_libs" + } else { + native_libraries_list_file = "${_prefix}.native_libs" + } + } else if (_native_libs_deps != []) { native_libraries_list_file = _shared_library_list_file - } else { + } else if (_secondary_abi_native_libs_deps != []) { native_libraries_list_file = _secondary_abi_shared_library_list_file } enable_chromium_linker = _use_chromium_linker - load_library_from_apk = _load_library_from_apk - use_modern_linker = _use_modern_linker use_final_fields = true } _srcjar_deps += [ ":${_template_name}__native_libraries" ] @@ -2663,6 +2973,11 @@ _loadable_modules += _sanitizer_runtimes } + _assertions_implicitly_enabled = defined(invoker.custom_assertion_handler) + + # Many possible paths where we wouldn't use this variable. + not_needed([ "_assertions_implicitly_enabled" ]) + if (_generate_buildconfig_java) { generate_build_config_srcjar("${_template_name}__build_config_srcjar") { forward_variables_from(invoker, @@ -2670,15 +2985,17 @@ "min_sdk_version", "isolated_splits_enabled", ]) - _bundles_supported = _is_bundle_module || _is_static_library_provider + _bundles_supported = _is_bundle_module if (defined(invoker.bundles_supported)) { _bundles_supported = invoker.bundles_supported } bundles_supported = _bundles_supported use_final_fields = true + assertions_implicitly_enabled = _assertions_implicitly_enabled enable_multidex = _enable_multidex is_incremental_install = _incremental_apk - if (defined(invoker.product_version_resources_dep)) { + if (defined(invoker.build_config_include_product_version_resource) && + invoker.build_config_include_product_version_resource) { resources_version_variable = "org.chromium.base.R.string.product_version" } @@ -2696,7 +3013,6 @@ build_config = _build_config java_package = _package use_chromium_linker = _use_chromium_linker - use_modern_linker = _use_modern_linker deps = [ ":$_build_config_target" ] } _srcjar_deps += [ ":$_locale_target_name" ] @@ -2707,6 +3023,7 @@ generate_jni_registration("${_template_name}__final_jni") { forward_variables_from(invoker, [ + "enable_jni_multiplexing", "enable_native_mocks", "require_native_mocks", ]) @@ -2715,30 +3032,25 @@ } else { targets = [ ":$_template_name" ] } - if (_is_static_library_provider) { - foreach(_target, invoker.static_library_dependent_targets) { - targets += [ _target.name ] - } + if (defined(invoker.jni_file_exclusions)) { + file_exclusions = invoker.jni_file_exclusions } - if (defined(invoker.jni_registration_header)) { - header_output = invoker.jni_registration_header - } - if (defined(invoker.jni_sources_exclusions)) { - sources_exclusions = invoker.jni_sources_exclusions - } + prevent_header_output = true } _srcjar_deps += [ ":${_template_name}__final_jni" ] } else { - not_needed(invoker, - [ - "enable_native_mocks", - "jni_registration_header", - ]) + not_needed(invoker, [ "enable_native_mocks" ]) } - _java_target = "${_template_name}__java" + if (_is_bundle_module) { + _add_view_trace_events = + defined(invoker.add_view_trace_events) && + invoker.add_view_trace_events && enable_trace_event_bytecode_rewriting + } - java_library_impl(_java_target) { + # We cannot skip this target when omit_dex = true because it writes the + # build_config.json. + java_library_impl(_java_target_name) { forward_variables_from(invoker, [ "alternative_android_sdk_dep", @@ -2748,41 +3060,30 @@ "apk_under_test", "base_module_target", "chromium_code", + "deps", "jacoco_never_instrument", "jar_excluded_patterns", "javac_args", + "mergeable_android_manifests", "native_lib_placeholders", + "parent_module_target", "processor_args_javac", "secondary_abi_loadable_modules", "secondary_native_lib_placeholders", "sources", - "static_library_dependent_targets", "library_always_compress", - "library_renames", ]) - deps = _deps - if (_uses_static_library_synchronized_proguard) { - if (!defined(jar_excluded_patterns)) { - jar_excluded_patterns = [] - } - - # The static library will provide all R.java files, but we still need to - # make the base module R.java files available at compile time since DFM - # R.java classes extend base module classes. - jar_excluded_patterns += [ - "*/R.class", - "*/R\$*.class", - ] - } + version_code = _version_code + version_name = _version_name if (_is_bundle_module) { type = "android_app_bundle_module" res_size_info_path = _res_size_info_path - is_base_module = _is_base_module - forward_variables_from(invoker, - [ - "version_code", - "version_name", - ]) + if (defined(invoker.module_name)) { + module_name = invoker.module_name + } else { + module_name = "base" + } + add_view_trace_events = _add_view_trace_events } else { type = "android_apk" } @@ -2791,6 +3092,7 @@ supports_android = true requires_android = true srcjar_deps = _srcjar_deps + merged_android_manifest = _android_manifest if (defined(_final_dex_path)) { final_dex_path = _final_dex_path } @@ -2817,10 +3119,8 @@ if (defined(invoker.proguard_configs)) { proguard_configs += invoker.proguard_configs } - if (_enable_main_dex_list) { - proguard_configs += [ "//build/android/multidex.flags" ] - } - if (!enable_java_asserts && (!defined(testonly) || !testonly) && + if (!_assertions_implicitly_enabled && !enable_java_asserts && + (!defined(testonly) || !testonly) && # Injected JaCoCo code causes -checkdiscards to fail. !use_jacoco_coverage) { proguard_configs += [ "//build/android/dcheck_is_off.flags" ] @@ -2843,204 +3143,106 @@ loadable_modules = _loadable_modules - uncompress_shared_libraries = _uncompress_shared_libraries - if (defined(_allowlist_r_txt_path) && _is_bundle_module) { - # Used to write the file path to the target's .build_config only. + # Used to write the file path to the target's .build_config.json only. base_allowlist_rtxt_path = _allowlist_r_txt_path } } - # TODO(cjhopman): This is only ever needed to calculate the list of tests to - # run. See build/android/pylib/instrumentation/test_jar.py. We should be - # able to just do that calculation at build time instead. - if (defined(invoker.dist_ijar_path)) { - _dist_ijar_path = invoker.dist_ijar_path - dist_jar("${_template_name}_dist_ijar") { - override_build_config = _build_config - output = _dist_ijar_path - data = [ _dist_ijar_path ] - use_interface_jars = true - deps = [ - ":$_build_config_target", - ":$_java_target", - ] - } - } - - if (_uses_static_library_synchronized_proguard) { - _final_dex_target_dep = "${invoker.static_library_provider}__dexsplitter" - } else if (_is_bundle_module && _proguard_enabled) { - _final_deps += [ ":$_java_target" ] + if (_is_bundle_module || _omit_dex) { + # Dex generation for app bundle modules take place in the + # android_app_bundle template. + not_needed(invoker, [ "custom_assertion_handler" ]) } else if (_incremental_apk) { - if (defined(invoker.enable_proguard_checks)) { - not_needed(invoker, [ "enable_proguard_checks" ]) - } - if (defined(invoker.disable_r8_outlining)) { - not_needed(invoker, [ "disable_r8_outlining" ]) - } - if (defined(invoker.dexlayout_profile)) { - not_needed(invoker, [ "dexlayout_profile" ]) - } + not_needed(invoker, + [ + "enable_proguard_checks", + "custom_assertion_handler", + ]) } else { - # Dex generation for app bundle modules with proguarding enabled takes - # place later due to synchronized proguarding. For more details, - # read build/android/docs/android_app_bundles.md _final_dex_target_name = "${_template_name}__final_dex" dex(_final_dex_target_name) { forward_variables_from(invoker, [ - "disable_r8_outlining", - "dexlayout_profile", "enable_proguard_checks", + "custom_assertion_handler", "proguard_enable_obfuscation", ]) min_sdk_version = _min_sdk_version proguard_enabled = _proguard_enabled build_config = _build_config + output = _final_dex_path + enable_multidex = _enable_multidex deps = [ ":$_build_config_target", - ":$_java_target", + ":$_java_target_name", ] if (_proguard_enabled) { - deps += _deps + [ ":$_compile_resources_target" ] + # Generates proguard configs + deps += [ ":$_compile_resources_target" ] proguard_mapping_path = _proguard_mapping_path - proguard_sourcefile_suffix = "$android_channel-$_version_code" has_apk_under_test = defined(invoker.apk_under_test) - } else if (_min_sdk_version >= default_min_sdk_version) { - # Enable dex merging only when min_sdk_version is >= what the library - # .dex files were created with. - input_dex_filearg = - "@FileArg(${_rebased_build_config}:final_dex:all_dex_files)" } else { - input_classes_filearg = - "@FileArg($_rebased_build_config:deps_info:device_classpath)" - } + if (_min_sdk_version >= default_min_sdk_version) { + # Enable dex merging only when min_sdk_version is >= what the library + # .dex files were created with. + input_dex_filearg = + "@FileArg(${_rebased_build_config}:deps_info:all_dex_files)" - if (_is_static_library_provider) { - # The list of input jars is already recorded in the .build_config, but - # we need to explicitly add the java deps here to ensure they're - # available to be used as inputs to the dex step. - foreach(_dep, invoker.static_library_dependent_targets) { - _target_label = get_label_info(_dep.name, "label_no_toolchain") - deps += [ "${_target_label}__java" ] + # Pure dex-merge. + enable_desugar = false + } else { + input_classes_filearg = + "@FileArg($_rebased_build_config:deps_info:device_classpath)" } - output = _static_library_sync_dex_path - is_static_library = true - } else { - output = _final_dex_path } - enable_multidex = _enable_multidex # The individual dependencies would have caught real missing deps in # their respective dex steps. False positives that were suppressed at # per-target dex steps are emitted here since this may use jar files # rather than dex files. - ignore_desugar_missing_deps = true + if (!defined(enable_desugar)) { + ignore_desugar_missing_deps = true + } if (_enable_main_dex_list) { - extra_main_dex_proguard_config = _generated_proguard_main_dex_config + # Generates main-dex config. deps += [ ":$_compile_resources_target" ] + extra_main_dex_proguard_config = _generated_proguard_main_dex_config } } _final_dex_target_dep = ":$_final_dex_target_name" - # For static libraries, a single Proguard run is performed that includes - # code from the static library APK and the APKs that use the static - # library (done via. classpath merging in write_build_config.py). - # This dexsplitter target splits the synchronized dex output into dex - # files for each APK/Bundle. In the Bundle case, another dexsplitter step - # is later performed to split the dex further for each feature module. - if (_is_static_library_provider && _proguard_enabled) { - _static_library_modules = [] - foreach(_target, invoker.static_library_dependent_targets) { - _apk_as_module = _target.name - _module_config_target = "${_apk_as_module}$build_config_target_suffix" - _module_gen_dir = get_label_info(_apk_as_module, "target_gen_dir") - _module_name = get_label_info(_apk_as_module, "name") - _module_config = "$_module_gen_dir/$_module_name.build_config" - _static_library_modules += [ - { - name = _module_name - build_config = _module_config - build_config_target = _module_config_target - }, - ] - } - - _static_library_dexsplitter_target = "${_template_name}__dexsplitter" - dexsplitter(_static_library_dexsplitter_target) { - input_dex_zip = _static_library_sync_dex_path - proguard_mapping = _proguard_mapping_path + _use_baseline_profile = + _proguard_enabled && defined(invoker.baseline_profile_path) && + enable_baseline_profiles + if (_use_baseline_profile) { + _binary_profile_target = "${_template_name}__binary_baseline_profile" + _binary_baseline_profile_path = + "$target_out_dir/$_template_name.baseline.prof" + _binary_baseline_profile_metadata_path = + _binary_baseline_profile_path + "m" + create_binary_profile(_binary_profile_target) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + binary_baseline_profile_path = _binary_baseline_profile_path + binary_baseline_profile_metadata_path = + _binary_baseline_profile_metadata_path + proguard_mapping_path = _proguard_mapping_path + build_config = _build_config + input_profile_path = invoker.baseline_profile_path deps = [ ":$_build_config_target", - "$_final_dex_target_dep", + _final_dex_target_dep, ] - all_modules = [ - { - name = "base" - build_config = _build_config - build_config_target = ":$_build_config_target" - }, - ] + _static_library_modules - feature_jars_args = [ - "--feature-jars", - "@FileArg($_rebased_build_config:deps_info:" + - "static_library_dependent_classpath_configs:" + - "$_rebased_build_config)", - ] - foreach(_module, _static_library_modules) { - _rebased_module_config = - rebase_path(_module.build_config, root_build_dir) - feature_jars_args += [ - "--feature-jars", - "@FileArg($_rebased_build_config:deps_info:" + - "static_library_dependent_classpath_configs:" + - "$_rebased_module_config)", - ] - } } - _final_deps += [ ":$_static_library_dexsplitter_target" ] - _validate_dex_target = "${_template_name}__validate_dex" - action_with_pydeps(_validate_dex_target) { - depfile = "$target_gen_dir/$target_name.d" - script = - "//build/android/gyp/validate_static_library_dex_references.py" - inputs = [ _build_config ] - _stamp = "$target_gen_dir/$target_name.stamp" - outputs = [ _stamp ] - deps = [ - ":$_build_config_target", - ":$_static_library_dexsplitter_target", - ] - args = [ - "--depfile", - rebase_path(depfile, root_build_dir), - "--stamp", - rebase_path(_stamp, root_build_dir), - "--static-library-dex", - "@FileArg($_rebased_build_config:final_dex:path)", - ] - foreach(_module, _static_library_modules) { - inputs += [ _module.build_config ] - _rebased_config = rebase_path(_module.build_config, root_build_dir) - deps += [ _module.build_config_target ] - args += [ - "--static-library-dependent-dex", - "@FileArg($_rebased_config:final_dex:path)", - ] - } - } - - # TODO(crbug.com/1032609): Switch to using R8's support for feature - # aware ProGuard and get rid of "_validate_dex_target" or figure out - # why some classes aren't properly being kept. - # _final_deps += [ ":$_validate_dex_target" ] - _final_dex_target_dep = ":$_static_library_dexsplitter_target" } } + if (!defined(_use_baseline_profile) || !_use_baseline_profile) { + not_needed(invoker, [ "baseline_profile_path" ]) + } + _all_native_libs_deps = _native_libs_deps + _secondary_abi_native_libs_deps if (_all_native_libs_deps != []) { _native_libs_filearg_dep = ":$_build_config_target" @@ -3054,10 +3256,13 @@ if (_is_bundle_module) { _final_deps += [ - ":$_merge_manifest_target", ":$_build_config_target", ":$_compile_resources_target", + ":$_merge_manifest_target", ] + _all_native_libs_deps + if (_optimize_resources) { + _final_deps += [ ":$_optimize_resources_target" ] + } if (defined(_final_dex_target_dep)) { not_needed([ "_final_dex_target_dep" ]) } @@ -3074,11 +3279,11 @@ name = "${invoker.name}.apk" build_config = _build_config res_size_info_path = _res_size_info_path - deps = _deps + [ - ":$_build_config_target", - ":$_compile_resources_target", - ":$_java_target", - ] + deps = [ + ":$_build_config_target", + ":$_compile_resources_target", + ":$_java_target_name", + ] } _final_deps += [ ":$_size_info_target" ] } else { @@ -3086,51 +3291,6 @@ } } - _keystore_path = android_keystore_path - _keystore_name = android_keystore_name - _keystore_password = android_keystore_password - - if (defined(invoker.keystore_path)) { - _keystore_path = invoker.keystore_path - _keystore_name = invoker.keystore_name - _keystore_password = invoker.keystore_password - } - - if (_incremental_apk) { - _incremental_compiled_resources_path = "${_base_path}_incremental.ap_" - _incremental_compile_resources_target_name = - "${target_name}__compile_incremental_resources" - - action_with_pydeps(_incremental_compile_resources_target_name) { - deps = [ - ":$_build_config_target", - ":$_compile_resources_target", - ":$_merge_manifest_target", - ] - script = - "//build/android/incremental_install/generate_android_manifest.py" - inputs = [ - _android_manifest, - _build_config, - _arsc_resources_path, - ] - outputs = [ _incremental_compiled_resources_path ] - - args = [ - "--disable-isolated-processes", - "--src-manifest", - rebase_path(_android_manifest, root_build_dir), - "--in-apk", - rebase_path(_arsc_resources_path, root_build_dir), - "--out-apk", - rebase_path(_incremental_compiled_resources_path, root_build_dir), - "--aapt2-path", - rebase_path(android_sdk_tools_bundle_aapt2, root_build_dir), - "--android-sdk-jars=@FileArg($_rebased_build_config:android:sdk_jars)", - ] - } - } - _create_apk_target = "${_template_name}__create" _final_deps += [ ":$_create_apk_target" ] package_apk("$_create_apk_target") { @@ -3138,13 +3298,14 @@ [ "expected_libs_and_assets", "expected_libs_and_assets_base", + "keystore_name", + "keystore_path", + "keystore_password", "native_lib_placeholders", "secondary_abi_loadable_modules", "secondary_native_lib_placeholders", "uncompress_dex", - "uncompress_shared_libraries", "library_always_compress", - "library_renames", ]) if (defined(expected_libs_and_assets)) { @@ -3153,32 +3314,27 @@ } build_config = _build_config - keystore_name = _keystore_name - keystore_path = _keystore_path - keystore_password = _keystore_password min_sdk_version = _min_sdk_version - uncompress_shared_libraries = _uncompress_shared_libraries + packaged_resources_path = _arsc_resources_path - deps = _deps + [ ":$_build_config_target" ] - - if ((!_proguard_enabled || _incremental_apk) && - enable_jdk_library_desugaring) { - _all_jdk_libs = "//build/android:all_jdk_libs" - deps += [ _all_jdk_libs ] - jdk_libs_dex = get_label_info(_all_jdk_libs, "target_out_dir") + - "/all_jdk_libs.l8.dex" - } + # Need full deps rather than _non_java_deps, because loadable_modules + # may include .so files extracted by __unpack_aar targets. + deps = _invoker_deps + [ ":$_build_config_target" ] if (_incremental_apk) { _dex_target = "//build/android/incremental_install:apk_dex" deps += [ - ":${_incremental_compile_resources_target_name}", + ":$_compile_resources_target", _dex_target, ] dex_path = get_label_info(_dex_target, "target_out_dir") + "/apk.dex" + # Incremental APKs cannot be installed via `adb install` as such they + # should be clearly named/labeled "incremental". + output_apk_path = _incremental_apk_path + # All native libraries are side-loaded, so use a placeholder to force # the proper bitness for the app. _has_native_libs = @@ -3186,32 +3342,34 @@ if (_has_native_libs && !defined(native_lib_placeholders)) { native_lib_placeholders = [ "libfix.crbug.384638.so" ] } - - packaged_resources_path = _incremental_compiled_resources_path - output_apk_path = _incremental_apk_path } else { loadable_modules = _loadable_modules deps += _all_native_libs_deps + [ - ":$_merge_manifest_target", ":$_compile_resources_target", + ":$_merge_manifest_target", ] if (defined(_final_dex_path)) { dex_path = _final_dex_path deps += [ _final_dex_target_dep ] + if (_use_baseline_profile) { + # extra_assets is a list of ["{src_path}:{dst_path}"] + extra_assets = [ + rebase_path(_binary_baseline_profile_path, root_build_dir) + + ":dexopt/baseline.prof", + rebase_path(_binary_baseline_profile_metadata_path, + root_build_dir) + ":dexopt/baseline.profm", + ] + deps += [ ":$_binary_profile_target" ] + } } - if (_optimize_resources) { - packaged_resources_path = _optimized_arsc_resources_path - } else { - packaged_resources_path = _arsc_resources_path - } + output_apk_path = _final_apk_path if (defined(_native_libs_filearg)) { native_libs_filearg = _native_libs_filearg secondary_abi_native_libs_filearg = "@FileArg($_rebased_build_config:native:secondary_abi_libraries)" } - output_apk_path = _final_apk_path } } } @@ -3233,7 +3391,7 @@ args = [ "--apk-path=$_rebased_incremental_apk_path", "--output-path=$_rebased_incremental_install_json_path", - "--dex-file=@FileArg($_rebased_build_config:final_dex:all_dex_files)", + "--dex-file=@FileArg($_rebased_build_config:deps_info:all_dex_files)", ] if (_proguard_enabled) { args += [ "--show-proguard-warning" ] @@ -3248,10 +3406,7 @@ args += [ "--native-libs=$_rebased_loadable_modules" ] } } - _final_deps += [ - ":$_java_target", - ":$_write_installer_json_rule_name", - ] + _final_deps += [ ":$_write_installer_json_rule_name" ] } # Generate apk operation related script. @@ -3338,7 +3493,9 @@ ]) build_config = _build_config build_config_dep = ":$_build_config_target" - deps = [ ":$_java_target" ] + + # This will use library subtargets under-the-hood + deps = [ ":$_java_target_name" ] if (defined(invoker.lint_suppressions_dep)) { deps += [ invoker.lint_suppressions_dep ] } @@ -3350,6 +3507,7 @@ not_needed(invoker, [ "lint_baseline_file", + "lint_jar_path", "lint_min_sdk_version", "lint_suppressions_dep", "lint_suppressions_file", @@ -3373,8 +3531,7 @@ } # Include unstripped native libraries so tests can symbolize stacks. - data_deps += _all_native_libs_deps - + data_deps += _all_native_libs_deps + [ ":${_java_target_name}__validate" ] if (_enable_lint) { data_deps += [ ":${target_name}__lint" ] } @@ -3416,102 +3573,97 @@ # TODO(crbug.com/1042017): Remove. not_needed(invoker, [ "no_build_hooks" ]) android_apk_or_module(target_name) { - forward_variables_from(invoker, - [ - "aapt_locale_allowlist", - "additional_jar_files", - "alternative_android_sdk_dep", - "android_manifest", - "android_manifest_dep", - "annotation_processor_deps", - "apk_under_test", - "app_as_shared_lib", - "assert_no_deps", - "bundles_supported", - "chromium_code", - "command_line_flags_file", - "create_apk_script", - "data", - "data_deps", - "deps", - "dexlayout_profile", - "disable_r8_outlining", - "dist_ijar_path", - "enable_lint", - "enable_multidex", - "enable_native_mocks", - "enable_proguard_checks", - "enforce_resource_overlays_in_tests", - "expected_android_manifest", - "expected_android_manifest_base", - "expected_libs_and_assets", - "expected_libs_and_assets_base", - "generate_buildconfig_java", - "generate_final_jni", - "include_size_info", - "input_jars_paths", - "use_modern_linker", - "jacoco_never_instrument", - "javac_args", - "jni_registration_header", - "jni_sources_exclusions", - "keystore_name", - "keystore_password", - "keystore_path", - "lint_baseline_file", - "lint_min_sdk_version", - "lint_suppressions_dep", - "lint_suppressions_file", - "load_library_from_apk", - "loadable_modules", - "manifest_package", - "max_sdk_version", - "product_config_java_packages", - "main_component_library", - "min_sdk_version", - "native_lib_placeholders", - "never_incremental", - "no_xml_namespaces", - "png_to_webp", - "post_process_package_resources_script", - "processor_args_javac", - "product_version_resources_dep", - "proguard_configs", - "proguard_enabled", - "proguard_enable_obfuscation", - "r_java_root_package_name", - "resource_exclusion_exceptions", - "resource_exclusion_regex", - "resource_ids_provider_dep", - "resource_values_filter_rules", - "resources_config_paths", - "require_native_mocks", - "secondary_abi_loadable_modules", - "secondary_abi_shared_libraries", - "secondary_native_lib_placeholders", - "shared_libraries", - "shared_resources", - "shared_resources_allowlist_locales", - "shared_resources_allowlist_target", - "short_resource_paths", - "sources", - "srcjar_deps", - "static_library_dependent_targets", - "static_library_provider", - "static_library_synchronized_proguard", - "strip_resource_names", - "support_zh_hk", - "target_sdk_version", - "testonly", - "uncompress_dex", - "uncompress_shared_libraries", - "library_always_compress", - "library_renames", - "use_chromium_linker", - "version_code", - "version_name", - "visibility", - ]) + forward_variables_from( + invoker, + [ + "aapt_locale_allowlist", + "additional_jar_files", + "alternative_android_sdk_dep", + "android_manifest", + "android_manifest_dep", + "annotation_processor_deps", + "apk_under_test", + "app_as_shared_lib", + "assert_no_deps", + "baseline_profile_path", + "build_config_include_product_version_resource", + "bundles_supported", + "chromium_code", + "command_line_flags_file", + "create_apk_script", + "custom_assertion_handler", + "data", + "data_deps", + "deps", + "enable_lint", + "enable_jni_multiplexing", + "enable_multidex", + "enable_native_mocks", + "enable_proguard_checks", + "enforce_resource_overlays_in_tests", + "expected_android_manifest", + "expected_android_manifest_base", + "expected_android_manifest_library_version_offset", + "expected_android_manifest_version_code_offset", + "expected_libs_and_assets", + "expected_libs_and_assets_base", + "generate_buildconfig_java", + "generate_final_jni", + "generate_native_libraries_java", + "include_size_info", + "input_jars_paths", + "jacoco_never_instrument", + "javac_args", + "jni_file_exclusions", + "keystore_name", + "keystore_password", + "keystore_path", + "lint_baseline_file", + "lint_min_sdk_version", + "lint_suppressions_dep", + "lint_suppressions_file", + "loadable_modules", + "manifest_package", + "max_sdk_version", + "mergeable_android_manifests", + "product_config_java_packages", + "main_component_library", + "min_sdk_version", + "native_lib_placeholders", + "never_incremental", + "omit_dex", + "png_to_webp", + "post_process_package_resources_script", + "processor_args_javac", + "proguard_configs", + "proguard_enabled", + "proguard_enable_obfuscation", + "r_java_root_package_name", + "resource_exclusion_exceptions", + "resource_exclusion_regex", + "resource_ids_provider_dep", + "resource_values_filter_rules", + "require_native_mocks", + "secondary_abi_loadable_modules", + "secondary_abi_shared_libraries", + "secondary_native_lib_placeholders", + "shared_libraries", + "shared_resources", + "shared_resources_allowlist_locales", + "shared_resources_allowlist_target", + "sources", + "srcjar_deps", + "static_library_provider", + "static_library_provider_use_secondary_abi", + "target_sdk_version", + "testonly", + "uncompress_dex", + "library_always_compress", + "use_chromium_linker", + "version_code", + "version_name", + "visibility", + ]) is_bundle_module = false name = invoker.apk_name if (defined(invoker.final_apk_path)) { @@ -3554,6 +3706,15 @@ assert(!defined(invoker.bundle_target)) } + # android_app_bundle's write_build_config expects module targets to be named + # according to java_target_patterns otherwise it ignores them when listed in + # possible_config_deps. See https://crbug.com/1418398. + if (filter_exclude([ target_name ], [ "*_bundle_module" ]) != []) { + assert(false, + "Invalid android_app_bundle_module target name ($target_name), " + + "must end in _bundle_module.") + } + # TODO(tiborg): We have several flags that are necessary for workarounds # that come from the fact that the resources get compiled in the bundle # module target, but bundle modules have to have certain flags in @@ -3563,85 +3724,91 @@ # target. Doing so would keep the bundle modules independent from the bundle # and potentially reuse the same bundle modules for multiple bundles. android_apk_or_module(target_name) { - forward_variables_from(invoker, - [ - "aapt_locale_allowlist", - "additional_jar_files", - "alternative_android_sdk_dep", - "android_manifest", - "android_manifest_dep", - "annotation_processor_deps", - "app_as_shared_lib", - "assert_no_deps", - "base_module_target", - "bundle_target", - "chromium_code", - "data", - "data_deps", - "deps", - "enable_multidex", - "expected_android_manifest", - "expected_android_manifest_base", - "extra_verification_manifest", - "extra_verification_manifest_dep", - "generate_buildconfig_java", - "generate_final_jni", - "input_jars_paths", - "isolated_splits_enabled", - "is_base_module", - "jacoco_never_instrument", - "jar_excluded_patterns", - "javac_args", - "jni_registration_header", - "jni_sources_exclusions", - "load_library_from_apk", - "loadable_modules", - "product_config_java_packages", - "manifest_package", - "max_sdk_version", - "min_sdk_version", - "native_lib_placeholders", - "no_xml_namespaces", - "package_id", - "package_name", - "png_to_webp", - "processor_args_javac", - "product_version_resources_dep", - "proguard_configs", - "proguard_enabled", - "proguard_enable_obfuscation", - "resource_exclusion_exceptions", - "resource_exclusion_regex", - "resource_ids_provider_dep", - "resource_values_filter_rules", - "resources_config_paths", - "secondary_abi_loadable_modules", - "secondary_abi_shared_libraries", - "secondary_native_lib_placeholders", - "shared_libraries", - "shared_resources", - "shared_resources_allowlist_locales", - "shared_resources_allowlist_target", - "short_resource_paths", - "srcjar_deps", - "static_library_provider", - "static_library_synchronized_proguard", - "strip_resource_names", - "support_zh_hk", - "target_sdk_version", - "testonly", - "uncompress_shared_libraries", - "library_always_compress", - "library_renames", - "use_chromium_linker", - "use_modern_linker", - "uses_split", - "version_code", - "version_name", - "visibility", - ]) + forward_variables_from( + invoker, + [ + "add_view_trace_events", + "aapt_locale_allowlist", + "additional_jar_files", + "alternative_android_sdk_dep", + "android_manifest", + "android_manifest_dep", + "annotation_processor_deps", + "app_as_shared_lib", + "assert_no_deps", + "base_module_target", + "build_config_include_product_version_resource", + "bundle_target", + "chromium_code", + "custom_assertion_handler", + "data", + "data_deps", + "deps", + "enable_jni_multiplexing", + "enable_multidex", + "expected_android_manifest", + "expected_android_manifest_base", + "expected_android_manifest_library_version_offset", + "expected_android_manifest_version_code_offset", + "generate_buildconfig_java", + "generate_final_jni", + "generate_native_libraries_java", + "input_jars_paths", + "isolated_splits_enabled", + "is_base_module", + "jacoco_never_instrument", + "jar_excluded_patterns", + "javac_args", + "jni_file_exclusions", + "loadable_modules", + "product_config_java_packages", + "main_component_library", + "manifest_package", + "max_sdk_version", + "min_sdk_version", + "mergeable_android_manifests", + "module_name", + "native_lib_placeholders", + "package_id", + "parent_module_target", + "png_to_webp", + "processor_args_javac", + "proguard_configs", + "proguard_enabled", + "proguard_enable_obfuscation", + "resource_exclusion_exceptions", + "resource_exclusion_regex", + "resource_ids_provider_dep", + "resource_values_filter_rules", + "resources_config_paths", + "secondary_abi_loadable_modules", + "secondary_abi_shared_libraries", + "secondary_native_lib_placeholders", + "shared_libraries", + "shared_resources", + "shared_resources_allowlist_locales", + "shared_resources_allowlist_target", + "short_resource_paths", + "srcjar_deps", + "static_library_provider", + "static_library_provider_use_secondary_abi", + "strip_resource_names", + "strip_unused_resources", + "target_sdk_version", + "testonly", + "library_always_compress", + "use_chromium_linker", + "uses_split", + "version_code", + "version_name", + "visibility", + ]) is_bundle_module = true generate_buildconfig_java = _is_base_module + if (defined(uses_split)) { + assert(defined(parent_module_target), + "Must set parent_module_target when uses_split is set") + } } } @@ -3651,7 +3818,6 @@ # # Arguments: # android_test_apk: The target containing the tests. - # android_test_apk_name: The apk_name in android_test_apk # # The following args are optional: # apk_under_test: The target being tested. @@ -3671,10 +3837,106 @@ # Example # instrumentation_test_runner("foo_test_for_bar") { # android_test_apk: ":foo" - # android_test_apk_name: "Foo" # apk_under_test: ":bar" # } template("instrumentation_test_runner") { + if (use_rts) { + action("${invoker.target_name}__rts_filters") { + script = "//build/add_rts_filters.py" + rts_file = "${root_build_dir}/gen/rts/${invoker.target_name}.filter" + inverted_rts_file = + "${root_build_dir}/gen/rts/${invoker.target_name}_inverted.filter" + args = [ + rebase_path(rts_file, root_build_dir), + rebase_path(inverted_rts_file, root_build_dir), + ] + outputs = [ + rts_file, + inverted_rts_file, + ] + } + } + _incremental_apk = !(defined(invoker.never_incremental) && + invoker.never_incremental) && incremental_install + _apk_operations_target_name = "${target_name}__apk_operations" + _apk_target = invoker.android_test_apk + if (defined(invoker.apk_under_test) && !_incremental_apk) { + # The actual target is defined in the test_runner_script template. + _install_artifacts_json = + "${target_gen_dir}/${target_name}.install_artifacts" + _install_artifacts_target_name = "${target_name}__install_artifacts" + } + + action_with_pydeps(_apk_operations_target_name) { + testonly = true + script = "//build/android/gyp/create_test_apk_wrapper_script.py" + deps = [] + _generated_script = "$root_build_dir/bin/${invoker.target_name}" + outputs = [ _generated_script ] + _apk_build_config = + get_label_info(_apk_target, "target_gen_dir") + "/" + + get_label_info(_apk_target, "name") + ".build_config.json" + _rebased_apk_build_config = rebase_path(_apk_build_config, root_build_dir) + args = [ + "--script-output-path", + rebase_path(_generated_script, root_build_dir), + "--package-name", + "@FileArg($_rebased_apk_build_config:deps_info:package_name)", + ] + deps += [ "${_apk_target}$build_config_target_suffix" ] + if (_incremental_apk) { + args += [ + "--test-apk-incremental-install-json", + "@FileArg($_rebased_apk_build_config:deps_info:incremental_install_json_path)", + ] + } else { + args += [ + "--test-apk", + "@FileArg($_rebased_apk_build_config:deps_info:apk_path)", + ] + } + if (defined(invoker.proguard_mapping_path) && !_incremental_apk) { + args += [ + "--proguard-mapping-path", + rebase_path(invoker.proguard_mapping_path, root_build_dir), + ] + } + if (defined(invoker.apk_under_test)) { + if (_incremental_apk) { + deps += [ "${invoker.apk_under_test}$build_config_target_suffix" ] + _apk_under_test_build_config = + get_label_info(invoker.apk_under_test, "target_gen_dir") + "/" + + get_label_info(invoker.apk_under_test, "name") + + ".build_config.json" + _rebased_apk_under_test_build_config = + rebase_path(_apk_under_test_build_config, root_build_dir) + _apk_under_test = "@FileArg($_rebased_apk_under_test_build_config:deps_info:incremental_apk_path)" + } else { + deps += [ ":${_install_artifacts_target_name}" ] + _rebased_install_artifacts_json = + rebase_path(_install_artifacts_json, root_build_dir) + _apk_under_test = "@FileArg($_rebased_install_artifacts_json[])" + } + args += [ + "--additional-apk", + _apk_under_test, + ] + } + if (defined(invoker.additional_apks)) { + foreach(additional_apk, invoker.additional_apks) { + deps += [ "$additional_apk$build_config_target_suffix" ] + _build_config = + get_label_info(additional_apk, "target_gen_dir") + "/" + + get_label_info(additional_apk, "name") + ".build_config.json" + _rebased_build_config = rebase_path(_build_config, root_build_dir) + args += [ + "--additional-apk", + "@FileArg($_rebased_build_config:deps_info:apk_path)", + ] + } + deps += invoker.additional_apks + } + } test_runner_script(target_name) { forward_variables_from(invoker, [ @@ -3687,25 +3949,19 @@ "extra_args", "fake_modules", "ignore_all_data_deps", + "is_unit_test", "modules", - "proguard_enabled", - "public_deps", + "proguard_mapping_path", "use_webview_provider", ]) test_name = invoker.target_name test_type = "instrumentation" - _apk_target_name = get_label_info(invoker.android_test_apk, "name") - apk_target = ":$_apk_target_name" - test_jar = "$root_build_dir/test.lib.java/" + - invoker.android_test_apk_name + ".jar" - incremental_apk = !(defined(invoker.never_incremental) && - invoker.never_incremental) && incremental_install + apk_target = invoker.android_test_apk + incremental_apk = _incremental_apk public_deps = [ - ":$_apk_target_name", - - # Required by test runner to enumerate test list. - ":${_apk_target_name}_dist_ijar", + ":$_apk_operations_target_name", + apk_target, ] if (defined(invoker.apk_under_test)) { public_deps += [ invoker.apk_under_test ] @@ -3713,6 +3969,12 @@ if (defined(invoker.additional_apks)) { public_deps += invoker.additional_apks } + if (use_rts) { + if (!defined(data_deps)) { + data_deps = [] + } + data_deps += [ ":${invoker.target_name}__rts_filters" ] + } } } @@ -3804,7 +4066,6 @@ data += [ "$_final_apk_path.mapping" ] } - dist_ijar_path = "$root_build_dir/test.lib.java/${invoker.apk_name}.jar" create_apk_script = false forward_variables_from(invoker, @@ -3813,6 +4074,8 @@ "data", "data_deps", "deps", + "extra_args", + "is_unit_test", "proguard_configs", ]) } @@ -3841,15 +4104,17 @@ "deps", "extra_args", "ignore_all_data_deps", + "is_unit_test", "modules", "never_incremental", - "proguard_enabled", - "proguard_enable_obfuscation", "public_deps", "use_webview_provider", ]) android_test_apk = ":${_apk_target_name}" - android_test_apk_name = invoker.apk_name + if (defined(invoker.proguard_enabled) && invoker.proguard_enabled) { + proguard_mapping_path = + "$root_build_dir/apks/${invoker.apk_name}.apk.mapping" + } } } @@ -3863,9 +4128,11 @@ # resource dependencies of the apk. # shared_library: shared_library target that contains the unit tests. # apk_name: The name of the produced apk. If unspecified, it uses the name - # of the shared_library target suffixed with "_apk" + # of the shared_library target suffixed with "_apk". # use_default_launcher: Whether the default activity (NativeUnitTestActivity) # should be used for launching tests. + # allow_cleartext_traffic: (Optional) Whether to allow cleartext network + # requests during the test. # use_native_activity: Test implements ANativeActivity_onCreate(). # # Example @@ -3885,6 +4152,8 @@ assert(_use_native_activity != "" && _android_manifest != "") if (!defined(invoker.android_manifest)) { + _allow_cleartext_traffic = defined(invoker.allow_cleartext_traffic) && + invoker.allow_cleartext_traffic jinja_template("${target_name}_manifest") { _native_library_name = get_label_info(invoker.shared_library, "name") if (defined(invoker.android_manifest_template)) { @@ -3898,6 +4167,7 @@ "is_component_build=${is_component_build}", "native_library_name=${_native_library_name}", "use_native_activity=${_use_native_activity}", + "allow_cleartext_traffic=${_allow_cleartext_traffic}", ] } } @@ -3931,7 +4201,6 @@ if (!defined(use_default_launcher) || use_default_launcher) { deps += [ - "//base:base_java", "//build/android/gtest_apk:native_test_instrumentation_test_runner_java", "//testing/android/native_test:native_test_java", ] @@ -4029,6 +4298,10 @@ # absolute paths, such as for third_party or generated .proto files. # http://crbug.com/691451 tracks fixing this. # + # generator_plugin_label (optional) + # GN label for plugin executable which generates custom cc stubs. + # Don't specify a toolchain, host toolchain is assumed. + # # Example: # proto_java_library("foo_proto_java") { # proto_path = "src/foo" @@ -4045,24 +4318,47 @@ _srcjar_path = "$target_gen_dir/$target_name.srcjar" script = "//build/protoc_java.py" - deps = [] if (defined(invoker.deps)) { - deps += invoker.deps + # Need to care only about targets that might generate .proto files. + # No need to depend on java_library or android_resource targets. + deps = filter_exclude(invoker.deps, java_target_patterns) } sources = invoker.sources depfile = "$target_gen_dir/$target_name.d" outputs = [ _srcjar_path ] args = [ - "--depfile", - rebase_path(depfile, root_build_dir), - "--protoc", - rebase_path(android_protoc_bin, root_build_dir), - "--proto-path", - rebase_path(invoker.proto_path, root_build_dir), - "--srcjar", - rebase_path(_srcjar_path, root_build_dir), - ] + rebase_path(sources, root_build_dir) + "--depfile", + rebase_path(depfile, root_build_dir), + "--protoc", + rebase_path(android_protoc_bin, root_build_dir), + "--proto-path", + rebase_path(invoker.proto_path, root_build_dir), + "--srcjar", + rebase_path(_srcjar_path, root_build_dir), + ] + + if (defined(invoker.generator_plugin_label)) { + if (host_os == "win") { + _host_executable_suffix = ".exe" + } else { + _host_executable_suffix = "" + } + + _plugin_host_label = + invoker.generator_plugin_label + "($host_toolchain)" + _plugin_path = + get_label_info(_plugin_host_label, "root_out_dir") + "/" + + get_label_info(_plugin_host_label, "name") + _host_executable_suffix + args += [ + "--plugin", + rebase_path(_plugin_path, root_build_dir), + ] + deps += [ _plugin_host_label ] + inputs = [ _plugin_path ] + } + + args += rebase_path(sources, root_build_dir) if (defined(invoker.import_dirs)) { foreach(_import_dir, invoker.import_dirs) { @@ -4085,6 +4381,81 @@ } } + # Compile a flatbuffer to java. + # + # This generates java files from flat buffers and creates an Android library + # containing the classes. + # + # Variables + # sources (required) + # Paths to .fbs files to compile. + # + # root_dir (required) + # Root directory of .fbs files. + # + # deps (optional) + # Additional dependencies. Passed through to both the action and the + # android_library targets. + # + # flatc_include_dirs (optional) + # A list of extra import directories to be passed to flatc compiler. + # + # + # Example: + # flatbuffer_java_library("foo_flatbuffer_java") { + # root_dir = "src/foo" + # sources = [ "$proto_path/foo.fbs" ] + # } + template("flatbuffer_java_library") { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + + _template_name = target_name + _flatc_dep = "//third_party/flatbuffers:flatc($host_toolchain)" + _flatc_out_dir = get_label_info(_flatc_dep, "root_out_dir") + _flatc_bin = "$_flatc_out_dir/flatc" + + action_with_pydeps("${_template_name}__flatc_java") { + _srcjar_path = "$target_gen_dir/$target_name.srcjar" + script = "//build/android/gyp/flatc_java.py" + + deps = [ _flatc_dep ] + if (defined(invoker.deps)) { + deps += invoker.deps + } + inputs = [ _flatc_bin ] + + sources = invoker.sources + outputs = [ _srcjar_path ] + args = [ + "--flatc", + rebase_path(_flatc_bin, root_build_dir), + "--import-dir", + rebase_path(invoker.root_dir, root_build_dir), + "--srcjar", + rebase_path(_srcjar_path, root_build_dir), + ] + rebase_path(sources, root_build_dir) + + if (defined(invoker.flatc_include_dirs)) { + foreach(_include_dir, invoker.flatc_include_dirs) { + args += [ + "--import-dir", + rebase_path(_include_dir, root_build_dir), + ] + } + } + } + + android_library(target_name) { + chromium_code = false + sources = [] + srcjar_deps = [ ":${_template_name}__flatc_java" ] + deps = [ "//third_party/flatbuffers:flatbuffers_java" ] + if (defined(invoker.deps)) { + deps += invoker.deps + } + } + } + # Declare an Android library target for a prebuilt AAR. # # This target creates an Android library containing java code and Android @@ -4107,9 +4478,6 @@ # ignore_native_libraries: Whether to ignore .so files found in the .aar. # See also extract_native_libraries. # ignore_proguard_configs: Whether to ignore proguard configs. - # ignore_info_updates: Whether to ignore the info file when - # update_android_aar_prebuilts gn arg is true. However, the info file - # will still be verified regardless of the value of this flag. # strip_resources: Whether to ignore android resources found in the .aar. # custom_package: Java package for generated R.java files. # extract_native_libraries: Whether to extract .so files found in the .aar. @@ -4129,8 +4497,14 @@ _info_path = invoker.info_path } _output_path = "${target_out_dir}/${target_name}" + + # Some targets only differ by _java with other targets so _java and _junit + # need to be replaced by non-empty strings to avoid duplicate targets. (e.g. + # androidx_window_window_java vs androidx_window_window_java_java). _target_name_without_java_or_junit = - string_replace(string_replace(target_name, "_java", ""), "_junit", "") + string_replace(string_replace(target_name, "_java", "_J"), + "_junit", + "_U") # This unpack target is a python action, not a valid java target. Since the # java targets below depend on it, its name must not match the java patterns @@ -4153,20 +4527,25 @@ # to keep the logic for generated 'android_aar_prebuilt' rules simple. not_needed(invoker, [ "resource_overlay" ]) - _ignore_info_updates = - defined(invoker.ignore_info_updates) && invoker.ignore_info_updates + _aar_common_args = [ rebase_path(invoker.aar_path, root_build_dir) ] + if (_strip_resources) { + _aar_common_args += [ "--ignore-resources" ] + } + if (defined(invoker.resource_exclusion_globs)) { + _aar_common_args += + [ "--resource-exclusion-globs=${invoker.resource_exclusion_globs}" ] + } # Scan the AAR file and determine the resources and jar files. # Some libraries might not have resources; others might have two jars. - if (!_ignore_info_updates && update_android_aar_prebuilts) { + if (update_android_aar_prebuilts) { print("Writing " + rebase_path(_info_path, "//")) exec_script("//build/android/gyp/aar.py", [ - "list", - rebase_path(invoker.aar_path, root_build_dir), - "--output", - rebase_path(_info_path, root_build_dir), - ]) + "list", + "--output", + rebase_path(_info_path, root_build_dir), + ] + _aar_common_args) } # If "gn gen" is failing on the following line, you need to generate an @@ -4176,6 +4555,7 @@ _scanned_files = read_file(_info_path, "scope") _use_scanned_assets = !_ignore_assets && _scanned_files.assets != [] + _has_resources = _scanned_files.resources != [] assert(_ignore_aidl || _scanned_files.aidl == [], "android_aar_prebuilt() aidl not yet supported." + @@ -4196,30 +4576,24 @@ action_with_pydeps(_unpack_target_name) { script = "//build/android/gyp/aar.py" # Unzips the AAR args = [ - "extract", - rebase_path(invoker.aar_path, root_build_dir), - "--output-dir", - rebase_path(_output_path, root_build_dir), - "--assert-info-file", - rebase_path(_info_path, root_build_dir), - ] - if (_strip_resources) { - args += [ "--ignore-resources" ] - } + "extract", + "--output-dir", + rebase_path(_output_path, root_build_dir), + "--assert-info-file", + rebase_path(_info_path, root_build_dir), + ] + _aar_common_args inputs = [ invoker.aar_path ] outputs = [ "${_output_path}/AndroidManifest.xml" ] - if (!_strip_resources && _scanned_files.has_r_text_file) { + outputs += + get_path_info(rebase_path(_scanned_files.resources, "", _output_path), + "abspath") + if (_scanned_files.has_r_text_file) { # Certain packages, in particular Play Services have no R.txt even # though its presence is mandated by AAR spec. Such packages cause # spurious rebuilds if this output is specified unconditionally. outputs += [ "${_output_path}/R.txt" ] } - if (!_strip_resources && _scanned_files.resources != []) { - outputs += get_path_info( - rebase_path(_scanned_files.resources, "", _output_path), - "abspath") - } if (_scanned_files.has_classes_jar) { outputs += [ "${_output_path}/classes.jar" ] } @@ -4244,15 +4618,11 @@ } } - _has_unignored_resources = - !_strip_resources && - (_scanned_files.resources != [] || _scanned_files.has_r_text_file) - _should_process_manifest = !_ignore_manifest && !_scanned_files.is_manifest_empty # Create the android_resources target for resources. - if (_has_unignored_resources || _should_process_manifest) { + if (_has_resources || _should_process_manifest) { _res_target_name = "${target_name}__resources" android_resources(_res_target_name) { forward_variables_from(invoker, @@ -4262,7 +4632,7 @@ "testonly", "strip_drawables", ]) - deps = [ ":$_unpack_target_name" ] + public_deps = [ ":$_unpack_target_name" ] if (_should_process_manifest) { android_manifest_dep = ":$_unpack_target_name" android_manifest = "${_output_path}/AndroidManifest.xml" @@ -4271,11 +4641,8 @@ custom_package = _scanned_files.manifest_package } - sources = [] - if (!_strip_resources) { - sources = rebase_path(_scanned_files.resources, "", _output_path) - } - if (!_strip_resources && _scanned_files.has_r_text_file) { + sources = rebase_path(_scanned_files.resources, "", _output_path) + if (_scanned_files.has_r_text_file) { r_text_file = "${_output_path}/R.txt" } } @@ -4299,6 +4666,7 @@ _assets_target_name = "${target_name}__assets" android_assets(_assets_target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + deps = [ ":$_unpack_target_name" ] renaming_sources = [] renaming_destinations = [] foreach(_asset_file, _scanned_files.assets) { @@ -4312,9 +4680,12 @@ } } + _target_label = get_label_info(":$target_name", "label_no_toolchain") + # Create android_java_prebuilt target for classes.jar. if (_scanned_files.has_classes_jar) { _java_library_vars = [ + "alternative_android_sdk_dep", "bytecode_rewriter_target", "enable_bytecode_checks", "jar_excluded_patterns", @@ -4339,7 +4710,7 @@ jar_path = "$_output_path/${_tuple[1]}" _base_output_name = get_path_info(jar_path, "name") output_name = "${invoker.target_name}-$_base_output_name" - public_target_label = invoker.target_name + public_target_label = _target_label } } @@ -4350,6 +4721,7 @@ [ "deps", "input_jars_paths", + "mergeable_android_manifests", "proguard_configs", ]) if (!defined(deps)) { @@ -4375,13 +4747,16 @@ proguard_configs += [ "$_output_path/proguard.txt" ] } } - public_target_label = invoker.target_name + public_target_label = _target_label } } java_group(target_name) { forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) public_deps = [ ":$_unpack_target_name" ] + if (defined(invoker.public_deps)) { + public_deps += invoker.public_deps + } deps = [] if (defined(_jar_target_name)) { deps += [ ":$_jar_target_name" ] @@ -4428,6 +4803,7 @@ # keystore_name: optional keystore name, used only when generating APKs. # keystore_password: optional keystore password, used only when # generating APKs. + # rotation_config: optional .textproto to enable key rotation. # # command_line_flags_file: Optional. If provided, named of the on-device # file that will be used to store command-line arguments. The default @@ -4449,8 +4825,7 @@ # used as a library jar for synchronized proguarding. # # compress_shared_libraries: Optional. Whether to compress shared libraries - # such that they are extracted upon install. Libraries prefixed with - # "crazy." are never compressed. + # such that they are extracted upon install. # # system_image_locale_allowlist: List of locales that should be included # on system APKs generated from this bundle. @@ -4479,6 +4854,11 @@ # default_modules_for_testing: (optional): A list of DFM that the wrapper # script should install. This is for local testing only, and does not # affect the actual DFM in production. + # + # add_view_trace_events: (optional): If true will add an additional step to + # add trace events to all Android views contained in the bundle. It also + # requires build argument enable_trace_event_bytecode_rewriting = true. + # # Example: # android_app_bundle("chrome_public_bundle") { # base_module_target = "//chrome/android:chrome_public_apk" @@ -4496,19 +4876,11 @@ _proguard_enabled = defined(invoker.proguard_enabled) && invoker.proguard_enabled - if (defined(invoker.version_code)) { - _version_code = invoker.version_code - } else { - _version_code = android_default_version_code + _min_sdk_version = default_min_sdk_version + if (defined(invoker.min_sdk_version)) { + _min_sdk_version = invoker.min_sdk_version } - if (android_override_version_code != "") { - _version_code = android_override_version_code - } - - # Prevent "unused variable". - not_needed([ "_version_code" ]) - _bundle_base_path = "$root_build_dir/apks" if (defined(invoker.bundle_base_path)) { _bundle_base_path = invoker.bundle_base_path @@ -4525,7 +4897,7 @@ _base_target_gen_dir = get_label_info(invoker.base_module_target, "target_gen_dir") _base_module_build_config = - "$_base_target_gen_dir/${_base_target_name}.build_config" + "$_base_target_gen_dir/${_base_target_name}.build_config.json" _base_module_build_config_target = "${invoker.base_module_target}$build_config_target_suffix" _rebased_base_module_build_config = @@ -4543,22 +4915,7 @@ }, ] - _enable_multidex = - !defined(invoker.enable_multidex) || invoker.enable_multidex - - if (!_proguard_enabled && defined(invoker.min_sdk_version)) { - not_needed(invoker, [ "min_sdk_version" ]) - } - - # Prevent "unused variable". - not_needed([ "_enable_multidex" ]) - if (_proguard_enabled) { - _uses_static_library_synchronized_proguard = - defined(invoker.static_library_synchronized_proguard) && - invoker.static_library_synchronized_proguard - - # TODO(crbug.com/1032609): Remove dexsplitter from Trichrome Proguard. _dex_target = "${_target_name}__dex" _proguard_mapping_path = "${_bundle_path}.mapping" } @@ -4584,7 +4941,7 @@ _module_target_gen_dir = get_label_info(_module_target, "target_gen_dir") _module.build_config = - "$_module_target_gen_dir/${_module_target_name}.build_config" + "$_module_target_gen_dir/${_module_target_name}.build_config.json" _module.build_config_target = "$_module_target$build_config_target_suffix" _module.parent = "base" @@ -4610,8 +4967,7 @@ get_label_info(invoker.static_library_provider, "name") _static_library_gen_dir = get_label_info(invoker.static_library_provider, "target_gen_dir") - _lib_proxy_module.build_config = - "$_static_library_gen_dir/$_static_library_target_name.build_config" + _lib_proxy_module.build_config = "$_static_library_gen_dir/$_static_library_target_name.build_config.json" _lib_proxy_module.build_config_target = "${invoker.static_library_provider}$build_config_target_suffix" } @@ -4662,74 +5018,156 @@ deps = [ "${invoker.base_module_target}__compile_resources" ] } - _build_config = "$target_gen_dir/${_target_name}.build_config" + _build_config = "$target_gen_dir/${_target_name}.build_config.json" _rebased_build_config = rebase_path(_build_config, root_build_dir) _build_config_target = "$_target_name$build_config_target_suffix" if (defined(invoker.proguard_android_sdk_dep)) { - proguard_android_sdk_dep_ = invoker.proguard_android_sdk_dep + _android_sdk_dep = invoker.proguard_android_sdk_dep } else { - proguard_android_sdk_dep_ = "//third_party/android_sdk:android_sdk_java" + _android_sdk_dep = default_android_sdk_dep } if (_proguard_enabled) { _proguard_mapping_path = "${_bundle_path}.mapping" + _add_view_trace_events = + defined(invoker.add_view_trace_events) && + invoker.add_view_trace_events && enable_trace_event_bytecode_rewriting + } else { + not_needed(invoker, [ "add_view_trace_events" ]) } write_build_config(_build_config_target) { type = "android_app_bundle" - possible_config_deps = _module_targets + [ proguard_android_sdk_dep_ ] + possible_config_deps = _module_targets + [ _android_sdk_dep ] build_config = _build_config proguard_enabled = _proguard_enabled module_build_configs = _module_build_configs + modules = _modules if (_proguard_enabled) { + add_view_trace_events = _add_view_trace_events proguard_mapping_path = _proguard_mapping_path } } if (_proguard_enabled) { - # If this Bundle uses a static library, the static library APK will - # create the synchronized dex file path. - if (_uses_static_library_synchronized_proguard) { - if (defined(invoker.min_sdk_version)) { - not_needed(invoker, [ "min_sdk_version" ]) + if (_add_view_trace_events) { + _trace_event_rewriter_target = + "//build/android/bytecode:trace_event_adder" + _rewritten_jar_target_name = "${target_name}__trace_event_rewritten" + _rewriter_path = root_build_dir + "/bin/helper/trace_event_adder" + _stamp = "${target_out_dir}/${target_name}.trace_event_rewrite.stamp" + action_with_pydeps(_rewritten_jar_target_name) { + script = "//build/android/gyp/trace_event_bytecode_rewriter.py" + inputs = [ + _rewriter_path, + _build_config, + ] + outputs = [ _stamp ] + depfile = "$target_gen_dir/$_rewritten_jar_target_name.d" + args = [ + "--stamp", + rebase_path(_stamp, root_build_dir), + "--depfile", + rebase_path(depfile, root_build_dir), + "--script", + rebase_path(_rewriter_path, root_build_dir), + "--classpath", + "@FileArg($_rebased_build_config:deps_info:javac_full_classpath)", + "--classpath", + "@FileArg($_rebased_build_config:android:sdk_jars)", + "--input-jars", + "@FileArg($_rebased_build_config:deps_info:device_classpath)", + "--output-jars", + "@FileArg($_rebased_build_config:deps_info:trace_event_rewritten_device_classpath)", + ] + deps = [ + ":$_build_config_target", + _trace_event_rewriter_target, + ] + _module_java_targets } - } else { - dex(_dex_target) { - forward_variables_from(invoker, - [ - "expected_proguard_config", - "expected_proguard_config_base", - "min_sdk_version", - "proguard_enable_obfuscation", - ]) - if (defined(expected_proguard_config)) { - top_target_name = _target_name - } - enable_multidex = _enable_multidex - proguard_enabled = true - proguard_mapping_path = _proguard_mapping_path - proguard_sourcefile_suffix = "$android_channel-$_version_code" - build_config = _build_config + } - deps = _module_java_targets + [ ":$_build_config_target" ] - modules = _modules + dex(_dex_target) { + forward_variables_from(invoker, + [ + "custom_assertion_handler", + "expected_proguard_config", + "expected_proguard_config_base", + "proguard_enable_obfuscation", + ]) + if (defined(expected_proguard_config)) { + top_target_name = _target_name } + min_sdk_version = _min_sdk_version + add_view_trace_events = _add_view_trace_events + proguard_enabled = true + proguard_mapping_path = _proguard_mapping_path + build_config = _build_config + + deps = _module_java_targets + [ ":$_build_config_target" ] + if (_add_view_trace_events) { + deps += [ ":${_rewritten_jar_target_name}" ] + } + modules = _modules } } _all_create_module_targets = [] _all_module_zip_paths = [] _all_module_build_configs = [] + _all_module_unused_resources_deps = [] foreach(_module, _modules) { _module_target = _module.module_target _module_build_config = _module.build_config _module_build_config_target = _module.build_config_target + _module_target_name = get_label_info(_module_target, "name") if (!_proguard_enabled) { - _dex_target_for_module = "${_module_target}__final_dex" + _dex_target = "${_module_target_name}__final_dex" + _dex_path = "$target_out_dir/$_module_target_name/$_module_target_name.mergeddex.jar" + dex(_dex_target) { + forward_variables_from(invoker, [ "custom_assertion_handler" ]) + min_sdk_version = _min_sdk_version + output = _dex_path + build_config = _build_config + + # This will be a pure dex-merge. + input_dex_filearg = "@FileArg($_rebased_build_config:modules:${_module.name}:all_dex_files)" + enable_desugar = false + + deps = [ + ":$_build_config_target", + ":${_module_target_name}__java", + ] + } + } + _dex_target_for_module = ":$_dex_target" + + _use_baseline_profile = + _proguard_enabled && defined(invoker.baseline_profile_path) && + enable_baseline_profiles + if (_use_baseline_profile) { + _binary_profile_target = + "${_module_target_name}__binary_baseline_profile" + _binary_baseline_profile_path = "$target_out_dir/$_module_target_name/$_module_target_name.baseline.prof" + _binary_baseline_profile_metadata_path = + _binary_baseline_profile_path + "m" + create_binary_profile(_binary_profile_target) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + binary_baseline_profile_path = _binary_baseline_profile_path + binary_baseline_profile_metadata_path = + _binary_baseline_profile_metadata_path + proguard_mapping_path = _proguard_mapping_path + build_config = _module_build_config + input_profile_path = invoker.baseline_profile_path + deps = [ + _dex_target_for_module, + _module_build_config_target, + ] + } } else { - _dex_target_for_module = ":$_dex_target" + not_needed(invoker, [ "baseline_profile_path" ]) } # Generate one module .zip file per bundle module. @@ -4738,20 +5176,22 @@ # the internal module name inside the final bundle, in other words, # this file *must* be named ${_module.name}.zip _create_module_target = "${_target_name}__${_module.name}__create" - _module_zip_path = "$target_gen_dir/$target_name/${_module.name}.zip" - + _module_zip_path = "$target_out_dir/$target_name/${_module.name}.zip" create_android_app_bundle_module(_create_module_target) { forward_variables_from(invoker, [ "is_multi_abi", - "min_sdk_version", "uncompress_dex", - "proguard_enabled", ]) module_name = _module.name + min_sdk_version = _min_sdk_version build_config = _module_build_config module_zip_path = _module_zip_path native_libraries_config = _native_libraries_config + if (!_proguard_enabled) { + dex_path = _dex_path + # dex_path is read from the build_config in the proguard case. + } if (module_name == "base" && defined(invoker.expected_libs_and_assets)) { @@ -4781,6 +5221,17 @@ _secondary_abi_native_libraries_config deps += [ ":$_secondary_abi_native_libraries_config_target" ] } + + if (_use_baseline_profile) { + # extra_assets is a list of ["{src_path}:{dst_path}"] + extra_assets = [ + rebase_path(_binary_baseline_profile_path, root_build_dir) + + ":dexopt/baseline.prof", + rebase_path(_binary_baseline_profile_metadata_path, + root_build_dir) + ":dexopt/baseline.profm", + ] + deps += [ ":$_binary_profile_target" ] + } } _all_create_module_targets += [ @@ -4790,6 +5241,40 @@ ] _all_module_zip_paths += [ _module_zip_path ] _all_module_build_configs += [ _module_build_config ] + _all_module_unused_resources_deps += [ + "${_module_target}__compile_resources", + _dex_target_for_module, + _module_build_config_target, + ] + } + _strip_unused_resources = defined(invoker.strip_unused_resources) && + invoker.strip_unused_resources + if (_strip_unused_resources) { + # Resources only live in the base module so we define the unused resources + # target only on the base module target. + _unused_resources_target = "${_base_target_name}__unused_resources" + _unused_resources_config = + "${_base_target_gen_dir}/${_base_target_name}_unused_resources.config" + _unused_resources_r_txt_out = + "${_base_target_gen_dir}/${_base_target_name}_unused_resources.R.txt" + unused_resources(_unused_resources_target) { + deps = _all_module_unused_resources_deps + all_module_build_configs = _all_module_build_configs + build_config = _base_module_build_config + if (_proguard_enabled) { + proguard_mapping_path = _proguard_mapping_path + } + output_config = _unused_resources_config + output_r_txt = _unused_resources_r_txt_out + } + _unused_resources_final_path = "${_bundle_path}.unused_resources" + _copy_unused_resources_target = + "${_base_target_name}__copy_unused_resources" + copy(_copy_unused_resources_target) { + deps = [ ":$_unused_resources_target" ] + sources = [ _unused_resources_config ] + outputs = [ _unused_resources_final_path ] + } } _all_rebased_module_zip_paths = @@ -4818,7 +5303,8 @@ _bundle_target_name = "${_target_name}__bundle" action_with_pydeps(_bundle_target_name) { script = "//build/android/gyp/create_app_bundle.py" - inputs = _all_module_zip_paths + _all_module_build_configs + inputs = _all_module_zip_paths + _all_module_build_configs + + [ _BUNDLETOOL_JAR_PATH ] outputs = [ _bundle_path ] deps = _all_create_module_targets + [ ":$_build_config_target" ] args = [ @@ -4834,16 +5320,37 @@ invoker.compress_shared_libraries) { args += [ "--compress-shared-libraries" ] } + + # Android P+ support loading from stored dex. + if (_min_sdk_version < 27) { + args += [ "--compress-dex" ] + } + + if (defined(invoker.rotation_config)) { + args += [ + "--rotation-config", + rebase_path(invoker.rotation_config, root_build_dir), + ] + } + if (treat_warnings_as_errors) { args += [ "--warnings-as-errors" ] } if (_enable_language_splits) { - args += [ - "--base-allowlist-rtxt-path=@FileArg(" + "${_rebased_base_module_build_config}:deps_info:base_allowlist_rtxt_path)", - "--base-module-rtxt-path=@FileArg(" + - "${_rebased_base_module_build_config}:deps_info:r_text_path)", - ] + args += [ "--base-allowlist-rtxt-path=@FileArg($_rebased_base_module_build_config:deps_info:base_allowlist_rtxt_path)" ] + if (_strip_unused_resources) { + # Use the stripped out rtxt file to set resources that are pinned to + # the default language split. + _rebased_unused_resources_r_txt_out = + rebase_path(_unused_resources_r_txt_out, root_build_dir) + inputs += [ _unused_resources_r_txt_out ] + deps += [ ":$_unused_resources_target" ] + args += + [ "--base-module-rtxt-path=$_rebased_unused_resources_r_txt_out" ] + } else { + args += [ "--base-module-rtxt-path=@FileArg($_rebased_base_module_build_config:deps_info:r_text_path)" ] + } } if (defined(invoker.validate_services) && invoker.validate_services) { args += [ "--validate-services" ] @@ -4926,8 +5433,7 @@ args = [ "--script-output-path", rebase_path(_bundle_wrapper_script_path, root_build_dir), - "--package-name=@FileArg(" + - "$_rebased_base_module_build_config:deps_info:package_name)", + "--package-name=@FileArg($_rebased_base_module_build_config:deps_info:package_name)", "--aapt2", rebase_path(_android_aapt2_path, root_build_dir), "--bundle-path", @@ -4987,8 +5493,8 @@ forward_variables_from(invoker, [ "lint_baseline_file", + "lint_jar_path", "lint_suppressions_file", - "min_sdk_version", ]) build_config = _build_config build_config_dep = ":$_build_config_target" @@ -4998,12 +5504,15 @@ } if (defined(invoker.lint_min_sdk_version)) { min_sdk_version = invoker.lint_min_sdk_version + } else { + min_sdk_version = _min_sdk_version } } } else { not_needed(invoker, [ "lint_baseline_file", + "lint_jar_path", "lint_min_sdk_version", "lint_suppressions_dep", "lint_suppressions_file", @@ -5029,7 +5538,10 @@ _apks_path = "$root_build_dir/apks/$_bundle_name.apks" action_with_pydeps("${_target_name}_apks") { script = "//build/android/gyp/create_app_bundle_apks.py" - inputs = [ _bundle_path ] + inputs = [ + _bundle_path, + _BUNDLETOOL_JAR_PATH, + ] outputs = [ _apks_path ] data = [ _apks_path ] args = [ @@ -5046,6 +5558,9 @@ "--keystore-password", android_keystore_password, ] + if (debuggable_apks) { + args += [ "--local-testing" ] + } deps = [ ":$_bundle_target_name" ] metadata = { install_artifacts = [ _apks_path ]
diff --git a/build/config/android/sdk.gni b/build/config/android/sdk.gni index d2e67a7..fb39315 100644 --- a/build/config/android/sdk.gni +++ b/build/config/android/sdk.gni
@@ -1,10 +1,13 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # The default SDK release used by public builds. Value may differ in # internal builds. -default_android_sdk_release = "r" +default_android_sdk_release = "t" # SDK releases against which public builds are supported. -public_sdk_releases = [ "r" ] +public_sdk_releases = [ + "t", + "tprivacysandbox", +]
diff --git a/build/config/android/system_image.gni b/build/config/android/system_image.gni new file mode 100644 index 0000000..79f8560 --- /dev/null +++ b/build/config/android/system_image.gni
@@ -0,0 +1,174 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/android/rules.gni") + +# Creates a stub .apk suitable for use with compressed system APKs. +# +# Variables: +# package_name: Package name to use for the stub. +# version_code: Version code for the stub. +# version_name: Version name for the stub. +# package_info_from_target: Use the package name and version_code from this +# apk/bundle target. +# static_library_name: For static library apks, name for the <static-library>. +# static_library_version: For static library apks, version for the +# <static-library> tag (for TrichromeLibrary, we set this to be the same +# as the package's version_code) +# stub_output: Path to output stub apk (default: do not create a stub). +# +# package_name and package_info_from_target are mutually exclusive. +template("system_image_stub_apk") { + # Android requires stubs end with -Stub.apk. + assert(filter_exclude([ invoker.stub_output ], [ "*-Stub.apk" ]) == [], + "stub_output \"${invoker.stub_output}\" must end with \"-Stub.apk\"") + + _resource_apk_path = "${target_out_dir}/$target_name.ap_" + _resource_apk_target_name = "${target_name}__compile_resources" + + _manifest_target_name = "${target_name}__manifest" + _manifest_path = "$target_gen_dir/$_manifest_target_name.xml" + action("$_manifest_target_name") { + outputs = [ _manifest_path ] + script = "//build/android/gyp/create_stub_manifest.py" + args = [ + "--output", + rebase_path(_manifest_path, root_build_dir), + ] + if (defined(invoker.static_library_name)) { + args += [ + "--static-library-name", + invoker.static_library_name, + ] + + # TODO(crbug.com/1408164): Make static_library_version mandatory. + if (defined(invoker.static_library_version)) { + args += [ + "--static-library-version", + invoker.static_library_version, + ] + } else { + args += [ "--static-library-version=1" ] + } + } + } + + action_with_pydeps(_resource_apk_target_name) { + script = "//build/android/gyp/compile_resources.py" + inputs = [ + _manifest_path, + android_sdk_jar, + ] + outputs = [ _resource_apk_path ] + args = [ + "--aapt2-path", + rebase_path(android_sdk_tools_bundle_aapt2, root_build_dir), + "--min-sdk-version=$default_min_sdk_version", + "--target-sdk-version=$default_android_sdk_version", + "--android-manifest", + rebase_path(_manifest_path, root_build_dir), + "--arsc-path", + rebase_path(_resource_apk_path, root_build_dir), + ] + deps = [ ":$_manifest_target_name" ] + if (defined(invoker.package_name)) { + _package_name = invoker.package_name + _version_code = invoker.version_code + _version_name = invoker.version_name + } else { + _target = invoker.package_info_from_target + deps += [ "${_target}$build_config_target_suffix" ] + _build_config = get_label_info(_target, "target_gen_dir") + "/" + + get_label_info(_target, "name") + ".build_config.json" + inputs += [ _build_config ] + _rebased_build_config = rebase_path(_build_config, root_build_dir) + _package_name = "@FileArg($_rebased_build_config:deps_info:package_name)" + _version_code = "@FileArg($_rebased_build_config:deps_info:version_code)" + _version_name = "@FileArg($_rebased_build_config:deps_info:version_name)" + } + args += [ + "--rename-manifest-package=$_package_name", + "--arsc-package-name=$_package_name", + "--version-code=$_version_code", + "--version-name=$_version_name", + "--include-resources", + rebase_path(android_sdk_jar, root_build_dir), + ] + } + + package_apk(target_name) { + forward_variables_from(invoker, + [ + "keystore_name", + "keystore_path", + "keystore_password", + ]) + min_sdk_version = default_min_sdk_version + deps = [ ":$_resource_apk_target_name" ] + + packaged_resources_path = _resource_apk_path + output_apk_path = invoker.stub_output + } +} + +# Generates artifacts for system APKs. +# +# Variables: +# apk_or_bundle_target: Target that creates input bundle or apk. +# input_apk_or_bundle: Path to input .apk or .aab. +# static_library_name: For static library apks, name for the <static-library>. +# static_library_version: For static library apks, version for the +# <static-library> tag (for TrichromeLibrary, we set this to be the same +# as the package's version_code) +# output: Path to the output system .apk or .zip. +# fuse_apk: Fuse all apk splits into a single .apk (default: false). +# stub_output: Path to output stub apk (default: do not create a stub). +# +template("system_image_apks") { + if (defined(invoker.stub_output)) { + _stub_apk_target_name = "${target_name}__stub" + system_image_stub_apk(_stub_apk_target_name) { + forward_variables_from(invoker, + [ + "static_library_name", + "static_library_version", + ]) + package_info_from_target = invoker.apk_or_bundle_target + stub_output = invoker.stub_output + } + } + + action_with_pydeps(target_name) { + script = "//build/android/gyp/system_image_apks.py" + deps = [ invoker.apk_or_bundle_target ] + inputs = [ invoker.input_apk_or_bundle ] + if (defined(invoker.stub_output)) { + public_deps = [ ":$_stub_apk_target_name" ] + } + outputs = [ invoker.output ] + args = [ + "--input", + rebase_path(invoker.input_apk_or_bundle, root_out_dir), + "--output", + rebase_path(invoker.output, root_out_dir), + ] + + _is_bundle = + filter_exclude([ invoker.input_apk_or_bundle ], [ "*.aab" ]) == [] + + if (_is_bundle) { + _wrapper_path = "$root_out_dir/bin/" + + get_label_info(invoker.apk_or_bundle_target, "name") + args += [ + "--bundle-wrapper", + rebase_path(_wrapper_path, root_out_dir), + ] + inputs += [ _wrapper_path ] + deps += [ "//build/android:apk_operations_py" ] + if (defined(invoker.fuse_apk) && invoker.fuse_apk) { + args += [ "--fuse-apk" ] + } + } + } +}
diff --git a/build/config/android/test/classpath_order/BUILD.gn b/build/config/android/test/classpath_order/BUILD.gn deleted file mode 100644 index decd1a8..0000000 --- a/build/config/android/test/classpath_order/BUILD.gn +++ /dev/null
@@ -1,111 +0,0 @@ -# Copyright 2021 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import("//build/config/android/rules.gni") - -template("test_resources") { - jinja_template_resources(target_name) { - forward_variables_from(invoker, "*") - testonly = true - variables = [ "resource_name=$resource_name" ] - res_dir = "java/res_template" - resources = [ "java/res_template/values/values.xml" ] - } -} - -template("generate_dummy_android_library") { - # No underscores to avoid crbug.com/908819. - _generate_java_source_target_name = "${target_name}generatejavasource" - jinja_template(_generate_java_source_target_name) { - testonly = true - input = "java/src/org/chromium/build/classpath_order/Dummy.java.jinja2" - output = "$target_gen_dir/java/src/org/chromium/build/classpath_order/${invoker.class_name}.java" - variables = [ "class_name=${invoker.class_name}" ] - } - - android_library(target_name) { - forward_variables_from(invoker, "*") - - if (!defined(invoker.deps)) { - deps = [] - } - - sources = get_target_outputs(":${_generate_java_source_target_name}") - deps += [ ":${_generate_java_source_target_name}" ] - } -} - -# Test that classpath order keeps resources accessible when multiple targets generate -# resources for the same package. Specifically, test that an android_library precedes -# its dependencies regardless of the relative lexographic order. - -test_resources("a1_dependency_resources") { - resource_name = "a1_dependency_resource" -} - -generate_dummy_android_library("a1_dependency_java") { - testonly = true - class_name = "A1Dependency" - resources_package = "org.chromium.build.classpath_order.test1" - deps = [ ":a1_dependency_resources" ] -} - -test_resources("z1_master_resources") { - resource_name = "z1_master_resource" - deps = [ ":a1_dependency_resources" ] -} - -generate_dummy_android_library("z1_master_java") { - testonly = true - class_name = "Z1Master" - resources_package = "org.chromium.build.classpath_order.test1" - deps = [ - ":a1_dependency_java", - ":z1_master_resources", - ] -} - -test_resources("z2_dependency_resources") { - resource_name = "z2_dependency_resource" -} - -generate_dummy_android_library("z2_dependency_java") { - testonly = true - class_name = "Z2Dependency" - resources_package = "org.chromium.build.classpath_order.test2" - deps = [ ":z2_dependency_resources" ] -} - -test_resources("a2_master_resources") { - resource_name = "a2_master_resource" - deps = [ ":z2_dependency_resources" ] -} - -generate_dummy_android_library("a2_master_java") { - testonly = true - class_name = "A2Master" - resources_package = "org.chromium.build.classpath_order.test2" - deps = [ - ":a2_master_resources", - ":z2_dependency_java", - ] -} - -java_library("junit_tests") { - bypass_platform_checks = true - testonly = true - sources = - [ "java/src/org/chromium/build/classpath_order/ClassPathOrderTest.java" ] - deps = [ - ":a1_dependency_java", - ":a2_master_java", - ":z1_master_java", - ":z2_dependency_java", - "//testing/android/junit:junit_test_support", - "//third_party/android_deps:robolectric_all_java", - "//third_party/android_support_test_runner:runner_java", - "//third_party/androidx:androidx_test_runner_java", - "//third_party/junit", - ] -}
diff --git a/build/config/android/test/classpath_order/java/res_template/values/values.xml b/build/config/android/test/classpath_order/java/res_template/values/values.xml deleted file mode 100644 index ee706b2..0000000 --- a/build/config/android/test/classpath_order/java/res_template/values/values.xml +++ /dev/null
@@ -1,9 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright 2021 The Chromium Authors. All rights reserved. - Use of this source code is governed by a BSD-style license that can be - found in the LICENSE file. --> - - -<resources xmlns:android="http://schemas.android.com/apk/res/android"> - <integer name="{{resource_name}}">42</integer> -</resources>
diff --git a/build/config/android/test/classpath_order/java/src/org/chromium/build/classpath_order/ClassPathOrderTest.java b/build/config/android/test/classpath_order/java/src/org/chromium/build/classpath_order/ClassPathOrderTest.java deleted file mode 100644 index c5a9202..0000000 --- a/build/config/android/test/classpath_order/java/src/org/chromium/build/classpath_order/ClassPathOrderTest.java +++ /dev/null
@@ -1,32 +0,0 @@ -// Copyright 2021 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package org.chromium.build.classpath_order; - -import static org.junit.Assert.assertTrue; - -import androidx.test.filters.SmallTest; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.robolectric.annotation.Config; - -import org.chromium.testing.local.LocalRobolectricTestRunner; - -/** - * Test that resources defined in different android_resources() targets but with the same - * package are accessible. - */ -@RunWith(LocalRobolectricTestRunner.class) -@Config(manifest = Config.NONE) -public final class ClassPathOrderTest { - @Test - @SmallTest - public void testAll() { - assertTrue(org.chromium.build.classpath_order.test1.R.integer.a1_dependency_resource >= 0); - assertTrue(org.chromium.build.classpath_order.test1.R.integer.z1_master_resource >= 0); - assertTrue(org.chromium.build.classpath_order.test2.R.integer.z2_dependency_resource >= 0); - assertTrue(org.chromium.build.classpath_order.test2.R.integer.a2_master_resource >= 0); - } -}
diff --git a/build/config/android/test/classpath_order/java/src/org/chromium/build/classpath_order/Dummy.java.jinja2 b/build/config/android/test/classpath_order/java/src/org/chromium/build/classpath_order/Dummy.java.jinja2 deleted file mode 100644 index 0ccf28b..0000000 --- a/build/config/android/test/classpath_order/java/src/org/chromium/build/classpath_order/Dummy.java.jinja2 +++ /dev/null
@@ -1,8 +0,0 @@ -// Copyright 2021 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package org.chromium.build.classpath_order; - -public class {{class_name}} { -}
diff --git a/build/config/android/test/proto/BUILD.gn b/build/config/android/test/proto/BUILD.gn index a28111a..1d0f37a 100644 --- a/build/config/android/test/proto/BUILD.gn +++ b/build/config/android/test/proto/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/android/test/proto/absolute_dep/absolute_dep.proto b/build/config/android/test/proto/absolute_dep/absolute_dep.proto index 46dcce7..f4aa92a 100644 --- a/build/config/android/test/proto/absolute_dep/absolute_dep.proto +++ b/build/config/android/test/proto/absolute_dep/absolute_dep.proto
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/android/test/proto/relative_dep/relative_dep.proto b/build/config/android/test/proto/relative_dep/relative_dep.proto index 600b6ca..917d2c3 100644 --- a/build/config/android/test/proto/relative_dep/relative_dep.proto +++ b/build/config/android/test/proto/relative_dep/relative_dep.proto
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/android/test/proto/root/absolute_child.proto b/build/config/android/test/proto/root/absolute_child.proto index d6a6a13..389538c 100644 --- a/build/config/android/test/proto/root/absolute_child.proto +++ b/build/config/android/test/proto/root/absolute_child.proto
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/android/test/proto/root/absolute_root.proto b/build/config/android/test/proto/root/absolute_root.proto index 3e20097..ad138ab 100644 --- a/build/config/android/test/proto/root/absolute_root.proto +++ b/build/config/android/test/proto/root/absolute_root.proto
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/android/test/proto/root/relative_child.proto b/build/config/android/test/proto/root/relative_child.proto index 10f7ed4..225758c 100644 --- a/build/config/android/test/proto/root/relative_child.proto +++ b/build/config/android/test/proto/root/relative_child.proto
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/android/test/proto/root/relative_root.proto b/build/config/android/test/proto/root/relative_root.proto index a37a268..9644fa1 100644 --- a/build/config/android/test/proto/root/relative_root.proto +++ b/build/config/android/test/proto/root/relative_root.proto
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/android/test/resource_overlay/BUILD.gn b/build/config/android/test/resource_overlay/BUILD.gn index 4a063d2..3b79363 100644 --- a/build/config/android/test/resource_overlay/BUILD.gn +++ b/build/config/android/test/resource_overlay/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -43,7 +43,7 @@ deps = [ ":root_tagged_dependency_resources" ] } -android_library("javatests") { +android_library("unit_device_javatests") { testonly = true sources = [ "java/src/org/chromium/build/resource_overlay/ResourceOverlayTest.java", @@ -53,7 +53,7 @@ ":dependency_tagged_root_resources", ":root_tagged_root_resources", "//base:base_java_test_support", - "//third_party/android_support_test_runner:runner_java", + "//third_party/androidx:androidx_test_monitor_java", "//third_party/androidx:androidx_test_runner_java", "//third_party/junit", ]
diff --git a/build/config/android/test/resource_overlay/java/res_template/values/values.xml b/build/config/android/test/resource_overlay/java/res_template/values/values.xml index 973f855..13ff516 100644 --- a/build/config/android/test/resource_overlay/java/res_template/values/values.xml +++ b/build/config/android/test/resource_overlay/java/res_template/values/values.xml
@@ -1,5 +1,5 @@ <?xml version="1.0" encoding="utf-8"?> -<!-- Copyright 2020 The Chromium Authors. All rights reserved. +<!-- Copyright 2020 The Chromium Authors Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
diff --git a/build/config/android/test/resource_overlay/java/src/org/chromium/build/resource_overlay/ResourceOverlayTest.java b/build/config/android/test/resource_overlay/java/src/org/chromium/build/resource_overlay/ResourceOverlayTest.java index 794cafa..d42450e 100644 --- a/build/config/android/test/resource_overlay/java/src/org/chromium/build/resource_overlay/ResourceOverlayTest.java +++ b/build/config/android/test/resource_overlay/java/src/org/chromium/build/resource_overlay/ResourceOverlayTest.java
@@ -1,4 +1,4 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. +// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -7,8 +7,8 @@ import static org.junit.Assert.assertEquals; import android.content.res.Resources; -import android.support.test.InstrumentationRegistry; +import androidx.test.InstrumentationRegistry; import androidx.test.filters.SmallTest; import org.junit.Test;
diff --git a/build/config/apple/BUILD.gn b/build/config/apple/BUILD.gn new file mode 100644 index 0000000..add2395 --- /dev/null +++ b/build/config/apple/BUILD.gn
@@ -0,0 +1,17 @@ +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/apple/symbols.gni") + +# The ldflags referenced below are handled by +# //build/toolchain/apple/linker_driver.py. +# Remove this config if a target wishes to change the arguments passed to the +# strip command during linking. This config by default strips all symbols +# from a binary, but some targets may wish to specify an exports file to +# preserve specific symbols. +config("strip_all") { + if (enable_stripping) { + ldflags = [ "-Wcrl,strip,-x,-S" ] + } +}
diff --git a/build/config/apple/sdk_info.py b/build/config/apple/sdk_info.py old mode 100644 new mode 100755 index fea6801..7928dbf --- a/build/config/apple/sdk_info.py +++ b/build/config/apple/sdk_info.py
@@ -1,9 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function - import argparse import doctest import itertools @@ -13,30 +12,12 @@ import subprocess import sys -if sys.version_info.major < 3: - basestring_compat = basestring -else: - basestring_compat = str - -# src directory -ROOT_SRC_DIR = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname( - os.path.realpath(__file__))))) # This script prints information about the build system, the operating # system and the iOS or Mac SDK (depending on the platform "iphonesimulator", # "iphoneos" or "macosx" generally). -def LoadPList(path): - """Loads Plist at |path| and returns it as a dictionary.""" - # Cloned from //build/apple/plist_util.py. - if sys.version_info.major == 2: - return plistlib.readPlist(path) - with open(path, 'rb') as f: - return plistlib.load(f) - - def SplitVersion(version): """Splits the Xcode version to 3 values. @@ -70,7 +51,8 @@ if developer_dir: xcode_version_plist_path = os.path.join(developer_dir, 'Contents/version.plist') - version_plist = LoadPList(xcode_version_plist_path) + with open(xcode_version_plist_path, 'rb') as f: + version_plist = plistlib.load(f) settings['xcode_version'] = FormatVersion( version_plist['CFBundleShortVersionString']) settings['xcode_version_int'] = int(settings['xcode_version'], 10) @@ -110,14 +92,14 @@ 'Toolchains/XcodeDefault.xctoolchain') -def CreateXcodeSymlinkAt(src, dst): +def CreateXcodeSymlinkAt(src, dst, root_build_dir): """Create symlink to Xcode directory at target location.""" if not os.path.isdir(dst): os.makedirs(dst) dst = os.path.join(dst, os.path.basename(src)) - updated_value = '//' + os.path.relpath(dst, ROOT_SRC_DIR) + updated_value = os.path.join(root_build_dir, dst) # Update the symlink only if it is different from the current destination. if os.path.islink(dst): @@ -131,47 +113,49 @@ return updated_value -if __name__ == '__main__': +def main(): doctest.testmod() parser = argparse.ArgumentParser() - parser.add_argument("--developer_dir", dest="developer_dir", required=False) - parser.add_argument("--get_sdk_info", - action="store_true", - dest="get_sdk_info", + parser.add_argument('--developer_dir') + parser.add_argument('--get_sdk_info', + action='store_true', default=False, - help="Returns SDK info in addition to xcode info.") - parser.add_argument("--get_machine_info", - action="store_true", - dest="get_machine_info", + help='Returns SDK info in addition to xcode info.') + parser.add_argument('--get_machine_info', + action='store_true', default=False, - help="Returns machine info in addition to xcode info.") - parser.add_argument("--create_symlink_at", - action="store", - dest="create_symlink_at", - help="Create symlink of SDK at given location and " - "returns the symlinked paths as SDK info instead " - "of the original location.") - args, unknownargs = parser.parse_known_args() + help='Returns machine info in addition to xcode info.') + parser.add_argument('--create_symlink_at', + help='Create symlink of SDK at given location and ' + 'returns the symlinked paths as SDK info instead ' + 'of the original location.') + parser.add_argument('--root_build_dir', + default='.', + help='Value of gn $root_build_dir') + parser.add_argument('platform', + choices=['iphoneos', 'iphonesimulator', 'macosx', + 'appletvos']) # Cobalt: for internal build + args = parser.parse_args() if args.developer_dir: os.environ['DEVELOPER_DIR'] = args.developer_dir - if len(unknownargs) != 1: - sys.stderr.write('usage: %s [iphoneos|iphonesimulator|macosx]\n' % - os.path.basename(sys.argv[0])) - sys.exit(1) - settings = {} if args.get_machine_info: FillMachineOSBuild(settings) FillXcodeVersion(settings, args.developer_dir) if args.get_sdk_info: - FillSDKPathAndVersion(settings, unknownargs[0], settings['xcode_version']) + FillSDKPathAndVersion(settings, args.platform, settings['xcode_version']) for key in sorted(settings): value = settings[key] if args.create_symlink_at and '_path' in key: - value = CreateXcodeSymlinkAt(value, args.create_symlink_at) - if isinstance(value, basestring_compat): + value = CreateXcodeSymlinkAt(value, args.create_symlink_at, + args.root_build_dir) + if isinstance(value, str): value = '"%s"' % value print('%s=%s' % (key, value)) + + +if __name__ == '__main__': + sys.exit(main())
diff --git a/build/config/apple/symbols.gni b/build/config/apple/symbols.gni index dd1d796..3b4dee4 100644 --- a/build/config/apple/symbols.gni +++ b/build/config/apple/symbols.gni
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -17,7 +17,7 @@ enable_dsyms = is_official_build || using_sanitizer # Strip symbols from linked targets by default. If this is enabled, the - # //build/config/mac:strip_all config will be applied to all linked targets. + # //build/config/apple:strip_all config will be applied to all linked targets. # If custom stripping parameters are required, remove that config from a # linked target and apply custom -Wcrl,strip flags. See # //build/toolchain/apple/linker_driver.py for more information.
diff --git a/build/config/arm.gni b/build/config/arm.gni index 60ef642..bddffcb 100644 --- a/build/config/arm.gni +++ b/build/config/arm.gni
@@ -1,7 +1,8 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/config/chromeos/ui_mode.gni") import("//build/config/v8_target_cpu.gni") if (is_starboard) { @@ -42,20 +43,26 @@ arm_use_thumb = true } + # For lacros build, we use ARM v8 by default. + if (is_chromeos_lacros && arm_arch == "") { + arm_version = 8 + arm_arch = "armv8-a+crc" + } + if (is_starboard) { arm_float_abi = sabi_variables.floating_point_abi arm_fpu = sabi_variables.floating_point_fpu } else { - if (current_os == "android" || target_os == "android") { - arm_float_abi = "softfp" - } else { + if (current_os == "android" || target_os == "android") { + arm_float_abi = "softfp" + } else { declare_args() { # The ARM floating point mode. This is either the string "hard", "soft", # or "softfp". An empty string means to use the default one for the # arm_version. arm_float_abi = "" } - } + } } assert(arm_float_abi == "" || arm_float_abi == "hard" || arm_float_abi == "soft" || arm_float_abi == "softfp") @@ -71,7 +78,8 @@ if (arm_version == 6) { if (arm_arch == "") { - arm_arch = "armv6" + # v8 can still with version 6 but only with the armv6k extension. + arm_arch = "armv6k" } if (arm_tune != "") { arm_tune = "" @@ -88,9 +96,6 @@ if (arm_arch == "") { arm_arch = "armv7-a" } - if (arm_tune == "") { - arm_tune = "generic-armv7-a" - } if (arm_float_abi == "") { if (current_os == "linux" && target_cpu != v8_target_cpu) { @@ -132,4 +137,23 @@ # arm64 supports only "hard". arm_float_abi = "hard" arm_use_neon = true + declare_args() { + # Enables the new Armv8 branch protection features. Valid strings are: + # - "pac": Enables Pointer Authentication Code (PAC, featured in Armv8.3) + # - "standard": Enables both PAC and Branch Target Identification (Armv8.5). + # - "none": No branch protection. + arm_control_flow_integrity = "none" + + if ((is_android || is_linux) && target_cpu == "arm64") { + # Enable PAC and BTI on AArch64 Linux/Android systems. + # target_cpu == "arm64" filters out some cases (e.g. the ChromeOS x64 + # MSAN build) where the target platform is x64, but V8 is configured to + # use the arm64 simulator. + arm_control_flow_integrity = "standard" + } + } + assert(arm_control_flow_integrity == "none" || + arm_control_flow_integrity == "standard" || + arm_control_flow_integrity == "pac", + "Invalid branch protection option") }
diff --git a/build/config/buildflags_paint_preview.gni b/build/config/buildflags_paint_preview.gni index 7129e76..951b660 100644 --- a/build/config/buildflags_paint_preview.gni +++ b/build/config/buildflags_paint_preview.gni
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,11 +6,11 @@ import("//build/config/features.gni") declare_args() { - # Enable basic paint preview support. Does not work on iOS or Fuchsia. Should - # not be included with Chromecast. Not ready for shipping builds yet so - # include in unofficial builds. + # Enable basic paint preview support. Does not work on iOS. Should + # not be included with Chromecast hardware devices. # Used by //components/paint_preview and //third_party/harfbuzz-ng. - # TODO(bug/webrtc:11223) Move back this file in //components/paint_preview/ - # once WebRTC doesn't roll harfbuzz-ng anymore, for consistency sake. - enable_paint_preview = !is_chromecast && !is_ios && !is_fuchsia + # TODO(crbug.com/webrtc/11223) Move back this file in + # //components/paint_preview/ once WebRTC doesn't roll harfbuzz-ng anymore, + # for consistency sake. + enable_paint_preview = !is_castos && !is_cast_android && !is_ios }
diff --git a/build/config/c++/BUILD.gn b/build/config/c++/BUILD.gn index 6494bb0..c00dcef 100644 --- a/build/config/c++/BUILD.gn +++ b/build/config/c++/BUILD.gn
@@ -1,30 +1,12 @@ import("//build/config/c++/c++.gni") import("//build/config/chrome_build.gni") import("//build/config/chromeos/ui_mode.gni") +import("//build/config/compiler/compiler.gni") import("//build/config/dcheck_always_on.gni") import("//buildtools/deps_revisions.gni") assert(use_custom_libcxx, "should only be used if use_custom_libcxx is set") -declare_args() { - # lldb pretty printing only works when libc++ is built in the __1 (or __ndk1) - # namespaces. For pretty printing to work out-of-the-box on Mac (where lldb - # is primarily used), this flag is set to false to build with the __1 - # namespace (to maintain ABI compatibility, this implies building without - # _LIBCPP_ABI_UNSTABLE). This is not necessary on non-component builds - # because we leave the ABI version set to __1 in that case because libc++ - # symbols are not exported. - # TODO(thomasanderson): Set this to true by default once rL352899 is available - # in MacOS's lldb. - libcxx_abi_unstable = !(is_apple && is_debug && is_component_build) -} - -# TODO(xiaohuic): https://crbug/917533 Crashes on internal ChromeOS build. -# Do unconditionally once the underlying problem is fixed. -if (is_chromeos_ash && is_chrome_branded) { - libcxx_abi_unstable = false -} - # This is included by reference in the //build/config/compiler:runtime_library # config that is applied to all targets. It is here to separate out the logic # that is specific to libc++. Please see that target for advice on what should @@ -33,26 +15,15 @@ cflags = [] cflags_cc = [] defines = [] + include_dirs = [] ldflags = [] libs = [] - if (libcxx_abi_unstable) { - defines += [ "_LIBCPP_ABI_UNSTABLE" ] - } + # Fixed libc++ configuration macros are in + # buildtools/third_party/libc++/__config_site. This config only has defines + # that vary depending on gn args, and non-define flags. - if (libcxx_is_shared) { - # When libcxx_is_shared is true, symbols from libc++.so are exported for - # all DSOs to use. If the system libc++ gets loaded (indirectly through - # a system library), then it will conflict with our libc++.so. Add a - # custom ABI version if we're building with _LIBCPP_ABI_UNSTABLE to avoid - # conflicts. - # - # Windows doesn't need to set _LIBCPP_ABI_VERSION since there's no system - # C++ library we could conflict with. - if (libcxx_abi_unstable && !is_win) { - defines += [ "_LIBCPP_ABI_VERSION=Cr" ] - } - } else { + if (!libcxx_is_shared) { # Don't leak any symbols on a static build. defines += [ "_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS" ] if (!export_libcxxabi_from_executables && !is_win) { @@ -60,32 +31,18 @@ } } - defines += [ - "_LIBCPP_ENABLE_NODISCARD", + include_dirs += [ "//buildtools/third_party/libc++" ] - # TODO(crbug.com/1166707): libc++ requires this macro. - "_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS", - ] - - # Work around a symbol conflict between GRPC and the Fuchsia SDK. - # TODO(crbug.com/1166970): Remove this when resolved. - if (is_fuchsia) { - defines += [ "_LIBCPP_NO_NATIVE_SEMAPHORES" ] + # libc++ has two levels of additional checking: + # 1. _LIBCPP_ENABLE_ASSERTIONS enables assertions for bounds checking. + # We always enable this in __config_site, in all build configurations. + # 2. _LIBCPP_ENABLE_DEBUG_MODE enables iterator debugging and other + # expensive checks. Enable these only if enable_iterator_debugging is on. + if (enable_iterator_debugging) { + defines += [ "_LIBCPP_ENABLE_DEBUG_MODE" ] } - # The Windows component build fails to link with libc++'s debug mode. See - # https://crbug.com/923166#c33, https://crbug.com/923166#c44, and - # https://llvm.org/PR41018. - if (!(is_win && is_component_build)) { - # libc++ has two levels of debug mode. Setting _LIBCPP_DEBUG to zero - # enables most assertions. Setting it to one additionally enables iterator - # debugging. See https://libcxx.llvm.org/docs/DesignDocs/DebugMode.html - if (enable_iterator_debugging) { - defines += [ "_LIBCPP_DEBUG=1" ] - } else if (is_debug || dcheck_always_on) { - defines += [ "_LIBCPP_DEBUG=0" ] - } - } + defines += [ "CR_LIBCXX_REVISION=$libcxx_revision" ] if (is_win) { # Intentionally not using libc++abi on Windows because libc++abi only @@ -98,19 +55,6 @@ cflags_cc += [ "-I" + rebase_path("$libcxx_prefix/include", root_build_dir) ] - # Prevent libc++ from embedding linker flags to try to automatically link - # against its runtime library. This is unnecessary with our build system, - # and can also result in build failures if libc++'s name for a library - # does not match ours. - defines += [ "_LIBCPP_NO_AUTO_LINK" ] - - if (is_component_build) { - # TODO(crbug.com/1090975): Disable the exclude_from_explicit_instantiation - # to work around compiler bugs in the interaction between it and - # dllimport/dllexport. - defines += [ "_LIBCPP_HIDE_FROM_ABI=_LIBCPP_HIDDEN" ] - } - # Add a debug visualizer for Microsoft's debuggers so that they can display # libc++ types well. if (libcxx_natvis_include) { @@ -124,9 +68,8 @@ "-isystem" + rebase_path("$libcxx_prefix/include", root_build_dir), "-isystem" + rebase_path("$libcxxabi_prefix/include", root_build_dir), ] - cflags_objcc = cflags_cc - defines += [ "CR_LIBCXX_REVISION=$libcxx_revision" ] + cflags_objcc = cflags_cc # Make sure we don't link against the system libstdc++ or libc++. if (is_clang) { @@ -151,4 +94,7 @@ ] } } + if (use_custom_libcxx && enable_safe_libcxx) { + defines += [ "_LIBCPP_ENABLE_ASSERTIONS=1" ] + } }
diff --git a/build/config/c++/c++.gni b/build/config/c++/c++.gni index a7448f3..25ece4c 100644 --- a/build/config/c++/c++.gni +++ b/build/config/c++/c++.gni
@@ -1,9 +1,9 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("//build/config/chromeos/ui_mode.gni") import("//build/config/sanitizers/sanitizers.gni") +import("//build_overrides/build.gni") declare_args() { # Use in-tree libc++ (buildtools/third_party/libc++ and @@ -11,11 +11,8 @@ # standard library support. # Don't check in changes that set this to false for more platforms; doing so # is not supported. - use_custom_libcxx = is_fuchsia || is_android || is_mac || - (is_ios && !use_xcode_clang) || (is_win && is_clang) || - ((is_linux || is_chromeos) && - (!is_chromeos_ash || - default_toolchain != "//build/toolchain/cros:target")) + use_custom_libcxx = is_fuchsia || is_android || is_apple || is_linux || + is_chromeos || (is_win && is_clang) # Use libc++ instead of stdlibc++ when using the host_cpu toolchain, even if # use_custom_libcxx is false. This is useful for cross-compiles where a custom @@ -56,10 +53,17 @@ libcxx_is_shared = use_custom_libcxx && is_component_build } +# TODO(https://crbug.com/1385662): This is temporarily guarded to make it easier +# to roll out this change. Once the various projects (ANGLE, v8, et cetera) +# rolling in Chrome's //build have updated, remove this entirely. +if (!defined(enable_safe_libcxx)) { + enable_safe_libcxx = true +} + # libc++abi needs to be exported from executables to be picked up by shared # libraries on certain instrumented builds. export_libcxxabi_from_executables = - use_custom_libcxx && !is_ios && !is_win && !is_component_build && + use_custom_libcxx && !is_apple && !is_win && !is_component_build && (is_asan || is_ubsan_vptr) # On Android, many shared libraries get loaded from the context of a JRE. In @@ -80,3 +84,6 @@ libcxx_prefix = "//buildtools/third_party/libc++/trunk" libcxxabi_prefix = "//buildtools/third_party/libc++abi/trunk" + +assert(!(is_ios && libcxx_is_shared), + "Can't build libc++ as a shared library on iOS.")
diff --git a/build/config/c++/libc++.natvis b/build/config/c++/libc++.natvis index 9a49a29..6378548 100644 --- a/build/config/c++/libc++.natvis +++ b/build/config/c++/libc++.natvis
@@ -11,20 +11,20 @@ * if T is empty and non-final the 3rd param is 1 and it derives from T * else it has a member variable __value_ of type T --> - <Type Name="std::__1::__compressed_pair_elem<*,*,0>"> + <Type Name="std::Cr::__compressed_pair_elem<*,*,0>"> <DisplayString>{__value_}</DisplayString> <Expand> <ExpandedItem>__value_</ExpandedItem> </Expand> </Type> - <Type Name="std::__1::__compressed_pair_elem<*,*,1>"> + <Type Name="std::Cr::__compressed_pair_elem<*,*,1>"> <DisplayString>{*($T1*)this}</DisplayString> <Expand> <ExpandedItem>*($T1*)this</ExpandedItem> </Expand> </Type> - <Type Name="std::__1::array<*,*>"> + <Type Name="std::Cr::array<*,*>"> <DisplayString>{{ size={$T2} }}</DisplayString> <Expand> <ArrayItems> @@ -50,7 +50,7 @@ and the inline data in the remaining storage.) --> - <Type Name="std::__1::basic_string<char,*>"> + <Type Name="std::Cr::basic_string<char,*>"> <!--<Intrinsic Name="is_long" Expression="((__rep*)&__r_)->__s.__size_ & 0x80" />--> <!-- The above doesn't work because of https://llvm.org/PR41615 @@ -85,7 +85,7 @@ </Expand> </Type> - <Type Name="std::__1::basic_string<wchar_t,*>"> + <Type Name="std::Cr::basic_string<wchar_t,*>"> <Intrinsic Name="is_long" Expression="*(((char*)this) + 3*sizeof(size_t) - 1) & 0x80" /> <DisplayString Condition="is_long()">{*(wchar_t**)this}</DisplayString> @@ -113,7 +113,7 @@ </Expand> </Type> - <Type Name="std::__1::deque<*,*>"> + <Type Name="std::Cr::deque<*,*>"> <Intrinsic Name="size" Expression="*(size_type*)&__size_" /> <Intrinsic Name="block_size" Expression="sizeof($T1) < 256 ? 4096 / sizeof($T1) : 16" /> @@ -129,7 +129,7 @@ </Expand> </Type> - <Type Name="std::__1::forward_list<*>"> + <Type Name="std::Cr::forward_list<*>"> <Intrinsic Name="head" Expression="((__node_pointer)&__before_begin_)->__next_" /> <DisplayString Condition="head() == 0">empty</DisplayString> @@ -143,7 +143,7 @@ </Expand> </Type> - <!-- Note: Not in __1! But will win over the one in stl.natvis --> + <!-- Note: Not in Cr! But will win over the one in stl.natvis --> <Type Name="std::initializer_list<*>"> <DisplayString>{{ size={__size_} }}</DisplayString> <Expand> @@ -154,7 +154,7 @@ </Expand> </Type> - <Type Name="std::__1::list<*>"> + <Type Name="std::Cr::list<*>"> <Intrinsic Name="size" Expression="*(size_type*)&__size_alloc_" /> <DisplayString>{{ size={size()} }}</DisplayString> <Expand> @@ -163,14 +163,14 @@ <HeadPointer>__end_.__next_</HeadPointer> <NextPointer>__next_</NextPointer> <ValueNode> - ((std::__1::list<$T1,$T2>::__node_pointer)this) + ((std::Cr::list<$T1,$T2>::__node_pointer)this) ->__value_ </ValueNode> </LinkedListItems> </Expand> </Type> - <Type Name="std::__1::map<*>"> + <Type Name="std::Cr::map<*>"> <Intrinsic Name="size" Expression="*(size_type*)&__tree_.__pair3_" /> <DisplayString>{{ size={size()} }}</DisplayString> <Expand> @@ -181,22 +181,22 @@ ((__node_pointer)&__tree_.__pair1_)->__left_ </HeadPointer> <LeftPointer> - ((std::__1::map<$T1,$T2,$T3,$T4>::__node_pointer)this) + ((std::Cr::map<$T1,$T2,$T3,$T4>::__node_pointer)this) ->__left_ </LeftPointer> <RightPointer> - ((std::__1::map<$T1,$T2,$T3,$T4>::__node_pointer)this) + ((std::Cr::map<$T1,$T2,$T3,$T4>::__node_pointer)this) ->__right_ </RightPointer> <ValueNode> - ((std::__1::map<$T1,$T2,$T3,$T4>::__node_pointer)this) - ->__value_.__cc + ((std::Cr::map<$T1,$T2,$T3,$T4>::__node_pointer)this) + ->__value_.__cc_ </ValueNode> </TreeItems> </Expand> </Type> - <Type Name="std::__1::multimap<*>"> + <Type Name="std::Cr::multimap<*>"> <Intrinsic Name="size" Expression="*(size_type*)&__tree_.__pair3_" /> <DisplayString>{{ size={size()} }}</DisplayString> <Expand> @@ -207,22 +207,22 @@ ((__node_pointer)&__tree_.__pair1_)->__left_ </HeadPointer> <LeftPointer> - ((std::__1::multimap<$T1,$T2,$T3,$T4>::__node_pointer)this) + ((std::Cr::multimap<$T1,$T2,$T3,$T4>::__node_pointer)this) ->__left_ </LeftPointer> <RightPointer> - ((std::__1::multimap<$T1,$T2,$T3,$T4>::__node_pointer)this) + ((std::Cr::multimap<$T1,$T2,$T3,$T4>::__node_pointer)this) ->__right_ </RightPointer> <ValueNode> - ((std::__1::multimap<$T1,$T2,$T3,$T4>::__node_pointer)this) - ->__value_.__cc + ((std::Cr::multimap<$T1,$T2,$T3,$T4>::__node_pointer)this) + ->__value_.__cc_ </ValueNode> </TreeItems> </Expand> </Type> - <Type Name="std::__1::multiset<*>"> + <Type Name="std::Cr::multiset<*>"> <Intrinsic Name="size" Expression="*(size_type*)&__tree_.__pair3_" /> <DisplayString>{{ size={size()} }}</DisplayString> <Expand> @@ -233,22 +233,22 @@ ((__base::__node_pointer)&__tree_.__pair1_)->__left_ </HeadPointer> <LeftPointer> - ((std::__1::multiset<$T1,$T2,$T3>::__base::__node_pointer)this) + ((std::Cr::multiset<$T1,$T2,$T3>::__base::__node_pointer)this) ->__left_ </LeftPointer> <RightPointer> - ((std::__1::multiset<$T1,$T2,$T3>::__base::__node_pointer)this) + ((std::Cr::multiset<$T1,$T2,$T3>::__base::__node_pointer)this) ->__right_ </RightPointer> <ValueNode> - ((std::__1::multiset<$T1,$T2,$T3>::__base::__node_pointer)this) + ((std::Cr::multiset<$T1,$T2,$T3>::__base::__node_pointer)this) ->__value_ </ValueNode> </TreeItems> </Expand> </Type> - <Type Name="std::__1::priority_queue<*>"> + <Type Name="std::Cr::priority_queue<*>"> <DisplayString>{c}</DisplayString> <Expand> <ExpandedItem>c</ExpandedItem> @@ -256,7 +256,7 @@ </Expand> </Type> - <Type Name="std::__1::set<*>"> + <Type Name="std::Cr::set<*>"> <Intrinsic Name="size" Expression="*(size_type*)&__tree_.__pair3_" /> <DisplayString>{{ size={size()} }}</DisplayString> <Expand> @@ -267,108 +267,108 @@ ((__base::__node_pointer)&__tree_.__pair1_)->__left_ </HeadPointer> <LeftPointer> - ((std::__1::set<$T1,$T2,$T3>::__base::__node_pointer)this) + ((std::Cr::set<$T1,$T2,$T3>::__base::__node_pointer)this) ->__left_ </LeftPointer> <RightPointer> - ((std::__1::set<$T1,$T2,$T3>::__base::__node_pointer)this) + ((std::Cr::set<$T1,$T2,$T3>::__base::__node_pointer)this) ->__right_ </RightPointer> <ValueNode> - ((std::__1::set<$T1,$T2,$T3>::__base::__node_pointer)this) + ((std::Cr::set<$T1,$T2,$T3>::__base::__node_pointer)this) ->__value_ </ValueNode> </TreeItems> </Expand> </Type> - <Type Name="std::__1::stack<*>"> - <AlternativeType Name="std::__1::queue<*>" /> + <Type Name="std::Cr::stack<*>"> + <AlternativeType Name="std::Cr::queue<*>" /> <DisplayString>{c}</DisplayString> <Expand> <ExpandedItem>c</ExpandedItem> </Expand> </Type> - <Type Name="std::__1::__tuple_leaf<*,*,0>"> + <Type Name="std::Cr::__tuple_leaf<*,*,0>"> <DisplayString>{__value_}</DisplayString> </Type> - <Type Name="std::__1::tuple<>"> + <Type Name="std::Cr::tuple<>"> <DisplayString>()</DisplayString> </Type> - <Type Name="std::__1::tuple<*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::tuple<*,*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_}, {(std::__1::__tuple_leaf<1,$T2,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*,*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_}, {(std::Cr::__tuple_leaf<1,$T2,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> - <Item Name="[1]">(std::__1::__tuple_leaf<1,$T2,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[1]">(std::Cr::__tuple_leaf<1,$T2,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::tuple<*,*,*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_}, {(std::__1::__tuple_leaf<1,$T2,0>)__base_}, {(std::__1::__tuple_leaf<2,$T3,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*,*,*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_}, {(std::Cr::__tuple_leaf<1,$T2,0>)__base_}, {(std::Cr::__tuple_leaf<2,$T3,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> - <Item Name="[1]">(std::__1::__tuple_leaf<1,$T2,0>)__base_</Item> - <Item Name="[2]">(std::__1::__tuple_leaf<2,$T3,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[1]">(std::Cr::__tuple_leaf<1,$T2,0>)__base_</Item> + <Item Name="[2]">(std::Cr::__tuple_leaf<2,$T3,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::tuple<*,*,*,*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_}, {(std::__1::__tuple_leaf<1,$T2,0>)__base_}, {(std::__1::__tuple_leaf<2,$T3,0>)__base_}, {(std::__1::__tuple_leaf<3,$T4,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*,*,*,*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_}, {(std::Cr::__tuple_leaf<1,$T2,0>)__base_}, {(std::Cr::__tuple_leaf<2,$T3,0>)__base_}, {(std::Cr::__tuple_leaf<3,$T4,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> - <Item Name="[1]">(std::__1::__tuple_leaf<1,$T2,0>)__base_</Item> - <Item Name="[2]">(std::__1::__tuple_leaf<2,$T3,0>)__base_</Item> - <Item Name="[3]">(std::__1::__tuple_leaf<3,$T4,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[1]">(std::Cr::__tuple_leaf<1,$T2,0>)__base_</Item> + <Item Name="[2]">(std::Cr::__tuple_leaf<2,$T3,0>)__base_</Item> + <Item Name="[3]">(std::Cr::__tuple_leaf<3,$T4,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::tuple<*,*,*,*,*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_}, {(std::__1::__tuple_leaf<1,$T2,0>)__base_}, {(std::__1::__tuple_leaf<2,$T3,0>)__base_}, {(std::__1::__tuple_leaf<3,$T4,0>)__base_}, {(std::__1::__tuple_leaf<4,$T5,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*,*,*,*,*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_}, {(std::Cr::__tuple_leaf<1,$T2,0>)__base_}, {(std::Cr::__tuple_leaf<2,$T3,0>)__base_}, {(std::Cr::__tuple_leaf<3,$T4,0>)__base_}, {(std::Cr::__tuple_leaf<4,$T5,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> - <Item Name="[1]">(std::__1::__tuple_leaf<1,$T2,0>)__base_</Item> - <Item Name="[2]">(std::__1::__tuple_leaf<2,$T3,0>)__base_</Item> - <Item Name="[3]">(std::__1::__tuple_leaf<3,$T4,0>)__base_</Item> - <Item Name="[4]">(std::__1::__tuple_leaf<4,$T5,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[1]">(std::Cr::__tuple_leaf<1,$T2,0>)__base_</Item> + <Item Name="[2]">(std::Cr::__tuple_leaf<2,$T3,0>)__base_</Item> + <Item Name="[3]">(std::Cr::__tuple_leaf<3,$T4,0>)__base_</Item> + <Item Name="[4]">(std::Cr::__tuple_leaf<4,$T5,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::tuple<*,*,*,*,*,*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_}, {(std::__1::__tuple_leaf<1,$T2,0>)__base_}, {(std::__1::__tuple_leaf<2,$T3,0>)__base_}, {(std::__1::__tuple_leaf<3,$T4,0>)__base_}, {(std::__1::__tuple_leaf<4,$T5,0>)__base_}, {(std::__1::__tuple_leaf<5,$T6,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*,*,*,*,*,*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_}, {(std::Cr::__tuple_leaf<1,$T2,0>)__base_}, {(std::Cr::__tuple_leaf<2,$T3,0>)__base_}, {(std::Cr::__tuple_leaf<3,$T4,0>)__base_}, {(std::Cr::__tuple_leaf<4,$T5,0>)__base_}, {(std::Cr::__tuple_leaf<5,$T6,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> - <Item Name="[1]">(std::__1::__tuple_leaf<1,$T2,0>)__base_</Item> - <Item Name="[2]">(std::__1::__tuple_leaf<2,$T3,0>)__base_</Item> - <Item Name="[3]">(std::__1::__tuple_leaf<3,$T4,0>)__base_</Item> - <Item Name="[4]">(std::__1::__tuple_leaf<4,$T5,0>)__base_</Item> - <Item Name="[5]">(std::__1::__tuple_leaf<5,$T6,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[1]">(std::Cr::__tuple_leaf<1,$T2,0>)__base_</Item> + <Item Name="[2]">(std::Cr::__tuple_leaf<2,$T3,0>)__base_</Item> + <Item Name="[3]">(std::Cr::__tuple_leaf<3,$T4,0>)__base_</Item> + <Item Name="[4]">(std::Cr::__tuple_leaf<4,$T5,0>)__base_</Item> + <Item Name="[5]">(std::Cr::__tuple_leaf<5,$T6,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::tuple<*,*,*,*,*,*,*>"> - <DisplayString>({(std::__1::__tuple_leaf<0,$T1,0>)__base_}, {(std::__1::__tuple_leaf<1,$T2,0>)__base_}, {(std::__1::__tuple_leaf<2,$T3,0>)__base_}, {(std::__1::__tuple_leaf<3,$T4,0>)__base_}, {(std::__1::__tuple_leaf<4,$T5,0>)__base_}, {(std::__1::__tuple_leaf<5,$T6,0>)__base_}, {(std::__1::__tuple_leaf<6,$T7,0>)__base_})</DisplayString> + <Type Name="std::Cr::tuple<*,*,*,*,*,*,*>"> + <DisplayString>({(std::Cr::__tuple_leaf<0,$T1,0>)__base_}, {(std::Cr::__tuple_leaf<1,$T2,0>)__base_}, {(std::Cr::__tuple_leaf<2,$T3,0>)__base_}, {(std::Cr::__tuple_leaf<3,$T4,0>)__base_}, {(std::Cr::__tuple_leaf<4,$T5,0>)__base_}, {(std::Cr::__tuple_leaf<5,$T6,0>)__base_}, {(std::Cr::__tuple_leaf<6,$T7,0>)__base_})</DisplayString> <Expand> - <Item Name="[0]">(std::__1::__tuple_leaf<0,$T1,0>)__base_</Item> - <Item Name="[1]">(std::__1::__tuple_leaf<1,$T2,0>)__base_</Item> - <Item Name="[2]">(std::__1::__tuple_leaf<2,$T3,0>)__base_</Item> - <Item Name="[3]">(std::__1::__tuple_leaf<3,$T4,0>)__base_</Item> - <Item Name="[4]">(std::__1::__tuple_leaf<4,$T5,0>)__base_</Item> - <Item Name="[5]">(std::__1::__tuple_leaf<5,$T6,0>)__base_</Item> - <Item Name="[6]">(std::__1::__tuple_leaf<6,$T7,0>)__base_</Item> + <Item Name="[0]">(std::Cr::__tuple_leaf<0,$T1,0>)__base_</Item> + <Item Name="[1]">(std::Cr::__tuple_leaf<1,$T2,0>)__base_</Item> + <Item Name="[2]">(std::Cr::__tuple_leaf<2,$T3,0>)__base_</Item> + <Item Name="[3]">(std::Cr::__tuple_leaf<3,$T4,0>)__base_</Item> + <Item Name="[4]">(std::Cr::__tuple_leaf<4,$T5,0>)__base_</Item> + <Item Name="[5]">(std::Cr::__tuple_leaf<5,$T6,0>)__base_</Item> + <Item Name="[6]">(std::Cr::__tuple_leaf<6,$T7,0>)__base_</Item> </Expand> </Type> - <Type Name="std::__1::unique_ptr<*>"> + <Type Name="std::Cr::unique_ptr<*>"> <Intrinsic Name="value" Expression="*($T1**)&__ptr_" /> <SmartPointer Usage="Minimal">value()</SmartPointer> <DisplayString Condition="value() == 0">empty</DisplayString> @@ -379,10 +379,10 @@ </Expand> </Type> -<Type Name="std::__1::unordered_map<*>"> - <AlternativeType Name="std::__1::unordered_multimap<*>" /> - <AlternativeType Name="std::__1::unordered_multiset<*>" /> - <AlternativeType Name="std::__1::unordered_set<*>" /> +<Type Name="std::Cr::unordered_map<*>"> + <AlternativeType Name="std::Cr::unordered_multimap<*>" /> + <AlternativeType Name="std::Cr::unordered_multiset<*>" /> + <AlternativeType Name="std::Cr::unordered_set<*>" /> <Intrinsic Name="size" Expression="*(size_type*)&__table_.__p2_" /> <Intrinsic Name="bucket_count" Expression="*(size_type*)& @@ -415,14 +415,14 @@ <!-- This is the node __value_ of an unordered_(multi)map. Expand it through a separate formatter instead of in the <Item> expression above so that the same <Type> works for unordered_(multi)set and unordered_(multi)map. --> - <Type Name="std::__1::__hash_value_type<*>"> + <Type Name="std::Cr::__hash_value_type<*>"> <DisplayString>{__cc}</DisplayString> <Expand> <ExpandedItem>__cc</ExpandedItem> </Expand> </Type> - <Type Name="std::__1::vector<*>"> + <Type Name="std::Cr::vector<*>"> <Intrinsic Name="size" Expression="__end_ - __begin_" /> <DisplayString>{{ size={size()} }}</DisplayString> <Expand>
diff --git a/build/config/chrome_build.gni b/build/config/chrome_build.gni index 5c51d7f..b5156d5 100644 --- a/build/config/chrome_build.gni +++ b/build/config/chrome_build.gni
@@ -1,4 +1,4 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,15 +7,58 @@ # true means official Google Chrome branding (requires extra Google-internal # resources). is_chrome_branded = false + + # Whether to enable the Chrome for Testing (CfT) flavor. This arg is not + # compatible with `is_chrome_branded`. + # + # Design document: https://goo.gle/chrome-for-testing + is_chrome_for_testing = false + + # Whether to use internal Chrome for Testing (CfT). + # Requires `src-internal/` and `is_chrome_for_testing = true`. + # + # When true, use Google-internal icons, otherwise fall back to Chromium icons. + is_chrome_for_testing_branded = false + + # Set to true to enable settings for high end Android devices, typically + # enhancing speed at the expense of resources such as binary sizes and memory. + is_high_end_android = false + + if (is_android) { + # By default, Trichrome channels are compiled using separate package names. + # Set this to 'true' to compile Trichrome channels using the Stable channel's + # package name. This currently only affects builds with `android_channel = + # "beta"`. + use_stable_package_name_for_trichrome = false + } } +assert( + !is_chrome_for_testing || !is_chrome_branded, + "`is_chrome_for_testing = true` is incompatible with `is_chrome_branded = true`") + +assert( + is_chrome_for_testing || !is_chrome_for_testing_branded, + "`is_chrome_for_testing_branded = true` requires `is_chrome_for_testing = true`") + declare_args() { # Refers to the subdirectory for branding in various places including # chrome/app/theme. - if (is_chrome_branded) { + # + # `branding_path_product` must not contain slashes. + if (is_chrome_for_testing) { + if (is_chrome_for_testing_branded) { + branding_path_component = "google_chrome/google_chrome_for_testing" + } else { + branding_path_component = "chromium" + } + branding_path_product = "chromium" + } else if (is_chrome_branded) { branding_path_component = "google_chrome" + branding_path_product = "google_chrome" } else { branding_path_component = "chromium" + branding_path_product = "chromium" } }
diff --git a/build/config/chromebox_for_meetings/BUILD.gn b/build/config/chromebox_for_meetings/BUILD.gn new file mode 100644 index 0000000..08d74f9 --- /dev/null +++ b/build/config/chromebox_for_meetings/BUILD.gn
@@ -0,0 +1,11 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/buildflag_header.gni") +import("//build/config/chromebox_for_meetings/buildflags.gni") + +buildflag_header("buildflags") { + header = "buildflags.h" + flags = [ "PLATFORM_CFM=$is_cfm" ] +}
diff --git a/build/config/chromebox_for_meetings/OWNERS b/build/config/chromebox_for_meetings/OWNERS new file mode 100644 index 0000000..985da0c --- /dev/null +++ b/build/config/chromebox_for_meetings/OWNERS
@@ -0,0 +1 @@ +file://chromeos/ash/components/chromebox_for_meetings/OWNERS
diff --git a/build/config/chromebox_for_meetings/README.md b/build/config/chromebox_for_meetings/README.md new file mode 100644 index 0000000..ddbe3c9 --- /dev/null +++ b/build/config/chromebox_for_meetings/README.md
@@ -0,0 +1,31 @@ +# CfM GN Build Flags + +Note: GN Flags are Build time flags + +You can get a comprehensive list of all arguments supported by gn by running the +command gn args --list out/some-directory (the directory passed to gn args is +required as gn args will invokes gn gen to generate the build.ninja files). + +## is_cfm (BUILDFLAG(PLATFORM_CFM)) + +Flag for building chromium for CfM devices. + +### Query Flag +```bash +$ gn args out_<cfm_overlay>/{Release||Debug} --list=is_cfm +``` + +### Enable Flag +```bash +$ gn args out_<cfm_overlay>/{Release||Debug} +$ Editor will open add is_cfm=true save and exit +``` + +### Alt: EnrollmentRequisitionManager + +We can alternatively use the EnrollmentRequisitionManager to determine if +chromium is running a CfM enabled Platform in source code + +```cpp +policy::EnrollmentRequisitionManager::IsRemoraRequisition(); +```
diff --git a/build/config/chromebox_for_meetings/buildflags.gni b/build/config/chromebox_for_meetings/buildflags.gni new file mode 100644 index 0000000..22ad88a --- /dev/null +++ b/build/config/chromebox_for_meetings/buildflags.gni
@@ -0,0 +1,8 @@ +# Copyright 2020 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +declare_args() { + # True if compiling for Chromebox for Meeting devices. + is_cfm = false +}
diff --git a/build/config/chromecast/BUILD.gn b/build/config/chromecast/BUILD.gn index 0c3b2cb..acaf990 100644 --- a/build/config/chromecast/BUILD.gn +++ b/build/config/chromecast/BUILD.gn
@@ -1,10 +1,10 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/chromecast_build.gni") -assert(is_chromecast) +assert(is_castos || is_cast_android) config("static_config") { if (!is_clang) { @@ -28,6 +28,7 @@ config("ldconfig") { visibility = [ ":*" ] + configs = [] # Chromecast executables depend on several shared libraries in # /oem_cast_shlib, $ORIGIN, and $ORIGIN/lib. Add these rpaths to each binary.
diff --git a/build/config/chromecast/OWNERS b/build/config/chromecast/OWNERS new file mode 100644 index 0000000..253037d --- /dev/null +++ b/build/config/chromecast/OWNERS
@@ -0,0 +1,3 @@ +mfoltz@chromium.org +rwkeane@google.com +seantopping@chromium.org
diff --git a/build/config/chromecast_build.gni b/build/config/chromecast_build.gni index deecdb5..e8294ce 100644 --- a/build/config/chromecast_build.gni +++ b/build/config/chromecast_build.gni
@@ -1,25 +1,16 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # The args declared in this file should be referenced by components outside of # //chromecast. Args needed only in //chromecast should be declared in # //chromecast/chromecast.gni. +# +# TODO(crbug.com/1294964): Rename this file after is_chromecast is removed. declare_args() { - # Set this true for a Chromecast build. Chromecast builds are supported on - # Linux and Android. - is_chromecast = false - - # If true, IS_CAST_DEBUG_BUILD() will evaluate to 1 in version.h. Otherwise, - # it will evaluate to 0. Overriding this when is_debug=false is useful for - # doing engineering builds. - cast_is_debug = is_debug - - # chromecast_branding is used to include or exclude Google-branded components. - # Set it to "public" for a Chromium build. - chromecast_branding = "public" - # Set this true for an audio-only Chromecast build. + # TODO(crbug.com/1293538): Replace with a buildflag for speaker-only builds not + # specific to Cast. is_cast_audio_only = false # If non empty, rpath of executables is set to this. @@ -28,33 +19,63 @@ # Set true to enable modular_updater. enable_modular_updater = false + + # True to enable the cast audio renderer. + # + # TODO(crbug.com/1293520): Remove this buildflag. + enable_cast_audio_renderer = false + + # Set this to true to build for Nest hardware running Linux (aka "CastOS"). + # Set this to false to use the defaults for Linux. + is_castos = false + + # Set this to true to build for Android-based Cast devices. + # Set this to false to use the defaults for Android. + is_cast_android = false } -# Note(slan): This arg depends on the value of is_chromecast, and thus must be -# declared in a separate block. These blocks can be combined when/if -# crbug.com/542846 is resolved. +# Restrict is_castos and is_cast_android to only be set on the target toolchain. +is_castos = is_castos && current_toolchain == default_toolchain +is_cast_android = is_cast_android && current_toolchain == default_toolchain + declare_args() { - # True if Chromecast build is targeted for linux desktop. This type of build - # is useful for testing and development, but currently supports only a subset - # of Cast functionality. Though this defaults to true for x86 Linux devices, - # this should be overriden manually for an embedded x86 build. - # TODO(slan): Remove instances of this when x86 is a fully supported platform. - is_cast_desktop_build = is_chromecast && target_os == "linux" && - (target_cpu == "x86" || target_cpu == "x64") + # Set this true for a Chromecast build. Chromecast builds are supported on + # Linux, Android, ChromeOS, and Fuchsia. + enable_cast_receiver = false } declare_args() { - # True to enable the cast renderer. It is enabled by default for linux and - # android audio only builds. - enable_cast_renderer = is_chromecast && (is_linux || is_chromeos || - (is_cast_audio_only && is_android)) + # True to enable the cast renderer. It is enabled by default for Linux and + # Android audio only builds. + # + # TODO(crbug.com/1293520): Remove this buildflag. + enable_cast_renderer = + enable_cast_receiver && + (is_linux || is_chromeos || (is_cast_audio_only && is_android)) } # Configures media options for cast. See media/media_options.gni cast_mojo_media_services = [] cast_mojo_media_host = "" -if (enable_cast_renderer) { +if (enable_cast_audio_renderer) { + if (is_android) { + cast_mojo_media_services = [ + "cdm", + "audio_decoder", + ] + } + + if (!is_cast_audio_only) { + cast_mojo_media_services += [ "video_decoder" ] + } + + if (is_android && is_cast_audio_only) { + cast_mojo_media_host = "browser" + } else { + cast_mojo_media_host = "gpu" + } +} else if (enable_cast_renderer) { # In this path, mojo media services are hosted in two processes: # 1. "renderer" and "cdm" run in browser process. This is hard coded in the # code. @@ -63,7 +84,6 @@ "cdm", "renderer", ] - if (!is_cast_audio_only) { cast_mojo_media_services += [ "video_decoder" ] } @@ -87,9 +107,18 @@ } # Assert that Chromecast is being built for a supported platform. -assert(is_linux || is_chromeos || is_android || is_fuchsia || !is_chromecast, - "Chromecast builds are not supported on $target_os") +assert(is_linux || is_chromeos || is_android || is_fuchsia || + !enable_cast_receiver, + "Cast receiver builds are not supported on $current_os") -# Assert that is_cast_audio_only and is_cast_desktop_build are both false on a -# non-Chromecast build. -assert(is_chromecast || (!is_cast_audio_only && !is_cast_desktop_build)) +assert(enable_cast_receiver || !is_cast_audio_only, + "is_cast_audio_only = true requires enable_cast_receiver = true.") + +assert(enable_cast_receiver || !is_castos, + "is_castos = true requires enable_cast_receiver = true.") +assert(is_linux || !is_castos, "is_castos = true requires is_linux = true.") + +assert(enable_cast_receiver || !is_cast_android, + "is_cast_android = true requires enable_cast_receiver = true.") +assert(is_android || !is_cast_android, + "is_cast_android = true requires is_android = true.")
diff --git a/build/config/chromeos/BUILD.gn b/build/config/chromeos/BUILD.gn index f3dfe70..0606072 100644 --- a/build/config/chromeos/BUILD.gn +++ b/build/config/chromeos/BUILD.gn
@@ -1,10 +1,10 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/chromeos/ui_mode.gni") -assert(is_chromeos_ash) +assert(is_chromeos) declare_args() { # The location to a file used to dump symbols ordered by Call-Chain Clustering (C3) @@ -13,11 +13,6 @@ dump_call_chain_clustering_order = "" } -declare_args() { - # Whether or not we're using new pass manager to build and link Chrome - use_new_pass_manager = dump_call_chain_clustering_order != "" -} - config("print_orderfile") { if (dump_call_chain_clustering_order != "") { _output_orderfile = @@ -26,9 +21,42 @@ } } -config("compiler") { - if (use_new_pass_manager) { - cflags = [ "-fexperimental-new-pass-manager" ] - ldflags = [ "-fexperimental-new-pass-manager" ] +config("compiler_cpu_abi") { + # Lacros currently uses the *-generic-crosstoolchain.gni files generated + # by the simplechrome sdk in build/args/chromeos. These target triples + # match the target toolchain defaults in these directories. Passing them + # redundantly is harmless and prepares for using Chromium's toolchain. + # Non-Lacros Chrome OS builds use per-board toolchains, which might use + # different triples. So don't do this there. + if (is_chromeos_device && is_chromeos_lacros) { + if (current_cpu == "x64") { + asmflags = [ "--target=x86_64-cros-linux-gnu" ] + cflags = [ "--target=x86_64-cros-linux-gnu" ] + ldflags = [ "--target=x86_64-cros-linux-gnu" ] + } else if (current_cpu == "arm") { + asmflags = [ "--target=armv7a-cros-linux-gnueabihf" ] + cflags = [ "--target=armv7a-cros-linux-gnueabihf" ] + ldflags = [ "--target=armv7a-cros-linux-gnueabihf" ] + } else if (current_cpu == "arm64") { + asmflags = [ "--target=aarch64-cros-linux-gnu" ] + cflags = [ "--target=aarch64-cros-linux-gnu" ] + ldflags = [ "--target=aarch64-cros-linux-gnu" ] + } else { + assert(false, "add support for $current_cpu here") + } + } +} + +config("runtime_library") { + # These flags are added by the Chrome OS toolchain compiler wrapper, + # or are implicitly passed by Chome OS's toolchain's clang due to the cmake + # flags that clang was built with. + # Passing them redundantly is harmless and prepares for using Chromium's + # toolchain for Lacros. + if (is_chromeos_device) { + ldflags = [ + "--rtlib=compiler-rt", + "--unwindlib=libunwind", + ] } }
diff --git a/build/config/chromeos/args.gni b/build/config/chromeos/args.gni index 3be4f27..8fb5053 100644 --- a/build/config/chromeos/args.gni +++ b/build/config/chromeos/args.gni
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -21,7 +21,21 @@ # linux-chromeos, so some have compile-time asserts that intentionally fail # when this build flag is set. Build and run the tests for linux-chromeos # instead. - # https://chromium.googlesource.com/chromium/src/+/master/docs/chromeos_build_instructions.md - # https://chromium.googlesource.com/chromiumos/docs/+/master/simple_chrome_workflow.md + # https://chromium.googlesource.com/chromium/src/+/main/docs/chromeos_build_instructions.md + # https://chromium.googlesource.com/chromiumos/docs/+/main/simple_chrome_workflow.md is_chromeos_device = false + + # Determines if we run the test in skylab, aka the CrOS labs. + is_skylab = false + + # Determines if we collect hardware information in chrome://system and + # feedback logs. A similar build flag "hw_details" is defined in Chrome OS + # (see https://crrev.com/c/3123455). + is_chromeos_with_hw_details = false + + # Determines if we're willing to link against libinput + use_libinput = false + + # Refers to the separate branding required for the reven build. + is_reven = false }
diff --git a/build/config/chromeos/rules.gni b/build/config/chromeos/rules.gni index c8693ba..10af886 100644 --- a/build/config/chromeos/rules.gni +++ b/build/config/chromeos/rules.gni
@@ -1,15 +1,16 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/chrome_build.gni") import("//build/config/chromeos/args.gni") import("//build/config/chromeos/ui_mode.gni") +import("//build/config/dcheck_always_on.gni") import("//build/config/gclient_args.gni") import("//build/config/python.gni") import("//build/util/generate_wrapper.gni") -assert((is_chromeos_ash || is_chromeos_lacros) && is_chromeos_device) +assert(is_chromeos && is_chromeos_device) # Determine the real paths for various items in the SDK, which may be used # in the 'generate_runner_script' template below. We do so outside the template @@ -57,8 +58,7 @@ _symlinks = [] _symlinks = [ # Tast harness & test data. - rebase_path("${_cache_path_prefix}+chromeos-base/tast-cmd"), - rebase_path("${_cache_path_prefix}+chromeos-base/tast-remote-tests-cros"), + rebase_path("${_cache_path_prefix}+autotest_server_package.tar.bz2"), # Binutils (and other toolchain tools) used to deploy Chrome to the device. rebase_path( @@ -69,7 +69,7 @@ # VM-related tools. _symlinks += [ rebase_path("${_cache_path_prefix}+sys-firmware/seabios"), - rebase_path("${_cache_path_prefix}+chromiumos_qemu_image.tar.xz"), + rebase_path("${_cache_path_prefix}+chromiumos_test_image.tar.xz"), rebase_path("${_cache_path_prefix}+app-emulation/qemu"), ] } @@ -100,6 +100,133 @@ } } +# Creates dependencies required by skylab testing. If passed the +# generated_script and test_exe this will generate the skylab runner script. +# If passed tast_attr_expr, tast_tests or tast_disabled_tests this will +# generate a filter file containing the expression for running tests in skylab. +# Args: +# generated_script: Name of the generated runner script created for test_exe +# test_exe: Name of the executable to run with the generated script. +# tast_attr_expr: Tast expression to determine tests to run. This creates the +# initial set of tests that can be further filtered.. +# tast_tests: Names of tests to enable in tast. All other tests will be +# disabled that are not listed. +# tast_disabled_tests: Names of tests to disable in tast. All other tests that +# match the tast expression will still run. +# tast_control: gni file with collections of tests to be used for specific +# filters (e.g. "//chromeos/tast_control.gni"). Any lists of strings in +# this file will be used to generate additional tast expressions with +# those strings expanded into tests to disable (i.e. as && !"name:test"). +# The name of those lists are then intended to be used to specify in +# test_suites.pyl which collection to be used on specific test suites. +template("generate_skylab_deps") { + forward_variables_from(invoker, + [ + "generated_script", + "test_exe", + "tast_attr_expr", + "tast_tests", + "tast_disabled_tests", + "tast_control", + ]) + if (defined(test_exe) || defined(generated_script)) { + assert(defined(test_exe) && defined(generated_script), + "The test_exe and generated_script must both be defined when " + + "generating the skylab runner script") + action(target_name) { + script = "//build/chromeos/generate_skylab_deps.py" + outputs = [ generated_script ] + args = [ + "generate-runner", + "--test-exe", + test_exe, + "--output", + rebase_path(generated_script, root_build_dir), + ] + + deps = [ "//testing/buildbot/filters:chromeos_filters" ] + if (defined(invoker.deps)) { + deps += invoker.deps + } + + data = [ generated_script ] + if (defined(invoker.data)) { + data += invoker.data + } + + data_deps = [ "//testing:test_scripts_shared" ] + if (defined(invoker.data_deps)) { + data_deps += invoker.data_deps + } + } + } + if (defined(tast_attr_expr) || defined(tast_tests) || + defined(tast_disabled_tests)) { + if (defined(tast_disabled_tests)) { + assert(defined(tast_attr_expr), + "tast_attr_expr must be used when specifying tast_disabled_tests.") + } + _generated_filter = "$root_build_dir/bin/${target_name}.filter" + _skylab_args = [ + "generate-filter", + "--output", + rebase_path(_generated_filter), + ] + if (defined(tast_control)) { + _skylab_args += [ + "--tast-control", + rebase_path(tast_control), + ] + } + if (defined(tast_attr_expr)) { + _skylab_args += [ + "--tast-expr", + tast_attr_expr, + ] + } + if (defined(tast_tests)) { + foreach(_test, tast_tests) { + _skylab_args += [ + "--enabled-tests", + _test, + ] + } + } + if (defined(tast_disabled_tests)) { + foreach(_test, tast_disabled_tests) { + _excluded_test_name_and_board = [] + _excluded_test_name_and_board = string_split(_test, "@") + [ "" ] + _excluded_test_name = _excluded_test_name_and_board[0] + _excluded_board = _excluded_test_name_and_board[1] + if (_excluded_board == "" || _excluded_board == cros_board) { + _skylab_args += [ + "--disabled-tests", + _excluded_test_name, + ] + } + } + } + action(target_name) { + script = "//build/chromeos/generate_skylab_deps.py" + if (defined(tast_control)) { + sources = [ tast_control ] + } + outputs = [ _generated_filter ] + args = _skylab_args + if (defined(invoker.data_deps)) { + data_deps = invoker.data_deps + } + data = [ _generated_filter ] + if (defined(invoker.data)) { + data += invoker.data + } + if (defined(invoker.deps)) { + deps = invoker.deps + } + } + } +} + # Creates a script at $generated_script that can be used to launch a cros VM # and optionally run a test within it. # Args: @@ -123,6 +250,7 @@ # them, and it's designed for use cases where one builds for one board # (e.g. amd64-generic), but tests on a different board (e.g. eve). # tast_vars: A list of "key=value" runtime variable pairs to pass to invoke +# strip_chrome: If true, strips Chrome before deploying it for non-Tast tests. # the Tast tests. For more details, please see: # https://chromium.googlesource.com/chromiumos/platform/tast/+/HEAD/docs/writing_tests.md#Runtime-variables template("generate_runner_script") { @@ -133,6 +261,7 @@ "generated_script", "runtime_deps_file", "skip_generating_board_args", + "strip_chrome", "tast_attr_expr", "tast_tests", "tast_vars", @@ -157,9 +286,9 @@ if (!defined(deploy_lacros)) { deploy_lacros = false } - assert(!(deploy_chrome && deploy_lacros), - "deploy_chrome and deploy_lacros are exclusive.") - + if (!defined(strip_chrome)) { + strip_chrome = false + } is_tast = defined(tast_attr_expr) || defined(tast_tests) assert(!(is_tast && defined(test_exe)), "Tast tests are invoked from binaries shipped with the VM image. " + @@ -167,6 +296,10 @@ assert(is_tast || !defined(tast_vars), "tast_vars is only support for Tast tests") + if (is_tast) { + not_needed([ "strip_chrome" ]) + } + # If we're in the cros chrome-sdk (and not the raw ebuild), the test will # need some additional runtime data located in the SDK cache. _sdk_data = [] @@ -196,26 +329,23 @@ if (is_tast) { # Add tast sdk items. - _sdk_data += [ - _symlink_targets[0], - _symlink_targets[1], - ] + _sdk_data += [ _symlink_targets[0] ] } if (deploy_chrome) { # To deploy chrome to the VM, it needs to be stripped down to fit into # the VM. This is done by using binutils in the toolchain. So add the # toolchain to the data. _sdk_data += [ + _symlink_targets[1], _symlink_targets[2], - _symlink_targets[3], ] } if (_cros_is_vm) { # Add vm sdk items. _sdk_data += [ + _symlink_targets[3], _symlink_targets[4], _symlink_targets[5], - _symlink_targets[6], ] } } @@ -224,7 +354,6 @@ generate_wrapper(target_name) { executable = "//build/chromeos/test_runner.py" - use_vpython3 = true wrapper_script = generated_script executable_args = [] @@ -275,6 +404,12 @@ ] } } + if (dcheck_always_on) { + executable_args += [ + "--tast-extra-use-flags", + "chrome_dcheck", + ] + } } else { executable_args += [ "host-cmd" ] } @@ -286,6 +421,10 @@ "-v", ] + if (!is_tast && strip_chrome) { + executable_args += [ "--strip-chrome" ] + } + if (!skip_generating_board_args) { executable_args += [ "--board", @@ -315,7 +454,7 @@ executable_args += [ "--deploy-lacros" ] } - if (deploy_chrome && !defined(test_exe) && !is_tast) { + if (deploy_chrome && !defined(test_exe)) { executable_args += [ "--deploy-chrome" ] } @@ -325,7 +464,6 @@ deps += invoker.deps } data = [ - "//.vpython", "//.vpython3", # We use android test-runner's results libs to construct gtest output @@ -334,10 +472,12 @@ "//build/android/pylib/base/", "//build/android/pylib/results/", "//build/chromeos/", + "//build/util/", # Needed for various SDK components used below. "//build/cros_cache/chrome-sdk/misc/", "//build/cros_cache/chrome-sdk/symlinks/", + "//chrome/VERSION", # The LKGM file controls what version of the VM image to download. Add it # as data here so that changes to it will trigger analyze. @@ -361,11 +501,15 @@ template("tast_test") { forward_variables_from(invoker, "*") + if (!defined(deploy_lacros_chrome)) { + deploy_lacros_chrome = false + } + # Default the expression to match any chrome-related test. if (!defined(tast_attr_expr) && !defined(tast_tests)) { # The following expression filters out all non-critical tests. See the link # below for more details: - # https://chromium.googlesource.com/chromiumos/platform/tast/+/master/docs/test_attributes.md + # https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/test_attributes.md tast_attr_expr = "\"group:mainline\" && \"dep:chrome\"" if (defined(enable_tast_informational_tests) && @@ -386,8 +530,14 @@ if (defined(tast_disabled_tests)) { assert(defined(tast_attr_expr), "tast_attr_expr must be used when specifying tast_disabled_tests.") - foreach(test, tast_disabled_tests) { - tast_attr_expr += " && !\"name:${test}\"" + foreach(_test, tast_disabled_tests) { + _excluded_test_name_and_board = [] + _excluded_test_name_and_board = string_split(_test, "@") + [ "" ] + _excluded_test_name = _excluded_test_name_and_board[0] + _excluded_board = _excluded_test_name_and_board[1] + if (_excluded_board == "" || _excluded_board == cros_board) { + tast_attr_expr += " && !\"name:${_excluded_test_name}\"" + } } } if (defined(tast_attr_expr)) { @@ -398,6 +548,7 @@ generated_script = "$root_build_dir/bin/run_${target_name}" runtime_deps_file = "$root_out_dir/${target_name}.runtime_deps" deploy_chrome = true + deploy_lacros = deploy_lacros_chrome data_deps = [ "//:chromiumos_preflight", # Builds the browser. "//chromeos:cros_chrome_deploy", # Adds additional browser run-time deps. @@ -411,6 +562,12 @@ ] data = [ "//components/crash/content/tools/generate_breakpad_symbols.py" ] + if (deploy_lacros_chrome) { + data += [ + # A script needed to launch Lacros in Lacros Tast tests. + "//build/lacros/mojo_connection_lacros_launcher.py", + ] + } } } @@ -420,60 +577,77 @@ "tast_attr_expr", "tast_disabled_tests", "tast_tests", + "tast_control", ]) assert(defined(tast_attr_expr) != defined(tast_tests), "Specify one of tast_tests or tast_attr_expr.") - # Append any disabled tests to the expression. - if (defined(tast_disabled_tests)) { - assert(defined(tast_attr_expr), - "tast_attr_expr must be used when specifying tast_disabled_tests.") - foreach(test, tast_disabled_tests) { - tast_attr_expr += " && !\"name:${test}\"" + _lacros_data_deps = [ + "//chrome", # Builds the browser. + + # Tools used to symbolize Chrome crash dumps. + # TODO(crbug.com/1156772): Remove these if/when all tests pick them up by + # default. + "//third_party/breakpad:dump_syms", + "//third_party/breakpad:minidump_dump", + "//third_party/breakpad:minidump_stackwalk", + ] + + _lacros_data = [ + "//components/crash/content/tools/generate_breakpad_symbols.py", + + # A script needed to launch Lacros in Lacros Tast tests. + "//build/lacros/mojo_connection_lacros_launcher.py", + ] + + if (is_skylab) { + generate_skylab_deps(target_name) { + data = _lacros_data + data_deps = _lacros_data_deps + + # To disable a test on specific milestones, add it to the appropriate + # collection in the following file + tast_control = "//chromeos/tast_control.gni" } - } - if (defined(tast_attr_expr)) { - tast_attr_expr = "( " + tast_attr_expr + " )" - } + } else { + # Append any disabled tests to the expression. + if (defined(tast_disabled_tests)) { + assert(defined(tast_attr_expr), + "tast_attr_expr must be used when specifying tast_disabled_tests.") + foreach(_test, tast_disabled_tests) { + _excluded_test_name_and_board = [] + _excluded_test_name_and_board = string_split(_test, "@") + [ "" ] + _excluded_test_name = _excluded_test_name_and_board[0] + _excluded_board = _excluded_test_name_and_board[1] + if (_excluded_board == "" || _excluded_board == cros_board) { + tast_attr_expr += " && !\"name:${_excluded_test_name}\"" + } + } + } + if (defined(tast_attr_expr)) { + tast_attr_expr = "( " + tast_attr_expr + " )" + } + generate_runner_script(target_name) { + testonly = true + deploy_lacros = true + generated_script = "$root_build_dir/bin/run_${target_name}" + runtime_deps_file = "$root_out_dir/${target_name}.runtime_deps" - generate_runner_script(target_name) { - testonly = true - deploy_lacros = true - generated_script = "$root_build_dir/bin/run_${target_name}" - runtime_deps_file = "$root_out_dir/${target_name}.runtime_deps" + # At build time, Lacros tests don't know whether they'll run on VM or HW, + # and instead, these flags are specified at runtime when invoking the + # generated runner script. + skip_generating_board_args = true - # At build time, Lacros tests don't know whether they'll run on VM or HW, - # and instead, these flags are specified at runtime when invoking the - # generated runner script. - skip_generating_board_args = true + # By default, tast tests download a lacros-chrome from a gcs location and + # use it for testing. To support running lacros tast tests from Chromium CI, + # a Var is added to support pointing the tast tests to use a specified + # pre-deployed lacros-chrome. The location is decided by: + # https://source.chromium.org/chromium/chromium/src/+/main:third_party/chromite/scripts/deploy_chrome.py;l=80;drc=86f1234a4be8e9574442e076cdc835897f7bea61 + tast_vars = [ "lacros.DeployedBinary=/usr/local/lacros-chrome" ] - # By default, tast tests download a lacros-chrome from a gcs location and - # use it for testing. To support running lacros tast tests from Chromium CI, - # a Var is added to support pointing the tast tests to use a specified - # pre-deployed lacros-chrome. The location is decided by: - # https://source.chromium.org/chromium/chromium/src/+/master:third_party/chromite/scripts/deploy_chrome.py;l=80;drc=86f1234a4be8e9574442e076cdc835897f7bea61 - tast_vars = [ "lacrosDeployedBinary=/usr/local/lacros-chrome" ] + data_deps = _lacros_data_deps - # Lacros tast tests may have different test expectations based on whether - # they're for Chromium or Chrome. - tast_vars += [ "lacrosIsChromeBranded=$is_chrome_branded" ] - - data_deps = [ - "//chrome", # Builds the browser. - - # Tools used to symbolize Chrome crash dumps. - # TODO(crbug.com/1156772): Remove these if/when all tests pick them up by - # default. - "//third_party/breakpad:dump_syms", - "//third_party/breakpad:minidump_dump", - "//third_party/breakpad:minidump_stackwalk", - ] - - data = [ - "//components/crash/content/tools/generate_breakpad_symbols.py", - - # A script needed to launch Lacros in Lacros Tast tests. - "//build/lacros/mojo_connection_lacros_launcher.py", - ] + data = _lacros_data + } } }
diff --git a/build/config/chromeos/ui_mode.gni b/build/config/chromeos/ui_mode.gni index df578bc..ce8fa8b 100644 --- a/build/config/chromeos/ui_mode.gni +++ b/build/config/chromeos/ui_mode.gni
@@ -1,9 +1,11 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/config/chromeos/args.gni") + declare_args() { - # Deprecated, use is_lacros. + # Deprecated, use is_chromeos_lacros. # # This controls UI configuration for Chrome. # If this flag is set, we assume Chrome runs on Chrome OS devices, using @@ -12,15 +14,25 @@ # TODO(crbug.com/1052397): # Define chromeos_product instead, which takes either "browser" or "ash". # Re-define the following variables as: - # is_lacros = chromeos_product == "browser" - # is_ash = chromeos_product == "ash" + # is_chromeos_lacros = chromeos_product == "browser" + # is_chromeos_ash = chromeos_product == "ash" chromeos_is_browser_only = false - # Setting this to true when building LaCrOS-chrome will cause it to - # *also* build ash-chrome in a subdirectory using an alternate toolchain. + # Setting this to true when building linux Lacros-chrome will cause it to + # *also* build linux ash-chrome in a subdirectory using an alternate + # toolchain. # Don't set this unless you're sure you want it, because it'll double # your build time. also_build_ash_chrome = false + + # Setting this to true when building linux ash-chrome will cause it to + # *also* build linux Lacros-chrome in a subdirectory using an alternate toolchain. + also_build_lacros_chrome = false + + # Setting this when building ash-chrome will cause it to + # *also* build Lacros-chrome in a subdirectory using an alternate toolchain. + # You can set this to either "amd64" or "arm". + also_build_lacros_chrome_for_architecture = "" } # is_chromeos_{ash,lacros} is used to specify that it is specific to either @@ -30,7 +42,19 @@ # is_chromeos_{ash,lacros} should be set true only for builds with target # toolchains. is_chromeos_ash = is_chromeos && !chromeos_is_browser_only +is_chromeos_lacros = is_chromeos && chromeos_is_browser_only -# TODO(crbug.com/1052397): Remove is_linux once lacros-chrome switches -# to target_os=chromeos -is_chromeos_lacros = (is_chromeos || is_linux) && chromeos_is_browser_only +# also_build_ash_chrome and also_build_lacros_chrome cannot be both true. +assert(!(also_build_ash_chrome && also_build_lacros_chrome)) + +# Can't set both also_build_lacros_chrome and +# also_build_lacros_chrome_for_architecture. +assert(!(also_build_lacros_chrome == true && + also_build_lacros_chrome_for_architecture != "")) + +# also_build_lacros_chrome_for_architecture is for device only. +assert(is_chromeos_device || also_build_lacros_chrome_for_architecture == "") + +# also_build_lacros_chrome_for_architecture is for ash build only. +assert(!chromeos_is_browser_only || + also_build_lacros_chrome_for_architecture == "")
diff --git a/build/config/clang/BUILD.gn b/build/config/clang/BUILD.gn index 180e2e6..ed39cc6 100644 --- a/build/config/clang/BUILD.gn +++ b/build/config/clang/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -15,10 +15,16 @@ "-add-plugin", "-Xclang", "find-bad-constructs", + "-Xclang", "-plugin-arg-find-bad-constructs", "-Xclang", - "checked-ptr-as-trivial-member", + "raw-ref-template-as-trivial-member", + + "-Xclang", + "-plugin-arg-find-bad-constructs", + "-Xclang", + "check-stack-allocated", ] if (is_linux || is_chromeos || is_android || is_fuchsia) { @@ -29,6 +35,27 @@ "check-ipc", ] } + + if (enable_check_raw_ptr_fields) { + cflags += [ + "-Xclang", + "-plugin-arg-find-bad-constructs", + "-Xclang", + "check-raw-ptr-fields", + + # TODO(keishi): Remove this once crrev.com/c/4387753 is rolled out. + "-Xclang", + "-plugin-arg-find-bad-constructs", + "-Xclang", + "raw-ptr-exclude-path=base/no_destructor.h", + + # TODO(keishi): Remove this once crrev.com/c/4086161 lands. + "-Xclang", + "-plugin-arg-find-bad-constructs", + "-Xclang", + "raw-ptr-exclude-path=base/containers/span.h", + ] + } } } @@ -50,10 +77,5 @@ data = [ "$clang_base_path/bin/llvm-symbolizer.exe" ] } else { data = [ "$clang_base_path/bin/llvm-symbolizer" ] - - if (!is_apple) { - # llvm-symbolizer uses libstdc++ from the clang package. - data += [ "$clang_base_path/lib/libstdc++.so.6" ] - } } }
diff --git a/build/config/clang/clang.gni b/build/config/clang/clang.gni index 9775296..1be5447 100644 --- a/build/config/clang/clang.gni +++ b/build/config/clang/clang.gni
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -13,10 +13,14 @@ # coding guidelines, etc. Only used when compiling with Chrome's Clang, not # Chrome OS's. clang_use_chrome_plugins = - is_clang && !is_nacl && !use_xcode_clang && + is_clang && !is_nacl && current_os != "zos" && default_toolchain != "//build/toolchain/cros:target" -if (!use_cobalt_customizations) { + enable_check_raw_ptr_fields = + build_with_chromium && !is_official_build && + ((is_linux && !is_castos) || (is_android && !is_cast_android)) + + if (!use_cobalt_customizations) { clang_base_path = default_clang_base_path -} + } }
diff --git a/build/config/compiler/BUILD.gn b/build/config/compiler/BUILD.gn index 36702d9..6c6fd68 100644 --- a/build/config/compiler/BUILD.gn +++ b/build/config/compiler/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,7 +6,6 @@ import("//build/config/android/config.gni") import("//build/config/c++/c++.gni") import("//build/config/chrome_build.gni") -import("//build/config/chromecast_build.gni") import("//build/config/chromeos/args.gni") import("//build/config/chromeos/ui_mode.gni") import("//build/config/clang/clang.gni") @@ -17,6 +16,7 @@ import("//build/config/gclient_args.gni") } import("//build/config/host_byteorder.gni") +import("//build/config/rust.gni") import("//build/config/sanitizers/sanitizers.gni") import("//build/config/ui.gni") import("//build/toolchain/cc_wrapper.gni") @@ -32,9 +32,6 @@ current_cpu == "mips" || current_cpu == "mips64") { import("//build/config/mips.gni") } -if (current_cpu == "x64") { - import("//build/config/x64.gni") -} if (is_mac) { import("//build/config/apple/symbols.gni") } @@ -48,6 +45,17 @@ import("//build/config/nacl/config.gni") } +lld_path = "" +if (!is_clang) { + declare_args() { + # This allows overriding the location of lld. + lld_path = rebase_path("$clang_base_path/bin", root_build_dir) + } +} else { + # clang looks for lld next to it, no need for -B. + lld_path = "" +} + declare_args() { # Normally, Android builds are lightly optimized, even for debug builds, to # keep binary size down. Setting this flag to true disables such optimization @@ -66,9 +74,7 @@ # Enable fatal linker warnings. Building Chromium with certain versions # of binutils can cause linker warning. - # TODO(thakis): Set this to true unconditionally once lld/MachO bring-up - # is along far enough that it no longer emits linker warnings. - fatal_linker_warnings = !(is_apple && use_lld) + fatal_linker_warnings = true # Build with C++ RTTI enabled. Chromium builds without RTTI by default, # but some sanitizers are known to require it, like CFI diagnostics @@ -81,9 +87,6 @@ # the needed gcov profiling data. auto_profile_path = "" - # Allow projects that wish to stay on C++11 to override Chromium's default. - use_cxx11 = false - # Path to an AFDO profile to use while building with clang, if any. Empty # implies none. clang_sample_profile_path = "" @@ -96,7 +99,7 @@ # nonsensical for said projects. clang_use_default_sample_profile = chrome_pgo_phase == 0 && build_with_chromium && is_official_build && - (is_android || chromeos_is_browser_only || is_chromecast) + (is_android || chromeos_is_browser_only) # This configuration is used to select a default profile in Chrome OS based on # the microarchitectures we are using. This is only used if @@ -105,7 +108,8 @@ chromeos_afdo_platform = "atom" # Emit debug information for profiling wile building with clang. - clang_emit_debug_info_for_profiling = false + # Only enable this for ChromeOS official builds for AFDO. + clang_emit_debug_info_for_profiling = is_chromeos_device && is_official_build # Turn this on to have the compiler output extra timing information. compiler_timing = false @@ -122,7 +126,8 @@ # the space overhead is too great. We should use some mixture of profiles and # optimization settings to better tune the size increase. thin_lto_enable_optimizations = - (is_chromeos_ash || is_android || is_win || is_linux) && is_official_build + (is_chromeos || is_android || is_win || is_linux || is_mac || + (is_ios && use_lld)) && is_official_build # Initialize all local variables with a pattern. This flag will fill # uninitialized floating-point types (and 32-bit pointers) with 0xFF and the @@ -130,12 +135,13 @@ # recognizable in the debugger, and crashes on memory accesses through # uninitialized pointers. # - # TODO(crbug.com/1131993): Enabling this when 'is_android' is true breaks - # content_shell_test_apk on both ARM and x86. - # - # TODO(crbug.com/977230): Enabling this when 'use_xcode_clang' is true may - # call an old clang that doesn't support auto-init. - init_stack_vars = !is_android && !use_xcode_clang + # TODO(crbug.com/1131993): This regresses binary size by ~1MB on Android and + # needs to be evaluated before enabling it there as well. + init_stack_vars = !(is_android && is_official_build) + + # Zero init has favorable performance/size tradeoffs for Chrome OS + # but was not evaluated for other platforms. + init_stack_vars_zero = is_chromeos # This argument is to control whether enabling text section splitting in the # final binary. When enabled, the separated text sections with prefix @@ -146,16 +152,6 @@ # The gold linker by default has text section splitting enabled. use_text_section_splitting = false - # Token limits may not be accurate for build configs not covered by the CQ, - # so only enable them by default for mainstream build configs. - enable_wmax_tokens = - !is_official_build && - ((is_mac && target_cpu == "x64" && !use_system_xcode) || - (is_linux && !is_chromeos && target_cpu == "x64") || - (is_win && target_cpu == "x86") || (is_win && target_cpu == "x64") || - (is_android && target_cpu == "arm") || - (is_android && target_cpu == "arm64")) - # Turn off the --call-graph-profile-sort flag for lld by default. Enable # selectively for targets where it's beneficial. enable_call_graph_profile_sort = chrome_pgo_phase == 2 @@ -174,11 +170,19 @@ # Enable -H, which prints the include tree during compilation. # For use by tools/clang/scripts/analyze_includes.py show_includes = false -} -declare_args() { - # C++11 may not be an option if Android test infrastructure is used. - use_cxx11_on_android = use_cxx11 + # Enable Profi algorithm. Profi can infer block and edge counts. + # https://clang.llvm.org/docs/UsersManual.html#using-sampling-profilers + # TODO(crbug.com/1375958i:) Possibly enable this for Android too. + use_profi = is_chromeos + + # If true, linker crashes will be rerun with `--reproduce` which causes + # a reproducer file to be saved. + save_reproducers_on_lld_crash = false + + # Allow projects that wish to stay on C++17 to override Chromium's default. + # TODO(crbug.com/1402249): evaluate removing this end of 2023 + use_cxx17 = false } declare_args() { @@ -191,11 +195,11 @@ # other architectures. # # lld doesn't have the bug. - use_icf = - (is_posix || is_fuchsia) && !is_debug && !using_sanitizer && - !use_clang_coverage && !(is_android && use_order_profiling) && - (use_lld || (use_gold && (is_chromeos_ash || !(current_cpu == "x86" || - current_cpu == "x64")))) + use_icf = (is_posix || is_fuchsia) && !is_debug && !using_sanitizer && + !use_clang_coverage && current_os != "zos" && + !(is_android && use_order_profiling) && + (use_lld || (use_gold && (is_chromeos || !(current_cpu == "x86" || + current_cpu == "x64")))) } if (is_android || (is_chromeos_ash && is_chromeos_device)) { @@ -217,7 +221,8 @@ assert(!(llvm_force_head_revision && use_goma), "can't use goma with trunk clang") -assert(!(llvm_force_head_revision && use_rbe), "can't use rbe with trunk clang") +assert(!(llvm_force_head_revision && use_remoteexec), + "can't use rbe with trunk clang") # default_include_dirs --------------------------------------------------------- # @@ -231,6 +236,18 @@ ] } +# Compiler instrumentation can introduce dependencies in DSOs to symbols in +# the executable they are loaded into, so they are unresolved at link-time. +config("no_unresolved_symbols") { + if (!using_sanitizer && + (is_linux || is_chromeos || is_android || is_fuchsia)) { + ldflags = [ + "-Wl,-z,defs", + "-Wl,--as-needed", + ] + } +} + # compiler --------------------------------------------------------------------- # # Base compiler configuration. @@ -258,9 +275,6 @@ configs += [ "//build/config/android:compiler" ] } else if (is_linux || is_chromeos) { configs += [ "//build/config/linux:compiler" ] - if (is_chromeos_ash) { - configs += [ "//build/config/chromeos:compiler" ] - } } else if (is_nacl) { configs += [ "//build/config/nacl:compiler" ] } else if (is_mac) { @@ -271,11 +285,14 @@ configs += [ "//build/config/fuchsia:compiler" ] } else if (current_os == "aix") { configs += [ "//build/config/aix:compiler" ] + } else if (current_os == "zos") { + configs += [ "//build/config/zos:compiler" ] } configs += [ # See the definitions below. ":clang_revision", + ":rustc_revision", ":compiler_cpu_abi", ":compiler_codegen", ":compiler_deterministic", @@ -308,7 +325,7 @@ cflags += [ "-fno-strict-aliasing" ] # See http://crbug.com/32204 # Stack protection. - if (is_mac) { + if (is_apple) { # The strong variant of the stack protector significantly increases # binary size, so only enable it in debug mode. if (is_debug) { @@ -316,10 +333,12 @@ } else { cflags += [ "-fstack-protector" ] } - } else if ((is_posix && !is_chromeos_ash && !is_nacl) || is_fuchsia) { + } else if ((is_posix && !is_chromeos && !is_nacl) || is_fuchsia) { # TODO(phajdan.jr): Use -fstack-protector-strong when our gcc supports it. # See also https://crbug.com/533294 - cflags += [ "--param=ssp-buffer-size=4" ] + if (current_os != "zos") { + cflags += [ "--param=ssp-buffer-size=4" ] + } # The x86 toolchain currently has problems with stack-protector. if (is_android && current_cpu == "x86") { @@ -332,10 +351,14 @@ if (use_lld) { ldflags += [ "-fuse-ld=lld" ] + if (lld_path != "") { + ldflags += [ "-B$lld_path" ] + } } # Linker warnings. - if (fatal_linker_warnings && !is_apple && current_os != "aix") { + if (fatal_linker_warnings && !is_apple && current_os != "aix" && + current_os != "zos") { ldflags += [ "-Wl,--fatal-warnings" ] } if (fatal_linker_warnings && is_apple) { @@ -354,7 +377,7 @@ ] } - # Non-Mac Posix and Fuchsia compiler flags setup. + # Non-Apple Posix and Fuchsia compiler flags setup. # ----------------------------------- if ((is_posix && !is_apple) || is_fuchsia) { if (enable_profiling) { @@ -378,7 +401,7 @@ # compute, so only use it in the official build to avoid slowing down # links. ldflags += [ "-Wl,--build-id=sha1" ] - } else if (current_os != "aix") { + } else if (current_os != "aix" && current_os != "zos") { ldflags += [ "-Wl,--build-id" ] } @@ -406,6 +429,23 @@ } } + # Apple compiler flags setup. + # --------------------------------- + if (is_apple) { + # On Intel, clang emits both Apple's "compact unwind" information and + # DWARF eh_frame unwind information by default, for compatibility reasons. + # This flag limits emission of eh_frame information to functions + # whose unwind information can't be expressed in the compact unwind format + # (which in practice means almost everything gets only compact unwind + # entries). This reduces object file size a bit and makes linking a bit + # faster. + # On arm64, this is already the default behavior. + if (current_cpu == "x64") { + asmflags += [ "-femit-dwarf-unwind=no-compact-unwind" ] + cflags += [ "-femit-dwarf-unwind=no-compact-unwind" ] + } + } + # Linux/Android/Fuchsia common flags setup. # --------------------------------- if (is_linux || is_chromeos || is_android || is_fuchsia) { @@ -427,27 +467,11 @@ if (!is_component_build) { ldflags += [ "-Wl,-z,now" ] } - - # Compiler instrumentation can introduce dependencies in DSOs to symbols in - # the executable they are loaded into, so they are unresolved at link-time. - if (!using_sanitizer) { - ldflags += [ - "-Wl,-z,defs", - "-Wl,--as-needed", - ] - } } # Linux-specific compiler flags setup. # ------------------------------------ - if ((is_posix || is_fuchsia) && use_lld) { - if (current_cpu == "arm64") { - # Reduce the page size from 65536 in order to reduce binary size slightly - # by shrinking the alignment gap between segments. This also causes all - # segments to be mapped adjacently, which breakpad relies on. - ldflags += [ "-Wl,-z,max-page-size=4096" ] - } - } else if (use_gold) { + if (use_gold) { ldflags += [ "-fuse-ld=gold" ] if (!is_android) { # On Android, this isn't needed. gcc in the NDK knows to look next to @@ -476,7 +500,7 @@ #} } - if (use_icf && !is_apple) { + if (use_icf && (!is_apple || use_lld)) { ldflags += [ "-Wl,--icf=all" ] } @@ -520,8 +544,14 @@ ldflags += [ "-Wl,-z,keep-text-section-prefix" ] } - if (is_clang && !is_nacl && !use_xcode_clang) { + if (is_clang && !is_nacl && current_os != "zos") { cflags += [ "-fcrash-diagnostics-dir=" + clang_diagnostic_dir ] + if (save_reproducers_on_lld_crash && use_lld) { + ldflags += [ + "-fcrash-diagnostics=all", + "-fcrash-diagnostics-dir=" + clang_diagnostic_dir, + ] + } # TODO(hans): Remove this once Clang generates better optimized debug info # by default. https://crbug.com/765793 @@ -536,18 +566,19 @@ ldflags += [ "-Wl,-mllvm,-instcombine-lower-dbg-declare=0" ] } } + + # TODO(crbug.com/1235145): Investigate why/if this should be needed. + if (is_win) { + cflags += [ "/clang:-ffp-contract=off" ] + } else { + cflags += [ "-ffp-contract=off" ] + } } # C11/C++11 compiler flags setup. # --------------------------- if (is_linux || is_chromeos || is_android || (is_nacl && is_clang) || current_os == "aix") { - if (target_os == "android") { - cxx11_override = use_cxx11_on_android - } else { - cxx11_override = use_cxx11 - } - if (is_clang) { standard_prefix = "c" @@ -571,53 +602,56 @@ } cflags_c += [ "-std=${standard_prefix}11" ] - if (cxx11_override) { - # Override Chromium's default for projects that wish to stay on C++11. - cflags_cc += [ "-std=${standard_prefix}++11" ] - } else { + if (is_nacl && !is_nacl_saigo) { + # This is for the pnacl_newlib toolchain. It's only used to build + # a few independent ppapi test files that don't pull in any other + # dependencies. cflags_cc += [ "-std=${standard_prefix}++14" ] - } - } else if (!is_win && !is_nacl) { - if (target_os == "android") { - cxx11_override = use_cxx11_on_android + if (is_clang) { + cflags_cc += [ "-fno-trigraphs" ] + } + } else if (is_clang) { + if (use_cxx17) { + cflags_cc += [ "-std=${standard_prefix}++17" ] + } else { + cflags_cc += [ "-std=${standard_prefix}++20" ] + } } else { - cxx11_override = use_cxx11 + # The gcc bots are currently using GCC 9, which is not new enough to + # support "c++20"/"gnu++20". + cflags_cc += [ "-std=${standard_prefix}++2a" ] } - - # TODO(mcgrathr) - the NaCl GCC toolchain doesn't support either gnu11/gnu++11 - # or c11/c++11; we technically don't need this toolchain any more, but there - # are still a few buildbots using it, so until those are turned off - # we need the !is_nacl clause and the (is_nacl && is_clang) clause, above. + } else if (is_win) { + cflags_c += [ "/std:c11" ] + if (use_cxx17 || (!is_clang && defined(msvc_use_cxx17) && msvc_use_cxx17)) { + cflags_cc += [ "/std:c++17" ] + } else { + cflags_cc += [ "/std:c++20" ] + } + } else if (!is_nacl) { + # TODO(mcgrathr) - the NaCl GCC toolchain doesn't support either + # gnu11/gnu++11 or c11/c++11; we technically don't need this toolchain any + # more, but there are still a few buildbots using it, so until those are + # turned off we need the !is_nacl clause and the (is_nacl && is_clang) + # clause, above. cflags_c += [ "-std=c11" ] - if (cxx11_override) { - cflags_cc += [ "-std=c++11" ] + + if (use_cxx17) { + cflags_cc += [ "-std=c++17" ] } else { - cflags_cc += [ "-std=c++14" ] + cflags_cc += [ "-std=c++20" ] } } - # C++17 removes trigraph support, so preemptively disable trigraphs. This is - # especially useful given the collision with ecmascript's logical assignment - # operators: https://github.com/tc39/proposal-logical-assignment - if (is_clang) { - # clang-cl disables trigraphs by default - if (!is_win) { - # The gnu variants of C++11 and C++14 already disable trigraph support, - # but when building with clang, we use -std=c++11 / -std=c++14, which - # enables trigraph support: override that here. - cflags_cc += [ "-fno-trigraphs" ] - } - - # Don't warn that trigraphs are ignored, since trigraphs are disabled - # anyway. + if (is_clang && current_os != "zos") { + # C++17 removes trigraph support, but clang still warns that it ignores + # them when seeing them. Don't. cflags_cc += [ "-Wno-trigraphs" ] } - if (is_mac) { - # The system libc++ on Mac doesn't have aligned allocation in C++17. - defines += [ "_LIBCPP_HAS_NO_ALIGNED_ALLOCATION" ] - cflags_cc += [ "-stdlib=libc++" ] - ldflags += [ "-stdlib=libc++" ] + if (use_relative_vtables_abi) { + cflags_cc += [ "-fexperimental-relative-c++-abi-vtables" ] + ldflags += [ "-fexperimental-relative-c++-abi-vtables" ] } # Add flags for link-time optimization. These flags enable @@ -640,9 +674,11 @@ # available disk space, 40GB and 100000 files. cache_policy = "cache_size=10%:cache_size_bytes=40g:cache_size_files=100000" + # An import limit of 30 has better performance (per speedometer) and lower + # binary size than the default setting of 100. # TODO(gbiv): We ideally shouldn't need to specify this; ThinLTO # should be able to better manage binary size increases on its own. - import_instr_limit = 5 + import_instr_limit = 30 if (is_win) { ldflags += [ @@ -651,6 +687,7 @@ "/lldltocache:" + rebase_path("$root_out_dir/thinlto-cache", root_build_dir), "/lldltocachepolicy:$cache_policy", + "-mllvm:-disable-auto-upgrade-debug-info", ] } else { ldflags += [ "-flto=thin" ] @@ -664,31 +701,45 @@ # TODO(thakis): Check if '=0' (that is, number of cores, instead # of "all" which means number of hardware threads) is faster. ldflags += [ "-Wl,--thinlto-jobs=all" ] + if (is_apple) { + ldflags += [ + "-Wl,-cache_path_lto," + + rebase_path("$root_out_dir/thinlto-cache", root_build_dir), + "-Wcrl,object_path_lto", + ] + } else { + ldflags += + [ "-Wl,--thinlto-cache-dir=" + + rebase_path("$root_out_dir/thinlto-cache", root_build_dir) ] + } - ldflags += [ - "-Wl,--thinlto-cache-dir=" + - rebase_path("$root_out_dir/thinlto-cache", root_build_dir), - "-Wl,--thinlto-cache-policy,$cache_policy", - ] + ldflags += [ "-Wl,--thinlto-cache-policy=$cache_policy" ] - if (is_chromeos_ash) { - # Not much performance difference was noted between the default (100) - # and these. ARM was originally set lower than x86 to keep the size + if (is_chromeos) { + # ARM was originally set lower than x86 to keep the size # bloat of ThinLTO to <10%, but that's potentially no longer true. # FIXME(inglorion): maybe tune these? - if (target_cpu == "arm" || target_cpu == "arm64") { - import_instr_limit = 20 - } else { - import_instr_limit = 30 - } + # TODO(b/271459198): Revert limit on amd64 to 30 when fixed. + import_instr_limit = 20 + } else if (is_android) { + # TODO(crbug.com/1308318): Investigate if we can get the > 6% perf win + # of import_instr_limit 30 with a binary size hit smaller than ~2 MiB. + import_instr_limit = 5 } ldflags += [ "-Wl,-mllvm,-import-instr-limit=$import_instr_limit" ] + + if (!is_chromeos) { + # TODO(https://crbug.com/972449): turn on for ChromeOS when that + # toolchain has this flag. + # We only use one version of LLVM within a build so there's no need to + # upgrade debug info, which can be expensive since it runs the verifier. + ldflags += [ "-Wl,-mllvm,-disable-auto-upgrade-debug-info" ] + } } - # Whole-program devirtualization increases android libchrome.so size - # by ~100kb on arm32 and reduces it by ~108kb on arm64 instead. - # Tracked by llvm bug: https://bugs.llvm.org/show_bug.cgi?id=48245 + # TODO(https://crbug.com/1211155): investigate why this isn't effective on + # arm32. if (!is_android || current_cpu == "arm64") { cflags += [ "-fwhole-program-vtables" ] if (!is_win) { @@ -724,6 +775,16 @@ ldflags += [ "-Wl,--no-rosegment" ] } + # TODO(crbug.com/1374347): Cleanup undefined symbol errors caught by + # --no-undefined-version. + if (use_lld && !is_win && !is_mac && !is_ios) { + ldflags += [ "-Wl,--undefined-version" ] + } + + if (use_lld && is_apple) { + ldflags += [ "-Wl,--strict-auto-link" ] + } + # LLD does call-graph-sorted binary layout by default when profile data is # present. On Android this increases binary size due to more thinks for long # jumps. Turn it off by default and enable selectively for targets where it's @@ -731,32 +792,63 @@ if (use_lld && !enable_call_graph_profile_sort) { if (is_win) { ldflags += [ "/call-graph-profile-sort:no" ] - } else if (!is_apple) { - # TODO(thakis): Once LLD's Mach-O port basically works, implement call - # graph profile sorting for it, add an opt-out flag, and pass it here. + } else { ldflags += [ "-Wl,--no-call-graph-profile-sort" ] } } if (is_clang && !is_nacl && show_includes) { - assert(!is_win, "show_includes is not supported on Windows") - cflags += [ - "-H", - "-Xclang", - "-show-skipped-includes", - ] + if (is_win) { + # TODO(crbug.com/1223741): Goma mixes the -H and /showIncludes output. + assert(!use_goma, "show_includes on Windows is not reliable with goma") + cflags += [ + "/clang:-H", + "/clang:-fshow-skipped-includes", + ] + } else { + cflags += [ + "-H", + "-fshow-skipped-includes", + ] + } } # This flag enforces that member pointer base types are complete. It helps # prevent us from running into problems in the Microsoft C++ ABI (see # https://crbug.com/847724). - # TODO(crbug/1052397): Remove is_chromeos_lacros once lacros-chrome switches - # to target_os="chromeos". - if (is_clang && !is_nacl && target_os != "chromeos" && !use_xcode_clang && - !is_chromeos_lacros && (is_win || use_custom_libcxx)) { + if (is_clang && !is_nacl && target_os != "chromeos" && + (is_win || use_custom_libcxx)) { cflags += [ "-fcomplete-member-pointers" ] } + # Use DWARF simple template names, with the following exceptions: + # + # * Windows is not supported as it doesn't use DWARF. + # * Apple platforms (e.g. MacOS, iPhone, iPad) aren't supported because xcode + # lldb doesn't have the needed changes yet. + # TODO(crbug.com/1379070): Remove if the upstream default ever changes. + if (is_clang && !is_nacl && !is_win && !is_apple) { + cflags_cc += [ "-gsimple-template-names" ] + } + + # MLGO specific flags. These flags enable an ML-based inliner trained on + # Chrome on Android (arm32) with ThinLTO enabled, optimizing for size. + # The "release" ML model is embedded into clang as part of its build. + # Currently, the ML inliner is only enabled when targeting Android due to: + # a) Android is where size matters the most. + # b) MLGO presently has the limitation of only being able to embed one model + # at a time; It is unclear if the embedded model is beneficial for + # non-Android targets. + # MLGO is only officially supported on linux. + if (use_ml_inliner && is_a_target_toolchain) { + assert( + is_android && host_os == "linux", + "MLGO is currently only supported for targeting Android on a linux host") + if (use_thin_lto) { + ldflags += [ "-Wl,-mllvm,-enable-ml-inliner=release" ] + } + } + # Pass the same C/C++ flags to the objective C/C++ compiler. cflags_objc += cflags_c cflags_objcc += cflags_cc @@ -768,6 +860,69 @@ asmflags += cflags asmflags += cflags_c } + + # Rust compiler flags setup. + # --------------------------- + rustflags = [ + # Overflow checks are optional in Rust, but even if switched + # off they do not cause undefined behavior (the overflowing + # behavior is defined). Because containers are bounds-checked + # in safe Rust, they also can't provoke buffer overflows. + # As such these checks may be less important in Rust than C++. + # But in (simplistic) testing they have negligible performance + # overhead, and this helps to provide consistent behavior + # between different configurations, so we'll keep them on until + # we discover a reason to turn them off. + "-Coverflow-checks=on", + + # By default Rust passes `-nodefaultlibs` to the linker, however this + # conflicts with our `--unwind=none` flag for Android dylibs, as the latter + # is then unused and produces a warning/error. So this removes the + # `-nodefaultlibs` from the linker invocation from Rust, which would be used + # to compile dylibs on Android, such as for constructing unit test APKs. + "-Cdefault-linker-libraries", + + # Require `unsafe` blocks even in `unsafe` fns. This is intended to become + # an error by default eventually; see + # https://github.com/rust-lang/rust/issues/71668 + "-Dunsafe_op_in_unsafe_fn", + + # To make Rust .d files compatible with ninja + "-Zdep-info-omit-d-target", + + # If a macro panics during compilation, show which macro and where it is + # defined. + "-Zmacro-backtrace", + + # For deterministic builds, keep the local machine's current working + # directory from appearing in build outputs. + "-Zremap-cwd-prefix=.", + ] + if (rust_abi_target != "") { + rustflags += [ "--target=$rust_abi_target" ] + } + if (!use_thin_lto) { + # Don't include bitcode if it won't be used. + rustflags += [ "-Cembed-bitcode=no" ] + } + if (is_official_build) { + rustflags += [ "-Ccodegen-units=1" ] + } +} + +# Defers LTO optimization to the linker, for use when: +# * Having the C++ toolchain do the linking against Rust staticlibs, and it +# will be using LTO. +# * Having Rust toolchain invoke the linker, and you're linking Rust and C++ +# together, so this defers LTO to the linker. +# +# Otherwise, Rust does LTO during compilation. +# +# https://doc.rust-lang.org/rustc/linker-plugin-lto.html +config("rust_defer_lto_to_linker") { + if (!is_debug && use_thin_lto && is_a_target_toolchain) { + rustflags = [ "-Clinker-plugin-lto" ] + } } # The BUILDCONFIG file sets this config on targets by default, which means when @@ -781,6 +936,8 @@ } else { ldflags = [ "-Wl,--lto-O" + lto_opt_level ] } + + rustflags = [ "-Clinker-plugin-lto=yes" ] } } @@ -805,6 +962,8 @@ } else { ldflags = [ "-Wl,--lto-O" + lto_opt_level ] } + + rustflags = [ "-Clinker-plugin-lto=yes" ] } } @@ -818,13 +977,23 @@ ldflags = [] defines = [] + configs = [] + if (is_chromeos) { + configs += [ "//build/config/chromeos:compiler_cpu_abi" ] + } + + # TODO(https://crbug.com/1383873): Remove this once figured out. + if (is_apple && current_cpu == "arm64") { + cflags += [ "-fno-global-isel" ] + ldflags += [ "-fno-global-isel" ] + } + if ((is_posix && !is_apple) || is_fuchsia) { # CPU architecture. We may or may not be doing a cross compile now, so for # simplicity we always explicitly set the architecture. if (current_cpu == "x64") { cflags += [ "-m64", - "-march=$x64_arch", "-msse3", ] ldflags += [ "-m64" ] @@ -838,7 +1007,8 @@ ] } } else if (current_cpu == "arm") { - if (is_clang && !is_android && !is_nacl) { + if (is_clang && !is_android && !is_nacl && + !(is_chromeos_lacros && is_chromeos_device)) { cflags += [ "--target=arm-linux-gnueabihf" ] ldflags += [ "--target=arm-linux-gnueabihf" ] } @@ -852,7 +1022,8 @@ cflags += [ "-mtune=$arm_tune" ] } } else if (current_cpu == "arm64") { - if (is_clang && !is_android && !is_nacl && !is_fuchsia) { + if (is_clang && !is_android && !is_nacl && !is_fuchsia && + !(is_chromeos_lacros && is_chromeos_device)) { cflags += [ "--target=aarch64-linux-gnu" ] ldflags += [ "--target=aarch64-linux-gnu" ] } @@ -1088,33 +1259,6 @@ ] ldflags += [ "-mips64r2" ] } - } else if (current_cpu == "pnacl" && is_nacl_nonsfi) { - if (target_cpu == "x86" || target_cpu == "x64") { - cflags += [ - "-arch", - "x86-32-nonsfi", - "--pnacl-bias=x86-32-nonsfi", - "--target=i686-unknown-nacl", - ] - ldflags += [ - "-arch", - "x86-32-nonsfi", - "--target=i686-unknown-nacl", - ] - } else if (target_cpu == "arm") { - cflags += [ - "-arch", - "arm-nonsfi", - "-mfloat-abi=hard", - "--pnacl-bias=arm-nonsfi", - "--target=armv7-unknown-nacl-gnueabihf", - ] - ldflags += [ - "-arch", - "arm-nonsfi", - "--target=armv7-unknown-nacl-gnueabihf", - ] - } } else if (current_cpu == "ppc64") { if (current_os == "aix") { cflags += [ "-maix64" ] @@ -1123,6 +1267,21 @@ cflags += [ "-m64" ] ldflags += [ "-m64" ] } + } else if (current_cpu == "riscv64") { + if (is_clang) { + cflags += [ "--target=riscv64-linux-gnu" ] + ldflags += [ "--target=riscv64-linux-gnu" ] + } + cflags += [ "-mabi=lp64d" ] + } else if (current_cpu == "loong64") { + if (is_clang) { + cflags += [ "--target=loongarch64-linux-gnu" ] + ldflags += [ "--target=loongarch64-linux-gnu" ] + } + cflags += [ + "-mabi=lp64d", + "-mcmodel=medium", + ] } else if (current_cpu == "s390x") { cflags += [ "-m64" ] ldflags += [ "-m64" ] @@ -1144,9 +1303,10 @@ configs += [ "//build/config/nacl:compiler_codegen" ] } - if (current_cpu == "arm64" && is_android) { - # On arm64 disable outlining for Android. See crbug.com/931297 for more - # information. + if (current_cpu == "arm64" && !is_win && is_clang) { + # Disable outlining everywhere on arm64 except Win. For more information see + # crbug.com/931297 for Android and crbug.com/1410297 for iOS. + # TODO(crbug.com/1411363): Enable this on Windows if possible. cflags += [ "-mno-outline" ] # This can be removed once https://bugs.llvm.org/show_bug.cgi?id=40348 @@ -1168,6 +1328,7 @@ config("compiler_deterministic") { cflags = [] ldflags = [] + swiftflags = [] # Eliminate build metadata (__DATE__, __TIME__ and __TIMESTAMP__) for # deterministic build. See https://crbug.com/314403 @@ -1196,12 +1357,20 @@ # different build directory like "out/feature_a" and "out/feature_b" if # we build same files with same compile flag. # Other paths are already given in relative, no need to normalize them. - cflags += [ - "-Xclang", - "-fdebug-compilation-dir", - "-Xclang", - ".", - ] + if (is_nacl) { + # TODO(https://crbug.com/1231236): Use -ffile-compilation-dir= here. + cflags += [ + "-Xclang", + "-fdebug-compilation-dir", + "-Xclang", + ".", + ] + } else { + # -ffile-compilation-dir is an alias for both -fdebug-compilation-dir= + # and -fcoverage-compilation-dir=. + cflags += [ "-ffile-compilation-dir=." ] + swiftflags += [ "-file-compilation-dir=." ] + } if (!is_win) { # We don't use clang -cc1as on Windows (yet? https://crbug.com/762167) asmflags = [ "-Wa,-fdebug-compilation-dir,." ] @@ -1224,8 +1393,18 @@ # Tells the compiler not to use absolute paths when passing the default # paths to the tools it invokes. We don't want this because we don't # really need it and it can mess up the goma cache entries. - if (is_clang && !is_nacl) { + if (is_clang && (!is_nacl || is_nacl_saigo)) { cflags += [ "-no-canonical-prefixes" ] + + # Same for links: Let the compiler driver invoke the linker + # with a relative path and pass relative paths to built-in + # libraries. Not needed on Windows because we call the linker + # directly there, not through the compiler driver. + # We don't link on goma, so this change is just for cleaner + # internal linker invocations, for people who work on the build. + if (!is_win) { + ldflags += [ "-no-canonical-prefixes" ] + } } } @@ -1251,6 +1430,18 @@ } } +config("rustc_revision") { + if (rustc_revision != "") { + # Similar to the above config, this is here so that all files get recompiled + # after a rustc roll. Nothing should ever read this cfg. This will not be + # set if a custom toolchain is used. + rustflags = [ + "--cfg", + "cr_rustc_revision=\"$rustc_revision\"", + ] + } +} + config("compiler_arm_fpu") { if (current_cpu == "arm" && !is_ios && !is_nacl) { cflags = [ "-mfpu=$arm_fpu" ] @@ -1269,7 +1460,7 @@ } config("compiler_arm") { - if (current_cpu == "arm" && (is_chromeos_ash || is_chromeos_lacros)) { + if (current_cpu == "arm" && is_chromeos) { # arm is normally the default mode for clang, but on chromeos a wrapper # is used to pass -mthumb, and therefor change the default. cflags = [ "-marm" ] @@ -1310,6 +1501,9 @@ configs += [ "//build/config/win:runtime_library" ] } else if (is_linux || is_chromeos) { configs += [ "//build/config/linux:runtime_library" ] + if (is_chromeos) { + configs += [ "//build/config/chromeos:runtime_library" ] + } } else if (is_ios) { configs += [ "//build/config/ios:runtime_library" ] } else if (is_mac) { @@ -1339,160 +1533,25 @@ cflags += [ "/WX" ] } if (fatal_linker_warnings) { + arflags = [ "/WX" ] ldflags = [ "/WX" ] } + defines = [ + # Without this, Windows headers warn that functions like wcsnicmp + # should be spelled _wcsnicmp. But all other platforms keep spelling + # it wcsnicmp, making this warning unhelpful. We don't want it. + "_CRT_NONSTDC_NO_WARNINGS", - cflags += [ - # Warnings permanently disabled: - - # C4091: 'typedef ': ignored on left of 'X' when no variable is - # declared. - # This happens in a number of Windows headers. Dumb. - "/wd4091", - - # C4127: conditional expression is constant - # This warning can in theory catch dead code and other problems, but - # triggers in far too many desirable cases where the conditional - # expression is either set by macros or corresponds some legitimate - # compile-time constant expression (due to constant template args, - # conditionals comparing the sizes of different types, etc.). Some of - # these can be worked around, but it's not worth it. - "/wd4127", - - # C4251: 'identifier' : class 'type' needs to have dll-interface to be - # used by clients of class 'type2' - # This is necessary for the shared library build. - "/wd4251", - - # C4275: non dll-interface class used as base for dll-interface class - # This points out a potential (but rare) problem with referencing static - # fields of a non-exported base, through the base's non-exported inline - # functions, or directly. The warning is subtle enough that people just - # suppressed it when they saw it, so it's not worth it. - "/wd4275", - - # C4312 is a VS 2015 64-bit warning for integer to larger pointer. - # TODO(brucedawson): fix warnings, crbug.com/554200 - "/wd4312", - - # C4324 warns when padding is added to fulfill alignas requirements, - # but can trigger in benign cases that are difficult to individually - # suppress. - "/wd4324", - - # C4351: new behavior: elements of array 'array' will be default - # initialized - # This is a silly "warning" that basically just alerts you that the - # compiler is going to actually follow the language spec like it's - # supposed to, instead of not following it like old buggy versions did. - # There's absolutely no reason to turn this on. - "/wd4351", - - # C4355: 'this': used in base member initializer list - # It's commonly useful to pass |this| to objects in a class' initializer - # list. While this warning can catch real bugs, most of the time the - # constructors in question don't attempt to call methods on the passed-in - # pointer (until later), and annotating every legit usage of this is - # simply more hassle than the warning is worth. - "/wd4355", - - # C4503: 'identifier': decorated name length exceeded, name was - # truncated - # This only means that some long error messages might have truncated - # identifiers in the presence of lots of templates. It has no effect on - # program correctness and there's no real reason to waste time trying to - # prevent it. - "/wd4503", - - # Warning C4589 says: "Constructor of abstract class ignores - # initializer for virtual base class." Disable this warning because it - # is flaky in VS 2015 RTM. It triggers on compiler generated - # copy-constructors in some cases. - "/wd4589", - - # C4611: interaction between 'function' and C++ object destruction is - # non-portable - # This warning is unavoidable when using e.g. setjmp/longjmp. MSDN - # suggests using exceptions instead of setjmp/longjmp for C++, but - # Chromium code compiles without exception support. We therefore have to - # use setjmp/longjmp for e.g. JPEG decode error handling, which means we - # have to turn off this warning (and be careful about how object - # destruction happens in such cases). - "/wd4611", - - # Warnings to evaluate and possibly fix/reenable later: - - "/wd4100", # Unreferenced formal function parameter. - "/wd4121", # Alignment of a member was sensitive to packing. - "/wd4244", # Conversion: possible loss of data. - "/wd4505", # Unreferenced local function has been removed. - "/wd4510", # Default constructor could not be generated. - "/wd4512", # Assignment operator could not be generated. - "/wd4610", # Class can never be instantiated, constructor required. - "/wd4838", # Narrowing conversion. Doesn't seem to be very useful. - "/wd4995", # 'X': name was marked as #pragma deprecated - "/wd4996", # Deprecated function warning. - - # These are variable shadowing warnings that are new in VS2015. We - # should work through these at some point -- they may be removed from - # the RTM release in the /W4 set. - "/wd4456", - "/wd4457", - "/wd4458", - "/wd4459", - - # All of our compilers support the extensions below. - "/wd4200", # nonstandard extension used: zero-sized array in struct/union - "/wd4201", # nonstandard extension used: nameless struct/union - "/wd4204", # nonstandard extension used : non-constant aggregate - # initializer - - "/wd4221", # nonstandard extension used : 'identifier' : cannot be - # initialized using address of automatic variable - - # http://crbug.com/588506 - Conversion suppressions waiting on Clang - # -Wconversion. - "/wd4245", # 'conversion' : conversion from 'type1' to 'type2', - # signed/unsigned mismatch - - "/wd4267", # 'var' : conversion from 'size_t' to 'type', possible loss of - # data - - "/wd4305", # 'identifier' : truncation from 'type1' to 'type2' - "/wd4389", # 'operator' : signed/unsigned mismatch - - "/wd4702", # unreachable code - - # http://crbug.com/848979 - MSVC is more conservative than Clang with - # regards to variables initialized and consumed in different branches. - "/wd4701", # Potentially uninitialized local variable 'name' used - "/wd4703", # Potentially uninitialized local pointer variable 'name' used - - # http://crbug.com/848979 - Remaining Clang permitted warnings. - "/wd4661", # 'identifier' : no suitable definition provided for explicit - # template instantiation request - - "/wd4706", # assignment within conditional expression - # MSVC is stricter and requires a boolean expression. - - "/wd4715", # 'function' : not all control paths return a value' - # MSVC does not analyze switch (enum) for completeness. + # TODO(thakis): winsock wants us to use getaddrinfo instead of + # gethostbyname. Fires mostly in non-Chromium code. We probably + # want to remove this define eventually. + "_WINSOCK_DEPRECATED_NO_WARNINGS", ] - - cflags_cc += [ - # Allow "noexcept" annotations even though we compile with exceptions - # disabled. - "/wd4577", - ] - - if (current_cpu == "x86") { - cflags += [ - # VC++ 2015 changes 32-bit size_t truncation warnings from 4244 to - # 4267. Example: short TruncTest(size_t x) { return x; } - # Since we disable 4244 we need to disable 4267 during migration. - # TODO(jschuh): crbug.com/167187 fix size_t to int truncations. - "/wd4267", - ] + if (!is_clang) { + # TODO(thakis): Remove this once + # https://swiftshader-review.googlesource.com/c/SwiftShader/+/57968 has + # rolled into angle. + cflags += [ "/wd4244" ] } } else { if (is_apple && !is_nacl) { @@ -1509,7 +1568,7 @@ # Suppress warnings about ABI changes on ARM (Clang doesn't give this # warning). - if (current_cpu == "arm" && !is_clang) { + if (!is_starboard && current_cpu == "arm" && !is_clang) { cflags += [ "-Wno-psabi" ] } @@ -1553,18 +1612,20 @@ "-Wno-missing-field-initializers", # "struct foo f = {0};" "-Wno-unused-parameter", # Unused function parameters. ] + + if (!is_starboard && (!is_nacl || is_nacl_saigo)) { + cflags += [ + # An ABI compat warning we don't care about, https://crbug.com/1102157 + # TODO(thakis): Push this to the (few) targets that need it, + # instead of having a global flag. + "-Wno-psabi", + ] + } } if (is_clang) { cflags += [ - # TODO(thakis): Consider -Wloop-analysis (turns on - # -Wrange-loop-analysis too). - - # This warns on using ints as initializers for floats in - # initializer lists (e.g. |int a = f(); CGSize s = { a, a };|), - # which happens in several places in chrome code. Not sure if - # this is worth fixing. - "-Wno-c++11-narrowing", + "-Wloop-analysis", # TODO(thakis): This used to be implied by -Wno-unused-function, # which we no longer use. Check if it makes sense to remove @@ -1572,67 +1633,91 @@ "-Wno-unneeded-internal-declaration", ] - # use_xcode_clang only refers to the iOS toolchain, host binaries use - # chromium's clang always. - if (!is_nacl) { + if (use_cobalt_customizations) { cflags += [ - # TODO(thakis): https://crbug.com/604888 - "-Wno-undefined-var-template", + "-Wno-extra-semi", + "-Wno-pessimizing-move", + "-Wno-shadow", + "-Wno-range-loop-bind-reference", + "-Wno-range-loop-construct", ] + } + if (!is_nacl || is_nacl_saigo) { if (is_win) { # TODO(thakis): https://crbug.com/617318 # Currently goma can not handle case sensitiveness for windows well. cflags += [ "-Wno-nonportable-include-path" ] } - if ((current_toolchain == host_toolchain || !use_xcode_clang) && - !using_old_compiler) { - # Flags NaCl (Clang 3.7) and Xcode 9.2 (Clang clang-900.0.39.2) do not - # recognize. - cflags += [ - # Ignore warnings about MSVC optimization pragmas. - # TODO(thakis): Only for no_chromium_code? http://crbug.com/912662 - "-Wno-ignored-pragma-optimize", - ] - if (!use_cobalt_customizations) { - cflags += [ - # An ABI compat warning we don't care about, https://crbug.com/1102157 - # TODO(thakis): Push this to the (few) targets that need it, - # instead of having a global flag. - "-Wno-psabi", + if (!use_cobalt_customizations) { + cflags += [ + "-Wenum-compare-conditional", - # TODO(https://crbug.com/989932): Evaluate and possibly enable. - "-Wno-implicit-int-float-conversion", - - # TODO(https://crbug.com/999886): Clean up, enable. - "-Wno-final-dtor-non-final-class", - - # TODO(https://crbug.com/1016945) Clean up, enable. - "-Wno-builtin-assume-aligned-alignment", - - # TODO(https://crbug.com/1028110): Evaluate and possible enable. - "-Wno-deprecated-copy", - - # TODO(https://crbug.com/1050281): Clean up, enable. - "-Wno-non-c-typedef-for-linkage", - ] - } - - cflags_c += [ - # TODO(https://crbug.com/995993): Clean up and enable. - "-Wno-implicit-fallthrough", - ] - - if (!use_cobalt_customizations) { - if (enable_wmax_tokens) { - cflags += [ "-Wmax-tokens" ] - } else { - # TODO(https://crbug.com/1049569): Remove after Clang 87b235db. - cflags += [ "-Wno-max-tokens" ] - } - } + # Ignore warnings about MSVC optimization pragmas. + # TODO(thakis): Only for no_chromium_code? http://crbug.com/912662 + "-Wno-ignored-pragma-optimize", + ] } + + if (!use_cobalt_customizations && !is_nacl) { + cflags += [ + # TODO(crbug.com/1343975) Evaluate and possibly enable. + "-Wno-deprecated-builtins", + + # TODO(crbug.com/1352183) Evaluate and possibly enable. + "-Wno-bitfield-constant-conversion", + + # TODO(crbug.com/1412713) Evaluate and possibly enable. + "-Wno-deprecated-this-capture", + ] + } + } + } +} + +# prevent_unsafe_narrowing ---------------------------------------------------- +# +# Warnings that prevent narrowing or comparisons of integer types that are +# likely to cause out-of-bound read/writes or Undefined Behaviour. In +# particular, size_t is used for memory sizes, allocation, indexing, and +# offsets. Using other integer types along with size_t produces risk of +# memory-safety bugs and thus security exploits. +# +# In order to prevent these bugs, allocation sizes were historically limited to +# sizes that can be represented within 31 bits of information, allowing `int` to +# be safely misused instead of `size_t` (https://crbug.com/169327). In order to +# support increasing the allocation limit we require strictly adherence to +# using the correct types, avoiding lossy conversions, and preventing overflow. +# To do so, enable this config and fix errors by converting types to be +# `size_t`, which is both large enough and unsigned, when dealing with memory +# sizes, allocations, indices, or offsets.In cases where type conversion is not +# possible or is superfluous, use base::strict_cast<> or base::checked_cast<> +# to convert to size_t as needed. +# See also: https://docs.google.com/document/d/1CTbQ-5cQjnjU8aCOtLiA7G6P0i5C6HpSDNlSNq6nl5E +# +# To enable in a GN target, use: +# configs += [ "//build/config/compiler:prevent_unsafe_narrowing" ] + +config("prevent_unsafe_narrowing") { + cflags = [] + if (is_clang) { + cflags += [ + "-Wimplicit-int-conversion", + "-Wsign-compare", + "-Wsign-conversion", + ] + if (!is_starboard) { + cflags += [ + "-Wshorten-64-to-32", + ] + } + if (!is_nacl) { + cflags += [ + # Avoid bugs of the form `if (size_t i = size; i >= 0; --i)` while + # fixing types to be sign-correct. + "-Wtautological-unsigned-zero-compare", + ] } } } @@ -1648,13 +1733,13 @@ # The platform should set warning flags. cflags = [] } else { - cflags = [ "/W4" ] # Warning level 4. - } - if (is_clang) { + cflags = [ "/W4" ] # Warning level 4. + # Opt in to additional [[nodiscard]] on standard library methods. defines = [ "_HAS_NODISCARD" ] } + } } else if (is_starboard) { # TODO(b/205790602): Revisit this code to be more compatible with platforms. defines = [ @@ -1677,6 +1762,15 @@ cflags += [ "-Wextra" ] } + if (treat_warnings_as_errors) { + # Turn rustc warnings into the "deny" lint level, which produce compiler + # errors. The equivalent of -Werror for clang/gcc. + # + # Note we apply the actual lint flags in config("compiler"). All warnings + # are suppressed in third-party crates. + rustflags = [ "-Dwarnings" ] + } + # In Chromium code, we define __STDC_foo_MACROS in order to get the # C99 macros on Mac and Linux. defines = [ @@ -1686,61 +1780,84 @@ if (!is_debug && !using_sanitizer && current_cpu != "s390x" && current_cpu != "s390" && current_cpu != "ppc64" && - current_cpu != "mips" && current_cpu != "mips64") { + current_cpu != "mips" && current_cpu != "mips64" && + current_cpu != "riscv64" && current_cpu != "loong64") { # Non-chromium code is not guaranteed to compile cleanly with # _FORTIFY_SOURCE. Also, fortified build may fail when optimizations are # disabled, so only do that for Release build. defines += [ "_FORTIFY_SOURCE=2" ] } - if (is_mac) { - cflags_objc = [ "-Wobjc-missing-property-synthesis" ] - cflags_objcc = [ "-Wobjc-missing-property-synthesis" ] + if (is_apple) { + cflags_objc = [ "-Wimplicit-retain-self" ] + cflags_objcc = [ "-Wimplicit-retain-self" ] } - if (is_ios) { - cflags_objc = [ "-Wimplicit-retain-self" ] - cflags_objcc = cflags_objc + if (is_mac) { + cflags_objc += [ "-Wobjc-missing-property-synthesis" ] + cflags_objcc += [ "-Wobjc-missing-property-synthesis" ] } } if (is_clang) { cflags += [ # Warn on missing break statements at the end of switch cases. - # For intentional fallthrough, use FALLTHROUGH; from - # base/compiler_specific.h + # For intentional fallthrough, use [[fallthrough]]. "-Wimplicit-fallthrough", ] + if (!is_starboard) { + cflags += [ + # Warn on unnecessary extra semicolons outside of function definitions. + "-Wextra-semi", + ] + } + + # Suppress warning in old //net. + if (is_starboard) { + cflags += [ + "-Wno-reorder-ctor", + "-Wno-unused-const-variable", + "-Wno-unused-variable", + "-Wno-unused-private-field", + "-Wno-missing-braces", + "-Wno-string-concatenation", + ] + } + + # Suppress warnings in old //base and //net. + if (is_starboard) { + cflags += [ + "-Wno-sign-compare", + "-Wno-shorten-64-to-32", + ] + } + # TODO(thakis): Enable this more often, https://crbug.com/346399 - # use_libfuzzer: https://crbug.com/1063180 - if (!is_starboard && !is_nacl && !use_libfuzzer) { - cflags += [ "-Wunreachable-code" ] + # use_fuzzing_engine_with_lpm: https://crbug.com/1063180 + if (!is_starboard && (!is_nacl || is_nacl_saigo) && !use_fuzzing_engine_with_lpm) { + cflags += [ "-Wunreachable-code-aggressive" ] } # Thread safety analysis is broken under nacl: https://crbug.com/982423. - if (!is_nacl) { + if (!is_nacl || is_nacl_saigo) { cflags += [ # Thread safety analysis. See base/thread_annotations.h and # https://clang.llvm.org/docs/ThreadSafetyAnalysis.html "-Wthread-safety", ] } - - # TODO(thakis): Enable this for more platforms, https://crbug.com/926235 - # ChromeOS: http://crbug.com/940863 - # Chromecast: http://crbug.com/942554 - has_dchecks = is_debug || dcheck_always_on - if (!has_dchecks && is_chromeos_ash && is_chrome_branded) { - # Temporarily disable -Wextra-semi for Chrome on Chrome OS. - } else if (is_chromecast && chromecast_branding != "public") { - # Temporarily disable -Wextra-semi for Chromecast. - } else if (!is_starboard) { - cflags += [ "-Wextra-semi" ] - } } - configs = [ ":default_warnings" ] + configs = [ + ":default_warnings", + ] + + if (!is_starboard) { + configs += [ + ":noshadowing", + ] + } } config("no_chromium_code") { @@ -1749,19 +1866,12 @@ defines = [] if (is_win) { + if (!is_starboard && is_clang) { + cflags += [ "/W3" ] # Warning level 3. + } cflags += [ - "/W3", # Warning level 3. "/wd4800", # Disable warning when forcing value to bool. "/wd4267", # TODO(jschuh): size_t to int. - "/wd4996", # Deprecated function warning. - ] - if (is_starboard) { - # The platform should set warning flags. - cflags -= [ "/W3" ] - } - defines += [ - "_CRT_NONSTDC_NO_WARNINGS", - "_CRT_NONSTDC_NO_DEPRECATE", ] } else { # GCC may emit unsuppressible warnings so don't add -Werror for no chromium @@ -1782,16 +1892,27 @@ # Lots of third-party libraries have unused variables. Instead of # suppressing them individually, we just blanket suppress them here. "-Wno-unused-variable", + + # Similarly, we're not going to fix all the C++11 narrowing issues in + # third-party libraries. + "-Wno-c++11-narrowing", ] if (!use_cobalt_customizations && !is_nacl && (current_toolchain == host_toolchain || !use_xcode_clang)) { cflags += [ + # Disabled for similar reasons as -Wunused-variable. + "-Wno-unused-but-set-variable", + # TODO(https://crbug.com/1202159): Clean up and enable. "-Wno-misleading-indentation", ] } } + # Suppress all warnings in third party, as Cargo does: + # https://doc.rust-lang.org/rustc/lints/levels.html#capping-lints + rustflags = [ "--cap-lints=allow" ] + configs = [ ":default_warnings" ] } @@ -1802,7 +1923,7 @@ config("noshadowing") { # This flag has to be disabled for nacl because the nacl compiler is too # strict about shadowing. - if (is_clang && !is_nacl) { + if (is_clang && (!is_nacl || is_nacl_saigo)) { cflags = [ "-Wshadow" ] } } @@ -1862,7 +1983,9 @@ config("thin_archive") { # The macOS and iOS default linker ld64 does not support reading thin # archives. - if ((is_posix && !is_nacl && (!is_apple || use_lld)) || is_fuchsia) { + # TODO(crbug.com/1221615): Enable on is_apple if use_lld once that no longer + # confuses lldb. + if ((is_posix && !is_nacl && !is_apple) || is_fuchsia) { arflags = [ "-T" ] } else if (is_win && use_lld) { arflags = [ "/llvmlibthin" ] @@ -2054,7 +2177,7 @@ "-Wl,-no_function_starts", ] } - } else if (current_os != "aix") { + } else if (current_os != "aix" && current_os != "zos") { # Non-Mac Posix flags. # Aix does not support these. @@ -2064,6 +2187,11 @@ "-fdata-sections", "-ffunction-sections", ] + if ((!is_nacl || is_nacl_saigo) && is_clang) { + # We don't care about unique section names, this makes object files a bit + # smaller. + common_optimize_on_cflags += [ "-fno-unique-section-names" ] + } common_optimize_on_ldflags += [ # Specifically tell the linker to perform optimizations. @@ -2076,7 +2204,7 @@ } config("default_stack_frames") { - if (is_posix || is_fuchsia) { + if (!is_win) { if (enable_frame_pointers) { cflags = [ "-fno-omit-frame-pointer" ] @@ -2108,21 +2236,36 @@ # Favor size over speed, /O1 must be before the common flags. # /O1 implies /Os and /GF. cflags = [ "/O1" ] + common_optimize_on_cflags + [ "/Oi" ] + rustflags = [ "-Copt-level=s" ] } else { # PGO requires all translation units to be compiled with /O2. The actual # optimization level will be decided based on the profiling data. cflags = [ "/O2" ] + common_optimize_on_cflags + [ "/Oi" ] + + # https://doc.rust-lang.org/rustc/profile-guided-optimization.html#usage + # suggests not using an explicit `-Copt-level` at all, and the default is + # to optimize for performance like `/O2` for clang. + rustflags = [] } - } else if (optimize_for_size && !is_nacl) { + } else if (optimize_for_size) { # Favor size over speed. - # TODO(crbug.com/718650): Fix -Os in PNaCl compiler and remove the is_nacl - # guard above. if (is_clang) { cflags = [ "-Oz" ] + common_optimize_on_cflags + + if (use_ml_inliner && is_a_target_toolchain) { + cflags += [ + "-mllvm", + "-enable-ml-inliner=release", + ] + } } else { cflags = [ "-Os" ] + common_optimize_on_cflags } - } else if (is_chromeos_ash) { + + # Like with `-Oz` on Clang, `-Copt-level=z` will also turn off loop + # vectorization. + rustflags = [ "-Copt-level=z" ] + } else if (is_chromeos) { # TODO(gbiv): This is partially favoring size over speed. CrOS exclusively # uses clang, and -Os in clang is more of a size-conscious -O2 than "size at # any cost" (AKA -Oz). It'd be nice to: @@ -2130,8 +2273,17 @@ # for size by default (so, also Windows) # - Investigate -Oz here, maybe just for ARM? cflags = [ "-Os" ] + common_optimize_on_cflags + + # Similar to clang, we optimize with `-Copt-level=s` to keep loop + # vectorization while otherwise optimizing for size. + rustflags = [ "-Copt-level=s" ] } else { cflags = [ "-O2" ] + common_optimize_on_cflags + + # The `-O3` for clang turns on extra optimizations compared to the standard + # `-O2`. But for rust, `-Copt-level=3` is the default and is thus reliable + # to use. + rustflags = [ "-Copt-level=3" ] } ldflags = common_optimize_on_ldflags } @@ -2199,6 +2351,7 @@ } else { cflags = [ "-O2" ] + common_optimize_on_cflags } + rustflags = [ "-Copt-level=3" ] } } @@ -2231,11 +2384,13 @@ } else { cflags = [ "-O3" ] + common_optimize_on_cflags } + rustflags = [ "-Copt-level=3" ] } } config("optimize_fuzzing") { cflags = [ "-O1" ] + common_optimize_on_cflags + rustflags = [ "-Copt-level=1" ] ldflags = common_optimize_on_ldflags visibility = [ ":default_optimization" ] } @@ -2270,14 +2425,17 @@ } else if (clang_use_default_sample_profile) { assert(build_with_chromium, "Our default profiles currently only apply to Chromium") - assert(is_android || is_chromeos_lacros || is_chromeos_ash || is_chromecast, + assert(is_android || is_chromeos || is_castos, "The current platform has no default profile") - if (is_android || is_chromecast) { + if (is_android || is_castos) { _clang_sample_profile = "//chrome/android/profiles/afdo.prof" } else { - assert(chromeos_afdo_platform == "atom" || - chromeos_afdo_platform == "bigcore", - "Only atom and bigcore are valid Chrome OS profiles.") + assert( + chromeos_afdo_platform == "atom" || + chromeos_afdo_platform == "bigcore" || + chromeos_afdo_platform == "arm" || + chromeos_afdo_platform == "arm-exp", + "Only 'atom', 'bigcore', 'arm' and 'arm-exp' are valid ChromeOS profiles.") _clang_sample_profile = "//chromeos/profiles/${chromeos_afdo_platform}.afdo.prof" } @@ -2295,8 +2453,7 @@ } # GCC and clang support a form of profile-guided optimization called AFDO. -# There are some targeted places that AFDO regresses (and an icky interaction -# between //base/allocator:tcmalloc and AFDO on GCC), so we provide a separate +# There are some targeted places that AFDO regresses, so we provide a separate # config to allow AFDO to be disabled per-target. config("afdo") { if (is_clang) { @@ -2313,6 +2470,9 @@ rebased_clang_sample_profile = rebase_path(_clang_sample_profile, root_build_dir) cflags += [ "-fprofile-sample-use=${rebased_clang_sample_profile}" ] + if (use_profi) { + cflags += [ "-fsample-profile-use-profi" ] + } inputs = [ _clang_sample_profile ] } } else if (auto_profile_path != "" && is_a_target_toolchain) { @@ -2354,9 +2514,17 @@ # Full symbols. config("symbols") { + rustflags = [] if (is_win) { if (is_clang) { - cflags = [ "/Z7" ] # Debug information in the .obj files. + cflags = [ + # Debug information in the .obj files. + "/Z7", + + # Disable putting the compiler command line into the debug info to + # prevent some types of non-determinism. + "-gno-codeview-command-line", + ] } else { cflags = [ "/Zi" ] # Produce PDB file, no edit and continue. } @@ -2370,9 +2538,6 @@ # All configs using /DEBUG should include this: configs = [ ":win_pdbaltpath" ] - - # TODO(crbug.com/1138553): Re-enable constructor homing on windows after - # libc++ fix is in. } else { cflags = [] if (is_mac && enable_dsyms) { @@ -2382,33 +2547,41 @@ # version 7 also produces debug data that is incompatible with Breakpad # dump_syms, so this is still required (https://crbug.com/622406). cflags += [ "-fno-standalone-debug" ] - } else if (is_mac && !use_dwarf5) { - # clang defaults to DWARF2 on macOS unless mac_deployment_target is - # at least 10.11. - # TODO(thakis): Remove this once mac_deployment_target is 10.11. - cflags += [ "-gdwarf-4" ] } - if (use_dwarf5 && !is_nacl) { - cflags += [ "-gdwarf-5" ] + # On aix -gdwarf causes linker failures due to thread_local variables. + if (!is_nacl && current_os != "aix") { + if (use_dwarf5) { + cflags += [ "-gdwarf-5" ] + rustflags += [ "-Zdwarf-version=5" ] + } else if (!is_apple) { + # Recent clang versions default to DWARF5 on Linux, and Android is about + # to switch. TODO: Adopt that in controlled way. + # Apple platforms still default to 4, so the flag is not needed there. + cflags += [ "-gdwarf-4" ] + rustflags += [ "-Zdwarf-version=4" ] + } } # The gcc-based nacl compilers don't support -fdebug-compilation-dir (see # elsewhere in this file), so they can't have build-dir-independent output. + # Moreover pnacl does not support newer flags such as -fdebug-prefix-map # Disable symbols for nacl object files to get deterministic, - # build-directory-independent output. pnacl and nacl-clang do support that - # flag, so we can use use -g1 for pnacl and nacl-clang compiles. - # gcc nacl is is_nacl && !is_clang, pnacl and nacl-clang are && is_clang. - if (!is_nacl || is_clang) { + # build-directory-independent output. + # Keeping -g2 for saigo as it's the only toolchain whose artifacts that are + # part of chromium release (other nacl toolchains are used only for tests). + if ((!is_nacl || is_nacl_saigo) && current_os != "zos") { cflags += [ "-g2" ] } - # TODO(https://crbug.com/1050118): Investigate missing debug info on mac. - if (is_clang && !is_nacl && !use_xcode_clang && !is_apple) { - cflags += [ - "-Xclang", - "-debug-info-kind=constructor", - ] + if (!is_nacl && is_clang && !is_tsan && !is_asan) { + # gcc generates dwarf-aranges by default on -g1 and -g2. On clang it has + # to be manually enabled. + # + # It is skipped in tsan and asan because enabling it causes some + # formatting changes in the output which would require fixing bunches + # of expectation regexps. + cflags += [ "-gdwarf-aranges" ] } if (is_apple) { @@ -2432,8 +2605,14 @@ # obj/native_client/src/trusted/service_runtime/sel_asm/nacl_switch_32.o: # DWARF info may be corrupt; offsets in a range list entry are in different # sections" there. Maybe just a bug in nacl_switch_32.S. - if (!is_apple && !is_nacl && current_cpu != "x86" && - (use_gold || use_lld)) { + _enable_gdb_index = + symbol_level == 2 && !is_apple && !is_nacl && current_cpu != "x86" && + current_os != "zos" && (use_gold || use_lld) && + # Disable on non-fission 32-bit Android because it pushes + # libcomponents_unittests over the 4gb size limit. + !(is_android && !use_debug_fission && current_cpu != "x64" && + current_cpu != "arm64") + if (_enable_gdb_index) { if (is_clang) { # This flag enables the GNU-format pubnames and pubtypes sections, # which lld needs in order to generate a correct GDB index. @@ -2444,6 +2623,34 @@ ldflags += [ "-Wl,--gdb-index" ] } } + + configs = [] + + # Compress debug on 32-bit ARM to stay under 4GB for ChromeOS + # https://b/243982712. + if (symbol_level == 2 && is_chromeos_device && !use_debug_fission && + !is_nacl && current_cpu == "arm") { + configs += [ "//build/config:compress_debug_sections" ] + } + + if (is_clang && (!is_nacl || is_nacl_saigo) && current_os != "zos") { + if (is_apple) { + # TODO(https://crbug.com/1050118): Investigate missing debug info on mac. + # Make sure we don't use constructor homing on mac. + cflags += [ + "-Xclang", + "-debug-info-kind=limited", + ] + } else { + # Use constructor homing for debug info. This option reduces debug info + # by emitting class type info only when constructors are emitted. + cflags += [ + "-Xclang", + "-fuse-ctor-homing", + ] + } + } + rustflags += [ "-g" ] } # Minimal symbols. @@ -2453,7 +2660,13 @@ if (is_win) { # Functions, files, and line tables only. cflags = [] - ldflags = [ "/DEBUG" ] + + if (is_clang && use_lld && use_ghash) { + cflags += [ "-gcodeview-ghash" ] + ldflags = [ "/DEBUG:GHASH" ] + } else { + ldflags = [ "/DEBUG" ] + } # All configs using /DEBUG should include this: configs = [ ":win_pdbaltpath" ] @@ -2471,6 +2684,11 @@ # at least 10.11. # TODO(thakis): Remove this once mac_deployment_target is 10.11. cflags += [ "-gdwarf-4" ] + } else if (!use_dwarf5 && !is_nacl && current_os != "aix") { + # On aix -gdwarf causes linker failures due to thread_local variables. + # Recent clang versions default to DWARF5 on Linux, and Android is about + # to switch. TODO: Adopt that in controlled way. + cflags += [ "-gdwarf-4" ] } if (use_dwarf5 && !is_nacl) { @@ -2479,30 +2697,34 @@ # The gcc-based nacl compilers don't support -fdebug-compilation-dir (see # elsewhere in this file), so they can't have build-dir-independent output. + # Moreover pnacl does not support newer flags such as -fdebug-prefix-map # Disable symbols for nacl object files to get deterministic, - # build-directory-independent output. pnacl and nacl-clang do support that - # flag, so we can use use -g1 for pnacl and nacl-clang compiles. - # gcc nacl is is_nacl && !is_clang, pnacl and nacl-clang are && is_clang. - if (!is_nacl || is_clang) { + # build-directory-independent output. + # Keeping -g1 for saigo as it's the only toolchain whose artifacts that are + # part of chromium release (other nacl toolchains are used only for tests). + if (!is_nacl || is_nacl_saigo) { cflags += [ "-g1" ] } + + if (!is_nacl && is_clang && !is_tsan && !is_asan) { + # See comment for -gdwarf-aranges in config("symbols"). + cflags += [ "-gdwarf-aranges" ] + } + ldflags = [] if (is_android && is_clang) { - # Android defaults to symbol_level=1 builds in production builds - # (https://crbug.com/648948), but clang, unlike gcc, doesn't emit - # DW_AT_linkage_name in -g1 builds. -fdebug-info-for-profiling enables - # that (and a bunch of other things we don't need), so that we get - # qualified names in stacks. + # Android defaults to symbol_level=1 builds, but clang, unlike gcc, + # doesn't emit DW_AT_linkage_name in -g1 builds. + # -fdebug-info-for-profiling enables that (and a bunch of other things we + # don't need), so that we get qualified names in stacks. # TODO(thakis): Consider making clang emit DW_AT_linkage_name in -g1 mode; # failing that consider doing this on non-Android too. cflags += [ "-fdebug-info-for-profiling" ] } - # Note: debug_fission is no-op with symbol_level=1 since all -g1 debug_info - # will stay in the executable. - asmflags = cflags } + rustflags = [ "-Cdebuginfo=1" ] } # This configuration contains function names only. That is, the compiler is @@ -2570,9 +2792,12 @@ if (is_android || (is_chromeos_ash && is_chromeos_device)) { # Use orderfile for linking Chrome on Android and Chrome OS. # This config enables using an orderfile for linking in LLD. - # TODO: Consider using call graph sort instead, at least on Android. config("chrome_orderfile_config") { - if (chrome_orderfile_path != "" && !enable_call_graph_profile_sort) { + # Don't try to use an orderfile with call graph sorting, except on Android, + # where we care about memory used by code, so we still want to mandate + # ordering. + if (chrome_orderfile_path != "" && + (is_android || !enable_call_graph_profile_sort)) { assert(use_lld) _rebased_orderfile = rebase_path(chrome_orderfile_path, root_build_dir) ldflags = [ @@ -2589,14 +2814,21 @@ config("default_init_stack_vars") { cflags = [] if (init_stack_vars && is_clang && !is_nacl && !using_sanitizer) { - cflags += [ "-ftrivial-auto-var-init=pattern" ] + if (init_stack_vars_zero) { + cflags += [ "-ftrivial-auto-var-init=zero" ] + } else { + cflags += [ "-ftrivial-auto-var-init=pattern" ] + } } } buildflag_header("compiler_buildflags") { header = "compiler_buildflags.h" - flags = [ "CLANG_PGO=$chrome_pgo_phase" ] + flags = [ + "CLANG_PGO=$chrome_pgo_phase", + "SYMBOL_LEVEL=$symbol_level", + ] } config("cet_shadow_stack") {
diff --git a/build/config/compiler/compiler.gni b/build/config/compiler/compiler.gni index c848dd7..aa5f37f 100644 --- a/build/config/compiler/compiler.gni +++ b/build/config/compiler/compiler.gni
@@ -1,12 +1,14 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/config/c++/c++.gni") import("//build/config/chrome_build.gni") import("//build/config/chromecast_build.gni") import("//build/config/chromeos/args.gni") import("//build/config/chromeos/ui_mode.gni") import("//build/config/compiler/pgo/pgo.gni") +import("//build/config/cronet/config.gni") import("//build/config/sanitizers/sanitizers.gni") import("//build/toolchain/cc_wrapper.gni") import("//build/toolchain/goma.gni") @@ -24,6 +26,27 @@ import("//build/config/apple/symbols.gni") } +if (is_ios) { + import("//build/config/ios/config.gni") +} + +declare_args() { + # Set to true to use lld, the LLVM linker. + # In late bring-up on macOS (see docs/mac_lld.md). + # Tentatively used on iOS. + # The default linker everywhere else. + use_lld = is_clang && current_os != "zos" + + if (use_cobalt_customizations && is_apple) { + use_lld = false + } + + # If true, optimize for size. + # Default to favoring speed over size for platforms not listed below. + optimize_for_size = + !is_high_end_android && (is_android || is_ios || is_castos) +} + declare_args() { # Default to warnings as errors for default workflow, where we catch # warnings with known toolchains. Allow overriding this e.g. for Chromium @@ -67,11 +90,15 @@ # Use it by default on official-optimized android and Chrome OS builds, but # not ARC or linux-chromeos since it's been seen to not play nicely with # Chrome's clang. crbug.com/1033839 + # Disabled in iOS cronet builds since build step cronet_static_complete + # wants to build a .a file consumable by external clients, and they won't + # have the same LLVM revisions as us, making bitcode useless to them. use_thin_lto = - is_cfi || - (is_official_build && chrome_pgo_phase != 1 && - (is_linux || is_win || (is_android && target_os != "chromeos") || - ((is_chromeos_ash || is_chromeos_lacros) && is_chromeos_device))) + is_cfi || (is_clang && is_official_build && chrome_pgo_phase != 1 && + (is_linux || is_win || is_mac || + (is_ios && use_lld && !is_cronet_build) || + (is_android && target_os != "chromeos") || + (is_chromeos && is_chromeos_device))) # If true, use Goma for ThinLTO code generation where applicable. use_goma_thin_lto = false @@ -95,7 +122,11 @@ # For unofficial (e.g. development) builds and non-Chrome branded (e.g. Cronet # which doesn't use Crashpad, crbug.com/479283) builds it's useful to be able # to unwind at runtime. - exclude_unwind_tables = is_official_build + # Include the unwind tables on Android even for official builds, as otherwise + # the crash dumps generated by Android's debuggerd are largely useless, and + # having this additional mechanism to understand issues is particularly helpful + # to WebView. + exclude_unwind_tables = is_official_build && !is_android # Where to redirect clang crash diagnoses clang_diagnostic_dir = @@ -105,40 +136,59 @@ # Technology (CET). If Windows version and hardware supports the feature and # it's enabled by OS then additional validation of return address will be # performed as mitigation against Return-oriented programming (ROP). - # https://chromium.googlesource.com/chromium/src/+/master/docs/design/sandbox.md#cet-shadow-stack + # https://chromium.googlesource.com/chromium/src/+/main/docs/design/sandbox.md#cet-shadow-stack enable_cet_shadow_stack = target_cpu == "x64" -} -assert(!is_cfi || use_thin_lto, "CFI requires ThinLTO") + # Set to true to enable using the ML inliner in LLVM. This currently only + # enables the ML inliner when targeting Android. + # Currently the ML inliner is only supported on linux hosts + use_ml_inliner = host_os == "linux" && is_android -# If true, optimize for size. Does not affect windows builds. -# Linux & Mac favor speed over size. -# TODO(brettw) it's weird that Mac and desktop Linux are different. We should -# explore favoring size over speed in this case as well. -optimize_for_size = is_android || is_chromecast || is_fuchsia || is_ios + # Set to true to use the android unwinder V2 implementation. + use_android_unwinder_v2 = true -declare_args() { # Whether we should consider the profile we're using to be accurate. Accurate # profiles have the benefit of (potentially substantial) binary size # reductions, by instructing the compiler to optimize cold and uncovered # functions heavily for size. This often comes at the cost of performance. sample_profile_is_accurate = optimize_for_size + + # Use offsets rather than pointers in vtables in order to reduce the number of + # relocations. This is safe to enable only when all C++ code is built with the + # flag set to the same value. + use_relative_vtables_abi = is_android && current_cpu == "arm64" && + use_custom_libcxx && !is_component_build +} + +# To try out this combination, delete this assert. +assert( + !use_relative_vtables_abi || !is_cfi, + "is_cfi=true is known to conflict with use_relative_vtables_abi=true.\n" + + "See https://bugs.chromium.org/p/chromium/issues/detail?id=1375035#c53") + +assert(!is_cfi || use_thin_lto, "CFI requires ThinLTO") +assert(!enable_profiling || !is_component_build, + "Cannot profile component builds (crbug.com/1199271).") + +if (use_thin_lto && is_debug) { + print("WARNING: ThinLTO (use_thin_lto=true) doesn't work with debug" + + " (is_debug=true) build.") } # Determine whether to enable or disable frame pointers, based on the platform # and build arguments. -# TODO(crbug.com/1052397): Consider changing is_chromeos_ash to is_chromeos after -# lacros-chrome switches to target_os="chromeos". -if (is_chromeos_ash || is_chromeos_lacros) { +if (is_chromeos) { # ChromeOS generally prefers frame pointers, to support CWP. # However, Clang does not currently generate usable frame pointers in ARM # 32-bit builds (https://bugs.llvm.org/show_bug.cgi?id=18505) so disable them # there to avoid the unnecessary overhead. enable_frame_pointers = current_cpu != "arm" -} else if (is_apple || is_linux || is_chromeos) { +} else if (is_apple || is_linux) { enable_frame_pointers = true } else if (is_win) { # 64-bit Windows ABI doesn't support frame pointers. + # NOTE: This setting is actually not used in the BUILD.gn for Windows, + # but it still reflects correctly that we don't emit frame pointers on x64. if (current_cpu == "x64") { enable_frame_pointers = false } else { @@ -158,10 +208,12 @@ # For caller-callee instrumentation version which needs frame pointers to # get the caller address. use_call_graph +} else if (is_fuchsia) { + # Fuchsia on arm64 could use shadow call stack for unwinding. + enable_frame_pointers = current_cpu != "arm64" } else { - # Explicitly ask for frame pointers, otherwise: - # * Stacks may be missing for sanitizer and profiling builds. - # * Debug tcmalloc can crash (crbug.com/636489). + # Explicitly ask for frame pointers, otherwise stacks may be missing for + # sanitizer and profiling builds. enable_frame_pointers = using_sanitizer || enable_profiling || is_debug } @@ -193,16 +245,8 @@ enable_arm_cfi_table = is_android && !is_component_build && current_cpu == "arm" declare_args() { - # Set to true to use lld, the LLVM linker. - # Not supported for macOS (see docs/mac_lld.md), and not functional at all for - # iOS. But used for mac cross-compile on linux (may not work properly). - # The default linker everywhere else. - use_lld = is_clang && (!is_apple || host_os == "linux") -} - -declare_args() { # Whether to use the gold linker from binutils instead of lld or bfd. - use_gold = !use_lld && !(is_chromecast && is_linux && + use_gold = !use_lld && !(is_castos && (current_cpu == "arm" || current_cpu == "mipsel")) && (((is_linux || is_chromeos_lacros) && (current_cpu == "x64" || current_cpu == "x86" || @@ -216,17 +260,14 @@ # results independent of the checkout and build directory names, which # in turn is important for goma compile hit rate. # Setting this to true may make it harder to debug binaries on Linux, see -# https://chromium.googlesource.com/chromium/src/+/master/docs/linux/debugging.md#Source-level-debug-with-fdebug_compilation_dir +# https://chromium.googlesource.com/chromium/src/+/main/docs/linux/debugging.md#Source-level-debug-with-fdebug_compilation_dir # It's not clear if the crash server will correctly handle dSYMs with relative # paths, so we disable this feature for official benefit. The main benefit is # deterministic builds to reduce compile times, so this is less relevant for # official builders. strip_absolute_paths_from_debug_symbols_default = - # TODO(crbug.com/1010267): remove '!use_clang_coverage', coverage build has - # dependency to absolute path of source files. - !use_clang_coverage && - (is_android || is_fuchsia || is_nacl || (is_win && use_lld) || is_linux || - is_chromeos || (is_apple && !enable_dsyms)) + is_android || is_fuchsia || is_nacl || (is_win && use_lld) || is_linux || + is_chromeos || (is_apple && !enable_dsyms) # If the platform uses stripped absolute paths by default, then we don't expose # it as a configuration option. If this is causing problems, please file a bug. @@ -251,8 +292,7 @@ assert(symbol_level >= -1 && symbol_level <= 2, "Invalid symbol_level") if (symbol_level == -1) { if (is_android && !is_component_build && !use_debug_fission) { - # Reduce symbol level when it will cause invalid elf files to be created - # (due to file size). https://crbug.com/648948. + # Prefer faster & smaller release builds. symbol_level = 1 } else if (is_chromeos_device) { # Use lower symbol level in Simple Chrome build for faster link time. @@ -273,7 +313,7 @@ symbol_level = 1 } else if ((!is_nacl && !is_linux && !is_chromeos && !is_fuchsia && current_os != "aix") || is_debug || is_official_build || - is_chromecast) { + is_castos || is_cast_android) { # Linux builds slower by having symbols as part of the target binary, # whereas Mac and Windows have them separate, so in Release Linux, default # them off, but keep them on for Official builds and Chromecast builds. @@ -293,18 +333,15 @@ # the build (like nacl) and we don't want to assert on those. # iOS does not support component builds so add an exception for this platform. if (forbid_non_component_debug_builds) { - assert(symbol_level != 2 || current_toolchain != default_toolchain || - is_component_build || !is_debug || is_ios, - "Can't do non-component debug builds at symbol_level=2") + assert( + symbol_level != 2 || current_toolchain != default_toolchain || + is_component_build || !is_debug || is_ios || use_debug_fission, + "Can't do non-component debug builds at symbol_level=2 without use_debug_fission=true") } -# Assert that the configuration isn't going to hit https://crbug.com/648948. -# An exception is made when target_os == "chromeos" as we only use the Android -# toolchain there to build relatively small binaries. -assert( - ignore_elf32_limitations || !is_android || target_os == "chromeos" || - is_component_build || symbol_level < 2 || use_debug_fission, - "Android 32-bit non-component builds without DWARF Fission cannot " + - "have symbol_level=2 due to 4GiB file size limit, see " + - "https://crbug.com/648948. " + "If you really want to try this out, " + - "set ignore_elf32_limitations=true.") +# TODO(crbug.com/1341436) For Windows, to assemble lzma_sdk's assembly files, +# ml64.exe needs to be utilized as llvm-ml cannot yet assemble it. Once llvm-ml +# is able to assemble lzma_sdk assembly files, remove this. +# LzmaDecOpt.asm only works on x64 and not x86. +# https://sourceforge.net/p/sevenzip/discussion/45797/thread/768932e9dd/?limit=25#0d6c +disable_llvm_ml = host_os == "win" && target_cpu == "x64" && !is_msan
diff --git a/build/config/compiler/pgo/BUILD.gn b/build/config/compiler/pgo/BUILD.gn index 3e8502e..86e76a4 100644 --- a/build/config/compiler/pgo/BUILD.gn +++ b/build/config/compiler/pgo/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -42,33 +42,68 @@ if (is_win) { if (target_cpu == "x64") { _pgo_target = "win64" - inputs = [ "//chrome/build/win64.pgo.txt" ] } else { _pgo_target = "win32" - inputs = [ "//chrome/build/win32.pgo.txt" ] } } else if (is_mac) { - _pgo_target = "mac" - inputs = [ "//chrome/build/mac.pgo.txt" ] - } else if (is_linux || is_chromeos_lacros) { + if (target_cpu == "arm64") { + _pgo_target = "mac-arm" + } else { + _pgo_target = "mac" + } + } else if (is_linux) { _pgo_target = "linux" - inputs = [ "//chrome/build/linux.pgo.txt" ] + } else if (is_chromeos_lacros) { + if (target_cpu == "arm") { + _pgo_target = "lacros-arm" + } else if (target_cpu == "arm64") { + _pgo_target = "lacros-arm64" + } else { + _pgo_target = "lacros64" + } + } else if (is_android) { + # Temporarily use mac-arm profile until Android native PGO support works. + # TODO(crbug.com/1308749): fix this. + _pgo_target = "mac-arm" + } else if (is_fuchsia) { + if (target_cpu == "arm64") { + _pgo_target = "mac-arm" + } else { + _pgo_target = "mac" + } } - if (pgo_data_path == "" && _pgo_target != "") { - pgo_data_path = rebase_path(exec_script("//tools/update_pgo_profiles.py", - [ - "--target", - _pgo_target, - "get_profile_path", - ], - "value"), - root_build_dir) + if (_pgo_target == "win64") { + inputs = [ "//chrome/build/win64.pgo.txt" ] + } else if (_pgo_target == "win32") { + inputs = [ "//chrome/build/win32.pgo.txt" ] + } else if (_pgo_target == "mac-arm") { + inputs = [ "//chrome/build/mac-arm.pgo.txt" ] + } else if (_pgo_target == "mac") { + inputs = [ "//chrome/build/mac.pgo.txt" ] + } else if (_pgo_target == "linux") { + inputs = [ "//chrome/build/linux.pgo.txt" ] + } else if (_pgo_target == "lacros64") { + inputs = [ "//chrome/build/lacros64.pgo.txt" ] + } else if (_pgo_target == "lacros-arm") { + inputs = [ "//chrome/build/lacros-arm.pgo.txt" ] + } else if (_pgo_target == "lacros-arm64") { + inputs = [ "//chrome/build/lacros-arm64.pgo.txt" ] + } + + if (_pgo_target != "" && pgo_data_path == "") { + pgo_data_path = exec_script("//tools/update_pgo_profiles.py", + [ + "--target", + _pgo_target, + "get_profile_path", + ], + "value") } assert(pgo_data_path != "", "Please set pgo_data_path to point at the profile data") cflags = [ - "-fprofile-instr-use=$pgo_data_path", + "-fprofile-use=" + rebase_path(pgo_data_path, root_build_dir), # It's possible to have some profile data legitimately missing, # and at least some profile data always ends up being considered
diff --git a/build/config/compiler/pgo/pgo.gni b/build/config/compiler/pgo/pgo.gni index c053eb5..9e9a0c5 100644 --- a/build/config/compiler/pgo/pgo.gni +++ b/build/config/compiler/pgo/pgo.gni
@@ -1,9 +1,11 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//build/config/chrome_build.gni") import("//build/config/chromecast_build.gni") import("//build/config/chromeos/ui_mode.gni") +import("//build/config/dcheck_always_on.gni") declare_args() { # Specify the current PGO phase. @@ -11,12 +13,19 @@ # 0 : Means that PGO is turned off. # 1 : Used during the PGI (instrumentation) phase. # 2 : Used during the PGO (optimization) phase. + # PGO profiles are generated from `dcheck_always_on = false` builds. Mixing + # those profiles with `dcheck_always_on = true` builds can cause the compiler + # to think some code is hotter than it actually is, potentially causing very + # bad compile times. chrome_pgo_phase = 0 - if (is_official_build && + if (!dcheck_always_on && is_official_build && # TODO(crbug.com/1052397): Remove chromeos_is_browser_only once # target_os switch for lacros-chrome is completed. - (is_win || is_mac || - (is_linux && !chromeos_is_browser_only && !is_chromecast))) { + # TODO(crbug.com/1336055): Update this now-outdated condition with regard + # to chromecast and determine whether chromeos_is_browser_only is + # obsolete. + (is_high_end_android || is_win || is_mac || is_fuchsia || + (is_linux && !is_castos && !chromeos_is_browser_only))) { chrome_pgo_phase = 2 }
diff --git a/build/config/compute_inputs_for_analyze.gni b/build/config/compute_inputs_for_analyze.gni index 050ab70..1e32294 100644 --- a/build/config/compute_inputs_for_analyze.gni +++ b/build/config/compute_inputs_for_analyze.gni
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/coverage/BUILD.gn b/build/config/coverage/BUILD.gn index fa0833e..59941c3 100644 --- a/build/config/coverage/BUILD.gn +++ b/build/config/coverage/BUILD.gn
@@ -1,8 +1,9 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/coverage/coverage.gni") +import("//build/config/rust.gni") config("default_coverage") { if (use_clang_coverage) { @@ -26,17 +27,17 @@ "-limited-coverage-experimental=true", ] + # Rust coverage is gated on using the Chromium-built Rust toolchain as it + # needs to have a compatible LLVM version with the C++ compiler and the LLVM + # tools that will be used to process the coverage output. This is because + # the coverage file format is not stable. + if (use_chromium_rust_toolchain) { + rustflags = [ "-Cinstrument-coverage" ] + } + if (is_linux || is_chromeos) { # TODO(crbug.com/1194301): Remove this flag. cflags += [ "-fno-use-cxa-atexit" ] } - - if (using_old_compiler) { - # These compiler flags aren't supported by the older clang compiler. - cflags -= [ - "-limited-coverage-experimental=true", - "-fno-use-cxa-atexit", - ] - } } }
diff --git a/build/config/coverage/OWNERS b/build/config/coverage/OWNERS index 0fc481f..7b0fe27 100644 --- a/build/config/coverage/OWNERS +++ b/build/config/coverage/OWNERS
@@ -1,3 +1 @@ -inferno@chromium.org -liaoyuke@chromium.org -ochang@chromium.org +pasthana@google.com
diff --git a/build/config/coverage/coverage.gni b/build/config/coverage/coverage.gni index 9586d8d..2e5b7ab 100644 --- a/build/config/coverage/coverage.gni +++ b/build/config/coverage/coverage.gni
@@ -1,8 +1,11 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/toolchain/toolchain.gni") +if (is_fuchsia) { + import("//third_party/fuchsia-sdk/sdk/build/component.gni") +} # There are two ways to enable code coverage instrumentation: # 1. When |use_clang_coverage| or |use_jacoco_coverage| is true and @@ -14,7 +17,11 @@ # input file or Java class files related to source files are instrumented. declare_args() { # Enable Clang's Source-based Code Coverage. - use_clang_coverage = false + if (is_fuchsia) { + use_clang_coverage = fuchsia_code_coverage + } else { + use_clang_coverage = false + } # Enables JaCoCo Java code coverage. use_jacoco_coverage = false
diff --git a/build/config/cronet/OWNERS b/build/config/cronet/OWNERS new file mode 100644 index 0000000..78c2d80 --- /dev/null +++ b/build/config/cronet/OWNERS
@@ -0,0 +1 @@ +file://components/cronet/OWNERS
diff --git a/build/config/cronet/config.gni b/build/config/cronet/config.gni new file mode 100644 index 0000000..1468ec1 --- /dev/null +++ b/build/config/cronet/config.gni
@@ -0,0 +1,10 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +declare_args() { + # Control whether cronet is built (this is usually set by the script + # components/cronet/tools/cr_cronet.py as cronet requires specific + # gn args to build correctly). + is_cronet_build = false +}
diff --git a/build/config/crypto.gni b/build/config/crypto.gni deleted file mode 100644 index dc33c5e..0000000 --- a/build/config/crypto.gni +++ /dev/null
@@ -1,15 +0,0 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This file declares build flags for the SSL library configuration. -# -# TODO(brettw) this should probably be moved to src/crypto or somewhere, and -# the global build dependency on it should be removed. -# -# PLEASE TRY TO AVOID ADDING FLAGS TO THIS FILE in cases where grit isn't -# required. See the declare_args block of BUILDCONFIG.gn for advice on how -# to set up feature flags. - -# True if NSS is used for certificate handling. -use_nss_certs = (is_linux || is_chromeos) && !is_starboard
diff --git a/build/config/dcheck_always_on.gni b/build/config/dcheck_always_on.gni index e7d6a79..26cb76c 100644 --- a/build/config/dcheck_always_on.gni +++ b/build/config/dcheck_always_on.gni
@@ -1,20 +1,41 @@ -# Copyright (c) 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +if (!use_cobalt_customizations) { +# TODO(crbug.com/1233050): Until the bug is resolved we need to include +# gclient_args for the definition of build_with_chromium and build_overrides +# for client overrides of that flag. The latter should go away. +import("//build/config/gclient_args.gni") +} +import("//build_overrides/build.gni") declare_args() { # Enables DCHECKs to be built-in, but to default to being non-fatal/log-only. - # DCHECKS can then be set as fatal/non-fatal via the DCheckIsFatal feature. + # DCHECKS can then be set as fatal/non-fatal via the "DcheckIsFatal" feature. # See https://bit.ly/dcheck-albatross for details on how this is used. dcheck_is_configurable = false } declare_args() { - # Set to true to enable dcheck in Release builds. - dcheck_always_on = dcheck_is_configurable + # Set to false to disable DCHECK in Release builds. This is enabled by default + # for non-official builds on the below platforms. + # This default only affects Chromium as indicated by build_with_chromium. + # Other clients typically set this to false. If another client wants to use + # the same default value as Chromium, we'd need to add a separate gclient + # variable to replace build_with_chromium here. + dcheck_always_on = + (build_with_chromium && !is_official_build) || dcheck_is_configurable } declare_args() { - # Set to false to disable EXPENSIVE_DCHECK()s. - enable_expensive_dchecks = is_debug || dcheck_always_on + # Set to false to disable EXPENSIVE_DCHECK()s or to true to enable them in + # official builds. These are generally used for really useful DCHECKs that are + # too expensive to be enabled in user-facing official+DCHECK builds. + enable_expensive_dchecks = + is_debug || (dcheck_always_on && !is_official_build) } + +assert(!dcheck_is_configurable || (dcheck_always_on || is_debug), + "dcheck_is_configurable only makes sense with DCHECKs enabled") +assert(!enable_expensive_dchecks || (dcheck_always_on || is_debug), + "enable_expensive_dchecks only makes sense with DCHECKs enabled")
diff --git a/build/config/devtools.gni b/build/config/devtools.gni new file mode 100644 index 0000000..4338e25 --- /dev/null +++ b/build/config/devtools.gni
@@ -0,0 +1,37 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/chrome_build.gni") +import("//build_overrides/build.gni") + +declare_args() { + if (build_with_chromium) { + # devtools_location is used in DevTools to resolve to the correct location + # for any script/file referenced in the DevTools build scripts. Since + # DevTools supports both a standalone build and build integration with + # Chromium, we need to differentiate between the two versions. + # devtools_location points to the Chromium version in both Chrome-branded + # and not Chrome-branded builds. devtools_root_location points to the root + # of the Chrome-branded version when is_chrome_branded is true and to the root + # of the Chromium version when is_chrome_branded is false. + # devtools_grd_location is the location of the GRD file listing all DevTools + # resources. + if (is_chrome_branded) { + devtools_root_location = "third_party/devtools-frontend-internal" + devtools_location = "$devtools_root_location/devtools-frontend/" + devtools_grd_location = + "$devtools_root_location/chrome_devtools_resources.grd" + } else { + devtools_root_location = "third_party/devtools-frontend/src" + devtools_location = "third_party/devtools-frontend/src/" + devtools_grd_location = + "$devtools_root_location/front_end/devtools_resources.grd" + } + } else { + # DevTools is building a standalone version + devtools_location = "" + devtools_root_location = "" + devtools_grd_location = "" + } +}
diff --git a/build/config/features.gni b/build/config/features.gni index 62bf4bc..852ac56 100644 --- a/build/config/features.gni +++ b/build/config/features.gni
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -23,14 +23,21 @@ # # Note: this flag is used by WebRTC which is DEPSed into Chrome. Moving it # out of //build will require using the build_overrides directory. - proprietary_codecs = is_chrome_branded || is_chromecast + # + # Do not add any other conditions to the following line. + # + # TODO(crbug.com/1314528): Remove chromecast-related conditions and force + # builds to explicitly specify this. + proprietary_codecs = is_chrome_branded || is_castos || is_cast_android # libudev usage. This currently only affects the content layer. - use_udev = (is_linux || is_chromeos) && !is_chromecast + use_udev = (is_linux && !is_castos) || is_chromeos - use_dbus = (is_linux || is_chromeos) && !is_chromecast + use_dbus = is_linux || is_chromeos - use_gio = is_linux && !is_chromecast + use_gio = is_linux && !is_castos + + use_blink = !is_ios } # # =============================================
diff --git a/build/config/freetype/BUILD.gn b/build/config/freetype/BUILD.gn index 76cb025..88a9c59 100644 --- a/build/config/freetype/BUILD.gn +++ b/build/config/freetype/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/freetype/freetype.gni b/build/config/freetype/freetype.gni index b4eced2..60aeb04 100644 --- a/build/config/freetype/freetype.gni +++ b/build/config/freetype/freetype.gni
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/fuchsia/BUILD.gn b/build/config/fuchsia/BUILD.gn index 88922a1..bbcd708 100644 --- a/build/config/fuchsia/BUILD.gn +++ b/build/config/fuchsia/BUILD.gn
@@ -1,11 +1,14 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/chromecast_build.gni") +import("//build/config/clang/clang.gni") +import("//build/config/fuchsia/generate_runner_scripts.gni") +import("//third_party/fuchsia-sdk/sdk/build/config/config.gni") assert(is_fuchsia) -assert(!is_posix) +assert(!is_posix, "Fuchsia is not POSIX.") config("compiler") { configs = [ "//third_party/fuchsia-sdk/sdk/build/config:compiler" ] @@ -14,17 +17,84 @@ # https://fuchsia.googlesource.com/zircon/+/master/system/private/zircon/stack.h#9), # but on other platforms it's much higher, so a variety of code assumes more # will be available. Raise to 8M which matches e.g. macOS. - ldflags = [ "-Wl,-z,stack-size=0x800000" ] + ldflags = [ + "-Wl,-z,stack-size=0x800000", + "-fexperimental-relative-c++-abi-vtables", + ] + cflags_cc = [ "-fexperimental-relative-c++-abi-vtables" ] +} - # Allow this in chromium-only builds, but do not allow this in Chromecast - # builds. - if (!is_chromecast) { - cflags_cc = [ "-fexperimental-relative-c++-abi-vtables" ] - ldflags += [ "-fexperimental-relative-c++-abi-vtables" ] +# Files required to run on Fuchsia on isolated swarming clients. +group("deployment_resources") { + data = [ + "//build/fuchsia/", + "//build/util/lib/", + "//third_party/fuchsia-sdk/sdk/.build-id/", + "//third_party/fuchsia-sdk/sdk/bin/fuchsia-common.sh", + "//third_party/fuchsia-sdk/sdk/meta/manifest.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/ffx", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/ffx-meta.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/fvm", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/fvm-meta.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/merkleroot", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/merkleroot-meta.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/pm", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/pm-meta.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/symbolizer", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/symbolizer-meta.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/zbi", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/zbi-meta.json", + ] + + if (fuchsia_additional_boot_images == []) { + data += [ "${boot_image_root}" ] + } + + foreach(fuchsia_additional_boot_image, fuchsia_additional_boot_images) { + data += [ "${fuchsia_additional_boot_image}/" ] + } + + if (test_isolate_uses_emulator) { + data += [ + "//third_party/fuchsia-sdk/sdk/bin/device_launcher.version", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/fvdl", + ] + if (test_host_cpu == "x64") { + data += [ + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/aemu_internal", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/aemu_internal-meta.json", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/qemu_internal", + "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/qemu_internal-meta.json", + ] + } else if (test_host_cpu == "arm64") { + data += [ + "//third_party/qemu-${host_os}-${test_host_cpu}/", + + # TODO(https://crbug.com/1336776): remove when ffx has native support + # for starting emulator on arm64 host. + "//third_party/fuchsia-sdk/sdk/tools/x64/qemu_internal-meta.json", + ] + } } } -# Settings for executables. -config("executable_config") { - ldflags = [ "-pie" ] +# Copy the loader to place it at the expected path in the final package. +copy("sysroot_asan_libs") { + sources = + [ "${fuchsia_sdk}/arch/${target_cpu}/sysroot/dist/lib/asan/ld.so.1" ] + outputs = [ "${root_out_dir}/lib/asan/{{source_file_part}}" ] +} + +# Copy the loader to place it at the expected path in the final package. +copy("sysroot_asan_runtime_libs") { + sources = [ "$clang_base_path/lib/clang/$clang_version/lib/x86_64-unknown-fuchsia/libclang_rt.asan.so" ] + outputs = [ "${root_out_dir}/lib/{{source_file_part}}" ] +} + +# This adds the runtime deps for Fuchsia ASAN builds. +group("asan_runtime_library") { + data_deps = [ + ":sysroot_asan_libs", + ":sysroot_asan_runtime_libs", + ] }
diff --git a/build/config/fuchsia/DIR_METADATA b/build/config/fuchsia/DIR_METADATA index 6d8f079..210aa6a 100644 --- a/build/config/fuchsia/DIR_METADATA +++ b/build/config/fuchsia/DIR_METADATA
@@ -1,7 +1 @@ -monorail { - component: "Fuchsia" -} - -team_email: "cr-fuchsia@chromium.org" - -os: FUCHSIA +mixins: "//build/fuchsia/COMMON_METADATA"
diff --git a/build/config/fuchsia/OWNERS b/build/config/fuchsia/OWNERS index 3a1056b..565fda1 100644 --- a/build/config/fuchsia/OWNERS +++ b/build/config/fuchsia/OWNERS
@@ -1,4 +1,5 @@ file://build/fuchsia/OWNERS -per-file *.cmx=set noparent -per-file *.cmx=file://fuchsia/SECURITY_OWNERS +chonggu@google.com +rohpavone@chromium.org +zijiehe@google.com
diff --git a/build/config/fuchsia/add_DebugData_service.test-cmx b/build/config/fuchsia/add_DebugData_service.test-cmx deleted file mode 100644 index 33fb6b0..0000000 --- a/build/config/fuchsia/add_DebugData_service.test-cmx +++ /dev/null
@@ -1,7 +0,0 @@ -{ - "sandbox": { - "services": [ - "fuchsia.debugdata.DebugData" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/build_cmx_from_fragment.py b/build/config/fuchsia/build_cmx_from_fragment.py deleted file mode 100644 index ac7e349..0000000 --- a/build/config/fuchsia/build_cmx_from_fragment.py +++ /dev/null
@@ -1,49 +0,0 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Creates a complete CMX (v1) component manifest, from a program name and - manifest fragment file.""" - -import argparse -import json -import sys - - -def BuildCmxFromFragment(output_file, fragment_file, program_binary): - """Reads a CMX fragment specifying e.g. features & sandbox, and a program - binary's filename, and writes out the full CMX. - - output_file: Build-relative filename at which to write the full CMX. - fragment_file: Build-relative filename of the CMX fragment to read from. - program_binary: Package-relative filename of the program binary. - """ - - with open(output_file, 'w') as component_manifest_file: - component_manifest = json.load(open(fragment_file, 'r')) - component_manifest.update({ - 'program': { - 'binary': program_binary - }, - }) - json.dump(component_manifest, component_manifest_file) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--cmx-fragment', - required=True, - help='Path to the CMX fragment to read from') - parser.add_argument( - '--cmx', required=True, help='Path to write the complete CMX file to') - parser.add_argument( - '--program', - required=True, - help='Package-relative path to the program binary') - args = parser.parse_args() - - return BuildCmxFromFragment(args.cmx, args.cmx_fragment, args.program) - - -if __name__ == '__main__': - sys.exit(main())
diff --git a/build/config/fuchsia/build_symbol_archive.py b/build/config/fuchsia/build_symbol_archive.py index c763627..a595ed8 100755 --- a/build/config/fuchsia/build_symbol_archive.py +++ b/build/config/fuchsia/build_symbol_archive.py
@@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/fuchsia/config.gni b/build/config/fuchsia/config.gni index 8e9e2be..1efe24c 100644 --- a/build/config/fuchsia/config.gni +++ b/build/config/fuchsia/config.gni
@@ -1,11 +1,8 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. assert(is_fuchsia) -# Compute the AEMU path. -aemu_root = "//third_party/aemu-${host_os}-${host_cpu}" - # Compute the path to the arch-specific boot image directory. -boot_image_root = "//third_party/fuchsia-sdk/images/${target_cpu}" +boot_image_root = "//third_party/fuchsia-sdk/images/"
diff --git a/build/config/fuchsia/extend_fvm.py b/build/config/fuchsia/extend_fvm.py index 44e5ee3..ae95f67 100644 --- a/build/config/fuchsia/extend_fvm.py +++ b/build/config/fuchsia/extend_fvm.py
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/fuchsia/fuchsia_package_metadata.gni b/build/config/fuchsia/fuchsia_package_metadata.gni new file mode 100644 index 0000000..fb33bb2 --- /dev/null +++ b/build/config/fuchsia/fuchsia_package_metadata.gni
@@ -0,0 +1,38 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +assert(is_fuchsia) + +# Generates a metadata file under root_gen_dir which provides information about +# a Fuchsia package. +# Parameters: +# package_deps: An array of package_paths which specify the location of all +# .far files that the package depends on. +template("fuchsia_package_metadata") { + _pkg_dir = "$root_out_dir/gen/" + get_label_info(invoker.package, "dir") + + "/" + target_name + _pkg_path = "$_pkg_dir/${target_name}.far" + pkg_dep_paths = [ rebase_path(_pkg_path, root_build_dir) ] + if (defined(invoker.package_deps)) { + foreach(package_dep, invoker.package_deps) { + _pkg_dep_target = package_dep[0] + _pkg_dep_name = package_dep[1] + pkg_dep_path = + rebase_path(get_label_info(_pkg_dep_target, "target_gen_dir") + "/" + + _pkg_dep_name + "/" + _pkg_dep_name + ".far", + root_build_dir) + pkg_dep_paths += [ pkg_dep_path ] + } + } + + pkg_metadata = "${target_name}_script_meta" + generated_file(pkg_metadata) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + contents = { + packages = pkg_dep_paths + } + output_conversion = "json" + outputs = [ "$root_gen_dir/package_metadata/${invoker.target_name}.meta" ] + } +}
diff --git a/build/config/fuchsia/generate_runner_scripts.gni b/build/config/fuchsia/generate_runner_scripts.gni index 7fac16f..cf01659 100644 --- a/build/config/fuchsia/generate_runner_scripts.gni +++ b/build/config/fuchsia/generate_runner_scripts.gni
@@ -1,21 +1,26 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -assert(is_fuchsia) - import("//build/config/chromecast_build.gni") import("//build/config/fuchsia/config.gni") -import("//build/config/fuchsia/package.gni") +import("//build/config/fuchsia/fuchsia_package_metadata.gni") import("//build/config/gclient_args.gni") import("//build/config/sysroot.gni") import("//build/util/generate_wrapper.gni") +assert(is_fuchsia) + declare_args() { # Sets the Fuchsia Amber repository which will be used by default by the # generated installation scripts. If not specified, then no default directory # will be used. - default_fuchsia_build_dir_for_installation = "" + default_fuchsia_out_dir = "" + + # Sets the Fuchsia device node name which will be used by default by the + # generated runner scripts. If not specficed, then no default node name will + # be used. + default_fuchsia_device_node_name = "" # CPU architecture of the host used to run the tests. test_host_cpu = host_cpu @@ -25,245 +30,221 @@ # A list of additional Fuchsia boot images to include in the test isolates. fuchsia_additional_boot_images = [] + + # This variable controls the browser included in the Telemetry based test + # targets. + fuchsia_browser_type = "web_engine_shell" } -# Generates a script which deploys and optionally executes a package on a -# device. +# Generates a wrapper script under root_build_dir/bin that performs an +# operation, such as deployment or execution, using a package and its +# dependencies. # # Parameters: -# package: The package() target which will be run. -# package_name_override: Specifies the name of the generated package, if its -# name is different than the |package| target name. This value must match -# package_name_override in the |package| target. -# package_deps: An array of [package, package_name_override] array pairs +# output_name_format: The format string for the generated script's filename. +# The placeholder string %package% will be substituted +# with |package| (or |package_name|, if set). +# Examples: "run_%package%", "install_%package%" +# package: The package() target to run. +# package_name: Specifies the name of the generated package, if its +# filename is different than the |package| target name. This value must +# match package_name in the |package| target. +# package_deps: An array of [package, package_name] array pairs # which specify additional dependency packages to be installed # prior to execution. -# runner_script: The runner script implementation to use, relative to -# "build/fuchsia". Defaults to "test_runner.py". -# install_only: If true, executing the script will only install the package -# on the device, but not run it. -# is_test_exe: If true, the generated script will run the command under -# test_env.py and add arguments expected to be passed to test exes. -template("fuchsia_package_runner") { - forward_variables_from(invoker, TESTONLY_AND_VISIBILITY + [ "runner_script" ]) - - if (defined(invoker.package_name_override)) { - _pkg_shortname = invoker.package_name_override +# executable: The underlying script to be called by the script. +# executable_args: The list of arguments to pass to |executable|. +# Runtime commandline arguments can be passed to +# |executable| using the placeholder %args%. +# +# In addition, the script is passed the following +# executable_args: +# --package - the path to a .FAR package to install. +# --package_name - the name of the package to use as an +# entry point. +# include_fuchsia_out_dir: If true, adds |default_fuchsia_out_dir| +# to executable_args (when set in GN args). +template("fuchsia_run_script_with_packages") { + if (defined(invoker.package_name)) { + _pkg_shortname = invoker.package_name } else { _pkg_shortname = get_label_info(invoker.package, "name") } - _pkg_dir = "$root_out_dir/gen/" + get_label_info(invoker.package, "dir") + - "/" + _pkg_shortname - _package_path = "$_pkg_dir/${_pkg_shortname}.far" + _generated_script_path = + "$root_build_dir/bin/" + + string_replace(invoker.output_name_format, "%package%", _pkg_shortname) - generated_run_pkg_script_path = "$root_build_dir/bin/run_${_pkg_shortname}" - generated_install_pkg_script_path = - "$root_build_dir/bin/install_$_pkg_shortname" + generate_wrapper(target_name) { + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "executable", + "executable_args", + "data", + "include_fuchsia_out_dir", + "target", + ]) - _generate_runner_target = "${target_name}__generate_runner" - _generate_installer_target = "${target_name}__generate_installer" + wrapper_script = _generated_script_path + deps = [ invoker.package ] - # Generates a script which installs and runs a test. - generate_wrapper(_generate_runner_target) { - forward_variables_from(invoker, [ "target" ]) - - _is_test_exe = defined(invoker.is_test_exe) && invoker.is_test_exe - - if (defined(runner_script)) { - _runner_script = runner_script - } else { - _runner_script = "//build/fuchsia/test_runner.py" - } - - if (_is_test_exe) { - executable = "//testing/test_env.py" - executable_args = - [ "@WrappedPath(" + rebase_path(_runner_script, root_out_dir) + ")" ] - data = [ - _runner_script, - "//.vpython", - ] - data_deps = [ "//testing:test_scripts_shared" ] - } else { - executable = rebase_path(_runner_script) - executable_args = [] - data = [] + if (!defined(data_deps)) { data_deps = [] } + data_deps += [ "//build/config/fuchsia:deployment_resources" ] - if (defined(invoker.data)) { - data += invoker.data + _combined_package_list = [ invoker.package ] + + if (defined(invoker.package_deps)) { + foreach(package_dep, invoker.package_deps) { + _combined_package_list += [ package_dep[0] ] + } + } + foreach(package_dep, _combined_package_list) { + data_deps += [ + package_dep, + package_dep + "__archive-manifest", + package_dep + "__archive-metadata", + ] } - wrapper_script = generated_run_pkg_script_path - - data_deps += [ - invoker.package, - - # Runner scripts require access to "ids.txt" for symbolization, and to - # the "package" from which to get the name & version to deploy, which - # are outputs of the archive manifest generation action. - "${invoker.package}__archive-manifest", - - # Runner scripts require access to "meta.far" from which to calculate the - # expected Merkle root for the package, to verify it has been cached. - "${invoker.package}__archive-metadata", - ] if (defined(invoker.data_deps)) { data_deps += invoker.data_deps } - # Declares the files that are needed for test execution on the - # swarming test client. - data += [ - "//build/fuchsia/", - "//build/util/lib/", - "//third_party/fuchsia-sdk/sdk/.build-id/", - "//third_party/fuchsia-sdk/sdk/bin/fpave.sh", - "//third_party/fuchsia-sdk/sdk/bin/fuchsia-common.sh", - "//third_party/fuchsia-sdk/sdk/meta/manifest.json", - ] - - # TODO(crbug.com/1137662): Remove checkout_fuchsia_for_arm64_host from - # gclient_gn_args in //DEPS as well as this condition when builders have - # test_host_cpu set correctly. - if (checkout_fuchsia_for_arm64_host) { - test_host_cpu = "arm64" - } - - if (test_host_cpu == "x64") { - data_deps += - [ "//build/config/clang:llvm-symbolizer_data($host_toolchain)" ] - } - - data += [ - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/device-finder", - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/fvm", - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/merkleroot", - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/pm", - - # TODO(crbug.com/1162314) Remove "symbolize" when transition to - # "symbolizer" is complete. - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/symbolize", - - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/symbolizer", - "//third_party/fuchsia-sdk/sdk/tools/${test_host_cpu}/zbi", - ] - - if (test_isolate_uses_emulator) { - data += [ - "${boot_image_root}/qemu/qemu-kernel.kernel", - "${boot_image_root}/qemu/storage-full.blk", - "${boot_image_root}/qemu/zircon-a.zbi", - "//third_party/qemu-${host_os}-${test_host_cpu}/", - ] - - # Include AEMU for x64 emulator hosts. - if (test_host_cpu == "x64") { - data += [ "${aemu_root}/" ] - } - } - - foreach(fuchsia_additional_boot_image, fuchsia_additional_boot_images) { - data += [ "${fuchsia_additional_boot_image}/" ] - } - - package_paths = [ rebase_path(_package_path, root_build_dir) ] + # Compute the list of full paths to package files, including dependencies. if (defined(invoker.package_deps)) { foreach(package_dep, invoker.package_deps) { package_dep_target = package_dep[0] - package_dep_name = package_dep[1] - - data_deps += [ - package_dep_target, - package_dep_target + "__archive-manifest", - package_dep_target + "__archive-metadata", - ] - package_dep_path = rebase_path( - get_label_info(package_dep_target, "target_gen_dir") + "/" + - package_dep_name + "/" + package_dep_name + ".far", - root_build_dir) - package_paths += [ package_dep_path ] + deps += [ package_dep_target ] + data_deps += [ package_dep_target ] } } - foreach(package_path, package_paths) { - executable_args += [ - "--package", - "@WrappedPath(${package_path})", - ] + # Include package information inside the wrapper script. + if (!defined(executable_args)) { + executable_args = [] } - executable_args += [ + if (defined(include_fuchsia_out_dir) && include_fuchsia_out_dir && + default_fuchsia_out_dir != "") { + executable_args += [ + "--fuchsia-out-dir", + default_fuchsia_out_dir, + ] + } + } + + # Create a wrapper script rather than using a group() in order to ensure + # "ninja $target_name" always works. + if (defined(invoker.executable_wrapper)) { + generate_wrapper(invoker.executable_wrapper) { + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + executable = _generated_script_path + wrapper_script = "$root_build_dir/${invoker.executable_wrapper}" + deps = [ ":${invoker._run_target}" ] + } + } +} + +# Generates a script which deploys a package to the TUF repo of a Fuchsia +# build output directory. +template("fuchsia_package_installer") { + if (defined(invoker.package_name)) { + pkg_shortname = invoker.package_name + } else { + pkg_shortname = get_label_info(invoker.package, "name") + } + fuchsia_package_metadata(pkg_shortname) { + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "package", + "package_deps", + ]) + } + fuchsia_run_script_with_packages(target_name) { + forward_variables_from(invoker, + "*", + TESTONLY_AND_VISIBILITY + [ "executable_args" ]) + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + executable = rebase_path("//build/fuchsia/test/deploy_to_fuchsia.py") + executable_args = [ "--out-dir", "@WrappedPath(.)", - "--target-cpu", - target_cpu, - "--package-name", - _pkg_shortname, + pkg_shortname, ] + output_name_format = "deploy_%package%" + include_fuchsia_out_dir = true + } +} + +# Generates scripts for installing and running test packages. +# See fuchsia_run_script_with_packages() for the full list of parameters. +template("fuchsia_test_runner") { + _run_target = "${target_name}__runner" + _install_target = "${target_name}__installer" + + fuchsia_run_script_with_packages(_run_target) { + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "data", + "data_deps", + "package", + "package_name", + "package_deps", + ]) + + _test_runner_py = "//build/fuchsia/test/run_test.py" + + executable = rebase_path(_test_runner_py) + + if (defined(invoker.is_test_exe) && invoker.is_test_exe) { + data += [ "//.vpython3" ] + } + output_name_format = "run_%package%" + executable_wrapper = invoker.target_name + + # Populate the arguments used by the test runner, defined at build-time. + executable_args = [ + "--out-dir", + "@WrappedPath(.)", + ] + + executable_args += [ package_name ] if (defined(invoker.use_test_server) && invoker.use_test_server) { executable_args += [ "--enable-test-server" ] } - if (default_fuchsia_build_dir_for_installation != "") { + if (default_fuchsia_device_node_name != "") { executable_args += [ - "--fuchsia-out-dir", - default_fuchsia_build_dir_for_installation, + "--target-id", + default_fuchsia_device_node_name, ] } - } - # Produces a script which installs a package and its dependencies into the - # Amber repository of a pre-existing Fuchsia build directory. - generate_wrapper(_generate_installer_target) { - executable = rebase_path("//build/fuchsia/deploy_to_amber_repo.py") - wrapper_script = generated_install_pkg_script_path - - data_deps = [ invoker.package ] - if (defined(invoker.data_deps)) { - data_deps += invoker.data_deps + # Declare the files that are needed for test execution on LUCI swarming + # test clients, both directly (via data) or indirectly (via data_deps). + if (!defined(data)) { + data = [] } + data += [ + _test_runner_py, + "$root_gen_dir/package_metadata/${invoker.package_name}.meta", + ] - # Build a list of all packages to install, and pass the list to the runner - # script. - package_paths = [ rebase_path(_package_path, root_build_dir) ] - if (defined(invoker.package_deps)) { - foreach(package_dep, invoker.package_deps) { - package_dep_target = package_dep[0] - package_dep_name = package_dep[1] - - data_deps += [ package_dep_target ] - package_dep_path = rebase_path( - get_label_info(package_dep_target, "target_gen_dir") + "/" + - package_dep_name + "/" + package_dep_name + ".far", - root_build_dir) - package_paths += [ package_dep_path ] - } - } - executable_args = [] - foreach(package_path, package_paths) { - executable_args += [ - "--package", - "@WrappedPath(${package_path})", - ] - - if (default_fuchsia_build_dir_for_installation != "") { - executable_args += [ - "--fuchsia-out-dir", - default_fuchsia_build_dir_for_installation, - ] - } + # TODO(crbug.com/1256870): Remove this once all out-of-tree references + # to "package_name_override" are migrated to "package_name". + if (defined(invoker.package_name_override)) { + package_name = invoker.package_name_override } } - - group(target_name) { - deps = [ ":${_generate_installer_target}" ] - - if (!defined(invoker.install_only) || invoker.install_only == false) { - deps += [ ":${_generate_runner_target}" ] - } + fuchsia_package_installer(_install_target) { + forward_variables_from(invoker, + TESTONLY_AND_VISIBILITY + [ + "package", + "package_name", + "package_deps", + ]) } }
diff --git a/build/config/fuchsia/gfx_tests.cmx b/build/config/fuchsia/gfx_tests.cmx deleted file mode 100644 index c02975d..0000000 --- a/build/config/fuchsia/gfx_tests.cmx +++ /dev/null
@@ -1,30 +0,0 @@ -{ - "sandbox": { - "features": [ - "deprecated-ambient-replace-as-executable", - "isolated-persistent-storage", - "isolated-temp", - "vulkan" - ], - "dev": [ - "null", - "zero" - ], - "services": [ - "fuchsia.accessibility.semantics.SemanticsManager", - "fuchsia.device.NameProvider", - "fuchsia.fonts.Provider", - "fuchsia.intl.PropertyProvider", - "fuchsia.logger.LogSink", - "fuchsia.memorypressure.Provider", - "fuchsia.process.Launcher", - "fuchsia.sys.Environment", - "fuchsia.sys.Loader", - "fuchsia.sysmem.Allocator", - "fuchsia.tracing.provider.Registry", - "fuchsia.ui.policy.Presenter", - "fuchsia.ui.scenic.Scenic", - "fuchsia.vulkan.loader.Loader" - ] - } -}
diff --git a/build/config/fuchsia/package.gni b/build/config/fuchsia/package.gni deleted file mode 100644 index ff6ffd0..0000000 --- a/build/config/fuchsia/package.gni +++ /dev/null
@@ -1,114 +0,0 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import("//third_party/fuchsia-sdk/sdk/build/component.gni") -import("//third_party/fuchsia-sdk/sdk/build/package.gni") - -# DEPRECATED: Use the Fuchsia SDK's fuchsia_component() and fuchsia_package() -# templates directly, in new code. -# -# Creates a Fuchsia .far package file containing a Fuchsia component. -# -# Parameters are: -# package_name_override: Specifies the name of the package to generate, -# if different than |target_name|. -# binary: The executable target which should be launched. -# manifest: A path to the manifest that will be used. -# "testonly" targets default to using -# //build/config/fuchsia/tests-with-exec.cmx. -# Non-test targets must explicitly specify a |manifest|. -# additional_manifests: Manifest files that should be included in the package in -# the /meta directory. This allows to package more than one component per -# manifest. These manifest files must specify program/binary to run, which -# is not required for the main manifest file where this parameter is added -# during build. -# component_name_override: If set, specifies the name of the component. -# By default, the component name is the same as the package name. -# deps: Additional targets to build and include in the package (optional). -# -# TODO(https://crbug.com/1050703): Migrate consumers to GN SDK equivalents. -template("cr_fuchsia_package") { - assert(defined(invoker.binary)) - - if (defined(invoker.package_name_override)) { - _package_name = invoker.package_name_override - } else { - _package_name = invoker.target_name - } - - _package_contents = [ invoker.binary ] - if (defined(invoker.deps)) { - _package_contents += invoker.deps - } - - _component_cmx_target = target_name + "__cr-component-cmx" - _component_target = target_name + "__cr-component" - _package_components = [ ":${_component_target}" ] - _component_manifest = "${target_gen_dir}/${target_name}.cmx" - - # Process the CMX fragment in |manifest| to get a full manifest. - action(_component_cmx_target) { - forward_variables_from(invoker, - [ - "deps", - "testonly", - ]) - - script = "//build/config/fuchsia/build_cmx_from_fragment.py" - - inputs = [ invoker.manifest ] - outputs = [ _component_manifest ] - - args = [ - "--cmx-fragment", - rebase_path(invoker.manifest), - "--cmx", - rebase_path(_component_manifest), - "--program", - get_label_info(invoker.binary, "name"), - ] - } - - # Declare the primary component for this package. - fuchsia_component(_component_target) { - forward_variables_from(invoker, [ "testonly" ]) - - deps = [ ":${_component_cmx_target}" ] - data_deps = _package_contents - manifest = _component_manifest - - if (defined(invoker.component_name_override)) { - manifest_output_name = "${invoker.component_name_override}" - } else { - manifest_output_name = "${_package_name}" - } - } - - # Bundle manifests providing additional entrypoints into the package. - if (defined(invoker.additional_manifests)) { - foreach(filename, invoker.additional_manifests) { - _additional_component_target = - target_name + "_" + get_path_info(filename, "name") - _package_components += [ ":${_additional_component_target}" ] - fuchsia_component(_additional_component_target) { - forward_variables_from(invoker, [ "testonly" ]) - data_deps = _package_contents - manifest = filename - - # Depend upon the invoker's |deps|, in case they include a dependency - # responsible for generating this additional component's manifest file. - deps = _package_contents - } - } - } - - fuchsia_package(target_name) { - forward_variables_from(invoker, [ "testonly" ]) - package_name = _package_name - if (defined(invoker.excluded_files)) { - excluded_files = invoker.excluded_files - } - deps = _package_components - } -}
diff --git a/build/config/fuchsia/packaged_content_embedder_excluded_dirs.gni b/build/config/fuchsia/packaged_content_embedder_excluded_dirs.gni new file mode 100644 index 0000000..f179a66 --- /dev/null +++ b/build/config/fuchsia/packaged_content_embedder_excluded_dirs.gni
@@ -0,0 +1,16 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/devtools.gni") + +assert(is_fuchsia) + +# List of transitively included directories that should be stripped from +# released packages for size reasons. For use with the |excluded_dirs| variable +# of fuchsia_package(). +FUCHSIA_PACKAGED_CONTENT_EMBEDDER_EXCLUDED_DIRS = [ + # These are mistakenly being shipped in both PAK form and runtime data deps. + # TODO(crbug.com/1265660): Remove when DevTools stops leaking its source list. + devtools_root_location, +]
diff --git a/build/config/fuchsia/rules.gni b/build/config/fuchsia/rules.gni deleted file mode 100644 index 689e130..0000000 --- a/build/config/fuchsia/rules.gni +++ /dev/null
@@ -1,5 +0,0 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import("//build/config/fuchsia/generate_runner_scripts.gni")
diff --git a/build/config/fuchsia/size_optimized_cast_receiver_args.gn b/build/config/fuchsia/size_optimized_cast_receiver_args.gn new file mode 100644 index 0000000..9a366c7 --- /dev/null +++ b/build/config/fuchsia/size_optimized_cast_receiver_args.gn
@@ -0,0 +1,43 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This file contains feature and optimization overrides that are commonly +# required or useful for Cast Receiver implementations. +# It prioritizes size and disables unneeded features that may add size. +# +# To use it do one of the following: +# * Add the following to your `gn args`: +# import("//build/config/fuchsia/size_optimized_cast_receiver_args.gn") +# * Add the following to `gn_args` in a bot recipe: +# 'args_file': '//build/config/fuchsia/size_optimized_cast_receiver_args.gn' + +# There is no reason these values couldn't be used on other platforms, but this +# file is in a fuchsia/ directory and some refactoring would probably be +# appropriate before reusing this file. +# It is not possible to assert the platform because `target_os` is not defined +# when this file is imported. + +enable_printing = false +enable_cast_receiver = true +cast_streaming_enable_remoting = true +enable_dav1d_decoder = false +enable_v8_compile_hints = false + +# //chrome makes many assumptions that Extensions are enabled. +# TODO(crbug.com/1363742): Fix theses assumptions or avoid building it. +# enable_extensions = false + +enable_hidpi = false +enable_libaom = false +enable_library_cdms = false +enable_logging_override = true +enable_pdf = false +enable_plugins = false +optimize_for_size = true +optional_trace_events_enabled = false + +# Ensure PGO and ThinLTO are disabled as these optimizations increase the binary +# size (see crbug.com/1322959). +chrome_pgo_phase = 0 +use_thin_lto = false
diff --git a/build/config/fuchsia/size_optimized_cast_receiver_args_internal.gn b/build/config/fuchsia/size_optimized_cast_receiver_args_internal.gn new file mode 100644 index 0000000..b59ce96 --- /dev/null +++ b/build/config/fuchsia/size_optimized_cast_receiver_args_internal.gn
@@ -0,0 +1,18 @@ +# Copyright 2022 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This is a version of size_optimized_cast_receiver_args.gn that is intended for +# internal builds and requires src-internal. +# +# To use it do one of the following: +# * Add the following to your `gn args`: +# import("build/config/fuchsia/size_optimized_cast_receiver_args_internal.gn") +# * Add the following to `gn_args` in a bot recipe: +# 'args_file': '//build/config/fuchsia/size_optimized_cast_receiver_args_internal.gn' + +import("//build/config/fuchsia/size_optimized_cast_receiver_args.gn") + +enable_widevine = true +use_internal_isolated_origins = true +use_official_google_api_keys = false
diff --git a/build/config/fuchsia/sizes.gni b/build/config/fuchsia/sizes.gni index 20a5bf8..fc97676 100644 --- a/build/config/fuchsia/sizes.gni +++ b/build/config/fuchsia/sizes.gni
@@ -1,7 +1,9 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +assert(is_fuchsia) + import("//build/util/generate_wrapper.gni") template("compute_fuchsia_package_sizes") { @@ -28,11 +30,14 @@ # Declares the files that are needed for test execution on the # swarming test client. + # TODO(crbug.com/1347172): Remove arm64 once the execution of fuchsia_sizes + # has been migrated to x64 machines. data += [ "//build/fuchsia/", - "//fuchsia/release/size_tests/", + "//tools/fuchsia/size_tests/", "//third_party/fuchsia-sdk/sdk/arch/", - "//third_party/fuchsia-sdk/sdk/tools/${target_cpu}/", + "//third_party/fuchsia-sdk/sdk/tools/arm64/", + "//third_party/fuchsia-sdk/sdk/tools/x64/", ] executable_args = [
diff --git a/build/config/fuchsia/symbol_archive.gni b/build/config/fuchsia/symbol_archive.gni index 9dcb53c..e05af11 100644 --- a/build/config/fuchsia/symbol_archive.gni +++ b/build/config/fuchsia/symbol_archive.gni
@@ -1,4 +1,4 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. +# Copyright 2019 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -8,7 +8,7 @@ # ".build_ids" convention used by the symbolizer and GNU GDB. # # Parameters: -# deps: Must all be cr_fuchsia_package() or fuchsia_package() targets. +# deps: Must all be fuchsia_package() targets. # ids_txt: The "ids.txt" file which lists the relative paths to unstripped # executables and libraries, along with their build IDs. # archive_name: The path to the compressed tarball that will be generated.
diff --git a/build/config/fuchsia/test/OWNERS b/build/config/fuchsia/test/OWNERS index 3be17de..ac711c0 100644 --- a/build/config/fuchsia/test/OWNERS +++ b/build/config/fuchsia/test/OWNERS
@@ -1,7 +1,7 @@ file://build/fuchsia/OWNERS -per-file *.test-cmx=set noparent -per-file *.test-cmx=ddorwin@chromium.org -per-file *.test-cmx=wez@chromium.org +per-file *.test-cml=set noparent +per-file *.test-cml=ddorwin@chromium.org +per-file *.test-cml=wez@chromium.org # Please prefer the above when possible. -per-file *.test-cmx=file://fuchsia/SECURITY_OWNERS +per-file *.test-cml=file://build/fuchsia/SECURITY_OWNERS
diff --git a/build/config/fuchsia/test/README.md b/build/config/fuchsia/test/README.md index c5f1762..d21cdb7 100644 --- a/build/config/fuchsia/test/README.md +++ b/build/config/fuchsia/test/README.md
@@ -1,31 +1,82 @@ -## CMX Fragments +## Manifest Fragments -This directory contains the cmx fragments that are required for running -Fuchsia tests hermetically. Tests start from `minimum_capabilities.test-cmx` -and add additional capabilities as necessary by providing the +This directory contains the manifest fragments that are required for running +Fuchsia tests hermetically. Tests start from `minimum.shard.test-cml` and add +additional capabilities as necessary by providing the `additional_manifest_fragments` argument. Some fragments are explained in detail below: ### General Purpose Fragments -#### font_capabilities.test-cmx -For tests that test fonts by providing `fuchsia.fonts.Provider`. +#### archivist.shard.test-cml +Runs an `archivist-without-attribution` with custom protocol routing for tests +that want to intercept events written to a `LogSink` by a component. -#### jit_capabilities.test-cmx +#### chromium_test_facet.shard.test-cml +Runs tests in the `chromium` test realm, which is mostly hermetic but has access +to specific system services that cannot (currently) be faked. For more +information, see https://fxbug.dev/91934. This is generally required for all +Chromium tests not using the +[`chromium_system_test_facet`](#chromium_system_test_facetshardtest-cml). + +#### fonts.shard.test-cml +For tests that test fonts by providing `fuchsia.fonts.Provider`. This shard +runs an isolated font provider, but serves the fonts present on the system. + +#### test_fonts.shard.test-cml +For tests that use the fonts in `//third_party/test_fonts` by way of +`//skia:test_fonts_cfv2`. + +#### mark_vmo_executable.shard.test-cml Required by tests that execute JavaScript. Should only be required in a small number of tests. -#### minimum_capabilites.test-cmx -Capabilities required by anything that uses `//base/test`, used as the base -fragment for all test suites. +#### minimum.shard.test-cml +Capabilities required by anything that uses `//base/test` when running in the +(default) `chromium` test realm. It is the default base fragment for most +`test()` Components. -#### read_debug_data.test-cmx -Required by tests that need access to its debug directory. Should only be -required in a small number of tests. +The system-wide `config-data` directory capability is routed to tests running in +the realm so that individual tests may route subdirectories as needed. +TODO(crbug.com/1360077): Remove this after migrating to the new mechanism. -#### test_logger_capabilities.test-cmx +#### logger.shard.test-cml For tests that test logging functionality by providing `fuchsia.logger.Log`. +#### sysmem.shard.test-cml +For tests that depend on the sysmem service (e.g. to allocate image buffers to +share with Vulkan and Scenic). + +#### system_test_minimum.shard.test-cml +Capabilities required by anything that uses `//base/test` when running as a +system test in the `chromium-system` test realm. It is the base fragment for +`test()` Components that use the +[`chromium_system_test_facet`](#chromium_system_test_facetshardtest-cml). + +Most tests use the [`minimum`](#minimumshardtest-cml) shard. + +#### chromium_system_test_facet.shard.test-cml +Runs tests in the `chromium-system` test realm. This is required for Chromium +tests that are intended to run against the actual system and its real system +services. This is required for, for example, performance tests intended to +measure system performance. Another overlapping use case is tests that need to +be run in environments without access to the packages containing fake +implementations of required protocols that other tests use. +(https://crbug.com/1408597 should make that use case obsolete.) + +Most tests should use the +[`chromium_test_facet`](#chromium_test_facetshardtest-cml). + +#### test_ui_stack.shard.test-cml +For tests that need an isolated UI subsystem, that supports the Flatland +API set. This allows tests to e.g. run with view-focus unaffected by any +other tests running concurrently on the device, as well as providing test-only +functionality such as input-injection support. + +#### gfx_test_ui_stack.shard.test-cml +For tests that need an isolated display subsystem supporting the legacy +Scenic/GFX APIs. + ### WebEngine Fragments The following fragments are specific to WebEngine functionality as documented documentation at @@ -33,26 +84,29 @@ https://fuchsia.dev/reference/fidl/fuchsia.web#ContextFeatureFlags. Any test-specific exceptions are documented for each file. -#### audio_capabilities.test-cmx -Corresponds to the `AUDIO` flag. Required for enabling audio input and output. +#### audio_output.shard.test-cml +Required by tests that need to enable audio output. -#### network_capabilities.test-cmx +#### platform_video_codecs.shard.test-cml +Required by tests that need accelerated (e.g., hardware) video codecs. A private +(semi-isolated) instance of codec_factory is run for tests using this shard in +support of running on system images that don't run it. + +#### network.shard.test-cml +For tests that need access to network services, including those that access a +local HTTP server. + +#### network.shard.test-cml Corresponds to the `NETWORK` flag. Required for enabling network access. Note that access to the root SSL certificates is not needed if ContextProvider is used to launch the `Context`. The `fuchsia.device.NameProvider` dependency comes from fdio. -#### present_view_capabilities.test-cmx +#### present_view.shard.test-cml Services that are needed to render web content in a Scenic view and present it. Most services are required per the FIDL documentation. -`fuchsia.ui.policy.Presenter` is additionally required by tests that create -views. -#### vulkan_capabilities.test-cmx -Corresponds to the `VULKAN` flag. Required for enabling GPU-accelerated -rendering of the web content. - -#### web_engine_required_capabilities.test-cmx -Contains services that need to be present when creating a -`fuchsia.web.Context`. Note that the `fuchsia.scheduler.ProfileProvider` service -is only used in tests that encounter memory pressure code. +#### web_instance.shard.test-cml +Contains services that need to be present when creating a `fuchsia.web.Context`. +Note that the `fuchsia.scheduler.ProfileProvider` service is only used in tests +that encounter memory pressure code.
diff --git a/build/config/fuchsia/test/access_test_data_dir.test-cmx b/build/config/fuchsia/test/access_test_data_dir.test-cmx deleted file mode 100644 index 5757778..0000000 --- a/build/config/fuchsia/test/access_test_data_dir.test-cmx +++ /dev/null
@@ -1,7 +0,0 @@ -{ - "sandbox": { - "features": [ - "isolated-cache-storage" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/archivist.shard.test-cml b/build/config/fuchsia/test/archivist.shard.test-cml new file mode 100644 index 0000000..b85162f --- /dev/null +++ b/build/config/fuchsia/test/archivist.shard.test-cml
@@ -0,0 +1,28 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + children: [ + { + name: "isolated_archivist", + url: "fuchsia-pkg://fuchsia.com/archivist-without-attribution#meta/archivist-without-attribution.cm", + }, + ], + use: [ + { + protocol: "fuchsia.logger.Log", + path: "/svc/fuchsia.logger.Log.isolated", + from: "#isolated_archivist", + }, + { + protocol: "fuchsia.logger.LogSink", + path: "/svc/fuchsia.logger.LogSink.isolated", + from: "#isolated_archivist", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ "archivist-without-attribution" ], + }, + }, +}
diff --git a/build/config/fuchsia/test/audio_capabilities.test-cmx b/build/config/fuchsia/test/audio_capabilities.test-cmx deleted file mode 100644 index 2e2013f..0000000 --- a/build/config/fuchsia/test/audio_capabilities.test-cmx +++ /dev/null
@@ -1,18 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "injected-services": { - "fuchsia.mediacodec.CodecFactory": "fuchsia-pkg://fuchsia.com/codec_factory#meta/codec_factory.cmx" - }, - "system-services": [ - "fuchsia.media.Audio" - ] - } - }, - "sandbox": { - "services": [ - "fuchsia.media.Audio", - "fuchsia.mediacodec.CodecFactory" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/audio_output.shard.test-cml b/build/config/fuchsia/test/audio_output.shard.test-cml new file mode 100644 index 0000000..9176f6c --- /dev/null +++ b/build/config/fuchsia/test/audio_output.shard.test-cml
@@ -0,0 +1,16 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + use: [ + { + protocol: [ + // TODO(crbug.com/1348174): Rather than require the system to provide + // capabilities straight from audio_core, we should run Chromium tests + // against an audio stack with fake device(s). + "fuchsia.media.Audio", + "fuchsia.media.AudioDeviceEnumerator", + ] + }, + ], +}
diff --git a/build/config/fuchsia/test/chromium_system_test_facet.shard.test-cml b/build/config/fuchsia/test/chromium_system_test_facet.shard.test-cml new file mode 100644 index 0000000..cdf9ca7 --- /dev/null +++ b/build/config/fuchsia/test/chromium_system_test_facet.shard.test-cml
@@ -0,0 +1,8 @@ +// Copyright 2023 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + facets: { + "fuchsia.test": { type: "chromium-system" }, + }, +}
diff --git a/build/config/fuchsia/test/chromium_test_facet.shard.test-cml b/build/config/fuchsia/test/chromium_test_facet.shard.test-cml new file mode 100644 index 0000000..3628cf4 --- /dev/null +++ b/build/config/fuchsia/test/chromium_test_facet.shard.test-cml
@@ -0,0 +1,8 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + facets: { + "fuchsia.test": { type: "chromium" }, + }, +}
diff --git a/build/config/fuchsia/test/context_provider.shard.test-cml b/build/config/fuchsia/test/context_provider.shard.test-cml new file mode 100644 index 0000000..e5db2f1 --- /dev/null +++ b/build/config/fuchsia/test/context_provider.shard.test-cml
@@ -0,0 +1,30 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + children: [ + { + name: "context_provider", + url: "fuchsia-pkg://fuchsia.com/web_engine#meta/context_provider.cm", + }, + ], + use: [ + { + protocol: [ + "fuchsia.web.ContextProvider", + ], + from: "#context_provider", + dependency: "weak", + }, + ], + offer: [ + { + protocol: [ + "fuchsia.feedback.ComponentDataRegister", + "fuchsia.feedback.CrashReportingProductRegister", + ], + from: "parent", + to: "#context_provider", + }, + ], +}
diff --git a/build/config/fuchsia/test/elf_test_ambient_exec_runner.shard.test-cml b/build/config/fuchsia/test/elf_test_ambient_exec_runner.shard.test-cml new file mode 100644 index 0000000..c9328c5 --- /dev/null +++ b/build/config/fuchsia/test/elf_test_ambient_exec_runner.shard.test-cml
@@ -0,0 +1,17 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + program: { + runner: "elf_test_ambient_exec_runner", + }, + capabilities: [ + { protocol: "fuchsia.test.Suite" }, + ], + expose: [ + { + protocol: "fuchsia.test.Suite", + from: "self", + }, + ], +}
diff --git a/build/config/fuchsia/test/elf_test_runner.shard.test-cml b/build/config/fuchsia/test/elf_test_runner.shard.test-cml new file mode 100644 index 0000000..c97e6d7 --- /dev/null +++ b/build/config/fuchsia/test/elf_test_runner.shard.test-cml
@@ -0,0 +1,17 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + program: { + runner: "elf_test_runner", + }, + capabilities: [ + { protocol: "fuchsia.test.Suite" }, + ], + expose: [ + { + protocol: "fuchsia.test.Suite", + from: "self", + }, + ], +}
diff --git a/build/config/fuchsia/test/font_capabilities.test-cmx b/build/config/fuchsia/test/font_capabilities.test-cmx deleted file mode 100644 index 4c8661b..0000000 --- a/build/config/fuchsia/test/font_capabilities.test-cmx +++ /dev/null
@@ -1,14 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "injected-services": { - "fuchsia.fonts.Provider": "fuchsia-pkg://fuchsia.com/fonts#meta/fonts.cmx", - } - } - }, - "sandbox": { - "services": [ - "fuchsia.fonts.Provider" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/fonts.shard.test-cml b/build/config/fuchsia/test/fonts.shard.test-cml new file mode 100644 index 0000000..80fb0ca --- /dev/null +++ b/build/config/fuchsia/test/fonts.shard.test-cml
@@ -0,0 +1,38 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + children: [ + { + name: "isolated_font_provider", + url: "fuchsia-pkg://fuchsia.com/fonts#meta/fonts.cm", + }, + ], + use: [ + { + protocol: "fuchsia.fonts.Provider", + from: "#isolated_font_provider", + }, + ], + offer: [ + { + directory: "config-data", + from: "parent", + to: "#isolated_font_provider", + subdir: "fonts", + }, + { + protocol: [ + "fuchsia.logger.LogSink", + "fuchsia.tracing.provider.Registry", + ], + from: "parent", + to: "#isolated_font_provider", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ "fonts" ], + }, + }, +}
diff --git a/build/config/fuchsia/test/gfx_test_ui_stack.shard.test-cml b/build/config/fuchsia/test/gfx_test_ui_stack.shard.test-cml new file mode 100644 index 0000000..2e51f03 --- /dev/null +++ b/build/config/fuchsia/test/gfx_test_ui_stack.shard.test-cml
@@ -0,0 +1,49 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Used in tests which are hard-coded for the Scenic/GFX API-set. +// Use test_ui_stack.shard.test-cml when testing for Flatland, or when the +// choice of API-set is not important. +{ + include: [ + "//build/config/fuchsia/test/sysmem.shard.test-cml", + ], + children: [ + { + name: "test_ui_stack", + url: "fuchsia-pkg://fuchsia.com/gfx-scene-manager-test-ui-stack#meta/test-ui-stack.cm", + }, + ], + offer: [ + { + protocol: [ + "fuchsia.logger.LogSink", + "fuchsia.scheduler.ProfileProvider", + "fuchsia.sysmem.Allocator", + "fuchsia.tracing.provider.Registry", + "fuchsia.vulkan.loader.Loader", + ], + from: "parent", + to: "#test_ui_stack", + }, + ], + use: [ + { + protocol: [ + "fuchsia.accessibility.semantics.SemanticsManager", + "fuchsia.element.GraphicalPresenter", + "fuchsia.ui.composition.Allocator", + "fuchsia.ui.composition.Flatland", + "fuchsia.ui.input3.Keyboard", + "fuchsia.ui.scenic.Scenic", + ], + from: "#test_ui_stack", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ "gfx-scene-manager-test-ui-stack" ], + }, + }, +}
diff --git a/build/config/fuchsia/test/jit_capabilities.test-cmx b/build/config/fuchsia/test/jit_capabilities.test-cmx deleted file mode 100644 index ff70e25..0000000 --- a/build/config/fuchsia/test/jit_capabilities.test-cmx +++ /dev/null
@@ -1,7 +0,0 @@ -{ - "sandbox": { - "features": [ - "deprecated-ambient-replace-as-executable" - ] - } -}
diff --git a/build/config/fuchsia/test/logger.shard.test-cml b/build/config/fuchsia/test/logger.shard.test-cml new file mode 100644 index 0000000..be0881d --- /dev/null +++ b/build/config/fuchsia/test/logger.shard.test-cml
@@ -0,0 +1,8 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + use: [ + { protocol: [ "fuchsia.logger.Log" ] }, + ], +}
diff --git a/build/config/fuchsia/test/mark_vmo_executable.shard.test-cml b/build/config/fuchsia/test/mark_vmo_executable.shard.test-cml new file mode 100644 index 0000000..ac07c1b --- /dev/null +++ b/build/config/fuchsia/test/mark_vmo_executable.shard.test-cml
@@ -0,0 +1,12 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + use: [ + { + protocol: [ + "fuchsia.kernel.VmexResource", + ], + }, + ], +}
diff --git a/build/config/fuchsia/test/minimum.shard.test-cml b/build/config/fuchsia/test/minimum.shard.test-cml new file mode 100644 index 0000000..17b4927 --- /dev/null +++ b/build/config/fuchsia/test/minimum.shard.test-cml
@@ -0,0 +1,78 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + include: [ + "syslog/client.shard.cml", + ], + // Add capability providers. + children: [ + { + name: "build-info-service", + url: "fuchsia-pkg://fuchsia.com/fake-build-info#meta/fake_build_info.cm", + }, + { + name: "intl_property_manager", + url: "fuchsia-pkg://fuchsia.com/intl_property_manager#meta/intl_property_manager.cm", + }, + ], + offer: [ + { + protocol: "fuchsia.logger.LogSink", + from: "parent", + to: [ "#intl_property_manager" ], + } + ], + use: [ + { + directory: "config-data", + rights: [ "r*" ], + path: "/config/data", + }, + { + storage: "cache", + path: "/cache", + }, + { + storage: "custom_artifacts", + path: "/custom_artifacts", + }, + { + storage: "data", + path: "/data", + }, + { + storage: "tmp", + path: "/tmp", + }, + { + protocol: [ "fuchsia.buildinfo.Provider" ], + from: "#build-info-service", + }, + { + protocol: [ "fuchsia.intl.PropertyProvider" ], + from: "#intl_property_manager", + }, + { + protocol: [ + "fuchsia.hwinfo.Product", + "fuchsia.media.ProfileProvider", + "fuchsia.process.Launcher", + ], + }, + { + protocol: [ + "fuchsia.tracing.perfetto.ProducerConnector", + ], + availability: "optional", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ + "fake-build-info", + "intl_property_manager", + ], + }, + }, +}
diff --git a/build/config/fuchsia/test/minimum_capabilities.test-cmx b/build/config/fuchsia/test/minimum_capabilities.test-cmx deleted file mode 100644 index a1d469d..0000000 --- a/build/config/fuchsia/test/minimum_capabilities.test-cmx +++ /dev/null
@@ -1,29 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "injected-services": { - "fuchsia.intl.PropertyProvider": "fuchsia-pkg://fuchsia.com/intl_property_manager#meta/intl_property_manager.cmx" - }, - "system-services": [ - "fuchsia.boot.ReadOnlyLog" - ] - } - }, - "sandbox": { - "dev": [ - "null", - "zero" - ], - "features": [ - "isolated-persistent-storage", - "isolated-temp" - ], - "services": [ - "fuchsia.intl.PropertyProvider", - "fuchsia.logger.LogSink", - "fuchsia.process.Launcher", - "fuchsia.sys.Launcher", - "fuchsia.sys.Loader" - ] - } -}
diff --git a/build/config/fuchsia/test/network.shard.test-cml b/build/config/fuchsia/test/network.shard.test-cml new file mode 100644 index 0000000..1fd4fa7 --- /dev/null +++ b/build/config/fuchsia/test/network.shard.test-cml
@@ -0,0 +1,20 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + use: [ + { + directory: "root-ssl-certificates", + rights: [ "r*" ], + path: "/config/ssl", + }, + { + protocol: [ + "fuchsia.device.NameProvider", // Required by FDIO. + "fuchsia.net.interfaces.State", + "fuchsia.net.name.Lookup", + "fuchsia.posix.socket.Provider", + ], + }, + ], +}
diff --git a/build/config/fuchsia/test/network_capabilities.test-cmx b/build/config/fuchsia/test/network_capabilities.test-cmx deleted file mode 100644 index 54b9e41..0000000 --- a/build/config/fuchsia/test/network_capabilities.test-cmx +++ /dev/null
@@ -1,25 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "injected-services": { - "fuchsia.net.NameLookup": "fuchsia-pkg://fuchsia.com/dns-resolver#meta/dns-resolver.cmx", - "fuchsia.net.interfaces.State": "fuchsia-pkg://fuchsia.com/netstack#meta/netstack.cmx", - "fuchsia.posix.socket.Provider": "fuchsia-pkg://fuchsia.com/netstack#meta/netstack.cmx" - } - }, - "system-services": [ - "fuchsia.device.NameProvider" - ] - }, - "sandbox": { - "features": [ - "root-ssl-certificates" - ], - "services": [ - "fuchsia.device.NameProvider", - "fuchsia.net.NameLookup", - "fuchsia.net.interfaces.State", - "fuchsia.posix.socket.Provider" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/platform_video_codecs.shard.test-cml b/build/config/fuchsia/test/platform_video_codecs.shard.test-cml new file mode 100644 index 0000000..13b5a1b --- /dev/null +++ b/build/config/fuchsia/test/platform_video_codecs.shard.test-cml
@@ -0,0 +1,48 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + include: [ + "//build/config/fuchsia/test/sysmem.shard.test-cml", + ], + children: [ + { + // Run an isolated instance of codec_factory so that tests can run on + // system images that don't run it. + name: "isolated_codec_factory", + url: "fuchsia-pkg://fuchsia.com/codec_factory#meta/codec_factory.cm", + }, + ], + offer: [ + { + protocol: [ + "fuchsia.logger.LogSink", + "fuchsia.sysinfo.SysInfo", + "fuchsia.sysmem.Allocator", + ], + from: "parent", + to: "#isolated_codec_factory", + }, + { + directory: "dev-mediacodec", + from: "parent", + to: "#isolated_codec_factory", + }, + { + directory: "dev-gpu", + from: "parent", + to: "#isolated_codec_factory", + }, + ], + use: [ + { + protocol: "fuchsia.mediacodec.CodecFactory", + from: "#isolated_codec_factory", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ "codec_factory" ], + }, + }, +}
diff --git a/build/config/fuchsia/test/present_view.shard.test-cml b/build/config/fuchsia/test/present_view.shard.test-cml new file mode 100644 index 0000000..4e15ad5 --- /dev/null +++ b/build/config/fuchsia/test/present_view.shard.test-cml
@@ -0,0 +1,42 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + children: [ + { + name: "isolated_a11y_manager", + url: "fuchsia-pkg://fuchsia.com/a11y-manager#meta/a11y-manager.cm", + }, + { + name: "isolated_text_manager", + url: "fuchsia-pkg://fuchsia.com/text_manager#meta/text_manager.cm", + }, + ], + offer: [ + { + protocol: "fuchsia.logger.LogSink", + from: "parent", + to: [ + "#isolated_a11y_manager", + "#isolated_text_manager", + ], + }, + ], + use: [ + { + protocol: [ + "fuchsia.ui.composition.Allocator", + "fuchsia.ui.composition.Flatland", + "fuchsia.ui.scenic.Scenic", + ], + }, + { + protocol: "fuchsia.accessibility.semantics.SemanticsManager", + from: "#isolated_a11y_manager", + }, + { + protocol: "fuchsia.ui.input3.Keyboard", + from: "#isolated_text_manager", + }, + ], +}
diff --git a/build/config/fuchsia/test/present_view_capabilities.test-cmx b/build/config/fuchsia/test/present_view_capabilities.test-cmx deleted file mode 100644 index 201c8b2..0000000 --- a/build/config/fuchsia/test/present_view_capabilities.test-cmx +++ /dev/null
@@ -1,24 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "injected-services": { - "fuchsia.accessibility.semantics.SemanticsManager": "fuchsia-pkg://fuchsia.com/a11y-manager#meta/a11y-manager.cmx", - "fuchsia.ui.input3.Keyboard": "fuchsia-pkg://fuchsia.com/ime_service#meta/ime_service.cmx", - }, - "system-services": [ - "fuchsia.sysmem.Allocator", - "fuchsia.ui.policy.Presenter", - "fuchsia.ui.scenic.Scenic" - ] - } - }, - "sandbox": { - "services": [ - "fuchsia.accessibility.semantics.SemanticsManager", - "fuchsia.sysmem.Allocator", - "fuchsia.ui.input3.Keyboard", - "fuchsia.ui.policy.Presenter", - "fuchsia.ui.scenic.Scenic" - ] - } -}
diff --git a/build/config/fuchsia/test/read_debug_data.test-cmx b/build/config/fuchsia/test/read_debug_data.test-cmx deleted file mode 100644 index b0c95b0..0000000 --- a/build/config/fuchsia/test/read_debug_data.test-cmx +++ /dev/null
@@ -1,7 +0,0 @@ -{ - "sandbox": { - "features": [ - "hub" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/sysmem.shard.test-cml b/build/config/fuchsia/test/sysmem.shard.test-cml new file mode 100644 index 0000000..8bebd99 --- /dev/null +++ b/build/config/fuchsia/test/sysmem.shard.test-cml
@@ -0,0 +1,10 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + use: [ + { + protocol: "fuchsia.sysmem.Allocator", + }, + ], +}
diff --git a/build/config/fuchsia/test/system_test_minimum.shard.test-cml b/build/config/fuchsia/test/system_test_minimum.shard.test-cml new file mode 100644 index 0000000..6efde20 --- /dev/null +++ b/build/config/fuchsia/test/system_test_minimum.shard.test-cml
@@ -0,0 +1,46 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + include: [ + "syslog/client.shard.cml", + ], + use: [ + { + directory: "config-data", + rights: [ "r*" ], + path: "/config/data", + }, + { + storage: "cache", + path: "/cache", + }, + { + storage: "custom_artifacts", + path: "/custom_artifacts", + }, + { + storage: "data", + path: "/data", + }, + { + storage: "tmp", + path: "/tmp", + }, + { + protocol: [ + "fuchsia.buildinfo.Provider", + "fuchsia.hwinfo.Product", + "fuchsia.intl.PropertyProvider", + "fuchsia.media.ProfileProvider", + "fuchsia.process.Launcher", + ], + }, + { + protocol: [ + "fuchsia.tracing.perfetto.ProducerConnector", + ], + availability: "optional", + }, + ], +}
diff --git a/build/config/fuchsia/test/test_fonts.shard.test-cml b/build/config/fuchsia/test/test_fonts.shard.test-cml new file mode 100644 index 0000000..6610e31 --- /dev/null +++ b/build/config/fuchsia/test/test_fonts.shard.test-cml
@@ -0,0 +1,37 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + children: [ + { + name: "test_fonts", + url: "fuchsia-pkg://fuchsia.com/fonts#meta/fonts.cm", + }, + ], + offer: [ + { + protocol: "fuchsia.logger.LogSink", + from: "parent", + to: "#test_fonts", + }, + { + directory: "pkg", + subdir: "test_fonts", + from: "framework", + to: "#test_fonts", + as: "config-data", + rights: [ "r*" ], + } + ], + use: [ + { + protocol: "fuchsia.fonts.Provider", + from: "#test_fonts", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ "fonts" ], + }, + }, +}
diff --git a/build/config/fuchsia/test/test_logger_capabilities.test-cmx b/build/config/fuchsia/test/test_logger_capabilities.test-cmx deleted file mode 100644 index 68b2a67..0000000 --- a/build/config/fuchsia/test/test_logger_capabilities.test-cmx +++ /dev/null
@@ -1,7 +0,0 @@ -{ - "sandbox": { - "services": [ - "fuchsia.logger.Log" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/test_ui_stack.shard.test-cml b/build/config/fuchsia/test/test_ui_stack.shard.test-cml new file mode 100644 index 0000000..102867c --- /dev/null +++ b/build/config/fuchsia/test/test_ui_stack.shard.test-cml
@@ -0,0 +1,48 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + include: [ "//build/config/fuchsia/test/sysmem.shard.test-cml" ], + children: [ + { + name: "test_ui_stack", + url: "fuchsia-pkg://fuchsia.com/flatland-scene-manager-test-ui-stack#meta/test-ui-stack.cm", + }, + ], + use: [ + { + protocol: [ + "fuchsia.accessibility.semantics.SemanticsManager", + "fuchsia.element.GraphicalPresenter", + "fuchsia.ui.composition.Allocator", + "fuchsia.ui.composition.Flatland", + "fuchsia.ui.input3.Keyboard", + "fuchsia.ui.scenic.Scenic", + ], + from: "#test_ui_stack", + }, + ], + offer: [ + { + storage: "tmp", + from: "parent", + to: "#test_ui_stack", + }, + { + protocol: [ + "fuchsia.logger.LogSink", + "fuchsia.scheduler.ProfileProvider", + "fuchsia.sysmem.Allocator", + "fuchsia.tracing.provider.Registry", + "fuchsia.vulkan.loader.Loader", + ], + from: "parent", + to: "#test_ui_stack", + }, + ], + facets: { + "fuchsia.test": { + "deprecated-allowed-packages": [ "flatland-scene-manager-test-ui-stack" ], + }, + }, +}
diff --git a/build/config/fuchsia/test/vulkan_capabilities.test-cmx b/build/config/fuchsia/test/vulkan_capabilities.test-cmx deleted file mode 100644 index 0436ffd..0000000 --- a/build/config/fuchsia/test/vulkan_capabilities.test-cmx +++ /dev/null
@@ -1,19 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "system-services": [ - "fuchsia.sysmem.Allocator", - "fuchsia.vulkan.loader.Loader" - ] - } - }, - "sandbox": { - "features": [ - "vulkan" - ], - "services": [ - "fuchsia.sysmem.Allocator", - "fuchsia.vulkan.loader.Loader" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/web_engine_required_capabilities.test-cmx b/build/config/fuchsia/test/web_engine_required_capabilities.test-cmx deleted file mode 100644 index 4cb61fe..0000000 --- a/build/config/fuchsia/test/web_engine_required_capabilities.test-cmx +++ /dev/null
@@ -1,25 +0,0 @@ -{ - "facets": { - "fuchsia.test": { - "injected-services": { - "fuchsia.fonts.Provider": "fuchsia-pkg://fuchsia.com/fonts#meta/fonts.cmx", - "fuchsia.memorypressure.Provider": "fuchsia-pkg://fuchsia.com/memory_monitor#meta/memory_monitor.cmx", - "fuchsia.web.ContextProvider": "fuchsia-pkg://fuchsia.com/web_engine#meta/context_provider.cmx", - }, - "system-services": [ - "fuchsia.device.NameProvider", - "fuchsia.scheduler.ProfileProvider", - "fuchsia.sysmem.Allocator" - ] - } - }, - "sandbox": { - "services": [ - "fuchsia.device.NameProvider", - "fuchsia.fonts.Provider", - "fuchsia.memorypressure.Provider", - "fuchsia.sysmem.Allocator", - "fuchsia.web.ContextProvider" - ] - } -} \ No newline at end of file
diff --git a/build/config/fuchsia/test/web_instance.shard.test-cml b/build/config/fuchsia/test/web_instance.shard.test-cml new file mode 100644 index 0000000..b996f4a --- /dev/null +++ b/build/config/fuchsia/test/web_instance.shard.test-cml
@@ -0,0 +1,21 @@ +// Copyright 2022 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +{ + include: [ + "//build/config/fuchsia/test/audio_output.shard.test-cml", + "//build/config/fuchsia/test/fonts.shard.test-cml", + "//build/config/fuchsia/test/mark_vmo_executable.shard.test-cml", + "//build/config/fuchsia/test/network.shard.test-cml", + "//build/config/fuchsia/test/platform_video_codecs.shard.test-cml", + "//build/config/fuchsia/test/test_ui_stack.shard.test-cml", + "vulkan/client.shard.cml", + ], + use: [ + { + protocol: [ + "fuchsia.memorypressure.Provider", + ], + }, + ], +}
diff --git a/build/config/gcc/BUILD.gn b/build/config/gcc/BUILD.gn index 154b259..147ebfc 100644 --- a/build/config/gcc/BUILD.gn +++ b/build/config/gcc/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -63,8 +63,9 @@ # configs += [ "//build/config/gcc:rpath_for_built_shared_libraries" ] # } config("rpath_for_built_shared_libraries") { - if (!is_android) { - # Note: Android doesn't support rpath. + if (!is_android && current_os != "aix" && !is_castos) { + # Note: Android, Aix don't support rpath. Chromecast has its own logic for + # setting the rpath in //build/config/chromecast. if (current_toolchain != default_toolchain || gcc_target_rpath == "") { ldflags = [ # Want to pass "\$". GN will re-escape as required for ninja.
diff --git a/build/config/get_host_byteorder.py b/build/config/get_host_byteorder.py index fc01d85..7cc0cdf 100755 --- a/build/config/get_host_byteorder.py +++ b/build/config/get_host_byteorder.py
@@ -1,11 +1,10 @@ -#!/usr/bin/env python -# Copyright 2017 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Get Byteorder of host architecture""" -from __future__ import print_function import sys
diff --git a/build/config/host_byteorder.gni b/build/config/host_byteorder.gni index 48a1a7f..1c3c72d 100644 --- a/build/config/host_byteorder.gni +++ b/build/config/host_byteorder.gni
@@ -1,4 +1,4 @@ -# Copyright (c) 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/BUILD.gn b/build/config/ios/BUILD.gn index c4cd317..863d1d0 100644 --- a/build/config/ios/BUILD.gn +++ b/build/config/ios/BUILD.gn
@@ -1,29 +1,14 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/ios/ios_sdk.gni") +import("//build/toolchain/apple/toolchain.gni") import("//build/toolchain/goma.gni") +import("//build/toolchain/rbe.gni") import("//build/toolchain/toolchain.gni") import("//build_overrides/build.gni") -declare_args() { - # Enabling this option makes clang compile to an intermediate - # representation ("bitcode"), and not to native code. This is preferred - # when including WebRTC in the apps that will be sent to Apple's App Store - # and mandatory for the apps that run on watchOS or tvOS. - # The option only works when building with Xcode (use_xcode_clang = true). - # Mimicking how Xcode handles it, the production builds (is_debug = false) - # get real bitcode sections added, while the debug builds (is_debug = true) - # only get bitcode-section "markers" added in them. - # NOTE: This option is ignored when building versions for the iOS simulator, - # where a part of libvpx is compiled from the assembly code written using - # Intel assembly syntax; Yasm / Nasm do not support emitting bitcode parts. - # That is not a limitation for now as Xcode mandates the presence of bitcode - # only when building bitcode-enabled projects for real devices (ARM CPUs). - enable_ios_bitcode = false -} - # This is included by reference in the //build/config/compiler config that # is applied to all targets. It is here to separate out the logic. config("compiler") { @@ -77,9 +62,22 @@ "5", ] - # Without this, the constructors and destructors of a C++ object inside - # an Objective C struct won't be called, which is very bad. - cflags_objcc = [ "-fobjc-call-cxx-cdtors" ] + cflags_objcc = [ + # Without this, the constructors and destructors of a C++ object inside + # an Objective C struct won't be called, which is very bad. + "-fobjc-call-cxx-cdtors", + + # When using -std=c++20 or higher, clang automatically returns true for + # `__has_feature(modules)` as it enables cxx modules. This is problematic + # because Objective-C code uses this to detect whether `@import` can be + # used (this feature is also named modules). + # + # Since Chromium does not yet enable cxx modules, nor clang modules, + # force disable the cxx modules, which cause `__has_features(modules)` + # to return false unless clang modules are explicitly enabled. + "-Xclang", + "-fno-cxx-modules", + ] ldflags = common_flags } @@ -94,7 +92,7 @@ # Rebase the value in that case since gn does not convert paths in compiler # flags (since it is not aware they are paths). _sdk_root = ios_sdk_path - if (use_system_xcode && use_goma) { + if (use_system_xcode && (use_goma || use_remoteexec)) { _sdk_root = rebase_path(ios_sdk_path, root_build_dir) } @@ -114,14 +112,13 @@ "-iframework", "$_sdk_root/System/iOSSupport/System/Library/Frameworks", ] - } - if (use_xcode_clang && enable_ios_bitcode && target_environment == "device") { - if (is_debug) { - common_flags += [ "-fembed-bitcode-marker" ] - } else { - common_flags += [ "-fembed-bitcode" ] - } + swiftflags += [ + "-isystem", + "$_sdk_root/System/iOSSupport/usr/include", + "-Fsystem", + "$_sdk_root/System/iOSSupport/System/Library/Frameworks", + ] } asmflags = common_flags @@ -163,30 +160,45 @@ ldflags = [ # Always load Objective-C categories and class. "-Wl,-ObjC", - - # Uses version 2 of Objective-C ABI. - "-Wl,-objc_abi_version,2", ] # The path to the Swift compatibility libraries (required to run code built # with version N of the SDK on older version of the OS) is relative to the - # toolchains directory and changes with the environment. - _swift_compatibility_libs_dir_prefix = "$ios_toolchains_path/usr/lib/swift" + # toolchains directory and changes with the environment when using the + # system toolchain. When using the hermetic swift toolchain instead, those + # libraries are relative to $swift_toolchain_path. + if (swift_toolchain_path == "") { + _swift_compatibility_libs_prefix = ios_toolchains_path + } else { + _swift_compatibility_libs_prefix = swift_toolchain_path + } + if (target_environment == "simulator") { - _swift_compatibility_libs_dir = - "$_swift_compatibility_libs_dir_prefix/iphonesimulator" + _swift_compatibility_libs_suffix = "iphonesimulator" } else if (target_environment == "device") { - _swift_compatibility_libs_dir = - "$_swift_compatibility_libs_dir_prefix/iphoneos" + _swift_compatibility_libs_suffix = "iphoneos" } else if (target_environment == "catalyst") { - _swift_compatibility_libs_dir = - "$_swift_compatibility_libs_dir_prefix/maccatalyst" + # The Swift compatibility libraries have changed location starting with + # Xcode 13.0, so check the version of Xcode when deciding which path to + # use. + if (xcode_version_int >= 1300) { + _swift_compatibility_libs_suffix = "macosx" + } else { + _swift_compatibility_libs_suffix = "maccatalyst" + } } lib_dirs = [ "$ios_sdk_path/usr/lib/swift", - _swift_compatibility_libs_dir, + "$_swift_compatibility_libs_prefix/usr/lib/swift/" + + "$_swift_compatibility_libs_suffix", ] + + # When building for catalyst, some Swift support libraries are in a + # different directory which needs to be added to the search path. + if (target_environment == "catalyst") { + lib_dirs += [ "$ios_sdk_path/System/iOSSupport/usr/lib/swift" ] + } } config("ios_shared_library_flags") { @@ -196,20 +208,33 @@ ] } -config("disable_implicit_retain_self_warning") { - cflags_objc = [ "-Wno-implicit-retain-self" ] - cflags_objcc = cflags_objc -} - config("xctest_config") { - framework_dirs = [ "$ios_sdk_platform_path/Developer/Library/Frameworks" ] + # Add some directories to the system framework search path to make + # them available to the compiler while silencing warnings in the + # framework headers. This is required for XCTest. + common_flags = [ + "-iframework", + rebase_path("$ios_sdk_platform_path/Developer/Library/Frameworks", + root_build_dir), + "-iframework", + rebase_path("$ios_sdk_path/Developer/Library/Frameworks", root_build_dir), + ] + cflags = common_flags + ldflags = common_flags + swiftflags = common_flags + include_dirs = [ "$ios_sdk_platform_path/Developer/usr/lib" ] + lib_dirs = [ "$ios_sdk_platform_path/Developer/usr/lib" ] frameworks = [ "Foundation.framework", "XCTest.framework", ] } +config("enable_swift_cxx_interop") { + swiftflags = [ "-enable-experimental-cxx-interop" ] +} + group("xctest") { public_configs = [ ":xctest_config" ] } @@ -229,7 +254,7 @@ # # To workaround this, add a target that pretends to create those files # (but does nothing). See https://crbug.com/1061487 for why this is needed. -if (use_system_xcode && use_goma) { +if (use_system_xcode && (use_goma || use_remoteexec)) { action("copy_xctrunner_app") { testonly = true script = "//build/noop.py" @@ -258,7 +283,10 @@ xcode_version, ] - if (use_system_xcode && use_goma) { - deps = [ ":copy_xctrunner_app" ] + # When running under ASan, the ASan runtime library must be packaged alongside + # the test runner binary. + deps = [ "//build/config/sanitizers:deps" ] + if (use_system_xcode && (use_goma || use_remoteexec)) { + deps += [ ":copy_xctrunner_app" ] } }
diff --git a/build/config/ios/Host-Info.plist b/build/config/ios/Host-Info.plist index 9f6f5de..6898c15 100644 --- a/build/config/ios/Host-Info.plist +++ b/build/config/ios/Host-Info.plist
@@ -9,7 +9,7 @@ <key>CFBundleExecutable</key> <string>${EXECUTABLE_NAME}</string> <key>CFBundleIdentifier</key> - <string>${IOS_BUNDLE_ID_PREFIX}.test.${EXECUTABLE_NAME:rfc1034identifier}</string> + <string>${BUNDLE_IDENTIFIER}</string> <key>CFBundleInfoDictionaryVersion</key> <string>6.0</string> <key>CFBundleName</key>
diff --git a/build/config/ios/Module-Info.plist b/build/config/ios/Module-Info.plist index d1bf77f..e1b0984 100644 --- a/build/config/ios/Module-Info.plist +++ b/build/config/ios/Module-Info.plist
@@ -7,7 +7,7 @@ <key>CFBundleExecutable</key> <string>${EXECUTABLE_NAME}</string> <key>CFBundleIdentifier</key> - <string>${IOS_BUNDLE_ID_PREFIX}.${MODULE_BUNDLE_ID:rfc1034identifier}</string> + <string>${BUNDLE_IDENTIFIER}</string> <key>CFBundleInfoDictionaryVersion</key> <string>6.0</string> <key>CFBundleName</key>
diff --git a/build/config/ios/asset_catalog.gni b/build/config/ios/asset_catalog.gni index 84dd92c..8695bf7 100644 --- a/build/config/ios/asset_catalog.gni +++ b/build/config/ios/asset_catalog.gni
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -11,7 +11,7 @@ # The create_bundle target requires that all asset catalogs are part of an # .xcasset bundle. This requirement comes from actool that only receives # the path to the .xcasset bundle directory and not to the individual -# .imageset directories. +# assets directories. # # The requirement is a bit problematic as it prevents compiling only a # subset of the asset catakig that are contained in a .xcasset. This template @@ -48,69 +48,63 @@ assert(defined(invoker.asset_type) && invoker.asset_type != "", "asset_type must be defined and not empty for $target_name") - if (is_fat_secondary_toolchain) { - group(target_name) { - public_deps = [ ":$target_name($primary_fat_toolchain_name)" ] - } - } else { - _copy_target_name = target_name + "__copy" - _data_target_name = target_name + _copy_target_name = target_name + "__copy" + _data_target_name = target_name - _sources = invoker.sources - _outputs = [] + _sources = invoker.sources + _outputs = [] - # The compilation of resources into Assets.car is enabled automatically - # by the "create_bundle" target if any of the "bundle_data" sources's - # path is in a .xcassets directory and matches one of the know asset - # catalog type. - _xcassets_dir = "$target_gen_dir/${target_name}.xcassets" - _output_dir = "$_xcassets_dir/" + - get_path_info(get_path_info(_sources[0], "dir"), "file") + # The compilation of resources into Assets.car is enabled automatically + # by the "create_bundle" target if any of the "bundle_data" sources's + # path is in a .xcassets directory and matches one of the know asset + # catalog type. + _xcassets_dir = "$target_gen_dir/${target_name}.xcassets" + _output_dir = "$_xcassets_dir/" + + get_path_info(get_path_info(_sources[0], "dir"), "file") - foreach(_source, invoker.sources) { - _dir = get_path_info(_source, "dir") - _outputs += [ "$_output_dir/" + get_path_info(_source, "file") ] + foreach(_source, invoker.sources) { + _dir = get_path_info(_source, "dir") + _outputs += [ "$_output_dir/" + get_path_info(_source, "file") ] - assert(get_path_info(_dir, "extension") == invoker.asset_type, - "$_source dirname must have .${invoker.asset_type} extension") - } + assert(get_path_info(_dir, "extension") == invoker.asset_type, + "$_source dirname must have .${invoker.asset_type} extension") + } - action(_copy_target_name) { - # Forward "deps", "public_deps" and "testonly" in case some of the - # source files are generated. - forward_variables_from(invoker, - [ - "deps", - "public_deps", - "testonly", - ]) + action(_copy_target_name) { + # Forward "deps", "public_deps" and "testonly" in case some of the + # source files are generated. + forward_variables_from(invoker, + [ + "deps", + "public_deps", + "testonly", + ]) - script = "//build/config/ios/hardlink.py" + script = "//build/config/ios/hardlink.py" - visibility = [ ":$_data_target_name" ] - sources = _sources - outputs = _outputs + [ _xcassets_dir ] + visibility = [ ":$_data_target_name" ] + sources = _sources + outputs = _outputs + [ _xcassets_dir ] - args = [ - rebase_path(get_path_info(_sources[0], "dir"), root_build_dir), - rebase_path(_output_dir, root_build_dir), - ] - } + args = [ + rebase_path(get_path_info(_sources[0], "dir"), root_build_dir), + rebase_path(_output_dir, root_build_dir), + ] + } - bundle_data(_data_target_name) { - forward_variables_from(invoker, - "*", - [ - "deps", - "outputs", - "public_deps", - "sources", - ]) + bundle_data(_data_target_name) { + forward_variables_from(invoker, + "*", + [ + "deps", + "outputs", + "public_deps", + "sources", + ]) - sources = _outputs - outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] - public_deps = [ ":$_copy_target_name" ] - } + sources = _outputs + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + public_deps = [ ":$_copy_target_name" ] } } @@ -148,3 +142,9 @@ asset_type = "launchimage" } } +template("symbolset") { + asset_catalog(target_name) { + forward_variables_from(invoker, "*", [ "asset_type" ]) + asset_type = "symbolset" + } +}
diff --git a/build/config/ios/bundle_data_from_filelist.gni b/build/config/ios/bundle_data_from_filelist.gni new file mode 100644 index 0000000..763dc86 --- /dev/null +++ b/build/config/ios/bundle_data_from_filelist.gni
@@ -0,0 +1,24 @@ +# Copyright 2023 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +assert(current_os == "ios") + +template("bundle_data_from_filelist") { + assert(defined(invoker.filelist_name), "Requires setting filelist_name") + + _filelist_content = read_file(invoker.filelist_name, "list lines") + bundle_data(target_name) { + forward_variables_from(invoker, + "*", + [ + "filelist_name", + "sources", + ]) + sources = filter_exclude(_filelist_content, [ "#*" ]) + if (!defined(outputs)) { + outputs = [ "{{bundle_resources_dir}}/" + + "{{source_root_relative_dir}}/{{source_file_part}}" ] + } + } +}
diff --git a/build/config/ios/codesign.py b/build/config/ios/codesign.py index 15d25a7..fd96f31 100644 --- a/build/config/ios/codesign.py +++ b/build/config/ios/codesign.py
@@ -1,8 +1,7 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import argparse import codecs @@ -14,6 +13,7 @@ import plistlib import shutil import subprocess +import stat import sys import tempfile @@ -226,6 +226,8 @@ def Install(self, installation_path): """Copies mobile provisioning profile info to |installation_path|.""" shutil.copy2(self.path, installation_path) + st = os.stat(installation_path) + os.chmod(installation_path, st.st_mode | stat.S_IWUSR) class Entitlements(object): @@ -273,7 +275,8 @@ plistlib.dump(self._data, fp) -def FindProvisioningProfile(bundle_identifier, required): +def FindProvisioningProfile(provisioning_profile_paths, bundle_identifier, + required): """Finds mobile provisioning profile to use to sign bundle. Args: @@ -283,8 +286,9 @@ The ProvisioningProfile object that can be used to sign the Bundle object or None if no matching provisioning profile was found. """ - provisioning_profile_paths = glob.glob( - os.path.join(GetProvisioningProfilesDir(), '*.mobileprovision')) + if not provisioning_profile_paths: + provisioning_profile_paths = glob.glob( + os.path.join(GetProvisioningProfilesDir(), '*.mobileprovision')) # Iterate over all installed mobile provisioning profiles and filter those # that can be used to sign the bundle, ignoring expired ones. @@ -304,8 +308,8 @@ if not valid_provisioning_profiles: if required: sys.stderr.write( - 'Error: no mobile provisioning profile found for "%s".\n' % - bundle_identifier) + 'Error: no mobile provisioning profile found for "%s" in %s.\n' % + (bundle_identifier, provisioning_profile_paths)) sys.exit(1) return None @@ -401,7 +405,7 @@ # Invoke the plist_compiler script. It needs to be a python script. subprocess.check_call([ - 'python', + 'python3', plist_compiler, 'merge', '-f', @@ -459,6 +463,14 @@ parser.add_argument( '--plist-compiler-path', '-P', action='store', help='path to the plist compiler script (for --partial-info-plist)') + parser.add_argument( + '--mobileprovision', + '-m', + action='append', + default=[], + dest='mobileprovision_files', + help='list of mobileprovision files to use. If empty, uses the files ' + + 'in $HOME/Library/MobileDevice/Provisioning Profiles') parser.set_defaults(no_signature=False) @staticmethod @@ -554,7 +566,8 @@ # provisioning is found). provisioning_profile_required = args.identity != '-' provisioning_profile = FindProvisioningProfile( - bundle.identifier, provisioning_profile_required) + args.mobileprovision_files, bundle.identifier, + provisioning_profile_required) if provisioning_profile and args.platform != 'iphonesimulator': provisioning_profile.Install(embedded_provisioning_profile) @@ -629,12 +642,21 @@ parser.add_argument( '--info-plist', '-p', required=True, help='path to the bundle Info.plist') + parser.add_argument( + '--mobileprovision', + '-m', + action='append', + default=[], + dest='mobileprovision_files', + help='set of mobileprovision files to use. If empty, uses the files ' + + 'in $HOME/Library/MobileDevice/Provisioning Profiles') @staticmethod def _Execute(args): info_plist = LoadPlistFile(args.info_plist) bundle_identifier = info_plist['CFBundleIdentifier'] - provisioning_profile = FindProvisioningProfile(bundle_identifier, False) + provisioning_profile = FindProvisioningProfile(args.mobileprovision_files, + bundle_identifier, False) entitlements = GenerateEntitlements( args.entitlements_path, provisioning_profile, bundle_identifier) entitlements.WriteTo(args.path) @@ -652,11 +674,20 @@ '-b', required=True, help='bundle identifier') + parser.add_argument( + '--mobileprovision', + '-m', + action='append', + default=[], + dest='mobileprovision_files', + help='set of mobileprovision files to use. If empty, uses the files ' + + 'in $HOME/Library/MobileDevice/Provisioning Profiles') @staticmethod def _Execute(args): provisioning_profile_info = {} - provisioning_profile = FindProvisioningProfile(args.bundle_id, False) + provisioning_profile = FindProvisioningProfile(args.mobileprovision_files, + args.bundle_id, False) for key in ('team_identifier', 'name'): if provisioning_profile: provisioning_profile_info[key] = getattr(provisioning_profile, key)
diff --git a/build/config/ios/compile_ib_files.py b/build/config/ios/compile_ib_files.py index 84781c1..e420016 100644 --- a/build/config/ios/compile_ib_files.py +++ b/build/config/ios/compile_ib_files.py
@@ -1,8 +1,7 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import argparse import logging
diff --git a/build/config/ios/compile_xcassets_unittests.py b/build/config/ios/compile_xcassets_unittests.py index 7655df8..8537e4e 100644 --- a/build/config/ios/compile_xcassets_unittests.py +++ b/build/config/ios/compile_xcassets_unittests.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/config.gni b/build/config/ios/config.gni index b25ecd9..c5c10c3 100644 --- a/build/config/ios/config.gni +++ b/build/config/ios/config.gni
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/dummy.py b/build/config/ios/dummy.py index b23b7da..e88c788 100644 --- a/build/config/ios/dummy.py +++ b/build/config/ios/dummy.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/find_signing_identity.py b/build/config/ios/find_signing_identity.py index d508e2b..37b3284 100644 --- a/build/config/ios/find_signing_identity.py +++ b/build/config/ios/find_signing_identity.py
@@ -1,8 +1,7 @@ -# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import argparse import os @@ -45,7 +44,8 @@ def FindValidIdentity(pattern): """Find all identities matching the pattern.""" lines = list(l.strip() for l in ListIdentities().splitlines()) - # Look for something like "2) XYZ "iPhone Developer: Name (ABC)"" + # Look for something like + # 1) 123ABC123ABC123ABC****** "iPhone Developer: DeveloperName (Team)" regex = re.compile('[0-9]+\) ([A-F0-9]+) "([^"(]*) \(([^)"]*)\)"') result = [] @@ -53,8 +53,9 @@ res = regex.match(line) if res is None: continue - if pattern is None or pattern in res.group(2): - result.append(Identity(*res.groups())) + identifier, developer_name, team = res.groups() + if pattern is None or pattern in '%s (%s)' % (developer_name, team): + result.append(Identity(identifier, developer_name, team)) return result
diff --git a/build/config/ios/generate_umbrella_header.py b/build/config/ios/generate_umbrella_header.py index 8547e18..943c49c 100644 --- a/build/config/ios/generate_umbrella_header.py +++ b/build/config/ios/generate_umbrella_header.py
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/hardlink.py b/build/config/ios/hardlink.py index 38f60d4..7f1be59 100644 --- a/build/config/ios/hardlink.py +++ b/build/config/ios/hardlink.py
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/ios_sdk.gni b/build/config/ios/ios_sdk.gni index fbff8b4..1417469 100644 --- a/build/config/ios/ios_sdk.gni +++ b/build/config/ios/ios_sdk.gni
@@ -1,11 +1,11 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/ios/config.gni") import("//build/config/ios/ios_sdk_overrides.gni") import("//build/toolchain/goma.gni") -import("//build/toolchain/goma.gni") +import("//build/toolchain/rbe.gni") import("//build/toolchain/toolchain.gni") import("//build_overrides/build.gni") @@ -46,42 +46,16 @@ # Prefix for CFBundleIdentifier property of iOS bundles (correspond to the # "Organization Identifier" in Xcode). Code signing will fail if no mobile # provisioning for the selected code signing identify support that prefix. - ios_app_bundle_id_prefix = "org.chromium" + ios_app_bundle_id_prefix = "org.chromium.ost" - # If non-empty, this list must contain valid cpu architecture, and the final - # build will be a multi-architecture build (aka fat build) supporting the - # main $target_cpu architecture and all of $additional_target_cpus. - # - # For example to build an application that will run on both arm64 and armv7 - # devices, you would use the following in args.gn file when running "gn args": - # - # target_os = "ios" - # target_cpu = "arm64" - # additional_target_cpus = [ "arm" ] - # - # You can also pass the value via "--args" parameter for "gn gen" command by - # using the syntax --args='additional_target_cpus=["arm"] target_cpu="arm64"'. - additional_target_cpus = [] + # Paths to the mobileprovision files for the chosen code signing + # identity description and app bundle id prefix. + ios_mobileprovision_files = [] + + # Set to true if all test apps should use the same bundle id. + ios_use_shared_bundle_id_for_test_apps = true } -declare_args() { - # This variable is set by the toolchain. It is set to true if the toolchain - # is a secondary toolchain as part of a "fat" build. - is_fat_secondary_toolchain = false - - # This variable is set by the toolchain. It is the name of the primary - # toolchain for the fat build (could be current_toolchain). - primary_fat_toolchain_name = "" -} - -# Official builds may not use goma. -assert(!(use_goma && is_chrome_branded && is_official_build && - target_cpu == "arm64"), - "goma use is forbidden for official iOS builds.") - -assert(custom_toolchain == "" || additional_target_cpus == [], - "cannot define both custom_toolchain and additional_target_cpus") - # If codesigning is enabled, use must configure either a codesigning identity # or a filter to automatically select the codesigning identity. if (target_environment == "device" && ios_enable_code_signing) { @@ -94,25 +68,6 @@ "pattern to match the identity to use).") } -# Initialize additional_toolchains from additional_target_cpus. Assert here -# that the list does not contains $target_cpu nor duplicates as this would -# cause weird errors during the build. -additional_toolchains = [] -if (additional_target_cpus != []) { - foreach(_additional_target_cpu, additional_target_cpus) { - assert(_additional_target_cpu != target_cpu, - "target_cpu must not be listed in additional_target_cpus") - - _toolchain = "//build/toolchain/ios:ios_clang_${_additional_target_cpu}_fat" - foreach(_additional_toolchain, additional_toolchains) { - assert(_toolchain != _additional_toolchain, - "additional_target_cpus must not contains duplicate values") - } - - additional_toolchains += [ _toolchain ] - } -} - if (ios_sdk_path == "") { # Compute default target. if (target_environment == "simulator") { @@ -139,10 +94,12 @@ ios_sdk_developer_dir, ] } - if (use_system_xcode && use_goma) { + if (use_system_xcode && (use_goma || use_remoteexec)) { ios_sdk_info_args += [ "--create_symlink_at", "sdk/xcode_links", + "--root_build_dir", + root_build_dir, ] } script_name = "//build/config/apple/sdk_info.py" @@ -183,3 +140,8 @@ "trim string") } } + +if (ios_use_shared_bundle_id_for_test_apps) { + shared_bundle_id_for_test_apps = + "$ios_app_bundle_id_prefix.chrome.unittests.dev" +}
diff --git a/build/config/ios/ios_sdk_overrides.gni b/build/config/ios/ios_sdk_overrides.gni index bd990bc..a2373c6 100644 --- a/build/config/ios/ios_sdk_overrides.gni +++ b/build/config/ios/ios_sdk_overrides.gni
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,11 +7,11 @@ declare_args() { # Version of iOS that we're targeting. - ios_deployment_target = "12.2" + ios_deployment_target = "15.0" } # Always assert that ios_deployment_target is used on non-iOS platforms to # prevent unused args warnings. if (!is_ios) { - assert(ios_deployment_target == "12.2" || true) + assert(ios_deployment_target == "15.0" || true) }
diff --git a/build/config/ios/ios_test_runner_wrapper.gni b/build/config/ios/ios_test_runner_wrapper.gni index 8911071..378323c 100644 --- a/build/config/ios/ios_test_runner_wrapper.gni +++ b/build/config/ios/ios_test_runner_wrapper.gni
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -61,6 +61,8 @@ _rebased_mac_toolchain = rebase_path("//mac_toolchain", root_build_dir) _rebased_xcode_path = rebase_path("//Xcode.app", root_build_dir) + _rebased_ios_runtime_cache_prefix = + rebase_path("//Runtime-ios-", root_build_dir) # --out-dir argument is specified in gn_isolate_map.pyl because # ${ISOLATED_OUTDIR} doesn't get resolved through this wrapper. @@ -69,6 +71,8 @@ "@WrappedPath(${_rebased_xcode_path})", "--mac-toolchain-cmd", "@WrappedPath(${_rebased_mac_toolchain})", + "--runtime-cache-prefix", + "@WrappedPath(${_rebased_ios_runtime_cache_prefix})", ] # Default retries to 3 @@ -89,6 +93,13 @@ "${shards}", ] + if (xcode_version_int >= 1400) { + executable_args += [ + "--readline-timeout", + "600", + ] + } + data_deps = [ "//testing:test_scripts_shared" ] if (defined(invoker.data_deps)) { data_deps += invoker.data_deps @@ -120,19 +131,21 @@ _wrapper_output_name = wrapper_output_name } - # Test targets may attempt to generate multiple wrappers for a suite with - # multiple different toolchains when running with additional_target_cpus. - # Generate the wrapper script into root_out_dir rather than root_build_dir - # to ensure those wrappers are distinct. - wrapper_script = "${root_out_dir}/bin/${_wrapper_output_name}" + wrapper_script = "${root_build_dir}/bin/${_wrapper_output_name}" data = [] if (defined(invoker.data)) { data += invoker.data } data += [ - "//.vpython", "//ios/build/bots/scripts/", + "//ios/build/bots/scripts/plugin", + + # gRPC interface for iOS test plugin + "//ios/testing/plugin", + + # Variations test utilities used by variations_runner script. + "//testing/scripts/variations_seed_access_helper.py", "//testing/test_env.py", ] }
diff --git a/build/config/ios/ios_test_runner_xcuitest.gni b/build/config/ios/ios_test_runner_xcuitest.gni new file mode 100644 index 0000000..6aeb08b --- /dev/null +++ b/build/config/ios/ios_test_runner_xcuitest.gni
@@ -0,0 +1,72 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build/config/ios/ios_test_runner_wrapper.gni") +import("//build/config/ios/rules.gni") + +# ios_test_runner_xcuitest are just ios_xcuitest_test with an +# ios_test_runner_wrapper. Currently used by Crashpad tests, which do not depend +# on EG2 (and therefore do not use ios_eg2_test) +template("ios_test_runner_xcuitest") { + assert(defined(invoker.xcode_test_application_name), + "xcode_test_application_name must be defined for $target_name") + assert( + defined(invoker.deps), + "deps must be defined for $target_name to include at least one xctest" + + "file.") + + _target_name = target_name + _test_target = "${target_name}_test" + ios_xcuitest_test(_test_target) { + forward_variables_from(invoker, + [ + "xcode_test_application_name", + "xctest_bundle_principal_class", + "bundle_deps", + "deps", + "data_deps", + ]) + + # TODO(crbug.com/1056328) Because we change the target name, the subnodes + # are going to append with the _test in the naming, which won't be backwards + # compatible during migration from iOS recipe to Chromium. + output_name = "${_target_name}" + } + + ios_test_runner_wrapper(target_name) { + forward_variables_from(invoker, + [ + "data", + "data_deps", + "deps", + "executable_args", + "retries", + "shards", + "xcode_test_application_name", + ]) + _root_build_dir = rebase_path("${root_build_dir}", root_build_dir) + + if (!defined(data_deps)) { + data_deps = [] + } + + # Include the top ios_test_runner_xcuitest target, and the host app + data_deps += [ ":${_test_target}" ] + + if (!defined(executable_args)) { + executable_args = [] + } + + # The xcuitest module is bundled as *-Runner.app, while the host app is + # bundled as *.app. + executable_args += [ + "--app", + "@WrappedPath(${_root_build_dir}/${target_name}-Runner.app)", + ] + executable_args += [ + "--host-app", + "@WrappedPath(${_root_build_dir}/${xcode_test_application_name}.app)", + ] + } +}
diff --git a/build/config/ios/resources/XCTRunnerAddition+Info.plist b/build/config/ios/resources/XCTRunnerAddition+Info.plist index cf9463f..ed26f55 100644 --- a/build/config/ios/resources/XCTRunnerAddition+Info.plist +++ b/build/config/ios/resources/XCTRunnerAddition+Info.plist
@@ -3,7 +3,7 @@ <plist version="1.0"> <dict> <key>CFBundleIdentifier</key> - <string>com.apple.test.${EXECUTABLE_NAME}</string> + <string>${BUNDLE_IDENTIFIER}</string> <key>CFBundleName</key> <string>${PRODUCT_NAME}</string> <key>CFBundleExecutable</key>
diff --git a/build/config/ios/rules.gni b/build/config/ios/rules.gni index a572548..c6d4092 100644 --- a/build/config/ios/rules.gni +++ b/build/config/ios/rules.gni
@@ -1,23 +1,16 @@ -# Copyright 2015 The Chromium Authors. All rights reserved. +# Copyright 2015 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/apple/apple_info_plist.gni") import("//build/config/apple/symbols.gni") +import("//build/config/compiler/compiler.gni") import("//build/config/ios/ios_sdk.gni") import("//build/toolchain/goma.gni") +import("//build/toolchain/rbe.gni") import("//build/toolchain/toolchain.gni") import("//build_overrides/build.gni") -declare_args() { - # Set to true if an Xcode project is generated for this build. Set this to - # false if you do not plan to run `gn gen --ide=xcode` in this directory. - # This will speed up the generation at the cost of generating an invalid - # Xcode project if `gn gen --ide=xcode` is used. Defaults to true (favor - # correctness over speed). - ios_set_attributes_for_xcode_project_generation = true -} - # Constants corresponding to the bundle type identifiers use application, # application extension, XCTest and XCUITest targets respectively. _ios_xcode_app_bundle_id = "com.apple.product-type.application" @@ -25,110 +18,6 @@ _ios_xcode_xctest_bundle_id = "com.apple.product-type.bundle.unit-test" _ios_xcode_xcuitest_bundle_id = "com.apple.product-type.bundle.ui-testing" -# Invokes lipo on multiple arch-specific binaries to create a fat binary. -# -# Arguments -# -# arch_binary_target -# name of the target generating the arch-specific binaries, they must -# be named $target_out_dir/$toolchain_cpu/$arch_binary_output. -# -# arch_binary_output -# (optional, defaults to the name of $arch_binary_target) base name of -# the arch-specific binary generated by arch_binary_target. -# -# output_name -# (optional, defaults to $target_name) base name of the target output, -# the full path will be $target_out_dir/$output_name. -# -# configs -# (optional) a list of configurations, this is used to check whether -# the binary should be stripped, when "enable_stripping" is true. -# -template("lipo_binary") { - assert(defined(invoker.arch_binary_target), - "arch_binary_target must be defined for $target_name") - - _target_name = target_name - _output_name = target_name - if (defined(invoker.output_name)) { - _output_name = invoker.output_name - } - - _all_target_cpu = [ current_cpu ] + additional_target_cpus - _all_toolchains = [ current_toolchain ] + additional_toolchains - - _arch_binary_target = invoker.arch_binary_target - _arch_binary_output = get_label_info(_arch_binary_target, "name") - if (defined(invoker.arch_binary_output)) { - _arch_binary_output = invoker.arch_binary_output - } - - action(_target_name) { - forward_variables_from(invoker, - "*", - [ - "arch_binary_output", - "arch_binary_target", - "configs", - "output_name", - ]) - - script = "//build/toolchain/apple/linker_driver.py" - - # http://crbug.com/762840. Fix for bots running out of memory. - pool = "//build/toolchain:link_pool($default_toolchain)" - - outputs = [ "$target_out_dir/$_output_name" ] - - deps = [] - _index = 0 - inputs = [] - foreach(_cpu, _all_target_cpu) { - _toolchain = _all_toolchains[_index] - _index = _index + 1 - - inputs += - [ get_label_info("$_arch_binary_target($_toolchain)", - "target_out_dir") + "/$_cpu/$_arch_binary_output" ] - - deps += [ "$_arch_binary_target($_toolchain)" ] - } - - args = [ - "xcrun", - "lipo", - "-create", - "-output", - rebase_path("$target_out_dir/$_output_name", root_build_dir), - ] + rebase_path(inputs, root_build_dir) - - if (enable_dsyms) { - _dsyms_output_dir = "$root_out_dir/$_output_name.dSYM" - outputs += [ - "$_dsyms_output_dir/", - "$_dsyms_output_dir/Contents/Info.plist", - "$_dsyms_output_dir/Contents/Resources/DWARF/$_output_name", - ] - args += [ "-Wcrl,dsym," + rebase_path("$root_out_dir/.", root_build_dir) ] - if (!use_xcode_clang) { - args += [ "-Wcrl,dsymutilpath," + - rebase_path("//tools/clang/dsymutil/bin/dsymutil", - root_build_dir) ] - } - } - - if (enable_stripping) { - args += [ "-Wcrl,strip,-x,-S" ] - if (save_unstripped_output) { - outputs += [ "$root_out_dir/$_output_name.unstripped" ] - args += [ "-Wcrl,unstripped," + - rebase_path("$root_out_dir/.", root_build_dir) ] - } - } - } -} - # Wrapper around create_bundle taking care of code signature settings. # # Arguments @@ -300,14 +189,6 @@ _enable_code_signing = invoker.enable_code_signing } - if (!ios_set_attributes_for_xcode_project_generation) { - not_needed(invoker, - [ - "xcode_product_bundle_id", - "xcode_extra_attributes", - ]) - } - create_bundle(_target_name) { forward_variables_from(invoker, [ @@ -344,35 +225,25 @@ public_deps = [] } - if (ios_set_attributes_for_xcode_project_generation) { - _xcode_product_bundle_id = "" - if (defined(invoker.xcode_product_bundle_id)) { - _xcode_product_bundle_id = invoker.xcode_product_bundle_id - } + _bundle_identifier = "" + if (defined(invoker.xcode_product_bundle_id)) { + _bundle_identifier = invoker.xcode_product_bundle_id + assert(_bundle_identifier == string_replace(_bundle_identifier, "_", "-"), + "$target_name: bundle_identifier does not respect rfc1034: " + + _bundle_identifier) + } - if (_xcode_product_bundle_id != "") { - _ios_provisioning_profile_info = - exec_script("//build/config/ios/codesign.py", - [ - "find-provisioning-profile", - "-b=" + _xcode_product_bundle_id, - ], - "json") - } + xcode_extra_attributes = { + IPHONEOS_DEPLOYMENT_TARGET = ios_deployment_target + PRODUCT_BUNDLE_IDENTIFIER = _bundle_identifier + CODE_SIGNING_REQUIRED = "NO" + CODE_SIGNING_ALLOWED = "NO" + CODE_SIGN_IDENTITY = "" + DONT_GENERATE_INFOPLIST_FILE = "YES" - xcode_extra_attributes = { - IPHONEOS_DEPLOYMENT_TARGET = ios_deployment_target - if (_xcode_product_bundle_id != "") { - CODE_SIGN_IDENTITY = "iPhone Developer" - DEVELOPMENT_TEAM = _ios_provisioning_profile_info.team_identifier - PRODUCT_BUNDLE_IDENTIFIER = _xcode_product_bundle_id - PROVISIONING_PROFILE_SPECIFIER = _ios_provisioning_profile_info.name - } - - # If invoker has defined extra attributes, they override the defaults. - if (defined(invoker.xcode_extra_attributes)) { - forward_variables_from(invoker.xcode_extra_attributes, "*") - } + # If invoker has defined extra attributes, they override the defaults. + if (defined(invoker.xcode_extra_attributes)) { + forward_variables_from(invoker.xcode_extra_attributes, "*") } } @@ -439,6 +310,11 @@ "-i=" + ios_code_signing_identity, "-b=" + rebase_path(_bundle_binary_path, root_build_dir), ] + foreach(mobileprovision, ios_mobileprovision_files) { + code_signing_args += + [ "-m=" + rebase_path(mobileprovision, root_build_dir) ] + } + code_signing_sources += ios_mobileprovision_files if (_enable_entitlements) { code_signing_args += [ "-e=" + rebase_path(_entitlements_path, root_build_dir) ] @@ -456,7 +332,7 @@ # rebase_path here unless using Goma RBE and system Xcode (as in that # case the system framework are found via a symlink in root_build_dir). foreach(_framework, invoker.extra_system_frameworks) { - if (use_system_xcode && use_goma) { + if (use_system_xcode && (use_goma || use_remoteexec)) { _framework_path = rebase_path(_framework, root_build_dir) } else { _framework_path = _framework @@ -525,11 +401,7 @@ apple_info_plist(target_name) { format = "binary1" - extra_substitutions = [] - if (defined(invoker.extra_substitutions)) { - extra_substitutions = invoker.extra_substitutions - } - extra_substitutions += [ + extra_substitutions = [ "IOS_BUNDLE_ID_PREFIX=$ios_app_bundle_id_prefix", "IOS_PLATFORM_BUILD=$ios_platform_build", "IOS_PLATFORM_NAME=$ios_sdk_name", @@ -542,6 +414,9 @@ "XCODE_BUILD=$xcode_build", "XCODE_VERSION=$xcode_version", ] + if (defined(invoker.extra_substitutions)) { + extra_substitutions += invoker.extra_substitutions + } plist_templates = [ "//build/config/ios/BuildInfo.plist", _info_plist, @@ -612,10 +487,11 @@ # variant with the same binary but the correct bundle_deps, the bundle # at $target_out_dir/$output_name will be a copy of the first variant. # -# xcode_product_bundle_id: -# (optional) string, the bundle ID that will be added in the XCode -# attributes to enable some features when debugging (e.g. MetricKit). -# defaults to "$ios_app_bundle_id_prefix.$output_name". +# bundle_identifier: +# (optional) string, value of CFBundleIdentifier in the application +# Info.plist, defaults to "$ios_app_bundle_id_prefix.$output_name" +# if omitted. Will be used to set BUNDLE_IDENTIFIER when generating +# the application Info.plist # # For more information, see "gn help executable". template("ios_app_bundle") { @@ -625,32 +501,21 @@ _output_name = invoker.output_name } - _primary_toolchain = current_toolchain - if (is_fat_secondary_toolchain) { - _primary_toolchain = primary_fat_toolchain_name - } - assert( !defined(invoker.bundle_extension), "bundle_extension must not be set for ios_app_bundle template for $target_name") - _xcode_product_bundle_id = "$ios_app_bundle_id_prefix.$_output_name" - if (defined(invoker.xcode_product_bundle_id)) { - _xcode_product_bundle_id = invoker.xcode_product_bundle_id - _xcode_product_bundle_id = - "$ios_app_bundle_id_prefix.$_xcode_product_bundle_id" - } else if (defined(invoker.bundle_id)) { - _xcode_product_bundle_id = invoker.bundle_id + if (defined(invoker.bundle_identifier)) { + _bundle_identifier = invoker.bundle_identifier + assert(_bundle_identifier == string_replace(_bundle_identifier, "_", "-"), + "$target_name: bundle_identifier does not respect rfc1034: " + + _bundle_identifier) + } else { + # Bundle identifier should respect rfc1034, so replace "_" with "-". + _bundle_identifier = + "$ios_app_bundle_id_prefix." + string_replace(_output_name, "_", "-") } - # Bundle ID should respect rfc1034 and replace _ with -. - _xcode_product_bundle_id = - string_replace("$_xcode_product_bundle_id", "_", "-") - - _arch_executable_source = _target_name + "_arch_executable_sources" - _arch_executable_target = _target_name + "_arch_executable" - _lipo_executable_target = _target_name + "_executable" - if (defined(invoker.variants) && invoker.variants != []) { _variants = [] @@ -687,35 +552,11 @@ _default_variant = _variants[0] - source_set(_arch_executable_source) { - forward_variables_from(invoker, - "*", - [ - "bundle_deps", - "bundle_deps_filter", - "bundle_extension", - "enable_code_signing", - "entitlements_path", - "entitlements_target", - "extra_substitutions", - "extra_system_frameworks", - "info_plist", - "info_plist_target", - "output_name", - "product_type", - "visibility", - "xcode_extra_attributes", - ]) - - visibility = [ ":$_arch_executable_target" ] - } - - if (!is_fat_secondary_toolchain || target_environment == "simulator") { - _generate_entitlements_target = _target_name + "_gen_entitlements" - _generate_entitlements_output = - get_label_info(":$_generate_entitlements_target($_primary_toolchain)", - "target_out_dir") + "/$_output_name.xcent" - } + _executable_target = _target_name + "_executable" + _generate_entitlements_target = _target_name + "_gen_entitlements" + _generate_entitlements_output = + get_label_info(":$_generate_entitlements_target", "target_out_dir") + + "/$_output_name.xcent" _product_type = _ios_xcode_app_bundle_id if (defined(invoker.product_type)) { @@ -732,7 +573,7 @@ _is_app_bundle = _product_type == _ios_xcode_app_bundle_id - executable(_arch_executable_target) { + executable(_executable_target) { forward_variables_from(invoker, "*", [ @@ -748,28 +589,25 @@ "info_plist_target", "output_name", "product_type", - "sources", "visibility", "xcode_extra_attributes", ]) - visibility = [ ":$_lipo_executable_target($_primary_toolchain)" ] - if (is_fat_secondary_toolchain) { - visibility += [ ":$_target_name" ] + visibility = [] + foreach(_variant, _variants) { + visibility += [ ":${_variant.target_name}" ] } - if (!defined(deps)) { - deps = [] - } - deps += [ ":$_arch_executable_source" ] - if (!defined(frameworks)) { frameworks = [] } frameworks += [ "UIKit.framework" ] if (target_environment == "simulator") { - deps += [ ":$_generate_entitlements_target($_primary_toolchain)" ] + if (!defined(deps)) { + deps = [] + } + deps += [ ":$_generate_entitlements_target" ] if (!defined(inputs)) { inputs = [] @@ -785,196 +623,170 @@ output_name = _output_name output_prefix_override = true - output_dir = "$target_out_dir/$current_cpu" + output_dir = target_out_dir } - if (is_fat_secondary_toolchain) { - # For fat builds, only the default toolchain will generate an application - # bundle. For the other toolchains, the template is only used for building - # the arch-specific binary, thus the default target is just a group(). + _generate_info_plist = target_name + "_generate_info_plist" + ios_info_plist(_generate_info_plist) { + forward_variables_from(invoker, + [ + "info_plist", + "info_plist_target", + ]) - group(_target_name) { - forward_variables_from(invoker, - [ - "visibility", - "testonly", - ]) - public_deps = [ ":$_arch_executable_target" ] + executable_name = _output_name + + extra_substitutions = [ "BUNDLE_IDENTIFIER=$_bundle_identifier" ] + if (defined(invoker.extra_substitutions)) { + extra_substitutions += invoker.extra_substitutions + } + } + + if (!defined(invoker.entitlements_target)) { + _entitlements_path = "//build/config/ios/entitlements.plist" + if (defined(invoker.entitlements_path)) { + _entitlements_path = invoker.entitlements_path } } else { - lipo_binary(_lipo_executable_target) { + assert(!defined(invoker.entitlements_path), + "Cannot define both entitlements_path and entitlements_target" + + "for $_target_name") + + _entitlements_target_outputs = + get_target_outputs(invoker.entitlements_target) + _entitlements_path = _entitlements_target_outputs[0] + } + + action(_generate_entitlements_target) { + _gen_info_plist_outputs = get_target_outputs(":$_generate_info_plist") + _info_plist_path = _gen_info_plist_outputs[0] + + script = "//build/config/ios/codesign.py" + deps = [ ":$_generate_info_plist" ] + if (defined(invoker.entitlements_target)) { + deps += [ invoker.entitlements_target ] + } + sources = [ + _entitlements_path, + _info_plist_path, + ] + sources += ios_mobileprovision_files + + outputs = [ _generate_entitlements_output ] + + args = [ + "generate-entitlements", + "-e=" + rebase_path(_entitlements_path, root_build_dir), + "-p=" + rebase_path(_info_plist_path, root_build_dir), + ] + foreach(mobileprovision, ios_mobileprovision_files) { + args += [ "-m=" + rebase_path(mobileprovision, root_build_dir) ] + } + args += rebase_path(outputs, root_build_dir) + } + + # Only write PkgInfo for real application, not application extension. + if (_is_app_bundle) { + _create_pkg_info = target_name + "_pkg_info" + action(_create_pkg_info) { + forward_variables_from(invoker, [ "testonly" ]) + script = "//build/apple/write_pkg_info.py" + inputs = [ "//build/apple/plist_util.py" ] + sources = get_target_outputs(":$_generate_info_plist") + outputs = [ + # Cannot name the output PkgInfo as the name will not be unique if + # multiple ios_app_bundle are defined in the same BUILD.gn file. The + # file is renamed in the bundle_data outputs to the correct name. + "$target_gen_dir/$target_name", + ] + args = [ "--plist" ] + rebase_path(sources, root_build_dir) + + [ "--output" ] + rebase_path(outputs, root_build_dir) + deps = [ ":$_generate_info_plist" ] + } + + _bundle_data_pkg_info = target_name + "_bundle_data_pkg_info" + bundle_data(_bundle_data_pkg_info) { + forward_variables_from(invoker, [ "testonly" ]) + sources = get_target_outputs(":$_create_pkg_info") + outputs = [ "{{bundle_resources_dir}}/PkgInfo" ] + public_deps = [ ":$_create_pkg_info" ] + } + } + + foreach(_variant, _variants) { + create_signed_bundle(_variant.target_name) { forward_variables_from(invoker, [ - "configs", + "bundle_deps", + "bundle_deps_filter", + "data_deps", + "deps", + "enable_code_signing", + "entitlements_path", + "entitlements_target", + "extra_system_frameworks", + "public_configs", + "public_deps", "testonly", + "visibility", + "xcode_extra_attributes", ]) - visibility = [] - foreach(_variant, _variants) { - visibility += [ ":${_variant.target_name}" ] - } - output_name = _output_name - arch_binary_target = ":$_arch_executable_target" - arch_binary_output = _output_name - } + bundle_gen_dir = _variant.bundle_gen_dir + bundle_binary_target = ":$_executable_target" + bundle_binary_output = _output_name + bundle_extension = _bundle_extension + product_type = _product_type + xcode_product_bundle_id = _bundle_identifier - _generate_info_plist = target_name + "_generate_info_plist" - ios_info_plist(_generate_info_plist) { - forward_variables_from(invoker, - [ - "extra_substitutions", - "info_plist", - "info_plist_target", - ]) + _generate_info_plist_outputs = + get_target_outputs(":$_generate_info_plist") + primary_info_plist = _generate_info_plist_outputs[0] + partial_info_plist = + "$target_gen_dir/${_variant.target_name}_partial_info.plist" - executable_name = _output_name - } - - if (!is_fat_secondary_toolchain) { - if (!defined(invoker.entitlements_target)) { - _entitlements_path = "//build/config/ios/entitlements.plist" - if (defined(invoker.entitlements_path)) { - _entitlements_path = invoker.entitlements_path - } - } else { - assert(!defined(invoker.entitlements_path), - "Cannot define both entitlements_path and entitlements_target" + - "for $_target_name") - - _entitlements_target_outputs = - get_target_outputs(invoker.entitlements_target) - _entitlements_path = _entitlements_target_outputs[0] + if (!defined(deps)) { + deps = [] } + deps += [ ":$_generate_info_plist" ] - action(_generate_entitlements_target) { - _gen_info_plist_outputs = get_target_outputs(":$_generate_info_plist") - _info_plist_path = _gen_info_plist_outputs[0] - - script = "//build/config/ios/codesign.py" - deps = [ ":$_generate_info_plist" ] - if (defined(invoker.entitlements_target)) { - deps += [ invoker.entitlements_target ] - } - sources = [ - _entitlements_path, - _info_plist_path, - ] - outputs = [ _generate_entitlements_output ] - - args = [ - "generate-entitlements", - "-e=" + rebase_path(_entitlements_path, root_build_dir), - "-p=" + rebase_path(_info_plist_path, root_build_dir), - ] + rebase_path(outputs, root_build_dir) + if (!defined(bundle_deps)) { + bundle_deps = [] } - } - - # Only write PkgInfo for real application, not application extension. - if (_is_app_bundle) { - _create_pkg_info = target_name + "_pkg_info" - action(_create_pkg_info) { - forward_variables_from(invoker, [ "testonly" ]) - script = "//build/apple/write_pkg_info.py" - inputs = [ "//build/apple/plist_util.py" ] - sources = get_target_outputs(":$_generate_info_plist") - outputs = [ - # Cannot name the output PkgInfo as the name will not be unique if - # multiple ios_app_bundle are defined in the same BUILD.gn file. The - # file is renamed in the bundle_data outputs to the correct name. - "$target_gen_dir/$target_name", - ] - args = [ "--plist" ] + rebase_path(sources, root_build_dir) + - [ "--output" ] + rebase_path(outputs, root_build_dir) - deps = [ ":$_generate_info_plist" ] + if (_is_app_bundle) { + bundle_deps += [ ":$_bundle_data_pkg_info" ] } + bundle_deps += _variant.bundle_deps - _bundle_data_pkg_info = target_name + "_bundle_data_pkg_info" - bundle_data(_bundle_data_pkg_info) { - forward_variables_from(invoker, [ "testonly" ]) - sources = get_target_outputs(":$_create_pkg_info") - outputs = [ "{{bundle_resources_dir}}/PkgInfo" ] - public_deps = [ ":$_create_pkg_info" ] - } - } - - foreach(_variant, _variants) { - create_signed_bundle(_variant.target_name) { - forward_variables_from(invoker, - [ - "bundle_deps", - "bundle_deps_filter", - "data_deps", - "deps", - "enable_code_signing", - "entitlements_path", - "entitlements_target", - "extra_system_frameworks", - "public_configs", - "public_deps", - "testonly", - "visibility", - "xcode_extra_attributes", - ]) - - output_name = _output_name - bundle_gen_dir = _variant.bundle_gen_dir - bundle_binary_target = ":$_lipo_executable_target" - bundle_binary_output = _output_name - bundle_extension = _bundle_extension - product_type = _product_type - xcode_product_bundle_id = _xcode_product_bundle_id - - _generate_info_plist_outputs = - get_target_outputs(":$_generate_info_plist") - primary_info_plist = _generate_info_plist_outputs[0] - partial_info_plist = - "$target_gen_dir/${_variant.target_name}_partial_info.plist" - - if (!defined(deps)) { - deps = [] + if (target_environment == "simulator") { + if (!defined(data_deps)) { + data_deps = [] } - deps += [ ":$_generate_info_plist" ] - - if (!defined(bundle_deps)) { - bundle_deps = [] - } - if (_is_app_bundle) { - bundle_deps += [ ":$_bundle_data_pkg_info" ] - } - bundle_deps += _variant.bundle_deps - - if (target_environment == "simulator") { - if (!defined(data_deps)) { - data_deps = [] - } + if (build_with_chromium) { data_deps += [ "//testing/iossim" ] } } } - - if (_default_variant.name != "") { - _bundle_short_name = "$_output_name$_bundle_extension" - action(_target_name) { - forward_variables_from(invoker, [ "testonly" ]) - - script = "//build/config/ios/hardlink.py" - public_deps = [] - foreach(_variant, _variants) { - public_deps += [ ":${_variant.target_name}" ] - } - - sources = [ "${_default_variant.bundle_gen_dir}/$_bundle_short_name" ] - outputs = [ "$root_out_dir/$_bundle_short_name" ] - - args = rebase_path(sources, root_build_dir) + - rebase_path(outputs, root_build_dir) - } - } } - if (is_fat_secondary_toolchain) { - not_needed("*") + if (_default_variant.name != "") { + _bundle_short_name = "$_output_name$_bundle_extension" + action(_target_name) { + forward_variables_from(invoker, [ "testonly" ]) + + script = "//build/config/ios/hardlink.py" + public_deps = [] + foreach(_variant, _variants) { + public_deps += [ ":${_variant.target_name}" ] + } + + sources = [ "${_default_variant.bundle_gen_dir}/$_bundle_short_name" ] + outputs = [ "$root_out_dir/$_bundle_short_name" ] + + args = rebase_path(sources, root_build_dir) + + rebase_path(outputs, root_build_dir) + } } } @@ -1278,27 +1090,12 @@ _has_public_headers = defined(invoker.public_headers) && invoker.public_headers != [] - _primary_toolchain = current_toolchain - if (is_fat_secondary_toolchain) { - _primary_toolchain = primary_fat_toolchain_name - } - - # Public configs are not propagated across toolchain (see crbug.com/675224) - # so some configs have to be defined for both default_toolchain and all others - # toolchains when performing a fat build. Use "get_label_info" to construct - # the path since they need to be relative to the default_toolchain. - - _default_toolchain_root_out_dir = - get_label_info("$_target_name($_primary_toolchain)", "root_out_dir") - - _arch_shared_library_source = _target_name + "_arch_shared_library_sources" - _arch_shared_library_target = _target_name + "_arch_shared_library" - _lipo_shared_library_target = _target_name + "_shared_library" + _shared_library_target = _target_name + "_shared_library" _link_target_name = _target_name + "+link" if (_has_public_headers) { _default_toolchain_target_gen_dir = - get_label_info("$_target_name($_primary_toolchain)", "target_gen_dir") + get_label_info("$_target_name", "target_gen_dir") _framework_headers_target = _target_name + "_framework_headers" @@ -1307,7 +1104,7 @@ "$_default_toolchain_target_gen_dir/$_output_name.headers.hmap" config(_headers_map_config) { visibility = [ - ":${_arch_shared_library_source}", + ":${_shared_library_target}", ":${_target_name}_signed_bundle", ] include_dirs = [ _header_map_filename ] @@ -1316,7 +1113,7 @@ _framework_headers_config = _target_name + "_framework_headers_config" config(_framework_headers_config) { - framework_dirs = [ _default_toolchain_root_out_dir ] + framework_dirs = [ root_out_dir ] } _framework_public_config = _target_name + "_public_config" @@ -1325,7 +1122,7 @@ frameworks = [ "$_output_name.framework" ] } - source_set(_arch_shared_library_source) { + shared_library(_shared_library_target) { forward_variables_from(invoker, "*", [ @@ -1341,7 +1138,13 @@ "visibility", ]) - visibility = [ ":$_arch_shared_library_target" ] + visibility = [ ":${_target_name}_signed_bundle" ] + + if (!defined(ldflags)) { + ldflags = [] + } + ldflags += + [ "-Wl,-install_name,@rpath/$_output_name.framework/$_output_name" ] if (_has_public_headers) { configs += [ ":$_headers_map_config" ] @@ -1349,301 +1152,214 @@ if (!defined(deps)) { deps = [] } - deps += [ ":$_framework_headers_target($_primary_toolchain)" ] + deps += [ ":$_framework_headers_target" ] } - } - - shared_library(_arch_shared_library_target) { - forward_variables_from(invoker, - "*", - [ - "bundle_deps", - "bundle_deps_filter", - "data_deps", - "enable_code_signing", - "extra_substitutions", - "info_plist", - "info_plist_target", - "output_name", - "sources", - "public_configs", - "visibility", - ]) - - visibility = [ ":$_lipo_shared_library_target($_primary_toolchain)" ] - if (is_fat_secondary_toolchain) { - visibility += [ - ":${_target_name}", - ":${_target_name}_signed_bundle", - ] - } - - if (!defined(deps)) { - deps = [] - } - deps += [ ":$_arch_shared_library_source" ] - if (_has_public_headers) { - deps += [ ":$_framework_headers_target($_primary_toolchain)" ] - } - if (!defined(ldflags)) { - ldflags = [] - } - ldflags += - [ "-Wl,-install_name,@rpath/$_output_name.framework/$_output_name" ] output_extension = "" output_name = _output_name output_prefix_override = true - output_dir = "$target_out_dir/$current_cpu" + output_dir = target_out_dir } - if (is_fat_secondary_toolchain) { - # For fat builds, only the default toolchain will generate a framework - # bundle. For the other toolchains, the template is only used for building - # the arch-specific binary, thus the default target is just a group(). + if (_has_public_headers) { + _public_headers = invoker.public_headers - group(_target_name) { + _framework_root_dir = "$root_out_dir/$_output_name.framework" + if (target_environment == "simulator" || target_environment == "device") { + _framework_contents_dir = _framework_root_dir + } else if (target_environment == "catalyst") { + _framework_contents_dir = "$_framework_root_dir/Versions/A" + } + + _compile_headers_map_target = _target_name + "_compile_headers_map" + action(_compile_headers_map_target) { + visibility = [ ":$_framework_headers_target" ] forward_variables_from(invoker, [ - "visibility", - "testonly", - ]) - public_deps = [ ":$_arch_shared_library_target" ] - } - - group(_link_target_name) { - forward_variables_from(invoker, - [ - "public_configs", - "visibility", - "testonly", - ]) - public_deps = [ ":$_link_target_name($_primary_toolchain)" ] - - if (_has_public_headers) { - if (!defined(public_configs)) { - public_configs = [] - } - public_configs += [ ":$_framework_headers_config" ] - } - if (!defined(all_dependent_configs)) { - all_dependent_configs = [] - } - all_dependent_configs += [ ":$_framework_public_config" ] - } - - group("$_target_name+bundle") { - forward_variables_from(invoker, [ "testonly" ]) - public_deps = [ ":$_target_name+bundle($_primary_toolchain)" ] - } - - not_needed(invoker, "*") - } else { - if (_has_public_headers) { - _public_headers = invoker.public_headers - - _framework_root_dir = "$root_out_dir/$_output_name.framework" - if (target_environment == "simulator" || target_environment == "device") { - _framework_contents_dir = _framework_root_dir - } else if (target_environment == "catalyst") { - _framework_contents_dir = "$_framework_root_dir/Versions/A" - } - - _compile_headers_map_target = _target_name + "_compile_headers_map" - action(_compile_headers_map_target) { - visibility = [ ":$_framework_headers_target" ] - forward_variables_from(invoker, - [ - "deps", - "public_deps", - "testonly", - ]) - script = "//build/config/ios/write_framework_hmap.py" - outputs = [ _header_map_filename ] - - # The header map generation only wants the list of headers, not all of - # sources, so filter any non-header source files from "sources". It is - # less error prone that having the developer duplicate the list of all - # headers in addition to "sources". - sources = [] - foreach(_source, invoker.sources) { - if (get_path_info(_source, "extension") == "h") { - sources += [ _source ] - } - } - - args = [ - rebase_path(_header_map_filename), - rebase_path(_framework_root_dir, root_build_dir), - ] + rebase_path(sources, root_build_dir) - } - - _create_module_map_target = _target_name + "_module_map" - action(_create_module_map_target) { - visibility = [ ":$_framework_headers_target" ] - script = "//build/config/ios/write_framework_modulemap.py" - outputs = [ "$_framework_contents_dir/Modules/module.modulemap" ] - args = [ - _output_name, - rebase_path("$_framework_contents_dir/Modules", root_build_dir), - ] - } - - _copy_public_headers_target = _target_name + "_copy_public_headers" - copy(_copy_public_headers_target) { - forward_variables_from(invoker, - [ - "testonly", - "deps", - ]) - visibility = [ ":$_framework_headers_target" ] - sources = _public_headers - outputs = [ "$_framework_contents_dir/Headers/{{source_file_part}}" ] - - # Do not use forward_variables_from for "public_deps" as - # we do not want to forward those dependencies. - if (defined(invoker.public_deps)) { - if (!defined(deps)) { - deps = [] - } - deps += invoker.public_deps - } - } - - group(_framework_headers_target) { - forward_variables_from(invoker, [ "testonly" ]) - deps = [ - ":$_compile_headers_map_target", - ":$_create_module_map_target", - ] - public_deps = [ ":$_copy_public_headers_target" ] - } - } - - lipo_binary(_lipo_shared_library_target) { - forward_variables_from(invoker, - [ - "configs", - "testonly", - ]) - - visibility = [ ":${_target_name}_signed_bundle" ] - output_name = _output_name - arch_binary_target = ":$_arch_shared_library_target" - arch_binary_output = _output_name - } - - _info_plist_target = _target_name + "_info_plist" - _info_plist_bundle = _target_name + "_info_plist_bundle" - ios_info_plist(_info_plist_target) { - visibility = [ ":$_info_plist_bundle" ] - executable_name = _output_name - forward_variables_from(invoker, - [ - "extra_substitutions", - "info_plist", - "info_plist_target", - ]) - } - - bundle_data(_info_plist_bundle) { - visibility = [ ":${_target_name}_signed_bundle" ] - forward_variables_from(invoker, [ "testonly" ]) - sources = get_target_outputs(":$_info_plist_target") - public_deps = [ ":$_info_plist_target" ] - - if (target_environment != "catalyst") { - outputs = [ "{{bundle_contents_dir}}/Info.plist" ] - } else { - outputs = [ "{{bundle_resources_dir}}/Info.plist" ] - } - } - - create_signed_bundle(_target_name + "_signed_bundle") { - forward_variables_from(invoker, - [ - "bundle_deps", - "bundle_deps_filter", - "data_deps", "deps", - "enable_code_signing", - "public_configs", "public_deps", "testonly", - "visibility", ]) + script = "//build/config/ios/write_framework_hmap.py" + outputs = [ _header_map_filename ] - product_type = "com.apple.product-type.framework" - bundle_extension = ".framework" - - output_name = _output_name - bundle_binary_target = ":$_lipo_shared_library_target" - bundle_binary_output = _output_name - - has_public_headers = _has_public_headers - - # Framework do not have entitlements nor mobileprovision because they use - # the one from the bundle using them (.app or .appex) as they are just - # dynamic library with shared code. - disable_entitlements = true - disable_embedded_mobileprovision = true - - if (!defined(deps)) { - deps = [] - } - deps += [ ":$_info_plist_bundle" ] - } - - group(_target_name) { - forward_variables_from(invoker, - [ - "public_configs", - "public_deps", - "testonly", - "visibility", - ]) - if (!defined(public_deps)) { - public_deps = [] - } - public_deps += [ ":${_target_name}_signed_bundle" ] - - if (_has_public_headers) { - if (!defined(public_configs)) { - public_configs = [] + # The header map generation only wants the list of headers, not all of + # sources, so filter any non-header source files from "sources". It is + # less error prone that having the developer duplicate the list of all + # headers in addition to "sources". + sources = [] + foreach(_source, invoker.sources) { + if (get_path_info(_source, "extension") == "h") { + sources += [ _source ] } - public_configs += [ ":$_framework_headers_config" ] } + + args = [ + rebase_path(_header_map_filename, root_build_dir), + rebase_path(_framework_root_dir, root_build_dir), + ] + rebase_path(sources, root_build_dir) } - group(_link_target_name) { - forward_variables_from(invoker, - [ - "public_configs", - "public_deps", - "testonly", - "visibility", - ]) - if (!defined(public_deps)) { - public_deps = [] - } - public_deps += [ ":$_target_name" ] - - if (!defined(all_dependent_configs)) { - all_dependent_configs = [] - } - all_dependent_configs += [ ":$_framework_public_config" ] + _create_module_map_target = _target_name + "_module_map" + action(_create_module_map_target) { + visibility = [ ":$_framework_headers_target" ] + script = "//build/config/ios/write_framework_modulemap.py" + outputs = [ "$_framework_contents_dir/Modules/module.modulemap" ] + args = [ + _output_name, + rebase_path("$_framework_contents_dir/Modules", root_build_dir), + ] } - bundle_data(_target_name + "+bundle") { + _copy_public_headers_target = _target_name + "_copy_public_headers" + copy(_copy_public_headers_target) { forward_variables_from(invoker, [ "testonly", - "visibility", + "deps", ]) - public_deps = [ ":$_target_name" ] - sources = [ "$root_out_dir/$_output_name.framework" ] - outputs = [ "{{bundle_contents_dir}}/Frameworks/$_output_name.framework" ] + visibility = [ ":$_framework_headers_target" ] + sources = _public_headers + outputs = [ "$_framework_contents_dir/Headers/{{source_file_part}}" ] + + # Do not use forward_variables_from for "public_deps" as + # we do not want to forward those dependencies. + if (defined(invoker.public_deps)) { + if (!defined(deps)) { + deps = [] + } + deps += invoker.public_deps + } } + + group(_framework_headers_target) { + forward_variables_from(invoker, [ "testonly" ]) + deps = [ + ":$_compile_headers_map_target", + ":$_create_module_map_target", + ] + public_deps = [ ":$_copy_public_headers_target" ] + } + } + + # Bundle identifier should respect rfc1034, so replace "_" with "-". + _bundle_identifier = + "$ios_app_bundle_id_prefix." + string_replace(_output_name, "_", "-") + + _info_plist_target = _target_name + "_info_plist" + _info_plist_bundle = _target_name + "_info_plist_bundle" + ios_info_plist(_info_plist_target) { + visibility = [ ":$_info_plist_bundle" ] + executable_name = _output_name + forward_variables_from(invoker, + [ + "info_plist", + "info_plist_target", + ]) + + extra_substitutions = [ "BUNDLE_IDENTIFIER=$_bundle_identifier" ] + if (defined(invoker.extra_substitutions)) { + extra_substitutions += invoker.extra_substitutions + } + } + + bundle_data(_info_plist_bundle) { + visibility = [ ":${_target_name}_signed_bundle" ] + forward_variables_from(invoker, [ "testonly" ]) + sources = get_target_outputs(":$_info_plist_target") + public_deps = [ ":$_info_plist_target" ] + + if (target_environment != "catalyst") { + outputs = [ "{{bundle_contents_dir}}/Info.plist" ] + } else { + outputs = [ "{{bundle_resources_dir}}/Info.plist" ] + } + } + + create_signed_bundle(_target_name + "_signed_bundle") { + forward_variables_from(invoker, + [ + "bundle_deps", + "bundle_deps_filter", + "data_deps", + "deps", + "enable_code_signing", + "public_configs", + "public_deps", + "testonly", + "visibility", + ]) + + product_type = "com.apple.product-type.framework" + bundle_extension = ".framework" + + output_name = _output_name + bundle_binary_target = ":$_shared_library_target" + bundle_binary_output = _output_name + + has_public_headers = _has_public_headers + + # Framework do not have entitlements nor mobileprovision because they use + # the one from the bundle using them (.app or .appex) as they are just + # dynamic library with shared code. + disable_entitlements = true + disable_embedded_mobileprovision = true + + if (!defined(deps)) { + deps = [] + } + deps += [ ":$_info_plist_bundle" ] + } + + group(_target_name) { + forward_variables_from(invoker, + [ + "public_configs", + "public_deps", + "testonly", + "visibility", + ]) + if (!defined(public_deps)) { + public_deps = [] + } + public_deps += [ ":${_target_name}_signed_bundle" ] + + if (_has_public_headers) { + if (!defined(public_configs)) { + public_configs = [] + } + public_configs += [ ":$_framework_headers_config" ] + } + } + + group(_link_target_name) { + forward_variables_from(invoker, + [ + "public_configs", + "public_deps", + "testonly", + "visibility", + ]) + if (!defined(public_deps)) { + public_deps = [] + } + public_deps += [ ":$_target_name" ] + + if (!defined(all_dependent_configs)) { + all_dependent_configs = [] + } + all_dependent_configs += [ ":$_framework_public_config" ] + } + + bundle_data(_target_name + "+bundle") { + forward_variables_from(invoker, + [ + "testonly", + "visibility", + ]) + public_deps = [ ":$_target_name" ] + sources = [ "$root_out_dir/$_output_name.framework" ] + outputs = [ "{{bundle_contents_dir}}/Frameworks/$_output_name.framework" ] } } @@ -1691,13 +1407,6 @@ assert(defined(invoker.xcode_test_application_name), "xcode_test_application_name must be defined for $target_name") - # Silence "assignment had no effect" error for non-default toolchains as - # following variables are only used in the expansion of the template for the - # default toolchain. - if (is_fat_secondary_toolchain) { - not_needed(invoker, "*") - } - _target_name = target_name _output_name = target_name @@ -1705,173 +1414,134 @@ _output_name = invoker.output_name } - _arch_loadable_module_source = _target_name + "_arch_loadable_module_source" - _arch_loadable_module_target = _target_name + "_arch_loadable_module" - _lipo_loadable_module_target = _target_name + "_loadable_module" + _loadable_module_target = _target_name + "_loadable_module" - _primary_toolchain = current_toolchain - if (is_fat_secondary_toolchain) { - _primary_toolchain = primary_fat_toolchain_name - } - - source_set(_arch_loadable_module_source) { - forward_variables_from(invoker, [ "deps" ]) + loadable_module(_loadable_module_target) { + forward_variables_from(invoker, + "*", + [ + "host_target", + "output_dir", + "output_extension", + "output_name", + "output_prefix_override", + "product_type", + "testonly", + "visibility", + "xcode_test_application_name", + "xcode_test_application_output_name", + "xctest_bundle_principal_class", + "bundle_deps_filter", + ]) testonly = true - visibility = [ ":$_arch_loadable_module_target" ] - } + visibility = [ ":$_target_name" ] - loadable_module(_arch_loadable_module_target) { - testonly = true - visibility = [ ":$_lipo_loadable_module_target($_primary_toolchain)" ] - if (is_fat_secondary_toolchain) { - visibility += [ ":$_target_name" ] - } - - deps = [ ":$_arch_loadable_module_source" ] configs += [ "//build/config/ios:xctest_config" ] - output_dir = "$target_out_dir/$current_cpu" + output_dir = target_out_dir output_name = _output_name output_prefix_override = true output_extension = "" } - if (is_fat_secondary_toolchain) { - # For fat builds, only the default toolchain will generate a test bundle. - # For the other toolchains, the template is only used for building the - # arch-specific binary, thus the default target is just a group(). - group(_target_name) { - forward_variables_from(invoker, [ "visibility" ]) - testonly = true + _info_plist_target = _target_name + "_info_plist" + _info_plist_bundle = _target_name + "_info_plist_bundle" - public_deps = [ ":$_arch_loadable_module_target" ] + # Bundle identifier should respect rfc1034, so replace "_" with "-". + _bundle_identifier = "$ios_app_bundle_id_prefix.chrome." + + string_replace(_output_name, "_", "-") + + ios_info_plist(_info_plist_target) { + testonly = true + visibility = [ ":$_info_plist_bundle" ] + + info_plist = "//build/config/ios/Module-Info.plist" + executable_name = _output_name + + if (defined(invoker.xctest_bundle_principal_class)) { + _principal_class = invoker.xctest_bundle_principal_class + } else { + # Fall back to a reasonable default value. + _principal_class = "NSObject" } + extra_substitutions = [ + "XCTEST_BUNDLE_PRINCIPAL_CLASS=${_principal_class}", + "BUNDLE_IDENTIFIER=$_bundle_identifier", + ] + } - not_needed(invoker, "*") - } else { - _info_plist_target = _target_name + "_info_plist" - _info_plist_bundle = _target_name + "_info_plist_bundle" + bundle_data(_info_plist_bundle) { + testonly = true + visibility = [ ":$_target_name" ] - ios_info_plist(_info_plist_target) { - testonly = true - visibility = [ ":$_info_plist_bundle" ] + public_deps = [ ":$_info_plist_target" ] - info_plist = "//build/config/ios/Module-Info.plist" - executable_name = _output_name + sources = get_target_outputs(":$_info_plist_target") + outputs = [ "{{bundle_contents_dir}}/Info.plist" ] + } - if (defined(invoker.xctest_bundle_principal_class)) { - _principal_class = invoker.xctest_bundle_principal_class - } else { - # Fall back to a reasonable default value. - _principal_class = "NSObject" + _xctest_bundle = _target_name + "_bundle" + create_signed_bundle(_target_name) { + forward_variables_from(invoker, + [ + "bundle_id", + "data_deps", + "bundle_deps_filter", + "enable_code_signing", + "product_type", + "xcode_test_application_name", + ]) + + testonly = true + visibility = [ ":$_xctest_bundle" ] + + bundle_extension = ".xctest" + + output_name = _output_name + bundle_binary_target = ":$_loadable_module_target" + bundle_binary_output = _output_name + + xcode_extra_attributes = { + IPHONEOS_DEPLOYMENT_TARGET = ios_deployment_target + PRODUCT_BUNDLE_IDENTIFIER = _bundle_identifier + CODE_SIGNING_REQUIRED = "NO" + CODE_SIGNING_ALLOWED = "NO" + CODE_SIGN_IDENTITY = "" + DONT_GENERATE_INFOPLIST_FILE = "YES" + + # For XCUITest, Xcode requires specifying the host application name + # via the TEST_TARGET_NAME attribute. + if (invoker.product_type == _ios_xcode_xcuitest_bundle_id) { + TEST_TARGET_NAME = invoker.xcode_test_application_name } - extra_substitutions = [ - "XCTEST_BUNDLE_PRINCIPAL_CLASS=${_principal_class}", - "MODULE_BUNDLE_ID=gtest.$_output_name", - ] - } - bundle_data(_info_plist_bundle) { - testonly = true - visibility = [ ":$_target_name" ] - - public_deps = [ ":$_info_plist_target" ] - - sources = get_target_outputs(":$_info_plist_target") - outputs = [ "{{bundle_contents_dir}}/Info.plist" ] - } - - lipo_binary(_lipo_loadable_module_target) { - forward_variables_from(invoker, [ "configs" ]) - - testonly = true - visibility = [ ":$_target_name" ] - - output_name = _output_name - arch_binary_target = ":$_arch_loadable_module_target" - arch_binary_output = _output_name - } - - _xctest_bundle = _target_name + "_bundle" - create_signed_bundle(_target_name) { - forward_variables_from(invoker, - [ - "bundle_id", - "data_deps", - "enable_code_signing", - "product_type", - "xcode_test_application_name", - ]) - - testonly = true - visibility = [ ":$_xctest_bundle" ] - - bundle_extension = ".xctest" - - output_name = _output_name - bundle_binary_target = ":$_lipo_loadable_module_target" - bundle_binary_output = _output_name - - if (ios_set_attributes_for_xcode_project_generation) { - _xcode_product_bundle_id = - "$ios_app_bundle_id_prefix.gtest.$_output_name" - - _ios_provisioning_profile_info = - exec_script("//build/config/ios/codesign.py", - [ - "find-provisioning-profile", - "-b=" + _xcode_product_bundle_id, - ], - "json") - - xcode_extra_attributes = { - IPHONEOS_DEPLOYMENT_TARGET = ios_deployment_target - CODE_SIGN_IDENTITY = "iPhone Developer" - DEVELOPMENT_TEAM = _ios_provisioning_profile_info.team_identifier - PRODUCT_BUNDLE_IDENTIFIER = _xcode_product_bundle_id - PROVISIONING_PROFILE_SPECIFIER = _ios_provisioning_profile_info.name - - # For XCUITest, Xcode requires specifying the host application name - # via the TEST_TARGET_NAME attribute. - if (invoker.product_type == _ios_xcode_xcuitest_bundle_id) { - TEST_TARGET_NAME = invoker.xcode_test_application_name - } - - # For XCTest, Xcode requires specifying the host application path via - # both BUNDLE_LOADER and TEST_HOST attributes. - if (invoker.product_type == _ios_xcode_xctest_bundle_id) { - _xcode_app_name = invoker.xcode_test_application_name - if (defined(invoker.xcode_test_application_output_name)) { - _xcode_app_name = invoker.xcode_test_application_output_name - } - - BUNDLE_LOADER = "\$(TEST_HOST)" - TEST_HOST = "\$(BUILT_PRODUCTS_DIR)/" + - "${_xcode_app_name}.app/${_xcode_app_name}" - } + # For XCTest, Xcode requires specifying the host application path via + # both BUNDLE_LOADER and TEST_HOST attributes. + if (invoker.product_type == _ios_xcode_xctest_bundle_id) { + _xcode_app_name = invoker.xcode_test_application_name + if (defined(invoker.xcode_test_application_output_name)) { + _xcode_app_name = invoker.xcode_test_application_output_name } - } else { - not_needed(invoker, - [ - "xcode_test_application_name", - "xcode_test_application_output_name", - ]) + + BUNDLE_LOADER = "\$(TEST_HOST)" + TEST_HOST = "\$(BUILT_PRODUCTS_DIR)/" + + "${_xcode_app_name}.app/${_xcode_app_name}" } - - deps = [ ":$_info_plist_bundle" ] } - bundle_data(_xctest_bundle) { - forward_variables_from(invoker, [ "host_target" ]) + deps = [ ":$_info_plist_bundle" ] + } - testonly = true - visibility = [ ":$host_target" ] + bundle_data(_xctest_bundle) { + forward_variables_from(invoker, [ "host_target" ]) - public_deps = [ ":$_target_name" ] - sources = [ "$root_out_dir/$_output_name.xctest" ] - outputs = [ "{{bundle_contents_dir}}/PlugIns/$_output_name.xctest" ] - } + testonly = true + visibility = [ ":$host_target" ] + + public_deps = [ ":$_target_name" ] + sources = [ "$root_out_dir/$_output_name.xctest" ] + outputs = [ "{{bundle_contents_dir}}/PlugIns/$_output_name.xctest" ] } } @@ -1946,13 +1616,27 @@ "$ios_sdk_platform_path/Developer/usr/lib/libXCTestBundleInject.dylib", ] - _xctest_bundle = _xctest_target + "_bundle" - if (!is_fat_secondary_toolchain) { - if (!defined(bundle_deps)) { - bundle_deps = [] - } - bundle_deps += [ ":$_xctest_bundle" ] + # Xcode 13 now depends on XCTestCore. To keep things future proof, copy over + # everything that Xcode copies. + if (xcode_version_int >= 1300) { + extra_system_frameworks += [ + "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCTestCore.framework", + "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCUIAutomation.framework", + "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCUnit.framework", + "$ios_sdk_platform_path/Developer/usr/lib/libXCTestSwiftSupport.dylib", + ] } + + # XCTestSupport framework is required as of Xcode 14.3 or later. + if (xcode_version_int >= 1430) { + extra_system_frameworks += [ "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCTestSupport.framework" ] + } + + _xctest_bundle = _xctest_target + "_bundle" + if (!defined(bundle_deps)) { + bundle_deps = [] + } + bundle_deps += [ ":$_xctest_bundle" ] } } @@ -1987,6 +1671,10 @@ _output_name = invoker.output_name } + # Bundle identifier should respect rfc1034, so replace "_" with "-". + _bundle_identifier = "$ios_app_bundle_id_prefix.chrome." + + string_replace(_output_name, "_", "-") + _xctrunner_path = "$ios_sdk_platform_path/Developer/Library/Xcode/Agents/XCTRunner.app" @@ -2016,7 +1704,7 @@ "-o=" + rebase_path(_output_name, root_build_dir), ] + rebase_path(sources, root_build_dir) - if (use_system_xcode && use_goma) { + if (use_system_xcode && (use_goma || use_remoteexec)) { deps = [ "//build/config/ios:copy_xctrunner_app" ] } } @@ -2027,6 +1715,7 @@ executable_name = _output_name info_plist_target = ":$_info_plist_merge_plist" + extra_substitutions = [ "BUNDLE_IDENTIFIER=$_bundle_identifier" ] } bundle_data(_info_plist_bundle) { @@ -2048,7 +1737,7 @@ outputs = [ "{{bundle_contents_dir}}/PkgInfo" ] - if (use_system_xcode && use_goma) { + if (use_system_xcode && (use_goma || use_remoteexec)) { public_deps = [ "//build/config/ios:copy_xctrunner_app" ] } } @@ -2072,6 +1761,22 @@ "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCTAutomationSupport.framework", ] + # Xcode 13 now depends on XCTestCore. To keep things future proof, copy over + # everything that Xcode copies. + if (xcode_version_int >= 1300) { + extra_system_frameworks += [ + "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCTestCore.framework", + "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCUIAutomation.framework", + "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCUnit.framework", + "$ios_sdk_platform_path/Developer/usr/lib/libXCTestSwiftSupport.dylib", + ] + } + + # XCTestSupport framework is required as of Xcode 14.3 or later. + if (xcode_version_int >= 1430) { + extra_system_frameworks += [ "$ios_sdk_platform_path/Developer/Library/PrivateFrameworks/XCTestSupport.framework" ] + } + bundle_deps = [] if (defined(invoker.bundle_deps)) { bundle_deps += invoker.bundle_deps @@ -2096,6 +1801,12 @@ # xcode_test_application_name: # string, name of the test application for the ui test target. # +# runner_only_bundle_deps: +# list of labels of bundle target to include in the runner and +# exclude from the test module (the use case is a framework bundle +# that is used by the test module and thus needs to be packaged in +# the runner application bundle) +# # This template defines two targets, one named "${target_name}_module" is the # xctest dynamic library, and the other named "${target_name}_runner" is the # test runner application bundle. @@ -2133,6 +1844,10 @@ output_name = _xcuitest_module_output deps = invoker.deps + + if (defined(invoker.runner_only_bundle_deps)) { + bundle_deps_filter = invoker.runner_only_bundle_deps + } } _xcuitest_runner_output = _xcuitest_target + "-Runner" @@ -2140,6 +1855,13 @@ output_name = _xcuitest_runner_output xctest_bundle = _xcuitest_module_target + "_bundle" forward_variables_from(invoker, [ "bundle_deps" ]) + + if (defined(invoker.runner_only_bundle_deps)) { + if (!defined(bundle_deps)) { + bundle_deps = [] + } + bundle_deps += invoker.runner_only_bundle_deps + } } }
diff --git a/build/config/ios/strip_arm64e.py b/build/config/ios/strip_arm64e.py index f21baf4..56e684f 100644 --- a/build/config/ios/strip_arm64e.py +++ b/build/config/ios/strip_arm64e.py
@@ -1,4 +1,4 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. +# Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Strip arm64e architecture from a binary if present."""
diff --git a/build/config/ios/swift_source_set.gni b/build/config/ios/swift_source_set.gni new file mode 100644 index 0000000..0f5cc07 --- /dev/null +++ b/build/config/ios/swift_source_set.gni
@@ -0,0 +1,25 @@ +# Copyright 2021 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Defines a template for Swift source files. The default module_name +# of the target is the entire target label (without the leading //) +# with all "/" and ":" replaced with "_". +template("swift_source_set") { + _target_name = target_name + source_set(target_name) { + forward_variables_from(invoker, "*", TESTONLY_AND_VISIBILITY) + forward_variables_from(invoker, TESTONLY_AND_VISIBILITY) + if (!defined(module_name)) { + _target_label = get_label_info(":$_target_name", "label_no_toolchain") + + # Strip the // from the beginning of the label. + _target_label = string_replace(_target_label, "//", "", 1) + module_name = + string_replace(string_replace(_target_label, "/", "_"), ":", "_") + } + } +} +set_defaults("swift_source_set") { + configs = default_compiler_configs +}
diff --git a/build/config/ios/write_framework_hmap.py b/build/config/ios/write_framework_hmap.py index ac467ee..8889253 100644 --- a/build/config/ios/write_framework_hmap.py +++ b/build/config/ios/write_framework_hmap.py
@@ -1,8 +1,7 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import os import struct
diff --git a/build/config/ios/write_framework_modulemap.py b/build/config/ios/write_framework_modulemap.py index dcc88a8..49f3263 100644 --- a/build/config/ios/write_framework_modulemap.py +++ b/build/config/ios/write_framework_modulemap.py
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/ios/xctest_shell.mm b/build/config/ios/xctest_shell.mm index dcf5bad..0fd5cca 100644 --- a/build/config/ios/xctest_shell.mm +++ b/build/config/ios/xctest_shell.mm
@@ -1,4 +1,4 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. +// Copyright 2016 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file.
diff --git a/build/config/linux/BUILD.gn b/build/config/linux/BUILD.gn index 4770424..131bb71 100644 --- a/build/config/linux/BUILD.gn +++ b/build/config/linux/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -15,6 +15,18 @@ # is applied to all targets. It is here to separate out the logic that is # Linux-only. This is not applied to Android, but is applied to ChromeOS. config("compiler") { + if (current_cpu == "arm64") { + import("//build/config/arm.gni") + cflags = [] + asmflags = [] + if (arm_control_flow_integrity == "standard") { + cflags += [ "-mbranch-protection=standard" ] + asmflags += [ "-mbranch-protection=standard" ] + } else if (arm_control_flow_integrity == "pac") { + cflags += [ "-mbranch-protection=pac-ret" ] + asmflags += [ "-mbranch-protection=pac-ret" ] + } + } } # This is included by reference in the //build/config/compiler:runtime_library @@ -28,8 +40,7 @@ defines = [ "OS_CHROMEOS" ] } - if ((!(is_chromeos_ash || is_chromeos_lacros) || - default_toolchain != "//build/toolchain/cros:target") && + if ((!is_chromeos || default_toolchain != "//build/toolchain/cros:target") && (!use_custom_libcxx || current_cpu == "mipsel")) { libs = [ "atomic" ] } @@ -52,17 +63,8 @@ "gthread-2.0", ] defines = [ - "GLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40", - "GLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40", + "GLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_56", + "GLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_56", ] } } - -# Ensures all exported symbols are added to the dynamic symbol table. This is -# necessary to expose Chrome's custom operator new() and operator delete() (and -# other memory-related symbols) to libraries. Otherwise, they might -# (de)allocate memory on a different heap, which would spell trouble if pointers -# to heap-allocated memory are passed over shared library boundaries. -config("export_dynamic") { - ldflags = [ "-rdynamic" ] -}
diff --git a/build/config/linux/atk/BUILD.gn b/build/config/linux/atk/BUILD.gn index bc8e278..239c387 100644 --- a/build/config/linux/atk/BUILD.gn +++ b/build/config/linux/atk/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,9 +7,8 @@ import("//build/config/linux/pkg_config.gni") import("//build/config/ui.gni") -# TODO(crbug.com/1171629): Change this back to is_chromeos. # CrOS doesn't install GTK or any gnome packages. -assert(!is_chromeos_ash) +assert(!is_chromeos) # These packages should _only_ be expected when building for a target. assert(current_toolchain == default_toolchain) @@ -24,7 +23,7 @@ "atk-bridge-2.0", ] atk_lib_dir = exec_script(pkg_config_script, - pkg_config_args + [ + common_pkg_config_args + pkg_config_args + [ "--libdir", "atk", ],
diff --git a/build/config/linux/atspi2/BUILD.gn b/build/config/linux/atspi2/BUILD.gn index 988a995..51b6d33 100644 --- a/build/config/linux/atspi2/BUILD.gn +++ b/build/config/linux/atspi2/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -12,7 +12,7 @@ pkg_config("atspi2") { packages = [ "atspi-2" ] atspi_version = exec_script(pkg_config_script, - pkg_config_args + [ + common_pkg_config_args + pkg_config_args + [ "atspi-2", "--version-as-components", ],
diff --git a/build/config/linux/dbus/BUILD.gn b/build/config/linux/dbus/BUILD.gn index f11cf71..2414c34 100644 --- a/build/config/linux/dbus/BUILD.gn +++ b/build/config/linux/dbus/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/linux/dri/BUILD.gn b/build/config/linux/dri/BUILD.gn index 8e3efe6..e3a0a83 100644 --- a/build/config/linux/dri/BUILD.gn +++ b/build/config/linux/dri/BUILD.gn
@@ -1,15 +1,15 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/linux/pkg_config.gni") -assert(is_linux || is_chromeos, "This file should only be referenced on Linux") +assert(is_linux, "This file should only be referenced on Linux") pkg_config("dri") { packages = [ "dri" ] dri_driver_dir = exec_script(pkg_config_script, - pkg_config_args + [ + common_pkg_config_args + pkg_config_args + [ "--dridriverdir", "dri", ],
diff --git a/build/config/linux/gtk/BUILD.gn b/build/config/linux/gtk/BUILD.gn index ecf95dd..355067e 100644 --- a/build/config/linux/gtk/BUILD.gn +++ b/build/config/linux/gtk/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,12 +7,6 @@ assert(is_linux, "This file should only be referenced on Linux") -declare_args() { - # The (major) version of GTK to build against. A different version may be - # loaded at runtime. - gtk_version = 3 -} - # GN doesn't check visibility for configs so we give this an obviously internal # name to discourage random targets from accidentally depending on this and # bypassing the GTK target's visibility. @@ -33,36 +27,19 @@ group("gtk") { visibility = [ - # This is the only target that can depend on GTK. Do not add more targets - # to this list. - "//ui/gtk:gtk_stubs", - - # These are allow-listed for WebRTC builds. + # These are allow-listed for WebRTC builds. Nothing in else should depend + # on GTK. "//examples:peerconnection_client", "//remoting/host:common", "//remoting/host:remoting_me2me_host_static", "//remoting/host/file_transfer", "//remoting/host/it2me:common", - "//remoting/host/it2me:remote_assistance_host", + "//remoting/host/it2me:main", "//remoting/host/linux", + "//remoting/host/remote_open_url:common", "//remoting/test:it2me_standalone_host_main", "//webrtc/examples:peerconnection_client", ] public_configs = [ ":gtk_internal_config" ] } - -# Depend on "gtkprint" to get this. -pkg_config("gtkprint_internal_config") { - if (gtk_version == 3) { - packages = [ "gtk+-unix-print-3.0" ] - } else { - assert(gtk_version == 4) - packages = [ "gtk4-unix-print" ] - } -} - -group("gtkprint") { - visibility = [ "//ui/gtk:*" ] - public_configs = [ ":gtkprint_internal_config" ] -}
diff --git a/build/config/linux/gtk/gtk.gni b/build/config/linux/gtk/gtk.gni index 1e45248..9e6131d 100644 --- a/build/config/linux/gtk/gtk.gni +++ b/build/config/linux/gtk/gtk.gni
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,5 +6,9 @@ declare_args() { # Whether or not we should use libgtk. - use_gtk = is_linux && !is_chromecast + use_gtk = is_linux && !is_castos + + # The (major) version of GTK to build against. A different version may be + # loaded at runtime. + gtk_version = 3 }
diff --git a/build/config/linux/libdrm/BUILD.gn b/build/config/linux/libdrm/BUILD.gn index e9b4018..31ab0d8 100644 --- a/build/config/linux/libdrm/BUILD.gn +++ b/build/config/linux/libdrm/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/chromecast_build.gni") @@ -11,7 +11,7 @@ # Controls whether the build should use the version of libdrm library shipped # with the system. In release builds of desktop Linux and Chrome OS we use the # system version. Some Chromecast devices use this as well. - use_system_libdrm = is_chromeos_device || (is_linux && !is_chromecast) + use_system_libdrm = is_chromeos_device || (is_linux && !is_castos) } if (use_system_libdrm) {
diff --git a/build/config/linux/libffi/BUILD.gn b/build/config/linux/libffi/BUILD.gn index 59b7f04..771170c 100644 --- a/build/config/linux/libffi/BUILD.gn +++ b/build/config/linux/libffi/BUILD.gn
@@ -1,16 +1,24 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/linux/pkg_config.gni") -if (default_toolchain == "//build/toolchain/cros:target") { +declare_args() { + # Controls whether the build should use the version of libffi library shipped + # with the system. By default, we only use the system version on Chrome OS: + # on Linux, libffi must be statically linked to prevent a situation where the + # runtime version of libffi is different from the build-time version from the + # sysroot. + use_system_libffi = default_toolchain == "//build/toolchain/cros:target" +} + +if (use_system_libffi) { pkg_config("libffi") { packages = [ "libffi" ] } } else { - # On Linux, make sure we link against libffi version 6. config("libffi") { - libs = [ ":libffi.so.6" ] + libs = [ ":libffi_pic.a" ] } }
diff --git a/build/config/linux/libva/BUILD.gn b/build/config/linux/libva/BUILD.gn index ada5d66..380da04 100644 --- a/build/config/linux/libva/BUILD.gn +++ b/build/config/linux/libva/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2018 The Chromium Authors. All rights reserved. +# Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/linux/nss/BUILD.gn b/build/config/linux/nss/BUILD.gn index 8c27938..c67cefc 100644 --- a/build/config/linux/nss/BUILD.gn +++ b/build/config/linux/nss/BUILD.gn
@@ -1,18 +1,14 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/linux/pkg_config.gni") if (is_linux || is_chromeos) { - # This is a dependency on NSS with no libssl. On Linux we use a built-in SSL - # library but the system NSS libraries. Non-Linux platforms using NSS use the - # hermetic one in //third_party/nss. - # - # Generally you should depend on //crypto:platform instead of using this - # config since that will properly pick up NSS or OpenSSL depending on - # platform and build config. - pkg_config("system_nss_no_ssl_config") { + # This is a dependency on NSS with no libssl3. On Linux and Chrome OS, we use + # NSS for platform certificate integration. We use our own TLS library, so + # exclude the one from NSS. + pkg_config("nss") { packages = [ "nss" ] extra_args = [ "-v",
diff --git a/build/config/linux/pangocairo/BUILD.gn b/build/config/linux/pangocairo/BUILD.gn index ddcc754..e2030b8 100644 --- a/build/config/linux/pangocairo/BUILD.gn +++ b/build/config/linux/pangocairo/BUILD.gn
@@ -1,4 +1,4 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2016 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.
diff --git a/build/config/linux/pangocairo/pangocairo.gni b/build/config/linux/pangocairo/pangocairo.gni index ecfe663..c7662ac 100644 --- a/build/config/linux/pangocairo/pangocairo.gni +++ b/build/config/linux/pangocairo/pangocairo.gni
@@ -1,4 +1,4 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. +# Copyright 2017 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -6,8 +6,5 @@ import("//build/config/ui.gni") declare_args() { - use_pangocairo = - # TODO(crbug.com/1052397): Remove !chromeos_is_browser_only once - # lacros-chrome switches to target_os="chromeos" - is_linux && !is_chromecast && !chromeos_is_browser_only + use_pangocairo = is_linux && !is_castos }
diff --git a/build/config/linux/pkg-config.py b/build/config/linux/pkg-config.py index 5adf70c..2e38c7f 100755 --- a/build/config/linux/pkg-config.py +++ b/build/config/linux/pkg-config.py
@@ -1,9 +1,8 @@ -#!/usr/bin/env python -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -from __future__ import print_function import json import os
diff --git a/build/config/linux/pkg_config.gni b/build/config/linux/pkg_config.gni index 428e44a..cb9b460 100644 --- a/build/config/linux/pkg_config.gni +++ b/build/config/linux/pkg_config.gni
@@ -1,4 +1,4 @@ -# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Copyright 2013 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -53,9 +53,10 @@ # need to invoke it manually. pkg_config_args = [] +common_pkg_config_args = [] if (sysroot != "") { # Pass the sysroot if we're using one (it requires the CPU arch also). - pkg_config_args += [ + common_pkg_config_args += [ "-s", rebase_path(sysroot), "-a", @@ -92,9 +93,9 @@ "Variable |packages| must be defined to be a list in pkg_config.") config(target_name) { if (host_toolchain == current_toolchain) { - args = host_pkg_config_args + invoker.packages + args = common_pkg_config_args + host_pkg_config_args + invoker.packages } else { - args = pkg_config_args + invoker.packages + args = common_pkg_config_args + pkg_config_args + invoker.packages } if (defined(invoker.extra_args)) { args += invoker.extra_args
diff --git a/build/config/locales.gni b/build/config/locales.gni index e94e162..ed26f3d 100644 --- a/build/config/locales.gni +++ b/build/config/locales.gni
@@ -1,157 +1,136 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import("//build/config/chromeos/ui_mode.gni") -# This file creates the |locales| which is the set of current -# locales based on the current platform. Locales in this list are formated -# based on what .pak files expect. -# The |locales| variable *may* contain pseudolocales, depending on the -# |enable_pseudolocales| flag. +# This file creates |platform_pak_locales| which is the set of packed locales +# based on the current platform. Locales in this list are formatted based on +# what .pak files expect. The |platform_pak_locales| variable *may* contain +# pseudolocales, depending on the |enable_pseudolocales| flag. # If you specifically want to have the locales variable with or without # pseudolocales, then use |locales_with_pseudolocales| or # |locales_without_pseudolocales|. # The following additional platform specific lists are created: -# - |android_apk_locales| subset for Android based apk builds +# - |extended_locales| list of locales not shipped on desktop builds # - |android_bundle_locales_as_resources| locales formatted for XML output names -# - |locales_as_mac_outputs| formated for mac output bundles -# - |ios_packed_locales| subset for iOS -# - |ios_packed_locales_as_mac_outputs| subset for iOS output +# - |locales_as_apple_outputs| formatted for mac output bundles -# Android doesn't ship all locales in order to save space (but webview does). -# http://crbug.com/369218 -android_apk_omitted_locales = [ - "bn", - "et", - "gu", - "kn", - "ml", - "mr", - "ms", - "ta", - "te", +pseudolocales = [ + "ar-XB", + "en-XA", ] -# Chrome on iOS only ships with a subset of the locales supported by other -# version of Chrome as the corresponding locales are not supported by the -# operating system (but for simplicity, the corresponding .pak files are -# still generated). -ios_unsupported_locales = [ - "am", - "bn", - "et", - "fil", - "gu", - "kn", - "lv", - "ml", - "mr", - "sl", - "sw", - "ta", - "te", -] - -# These list are defined even when not building for Android or iOS for the -# sake of build/locale_tool.py. Ensure that GN doesn't complain about them -# being unused. -not_needed([ "android_apk_omitted_locales" ]) -not_needed([ "ios_unsupported_locales" ]) - # Superset of all locales used in Chrome with platform specific changes noted. -all_chrome_locales = [ - "af", - "am", - "ar", - "as", - "az", - "be", - "bg", - "bn", - "bs", - "ca", - "cs", - "da", - "de", - "el", - "en-GB", - "en-US", - "es", - "es-419", # "es-MX" in iOS (Mexico vs Latin America) "es-US" on Android - "et", - "eu", - "fa", - "fi", - "fil", # "tl" in .xml but "fil" in TC and .pak - "fr", - "fr-CA", - "gl", - "gu", - "he", # "iw" in .xml and TC but "he" in .pak - "hi", - "hr", - "hu", - "hy", - "id", # "in" in .xml but "id" in TC and .pak - "is", - "it", - "ja", - "ka", - "kk", - "km", - "kn", - "ko", - "ky", - "lo", - "lt", - "lv", - "mk", - "ml", - "mn", - "mr", - "ms", - "my", - "nb", # "no" in TC but "nb" in .xml and .pak - "ne", - "nl", - "or", - "pa", - "pl", - "pt-BR", # just "pt" in iOS - "pt-PT", - "ro", - "ru", - "si", - "sk", - "sl", - "sq", - "sr", - "sr-Latn", # -b+sr+Latn in .xml - "sv", - "sw", - "ta", - "te", - "th", - "tr", - "uk", - "ur", - "uz", - "vi", - "zh-CN", - "zh-HK", - "zh-TW", - "zu", -] +all_chrome_locales = + [ + "af", + "am", + "ar", + "as", + "az", + "be", + "bg", + "bn", + "bs", + "ca", + "cs", + "cy", + "da", + "de", + "el", + "en-GB", + "en-US", + "es", + "es-419", # "es-MX" in iOS (Mexico vs Latin America) "es-US" on Android + "et", + "eu", + "fa", + "fi", + "fil", # "tl" in .xml but "fil" in TC and .pak + "fr", + "fr-CA", + "gl", + "gu", + "he", # "iw" in .xml and TC but "he" in .pak + "hi", + "hr", + "hu", + "hy", + "id", # "in" in .xml but "id" in TC and .pak + "is", + "it", + "ja", + "ka", + "kk", + "km", + "kn", + "ko", + "ky", + "lo", + "lt", + "lv", + "mk", + "ml", + "mn", + "mr", + "ms", + "my", + "nb", # "no" in TC but "nb" in .xml and .pak + "ne", + "nl", + "or", + "pa", + "pl", + "pt-BR", # just "pt" in iOS + "pt-PT", + "ro", + "ru", + "si", + "sk", + "sl", + "sq", + "sr", + "sr-Latn", # -b+sr+Latn in .xml + "sv", + "sw", + "ta", + "te", + "th", + "tr", + "uk", + "ur", + "uz", + "vi", + "zh-CN", + "zh-HK", + "zh-TW", + "zu", + ] + pseudolocales -# New locales added to Chrome Android bundle builds. -android_bundle_only_locales = [ - "af", +if (is_ios) { + # Chrome on iOS uses "es-MX" and "pt" for "es-419" and "pt-BR". + all_chrome_locales -= [ + "es-419", + "pt-BR", + ] + all_chrome_locales += [ + "es-MX", + "pt", + ] +} + +# Chrome locales not on Windows, Mac, or Linux. +# This list is used for all platforms except Android. On Android, this list is +# modified to exclude locales that are not used on Android, so +# `platform_pak_locales - extended_locales` works as expected. +extended_locales = [ "as", "az", "be", "bs", + "cy", "eu", "fr-CA", "gl", @@ -171,28 +150,66 @@ "si", "sq", "sr-Latn", - "ur", "uz", "zh-HK", "zu", ] -# New locales added to ChromeOS builds. -chromeos_only_locales = [ "is" ] +# Chrome locales not on Android. +# These locales have not yet been tested yet. Specifically, AOSP has not been +# translated to Welsh at the time of writing (April 2022): +# https://cs.android.com/android/platform/superproject/+/master:build/make/target/product/languages_default.mk +# Due to this, the only way a user could see Welsh strings - assuming they were +# built - would be to manually switch their "Chrome language" in Chrome's +# language settings to Welsh, so Welsh usage would probably be very low. +_non_android_locales = [ "cy" ] +# Setup |platform_pak_locales| for each platform. +platform_pak_locales = all_chrome_locales if (is_android) { - locales = all_chrome_locales + platform_pak_locales -= _non_android_locales + extended_locales -= _non_android_locales +} else { + platform_pak_locales -= extended_locales +} - # Android doesn't ship all locales on KitKat in order to save space - # (but webview does). http://crbug.com/369218 - android_apk_locales = all_chrome_locales - android_bundle_only_locales - - android_apk_omitted_locales +# The base list for all platforms except Android excludes the extended locales. +# Add or subtract platform specific locales below. +if (is_chromeos) { + platform_pak_locales += [ + "cy", + "eu", + "gl", + "is", + "zu", + ] + platform_pak_locales -= [ "ur" ] +} else if (is_ios) { + platform_pak_locales -= [ + "af", + "am", + "bn", + "et", + "fil", + "gu", + "kn", + "lv", + "ml", + "mr", + "sl", + "sw", + "ta", + "te", + "ur", + ] +} - # List for Android locale names in .xml exports. Note: needs to stay in sync - # with |ToAndroidLocaleName| in build/android/gyp/util/resource_utils.py. +# List for Android locale names in .xml exports. Note: needs to stay in sync +# with |ToAndroidLocaleName| in build/android/gyp/util/resource_utils.py. +if (is_android) { # - add r: (e.g. zh-HK -> zh-rHK ) android_bundle_locales_as_resources = [] - foreach(_locale, locales) { + foreach(_locale, platform_pak_locales) { android_bundle_locales_as_resources += [ string_replace(_locale, "-", "-r") ] } @@ -215,58 +232,30 @@ "iw", "tl", ] -} else if (is_chromeos_ash || is_chromeos_lacros) { - # In ChromeOS we support a few more locales than standard Chrome. - locales = - all_chrome_locales - android_bundle_only_locales + chromeos_only_locales -} else { - # Change if other platforms support more locales. - locales = all_chrome_locales - android_bundle_only_locales } -# Chrome on iOS uses different names for "es-419" and "pt-BR" (called -# respectively "es-MX" and "pt" on iOS). -if (is_ios) { - locales -= [ - "es-419", - "pt-BR", - ] - locales += [ - "es-MX", - "pt", - ] -}