build/linux/sysroot_scripts/build_and_upload.py

#!/usr/bin/env python
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Automates running BuildPackageLists, BuildSysroot, and
UploadSysroot for each supported arch of each sysroot creator.
"""

from __future__ import print_function

import glob
import hashlib
import json
import multiprocessing
import os
import re
import string
import subprocess
import sys


def run_script(args):
  fnull = open(os.devnull, 'w')
  subprocess.check_call(args, stdout=fnull, stderr=fnull)


def sha1sumfile(filename):
  sha1 = hashlib.sha1()
  with open(filename, 'rb') as f:
    while True:
      data = f.read(65536)
      if not data:
        break
      sha1.update(data)
  return sha1.hexdigest()


def get_proc_output(args):
  return subprocess.check_output(args).strip()


def build_and_upload(script_path, distro, release, arch, lock):
  script_dir = os.path.dirname(os.path.realpath(__file__))

  run_script([script_path, 'BuildSysroot' + arch])
  run_script([script_path, 'UploadSysroot' + arch])

  tarball = '%s_%s_%s_sysroot.tar.xz' % (distro, release, arch.lower())
  tarxz_path = os.path.join(script_dir, "..", "..", "..", "out",
                            "sysroot-build", release, tarball)
  sha1sum = sha1sumfile(tarxz_path)
  sysroot_dir = '%s_%s_%s-sysroot' % (distro, release, arch.lower())

  sysroot_metadata = {
      'Tarball': tarball,
      'Sha1Sum': sha1sum,
      'SysrootDir': sysroot_dir,
  }
  with lock:
    with open(os.path.join(script_dir, 'sysroots.json'), 'rw+') as f:
      sysroots = json.load(f)
      sysroots["%s_%s" % (release, arch.lower())] = sysroot_metadata
      f.seek(0)
      f.truncate()
      f.write(
          json.dumps(
              sysroots, sort_keys=True, indent=4, separators=(',', ': ')))
      f.write('\n')


def main():
  script_dir = os.path.dirname(os.path.realpath(__file__))
  procs = []
  lock = multiprocessing.Lock()
  for filename in glob.glob(os.path.join(script_dir, 'sysroot-creator-*.sh')):
    script_path = os.path.join(script_dir, filename)
    distro = get_proc_output([script_path, 'PrintDistro'])
    release = get_proc_output([script_path, 'PrintRelease'])
    architectures = get_proc_output([script_path, 'PrintArchitectures'])
    for arch in architectures.split('\n'):
      proc = multiprocessing.Process(
          target=build_and_upload,
          args=(script_path, distro, release, arch, lock))
      procs.append(("%s %s (%s)" % (distro, release, arch), proc))
      proc.start()
  for _, proc in procs:
    proc.join()

  print("SYSROOT CREATION SUMMARY")
  failures = 0
  for name, proc in procs:
    if proc.exitcode:
      failures += 1
    status = "FAILURE" if proc.exitcode else "SUCCESS"
    print("%s sysroot creation\t%s" % (name, status))
  return failures


if __name__ == '__main__':
  sys.exit(main())