#!/usr/bin/python
# Copyright 2022 The Cobalt Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tools for creating a compressed tar archive.

Usage:
  create_archive.py -d <dest_path> -s <source_paths>

Args:
    dest_path: Path (can be absolute or relative) to the destination folder
      where the compressed archive is created.
    source_paths: Paths (can be absolute or relative) to the source folders
      which should archived.

Returns:
    0 on success.

Raises:
    ValueError: An error occurred when adding a source to the archive, when
      compressing the archive, or when the archive has uppercase symlink names.
"""

import argparse
import glob
import logging
import os
import subprocess
import sys
import time

# Use strict include filter to pass artifacts to Mobile Harness
TEST_PATTERNS = [
    'content/*',
    'install/*',
    # TODO: All of those should be built under deploy/
    '*.apk',
    'ds_archive.zip',
    'app_launcher.zip',
    'native_target/crashpad_handler',
    'elf_loader_sandbox'
]

SOURCE_DIR = r'<source_dir>'

# These files are not final build artifacts nor inetermediate build artifacts
# needed.
BUILDBOT_INTERMEDIATE_FILE_NAMES_WILDCARDS = (
    r'gypfiles',
    r'gyp-win-tool',
    r'environment.x64',
    r'environment.x86',
    r'obj',
    r'obj.host',
    r'gradle',  # Android intermediates
    SOURCE_DIR + r'lib/*.so',
    r'intermediates',
    r'*.ninja',
    r'*.ninja_deps',
    r'*.ninja_log',
    r'*.mp4',
    r'*.deb',
)

# These files are not final build artifacts. Values should match those in
# INTERMEDIATE_WILDCARD_FILE_NAMES_RE
INTERMEDIATE_FILE_NAMES_WILDCARDS = (
    r'gen',
    r'gypfiles',
    r'gyp-win-tool',
    r'environment.x64',
    r'environment.x86',
    r'obj',
    r'obj.host',
    r'gradle',  # Android intermediates
    SOURCE_DIR + r'lib/*.so',
    r'intermediates',
    r'*.ninja',
    r'*.ninja_deps',
    r'*.ninja_log',
    r'*.mp4',
    r'*.deb',
)

# Reference: https://sevenzip.osdn.jp/chm/cmdline/switches/
_7Z_PATH = r'%ProgramFiles%\7-Zip\7z.exe'  # pylint:disable=invalid-name
# 1 is fastest and least compression, max is 9, see -mx parameter in docs
_COMPRESSION_LEVEL = 1

_LINUX_PARALLEL_ZIP_EXTENSION = 'tar.xz'
_LINUX_PARALLEL_ZIP_PROGRAM = 'xz -T0'
_LINUX_SERIAL_ZIP_EXTENSION = 'tar.gz'
_LINUX_SERIAL_ZIP_PROGRAM = 'gzip'


# TODO(b/143570416): Refactor this method and similar ones in a common file.
def _IsWindows():
  return sys.platform in ['win32']


class InvalidArchiveExtensionException(RuntimeError):
  pass


def _CheckArchiveExtension(archive_path, is_parallel):
  if is_parallel and (_LINUX_PARALLEL_ZIP_EXTENSION not in archive_path):
    raise InvalidArchiveExtensionException(
        f'Parallelized zip requested, but path is {archive_path}')
  if not is_parallel and (_LINUX_PARALLEL_ZIP_EXTENSION in archive_path):
    raise InvalidArchiveExtensionException(
        f'Serialized zip requested, but path is: {archive_path}')


def _CreateCompressedArchive(source_paths,
                             dest_path,
                             patterns,
                             intermediate,
                             is_parallel=False):
  """Create a compressed tar archive.

  This function creates a compressed tar archive from all of the source
  directories in |source_paths|. The basename of each source directory is
  maintained within the archive to prevent file overwriting on extraction.

  Args:
    source_paths: Paths (that can be absolute or relative) to the source folders
      which should be archived.
    dest_path: Path (can be absolute or relative) that the compressed archive
      should be created at.
    patterns: Patterns of allowable files to archive.
    intermediate: Flag to archive intermediate build artifacts needed by
      Buildbot.

  Raises:
    ValueError: An error occurred when adding a source to the archive, when
      compressing the archive, or when the archive has uppercase symlink names.
  """
  # Ensure the directory where we will create our archive exists, and that the
  # directory does not contain a previous archive.
  dest_parent = os.path.dirname(os.path.abspath(dest_path))
  if os.path.exists(dest_path):
    os.remove(dest_path)
  elif not os.path.exists(dest_parent):
    os.makedirs(dest_parent)

  # Get the current working directory to return to once we are done archiving.
  cwd = os.getcwd()

  _CheckArchiveExtension(dest_path, is_parallel)

  try:
    intermediate_tar_path = os.path.join(
        dest_parent, 'create_archive_intermediate_artifacts.tar')

    # Generate the commands to archive each source directory and execute them
    # from the parent directory of the source we are adding to the archive.
    for source_path in source_paths:
      # Determine if only certain contents of the source_path are to be
      # included.
      if ',' in source_path:
        source_path_parts = source_path.split(',')
        source_path = source_path_parts[0]
        included_paths = source_path_parts[1:]
      else:
        included_paths = ['']

      logging.info('source path: %s', source_path)
      logging.info('included subpaths: %s', included_paths)

      if not os.path.exists(source_path):
        raise ValueError(f'Failed to find source path "{source_path}".')

      for target_path in included_paths:
        if not os.path.exists(os.path.join(source_path, target_path)):
          raise ValueError(f'Failed to find target path "{target_path}".')

      source_parent_dir, source_dir_name = os.path.split(
          os.path.abspath(source_path))

      os.chdir(source_parent_dir)
      t_start = time.time()
      for included_path in included_paths:
        # Check every path to make sure there is no symlink being created with
        # uppercase in it. This causes issues in Windows Archives.
        for dirpath, dirnames, _ in os.walk(included_path):
          for dirname in dirnames:
            iterpath = os.path.join(dirpath, dirname)
            if os.path.islink(iterpath) and iterpath.lower() != iterpath:
              raise ValueError('Archive contains uppercase symlink names.\n'
                               'Please change these names to lowercase.')
        # Iteratively add filtered source files to the final tar
        _AddSourceToTar(
            os.path.join(source_dir_name, included_path), intermediate_tar_path,
            patterns, intermediate)
        t_archive = time.time() - t_start
        logging.info('Archiving took %5.2f seconds', t_archive)
      os.chdir(cwd)

    archive_size = os.path.getsize(intermediate_tar_path) / 1e9
    logging.info('Size of %s: %5.2f Gi', intermediate_tar_path, archive_size)
    # Compress the archive.
    cmd = _CreateZipCommand(intermediate_tar_path, dest_path, is_parallel)
    logging.info('Compressing "%s" into %s with "%s".', intermediate_tar_path,
                 dest_path, cmd)
    t_start = time.time()
    subprocess.check_call(cmd, shell=True)
    t_compress = time.time() - t_start
    logging.info('Compressing took %5.2f seconds', t_compress)
    archive_size = os.path.getsize(dest_path) / 1e9
    logging.info('Size of %s: %5.2f Gi', dest_path, archive_size)
  finally:
    if os.path.exists(intermediate_tar_path):
      os.remove(intermediate_tar_path)


def UncompressArchive(source_path, dest_path, is_parallel=False):
  """Uncompress tar archive.

  Args:
    source_path: Path (that can be absolute or relative) to the archive.
    dest_path: Path (can be absolute or relative) where the archive should be
      uncompressed to.
  """
  _CheckArchiveExtension(source_path, is_parallel)

  # Creates and cleans up destination path
  dest_path = os.path.abspath(dest_path)
  if not os.path.exists(dest_path):
    os.makedirs(dest_path)

  try:
    intermediate_tar_path = os.path.join(
        dest_path, 'create_archive_intermediate_artifacts.tar')

    # Unzips archive
    cmd = _CreateUnzipCommand(source_path, intermediate_tar_path, is_parallel)
    logging.info('Unzipping "%s" into %s with "%s".', source_path,
                 intermediate_tar_path, cmd)
    subprocess.check_call(cmd, shell=True)

    # Untars archive
    cmd = _CreateUntarCommand(intermediate_tar_path)
    logging.info('Untarring "%s" into "%s" with "%s".', intermediate_tar_path,
                 dest_path, cmd)
    subprocess.check_call(cmd, shell=True, cwd=dest_path)
  finally:
    os.remove(intermediate_tar_path)


def _AddSourceToTar(source_path, intermediate_tar_path, patterns, intermediate):
  """Add a directory to an archive.

  The tarfile Python 2.7 module does not work with symlinks, so the operation
  is done in platform-specific ways.

  Args:
    source_path: Source directory to add to archive.
    intermediate_tar_path: Path to tar file to add to (or create).
    patterns: Patterns of allowable files to archive.
    intermediate: Flag to archive intermediate build artifacts needed by
      Buildbot.
  """
  cmd = _CreateTarCommand(source_path, intermediate_tar_path, patterns,
                          intermediate)
  logging.info('Adding "%s" to "%s" with "%s".', source_path,
               intermediate_tar_path, cmd)
  subprocess.check_call(cmd, shell=True)


def _CreateTarCommand(source_path, intermediate_tar_path, patterns,
                      intermediate):
  if _IsWindows():
    return _CreateWindowsTarCmd(source_path, intermediate_tar_path, patterns,
                                intermediate)
  return _CreateLinuxTarCmd(source_path, intermediate_tar_path, patterns,
                            intermediate)


def _CreateWindowsTarCmd(source_path, intermediate_tar_path, patterns,
                         intermediate):
  exclude_patterns = INTERMEDIATE_FILE_NAMES_WILDCARDS
  if intermediate:
    exclude_patterns = BUILDBOT_INTERMEDIATE_FILE_NAMES_WILDCARDS
  excludes = ' '.join(
      [f'-xr!{x.replace(SOURCE_DIR, source_path)}' for x in exclude_patterns])
  contents = [
      os.path.join(source_path, pattern)
      for pattern in patterns
      if glob.glob(os.path.join(source_path, pattern))
  ]
  files_to_tar = ' '.join(contents)
  return (f'"{_7Z_PATH}" a {excludes} -bsp1 -snl -ttar '
          f'{intermediate_tar_path} {files_to_tar}')


def _CreateLinuxTarCmd(source_path, intermediate_tar_path, patterns,
                       intermediate):
  exclude_patterns = INTERMEDIATE_FILE_NAMES_WILDCARDS
  if intermediate:
    exclude_patterns = BUILDBOT_INTERMEDIATE_FILE_NAMES_WILDCARDS
  excludes = ' '.join([
      f'--exclude="{x.replace(SOURCE_DIR, source_path)}"'
      for x in exclude_patterns
  ])
  mode = 'r' if os.path.exists(intermediate_tar_path) else 'c'
  contents = [
      os.path.join(source_path, pattern)
      for pattern in patterns
      if glob.glob(os.path.join(source_path, pattern))
  ]
  files_to_tar = ' '.join(contents)
  return (f'tar -{mode}f {intermediate_tar_path} --format=posix '
          f'{excludes} {files_to_tar}')


def _CreateUntarCommand(intermediate_tar_path):
  if _IsWindows():
    return f'"{_7Z_PATH}" x -bsp1 {intermediate_tar_path}'
  else:
    return f'tar -xf {intermediate_tar_path}'


def _CreateZipCommand(intermediate_tar_path, dest_path, is_parallel=False):
  if _IsWindows():
    return (f'"{_7Z_PATH}" a -bsp1 -mx={_COMPRESSION_LEVEL} '
            f'-mmt=on {dest_path} {intermediate_tar_path}')
  else:
    zip_program = (
        _LINUX_PARALLEL_ZIP_PROGRAM
        if is_parallel else _LINUX_SERIAL_ZIP_PROGRAM)
    return f'{zip_program} -vc -1 {intermediate_tar_path} > {dest_path}'


def _CreateUnzipCommand(source_path, intermediate_tar_path, is_parallel=False):
  if _IsWindows():
    tar_parent = os.path.dirname(intermediate_tar_path)
    return f'"{_7Z_PATH}" x -bsp1 {source_path} -o{tar_parent}'
  else:
    zip_program = (
        _LINUX_PARALLEL_ZIP_PROGRAM
        if is_parallel else _LINUX_SERIAL_ZIP_PROGRAM)
    return f'{zip_program} -dvc {source_path} > {intermediate_tar_path}'


def _CleanUp(intermediate_tar_path):
  if os.path.exists(intermediate_tar_path):
    os.remove(intermediate_tar_path)


def main():
  logging_format = '[%(levelname)s:%(filename)s:%(lineno)s] %(message)s'
  logging.basicConfig(
      level=logging.INFO, format=logging_format, datefmt='%H:%M:%S')

  parser = argparse.ArgumentParser()
  parser.add_argument(
      '-d',
      '--dest_path',
      type=str,
      required=True,
      help='The path to create the archive at.')
  parser.add_argument(
      '-s',
      '--source_paths',
      type=str,
      nargs='+',
      required=True,
      help='The paths to the source folders which should be archived. '
      'To include only specific subdirectories, but compress relative '
      'to some root directory, pass: "<root>,<target_subdir>,...".')
  parser.add_argument(
      '-x',
      '--extract',
      default=False,
      action='store_true',
      help='Uncompresses tar archive.')
  parser.add_argument(
      '-p',
      '--patterns',
      type=str,
      nargs='+',
      required=False,
      default=['*'],
      help='The patterns of allowable files to archive. Defaults to *.')
  parser.add_argument(
      '--test_infra',
      default=False,
      action='store_true',
      help='Utilizes predefined test infrastructure patterns.')
  parser.add_argument(
      '--intermediate',
      default=False,
      action='store_true',
      help='Archives intermediate build artifacts needed by Buildbot.')
  parser.add_argument(
      '--parallel',
      default=False,
      action='store_true',
      help='Archives using parallelized methods.')
  args = parser.parse_args()

  if args.extract:
    UncompressArchive(args.source_paths[0], args.dest_path, args.parallel)
  else:
    _CreateCompressedArchive(
        args.source_paths, args.dest_path,
        TEST_PATTERNS if args.test_infra else args.patterns, args.intermediate,
        args.parallel)


if __name__ == '__main__':
  main()