src/third_party/mozjs-45/js/src/devtools/rootAnalysis/ra.away/tools/lib/python/release/download.py

import os
from os import path
import urllib
import socket
from urllib import urlretrieve
from urllib2 import urlopen, HTTPError, URLError

from redo import retrier
import requests

from release.platforms import ftp_platform_map, buildbot2ftp
from release.l10n import makeReleaseRepackUrls
from release.paths import makeCandidatesDir
from util.paths import windows2msys, msys2windows
from util.file import directoryContains
from util.commands import run_cmd

import logging
log = logging.getLogger(__name__)

installer_ext_map = {
    'win32': ".exe",
    'win64': ".exe",
    'macosx': ".dmg",
    'macosx64': ".dmg",
    'linux': ".tar.bz2",
    'linux64': ".tar.bz2",
}


def getInstallerExt(platform):
    """ Return the file extension of the installer file on a given platform,
    raising a KeyError if the platform is not found """
    return installer_ext_map[platform]


def downloadReleaseBuilds(stageServer, productName, brandName, version,
                          buildNumber, platform, candidatesDir=None,
                          signed=False, usePymake=False):
    if candidatesDir is None:
        candidatesDir = makeCandidatesDir(productName, version, buildNumber,
                                          protocol='http', server=stageServer)
    files = makeReleaseRepackUrls(productName, brandName, version, platform,
                                  signed=signed)

    env = {}
    for fileName, remoteFile in files.iteritems():
        url = '/'.join([p.strip('/') for p in [candidatesDir,
                                               urllib.quote(remoteFile)]])
        log.info("Downloading %s to %s", url, fileName)
        for _ in retrier():
            with open(fileName, "wb") as f:
                try:
                    r = requests.get(url, stream=True, timeout=15)
                    r.raise_for_status()
                    for chunk in r.iter_content(chunk_size=5*1024**2):
                        f.write(chunk)
                    r.close()
                    break
                except (requests.HTTPError, requests.ConnectionError,
                        requests.Timeout):
                    log.exception("Caught exception downloading")

        if fileName.endswith('exe'):
            if usePymake:
                env['WIN32_INSTALLER_IN'] = msys2windows(path.join(os.getcwd(),
                                                         fileName))
            else:
                env['WIN32_INSTALLER_IN'] = windows2msys(path.join(os.getcwd(),
                                                         fileName))
        else:
            if platform.startswith('win') and not usePymake:
                env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName))
            else:
                env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName))

    return env


def downloadUpdate(stageServer, productName, version, buildNumber,
                   platform, locale, candidatesDir=None):
    if candidatesDir is None:
        candidatesDir = makeCandidatesDir(productName, version, buildNumber,
                                          protocol='http', server=stageServer)
    fileName = '%s-%s.complete.mar' % (productName, version)
    destFileName = '%s-%s.%s.complete.mar' % (productName, version, locale)
    platformDir = buildbot2ftp(platform)
    url = '/'.join([p.strip('/') for p in [
        candidatesDir, 'update', platformDir, locale, fileName]])
    log.info("Downloading %s to %s", url, destFileName)
    remote_f = urlopen(url)
    local_f = open(destFileName, "wb")
    local_f.write(remote_f.read())
    local_f.close()
    return destFileName


def downloadUpdateIgnore404(*args, **kwargs):
    try:
        return downloadUpdate(*args, **kwargs)
    except HTTPError, e:
        if e.code == 404:
            # New locale
            log.warning('Got 404. Skipping %s' % e.geturl())
            return None
        else:
            raise


def rsyncFilesByPattern(server, userName, sshKey, source_dir, target_dir,
                        pattern):
    cmd = ['rsync', '-e',
           'ssh -l %s -oIdentityFile=%s' % (userName, sshKey),
           '-av', '--include=%s' % pattern, '--include=*/', '--exclude=*',
           '%s:%s' % (server, source_dir), target_dir]
    run_cmd(cmd)


def rsyncFiles(files, server, userName, sshKey, target_dir):
    cmd = ['rsync', '-e',
           'ssh -l %s -oIdentityFile=%s' % (userName, sshKey),
           '-av'] + files + ['%s:%s' % (server, target_dir)]
    run_cmd(cmd)


def url_exists(url, timeout=10):
    """simple function that verifies if given a url exists.
    Args:
        url (str): url to check
        timeout (int): timeout in seconds
    Returns:
        bool: True if url exists. False if the url does not exists (any http
                error) or the operation does not complete in timeout seconds.
    """
    exists = True
    try:
        urlopen(url, timeout=timeout)
    except (HTTPError, URLError, socket.timeout):
        # socket.timeout is required for python > 2.7
        exists = False
    return exists