third_party/gsutil/gslib/util.py

# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Static data and helper functions."""

import math
import re
import os
import sys
import time

import boto
from third_party.retry_decorator.decorators import retry

# We don't use the oauth2 authentication plugin directly; importing it here
# ensures that it's loaded and available by default. Note: we made this static
# state instead of Command instance state because the top-level gsutil code
# needs to check it.
HAVE_OAUTH2 = False
try:
  from oauth2_plugin import oauth2_helper
  HAVE_OAUTH2 = True
except ImportError:
  pass

TWO_MB = 2 * 1024 * 1024

NO_MAX = sys.maxint

# Binary exponentiation strings.
_EXP_STRINGS = [
  (0, 'B', 'bit'),
  (10, 'KB', 'kbit'),
  (20, 'MB', 'Mbit'),
  (30, 'GB', 'Gbit'),
  (40, 'TB', 'Tbit'),
  (50, 'PB', 'Pbit'),
]

# Detect platform types.
IS_WINDOWS = 'win32' in str(sys.platform).lower()
IS_LINUX = 'linux' in str(sys.platform).lower()
IS_OSX = 'darwin' in str(sys.platform).lower()

Retry = retry

# Enum class for specifying listing style.
class ListingStyle(object):
  SHORT = 'SHORT'
  LONG = 'LONG'
  LONG_LONG = 'LONG_LONG'


def HasConfiguredCredentials(bypass_prodaccess):
  """Determines if boto credential/config file exists."""
  config = boto.config
  has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
                    config.has_option('Credentials', 'gs_secret_access_key'))
  has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
                    config.has_option('Credentials', 'aws_secret_access_key'))
  has_oauth_creds = (HAVE_OAUTH2 and
      config.has_option('Credentials', 'gs_oauth2_refresh_token'))
  has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
  # Pretend prodaccess doesn't exist if --bypass_prodaccess is passed in.
  has_prodaccess = HasExecutable('prodaccess') and not bypass_prodaccess
  return (has_goog_creds or has_amzn_creds or has_oauth_creds
          or has_auth_plugins or has_prodaccess)


def _RoundToNearestExponent(num):
  i = 0
  while i+1 < len(_EXP_STRINGS) and num >= (2 ** _EXP_STRINGS[i+1][0]):
    i += 1
  return i, round(float(num) / 2 ** _EXP_STRINGS[i][0], 2)

def MakeHumanReadable(num):
  """Generates human readable string for a number of bytes.

  Args:
    num: The number, in bytes.

  Returns:
    A string form of the number using size abbreviations (KB, MB, etc.).
  """
  i, rounded_val = _RoundToNearestExponent(num)
  return '%s %s' % (rounded_val, _EXP_STRINGS[i][1])

def MakeBitsHumanReadable(num):
  """Generates human readable string for a number of bits.

  Args:
    num: The number, in bits.

  Returns:
    A string form of the number using bit size abbreviations (kbit, Mbit, etc.)
  """
  i, rounded_val = _RoundToNearestExponent(num)
  return '%s %s' % (rounded_val, _EXP_STRINGS[i][2])

def Percentile(values, percent, key=lambda x:x):
  """Find the percentile of a list of values.

  Taken from: http://code.activestate.com/recipes/511478/

  Args:
    values: a list of numeric values. Note that the values MUST BE already
            sorted.
    percent: a float value from 0.0 to 1.0.
    key: optional key function to compute value from each element of the list
         of values.

  Returns:
    The percentile of the values.
  """
  if not values:
    return None
  k = (len(values) - 1) * percent
  f = math.floor(k)
  c = math.ceil(k)
  if f == c:
    return key(values[int(k)])
  d0 = key(values[int(f)]) * (c-k)
  d1 = key(values[int(c)]) * (k-f)
  return d0 + d1

def ExtractErrorDetail(e):
  """Extract <Details> text from XML content.

  Args:
    e: The GSResponseError that includes XML to be parsed.

  Returns:
    (exception_name, d), where d is <Details> text or None if not found.
  """
  exc_name_parts = re.split("[\.']", str(type(e)))
  if len(exc_name_parts) < 2:
    # Shouldn't happen, but have fallback in case.
    exc_name = str(type(e))
  else:
    exc_name = exc_name_parts[-2]
  if not hasattr(e, 'body'):
    return (exc_name, None)
  detail_start = e.body.find('<Details>')
  detail_end = e.body.find('</Details>')
  if detail_start != -1 and detail_end != -1:
    return (exc_name, e.body[detail_start+9:detail_end])
  return (exc_name, None)


def HasExecutable(filename):
  """Determines if an executable is available on the system."""
  for path in os.environ['PATH'].split(os.pathsep):
    exe_file = os.path.join(path, filename)
    if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
      return True
  return False