#!/usr/bin/env python
# coding=utf8
# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main module for Google Cloud Storage command line tool."""


import ConfigParser
import errno
import getopt
import logging
import os
import re
import signal
import socket
import sys
import traceback

third_party_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, os.path.dirname(third_party_dir))
sys.path.insert(0, third_party_dir)


def _OutputAndExit(message):
  global debug
  if debug == 4:
    stack_trace = traceback.format_exc()
    sys.stderr.write('DEBUG: Exception stack trace:\n    %s\n' %
                     re.sub('\\n', '\n    ', stack_trace))
  else:
    sys.stderr.write('%s\n' % message)
  sys.exit(1)


def _OutputUsageAndExit(command_runner):
  command_runner.RunNamedCommand('help')
  sys.exit(1)


debug = 0
# Before importing boto, find where gsutil is installed and include its
# boto sub-directory at the start of the PYTHONPATH, to ensure the versions of
# gsutil and boto stay in sync after software updates. This also allows gsutil
# to be used without explicitly adding it to the PYTHONPATH.
# We use realpath() below to unwind symlinks if any were used in the gsutil
# installation.
gsutil_bin_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
if not gsutil_bin_dir:
  _OutputAndExit('Unable to determine where gsutil is installed. Sorry, '
                'cannot run correctly without this.\n')
boto_lib_dir = os.path.join(gsutil_bin_dir, '..', 'boto')
if not os.path.isdir(boto_lib_dir):
  _OutputAndExit('There is no boto library under the gsutil install directory '
                '(%s).\nThe gsutil command cannot work properly when installed '
                'this way.\nPlease re-install gsutil per the installation '
                'instructions.' % gsutil_bin_dir)
# sys.path.insert(0, boto_lib_dir)
import boto
from boto.exception import BotoClientError
from boto.exception import InvalidAclError
from boto.exception import InvalidUriError
from boto.exception import ResumableUploadException
from boto.exception import StorageResponseError
from gslib.command_runner import CommandRunner
from gslib.exception import CommandException
from gslib.exception import ProjectIdException
from gslib import util
from gslib.util import ExtractErrorDetail
from gslib.util import HasConfiguredCredentials
from gslib.wildcard_iterator import WildcardException

# Load the gsutil version number and append it to boto.UserAgent so the value
# is set before anything instantiates boto. (If parts of boto were instantiated
# first those parts would have the old value of boto.UserAgent, so we wouldn't
# be guaranteed that all code paths send the correct user agent.)
ver_file_path = os.path.join(gsutil_bin_dir, 'VERSION')
if not os.path.isfile(ver_file_path):
  raise CommandException(
      '%s not found. Please reinstall gsutil from scratch' % ver_file_path)
ver_file = open(ver_file_path, 'r')
gsutil_ver = ver_file.read().rstrip()
ver_file.close()
boto.UserAgent += ' gsutil/%s (%s)' % (gsutil_ver, sys.platform)

# We don't use the oauth2 authentication plugin directly; importing it here
# ensures that it's loaded and available by default when an operation requiring
# authentication is performed.
try:
  from oauth2_plugin import oauth2_plugin
except ImportError:
  pass


def main():
  global debug

  if sys.version_info[:3] < (2, 6):
    raise CommandException('gsutil requires Python 2.6 or higher.')

  bypass_prodaccess = False
  config_file_list = _GetBotoConfigFileList()
  command_runner = CommandRunner(gsutil_bin_dir, boto_lib_dir, config_file_list,
                                 gsutil_ver)
  headers = {}
  parallel_operations = False
  debug = 0

  # If user enters no commands just print the usage info.
  if len(sys.argv) == 1:
    sys.argv.append('help')

  # Change the default of the 'https_validate_certificates' boto option to
  # True (it is currently False in boto).
  if not boto.config.has_option('Boto', 'https_validate_certificates'):
    if not boto.config.has_section('Boto'):
      boto.config.add_section('Boto')
    boto.config.setbool('Boto', 'https_validate_certificates', True)

  try:
    opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mb',
                               ['debug', 'detailedDebug', 'version', 'help',
                                'header', 'multithreaded',
                                'bypass_prodaccess'])
  except getopt.GetoptError, e:
    _HandleCommandException(CommandException(e.msg))
  for o, a in opts:
    if o in ('-d', '--debug'):
      # Passing debug=2 causes boto to include httplib header output.
      debug = 2
    if o in ('-D', '--detailedDebug'):
      # We use debug level 3 to ask gsutil code to output more detailed
      # debug output. This is a bit of a hack since it overloads the same
      # flag that was originally implemented for boto use. And we use -DD
      # to ask for really detailed debugging (i.e., including HTTP payload).
      if debug == 3:
        debug = 4
      else:
        debug = 3
    if o in ('-?', '--help'):
      _OutputUsageAndExit(command_runner)
    if o in ('-h', '--header'):
      (hdr_name, unused_ptn, hdr_val) = a.partition(':')
      if not hdr_name:
        _OutputUsageAndExit(command_runner)
      headers[hdr_name] = hdr_val
    if o in ('-m', '--multithreaded'):
      parallel_operations = True
    if o in ('-b', '--bypass_prodaccess'):
      bypass_prodaccess = True
  if debug > 1:
    sys.stderr.write(
        '***************************** WARNING *****************************\n'
        '*** You are running gsutil with debug output enabled.\n'
        '*** Be aware that debug output includes authentication '
        'credentials.\n'
        '*** Do not share (e.g., post to support forums) debug output\n'
        '*** unless you have sanitized authentication tokens in the\n'
        '*** output, or have revoked your credentials.\n'
        '***************************** WARNING *****************************\n')
  if debug == 2:
    logging.basicConfig(level=logging.INFO)
  elif debug > 2:
    logging.basicConfig(level=logging.DEBUG)
    command_runner.RunNamedCommand('ver')
    config_items = []
    try:
      config_items.extend(boto.config.items('Boto'))
      config_items.extend(boto.config.items('GSUtil'))
    except ConfigParser.NoSectionError:
      pass
    sys.stderr.write('config_file_list: %s\n' % config_file_list)
    sys.stderr.write('config: %s\n' % str(config_items))
  else:
    logging.basicConfig()

  if not args:
    command_name = 'help'
  else:
    command_name = args[0]

  if not bypass_prodaccess:
    import plugins.sso_auth

  return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
                                             args[1:], headers, debug,
                                             parallel_operations,
                                             bypass_prodaccess)


def _GetBotoConfigFileList():
  """Returns list of boto config files that exist."""
  config_paths = boto.pyami.config.BotoConfigLocations
  if 'AWS_CREDENTIAL_FILE' in os.environ:
    config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
  config_files = {}
  for config_path in config_paths:
    if os.path.exists(config_path):
      config_files[config_path] = 1
  cf_list = []
  for config_file in config_files:
    cf_list.append(config_file)
  return cf_list


def _HandleUnknownFailure(e):
  global debug
  # Called if we fall through all known/handled exceptions. Allows us to
  # print a stacktrace if -D option used.
  if debug > 2:
    stack_trace = traceback.format_exc()
    sys.stderr.write('DEBUG: Exception stack trace:\n    %s\n' %
                     re.sub('\\n', '\n    ', stack_trace))
  else:
    _OutputAndExit('Failure: %s.' % e)


def _HandleCommandException(e):
  if e.informational:
    _OutputAndExit(e.reason)
  else:
    _OutputAndExit('CommandException: %s' % e.reason)


def _HandleControlC(signal_num, cur_stack_frame):
  """Called when user hits ^C so we can print a brief message instead of
  the normal Python stack trace (unless -D option is used)."""
  global debug
  if debug > 2:
    stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
    _OutputAndExit('DEBUG: Caught signal %d - Exception stack trace:\n'
                  '    %s' % (signal_num, re.sub('\\n', '\n    ', stack_trace)))
  else:
    _OutputAndExit('Caught signal %d - exiting' % signal_num)


def _HandleSigQuit(signal_num, cur_stack_frame):
  """Called when user hits ^\, so we can force breakpoint a running gsutil."""
  import pdb; pdb.set_trace()


def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
                                        headers=None, debug=0,
                                        parallel_operations=False,
                                        bypass_prodaccess=False):
  try:
    # Catch ^C so we can print a brief message instead of the normal Python
    # stack trace.
    signal.signal(signal.SIGINT, _HandleControlC)
    # Catch ^\ so we can force a breakpoint in a running gsutil.
    if not util.IS_WINDOWS:
      signal.signal(signal.SIGQUIT, _HandleSigQuit)
    return command_runner.RunNamedCommand(command_name, args, headers, debug,
                                          parallel_operations,
                                          bypass_prodaccess=bypass_prodaccess)
  except AttributeError, e:
    if str(e).find('secret_access_key') != -1:
      _OutputAndExit('Missing credentials for the given URI(s). Does your '
                    'boto config file contain all needed credentials?')
    else:
      _OutputAndExit(str(e))
  except BotoClientError, e:
    _OutputAndExit('BotoClientError: %s.' % e.reason)
  except CommandException, e:
    _HandleCommandException(e)
  except getopt.GetoptError, e:
    _HandleCommandException(CommandException(e.msg))
  except InvalidAclError, e:
    _OutputAndExit('InvalidAclError: %s.' % str(e))
  except InvalidUriError, e:
    _OutputAndExit('InvalidUriError: %s.' % e.message)
  except ProjectIdException, e:
    _OutputAndExit('ProjectIdException: %s.' % e.reason)
  except boto.auth_handler.NotReadyToAuthenticate:
    _OutputAndExit('NotReadyToAuthenticate')
  except OSError, e:
    _OutputAndExit('OSError: %s.' % e.strerror)
  except WildcardException, e:
    _OutputAndExit(e.reason)
  except StorageResponseError, e:
    # Check for access denied, and provide detail to users who have no boto
    # config file (who might previously have been using gsutil only for
    # accessing publicly readable buckets and objects).
    if e.status == 403:
      if not HasConfiguredCredentials(bypass_prodaccess):
        _OutputAndExit(
            'You are attempting to access protected data with no configured '
            'credentials.\nPlease see '
            'http://code.google.com/apis/storage/docs/signup.html for\ndetails '
            'about activating the Google Cloud Storage service and then run '
            'the\n"gsutil config" command to configure gsutil to use these '
            'credentials.')
      elif (e.error_code == 'AccountProblem'
            and ','.join(args).find('gs://') != -1):
        default_project_id = boto.config.get_value('GSUtil',
                                                   'default_project_id')
        acct_help_part_1 = (
"""Your request resulted in an AccountProblem (403) error. Usually this happens
if you attempt to create a bucket or upload an object without having first
enabled billing for the project you are using. To remedy this problem, please do
the following:

1. Navigate to the Google APIs console (https://code.google.com/apis/console),
   and ensure the drop-down selector beneath "Google APIs" shows the project
   you're attempting to use.

""")
        acct_help_part_2 = '\n'
        if default_project_id:
          acct_help_part_2 = (
"""2. Click "Google Cloud Storage" on the left hand pane, and then check that
   the value listed for "x-goog-project-id" on this page matches the project ID
   (%s) from your boto config file.

""" % default_project_id)
        acct_help_part_3 = (
"""Check whether there's an "!" next to Billing. If so, click Billing and then
   enable billing for this project. Note that it can take up to one hour after
   enabling billing for the project to become activated for creating buckets and
   uploading objects.

If the above doesn't resolve your AccountProblem, please send mail to
gs-team@google.com requesting assistance, noting the exact command you ran, the
fact that you received a 403 AccountProblem error, and your project ID. Please
do not post your project ID on the public discussion forum (gs-discussion) or on
StackOverflow.

Note: It's possible to use Google Cloud Storage without enabling billing if
you're only listing or reading objects for which you're authorized, or if
you're uploading objects to a bucket billed to a project that has billing
enabled. But if you're attempting to create buckets or upload objects to a
bucket owned by your own project, you must first enable billing for that
project.""")
        if default_project_id:
            _OutputAndExit(acct_help_part_1 + acct_help_part_2 + '3. '
                           + acct_help_part_3)
        else:
            _OutputAndExit(acct_help_part_1 + '2. ' + acct_help_part_3)

    if not e.body:
      e.body = ''
    exc_name, error_detail = ExtractErrorDetail(e)
    if error_detail:
      _OutputAndExit('%s: status=%d, code=%s, reason=%s, detail=%s.' %
                    (exc_name, e.status, e.code, e.reason, error_detail))
    else:
      _OutputAndExit('%s: status=%d, code=%s, reason=%s.' %
                    (exc_name, e.status, e.code, e.reason))
  except ResumableUploadException, e:
    _OutputAndExit('ResumableUploadException: %s.' % e.message)
  except socket.error, e:
    if e.args[0] == errno.EPIPE:
      # Retrying with a smaller file (per suggestion below) works because
      # the library code send loop (in boto/s3/key.py) can get through the
      # entire file and then request the HTTP response before the socket
      # gets closed and the response lost.
      message = (
"""
Got a "Broken pipe" error. This can happen to clients using Python 2.x,
when the server sends an error response and then closes the socket (see
http://bugs.python.org/issue5542). If you are trying to upload a large
object you might retry with a small (say 200k) object, and see if you get
a more specific error code.
""")
      _OutputAndExit(message)
    else:
      _HandleUnknownFailure(e)
  except Exception, e:
    _HandleUnknownFailure(e)


if __name__ == '__main__':
  sys.exit(main())