| # Copyright 2011 Google Inc. All Rights Reserved. |
| # Copyright 2011, Nexenta Systems Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import boto |
| import errno |
| import gzip |
| import hashlib |
| import mimetypes |
| import os |
| import platform |
| import re |
| import subprocess |
| import stat |
| import sys |
| import tempfile |
| import threading |
| import time |
| |
| from boto import config |
| from boto.exception import GSResponseError |
| from boto.exception import ResumableUploadException |
| from boto.gs.resumable_upload_handler import ResumableUploadHandler |
| from boto.s3.keyfile import KeyFile |
| from boto.s3.resumable_download_handler import ResumableDownloadHandler |
| from boto.storage_uri import BucketStorageUri |
| from gslib.command import COMMAND_NAME |
| from gslib.command import COMMAND_NAME_ALIASES |
| from gslib.command import CONFIG_REQUIRED |
| from gslib.command import Command |
| from gslib.command import FILE_URIS_OK |
| from gslib.command import MAX_ARGS |
| from gslib.command import MIN_ARGS |
| from gslib.command import PROVIDER_URIS_OK |
| from gslib.command import SUPPORTED_SUB_ARGS |
| from gslib.command import URIS_START_ARG |
| from gslib.exception import CommandException |
| from gslib.help_provider import HELP_NAME |
| from gslib.help_provider import HELP_NAME_ALIASES |
| from gslib.help_provider import HELP_ONE_LINE_SUMMARY |
| from gslib.help_provider import HELP_TEXT |
| from gslib.help_provider import HELP_TYPE |
| from gslib.help_provider import HelpType |
| from gslib.name_expansion import NameExpansionIterator |
| from gslib.util import ExtractErrorDetail |
| from gslib.util import IS_WINDOWS |
| from gslib.util import MakeHumanReadable |
| from gslib.util import NO_MAX |
| from gslib.util import TWO_MB |
| from gslib.wildcard_iterator import ContainsWildcard |
| |
| _detailed_help_text = (""" |
| <B>SYNOPSIS</B> |
| gsutil cp [OPTION]... src_uri dst_uri |
| - or - |
| gsutil cp [OPTION]... src_uri... dst_uri |
| - or - |
| gsutil cp [OPTION]... -I dst_uri |
| |
| |
| <B>DESCRIPTION</B> |
| The gsutil cp command allows you to copy data between your local file |
| system and the cloud, copy data within the cloud, and copy data between |
| cloud storage providers. For example, to copy all text files from the |
| local directory to a bucket you could do: |
| |
| gsutil cp *.txt gs://my_bucket |
| |
| Similarly, you can download text files from a bucket by doing: |
| |
| gsutil cp gs://my_bucket/*.txt . |
| |
| If you want to copy an entire directory tree you need to use the -R option: |
| |
| gsutil cp -R dir gs://my_bucket |
| |
| If you have a large number of files to upload you might want to use the |
| gsutil -m option, to perform a parallel (multi-threaded/multi-processing) |
| copy: |
| |
| gsutil -m cp -R dir gs://my_bucket |
| |
| You can pass a list of URIs to copy on STDIN instead of as command line |
| arguments by using the -I option. This allows you to use gsutil in a |
| pipeline to copy files and objects as generated by a program, such as: |
| |
| some_program | gsutil -m cp -I gs://my_bucket |
| |
| The contents of STDIN can name files, cloud URIs, and wildcards of files |
| and cloud URIs. |
| |
| |
| <B>HOW NAMES ARE CONSTRUCTED</B> |
| The gsutil cp command strives to name objects in a way consistent with how |
| Linux cp works, which causes names to be constructed in varying ways depending |
| on whether you're performing a recursive directory copy or copying |
| individually named objects; and whether you're copying to an existing or |
| non-existent directory. |
| |
| When performing recursive directory copies, object names are constructed |
| that mirror the source directory structure starting at the point of |
| recursive processing. For example, the command: |
| |
| gsutil cp -R dir1/dir2 gs://my_bucket |
| |
| will create objects named like gs://my_bucket/dir2/a/b/c, assuming |
| dir1/dir2 contains the file a/b/c. |
| |
| In contrast, copying individually named files will result in objects named |
| by the final path component of the source files. For example, the command: |
| |
| gsutil cp dir1/dir2/** gs://my_bucket |
| |
| will create objects named like gs://my_bucket/c. |
| |
| The same rules apply for downloads: recursive copies of buckets and |
| bucket subdirectories produce a mirrored filename structure, while copying |
| individually (or wildcard) named objects produce flatly named files. |
| |
| Note that in the above example the '**' wildcard matches all names |
| anywhere under dir. The wildcard '*' will match names just one level deep. For |
| more details see 'gsutil help wildcards'. |
| |
| There's an additional wrinkle when working with subdirectories: the resulting |
| names depend on whether the destination subdirectory exists. For example, |
| if gs://my_bucket/subdir exists as a subdirectory, the command: |
| |
| gsutil cp -R dir1/dir2 gs://my_bucket/subdir |
| |
| will create objects named like gs://my_bucket/subdir/dir2/a/b/c. In contrast, |
| if gs://my_bucket/subdir does not exist, this same gsutil cp command will |
| create objects named like gs://my_bucket/subdir/a/b/c. |
| |
| |
| <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B> |
| You can use gsutil to copy to and from subdirectories by using a command like: |
| |
| gsutil cp -R dir gs://my_bucket/data |
| |
| This will cause dir and all of its files and nested subdirectories to be |
| copied under the specified destination, resulting in objects with names like |
| gs://my_bucket/data/dir/a/b/c. Similarly you can download from bucket |
| subdirectories by using a command like: |
| |
| gsutil cp -R gs://my_bucket/data dir |
| |
| This will cause everything nested under gs://my_bucket/data to be downloaded |
| into dir, resulting in files with names like dir/data/a/b/c. |
| |
| Copying subdirectories is useful if you want to add data to an existing |
| bucket directory structure over time. It's also useful if you want |
| to parallelize uploads and downloads across multiple machines (often |
| reducing overall transfer time compared with simply running gsutil -m |
| cp on one machine). For example, if your bucket contains this structure: |
| |
| gs://my_bucket/data/result_set_01/ |
| gs://my_bucket/data/result_set_02/ |
| ... |
| gs://my_bucket/data/result_set_99/ |
| |
| you could perform concurrent downloads across 3 machines by running these |
| commands on each machine, respectively: |
| |
| gsutil -m cp -R gs://my_bucket/data/result_set_[0-3]* dir |
| gsutil -m cp -R gs://my_bucket/data/result_set_[4-6]* dir |
| gsutil -m cp -R gs://my_bucket/data/result_set_[7-9]* dir |
| |
| Note that dir could be a local directory on each machine, or it could |
| be a directory mounted off of a shared file server; whether the latter |
| performs acceptably may depend on a number of things, so we recommend |
| you experiment and find out what works best for you. |
| |
| |
| <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> |
| If both the source and destination URI are cloud URIs from the same |
| provider, gsutil copies data "in the cloud" (i.e., without downloading |
| to and uploading from the machine where you run gsutil). In addition to |
| the performance and cost advantages of doing this, copying in the cloud |
| preserves metadata (like Content-Type and Cache-Control). In contrast, |
| when you download data from the cloud it ends up in a file, which has |
| no associated metadata. Thus, unless you have some way to hold on to |
| or re-create that metadata, downloading to a file will not retain the |
| metadata. |
| |
| Note that by default, the gsutil cp command does not copy the object |
| ACL to the new object, and instead will use the default bucket ACL (see |
| "gsutil help setdefacl"). You can override this behavior with the -p |
| option (see OPTIONS below). |
| |
| gsutil does not preserve metadata when copying objects between providers. |
| |
| |
| <B>RESUMABLE TRANSFERS</B> |
| gsutil automatically uses the Google Cloud Storage resumable upload |
| feature whenever you use the cp command to upload an object that is larger |
| than 2 MB. You do not need to specify any special command line options |
| to make this happen. If your upload is interrupted you can restart the |
| upload by running the same cp command that you ran to start the upload. |
| |
| Similarly, gsutil automatically performs resumable downloads (using HTTP |
| standard Range GET operations) whenever you use the cp command to download an |
| object larger than 2 MB. |
| |
| Resumable uploads and downloads store some state information in a file |
| in ~/.gsutil named by the destination object or file. If you attempt to |
| resume a transfer from a machine with a different directory, the transfer |
| will start over from scratch. |
| |
| See also "gsutil help prod" for details on using resumable transfers |
| in production. |
| |
| |
| <B>STREAMING TRANSFERS</B> |
| Use '-' in place of src_uri or dst_uri to perform a streaming |
| transfer. For example: |
| long_running_computation | gsutil cp - gs://my_bucket/obj |
| |
| Streaming transfers do not support resumable uploads/downloads. |
| (The Google resumable transfer protocol has a way to support streaming |
| transers, but gsutil doesn't currently implement support for this.) |
| |
| |
| <B>CHANGING TEMP DIRECTORIES</B> |
| gsutil writes data to a temporary directory in several cases: |
| - when compressing data to be uploaded (see the -z option) |
| - when decompressing data being downloaded (when the data has |
| Content-Encoding:gzip, e.g., as happens when uploaded using gsutil cp -z) |
| - when running integration tests (using the gsutil test command) |
| |
| In these cases it's possible the temp file location on your system that |
| gsutil selects by default may not have enough space. If you find that |
| gsutil runs out of space during one of these operations (e.g., raising |
| "CommandException: Inadequate temp space available to compress <your file>" |
| during a gsutil cp -z operation), you can change where it writes these |
| temp files by setting the TMPDIR environment variable. On Linux and MacOS |
| you can do this either by running gsutil this way: |
| |
| TMPDIR=/some/directory gsutil cp ... |
| |
| or by adding this line to your ~/.bashrc file and then restarting the shell |
| before running gsutil: |
| |
| export TMPDIR=/some/directory |
| |
| On Windows 7 you can change the TMPDIR environment variable from Start -> |
| Computer -> System -> Advanced System Settings -> Environment Variables. |
| You need to reboot after making this change for it to take effect. (Rebooting |
| is not necessary after running the export command on Linux and MacOS.) |
| |
| |
| <B>OPTIONS</B> |
| -a canned_acl Sets named canned_acl when uploaded objects created. See |
| 'gsutil help acls' for further details. |
| |
| -c If an error occurrs, continue to attempt to copy the remaining |
| files. |
| |
| -D Copy in "daisy chain" mode, i.e., copying between two buckets by |
| hooking a download to an upload, via the machine where gsutil is |
| run. By default, data are copied between two buckets "in the |
| cloud", i.e., without needing to copy via the machine where |
| gsutil runs. However, copy-in-the-cloud is not supported when |
| copying between different locations (like US and EU) or between |
| different storage classes (like STANDARD and |
| DURABLE_REDUCED_AVAILABILITY). For these cases, you can use the |
| -D option to copy data between buckets. |
| Note: Daisy chain mode is automatically used when copying |
| between providers (e.g., to copy data from Google Cloud Storage |
| to another provider). |
| |
| -e Exclude symlinks. When specified, symbolic links will not be |
| copied. |
| |
| -n No-clobber. When specified, existing files or objects at the |
| destination will not be overwritten. Any items that are skipped |
| by this option will be reported as being skipped. This option |
| will perform an additional HEAD request to check if an item |
| exists before attempting to upload the data. This will save |
| retransmitting data, but the additional HTTP requests may make |
| small object transfers slower and more expensive. |
| |
| This option can be combined with the -c option to build a script |
| that copies a large number of objects, allowing retries when |
| some failures occur from which gsutil doesn't automatically |
| recover, using a bash script like the following: |
| |
| status=1 |
| while [ $status -ne 0 ] ; do |
| gsutil cp -c -n -R ./dir gs://bucket |
| status=$? |
| done |
| |
| The -c option will cause copying to continue after failures |
| occur, and the -n option will cause objects already copied to be |
| skipped on subsequent iterations. The loop will continue running |
| as long as gsutil exits with a non-zero status (such a status |
| indicates there was at least one failure during the gsutil run). |
| |
| -p Causes ACLs to be preserved when copying in the cloud. Note that |
| this option has performance and cost implications, because it |
| is essentially performing three requests (getacl, cp, setacl). |
| (The performance issue can be mitigated to some degree by |
| using gsutil -m cp to cause parallel copying.) |
| |
| You can avoid the additional performance and cost of using cp -p |
| if you want all objects in the destination bucket to end up with |
| the same ACL by setting a default ACL on that bucket instead of |
| using cp -p. See "help gsutil setdefacl". |
| |
| Note that it's not valid to specify both the -a and -p options |
| together. |
| |
| -q Causes copies to be performed quietly, i.e., without reporting |
| progress indicators of files being copied. Errors are still |
| reported. This option can be useful for running gsutil from a |
| cron job that logs its output to a file, for which the only |
| information desired in the log is failures. |
| |
| -R, -r Causes directories, buckets, and bucket subdirectories to be |
| copied recursively. If you neglect to use this option for |
| an upload, gsutil will copy any files it finds and skip any |
| directories. Similarly, neglecting to specify -R for a download |
| will cause gsutil to copy any objects at the current bucket |
| directory level, and skip any subdirectories. |
| |
| -v Requests that the version-specific URI for each uploaded object |
| be printed. Given this URI you can make future upload requests |
| that are safe in the face of concurrent updates, because Google |
| Cloud Storage will refuse to perform the update if the current |
| object version doesn't match the version-specific URI. See |
| 'gsutil help versioning' for more details. Note: at present this |
| option does not work correctly for objects copied "in the cloud" |
| (e.g., gsutil cp gs://bucket/obj1 gs://bucket/obj2). |
| |
| -z ext1,... Compresses file uploads with the given extensions. If you are |
| uploading a large file with compressible content, such as |
| a .js, .css, or .html file, you can gzip-compress the file |
| during the upload process by specifying the -z <extensions> |
| option. Compressing data before upload saves on usage charges |
| because you are uploading a smaller amount of data. |
| |
| When you specify the -z option, the data from your files is |
| compressed before it is uploaded, but your actual files are left |
| uncompressed on the local disk. The uploaded objects retain the |
| original content type and name as the original files but are |
| given a Content-Encoding header with the value "gzip" to |
| indicate that the object data stored are compressed on the |
| Google Cloud Storage servers. |
| |
| For example, the following command: |
| |
| gsutil cp -z html -a public-read cattypes.html gs://mycats |
| |
| will do all of the following: |
| - Upload as the object gs://mycats/cattypes.html (cp command) |
| - Set the Content-Type to text/html (based on file extension) |
| - Compress the data in the file cattypes.html (-z option) |
| - Set the Content-Encoding to gzip (-z option) |
| - Set the ACL to public-read (-a option) |
| - If a user tries to view cattypes.html in a browser, the |
| browser will know to uncompress the data based on the |
| Content-Encoding header, and to render it as HTML based on |
| the Content-Type header. |
| """) |
| |
| class CpCommand(Command): |
| """ |
| Implementation of gsutil cp command. |
| |
| Note that CpCommand is run for both gsutil cp and gsutil mv. The latter |
| happens by MvCommand calling CpCommand and passing the hidden (undocumented) |
| -M option. This allows the copy and remove needed for each mv to run |
| together (rather than first running all the cp's and then all the rm's, as |
| we originally had implemented), which in turn avoids the following problem |
| with removing the wrong objects: starting with a bucket containing only |
| the object gs://bucket/obj, say the user does: |
| gsutil mv gs://bucket/* gs://bucket/d.txt |
| If we ran all the cp's and then all the rm's and we didn't expand the wildcard |
| first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt, |
| and the rm command would then remove that object. In the implementation |
| prior to gsutil release 3.12 we avoided this by building a list of objects |
| to process and then running the copies and then the removes; but building |
| the list up front limits scalability (compared with the current approach |
| of processing the bucket listing iterator on the fly). |
| """ |
| |
| # Set default Content-Type type. |
| DEFAULT_CONTENT_TYPE = 'application/octet-stream' |
| USE_MAGICFILE = boto.config.getbool('GSUtil', 'use_magicfile', False) |
| |
| # Command specification (processed by parent class). |
| command_spec = { |
| # Name of command. |
| COMMAND_NAME : 'cp', |
| # List of command name aliases. |
| COMMAND_NAME_ALIASES : ['copy'], |
| # Min number of args required by this command. |
| MIN_ARGS : 1, |
| # Max number of args required by this command, or NO_MAX. |
| MAX_ARGS : NO_MAX, |
| # Getopt-style string specifying acceptable sub args. |
| # -t is deprecated but leave intact for now to avoid breakage. |
| SUPPORTED_SUB_ARGS : 'a:cDeIMNnpqrRtvz:', |
| # True if file URIs acceptable for this command. |
| FILE_URIS_OK : True, |
| # True if provider-only URIs acceptable for this command. |
| PROVIDER_URIS_OK : False, |
| # Index in args of first URI arg. |
| URIS_START_ARG : 0, |
| # True if must configure gsutil before running command. |
| CONFIG_REQUIRED : True, |
| } |
| help_spec = { |
| # Name of command or auxiliary help info for which this help applies. |
| HELP_NAME : 'cp', |
| # List of help name aliases. |
| HELP_NAME_ALIASES : ['copy'], |
| # Type of help: |
| HELP_TYPE : HelpType.COMMAND_HELP, |
| # One line summary of this help. |
| HELP_ONE_LINE_SUMMARY : 'Copy files and objects', |
| # The full help text. |
| HELP_TEXT : _detailed_help_text, |
| } |
| |
| def _CheckFinalMd5(self, key, file_name): |
| """ |
| Checks that etag from server agrees with md5 computed after the |
| download completes. |
| """ |
| obj_md5 = key.etag.strip('"\'') |
| file_md5 = None |
| |
| if hasattr(key, 'md5') and key.md5: |
| file_md5 = key.md5 |
| else: |
| print 'Computing MD5 from scratch for resumed download' |
| |
| # Open file in binary mode to avoid surprises in Windows. |
| fp = open(file_name, 'rb') |
| try: |
| file_md5 = key.compute_md5(fp)[0] |
| finally: |
| fp.close() |
| |
| if self.debug: |
| print 'Checking file md5 against etag. (%s/%s)' % (file_md5, obj_md5) |
| if file_md5 != obj_md5: |
| # Checksums don't match - remove file and raise exception. |
| os.unlink(file_name) |
| raise CommandException( |
| 'File changed during download: md5 signature doesn\'t match ' |
| 'etag (incorrect downloaded file deleted)') |
| |
| def _CheckForDirFileConflict(self, exp_src_uri, dst_uri): |
| """Checks whether copying exp_src_uri into dst_uri is not possible. |
| |
| This happens if a directory exists in local file system where a file |
| needs to go or vice versa. In that case we print an error message and |
| exits. Example: if the file "./x" exists and you try to do: |
| gsutil cp gs://mybucket/x/y . |
| the request can't succeed because it requires a directory where |
| the file x exists. |
| |
| Note that we don't enforce any corresponding restrictions for buckets, |
| because the flat namespace semantics for buckets doesn't prohibit such |
| cases the way hierarchical file systems do. For example, if a bucket |
| contains an object called gs://bucket/dir and then you run the command: |
| gsutil cp file1 file2 gs://bucket/dir |
| you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and |
| gs://bucket/dir/file2. |
| |
| Args: |
| exp_src_uri: Expanded source StorageUri of copy. |
| dst_uri: Destination URI. |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| if dst_uri.is_cloud_uri(): |
| # The problem can only happen for file destination URIs. |
| return |
| dst_path = dst_uri.object_name |
| final_dir = os.path.dirname(dst_path) |
| if os.path.isfile(final_dir): |
| raise CommandException('Cannot retrieve %s because a file exists ' |
| 'where a directory needs to be created (%s).' % |
| (exp_src_uri, final_dir)) |
| if os.path.isdir(dst_path): |
| raise CommandException('Cannot retrieve %s because a directory exists ' |
| '(%s) where the file needs to be created.' % |
| (exp_src_uri, dst_path)) |
| |
| def _InsistDstUriNamesContainer(self, exp_dst_uri, |
| have_existing_dst_container, command_name): |
| """ |
| Raises an exception if URI doesn't name a directory, bucket, or bucket |
| subdir, with special exception for cp -R (see comments below). |
| |
| Args: |
| exp_dst_uri: Wildcard-expanding dst_uri. |
| have_existing_dst_container: bool indicator of whether exp_dst_uri |
| names a container (directory, bucket, or existing bucket subdir). |
| command_name: Name of command making call. May not be the same as |
| self.command_name in the case of commands implemented atop other |
| commands (like mv command). |
| |
| Raises: |
| CommandException: if the URI being checked does not name a container. |
| """ |
| if exp_dst_uri.is_file_uri(): |
| ok = exp_dst_uri.names_directory() |
| else: |
| if have_existing_dst_container: |
| ok = True |
| else: |
| # It's ok to specify a non-existing bucket subdir, for example: |
| # gsutil cp -R dir gs://bucket/abc |
| # where gs://bucket/abc isn't an existing subdir. |
| ok = exp_dst_uri.names_object() |
| if not ok: |
| raise CommandException('Destination URI must name a directory, bucket, ' |
| 'or bucket\nsubdirectory for the multiple ' |
| 'source form of the %s command.' % command_name) |
| |
| class _FileCopyCallbackHandler(object): |
| """Outputs progress info for large copy requests.""" |
| |
| def __init__(self, upload): |
| if upload: |
| self.announce_text = 'Uploading' |
| else: |
| self.announce_text = 'Downloading' |
| |
| def call(self, total_bytes_transferred, total_size): |
| sys.stderr.write('%s: %s/%s \r' % ( |
| self.announce_text, |
| MakeHumanReadable(total_bytes_transferred), |
| MakeHumanReadable(total_size))) |
| if total_bytes_transferred == total_size: |
| sys.stderr.write('\n') |
| |
| class _StreamCopyCallbackHandler(object): |
| """Outputs progress info for Stream copy to cloud. |
| Total Size of the stream is not known, so we output |
| only the bytes transferred. |
| """ |
| |
| def call(self, total_bytes_transferred, total_size): |
| sys.stderr.write('Uploading: %s \r' % ( |
| MakeHumanReadable(total_bytes_transferred))) |
| if total_size and total_bytes_transferred == total_size: |
| sys.stderr.write('\n') |
| |
| def _GetTransferHandlers(self, dst_uri, size, upload): |
| """ |
| Selects upload/download and callback handlers. |
| |
| We use a callback handler that shows a simple textual progress indicator |
| if size is above the configurable threshold. |
| |
| We use a resumable transfer handler if size is >= the configurable |
| threshold and resumable transfers are supported by the given provider. |
| boto supports resumable downloads for all providers, but resumable |
| uploads are currently only supported by GS. |
| |
| Args: |
| dst_uri: the destination URI. |
| size: size of file (object) being uploaded (downloaded). |
| upload: bool indication of whether transfer is an upload. |
| """ |
| config = boto.config |
| resumable_threshold = config.getint('GSUtil', 'resumable_threshold', TWO_MB) |
| transfer_handler = None |
| cb = None |
| num_cb = None |
| |
| # Checks whether the destination file is a "special" file, like /dev/null on |
| # Linux platforms or null on Windows platforms, so we can disable resumable |
| # download support since the file size of the destination won't ever be |
| # correct. |
| dst_is_special = False |
| if dst_uri.is_file_uri(): |
| # Check explicitly first because os.stat doesn't work on 'nul' in Windows. |
| if dst_uri.object_name == os.devnull: |
| dst_is_special = True |
| try: |
| mode = os.stat(dst_uri.object_name).st_mode |
| if stat.S_ISCHR(mode): |
| dst_is_special = True |
| except OSError: |
| pass |
| |
| if size >= resumable_threshold and not dst_is_special: |
| if not self.quiet: |
| cb = self._FileCopyCallbackHandler(upload).call |
| num_cb = int(size / TWO_MB) |
| |
| resumable_tracker_dir = config.get( |
| 'GSUtil', 'resumable_tracker_dir', |
| os.path.expanduser('~' + os.sep + '.gsutil')) |
| if not os.path.exists(resumable_tracker_dir): |
| os.makedirs(resumable_tracker_dir) |
| |
| if upload: |
| # Encode the dest bucket and object name into the tracker file name. |
| res_tracker_file_name = ( |
| re.sub('[/\\\\]', '_', 'resumable_upload__%s__%s.url' % |
| (dst_uri.bucket_name, dst_uri.object_name))) |
| else: |
| # Encode the fully-qualified dest file name into the tracker file name. |
| res_tracker_file_name = ( |
| re.sub('[/\\\\]', '_', 'resumable_download__%s.etag' % |
| (os.path.realpath(dst_uri.object_name)))) |
| |
| res_tracker_file_name = _hash_filename(res_tracker_file_name) |
| tracker_file = '%s%s%s' % (resumable_tracker_dir, os.sep, |
| res_tracker_file_name) |
| if upload: |
| if dst_uri.scheme == 'gs': |
| transfer_handler = ResumableUploadHandler(tracker_file) |
| else: |
| transfer_handler = ResumableDownloadHandler(tracker_file) |
| |
| return (cb, num_cb, transfer_handler) |
| |
| def _LogCopyOperation(self, src_uri, dst_uri, headers): |
| """ |
| Logs copy operation being performed, including Content-Type if appropriate. |
| """ |
| if self.quiet: |
| return |
| if 'Content-Type' in headers and dst_uri.is_cloud_uri(): |
| content_type_msg = ' [Content-Type=%s]' % headers['Content-Type'] |
| else: |
| content_type_msg = '' |
| if src_uri.is_stream(): |
| self.THREADED_LOGGER.info('Copying from <STDIN>%s...', content_type_msg) |
| else: |
| self.THREADED_LOGGER.info('Copying %s%s...', src_uri, content_type_msg) |
| |
| # We pass the headers explicitly to this call instead of using self.headers |
| # so we can set different metadata (like Content-Type type) for each object. |
| def _CopyObjToObjInTheCloud(self, src_key, src_uri, dst_uri, headers): |
| """Performs copy-in-the cloud from specified src to dest object. |
| |
| Args: |
| src_key: Source Key. |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| headers: A copy of the headers dictionary. |
| |
| Returns: |
| (elapsed_time, bytes_transferred, dst_uri) excluding overhead like initial |
| HEAD. Note: At present copy-in-the-cloud doesn't return the generation of |
| the created object, so the returned URI is actually not version-specific |
| (unlike other cp cases). |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| self._SetContentTypeHeader(src_uri, headers) |
| self._LogCopyOperation(src_uri, dst_uri, headers) |
| # Do Object -> object copy within same provider (uses |
| # x-<provider>-copy-source metadata HTTP header to request copying at the |
| # server). |
| src_bucket = src_uri.get_bucket(False, headers) |
| preserve_acl = False |
| canned_acl = None |
| if self.sub_opts: |
| for o, a in self.sub_opts: |
| if o == '-a': |
| canned_acls = dst_uri.canned_acls() |
| if a not in canned_acls: |
| raise CommandException('Invalid canned ACL "%s".' % a) |
| canned_acl = a |
| headers[dst_uri.get_provider().acl_header] = canned_acl |
| if o == '-p': |
| preserve_acl = True |
| if preserve_acl and canned_acl: |
| raise CommandException( |
| 'Specifying both the -p and -a options together is invalid.') |
| start_time = time.time() |
| # Pass headers in headers param not metadata param, so boto will copy |
| # existing key's metadata and just set the additional headers specified |
| # in the headers param (rather than using the headers to override existing |
| # metadata). In particular this allows us to copy the existing key's |
| # Content-Type and other metadata users need while still being able to |
| # set headers the API needs (like x-goog-project-id). Note that this means |
| # you can't do something like: |
| # gsutil cp -t Content-Type text/html gs://bucket/* gs://bucket2 |
| # to change the Content-Type while copying. |
| |
| try: |
| dst_key = dst_uri.copy_key( |
| src_bucket.name, src_uri.object_name, preserve_acl=False, |
| headers=headers, src_version_id=src_uri.version_id, |
| src_generation=src_uri.generation) |
| except GSResponseError as e: |
| exc_name, error_detail = ExtractErrorDetail(e) |
| if (exc_name == 'GSResponseError' |
| and ('Copy-in-the-cloud disallowed' in error_detail)): |
| raise CommandException('%s.\nNote: you can copy between locations ' |
| 'and between storage classes by using the ' |
| 'gsutil cp -D option.' % error_detail) |
| else: |
| raise |
| end_time = time.time() |
| return (end_time - start_time, src_key.size, |
| dst_uri.clone_replace_key(dst_key)) |
| |
| def _CheckFreeSpace(self, path): |
| """Return path/drive free space (in bytes).""" |
| if platform.system() == 'Windows': |
| from ctypes import c_int, c_uint64, c_wchar_p, windll, POINTER, WINFUNCTYPE, WinError |
| try: |
| GetDiskFreeSpaceEx = WINFUNCTYPE(c_int, c_wchar_p, POINTER(c_uint64), |
| POINTER(c_uint64), POINTER(c_uint64)) |
| GetDiskFreeSpaceEx = GetDiskFreeSpaceEx( |
| ('GetDiskFreeSpaceExW', windll.kernel32), ( |
| (1, 'lpszPathName'), |
| (2, 'lpFreeUserSpace'), |
| (2, 'lpTotalSpace'), |
| (2, 'lpFreeSpace'),)) |
| except AttributeError: |
| GetDiskFreeSpaceEx = WINFUNCTYPE(c_int, c_char_p, POINTER(c_uint64), |
| POINTER(c_uint64), POINTER(c_uint64)) |
| GetDiskFreeSpaceEx = GetDiskFreeSpaceEx( |
| ('GetDiskFreeSpaceExA', windll.kernel32), ( |
| (1, 'lpszPathName'), |
| (2, 'lpFreeUserSpace'), |
| (2, 'lpTotalSpace'), |
| (2, 'lpFreeSpace'),)) |
| |
| def GetDiskFreeSpaceEx_errcheck(result, func, args): |
| if not result: |
| raise WinError() |
| return args[1].value |
| GetDiskFreeSpaceEx.errcheck = GetDiskFreeSpaceEx_errcheck |
| |
| return GetDiskFreeSpaceEx(os.getenv('SystemDrive')) |
| else: |
| (_, f_frsize, _, _, f_bavail, _, _, _, _, _) = os.statvfs(path) |
| return f_frsize * f_bavail |
| |
| def _PerformResumableUploadIfApplies(self, fp, dst_uri, canned_acl, headers): |
| """ |
| Performs resumable upload if supported by provider and file is above |
| threshold, else performs non-resumable upload. |
| |
| Returns (elapsed_time, bytes_transferred, version-specific dst_uri). |
| """ |
| start_time = time.time() |
| # Determine file size different ways for case where fp is actually a wrapper |
| # around a Key vs an actual file. |
| if isinstance(fp, KeyFile): |
| file_size = fp.getkey().size |
| else: |
| file_size = os.path.getsize(fp.name) |
| (cb, num_cb, res_upload_handler) = self._GetTransferHandlers( |
| dst_uri, file_size, True) |
| if dst_uri.scheme == 'gs': |
| # Resumable upload protocol is Google Cloud Storage-specific. |
| dst_uri.set_contents_from_file(fp, headers, policy=canned_acl, |
| cb=cb, num_cb=num_cb, |
| res_upload_handler=res_upload_handler) |
| else: |
| dst_uri.set_contents_from_file(fp, headers, policy=canned_acl, |
| cb=cb, num_cb=num_cb) |
| if res_upload_handler: |
| # ResumableUploadHandler does not update upload_start_point from its |
| # initial value of -1 if transferring the whole file, so clamp at 0 |
| bytes_transferred = file_size - max( |
| res_upload_handler.upload_start_point, 0) |
| else: |
| bytes_transferred = file_size |
| end_time = time.time() |
| return (end_time - start_time, bytes_transferred, dst_uri) |
| |
| def _PerformStreamingUpload(self, fp, dst_uri, headers, canned_acl=None): |
| """ |
| Performs a streaming upload to the cloud. |
| |
| Args: |
| fp: The file whose contents to upload. |
| dst_uri: Destination StorageUri. |
| headers: A copy of the headers dictionary. |
| canned_acl: Optional canned ACL to set on the object. |
| |
| Returns (elapsed_time, bytes_transferred, version-specific dst_uri). |
| """ |
| start_time = time.time() |
| |
| if self.quiet: |
| cb = None |
| else: |
| cb = self._StreamCopyCallbackHandler().call |
| dst_uri.set_contents_from_stream( |
| fp, headers, policy=canned_acl, cb=cb) |
| try: |
| bytes_transferred = fp.tell() |
| except: |
| bytes_transferred = 0 |
| |
| end_time = time.time() |
| return (end_time - start_time, bytes_transferred, dst_uri) |
| |
| def _SetContentTypeHeader(self, src_uri, headers): |
| """ |
| Sets content type header to value specified in '-h Content-Type' option (if |
| specified); else sets using Content-Type detection. |
| """ |
| if 'Content-Type' in headers: |
| # If empty string specified (i.e., -h "Content-Type:") set header to None, |
| # which will inhibit boto from sending the CT header. Otherwise, boto will |
| # pass through the user specified CT header. |
| if not headers['Content-Type']: |
| headers['Content-Type'] = None |
| # else we'll keep the value passed in via -h option (not performing |
| # content type detection). |
| else: |
| # Only do content type recognition is src_uri is a file. Object-to-object |
| # copies with no -h Content-Type specified re-use the content type of the |
| # source object. |
| if src_uri.is_file_uri(): |
| object_name = src_uri.object_name |
| content_type = None |
| # Streams (denoted by '-') are expected to be 'application/octet-stream' |
| # and 'file' would partially consume them. |
| if object_name != '-': |
| if self.USE_MAGICFILE: |
| p = subprocess.Popen(['file', '--mime-type', object_name], |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| output, error = p.communicate() |
| if p.returncode != 0 or error: |
| raise CommandException( |
| 'Encountered error running "file --mime-type %s" ' |
| '(returncode=%d).\n%s' % (object_name, p.returncode, error)) |
| # Parse output by removing line delimiter and splitting on last ": |
| content_type = output.rstrip().rpartition(': ')[2] |
| else: |
| content_type = mimetypes.guess_type(object_name)[0] |
| if not content_type: |
| content_type = self.DEFAULT_CONTENT_TYPE |
| headers['Content-Type'] = content_type |
| |
| def _UploadFileToObject(self, src_key, src_uri, dst_uri, headers, |
| should_log=True): |
| """Uploads a local file to an object. |
| |
| Args: |
| src_key: Source StorageUri. Must be a file URI. |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| headers: The headers dictionary. |
| should_log: bool indicator whether we should log this operation. |
| Returns: |
| (elapsed_time, bytes_transferred, version-specific dst_uri), excluding |
| overhead like initial HEAD. |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| gzip_exts = [] |
| canned_acl = None |
| if self.sub_opts: |
| for o, a in self.sub_opts: |
| if o == '-a': |
| canned_acls = dst_uri.canned_acls() |
| if a not in canned_acls: |
| raise CommandException('Invalid canned ACL "%s".' % a) |
| canned_acl = a |
| elif o == '-t': |
| print('Warning: -t is deprecated, and will be removed in the future. ' |
| 'Content type\ndetection is ' |
| 'now performed by default, unless inhibited by specifying ' |
| 'a\nContent-Type header via the -h option.') |
| elif o == '-z': |
| gzip_exts = a.split(',') |
| |
| self._SetContentTypeHeader(src_uri, headers) |
| if should_log: |
| self._LogCopyOperation(src_uri, dst_uri, headers) |
| |
| if 'Content-Language' not in headers: |
| content_language = config.get_value('GSUtil', 'content_language') |
| if content_language: |
| headers['Content-Language'] = content_language |
| |
| fname_parts = src_uri.object_name.split('.') |
| if len(fname_parts) > 1 and fname_parts[-1] in gzip_exts: |
| if self.debug: |
| print 'Compressing %s (to tmp)...' % src_key |
| (gzip_fh, gzip_path) = tempfile.mkstemp() |
| gzip_fp = None |
| try: |
| # Check for temp space. Assume the compressed object is at most 2x |
| # the size of the object (normally should compress to smaller than |
| # the object) |
| if (self._CheckFreeSpace(gzip_path) |
| < 2*int(os.path.getsize(src_key.name))): |
| raise CommandException('Inadequate temp space available to compress ' |
| '%s' % src_key.name) |
| gzip_fp = gzip.open(gzip_path, 'wb') |
| gzip_fp.writelines(src_key.fp) |
| finally: |
| if gzip_fp: |
| gzip_fp.close() |
| os.close(gzip_fh) |
| headers['Content-Encoding'] = 'gzip' |
| gzip_fp = open(gzip_path, 'rb') |
| try: |
| (elapsed_time, bytes_transferred, result_uri) = ( |
| self._PerformResumableUploadIfApplies(gzip_fp, dst_uri, |
| canned_acl, headers)) |
| finally: |
| gzip_fp.close() |
| try: |
| os.unlink(gzip_path) |
| # Windows sometimes complains the temp file is locked when you try to |
| # delete it. |
| except Exception, e: |
| pass |
| elif (src_key.is_stream() |
| and dst_uri.get_provider().supports_chunked_transfer()): |
| (elapsed_time, bytes_transferred, result_uri) = ( |
| self._PerformStreamingUpload(src_key.fp, dst_uri, headers, |
| canned_acl)) |
| else: |
| if src_key.is_stream(): |
| # For Providers that doesn't support chunked Transfers |
| tmp = tempfile.NamedTemporaryFile() |
| file_uri = self.suri_builder.StorageUri('file://%s' % tmp.name) |
| try: |
| file_uri.new_key(False, headers).set_contents_from_file( |
| src_key.fp, headers) |
| src_key = file_uri.get_key() |
| finally: |
| file_uri.close() |
| try: |
| (elapsed_time, bytes_transferred, result_uri) = ( |
| self._PerformResumableUploadIfApplies(src_key.fp, dst_uri, |
| canned_acl, headers)) |
| finally: |
| if src_key.is_stream(): |
| tmp.close() |
| else: |
| src_key.close() |
| |
| return (elapsed_time, bytes_transferred, result_uri) |
| |
| def _DownloadObjectToFile(self, src_key, src_uri, dst_uri, headers, |
| should_log=True): |
| """Downloads an object to a local file. |
| |
| Args: |
| src_key: Source StorageUri. Must be a file URI. |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| headers: The headers dictionary. |
| should_log: bool indicator whether we should log this operation. |
| Returns: |
| (elapsed_time, bytes_transferred, dst_uri), excluding overhead like |
| initial HEAD. |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| if should_log: |
| self._LogCopyOperation(src_uri, dst_uri, headers) |
| (cb, num_cb, res_download_handler) = self._GetTransferHandlers( |
| dst_uri, src_key.size, False) |
| file_name = dst_uri.object_name |
| dir_name = os.path.dirname(file_name) |
| if dir_name and not os.path.exists(dir_name): |
| # Do dir creation in try block so can ignore case where dir already |
| # exists. This is needed to avoid a race condition when running gsutil |
| # -m cp. |
| try: |
| os.makedirs(dir_name) |
| except OSError, e: |
| if e.errno != errno.EEXIST: |
| raise |
| # For gzipped objects not named *.gz download to a temp file and unzip. |
| if (hasattr(src_key, 'content_encoding') |
| and src_key.content_encoding == 'gzip' |
| and not file_name.endswith('.gz')): |
| # We can't use tempfile.mkstemp() here because we need a predictable |
| # filename for resumable downloads. |
| download_file_name = '%s_.gztmp' % file_name |
| need_to_unzip = True |
| else: |
| download_file_name = file_name |
| need_to_unzip = False |
| fp = None |
| try: |
| if res_download_handler: |
| fp = open(download_file_name, 'ab') |
| else: |
| fp = open(download_file_name, 'wb') |
| start_time = time.time() |
| src_key.get_contents_to_file(fp, headers, cb=cb, num_cb=num_cb, |
| res_download_handler=res_download_handler) |
| # If a custom test method is defined, call it here. For the copy command, |
| # test methods are expected to take one argument: an open file pointer, |
| # and are used to perturb the open file during download to exercise |
| # download error detection. |
| if self.test_method: |
| self.test_method(fp) |
| end_time = time.time() |
| finally: |
| if fp: |
| fp.close() |
| |
| # Discard the md5 if we are resuming a partial download. |
| if res_download_handler and res_download_handler.download_start_point: |
| src_key.md5 = None |
| |
| # Verify downloaded file checksum matched source object's checksum. |
| self._CheckFinalMd5(src_key, download_file_name) |
| |
| if res_download_handler: |
| bytes_transferred = ( |
| src_key.size - res_download_handler.download_start_point) |
| else: |
| bytes_transferred = src_key.size |
| if need_to_unzip: |
| # Log that we're uncompressing if the file is big enough that |
| # decompressing would make it look like the transfer "stalled" at the end. |
| if not self.quiet and bytes_transferred > 10 * 1024 * 1024: |
| self.THREADED_LOGGER.info('Uncompressing downloaded tmp file to %s...', |
| file_name) |
| # Downloaded gzipped file to a filename w/o .gz extension, so unzip. |
| f_in = gzip.open(download_file_name, 'rb') |
| f_out = open(file_name, 'wb') |
| try: |
| while True: |
| data = f_in.read(8192) |
| if not data: |
| break |
| f_out.write(data) |
| finally: |
| f_out.close() |
| f_in.close() |
| os.unlink(download_file_name) |
| return (end_time - start_time, bytes_transferred, dst_uri) |
| |
| def _PerformDownloadToStream(self, src_key, src_uri, str_fp, headers): |
| (cb, num_cb, res_download_handler) = self._GetTransferHandlers( |
| src_uri, src_key.size, False) |
| start_time = time.time() |
| src_key.get_contents_to_file(str_fp, headers, cb=cb, num_cb=num_cb) |
| end_time = time.time() |
| bytes_transferred = src_key.size |
| end_time = time.time() |
| return (end_time - start_time, bytes_transferred) |
| |
| def _CopyFileToFile(self, src_key, src_uri, dst_uri, headers): |
| """Copies a local file to a local file. |
| |
| Args: |
| src_key: Source StorageUri. Must be a file URI. |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| headers: The headers dictionary. |
| Returns: |
| (elapsed_time, bytes_transferred, dst_uri), excluding |
| overhead like initial HEAD. |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| self._LogCopyOperation(src_uri, dst_uri, headers) |
| dst_key = dst_uri.new_key(False, headers) |
| start_time = time.time() |
| dst_key.set_contents_from_file(src_key.fp, headers) |
| end_time = time.time() |
| return (end_time - start_time, os.path.getsize(src_key.fp.name), dst_uri) |
| |
| def _CopyObjToObjDaisyChainMode(self, src_key, src_uri, dst_uri, headers): |
| """Copies from src_uri to dst_uri in "daisy chain" mode. |
| See -D OPTION documentation about what daisy chain mode is. |
| |
| Args: |
| src_key: Source Key. |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| headers: A copy of the headers dictionary. |
| |
| Returns: |
| (elapsed_time, bytes_transferred, version-specific dst_uri) excluding |
| overhead like initial HEAD. |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| self._SetContentTypeHeader(src_uri, headers) |
| self._LogCopyOperation(src_uri, dst_uri, headers) |
| canned_acl = None |
| if self.sub_opts: |
| for o, a in self.sub_opts: |
| if o == '-a': |
| canned_acls = dst_uri.canned_acls() |
| if a not in canned_acls: |
| raise CommandException('Invalid canned ACL "%s".' % a) |
| canned_acl = a |
| elif o == '-p': |
| # We don't attempt to preserve ACLs across providers because |
| # GCS and S3 support different ACLs and disjoint principals. |
| raise NotImplementedError('Cross-provider cp -p not supported') |
| return self._PerformResumableUploadIfApplies(KeyFile(src_key), dst_uri, |
| canned_acl, headers) |
| |
| def _PerformCopy(self, src_uri, dst_uri): |
| """Performs copy from src_uri to dst_uri, handling various special cases. |
| |
| Args: |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| |
| Returns: |
| (elapsed_time, bytes_transferred, version-specific dst_uri) excluding |
| overhead like initial HEAD. |
| |
| Raises: |
| CommandException: if errors encountered. |
| """ |
| # Make a copy of the input headers each time so we can set a different |
| # content type for each object. |
| if self.headers: |
| headers = self.headers.copy() |
| else: |
| headers = {} |
| |
| src_key = src_uri.get_key(False, headers) |
| if not src_key: |
| raise CommandException('"%s" does not exist.' % src_uri) |
| |
| # On Windows, stdin is opened as text mode instead of binary which causes |
| # problems when piping a binary file, so this switches it to binary mode. |
| if IS_WINDOWS and src_uri.is_file_uri() and src_key.is_stream(): |
| import msvcrt |
| msvcrt.setmode(src_key.fp.fileno(), os.O_BINARY) |
| |
| if self.no_clobber: |
| # There are two checks to prevent clobbering: |
| # 1) The first check is to see if the item |
| # already exists at the destination and prevent the upload/download |
| # from happening. This is done by the exists() call. |
| # 2) The second check is only relevant if we are writing to gs. We can |
| # enforce that the server only writes the object if it doesn't exist |
| # by specifying the header below. This check only happens at the |
| # server after the complete file has been uploaded. We specify this |
| # header to prevent a race condition where a destination file may |
| # be created after the first check and before the file is fully |
| # uploaded. |
| # In order to save on unnecessary uploads/downloads we perform both |
| # checks. However, this may come at the cost of additional HTTP calls. |
| if dst_uri.exists(headers): |
| if not self.quiet: |
| self.THREADED_LOGGER.info('Skipping existing item: %s' % |
| dst_uri.uri) |
| return (0, 0, None) |
| if dst_uri.is_cloud_uri() and dst_uri.scheme == 'gs': |
| headers['x-goog-if-generation-match'] = '0' |
| |
| if src_uri.is_cloud_uri() and dst_uri.is_cloud_uri(): |
| if src_uri.scheme == dst_uri.scheme and not self.daisy_chain: |
| return self._CopyObjToObjInTheCloud(src_key, src_uri, dst_uri, headers) |
| else: |
| return self._CopyObjToObjDaisyChainMode(src_key, src_uri, dst_uri, |
| headers) |
| elif src_uri.is_file_uri() and dst_uri.is_cloud_uri(): |
| return self._UploadFileToObject(src_key, src_uri, dst_uri, headers) |
| elif src_uri.is_cloud_uri() and dst_uri.is_file_uri(): |
| return self._DownloadObjectToFile(src_key, src_uri, dst_uri, headers) |
| elif src_uri.is_file_uri() and dst_uri.is_file_uri(): |
| return self._CopyFileToFile(src_key, src_uri, dst_uri, headers) |
| else: |
| raise CommandException('Unexpected src/dest case') |
| |
| def _ExpandDstUri(self, dst_uri_str): |
| """ |
| Expands wildcard if present in dst_uri_str. |
| |
| Args: |
| dst_uri_str: String representation of requested dst_uri. |
| |
| Returns: |
| (exp_dst_uri, have_existing_dst_container) |
| where have_existing_dst_container is a bool indicating whether |
| exp_dst_uri names an existing directory, bucket, or bucket subdirectory. |
| |
| Raises: |
| CommandException: if dst_uri_str matched more than 1 URI. |
| """ |
| dst_uri = self.suri_builder.StorageUri(dst_uri_str) |
| |
| # Handle wildcarded dst_uri case. |
| if ContainsWildcard(dst_uri): |
| blr_expansion = list(self.WildcardIterator(dst_uri)) |
| if len(blr_expansion) != 1: |
| raise CommandException('Destination (%s) must match exactly 1 URI' % |
| dst_uri_str) |
| blr = blr_expansion[0] |
| uri = blr.GetUri() |
| if uri.is_cloud_uri(): |
| return (uri, uri.names_bucket() or blr.HasPrefix() |
| or blr.GetKey().endswith('/')) |
| else: |
| return (uri, uri.names_directory()) |
| |
| # Handle non-wildcarded dst_uri: |
| if dst_uri.is_file_uri(): |
| return (dst_uri, dst_uri.names_directory()) |
| if dst_uri.names_bucket(): |
| return (dst_uri, True) |
| # For object URIs check 3 cases: (a) if the name ends with '/' treat as a |
| # subdir; else, perform a wildcard expansion with dst_uri + "*" and then |
| # find if (b) there's a Prefix matching dst_uri, or (c) name is of form |
| # dir_$folder$ (and in both these cases also treat dir as a subdir). |
| if dst_uri.is_cloud_uri() and dst_uri_str.endswith('/'): |
| return (dst_uri, True) |
| blr_expansion = list(self.WildcardIterator( |
| '%s*' % dst_uri_str.rstrip(dst_uri.delim))) |
| for blr in blr_expansion: |
| if blr.GetRStrippedUriString().endswith('_$folder$'): |
| return (dst_uri, True) |
| if blr.GetRStrippedUriString() == dst_uri_str.rstrip(dst_uri.delim): |
| return (dst_uri, blr.HasPrefix()) |
| return (dst_uri, False) |
| |
| def _ConstructDstUri(self, src_uri, exp_src_uri, |
| src_uri_names_container, src_uri_expands_to_multi, |
| have_multiple_srcs, exp_dst_uri, |
| have_existing_dest_subdir): |
| """ |
| Constructs the destination URI for a given exp_src_uri/exp_dst_uri pair, |
| using context-dependent naming rules that mimic Linux cp and mv behavior. |
| |
| Args: |
| src_uri: src_uri to be copied. |
| exp_src_uri: Single StorageUri from wildcard expansion of src_uri. |
| src_uri_names_container: True if src_uri names a container (including the |
| case of a wildcard-named bucket subdir (like gs://bucket/abc, |
| where gs://bucket/abc/* matched some objects). Note that this is |
| additional semantics tha src_uri.names_container() doesn't understand |
| because the latter only understands StorageUris, not wildcards. |
| src_uri_expands_to_multi: True if src_uri expanded to multiple URIs. |
| have_multiple_srcs: True if this is a multi-source request. This can be |
| true if src_uri wildcard-expanded to multiple URIs or if there were |
| multiple source URIs in the request. |
| exp_dst_uri: the expanded StorageUri requested for the cp destination. |
| Final written path is constructed from this plus a context-dependent |
| variant of src_uri. |
| have_existing_dest_subdir: bool indicator whether dest is an existing |
| subdirectory. |
| |
| Returns: |
| StorageUri to use for copy. |
| |
| Raises: |
| CommandException if destination object name not specified for |
| source and source is a stream. |
| """ |
| if self._ShouldTreatDstUriAsSingleton( |
| have_multiple_srcs, have_existing_dest_subdir, exp_dst_uri): |
| # We're copying one file or object to one file or object. |
| return exp_dst_uri |
| |
| if exp_src_uri.is_stream(): |
| if exp_dst_uri.names_container(): |
| raise CommandException('Destination object name needed when ' |
| 'source is a stream') |
| return exp_dst_uri |
| |
| if not self.recursion_requested and not have_multiple_srcs: |
| # We're copying one file or object to a subdirectory. Append final comp |
| # of exp_src_uri to exp_dst_uri. |
| src_final_comp = exp_src_uri.object_name.rpartition(src_uri.delim)[-1] |
| return self.suri_builder.StorageUri('%s%s%s' % ( |
| exp_dst_uri.uri.rstrip(exp_dst_uri.delim), exp_dst_uri.delim, |
| src_final_comp)) |
| |
| # Else we're copying multiple sources to a directory, bucket, or a bucket |
| # "sub-directory". |
| |
| # Ensure exp_dst_uri ends in delim char if we're doing a multi-src copy or |
| # a copy to a directory. (The check for copying to a directory needs |
| # special-case handling so that the command: |
| # gsutil cp gs://bucket/obj dir |
| # will turn into file://dir/ instead of file://dir -- the latter would cause |
| # the file "dirobj" to be created.) |
| # Note: need to check have_multiple_srcs or src_uri.names_container() |
| # because src_uri could be a bucket containing a single object, named |
| # as gs://bucket. |
| if ((have_multiple_srcs or src_uri.names_container() |
| or os.path.isdir(exp_dst_uri.object_name)) |
| and not exp_dst_uri.uri.endswith(exp_dst_uri.delim)): |
| exp_dst_uri = exp_dst_uri.clone_replace_name( |
| '%s%s' % (exp_dst_uri.object_name, exp_dst_uri.delim) |
| ) |
| |
| # Making naming behavior match how things work with local Linux cp and mv |
| # operations depends on many factors, including whether the destination is a |
| # container, the plurality of the source(s), and whether the mv command is |
| # being used: |
| # 1. For the "mv" command that specifies a non-existent destination subdir, |
| # renaming should occur at the level of the src subdir, vs appending that |
| # subdir beneath the dst subdir like is done for copying. For example: |
| # gsutil rm -R gs://bucket |
| # gsutil cp -R dir1 gs://bucket |
| # gsutil cp -R dir2 gs://bucket/subdir1 |
| # gsutil mv gs://bucket/subdir1 gs://bucket/subdir2 |
| # would (if using cp naming behavior) end up with paths like: |
| # gs://bucket/subdir2/subdir1/dir2/.svn/all-wcprops |
| # whereas mv naming behavior should result in: |
| # gs://bucket/subdir2/dir2/.svn/all-wcprops |
| # 2. Copying from directories, buckets, or bucket subdirs should result in |
| # objects/files mirroring the source directory hierarchy. For example: |
| # gsutil cp dir1/dir2 gs://bucket |
| # should create the object gs://bucket/dir2/file2, assuming dir1/dir2 |
| # contains file2). |
| # To be consistent with Linux cp behavior, there's one more wrinkle when |
| # working with subdirs: The resulting object names depend on whether the |
| # destination subdirectory exists. For example, if gs://bucket/subdir |
| # exists, the command: |
| # gsutil cp -R dir1/dir2 gs://bucket/subdir |
| # should create objects named like gs://bucket/subdir/dir2/a/b/c. In |
| # contrast, if gs://bucket/subdir does not exist, this same command |
| # should create objects named like gs://bucket/subdir/a/b/c. |
| # 3. Copying individual files or objects to dirs, buckets or bucket subdirs |
| # should result in objects/files named by the final source file name |
| # component. Example: |
| # gsutil cp dir1/*.txt gs://bucket |
| # should create the objects gs://bucket/f1.txt and gs://bucket/f2.txt, |
| # assuming dir1 contains f1.txt and f2.txt. |
| |
| if (self.perform_mv and self.recursion_requested |
| and src_uri_expands_to_multi and not have_existing_dest_subdir): |
| # Case 1. Handle naming rules for bucket subdir mv. Here we want to |
| # line up the src_uri against its expansion, to find the base to build |
| # the new name. For example, running the command: |
| # gsutil mv gs://bucket/abcd gs://bucket/xyz |
| # when processing exp_src_uri=gs://bucket/abcd/123 |
| # exp_src_uri_tail should become /123 |
| # Note: mv.py code disallows wildcard specification of source URI. |
| exp_src_uri_tail = exp_src_uri.uri[len(src_uri.uri):] |
| dst_key_name = '%s/%s' % (exp_dst_uri.object_name.rstrip('/'), |
| exp_src_uri_tail.strip('/')) |
| return exp_dst_uri.clone_replace_name(dst_key_name) |
| |
| if src_uri_names_container and not exp_dst_uri.names_file(): |
| # Case 2. Build dst_key_name from subpath of exp_src_uri past |
| # where src_uri ends. For example, for src_uri=gs://bucket/ and |
| # exp_src_uri=gs://bucket/src_subdir/obj, dst_key_name should be |
| # src_subdir/obj. |
| src_uri_path_sans_final_dir = _GetPathBeforeFinalDir(src_uri) |
| dst_key_name = exp_src_uri.uri[ |
| len(src_uri_path_sans_final_dir):].lstrip(src_uri.delim) |
| # Handle case where dst_uri is a non-existent subdir. |
| if not have_existing_dest_subdir: |
| dst_key_name = dst_key_name.partition(src_uri.delim)[-1] |
| # Handle special case where src_uri was a directory named with '.' or |
| # './', so that running a command like: |
| # gsutil cp -r . gs://dest |
| # will produce obj names of the form gs://dest/abc instead of |
| # gs://dest/./abc. |
| if dst_key_name.startswith('.%s' % os.sep): |
| dst_key_name = dst_key_name[2:] |
| |
| else: |
| # Case 3. |
| dst_key_name = exp_src_uri.object_name.rpartition(src_uri.delim)[-1] |
| |
| if (exp_dst_uri.is_file_uri() |
| or self._ShouldTreatDstUriAsBucketSubDir( |
| have_multiple_srcs, exp_dst_uri, have_existing_dest_subdir)): |
| if exp_dst_uri.object_name.endswith(exp_dst_uri.delim): |
| dst_key_name = '%s%s%s' % ( |
| exp_dst_uri.object_name.rstrip(exp_dst_uri.delim), |
| exp_dst_uri.delim, dst_key_name) |
| else: |
| delim = exp_dst_uri.delim if exp_dst_uri.object_name else '' |
| dst_key_name = '%s%s%s' % (exp_dst_uri.object_name, delim, dst_key_name) |
| |
| return exp_dst_uri.clone_replace_name(dst_key_name) |
| |
| def _FixWindowsNaming(self, src_uri, dst_uri): |
| """ |
| Rewrites the destination URI built by _ConstructDstUri() to translate |
| Windows pathnames to cloud pathnames if needed. |
| |
| Args: |
| src_uri: Source URI to be copied. |
| dst_uri: The destination URI built by _ConstructDstUri(). |
| |
| Returns: |
| StorageUri to use for copy. |
| """ |
| if (src_uri.is_file_uri() and src_uri.delim == '\\' |
| and dst_uri.is_cloud_uri()): |
| trans_uri_str = re.sub(r'\\', '/', dst_uri.uri) |
| dst_uri = self.suri_builder.StorageUri(trans_uri_str) |
| return dst_uri |
| |
| # Command entry point. |
| def RunCommand(self): |
| |
| # Inner funcs. |
| def _CopyExceptionHandler(e): |
| """Simple exception handler to allow post-completion status.""" |
| self.THREADED_LOGGER.error(str(e)) |
| self.copy_failure_count += 1 |
| |
| def _CopyFunc(name_expansion_result): |
| """Worker function for performing the actual copy (and rm, for mv).""" |
| if self.perform_mv: |
| cmd_name = 'mv' |
| else: |
| cmd_name = self.command_name |
| src_uri = self.suri_builder.StorageUri( |
| name_expansion_result.GetSrcUriStr()) |
| exp_src_uri = self.suri_builder.StorageUri( |
| name_expansion_result.GetExpandedUriStr()) |
| src_uri_names_container = name_expansion_result.NamesContainer() |
| src_uri_expands_to_multi = name_expansion_result.NamesContainer() |
| have_multiple_srcs = name_expansion_result.IsMultiSrcRequest() |
| have_existing_dest_subdir = ( |
| name_expansion_result.HaveExistingDstContainer()) |
| |
| if src_uri.names_provider(): |
| raise CommandException( |
| 'The %s command does not allow provider-only source URIs (%s)' % |
| (cmd_name, src_uri)) |
| if have_multiple_srcs: |
| self._InsistDstUriNamesContainer(exp_dst_uri, |
| have_existing_dst_container, |
| cmd_name) |
| |
| if self.perform_mv: |
| if name_expansion_result.NamesContainer(): |
| # Use recursion_requested when performing name expansion for the |
| # directory mv case so we can determine if any of the source URIs are |
| # directories (and then use cp -R and rm -R to perform the move, to |
| # match the behavior of Linux mv (which when moving a directory moves |
| # all the contained files). |
| self.recursion_requested = True |
| # Disallow wildcard src URIs when moving directories, as supporting it |
| # would make the name transformation too complex and would also be |
| # dangerous (e.g., someone could accidentally move many objects to the |
| # wrong name, or accidentally overwrite many objects). |
| if ContainsWildcard(src_uri): |
| raise CommandException('The mv command disallows naming source ' |
| 'directories using wildcards') |
| |
| if (exp_dst_uri.is_file_uri() |
| and not os.path.exists(exp_dst_uri.object_name) |
| and have_multiple_srcs): |
| os.makedirs(exp_dst_uri.object_name) |
| |
| dst_uri = self._ConstructDstUri(src_uri, exp_src_uri, |
| src_uri_names_container, |
| src_uri_expands_to_multi, |
| have_multiple_srcs, exp_dst_uri, |
| have_existing_dest_subdir) |
| dst_uri = self._FixWindowsNaming(src_uri, dst_uri) |
| |
| self._CheckForDirFileConflict(exp_src_uri, dst_uri) |
| if self._SrcDstSame(exp_src_uri, dst_uri): |
| raise CommandException('%s: "%s" and "%s" are the same file - ' |
| 'abort.' % (cmd_name, exp_src_uri, dst_uri)) |
| |
| if dst_uri.is_cloud_uri() and dst_uri.is_version_specific: |
| raise CommandException('%s: a version-specific URI\n(%s)\ncannot be ' |
| 'the destination for gsutil cp - abort.' |
| % (cmd_name, dst_uri)) |
| |
| elapsed_time = bytes_transferred = 0 |
| try: |
| (elapsed_time, bytes_transferred, result_uri) = ( |
| self._PerformCopy(exp_src_uri, dst_uri)) |
| except Exception, e: |
| if self._IsNoClobberServerException(e): |
| if not self.quiet: |
| self.THREADED_LOGGER.info('Rejected (noclobber): %s' % dst_uri.uri) |
| elif self.continue_on_error: |
| if not self.quiet: |
| self.THREADED_LOGGER.error('Error copying %s: %s' % (src_uri.uri, |
| str(e))) |
| self.copy_failure_count += 1 |
| else: |
| raise |
| if self.print_ver: |
| # Some cases don't return a version-specific URI (e.g., if destination |
| # is a file). |
| if hasattr(result_uri, 'version_specific_uri'): |
| self.THREADED_LOGGER.info('Created: %s' % |
| result_uri.version_specific_uri) |
| else: |
| self.THREADED_LOGGER.info('Created: %s' % result_uri.uri) |
| |
| # TODO: If we ever use -n (noclobber) with -M (move) (not possible today |
| # since we call copy internally from move and don't specify the -n flag) |
| # we'll need to only remove the source when we have not skipped the |
| # destination. |
| if self.perform_mv: |
| if not self.quiet: |
| self.THREADED_LOGGER.info('Removing %s...', exp_src_uri) |
| exp_src_uri.delete_key(validate=False, headers=self.headers) |
| stats_lock.acquire() |
| self.total_elapsed_time += elapsed_time |
| self.total_bytes_transferred += bytes_transferred |
| stats_lock.release() |
| |
| # Start of RunCommand code. |
| self._ParseArgs() |
| |
| self.total_elapsed_time = self.total_bytes_transferred = 0 |
| if self.args[-1] == '-' or self.args[-1] == 'file://-': |
| self._HandleStreamingDownload() |
| return 0 |
| |
| if self.read_args_from_stdin: |
| if len(self.args) != 1: |
| raise CommandException('Source URIs cannot be specified with -I option') |
| uri_strs = self._StdinIterator() |
| else: |
| if len(self.args) < 2: |
| raise CommandException('Wrong number of arguments for "cp" command.') |
| uri_strs = self.args[0:len(self.args)-1] |
| |
| (exp_dst_uri, have_existing_dst_container) = self._ExpandDstUri( |
| self.args[-1]) |
| name_expansion_iterator = NameExpansionIterator( |
| self.command_name, self.proj_id_handler, self.headers, self.debug, |
| self.bucket_storage_uri_class, uri_strs, |
| self.recursion_requested or self.perform_mv, |
| have_existing_dst_container) |
| |
| # Use a lock to ensure accurate statistics in the face of |
| # multi-threading/multi-processing. |
| stats_lock = threading.Lock() |
| |
| # Tracks if any copies failed. |
| self.copy_failure_count = 0 |
| |
| # Start the clock. |
| start_time = time.time() |
| |
| # Tuple of attributes to share/manage across multiple processes in |
| # parallel (-m) mode. |
| shared_attrs = ('copy_failure_count', 'total_bytes_transferred') |
| |
| # Perform copy requests in parallel (-m) mode, if requested, using |
| # configured number of parallel processes and threads. Otherwise, |
| # perform requests with sequential function calls in current process. |
| self.Apply(_CopyFunc, name_expansion_iterator, _CopyExceptionHandler, |
| shared_attrs) |
| if self.debug: |
| print 'total_bytes_transferred:' + str(self.total_bytes_transferred) |
| |
| end_time = time.time() |
| self.total_elapsed_time = end_time - start_time |
| |
| # Sometimes, particularly when running unit tests, the total elapsed time |
| # is really small. On Windows, the timer resolution is too small and |
| # causes total_elapsed_time to be zero. |
| try: |
| float(self.total_bytes_transferred) / float(self.total_elapsed_time) |
| except ZeroDivisionError: |
| self.total_elapsed_time = 0.01 |
| |
| self.total_bytes_per_second = (float(self.total_bytes_transferred) / |
| float(self.total_elapsed_time)) |
| |
| if self.debug == 3: |
| # Note that this only counts the actual GET and PUT bytes for the copy |
| # - not any transfers for doing wildcard expansion, the initial HEAD |
| # request boto performs when doing a bucket.get_key() operation, etc. |
| if self.total_bytes_transferred != 0: |
| self.THREADED_LOGGER.info( |
| 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)', |
| self.total_bytes_transferred, self.total_elapsed_time, |
| MakeHumanReadable(self.total_bytes_per_second)) |
| if self.copy_failure_count: |
| plural_str = '' |
| if self.copy_failure_count > 1: |
| plural_str = 's' |
| raise CommandException('%d file%s/object%s could not be transferred.' % ( |
| self.copy_failure_count, plural_str, plural_str)) |
| |
| return 0 |
| |
| def _ParseArgs(self): |
| self.perform_mv = False |
| self.exclude_symlinks = False |
| self.quiet = False |
| self.no_clobber = False |
| self.continue_on_error = False |
| self.daisy_chain = False |
| self.read_args_from_stdin = False |
| self.print_ver = False |
| # self.recursion_requested initialized in command.py (so can be checked |
| # in parent class for all commands). |
| if self.sub_opts: |
| for o, unused_a in self.sub_opts: |
| if o == '-c': |
| self.continue_on_error = True |
| elif o == '-D': |
| self.daisy_chain = True |
| elif o == '-e': |
| self.exclude_symlinks = True |
| elif o == '-I': |
| self.read_args_from_stdin = True |
| elif o == '-M': |
| # Note that we signal to the cp command to perform a move (copy |
| # followed by remove) and use directory-move naming rules by passing |
| # the undocumented (for internal use) -M option when running the cp |
| # command from mv.py. |
| self.perform_mv = True |
| elif o == '-n': |
| self.no_clobber = True |
| elif o == '-q': |
| self.quiet = True |
| elif o == '-r' or o == '-R': |
| self.recursion_requested = True |
| elif o == '-v': |
| self.print_ver = True |
| |
| def _HandleStreamingDownload(self): |
| # Destination is <STDOUT>. Manipulate sys.stdout so as to redirect all |
| # debug messages to <STDERR>. |
| stdout_fp = sys.stdout |
| sys.stdout = sys.stderr |
| did_some_work = False |
| for uri_str in self.args[0:len(self.args)-1]: |
| for uri in self.WildcardIterator(uri_str).IterUris(): |
| did_some_work = True |
| key = uri.get_key(False, self.headers) |
| (elapsed_time, bytes_transferred) = self._PerformDownloadToStream( |
| key, uri, stdout_fp, self.headers) |
| self.total_elapsed_time += elapsed_time |
| self.total_bytes_transferred += bytes_transferred |
| if not did_some_work: |
| raise CommandException('No URIs matched') |
| if self.debug == 3: |
| if self.total_bytes_transferred != 0: |
| self.THREADED_LOGGER.info( |
| 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)', |
| self.total_bytes_transferred, self.total_elapsed_time, |
| MakeHumanReadable(float(self.total_bytes_transferred) / |
| float(self.total_elapsed_time))) |
| |
| def _StdinIterator(self): |
| """A generator function that returns lines from stdin.""" |
| for line in sys.stdin: |
| # Strip CRLF. |
| yield line.rstrip() |
| |
| def _SrcDstSame(self, src_uri, dst_uri): |
| """Checks if src_uri and dst_uri represent the same object or file. |
| |
| We don't handle anything about hard or symbolic links. |
| |
| Args: |
| src_uri: Source StorageUri. |
| dst_uri: Destination StorageUri. |
| |
| Returns: |
| Bool indicator. |
| """ |
| if src_uri.is_file_uri() and dst_uri.is_file_uri(): |
| # Translate a/b/./c to a/b/c, so src=dst comparison below works. |
| new_src_path = os.path.normpath(src_uri.object_name) |
| new_dst_path = os.path.normpath(dst_uri.object_name) |
| return (src_uri.clone_replace_name(new_src_path).uri == |
| dst_uri.clone_replace_name(new_dst_path).uri) |
| else: |
| return (src_uri.uri == dst_uri.uri and |
| src_uri.generation == dst_uri.generation and |
| src_uri.version_id == dst_uri.version_id) |
| |
| def _ShouldTreatDstUriAsBucketSubDir(self, have_multiple_srcs, dst_uri, |
| have_existing_dest_subdir): |
| """ |
| Checks whether dst_uri should be treated as a bucket "sub-directory". The |
| decision about whether something constitutes a bucket "sub-directory" |
| depends on whether there are multiple sources in this request and whether |
| there is an existing bucket subdirectory. For example, when running the |
| command: |
| gsutil cp file gs://bucket/abc |
| if there's no existing gs://bucket/abc bucket subdirectory we should copy |
| file to the object gs://bucket/abc. In contrast, if |
| there's an existing gs://bucket/abc bucket subdirectory we should copy |
| file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc |
| exists, when running the command: |
| gsutil cp file1 file2 gs://bucket/abc |
| we should copy file1 to gs://bucket/abc/file1 (and similarly for file2). |
| |
| Note that we don't disallow naming a bucket "sub-directory" where there's |
| already an object at that URI. For example it's legitimate (albeit |
| confusing) to have an object called gs://bucket/dir and |
| then run the command |
| gsutil cp file1 file2 gs://bucket/dir |
| Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1, |
| and gs://bucket/dir/file2. |
| |
| Args: |
| have_multiple_srcs: Bool indicator of whether this is a multi-source |
| operation. |
| dst_uri: StorageUri to check. |
| have_existing_dest_subdir: bool indicator whether dest is an existing |
| subdirectory. |
| |
| Returns: |
| bool indicator. |
| """ |
| return ((have_multiple_srcs and dst_uri.is_cloud_uri()) |
| or (have_existing_dest_subdir)) |
| |
| def _ShouldTreatDstUriAsSingleton(self, have_multiple_srcs, |
| have_existing_dest_subdir, dst_uri): |
| """ |
| Checks that dst_uri names a singleton (file or object) after |
| dir/wildcard expansion. The decision is more nuanced than simply |
| dst_uri.names_singleton()) because of the possibility that an object path |
| might name a bucket sub-directory. |
| |
| Args: |
| have_multiple_srcs: Bool indicator of whether this is a multi-source |
| operation. |
| have_existing_dest_subdir: bool indicator whether dest is an existing |
| subdirectory. |
| dst_uri: StorageUri to check. |
| |
| Returns: |
| bool indicator. |
| """ |
| if have_multiple_srcs: |
| # Only a file meets the criteria in this case. |
| return dst_uri.names_file() |
| return not have_existing_dest_subdir and dst_uri.names_singleton() |
| |
| def _IsNoClobberServerException(self, e): |
| """ |
| Checks to see if the server attempted to clobber a file after we specified |
| in the header that we didn't want the file clobbered. |
| |
| Args: |
| e: The Exception that was generated by a failed copy operation |
| |
| Returns: |
| bool indicator - True indicates that the server did attempt to clobber |
| an existing file. |
| """ |
| return self.no_clobber and ( |
| (isinstance(e, GSResponseError) and e.status==412) or |
| (isinstance(e, ResumableUploadException) and 'code 412' in e.message)) |
| |
| def _GetPathBeforeFinalDir(uri): |
| """ |
| Returns the part of the path before the final directory component for the |
| given URI, handling cases for file system directories, bucket, and bucket |
| subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket', |
| and for file://dir we'll return file:// |
| |
| Args: |
| uri: StorageUri. |
| |
| Returns: |
| String name of above-described path, sans final path separator. |
| """ |
| sep = uri.delim |
| assert not uri.names_file() |
| if uri.names_directory(): |
| past_scheme = uri.uri[len('file://'):] |
| if past_scheme.find(sep) == -1: |
| return 'file://' |
| else: |
| return 'file://%s' % past_scheme.rstrip(sep).rpartition(sep)[0] |
| if uri.names_bucket(): |
| return '%s://' % uri.scheme |
| # Else it names a bucket subdir. |
| return uri.uri.rstrip(sep).rpartition(sep)[0] |
| |
| def _hash_filename(filename): |
| """ |
| Apply a hash function (SHA1) to shorten the passed file name. The spec |
| for the hashed file name is as follows: |
| |
| TRACKER_<hash>_<trailing> |
| |
| where hash is a SHA1 hash on the original file name and trailing is |
| the last 16 chars from the original file name. Max file name lengths |
| vary by operating system so the goal of this function is to ensure |
| the hashed version takes fewer than 100 characters. |
| |
| Args: |
| filename: file name to be hashed. |
| |
| Returns: |
| shorter, hashed version of passed file name |
| """ |
| if not isinstance(filename, unicode): |
| filename = unicode(filename, 'utf8').encode('utf-8') |
| m = hashlib.sha1(filename) |
| return "TRACKER_" + m.hexdigest() + '.' + filename[-16:] |