In upload_to_google_storage, pass -z argument through to gsutil.
Also fix some latent bugs in the unit tests.
(cherry-picked by pasko@ from https://codereview.chromium.org/1048103002/)
BUG=467005
Review URL: https://codereview.chromium.org/1059723003
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@294674 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/tests/upload_to_google_storage_unittests.py b/tests/upload_to_google_storage_unittests.py
index 24ac6b8..3bac038 100755
--- a/tests/upload_to_google_storage_unittests.py
+++ b/tests/upload_to_google_storage_unittests.py
@@ -48,14 +48,14 @@
filenames = [self.lorem_ipsum]
output_filename = '%s.sha1' % self.lorem_ipsum
code = upload_to_google_storage.upload_to_google_storage(
- filenames, self.base_url, self.gsutil, True, False, 1, False)
+ filenames, self.base_url, self.gsutil, True, False, 1, False, 'txt')
self.assertEqual(
self.gsutil.history,
[('check_call',
('ls', '%s/%s' % (self.base_url, self.lorem_ipsum_sha1))),
('check_call',
- ('cp', '-q', filenames[0], '%s/%s' % (self.base_url,
- self.lorem_ipsum_sha1)))])
+ ('cp', '-z', 'txt', filenames[0],
+ '%s/%s' % (self.base_url, self.lorem_ipsum_sha1)))])
self.assertTrue(os.path.exists(output_filename))
self.assertEqual(
open(output_filename, 'rb').read(),
@@ -70,7 +70,7 @@
self.gsutil.add_expected(0, '', '')
self.gsutil.add_expected(0, etag_string, '')
code = upload_to_google_storage.upload_to_google_storage(
- filenames, self.base_url, self.gsutil, False, False, 1, False)
+ filenames, self.base_url, self.gsutil, False, False, 1, False, None)
self.assertEqual(
self.gsutil.history,
[('check_call',
@@ -100,7 +100,8 @@
False,
False,
self.stdout_queue,
- self.ret_codes)
+ self.ret_codes,
+ None)
expected_ret_codes = [
(20,
'Encountered error on uploading %s to %s/%s\nExpected error message' %
@@ -114,7 +115,7 @@
with open(output_filename, 'wb') as f:
f.write(fake_hash) # Fake hash.
code = upload_to_google_storage.upload_to_google_storage(
- filenames, self.base_url, self.gsutil, False, False, 1, True)
+ filenames, self.base_url, self.gsutil, False, False, 1, True, None)
self.assertEqual(
self.gsutil.history,
[('check_call',
@@ -122,7 +123,7 @@
('check_call',
('ls', '-L', '%s/%s' % (self.base_url, fake_hash))),
('check_call',
- ('cp', '-q', filenames[0], '%s/%s' % (self.base_url, fake_hash)))])
+ ('cp', filenames[0], '%s/%s' % (self.base_url, fake_hash)))])
self.assertEqual(
open(output_filename, 'rb').read(), fake_hash)
os.remove(output_filename)
@@ -162,4 +163,4 @@
if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py
index d88597c..4cf9d1a 100755
--- a/upload_to_google_storage.py
+++ b/upload_to_google_storage.py
@@ -67,7 +67,7 @@
def _upload_worker(
thread_num, upload_queue, base_url, gsutil, md5_lock, force,
- use_md5, stdout_queue, ret_codes):
+ use_md5, stdout_queue, ret_codes, gzip):
while True:
filename, sha1_sum = upload_queue.get()
if not filename:
@@ -92,7 +92,11 @@
continue
stdout_queue.put('%d> Uploading %s...' % (
thread_num, filename))
- code, _, err = gsutil.check_call('cp', filename, file_url)
+ gsutil_args = ['cp']
+ if gzip:
+ gsutil_args.extend(['-z', gzip])
+ gsutil_args.extend([filename, file_url])
+ code, _, err = gsutil.check_call(*gsutil_args)
if code != 0:
ret_codes.put(
(code,
@@ -129,7 +133,7 @@
def upload_to_google_storage(
input_filenames, base_url, gsutil, force,
- use_md5, num_threads, skip_hashing):
+ use_md5, num_threads, skip_hashing, gzip):
# We only want one MD5 calculation happening at a time to avoid HD thrashing.
md5_lock = threading.Lock()
@@ -147,7 +151,7 @@
t = threading.Thread(
target=_upload_worker,
args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
- force, use_md5, stdout_queue, ret_codes])
+ force, use_md5, stdout_queue, ret_codes, gzip])
t.daemon = True
t.start()
all_threads.append(t)
@@ -223,6 +227,9 @@
help='Use \\0 instead of \\n when parsing '
'the file list from stdin. This is useful if the input '
'is coming from "find ... -print0".')
+ parser.add_option('-z', '--gzip', metavar='ext',
+ help='Gzip files which end in ext. '
+ 'ext is a comma-separated list')
(options, args) = parser.parse_args()
# Enumerate our inputs.
@@ -244,7 +251,7 @@
return upload_to_google_storage(
input_filenames, base_url, gsutil, options.force, options.use_md5,
- options.num_threads, options.skip_hashing)
+ options.num_threads, options.skip_hashing, options.gzip)
if __name__ == '__main__':
