Add verification to downloaded files
This does one last check to see if the file downloaded by
download_from_google_storage.py actually matches its sha1
BUG=
Review URL: https://codereview.chromium.org/1252313005
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@296155 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/download_from_google_storage.py b/download_from_google_storage.py
index a45f387..44cc3a7 100755
--- a/download_from_google_storage.py
+++ b/download_from_google_storage.py
@@ -227,6 +227,15 @@
if code != 0:
out_q.put('%d> %s' % (thread_num, err))
ret_codes.put((code, err))
+ continue
+
+ remote_sha1 = get_sha1(output_filename)
+ if remote_sha1 != input_sha1_sum:
+ msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' %
+ (thread_num, remote_sha1, input_sha1_sum))
+ out_q.put(msg)
+ ret_codes.put((20, msg))
+ continue
# Set executable bit.
if sys.platform == 'cygwin':
diff --git a/tests/download_from_google_storage_unittests.py b/tests/download_from_google_storage_unittests.py
index a8af63b..f87c6a7 100755
--- a/tests/download_from_google_storage_unittests.py
+++ b/tests/download_from_google_storage_unittests.py
@@ -36,8 +36,8 @@
self.history = []
self.lock = threading.Lock()
- def add_expected(self, return_code, out, err):
- self.expected.append((return_code, out, err))
+ def add_expected(self, return_code, out, err, fn=None):
+ self.expected.append((return_code, out, err, fn))
def append_history(self, method, args):
self.history.append((method, args))
@@ -46,7 +46,10 @@
with self.lock:
self.append_history('call', args)
if self.expected:
- return self.expected.pop(0)[0]
+ code, _out, _err, fn = self.expected.pop(0)
+ if fn:
+ fn()
+ return code
else:
return 0
@@ -54,7 +57,10 @@
with self.lock:
self.append_history('check_call', args)
if self.expected:
- return self.expected.pop(0)
+ code, out, err, fn = self.expected.pop(0)
+ if fn:
+ fn()
+ return code, out, err
else:
return (0, '', '')
@@ -257,6 +263,32 @@
self.assertEqual(self.gsutil.history, expected_calls)
self.assertEqual(code, 101)
+ def test_corrupt_download(self):
+ q = Queue.Queue()
+ out_q = Queue.Queue()
+ ret_codes = Queue.Queue()
+ tmp_dir = tempfile.mkdtemp()
+ sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f'
+ output_filename = os.path.join(tmp_dir, 'lorem_ipsum.txt')
+ q.put(('7871c8e24da15bad8b0be2c36edc9dc77e37727f', output_filename))
+ q.put((None, None))
+ def _write_bad_file():
+ with open(output_filename, 'w') as f:
+ f.write('foobar')
+ self.gsutil.add_expected(0, '', '')
+ self.gsutil.add_expected(0, '', '', _write_bad_file)
+ download_from_google_storage._downloader_worker_thread(
+ 1, q, True, self.base_url, self.gsutil, out_q, ret_codes, True)
+ self.assertTrue(q.empty())
+ msg = ('1> ERROR remote sha1 (%s) does not match expected sha1 (%s).' %
+ ('8843d7f92416211de9ebb963ff4ce28125932878', sha1_hash))
+ self.assertEquals(out_q.get(), '1> Downloading %s...' % output_filename)
+ self.assertEquals(out_q.get(), msg)
+ self.assertEquals(ret_codes.get(), (20, msg))
+ self.assertTrue(out_q.empty())
+ self.assertTrue(ret_codes.empty())
+
+
def test_download_directory_no_recursive_non_force(self):
sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f'
input_filename = '%s/%s' % (self.base_url, sha1_hash)