Add bad gateway error to recognized retry cases.

Convert to a set of failure string and make retry backoff conform to
google standard.
Add additional logging to distinguish when we do not retry because it
is not a retry case (but it failed).

b/64988437

Change-Id: I1e86dde71623e2b1488c276809bab930e515599f
diff --git a/gclient_sync_retry.py b/gclient_sync_retry.py
index 232f657..88227be 100644
--- a/gclient_sync_retry.py
+++ b/gclient_sync_retry.py
@@ -35,15 +35,13 @@
 
   print command
 
-  max_runs = 3
-  fail_string = 'Failed to connect'
+  max_runs = 6
+  fail_strings = ['Failed to connect', '502 Bad Gateway']
   os.environ['GIT_CURL_VERBOSE'] = '1'
 
   # Whether any runs were successful or not.  Success is defined to be output
   # that does not contain |fail_string|, as well as a zero exit code, as the
   # gclient call could have failed for other reasons.
-  any_succeeded = False
-
   for run in range(1, max_runs + 1):
     sys.stdout.write('Attempt %d\n' % run)
     p = subprocess.Popen(
@@ -54,20 +52,21 @@
     p_stdout, p_stderr = p.communicate()
     sys.stdout.write(p_stdout)
     sys.stderr.write(p_stderr)
-    contains_fail_string = (fail_string in p_stderr) or (
-        fail_string in p_stdout)
+    contains_fail_string = ((f in p_stderr for f in fail_strings) or
+                            (f in p_stdout for f in fail_strings))
 
-    if p.returncode == 0 and not contains_fail_string:
-      any_succeeded = True
+    if not contains_fail_string:
+      if p.returncode == 0:
+        return 0
+      sys.stdout.write('Sync returned failure but no retry case detected.\n')
       break
 
-    if run < max_runs and contains_fail_string:
-      sys.stdout.write('Retrying gclient sync\n')
-      time.sleep(5 * random.randint(0, 2**run - 1))
-      continue
-    break
+    if run < max_runs:
+      sys.stdout.write('Sync failed. Retrying.\n')
+      time.sleep(2**(run - 1) + 2 * random.random())
 
-  return 0 if any_succeeded else 1
+  sys.stdout.write('Permanent gclient sync failure.\n')
+  return 1
 
 
 if __name__ == '__main__':