Make git_cache.py import-able.
Evidence indicates that running non-builtin git commands is very
slow in msysgit, slow enough to dominate the running time of
gclient sync. With this change, gclient never shells out to
git-cache; it import the lib directly instead.
R=agable@chromium.org,hinoka@chromium.org
BUG=
Review URL: https://codereview.chromium.org/229653002
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@262759 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/gclient.py b/gclient.py
index 5fa3565..5c5c5ca 100755
--- a/gclient.py
+++ b/gclient.py
@@ -97,6 +97,7 @@
import fix_encoding
import gclient_scm
import gclient_utils
+import git_cache
from third_party.repo.progress import Progress
import subcommand
import subprocess2
@@ -1094,6 +1095,7 @@
self._enforced_os = tuple(set(self._enforced_os).union(target_os))
gclient_scm.GitWrapper.cache_dir = config_dict.get('cache_dir')
+ git_cache.Mirror.SetCachePath(config_dict.get('cache_dir'))
if not target_os and config_dict.get('target_os_only', False):
raise gclient_utils.Error('Can\'t use target_os_only if target_os is '
diff --git a/gclient_scm.py b/gclient_scm.py
index 5f04df0..f0e877a 100644
--- a/gclient_scm.py
+++ b/gclient_scm.py
@@ -18,6 +18,7 @@
import download_from_google_storage
import gclient_utils
+import git_cache
import scm
import subprocess2
@@ -28,7 +29,7 @@
os.path.dirname(os.path.abspath(__file__)),
'third_party', 'gsutil', 'gsutil')
-
+CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
class DiffFiltererWrapper(object):
"""Simple base class which tracks which file is being diffed and
replaces instances of its file name in the original and
@@ -159,23 +160,18 @@
"""Attempt to determine the remote URL for this SCMWrapper."""
# Git
if os.path.exists(os.path.join(self.checkout_path, '.git')):
- actual_remote_url = shlex.split(scm.GIT.Capture(
+ actual_remote_url = shlex.split(self._Capture(
['config', '--local', '--get-regexp', r'remote.*.url'],
self.checkout_path))[1]
# If a cache_dir is used, obtain the actual remote URL from the cache.
if getattr(self, 'cache_dir', None):
- try:
- full_cache_dir = self._Run(['cache', 'exists', '--cache-dir',
- self.cache_dir, self.url],
- options, cwd=self._root_dir).strip()
- except subprocess2.CalledProcessError:
- full_cache_dir = None
- if (full_cache_dir.replace('\\', '/') ==
+ mirror = git_cache.Mirror(self.url)
+ if (mirror.exists() and mirror.mirror_path.replace('\\', '/') ==
actual_remote_url.replace('\\', '/')):
- actual_remote_url = shlex.split(scm.GIT.Capture(
+ actual_remote_url = shlex.split(self._Capture(
['config', '--local', '--get-regexp', r'remote.*.url'],
- os.path.join(self._root_dir, full_cache_dir)))[1]
+ cwd=mirror.mirror_path))[1]
return actual_remote_url
# Svn
@@ -206,12 +202,15 @@
cache_dir = None
- def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
- out_cb=None):
+ def __init__(self, url=None, *args):
"""Removes 'git+' fake prefix from git URL."""
if url.startswith('git+http://') or url.startswith('git+https://'):
url = url[4:]
- SCMWrapper.__init__(self, url, root_dir, relpath, out_fh, out_cb)
+ SCMWrapper.__init__(self, url, *args)
+ filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh }
+ if self.out_cb:
+ filter_kwargs['predicate'] = self.out_cb
+ self.filter = gclient_utils.GitFilter(**filter_kwargs)
@staticmethod
def BinaryExists():
@@ -468,7 +467,10 @@
# case 0
self._CheckClean(rev_str)
self._CheckDetachedHead(rev_str, options)
- self._Capture(['checkout', '--quiet', '%s' % revision])
+ if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision:
+ self.Print('Up-to-date; skipping checkout.')
+ else:
+ self._Capture(['checkout', '--quiet', '%s' % revision])
if not printed_path:
self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False)
elif current_type == 'hash':
@@ -743,11 +745,13 @@
"""
if not self.cache_dir:
return url
- v = ['-v'] if options.verbose else []
- self._Run(['cache', 'populate'] + v + ['--cache-dir', self.cache_dir, url],
- options, cwd=self._root_dir, retry=True)
- return self._Run(['cache', 'exists', '--cache-dir', self.cache_dir, url],
- options, cwd=self._root_dir, ).strip()
+ mirror_kwargs = { 'print_func': self.filter }
+ if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
+ mirror_kwargs['refs'] = ['refs/tags/lkgr', 'refs/tags/lkcr']
+ mirror = git_cache.Mirror(url, **mirror_kwargs)
+ mirror.populate(verbose=options.verbose, bootstrap=True)
+ mirror.unlock()
+ return mirror.mirror_path if mirror.exists() else None
def _Clone(self, revision, url, options):
"""Clone a git repository from the given URL.
@@ -983,10 +987,7 @@
def _Run(self, args, options, **kwargs):
cwd = kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stdout', self.out_fh)
- filter_kwargs = { 'time_throttle': 10, 'out_fh': self.out_fh }
- if self.out_cb:
- filter_kwargs['predicate'] = self.out_cb
- kwargs['filter_fn'] = git_filter = gclient_utils.GitFilter(**filter_kwargs)
+ kwargs['filter_fn'] = self.filter
kwargs.setdefault('print_stdout', False)
# Don't prompt for passwords; just fail quickly and noisily.
# By default, git will use an interactive terminal prompt when a username/
@@ -1002,7 +1003,7 @@
cmd = ['git'] + args
header = "running '%s' in '%s'" % (' '.join(cmd), cwd)
- git_filter(header)
+ self.filter(header)
return gclient_utils.CheckCallAndFilter(cmd, **kwargs)
diff --git a/git_cache.py b/git_cache.py
index be44ec5..b60073e 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -5,6 +5,7 @@
"""A git command for managing a local cache of git repositories."""
+from __future__ import print_function
import errno
import logging
import optparse
@@ -18,38 +19,12 @@
import gclient_utils
import subcommand
-
-GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
-BOOTSTRAP_BUCKET = 'chromium-git-cache'
-GSUTIL_DEFAULT_PATH = os.path.join(
- os.path.dirname(os.path.abspath(__file__)),
- 'third_party', 'gsutil', 'gsutil')
-
-
-def UrlToCacheDir(url):
- """Convert a git url to a normalized form for the cache dir path."""
- parsed = urlparse.urlparse(url)
- norm_url = parsed.netloc + parsed.path
- if norm_url.endswith('.git'):
- norm_url = norm_url[:-len('.git')]
- return norm_url.replace('-', '--').replace('/', '-').lower()
-
-
-def RunGit(cmd, **kwargs):
- """Run git in a subprocess."""
- kwargs.setdefault('cwd', os.getcwd())
- if kwargs.get('filter_fn'):
- kwargs['filter_fn'] = gclient_utils.GitFilter(kwargs.get('filter_fn'))
- kwargs.setdefault('print_stdout', False)
- env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
- env.setdefault('GIT_ASKPASS', 'true')
- env.setdefault('SSH_ASKPASS', 'true')
- else:
- kwargs.setdefault('print_stdout', True)
- stdout = kwargs.get('stdout', sys.stdout)
- print >> stdout, 'running "git %s" in "%s"' % (' '.join(cmd), kwargs['cwd'])
- gclient_utils.CheckCallAndFilter([GIT_EXECUTABLE] + cmd, **kwargs)
-
+try:
+ # pylint: disable=E0602
+ WinErr = WindowsError
+except NameError:
+ class WinErr(Exception):
+ pass
class LockError(Exception):
pass
@@ -81,7 +56,7 @@
open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY)
fd = os.open(self.lockfile, open_flags, 0o644)
f = os.fdopen(fd, 'w')
- print >> f, self.pid
+ print(self.pid, file=f)
f.close()
def _remove_lockfile(self):
@@ -138,20 +113,234 @@
return self
def __exit__(self, *_exc):
- self.unlock()
+ # Windows is unreliable when it comes to file locking. YMMV.
+ try:
+ self.unlock()
+ except WinErr:
+ pass
+class Mirror(object):
+
+ git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
+ gsutil_exe = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ 'third_party', 'gsutil', 'gsutil')
+ bootstrap_bucket = 'chromium-git-cache'
+
+ def __init__(self, url, refs=None, print_func=None):
+ self.url = url
+ self.refs = refs or []
+ self.basedir = self.UrlToCacheDir(url)
+ self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
+ self.print = print_func or print
+
+ @staticmethod
+ def UrlToCacheDir(url):
+ """Convert a git url to a normalized form for the cache dir path."""
+ parsed = urlparse.urlparse(url)
+ norm_url = parsed.netloc + parsed.path
+ if norm_url.endswith('.git'):
+ norm_url = norm_url[:-len('.git')]
+ return norm_url.replace('-', '--').replace('/', '-').lower()
+
+ @staticmethod
+ def FindExecutable(executable):
+ """This mimics the "which" utility."""
+ path_folders = os.environ.get('PATH').split(os.pathsep)
+
+ for path_folder in path_folders:
+ target = os.path.join(path_folder, executable)
+ # Just incase we have some ~/blah paths.
+ target = os.path.abspath(os.path.expanduser(target))
+ if os.path.isfile(target) and os.access(target, os.X_OK):
+ return target
+ return None
+
+ @classmethod
+ def SetCachePath(cls, cachepath):
+ setattr(cls, 'cachepath', cachepath)
+
+ @classmethod
+ def GetCachePath(cls):
+ if not hasattr(cls, 'cachepath'):
+ try:
+ cachepath = subprocess.check_output(
+ [cls.git_exe, 'config', '--global', 'cache.cachepath']).strip()
+ except subprocess.CalledProcessError:
+ cachepath = None
+ if not cachepath:
+ raise RuntimeError('No global cache.cachepath git configuration found.')
+ setattr(cls, 'cachepath', cachepath)
+ return getattr(cls, 'cachepath')
+
+ def RunGit(self, cmd, **kwargs):
+ """Run git in a subprocess."""
+ cwd = kwargs.setdefault('cwd', self.mirror_path)
+ kwargs.setdefault('print_stdout', False)
+ kwargs.setdefault('filter_fn', self.print)
+ env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
+ env.setdefault('GIT_ASKPASS', 'true')
+ env.setdefault('SSH_ASKPASS', 'true')
+ self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
+ gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
+
+ def config(self, cwd=None):
+ if cwd is None:
+ cwd = self.mirror_path
+ self.RunGit(['config', 'core.deltaBaseCacheLimit',
+ gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
+ self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
+ self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
+ '+refs/heads/*:refs/heads/*'], cwd=cwd)
+ for ref in self.refs:
+ ref = ref.lstrip('+').rstrip('/')
+ if ref.startswith('refs/'):
+ refspec = '+%s:%s' % (ref, ref)
+ else:
+ refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
+ self.RunGit(['config', '--add', 'remote.origin.fetch', refspec], cwd=cwd)
+
+ def bootstrap_repo(self, directory):
+ """Bootstrap the repo from Google Stroage if possible.
+
+ Requires 7z on Windows and Unzip on Linux/Mac.
+ """
+ if sys.platform.startswith('win'):
+ if not self.FindExecutable('7z'):
+ self.print('''
+Cannot find 7z in the path. If you want git cache to be able to bootstrap from
+Google Storage, please install 7z from:
+
+http://www.7-zip.org/download.html
+''')
+ return False
+ else:
+ if not self.FindExecutable('unzip'):
+ self.print('''
+Cannot find unzip in the path. If you want git cache to be able to bootstrap
+from Google Storage, please ensure unzip is present on your system.
+''')
+ return False
+
+ gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
+ gsutil = Gsutil(
+ self.gsutil_exe, boto_path=os.devnull, bypass_prodaccess=True)
+ # Get the most recent version of the zipfile.
+ _, ls_out, _ = gsutil.check_call('ls', gs_folder)
+ ls_out_sorted = sorted(ls_out.splitlines())
+ if not ls_out_sorted:
+ # This repo is not on Google Storage.
+ return False
+ latest_checkout = ls_out_sorted[-1]
+
+ # Download zip file to a temporary directory.
+ try:
+ tempdir = tempfile.mkdtemp()
+ self.print('Downloading %s' % latest_checkout)
+ code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+ if code:
+ self.print('%s\n%s' % (out, err))
+ return False
+ filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
+
+ # Unpack the file with 7z on Windows, or unzip everywhere else.
+ if sys.platform.startswith('win'):
+ cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
+ else:
+ cmd = ['unzip', filename, '-d', directory]
+ retcode = subprocess.call(cmd)
+ finally:
+ # Clean up the downloaded zipfile.
+ gclient_utils.rmtree(tempdir)
+
+ if retcode:
+ self.print(
+ 'Extracting bootstrap zipfile %s failed.\n'
+ 'Resuming normal operations.' % filename)
+ return False
+ return True
+
+ def exists(self):
+ return os.path.isfile(os.path.join(self.mirror_path, 'config'))
+
+ def populate(self, depth=None, shallow=False, bootstrap=False,
+ verbose=False):
+ if shallow and not depth:
+ depth = 10000
+ gclient_utils.safe_makedirs(self.GetCachePath())
+
+ v = []
+ if verbose:
+ v = ['-v', '--progress']
+
+ d = []
+ if depth:
+ d = ['--depth', str(depth)]
+
+
+ with Lockfile(self.mirror_path):
+ # Setup from scratch if the repo is new or is in a bad state.
+ tempdir = None
+ if not os.path.exists(os.path.join(self.mirror_path, 'config')):
+ gclient_utils.rmtree(self.mirror_path)
+ tempdir = tempfile.mkdtemp(
+ suffix=self.basedir, dir=self.GetCachePath())
+ bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
+ if not bootstrapped:
+ self.RunGit(['init', '--bare'], cwd=tempdir)
+ else:
+ if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
+ logging.warn(
+ 'Shallow fetch requested, but repo cache already exists.')
+ d = []
+
+ rundir = tempdir or self.mirror_path
+ self.config(rundir)
+ fetch_cmd = ['fetch'] + v + d + ['origin']
+ fetch_specs = subprocess.check_output(
+ [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+ cwd=rundir).strip().splitlines()
+ for spec in fetch_specs:
+ try:
+ self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
+ except subprocess.CalledProcessError:
+ logging.warn('Fetch of %s failed' % spec)
+ if tempdir:
+ os.rename(tempdir, self.mirror_path)
+
+ def update_bootstrap(self):
+ # The files are named <git number>.zip
+ gen_number = subprocess.check_output(
+ [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
+ self.RunGit(['gc']) # Run Garbage Collect to compress packfile.
+ # Creating a temp file and then deleting it ensures we can use this name.
+ _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
+ os.remove(tmp_zipfile)
+ subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
+ gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
+ dest_name = 'gs://%s/%s/%s.zip' % (
+ self.bootstrap_bucket, self.basedir, gen_number)
+ gsutil.call('cp', tmp_zipfile, dest_name)
+ os.remove(tmp_zipfile)
+
+ def unlock(self):
+ lf = Lockfile(self.mirror_path)
+ config_lock = os.path.join(self.mirror_path, 'config.lock')
+ if os.path.exists(config_lock):
+ os.remove(config_lock)
+ lf.break_lock()
+
@subcommand.usage('[url of repo to check for caching]')
def CMDexists(parser, args):
"""Check to see if there already is a cache of the given repo."""
- options, args = parser.parse_args(args)
+ _, args = parser.parse_args(args)
if not len(args) == 1:
parser.error('git cache exists only takes exactly one repo url.')
url = args[0]
- repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
- flag_file = os.path.join(repo_dir, 'config')
- if os.path.isdir(repo_dir) and os.path.isfile(flag_file):
- print repo_dir
+ mirror = Mirror(url)
+ if mirror.exists():
+ print(mirror.mirror_path)
return 0
return 1
@@ -161,7 +350,7 @@
"""Create and uploads a bootstrap tarball."""
# Lets just assert we can't do this on Windows.
if sys.platform.startswith('win'):
- print >> sys.stderr, 'Sorry, update bootstrap will not work on Windows.'
+ print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
return 1
# First, we need to ensure the cache is populated.
@@ -170,24 +359,11 @@
CMDpopulate(parser, populate_args)
# Get the repo directory.
- options, args = parser.parse_args(args)
+ _, args = parser.parse_args(args)
url = args[0]
- repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
-
- # The files are named <git number>.zip
- gen_number = subprocess.check_output(['git', 'number', 'master'],
- cwd=repo_dir).strip()
- RunGit(['gc'], cwd=repo_dir) # Run Garbage Collect to compress packfile.
- # Creating a temp file and then deleting it ensures we can use this name.
- _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
- os.remove(tmp_zipfile)
- subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=repo_dir)
- gsutil = Gsutil(path=GSUTIL_DEFAULT_PATH, boto_path=None)
- dest_name = 'gs://%s/%s/%s.zip' % (BOOTSTRAP_BUCKET,
- UrlToCacheDir(url),
- gen_number)
- gsutil.call('cp', tmp_zipfile, dest_name)
- os.remove(tmp_zipfile)
+ mirror = Mirror(url)
+ mirror.update_bootstrap()
+ return 0
@subcommand.usage('[url of repo to add to or update in cache]')
@@ -203,130 +379,19 @@
help='Don\'t bootstrap from Google Storage')
options, args = parser.parse_args(args)
- if options.shallow and not options.depth:
- options.depth = 10000
if not len(args) == 1:
parser.error('git cache populate only takes exactly one repo url.')
url = args[0]
- gclient_utils.safe_makedirs(options.cache_dir)
- repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
-
- v = []
- filter_fn = lambda l: '[up to date]' not in l
- if options.verbose:
- v = ['-v', '--progress']
- filter_fn = None
-
- d = []
+ mirror = Mirror(url, refs=options.ref)
+ kwargs = {
+ 'verbose': options.verbose,
+ 'shallow': options.shallow,
+ 'bootstrap': not options.no_bootstrap,
+ }
if options.depth:
- d = ['--depth', '%d' % options.depth]
-
- def _find(executable):
- """This mimics the "which" utility."""
- path_folders = os.environ.get('PATH').split(os.pathsep)
-
- for path_folder in path_folders:
- target = os.path.join(path_folder, executable)
- # Just incase we have some ~/blah paths.
- target = os.path.abspath(os.path.expanduser(target))
- if os.path.isfile(target) and os.access(target, os.X_OK):
- return target
- return False
-
- def _maybe_bootstrap_repo(directory):
- """Bootstrap the repo from Google Stroage if possible.
-
- Requires 7z on Windows and Unzip on Linux/Mac.
- """
- if options.no_bootstrap:
- return False
- if sys.platform.startswith('win'):
- if not _find('7z'):
- print 'Cannot find 7z in the path.'
- print 'If you want git cache to be able to bootstrap from '
- print 'Google Storage, please install 7z from:'
- print 'http://www.7-zip.org/download.html'
- return False
- else:
- if not _find('unzip'):
- print 'Cannot find unzip in the path.'
- print 'If you want git cache to be able to bootstrap from '
- print 'Google Storage, please ensure unzip is present on your system.'
- return False
-
- folder = UrlToCacheDir(url)
- gs_folder = 'gs://%s/%s' % (BOOTSTRAP_BUCKET, folder)
- gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=os.devnull,
- bypass_prodaccess=True)
- # Get the most recent version of the zipfile.
- _, ls_out, _ = gsutil.check_call('ls', gs_folder)
- ls_out_sorted = sorted(ls_out.splitlines())
- if not ls_out_sorted:
- # This repo is not on Google Storage.
- return False
- latest_checkout = ls_out_sorted[-1]
-
- # Download zip file to a temporary directory.
- tempdir = tempfile.mkdtemp()
- print 'Downloading %s...' % latest_checkout
- code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
- if code:
- print '%s\n%s' % (out, err)
- return False
- filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
-
- # Unpack the file with 7z on Windows, or unzip everywhere else.
- if sys.platform.startswith('win'):
- cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
- else:
- cmd = ['unzip', filename, '-d', directory]
- retcode = subprocess.call(cmd)
-
- # Clean up the downloaded zipfile.
- gclient_utils.rmtree(tempdir)
- if retcode:
- print 'Extracting bootstrap zipfile %s failed.' % filename
- print 'Resuming normal operations'
- return False
- return True
-
- def _config(directory):
- RunGit(['config', 'core.deltaBaseCacheLimit',
- gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=directory)
- RunGit(['config', 'remote.origin.url', url],
- cwd=directory)
- RunGit(['config', '--replace-all', 'remote.origin.fetch',
- '+refs/heads/*:refs/heads/*'],
- cwd=directory)
- RunGit(['config', '--add', 'remote.origin.fetch',
- '+refs/tags/*:refs/tags/*'],
- cwd=directory)
- for ref in options.ref or []:
- ref = ref.rstrip('/')
- refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
- RunGit(['config', '--add', 'remote.origin.fetch', refspec],
- cwd=directory)
-
- with Lockfile(repo_dir):
- # Setup from scratch if the repo is new or is in a bad state.
- if not os.path.exists(os.path.join(repo_dir, 'config')):
- gclient_utils.rmtree(repo_dir)
- tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
- dir=options.cache_dir)
- bootstrapped = _maybe_bootstrap_repo(tempdir)
- if not bootstrapped:
- RunGit(['init', '--bare'], cwd=tempdir)
- _config(tempdir)
- fetch_cmd = ['fetch'] + v + d + ['origin']
- RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
- os.rename(tempdir, repo_dir)
- else:
- _config(repo_dir)
- if options.depth and os.path.exists(os.path.join(repo_dir, 'shallow')):
- logging.warn('Shallow fetch requested, but repo cache already exists.')
- fetch_cmd = ['fetch'] + v + ['origin']
- RunGit(fetch_cmd, filter_fn=filter_fn, cwd=repo_dir, retry=True)
+ kwargs['depth'] = options.depth
+ mirror.populate(**kwargs)
@subcommand.usage('[url of repo to unlock, or -a|--all]')
@@ -340,20 +405,22 @@
if len(args) > 1 or (len(args) == 0 and not options.all):
parser.error('git cache unlock takes exactly one repo url, or --all')
+ repo_dirs = []
if not options.all:
url = args[0]
- repo_dirs = [os.path.join(options.cache_dir, UrlToCacheDir(url))]
+ repo_dirs.append(Mirror(url).mirror_path)
else:
- repo_dirs = [os.path.join(options.cache_dir, path)
- for path in os.listdir(options.cache_dir)
- if os.path.isdir(os.path.join(options.cache_dir, path))]
- repo_dirs.extend([os.path.join(options.cache_dir,
+ cachepath = Mirror.GetCachePath()
+ repo_dirs = [os.path.join(cachepath, path)
+ for path in os.listdir(cachepath)
+ if os.path.isdir(os.path.join(cachepath, path))]
+ repo_dirs.extend([os.path.join(cachepath,
lockfile.replace('.lock', ''))
- for lockfile in os.listdir(options.cache_dir)
- if os.path.isfile(os.path.join(options.cache_dir,
+ for lockfile in os.listdir(cachepath)
+ if os.path.isfile(os.path.join(cachepath,
lockfile))
and lockfile.endswith('.lock')
- and os.path.join(options.cache_dir, lockfile)
+ and os.path.join(cachepath, lockfile)
not in repo_dirs])
lockfiles = [repo_dir + '.lock' for repo_dir in repo_dirs
if os.path.exists(repo_dir + '.lock')]
@@ -363,8 +430,8 @@
'Refusing to unlock the following repo caches: '
', '.join(lockfiles))
- unlocked = []
- untouched = []
+ unlocked_repos = []
+ untouched_repos = []
for repo_dir in repo_dirs:
lf = Lockfile(repo_dir)
config_lock = os.path.join(repo_dir, 'config.lock')
@@ -376,14 +443,16 @@
unlocked = True
if unlocked:
- unlocked.append(repo_dir)
+ unlocked_repos.append(repo_dir)
else:
- untouched.append(repo_dir)
+ untouched_repos.append(repo_dir)
- if unlocked:
- logging.info('Broke locks on these caches: %s' % unlocked)
- if untouched:
- logging.debug('Did not touch these caches: %s' % untouched)
+ if unlocked_repos:
+ logging.info('Broke locks on these caches:\n %s' % '\n '.join(
+ unlocked_repos))
+ if untouched_repos:
+ logging.debug('Did not touch these caches:\n %s' % '\n '.join(
+ untouched_repos))
class OptionParser(optparse.OptionParser):
@@ -400,20 +469,15 @@
options, args = optparse.OptionParser.parse_args(self, args, values)
try:
- global_cache_dir = subprocess.check_output(
- [GIT_EXECUTABLE, 'config', '--global', 'cache.cachepath']).strip()
- if options.cache_dir:
- if global_cache_dir and (
- os.path.abspath(options.cache_dir) !=
- os.path.abspath(global_cache_dir)):
- logging.warn('Overriding globally-configured cache directory.')
- else:
- options.cache_dir = global_cache_dir
- except subprocess.CalledProcessError:
- if not options.cache_dir:
- self.error('No cache directory specified on command line '
- 'or in cache.cachepath.')
- options.cache_dir = os.path.abspath(options.cache_dir)
+ global_cache_dir = Mirror.GetCachePath()
+ except RuntimeError:
+ global_cache_dir = None
+ if options.cache_dir:
+ if global_cache_dir and (
+ os.path.abspath(options.cache_dir) !=
+ os.path.abspath(global_cache_dir)):
+ logging.warn('Overriding globally-configured cache directory.')
+ Mirror.SetCachePath(options.cache_dir)
levels = [logging.WARNING, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
