--- a/hgext/largefiles/lfutil.py Sat Oct 05 10:29:34 2019 -0400
+++ b/hgext/largefiles/lfutil.py Sun Oct 06 09:45:02 2019 -0400
@@ -37,6 +37,7 @@
# -- Private worker functions ------------------------------------------
+
def getminsize(ui, assumelfiles, opt, default=10):
lfsize = opt
if not lfsize and assumelfiles:
@@ -45,12 +46,14 @@
try:
lfsize = float(lfsize)
except ValueError:
- raise error.Abort(_('largefiles: size must be number (not %s)\n')
- % lfsize)
+ raise error.Abort(
+ _('largefiles: size must be number (not %s)\n') % lfsize
+ )
if lfsize is None:
raise error.Abort(_('minimum size for largefiles must be specified'))
return lfsize
+
def link(src, dest):
"""Try to create hardlink - if that fails, efficiently make a copy."""
util.makedirs(os.path.dirname(dest))
@@ -63,6 +66,7 @@
dstf.write(chunk)
os.chmod(dest, os.stat(src).st_mode)
+
def usercachepath(ui, hash):
'''Return the correct location in the "global" largefiles cache for a file
with the given hash.
@@ -70,14 +74,16 @@
to preserve download bandwidth and storage space.'''
return os.path.join(_usercachedir(ui), hash)
+
def _usercachedir(ui, name=longname):
'''Return the location of the "global" largefiles cache.'''
path = ui.configpath(name, 'usercache')
if path:
return path
if pycompat.iswindows:
- appdata = encoding.environ.get('LOCALAPPDATA',
- encoding.environ.get('APPDATA'))
+ appdata = encoding.environ.get(
+ 'LOCALAPPDATA', encoding.environ.get('APPDATA')
+ )
if appdata:
return os.path.join(appdata, name)
elif pycompat.isdarwin:
@@ -92,14 +98,15 @@
if home:
return os.path.join(home, '.cache', name)
else:
- raise error.Abort(_('unknown operating system: %s\n')
- % pycompat.osname)
+ raise error.Abort(_('unknown operating system: %s\n') % pycompat.osname)
raise error.Abort(_('unknown %s usercache location') % name)
+
def inusercache(ui, hash):
path = usercachepath(ui, hash)
return os.path.exists(path)
+
def findfile(repo, hash):
'''Return store path of the largefile with the specified hash.
As a side effect, the file might be linked from user cache.
@@ -115,29 +122,39 @@
return path
return None
+
class largefilesdirstate(dirstate.dirstate):
def __getitem__(self, key):
return super(largefilesdirstate, self).__getitem__(unixpath(key))
+
def normal(self, f):
return super(largefilesdirstate, self).normal(unixpath(f))
+
def remove(self, f):
return super(largefilesdirstate, self).remove(unixpath(f))
+
def add(self, f):
return super(largefilesdirstate, self).add(unixpath(f))
+
def drop(self, f):
return super(largefilesdirstate, self).drop(unixpath(f))
+
def forget(self, f):
return super(largefilesdirstate, self).forget(unixpath(f))
+
def normallookup(self, f):
return super(largefilesdirstate, self).normallookup(unixpath(f))
+
def _ignore(self, f):
return False
+
def write(self, tr=False):
# (1) disable PENDING mode always
# (lfdirstate isn't yet managed as a part of the transaction)
# (2) avoid develwarn 'use dirstate.write with ....'
super(largefilesdirstate, self).write(None)
+
def openlfdirstate(ui, repo, create=True):
'''
Return a dirstate object that tracks largefiles: i.e. its root is
@@ -146,17 +163,22 @@
vfs = repo.vfs
lfstoredir = longname
opener = vfsmod.vfs(vfs.join(lfstoredir))
- lfdirstate = largefilesdirstate(opener, ui, repo.root,
- repo.dirstate._validate,
- lambda: sparse.matcher(repo))
+ lfdirstate = largefilesdirstate(
+ opener,
+ ui,
+ repo.root,
+ repo.dirstate._validate,
+ lambda: sparse.matcher(repo),
+ )
# If the largefiles dirstate does not exist, populate and create
# it. This ensures that we create it on the first meaningful
# largefiles operation in a new clone.
if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')):
matcher = getstandinmatcher(repo)
- standins = repo.dirstate.walk(matcher, subrepos=[], unknown=False,
- ignored=False)
+ standins = repo.dirstate.walk(
+ matcher, subrepos=[], unknown=False, ignored=False
+ )
if len(standins) > 0:
vfs.makedirs(lfstoredir)
@@ -166,11 +188,13 @@
lfdirstate.normallookup(lfile)
return lfdirstate
+
def lfdirstatestatus(lfdirstate, repo):
pctx = repo['.']
match = matchmod.always()
- unsure, s = lfdirstate.status(match, subrepos=[], ignored=False,
- clean=False, unknown=False)
+ unsure, s = lfdirstate.status(
+ match, subrepos=[], ignored=False, clean=False, unknown=False
+ )
modified, clean = s.modified, s.clean
for lfile in unsure:
try:
@@ -184,6 +208,7 @@
lfdirstate.normal(lfile)
return s
+
def listlfiles(repo, rev=None, matcher=None):
'''return a list of largefiles in the working copy or the
specified changeset'''
@@ -192,14 +217,18 @@
matcher = getstandinmatcher(repo)
# ignore unknown files in working directory
- return [splitstandin(f)
- for f in repo[rev].walk(matcher)
- if rev is not None or repo.dirstate[f] != '?']
+ return [
+ splitstandin(f)
+ for f in repo[rev].walk(matcher)
+ if rev is not None or repo.dirstate[f] != '?'
+ ]
+
def instore(repo, hash, forcelocal=False):
'''Return true if a largefile with the given hash exists in the store'''
return os.path.exists(storepath(repo, hash, forcelocal))
+
def storepath(repo, hash, forcelocal=False):
'''Return the correct location in the repository largefiles store for a
file with the given hash.'''
@@ -207,6 +236,7 @@
return repo.vfs.reljoin(repo.sharedpath, longname, hash)
return repo.vfs.join(longname, hash)
+
def findstorepath(repo, hash):
'''Search through the local store path(s) to find the file for the given
hash. If the file is not found, its path in the primary store is returned.
@@ -224,6 +254,7 @@
return (path, False)
+
def copyfromcache(repo, hash, filename):
'''Copy the specified largefile from the repo or system cache to
filename in the repository. Return true on success or false if the
@@ -238,15 +269,17 @@
# The write may fail before the file is fully written, but we
# don't use atomic writes in the working copy.
with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd:
- gothash = copyandhash(
- util.filechunkiter(srcfd), destfd)
+ gothash = copyandhash(util.filechunkiter(srcfd), destfd)
if gothash != hash:
- repo.ui.warn(_('%s: data corruption in %s with hash %s\n')
- % (filename, path, gothash))
+ repo.ui.warn(
+ _('%s: data corruption in %s with hash %s\n')
+ % (filename, path, gothash)
+ )
wvfs.unlink(filename)
return False
return True
+
def copytostore(repo, ctx, file, fstandin):
wvfs = repo.wvfs
hash = readasstandin(ctx[fstandin])
@@ -255,8 +288,11 @@
if wvfs.exists(file):
copytostoreabsolute(repo, wvfs.join(file), hash)
else:
- repo.ui.warn(_("%s: largefile %s not available from local store\n") %
- (file, hash))
+ repo.ui.warn(
+ _("%s: largefile %s not available from local store\n")
+ % (file, hash)
+ )
+
def copyalltostore(repo, node):
'''Copy all largefiles in a given revision to the store'''
@@ -267,24 +303,28 @@
if realfile is not None and filename in ctx.manifest():
copytostore(repo, ctx, realfile, filename)
+
def copytostoreabsolute(repo, file, hash):
if inusercache(repo.ui, hash):
link(usercachepath(repo.ui, hash), storepath(repo, hash))
else:
util.makedirs(os.path.dirname(storepath(repo, hash)))
with open(file, 'rb') as srcf:
- with util.atomictempfile(storepath(repo, hash),
- createmode=repo.store.createmode) as dstf:
+ with util.atomictempfile(
+ storepath(repo, hash), createmode=repo.store.createmode
+ ) as dstf:
for chunk in util.filechunkiter(srcf):
dstf.write(chunk)
linktousercache(repo, hash)
+
def linktousercache(repo, hash):
'''Link / copy the largefile with the specified hash from the store
to the cache.'''
path = usercachepath(repo.ui, hash)
link(storepath(repo, hash), path)
+
def getstandinmatcher(repo, rmatcher=None):
'''Return a match object that applies rmatcher to the standin directory'''
wvfs = repo.wvfs
@@ -303,18 +343,22 @@
match = scmutil.match(repo[None], [wvfs.join(standindir)], badfn=badfn)
return match
+
def composestandinmatcher(repo, rmatcher):
'''Return a matcher that accepts standins corresponding to the
files accepted by rmatcher. Pass the list of files in the matcher
as the paths specified by the user.'''
smatcher = getstandinmatcher(repo, rmatcher)
isstandin = smatcher.matchfn
+
def composedmatchfn(f):
return isstandin(f) and rmatcher.matchfn(splitstandin(f))
+
smatcher.matchfn = composedmatchfn
return smatcher
+
def standin(filename):
'''Return the repo-relative path to the standin for the specified big
file.'''
@@ -327,11 +371,13 @@
# passed filenames from an external source (like the command line).
return shortnameslash + util.pconvert(filename)
+
def isstandin(filename):
'''Return true if filename is a big file standin. filename must be
in Mercurial's internal form (slash-separated).'''
return filename.startswith(shortnameslash)
+
def splitstandin(filename):
# Split on / because that's what dirstate always uses, even on Windows.
# Change local separator to / first just in case we are passed filenames
@@ -342,6 +388,7 @@
else:
return None
+
def updatestandin(repo, lfile, standin):
"""Re-calculate hash value of lfile and write it into standin
@@ -355,16 +402,19 @@
else:
raise error.Abort(_('%s: file not found!') % lfile)
+
def readasstandin(fctx):
'''read hex hash from given filectx of standin file
This encapsulates how "standin" data is stored into storage layer.'''
return fctx.data().strip()
+
def writestandin(repo, standin, hash, executable):
'''write hash to <repo.root>/<standin>'''
repo.wwrite(standin, hash + '\n', executable and 'x' or '')
+
def copyandhash(instream, outfile):
'''Read bytes from instream (iterable) and write them to outfile,
computing the SHA-1 hash of the data along the way. Return the hash.'''
@@ -374,17 +424,22 @@
outfile.write(data)
return hex(hasher.digest())
+
def hashfile(file):
if not os.path.exists(file):
return ''
with open(file, 'rb') as fd:
return hexsha1(fd)
+
def getexecutable(filename):
mode = os.stat(filename).st_mode
- return ((mode & stat.S_IXUSR) and
- (mode & stat.S_IXGRP) and
- (mode & stat.S_IXOTH))
+ return (
+ (mode & stat.S_IXUSR)
+ and (mode & stat.S_IXGRP)
+ and (mode & stat.S_IXOTH)
+ )
+
def urljoin(first, second, *arg):
def join(left, right):
@@ -399,6 +454,7 @@
url = join(url, a)
return url
+
def hexsha1(fileobj):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data"""
@@ -407,31 +463,38 @@
h.update(chunk)
return hex(h.digest())
+
def httpsendfile(ui, filename):
return httpconnection.httpsendfile(ui, filename, 'rb')
+
def unixpath(path):
'''Return a version of path normalized for use with the lfdirstate.'''
return util.pconvert(os.path.normpath(path))
+
def islfilesrepo(repo):
'''Return true if the repo is a largefile repo.'''
- if ('largefiles' in repo.requirements and
- any(shortnameslash in f[0] for f in repo.store.datafiles())):
+ if 'largefiles' in repo.requirements and any(
+ shortnameslash in f[0] for f in repo.store.datafiles()
+ ):
return True
return any(openlfdirstate(repo.ui, repo, False))
+
class storeprotonotcapable(Exception):
def __init__(self, storetypes):
self.storetypes = storetypes
+
def getstandinsstate(repo):
standins = []
matcher = getstandinmatcher(repo)
wctx = repo[None]
- for standin in repo.dirstate.walk(matcher, subrepos=[], unknown=False,
- ignored=False):
+ for standin in repo.dirstate.walk(
+ matcher, subrepos=[], unknown=False, ignored=False
+ ):
lfile = splitstandin(standin)
try:
hash = readasstandin(wctx[standin])
@@ -440,6 +503,7 @@
standins.append((lfile, hash))
return standins
+
def synclfdirstate(repo, lfdirstate, lfile, normallookup):
lfstandin = standin(lfile)
if lfstandin in repo.dirstate:
@@ -448,8 +512,7 @@
else:
state, mtime = '?', -1
if state == 'n':
- if (normallookup or mtime < 0 or
- not repo.wvfs.exists(lfile)):
+ if normallookup or mtime < 0 or not repo.wvfs.exists(lfile):
# state 'n' doesn't ensure 'clean' in this case
lfdirstate.normallookup(lfile)
else:
@@ -463,6 +526,7 @@
elif state == '?':
lfdirstate.drop(lfile)
+
def markcommitted(orig, ctx, node):
repo = ctx.repo()
@@ -492,6 +556,7 @@
# at merging.
copyalltostore(repo, node)
+
def getlfilestoupdate(oldstandins, newstandins):
changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
filelist = []
@@ -500,10 +565,14 @@
filelist.append(f[0])
return filelist
+
def getlfilestoupload(repo, missing, addfunc):
makeprogress = repo.ui.makeprogress
- with makeprogress(_('finding outgoing largefiles'),
- unit=_('revisions'), total=len(missing)) as progress:
+ with makeprogress(
+ _('finding outgoing largefiles'),
+ unit=_('revisions'),
+ total=len(missing),
+ ) as progress:
for i, n in enumerate(missing):
progress.update(i)
parents = [p for p in repo[n].parents() if p != node.nullid]
@@ -533,6 +602,7 @@
if isstandin(fn) and fn in ctx:
addfunc(fn, readasstandin(ctx[fn]))
+
def updatestandinsbymatch(repo, match):
'''Update standins in the working directory according to specified match
@@ -553,8 +623,9 @@
# large.
lfdirstate = openlfdirstate(ui, repo)
dirtymatch = matchmod.always()
- unsure, s = lfdirstate.status(dirtymatch, subrepos=[], ignored=False,
- clean=False, unknown=False)
+ unsure, s = lfdirstate.status(
+ dirtymatch, subrepos=[], ignored=False, clean=False, unknown=False
+ )
modifiedfiles = unsure + s.modified + s.added + s.removed
lfiles = listlfiles(repo)
# this only loops through largefiles that exist (not
@@ -577,8 +648,9 @@
# Case 2: user calls commit with specified patterns: refresh
# any matching big files.
smatcher = composestandinmatcher(repo, match)
- standins = repo.dirstate.walk(smatcher, subrepos=[], unknown=False,
- ignored=False)
+ standins = repo.dirstate.walk(
+ smatcher, subrepos=[], unknown=False, ignored=False
+ )
# No matching big files: get out of the way and pass control to
# the usual commit() method.
@@ -636,6 +708,7 @@
return match
+
class automatedcommithook(object):
'''Stateful hook to update standins at the 1st commit of resuming
@@ -647,16 +720,18 @@
--continue``) should update them, because largefiles may be
modified manually.
'''
+
def __init__(self, resuming):
self.resuming = resuming
def __call__(self, repo, match):
if self.resuming:
- self.resuming = False # avoids updating at subsequent commits
+ self.resuming = False # avoids updating at subsequent commits
return updatestandinsbymatch(repo, match)
else:
return match
+
def getstatuswriter(ui, repo, forcibly=None):
'''Return the function to write largefiles specific status out
@@ -670,6 +745,6 @@
return repo._lfstatuswriters[-1]
else:
if forcibly:
- return ui.status # forcibly WRITE OUT
+ return ui.status # forcibly WRITE OUT
else:
- return lambda *msg, **opts: None # forcibly IGNORE
+ return lambda *msg, **opts: None # forcibly IGNORE