Mercurial > public > mercurial-scm > hg-stable
diff hgext/largefiles/lfutil.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | aaad36b88298 |
children | 687b865b95ad |
line wrap: on
line diff
--- a/hgext/largefiles/lfutil.py Sat Oct 05 10:29:34 2019 -0400 +++ b/hgext/largefiles/lfutil.py Sun Oct 06 09:45:02 2019 -0400 @@ -37,6 +37,7 @@ # -- Private worker functions ------------------------------------------ + def getminsize(ui, assumelfiles, opt, default=10): lfsize = opt if not lfsize and assumelfiles: @@ -45,12 +46,14 @@ try: lfsize = float(lfsize) except ValueError: - raise error.Abort(_('largefiles: size must be number (not %s)\n') - % lfsize) + raise error.Abort( + _('largefiles: size must be number (not %s)\n') % lfsize + ) if lfsize is None: raise error.Abort(_('minimum size for largefiles must be specified')) return lfsize + def link(src, dest): """Try to create hardlink - if that fails, efficiently make a copy.""" util.makedirs(os.path.dirname(dest)) @@ -63,6 +66,7 @@ dstf.write(chunk) os.chmod(dest, os.stat(src).st_mode) + def usercachepath(ui, hash): '''Return the correct location in the "global" largefiles cache for a file with the given hash. @@ -70,14 +74,16 @@ to preserve download bandwidth and storage space.''' return os.path.join(_usercachedir(ui), hash) + def _usercachedir(ui, name=longname): '''Return the location of the "global" largefiles cache.''' path = ui.configpath(name, 'usercache') if path: return path if pycompat.iswindows: - appdata = encoding.environ.get('LOCALAPPDATA', - encoding.environ.get('APPDATA')) + appdata = encoding.environ.get( + 'LOCALAPPDATA', encoding.environ.get('APPDATA') + ) if appdata: return os.path.join(appdata, name) elif pycompat.isdarwin: @@ -92,14 +98,15 @@ if home: return os.path.join(home, '.cache', name) else: - raise error.Abort(_('unknown operating system: %s\n') - % pycompat.osname) + raise error.Abort(_('unknown operating system: %s\n') % pycompat.osname) raise error.Abort(_('unknown %s usercache location') % name) + def inusercache(ui, hash): path = usercachepath(ui, hash) return os.path.exists(path) + def findfile(repo, hash): '''Return store path of the largefile with the specified hash. As a side effect, the file might be linked from user cache. @@ -115,29 +122,39 @@ return path return None + class largefilesdirstate(dirstate.dirstate): def __getitem__(self, key): return super(largefilesdirstate, self).__getitem__(unixpath(key)) + def normal(self, f): return super(largefilesdirstate, self).normal(unixpath(f)) + def remove(self, f): return super(largefilesdirstate, self).remove(unixpath(f)) + def add(self, f): return super(largefilesdirstate, self).add(unixpath(f)) + def drop(self, f): return super(largefilesdirstate, self).drop(unixpath(f)) + def forget(self, f): return super(largefilesdirstate, self).forget(unixpath(f)) + def normallookup(self, f): return super(largefilesdirstate, self).normallookup(unixpath(f)) + def _ignore(self, f): return False + def write(self, tr=False): # (1) disable PENDING mode always # (lfdirstate isn't yet managed as a part of the transaction) # (2) avoid develwarn 'use dirstate.write with ....' super(largefilesdirstate, self).write(None) + def openlfdirstate(ui, repo, create=True): ''' Return a dirstate object that tracks largefiles: i.e. its root is @@ -146,17 +163,22 @@ vfs = repo.vfs lfstoredir = longname opener = vfsmod.vfs(vfs.join(lfstoredir)) - lfdirstate = largefilesdirstate(opener, ui, repo.root, - repo.dirstate._validate, - lambda: sparse.matcher(repo)) + lfdirstate = largefilesdirstate( + opener, + ui, + repo.root, + repo.dirstate._validate, + lambda: sparse.matcher(repo), + ) # If the largefiles dirstate does not exist, populate and create # it. This ensures that we create it on the first meaningful # largefiles operation in a new clone. if create and not vfs.exists(vfs.join(lfstoredir, 'dirstate')): matcher = getstandinmatcher(repo) - standins = repo.dirstate.walk(matcher, subrepos=[], unknown=False, - ignored=False) + standins = repo.dirstate.walk( + matcher, subrepos=[], unknown=False, ignored=False + ) if len(standins) > 0: vfs.makedirs(lfstoredir) @@ -166,11 +188,13 @@ lfdirstate.normallookup(lfile) return lfdirstate + def lfdirstatestatus(lfdirstate, repo): pctx = repo['.'] match = matchmod.always() - unsure, s = lfdirstate.status(match, subrepos=[], ignored=False, - clean=False, unknown=False) + unsure, s = lfdirstate.status( + match, subrepos=[], ignored=False, clean=False, unknown=False + ) modified, clean = s.modified, s.clean for lfile in unsure: try: @@ -184,6 +208,7 @@ lfdirstate.normal(lfile) return s + def listlfiles(repo, rev=None, matcher=None): '''return a list of largefiles in the working copy or the specified changeset''' @@ -192,14 +217,18 @@ matcher = getstandinmatcher(repo) # ignore unknown files in working directory - return [splitstandin(f) - for f in repo[rev].walk(matcher) - if rev is not None or repo.dirstate[f] != '?'] + return [ + splitstandin(f) + for f in repo[rev].walk(matcher) + if rev is not None or repo.dirstate[f] != '?' + ] + def instore(repo, hash, forcelocal=False): '''Return true if a largefile with the given hash exists in the store''' return os.path.exists(storepath(repo, hash, forcelocal)) + def storepath(repo, hash, forcelocal=False): '''Return the correct location in the repository largefiles store for a file with the given hash.''' @@ -207,6 +236,7 @@ return repo.vfs.reljoin(repo.sharedpath, longname, hash) return repo.vfs.join(longname, hash) + def findstorepath(repo, hash): '''Search through the local store path(s) to find the file for the given hash. If the file is not found, its path in the primary store is returned. @@ -224,6 +254,7 @@ return (path, False) + def copyfromcache(repo, hash, filename): '''Copy the specified largefile from the repo or system cache to filename in the repository. Return true on success or false if the @@ -238,15 +269,17 @@ # The write may fail before the file is fully written, but we # don't use atomic writes in the working copy. with open(path, 'rb') as srcfd, wvfs(filename, 'wb') as destfd: - gothash = copyandhash( - util.filechunkiter(srcfd), destfd) + gothash = copyandhash(util.filechunkiter(srcfd), destfd) if gothash != hash: - repo.ui.warn(_('%s: data corruption in %s with hash %s\n') - % (filename, path, gothash)) + repo.ui.warn( + _('%s: data corruption in %s with hash %s\n') + % (filename, path, gothash) + ) wvfs.unlink(filename) return False return True + def copytostore(repo, ctx, file, fstandin): wvfs = repo.wvfs hash = readasstandin(ctx[fstandin]) @@ -255,8 +288,11 @@ if wvfs.exists(file): copytostoreabsolute(repo, wvfs.join(file), hash) else: - repo.ui.warn(_("%s: largefile %s not available from local store\n") % - (file, hash)) + repo.ui.warn( + _("%s: largefile %s not available from local store\n") + % (file, hash) + ) + def copyalltostore(repo, node): '''Copy all largefiles in a given revision to the store''' @@ -267,24 +303,28 @@ if realfile is not None and filename in ctx.manifest(): copytostore(repo, ctx, realfile, filename) + def copytostoreabsolute(repo, file, hash): if inusercache(repo.ui, hash): link(usercachepath(repo.ui, hash), storepath(repo, hash)) else: util.makedirs(os.path.dirname(storepath(repo, hash))) with open(file, 'rb') as srcf: - with util.atomictempfile(storepath(repo, hash), - createmode=repo.store.createmode) as dstf: + with util.atomictempfile( + storepath(repo, hash), createmode=repo.store.createmode + ) as dstf: for chunk in util.filechunkiter(srcf): dstf.write(chunk) linktousercache(repo, hash) + def linktousercache(repo, hash): '''Link / copy the largefile with the specified hash from the store to the cache.''' path = usercachepath(repo.ui, hash) link(storepath(repo, hash), path) + def getstandinmatcher(repo, rmatcher=None): '''Return a match object that applies rmatcher to the standin directory''' wvfs = repo.wvfs @@ -303,18 +343,22 @@ match = scmutil.match(repo[None], [wvfs.join(standindir)], badfn=badfn) return match + def composestandinmatcher(repo, rmatcher): '''Return a matcher that accepts standins corresponding to the files accepted by rmatcher. Pass the list of files in the matcher as the paths specified by the user.''' smatcher = getstandinmatcher(repo, rmatcher) isstandin = smatcher.matchfn + def composedmatchfn(f): return isstandin(f) and rmatcher.matchfn(splitstandin(f)) + smatcher.matchfn = composedmatchfn return smatcher + def standin(filename): '''Return the repo-relative path to the standin for the specified big file.''' @@ -327,11 +371,13 @@ # passed filenames from an external source (like the command line). return shortnameslash + util.pconvert(filename) + def isstandin(filename): '''Return true if filename is a big file standin. filename must be in Mercurial's internal form (slash-separated).''' return filename.startswith(shortnameslash) + def splitstandin(filename): # Split on / because that's what dirstate always uses, even on Windows. # Change local separator to / first just in case we are passed filenames @@ -342,6 +388,7 @@ else: return None + def updatestandin(repo, lfile, standin): """Re-calculate hash value of lfile and write it into standin @@ -355,16 +402,19 @@ else: raise error.Abort(_('%s: file not found!') % lfile) + def readasstandin(fctx): '''read hex hash from given filectx of standin file This encapsulates how "standin" data is stored into storage layer.''' return fctx.data().strip() + def writestandin(repo, standin, hash, executable): '''write hash to <repo.root>/<standin>''' repo.wwrite(standin, hash + '\n', executable and 'x' or '') + def copyandhash(instream, outfile): '''Read bytes from instream (iterable) and write them to outfile, computing the SHA-1 hash of the data along the way. Return the hash.''' @@ -374,17 +424,22 @@ outfile.write(data) return hex(hasher.digest()) + def hashfile(file): if not os.path.exists(file): return '' with open(file, 'rb') as fd: return hexsha1(fd) + def getexecutable(filename): mode = os.stat(filename).st_mode - return ((mode & stat.S_IXUSR) and - (mode & stat.S_IXGRP) and - (mode & stat.S_IXOTH)) + return ( + (mode & stat.S_IXUSR) + and (mode & stat.S_IXGRP) + and (mode & stat.S_IXOTH) + ) + def urljoin(first, second, *arg): def join(left, right): @@ -399,6 +454,7 @@ url = join(url, a) return url + def hexsha1(fileobj): """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like object data""" @@ -407,31 +463,38 @@ h.update(chunk) return hex(h.digest()) + def httpsendfile(ui, filename): return httpconnection.httpsendfile(ui, filename, 'rb') + def unixpath(path): '''Return a version of path normalized for use with the lfdirstate.''' return util.pconvert(os.path.normpath(path)) + def islfilesrepo(repo): '''Return true if the repo is a largefile repo.''' - if ('largefiles' in repo.requirements and - any(shortnameslash in f[0] for f in repo.store.datafiles())): + if 'largefiles' in repo.requirements and any( + shortnameslash in f[0] for f in repo.store.datafiles() + ): return True return any(openlfdirstate(repo.ui, repo, False)) + class storeprotonotcapable(Exception): def __init__(self, storetypes): self.storetypes = storetypes + def getstandinsstate(repo): standins = [] matcher = getstandinmatcher(repo) wctx = repo[None] - for standin in repo.dirstate.walk(matcher, subrepos=[], unknown=False, - ignored=False): + for standin in repo.dirstate.walk( + matcher, subrepos=[], unknown=False, ignored=False + ): lfile = splitstandin(standin) try: hash = readasstandin(wctx[standin]) @@ -440,6 +503,7 @@ standins.append((lfile, hash)) return standins + def synclfdirstate(repo, lfdirstate, lfile, normallookup): lfstandin = standin(lfile) if lfstandin in repo.dirstate: @@ -448,8 +512,7 @@ else: state, mtime = '?', -1 if state == 'n': - if (normallookup or mtime < 0 or - not repo.wvfs.exists(lfile)): + if normallookup or mtime < 0 or not repo.wvfs.exists(lfile): # state 'n' doesn't ensure 'clean' in this case lfdirstate.normallookup(lfile) else: @@ -463,6 +526,7 @@ elif state == '?': lfdirstate.drop(lfile) + def markcommitted(orig, ctx, node): repo = ctx.repo() @@ -492,6 +556,7 @@ # at merging. copyalltostore(repo, node) + def getlfilestoupdate(oldstandins, newstandins): changedstandins = set(oldstandins).symmetric_difference(set(newstandins)) filelist = [] @@ -500,10 +565,14 @@ filelist.append(f[0]) return filelist + def getlfilestoupload(repo, missing, addfunc): makeprogress = repo.ui.makeprogress - with makeprogress(_('finding outgoing largefiles'), - unit=_('revisions'), total=len(missing)) as progress: + with makeprogress( + _('finding outgoing largefiles'), + unit=_('revisions'), + total=len(missing), + ) as progress: for i, n in enumerate(missing): progress.update(i) parents = [p for p in repo[n].parents() if p != node.nullid] @@ -533,6 +602,7 @@ if isstandin(fn) and fn in ctx: addfunc(fn, readasstandin(ctx[fn])) + def updatestandinsbymatch(repo, match): '''Update standins in the working directory according to specified match @@ -553,8 +623,9 @@ # large. lfdirstate = openlfdirstate(ui, repo) dirtymatch = matchmod.always() - unsure, s = lfdirstate.status(dirtymatch, subrepos=[], ignored=False, - clean=False, unknown=False) + unsure, s = lfdirstate.status( + dirtymatch, subrepos=[], ignored=False, clean=False, unknown=False + ) modifiedfiles = unsure + s.modified + s.added + s.removed lfiles = listlfiles(repo) # this only loops through largefiles that exist (not @@ -577,8 +648,9 @@ # Case 2: user calls commit with specified patterns: refresh # any matching big files. smatcher = composestandinmatcher(repo, match) - standins = repo.dirstate.walk(smatcher, subrepos=[], unknown=False, - ignored=False) + standins = repo.dirstate.walk( + smatcher, subrepos=[], unknown=False, ignored=False + ) # No matching big files: get out of the way and pass control to # the usual commit() method. @@ -636,6 +708,7 @@ return match + class automatedcommithook(object): '''Stateful hook to update standins at the 1st commit of resuming @@ -647,16 +720,18 @@ --continue``) should update them, because largefiles may be modified manually. ''' + def __init__(self, resuming): self.resuming = resuming def __call__(self, repo, match): if self.resuming: - self.resuming = False # avoids updating at subsequent commits + self.resuming = False # avoids updating at subsequent commits return updatestandinsbymatch(repo, match) else: return match + def getstatuswriter(ui, repo, forcibly=None): '''Return the function to write largefiles specific status out @@ -670,6 +745,6 @@ return repo._lfstatuswriters[-1] else: if forcibly: - return ui.status # forcibly WRITE OUT + return ui.status # forcibly WRITE OUT else: - return lambda *msg, **opts: None # forcibly IGNORE + return lambda *msg, **opts: None # forcibly IGNORE