mercurial-scm/hg-stable: mercurial/cmdutil.py comparison

comparison mercurial/cmdutil.py @ 34682:7e3001b74ab3

tersestatus: re-implement the functionality to terse the status The previous terse status implementation was hacking around os.listdir() and was flaky. There have been a lot of instances of mercurial buildbots failing and google's internal builds failing because of the hacky implementation of terse status. Even though I wrote the last implementation but it was hard for me to find the reason for the flake. The new implementation can be slower than the old one but is clean and easy to understand. In this we create a node object for each directory and create a tree like structure starting from the root of the working copy. While building the tree like structure we store some information on the nodes which will be helpful for deciding later whether we can terse the dir or not. Once the whole tree is build we traverse and built the list of files for each status with required tersing. There is no behaviour change as the old test, test-status-terse.t passes with the new implementation. Differential Revision: https://phab.mercurial-scm.org/D985

author	Pulkit Goyal <7895pulkit@gmail.com>
date	Fri, 06 Oct 2017 20:54:23 +0530
parents	2a360445afa0
children	3d6d4b12128e

comparison

equal deleted inserted replaced

-:4dc8a2ee0f4f
+:7e3001b74ab3
 with repo.wlock():
 return recordfunc(ui, repo, message, match, opts)
 return commit(ui, repo, recordinwlock, pats, opts)
-def tersestatus(root, statlist, status, ignorefn, ignore):
+# extracted at module level as it's required each time a file will be added
+# to dirnode class object below
+pathsep = pycompat.ossep
+class dirnode(object):
 """
-Returns a list of statuses with directory collapsed if all the files in the
+represents a directory in user working copy
-directory has the same status.
+stores information which is required for purpose of tersing the status
+path is the path to the directory
+statuses is a set of statuses of all files in this directory (this includes
+all the files in all the subdirectories too)
+files is a list of files which are direct child of this directory
+subdirs is a dictionary of sub-directory name as the key and it's own
+dirnode object as the value
 """
-def numfiles(dirname):
+def __init__(self, dirpath):
+self.path = dirpath
+self.statuses = set([])
+self.files = []
+self.subdirs = {}
+def _addfileindir(self, filename, status):
+""" adds a file in this directory as the direct child """
+self.files.append((filename, status))
+def addfile(self, filename, status):
 """
-Calculates the number of tracked files in a given directory which also
+adds a file which is present in this directory to its direct parent
-includes files which were removed or deleted. Considers ignored files
+dirnode object
-if ignore argument is True or 'i' is present in status argument.
+if the file is not direct child of this directory, we traverse to the
+directory of which this file is a direct child of and add the file there
 """
-if lencache.get(dirname):
-return lencache[dirname]
+# the filename contains a path separator, it means it's not the direct
-if 'i' in status or ignore:
+# child of this directory
-def match(localpath):
+if pathsep in filename:
-absolutepath = os.path.join(root, localpath)
+subdir, filep = filename.split(pathsep, 1)
-if os.path.isdir(absolutepath) and isemptydir(absolutepath):
-return True
+# does the dirnode object for subdir exists
-return False
+if subdir not in self.subdirs:
+subdirpath = os.path.join(self.path, subdir)
+self.subdirs[subdir] = dirnode(subdirpath)
+# try adding the file in subdir
+self.subdirs[subdir].addfile(filep, status)
 else:
-def match(localpath):
+self._addfileindir(filename, status)
-# there can be directory whose all the files are ignored and
-# hence the drectory should also be ignored while counting
+if status not in self.statuses:
-# number of files or subdirs in it's parent directory. This
+self.statuses.add(status)
-# checks the same.
-# XXX: We need a better logic here.
+def _addfilestotersed(path, files, tersedict):
-if os.path.isdir(os.path.join(root, localpath)):
+""" adds files to the their respective status list in the final tersed list
-return isignoreddir(localpath)
-else:
+path is the path of parent directory of the file
-# XXX: there can be files which have the ignored pattern but
+files is a list of tuple where each tuple is (filename, status)
-# are not ignored. That leads to bug in counting number of
+tersedict is a dictonary which contains each status abbreviation as key and
-# tracked files in the directory.
+list of files and tersed dirs in that status as value
-return ignorefn(localpath)
+"""
-lendir = 0
+for f, st in files:
-abspath = os.path.join(root, dirname)
+tersedict[st].append(os.path.join(path, f))
-# There might be cases when a directory does not exists as the whole
-# directory can be removed and/or deleted.
+def _processtersestatus(subdir, tersedict, terseargs):
-try:
+"""a recursive function which process status for a certain directory.
-for f in os.listdir(abspath):
-localpath = os.path.join(dirname, f)
+subdir is an oject of dirnode class defined below. each object of dirnode
-if not match(localpath):
+class has a set of statuses which files in that directory has. This ease our
-lendir += 1
+check whether we can terse that directory or not.
-except OSError:
-pass
+tersedict is a dictonary which contains each status abbreviation as key and
-lendir += len(absentdir.get(dirname, []))
+list of files and tersed dirs in that status as value. In each function call
-lencache[dirname] = lendir
+we are passing the same dict and adding files and dirs to it.
-return lendir
+terseargs is the string of arguments passed by the user with `--terse` flag.
-def isemptydir(abspath):
-"""
+Following are the cases which can happen:
-Check whether a directory is empty or not, i.e. there is no files in the
-directory and all its subdirectories.
+1) All the files in the directory (including all the files in its
-"""
+subdirectories) share the same status and the user has asked us to terse
-for f in os.listdir(abspath):
+that status. -> we add the directory name to status list and return
-fullpath = os.path.join(abspath, f)
-if os.path.isdir(fullpath):
+2) If '1)' does not happen, we do following:
-# recursion here
-ret = isemptydir(fullpath)
+a) Add all the files which are in this directory (only the ones in
-if not ret:
+this directory, not the subdirs) to their respective status list
-return False
-else:
+b) Recurse the function on all the subdirectories of this directory
-return False
+"""
-return True
+if len(subdir.statuses) == 1:
-def isignoreddir(localpath):
+onlyst = subdir.statuses.pop()
-"""Return True if `localpath` directory is ignored or contains only
-ignored files and should hence be considered ignored.
+# Making sure we terse only when the status abbreviation is passed as
-"""
+# terse argument
-dirpath = os.path.join(root, localpath)
+if onlyst in terseargs:
-if ignorefn(dirpath):
+tersedict[onlyst].append(subdir.path + pycompat.ossep)
-return True
+return
-for f in os.listdir(dirpath):
-filepath = os.path.join(dirpath, f)
+# add the files to status list
-if os.path.isdir(filepath):
+_addfilestotersed(subdir.path, subdir.files, tersedict)
-# recursion here
-ret = isignoreddir(os.path.join(localpath, f))
+#recurse on the subdirs
-if not ret:
+for dirobj in subdir.subdirs.values():
-return False
+_processtersestatus(dirobj, tersedict, terseargs)
-else:
-if not ignorefn(os.path.join(localpath, f)):
+def tersedir(statuslist, terseargs):
-return False
+"""
-return True
+terses the status if all the files in a directory shares the same status
-def absentones(removedfiles, missingfiles):
+statuslist is scmutil.status() object which contains a list of files for
-"""
+each status.
-Returns a dictionary of directories with files in it which are either
+terseargs is string which is passed by the user as the argument to `--terse`
-removed or missing (deleted) in them.
+flag.
-"""
-absentdir = {}
+The function makes a tree of objects of dirnode class, and at each node it
-absentfiles = removedfiles + missingfiles
+stores the information required to know whether we can terse a certain
-while absentfiles:
+directory or not.
-f = absentfiles.pop()
-par = os.path.dirname(f)
+tersedict (defined in the function) is a dictionary which has one word key
-if par == '':
+for each status and a list of files and dir in that status as the respective
-continue
+value. The dictionary is passed to other helper functions which builds it.
-# we need to store files rather than number of files as some files
+"""
-# or subdirectories in a directory can be counted twice. This is
+# the order matters here as that is used to produce final list
-# also we have used sets here.
+allst = ('m', 'a', 'r', 'd', 'u', 'i', 'c')
-try:
-absentdir[par].add(f)
+# checking the argument validity
-except KeyError:
+for s in terseargs:
-absentdir[par] = set([f])
+if s not in allst:
-absentfiles.append(par)
+raise error.Abort(_("'%s' not recognized") % s)
-return absentdir
+# creating a dirnode object for the root of the repo
-indexes = {'m': 0, 'a': 1, 'r': 2, 'd': 3, 'u': 4, 'i': 5, 'c': 6}
+rootobj = dirnode('')
-# get a dictonary of directories and files which are missing as os.listdir()
+pstatus = ('modified', 'added', 'deleted', 'clean', 'unknown',
-# won't be able to list them.
+'ignored', 'removed')
-absentdir = absentones(statlist[2], statlist[3])
-finalrs = [[]] * len(indexes)
+tersedict = {}
-didsomethingchanged = False
+for attrname in pstatus:
-# dictionary to store number of files and subdir in a directory so that we
+for f in getattr(statuslist, attrname):
-# don't compute that again.
+rootobj.addfile(f, attrname[0])
-lencache = {}
+tersedict[attrname[0]] = []
-for st in pycompat.bytestr(status):
+# we won't be tersing the root dir, so add files in it
+_addfilestotersed(rootobj.path, rootobj.files, tersedict)
-try:
-ind = indexes[st]
+# process each sub-directory and build tersedict
-except KeyError:
+for subdir in rootobj.subdirs.values():
-# TODO: Need a better error message here
+_processtersestatus(subdir, tersedict, terseargs)
-raise error.Abort("'%s' not recognized" % st)
+tersedlist = []
-sfiles = statlist[ind]
+for st in allst:
-if not sfiles:
+tersedict[st].sort()
-continue
+tersedlist.append(tersedict[st])
-pardict = {}
-for a in sfiles:
+return tersedlist
-par = os.path.dirname(a)
-pardict.setdefault(par, []).append(a)
-rs = []
-newls = []
-for par, files in sorted(pardict.iteritems()):
-lenpar = numfiles(par)
-if lenpar == len(files):
-newls.append(par)
-if not newls:
-continue
-while newls:
-newel = newls.pop()
-if newel == '':
-continue
-parn = os.path.dirname(newel)
-pardict[newel] = []
-# Adding pycompat.ossep as newel is a directory.
-pardict.setdefault(parn, []).append(newel + pycompat.ossep)
-lenpar = numfiles(parn)
-if lenpar == len(pardict[parn]):
-newls.append(parn)
-# dict.values() for Py3 compatibility
-for files in pardict.values():
-rs.extend(files)
-rs.sort()
-finalrs[ind] = rs
-didsomethingchanged = True
-# If nothing is changed, make sure the order of files is preserved.
-if not didsomethingchanged:
-return statlist
-for x in xrange(len(indexes)):
-if not finalrs[x]:
-finalrs[x] = statlist[x]
-return finalrs
 def _commentlines(raw):
 '''Surround lineswith a comment char and a new line'''
 lines = raw.splitlines()
 commentedlines = ['# %s' % line for line in lines]

Mercurial > public > mercurial-scm > hg-stable

comparison mercurial/cmdutil.py @ 34682:7e3001b74ab3