comparison mercurial/changegroup.py @ 35012:d80380ba8e7d

changegroup: use any node, not min(), in treemanifest's generatemanifests This is fixing quadratic behavior, which is probably not noticeable in the common case, but if a very large directory gets added here, it can get pretty bad. This was noticed because we had some pushes that spent >25s in changegroup generation calling min() here, according to profiling. The original reasoning for min() being used in 829d369fc5a8 was that, at that point in the series, we were adding almost everything to tmfnodes during the first iteration through the loop , so we needed to avoid sending child directories before parents. Later changes made it so that the child directories were added only when we visited the parent directory (not all of them on the first iteration), so this is no longer necessary - there won't be any child directories in tmfnodes before the parents have been sent. This does mean that the manifests are now exchanged unordered, whereas previously we would essentially do [a, b, b/c, b/c/d, e], we now can send a, b, and e in any order; b/c must still follow b, and b/c/d must still follow b/c. Differential Revision: https://phab.mercurial-scm.org/D1351
author Kyle Lippincott <spectral@google.com>
date Wed, 08 Nov 2017 18:24:43 -0800
parents 3572b2031cec
children fb0be099063f
comparison
equal deleted inserted replaced
35011:a2dfc723b6b5 35012:d80380ba8e7d
690 tmfnodes = {'': mfs} 690 tmfnodes = {'': mfs}
691 691
692 # Callback for the manifest, used to collect linkrevs for filelog 692 # Callback for the manifest, used to collect linkrevs for filelog
693 # revisions. 693 # revisions.
694 # Returns the linkrev node (collected in lookupcl). 694 # Returns the linkrev node (collected in lookupcl).
695 def makelookupmflinknode(dir): 695 def makelookupmflinknode(dir, nodes):
696 if fastpathlinkrev: 696 if fastpathlinkrev:
697 assert not dir 697 assert not dir
698 return mfs.__getitem__ 698 return mfs.__getitem__
699 699
700 def lookupmflinknode(x): 700 def lookupmflinknode(x):
711 711
712 Note that this means manifests must be completely sent to 712 Note that this means manifests must be completely sent to
713 the client before you can trust the list of files and 713 the client before you can trust the list of files and
714 treemanifests to send. 714 treemanifests to send.
715 """ 715 """
716 clnode = tmfnodes[dir][x] 716 clnode = nodes[x]
717 mdata = mfl.get(dir, x).readfast(shallow=True) 717 mdata = mfl.get(dir, x).readfast(shallow=True)
718 for p, n, fl in mdata.iterentries(): 718 for p, n, fl in mdata.iterentries():
719 if fl == 't': # subdirectory manifest 719 if fl == 't': # subdirectory manifest
720 subdir = dir + p + '/' 720 subdir = dir + p + '/'
721 tmfclnodes = tmfnodes.setdefault(subdir, {}) 721 tmfclnodes = tmfnodes.setdefault(subdir, {})
731 return clnode 731 return clnode
732 return lookupmflinknode 732 return lookupmflinknode
733 733
734 size = 0 734 size = 0
735 while tmfnodes: 735 while tmfnodes:
736 dir = min(tmfnodes) 736 dir, nodes = tmfnodes.popitem()
737 nodes = tmfnodes[dir]
738 prunednodes = self.prune(dirlog(dir), nodes, commonrevs) 737 prunednodes = self.prune(dirlog(dir), nodes, commonrevs)
739 if not dir or prunednodes: 738 if not dir or prunednodes:
740 for x in self._packmanifests(dir, prunednodes, 739 for x in self._packmanifests(dir, prunednodes,
741 makelookupmflinknode(dir)): 740 makelookupmflinknode(dir, nodes)):
742 size += len(x) 741 size += len(x)
743 yield x 742 yield x
744 del tmfnodes[dir]
745 self._verbosenote(_('%8.i (manifests)\n') % size) 743 self._verbosenote(_('%8.i (manifests)\n') % size)
746 yield self._manifestsdone() 744 yield self._manifestsdone()
747 745
748 # The 'source' parameter is useful for extensions 746 # The 'source' parameter is useful for extensions
749 def generatefiles(self, changedfiles, linknodes, commonrevs, source): 747 def generatefiles(self, changedfiles, linknodes, commonrevs, source):