Mercurial > public > mercurial-scm > hg
comparison mercurial/manifest.py @ 41153:2c3f69855ce8
manifest: convert a recursive function to iterative one using stacks
I am debugging a memory issue from yesterday where `hg update` goes upto taking
22GB of memory on our internal treemanifest repository. This is an interesting
function and I saw memory consumption increasing while this function was
running.
It's sometimes hard to understand a recursive function and also the profile
won't show you actual operations which took time, rather it will show you the
function again and again in profile.
I am yet to notice any memory consumption decrease with this patch, but I
believe this will help like in making this a generator.
Differential Revision: https://phab.mercurial-scm.org/D5413
| author | Pulkit Goyal <pulkit@yandex-team.ru> |
|---|---|
| date | Wed, 12 Dec 2018 16:26:58 +0300 |
| parents | 3913223417ea |
| children | 876494fd967d |
comparison
equal
deleted
inserted
replaced
| 41152:191fac9ff9d3 | 41153:2c3f69855ce8 |
|---|---|
| 1133 m1 = self.matches(match) | 1133 m1 = self.matches(match) |
| 1134 m2 = m2.matches(match) | 1134 m2 = m2.matches(match) |
| 1135 return m1.diff(m2, clean=clean) | 1135 return m1.diff(m2, clean=clean) |
| 1136 result = {} | 1136 result = {} |
| 1137 emptytree = treemanifest() | 1137 emptytree = treemanifest() |
| 1138 def _diff(t1, t2): | 1138 |
| 1139 def _iterativediff(t1, t2, stack): | |
| 1140 """compares two tree manifests and append new tree-manifests which | |
| 1141 needs to be compared to stack""" | |
| 1139 if t1._node == t2._node and not t1._dirty and not t2._dirty: | 1142 if t1._node == t2._node and not t1._dirty and not t2._dirty: |
| 1140 return | 1143 return |
| 1141 t1._load() | 1144 t1._load() |
| 1142 t2._load() | 1145 t2._load() |
| 1143 self._loaddifflazy(t1, t2) | 1146 self._loaddifflazy(t1, t2) |
| 1144 | 1147 |
| 1145 for d, m1 in t1._dirs.iteritems(): | 1148 for d, m1 in t1._dirs.iteritems(): |
| 1146 m2 = t2._dirs.get(d, emptytree) | 1149 m2 = t2._dirs.get(d, emptytree) |
| 1147 _diff(m1, m2) | 1150 stack.append((m1, m2)) |
| 1148 | 1151 |
| 1149 for d, m2 in t2._dirs.iteritems(): | 1152 for d, m2 in t2._dirs.iteritems(): |
| 1150 if d not in t1._dirs: | 1153 if d not in t1._dirs: |
| 1151 _diff(emptytree, m2) | 1154 stack.append((emptytree, m2)) |
| 1152 | 1155 |
| 1153 for fn, n1 in t1._files.iteritems(): | 1156 for fn, n1 in t1._files.iteritems(): |
| 1154 fl1 = t1._flags.get(fn, '') | 1157 fl1 = t1._flags.get(fn, '') |
| 1155 n2 = t2._files.get(fn, None) | 1158 n2 = t2._files.get(fn, None) |
| 1156 fl2 = t2._flags.get(fn, '') | 1159 fl2 = t2._flags.get(fn, '') |
| 1162 for fn, n2 in t2._files.iteritems(): | 1165 for fn, n2 in t2._files.iteritems(): |
| 1163 if fn not in t1._files: | 1166 if fn not in t1._files: |
| 1164 fl2 = t2._flags.get(fn, '') | 1167 fl2 = t2._flags.get(fn, '') |
| 1165 result[t2._subpath(fn)] = ((None, ''), (n2, fl2)) | 1168 result[t2._subpath(fn)] = ((None, ''), (n2, fl2)) |
| 1166 | 1169 |
| 1167 _diff(self, m2) | 1170 stackls = [] |
| 1171 _iterativediff(self, m2, stackls) | |
| 1172 while stackls: | |
| 1173 t1, t2 = stackls.pop() | |
| 1174 # stackls is populated in the function call | |
| 1175 _iterativediff(t1, t2, stackls) | |
| 1168 return result | 1176 return result |
| 1169 | 1177 |
| 1170 def unmodifiedsince(self, m2): | 1178 def unmodifiedsince(self, m2): |
| 1171 return not self._dirty and not m2._dirty and self._node == m2._node | 1179 return not self._dirty and not m2._dirty and self._node == m2._node |
| 1172 | 1180 |