Mercurial > public > mercurial-scm > hg-stable

comparison mercurial/upgrade_utils/engine.py @ 49544:1994842955db stable

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
upgrade: no longer keep all revlogs in memory at any point Keeping all object open is unsustainable, so we will open them on demand. This mean opening them multiple times, but this is a lesser evil. Each revlog consume a small amount of memory (index content, associated nodemap, etc). While there are few "big" revlog, the sheer amount of small filelog can become a significant issue memory wise, consuming multiple GB of memory. If you combines this extra usage with the use of multiprocessing, this usage can quickly get out of control. This can effectively block the upgrade of larger repository. This changeset fixes this issue.
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Fri, 04 Nov 2022 16:15:12 -0400
parents 2e726c934fcd
children b4b1791f36e4
comparison
equal deleted inserted replaced
49543:5f22c92dcf3d 49544:1994842955db
231 srcsize += datasize 231 srcsize += datasize
232 srcrawsize += rawsize 232 srcrawsize += rawsize
233 233
234 # This is for the separate progress bars. 234 # This is for the separate progress bars.
235 if rl_type & store.FILEFLAGS_CHANGELOG: 235 if rl_type & store.FILEFLAGS_CHANGELOG:
236 changelogs[unencoded] = (rl_type, rl) 236 changelogs[unencoded] = rl_type
237 crevcount += len(rl) 237 crevcount += len(rl)
238 csrcsize += datasize 238 csrcsize += datasize
239 crawsize += rawsize 239 crawsize += rawsize
240 elif rl_type & store.FILEFLAGS_MANIFESTLOG: 240 elif rl_type & store.FILEFLAGS_MANIFESTLOG:
241 manifests[unencoded] = (rl_type, rl) 241 manifests[unencoded] = rl_type
242 mcount += 1 242 mcount += 1
243 mrevcount += len(rl) 243 mrevcount += len(rl)
244 msrcsize += datasize 244 msrcsize += datasize
245 mrawsize += rawsize 245 mrawsize += rawsize
246 elif rl_type & store.FILEFLAGS_FILELOG: 246 elif rl_type & store.FILEFLAGS_FILELOG:
247 filelogs[unencoded] = (rl_type, rl) 247 filelogs[unencoded] = rl_type
248 fcount += 1 248 fcount += 1
249 frevcount += len(rl) 249 frevcount += len(rl)
250 fsrcsize += datasize 250 fsrcsize += datasize
251 frawsize += rawsize 251 frawsize += rawsize
252 else: 252 else:
287 util.bytecount(fsrcsize), 287 util.bytecount(fsrcsize),
288 util.bytecount(frawsize), 288 util.bytecount(frawsize),
289 ) 289 )
290 ) 290 )
291 progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount) 291 progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
292 for unencoded, (rl_type, oldrl) in sorted(filelogs.items()): 292 for unencoded, rl_type in sorted(filelogs.items()):
293 oldrl = _revlogfrompath(srcrepo, rl_type, unencoded)
294
293 newrl = _perform_clone( 295 newrl = _perform_clone(
294 ui, 296 ui,
295 dstrepo, 297 dstrepo,
296 tr, 298 tr,
297 oldrl, 299 oldrl,
327 if progress: 329 if progress:
328 progress.complete() 330 progress.complete()
329 progress = srcrepo.ui.makeprogress( 331 progress = srcrepo.ui.makeprogress(
330 _(b'manifest revisions'), total=mrevcount 332 _(b'manifest revisions'), total=mrevcount
331 ) 333 )
332 for unencoded, (rl_type, oldrl) in sorted(manifests.items()): 334 for unencoded, rl_type in sorted(manifests.items()):
335 oldrl = _revlogfrompath(srcrepo, rl_type, unencoded)
333 newrl = _perform_clone( 336 newrl = _perform_clone(
334 ui, 337 ui,
335 dstrepo, 338 dstrepo,
336 tr, 339 tr,
337 oldrl, 340 oldrl,
366 if progress: 369 if progress:
367 progress.complete() 370 progress.complete()
368 progress = srcrepo.ui.makeprogress( 371 progress = srcrepo.ui.makeprogress(
369 _(b'changelog revisions'), total=crevcount 372 _(b'changelog revisions'), total=crevcount
370 ) 373 )
371 for unencoded, (rl_type, oldrl) in sorted(changelogs.items()): 374 for unencoded, rl_type in sorted(changelogs.items()):
375 oldrl = _revlogfrompath(srcrepo, rl_type, unencoded)
372 newrl = _perform_clone( 376 newrl = _perform_clone(
373 ui, 377 ui,
374 dstrepo, 378 dstrepo,
375 tr, 379 tr,
376 oldrl, 380 oldrl,