Mercurial > public > mercurial-scm > hg-stable

comparison mercurial/upgrade_utils/engine.py @ 46050:f105c49e89cd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
upgrade: split actual upgrade code away from the main module The main module is getting big and hard to follow. So we are splitting all the logic to actually run an upgrade in a sub module. It nicely highlight that there are very few actual call point to the code we just moved. Differential Revision: https://phab.mercurial-scm.org/D9476
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Tue, 01 Dec 2020 09:13:08 +0100
parents mercurial/upgrade.py@6c960b708ac4
children c407513a44a3
comparison
equal deleted inserted replaced
46049:7905899c4f8f 46050:f105c49e89cd
1 # upgrade.py - functions for in place upgrade of Mercurial repository
2 #
3 # Copyright (c) 2016-present, Gregory Szorc
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8 from __future__ import absolute_import
9
10 import stat
11
12 from ..i18n import _
13 from ..pycompat import getattr
14 from .. import (
15 changelog,
16 error,
17 filelog,
18 manifest,
19 metadata,
20 pycompat,
21 requirements,
22 revlog,
23 scmutil,
24 util,
25 vfs as vfsmod,
26 )
27
28
29 def _revlogfrompath(repo, path):
30 """Obtain a revlog from a repo path.
31
32 An instance of the appropriate class is returned.
33 """
34 if path == b'00changelog.i':
35 return changelog.changelog(repo.svfs)
36 elif path.endswith(b'00manifest.i'):
37 mandir = path[: -len(b'00manifest.i')]
38 return manifest.manifestrevlog(repo.svfs, tree=mandir)
39 else:
40 # reverse of "/".join(("data", path + ".i"))
41 return filelog.filelog(repo.svfs, path[5:-2])
42
43
44 def _copyrevlog(tr, destrepo, oldrl, unencodedname):
45 """copy all relevant files for `oldrl` into `destrepo` store
46
47 Files are copied "as is" without any transformation. The copy is performed
48 without extra checks. Callers are responsible for making sure the copied
49 content is compatible with format of the destination repository.
50 """
51 oldrl = getattr(oldrl, '_revlog', oldrl)
52 newrl = _revlogfrompath(destrepo, unencodedname)
53 newrl = getattr(newrl, '_revlog', newrl)
54
55 oldvfs = oldrl.opener
56 newvfs = newrl.opener
57 oldindex = oldvfs.join(oldrl.indexfile)
58 newindex = newvfs.join(newrl.indexfile)
59 olddata = oldvfs.join(oldrl.datafile)
60 newdata = newvfs.join(newrl.datafile)
61
62 with newvfs(newrl.indexfile, b'w'):
63 pass # create all the directories
64
65 util.copyfile(oldindex, newindex)
66 copydata = oldrl.opener.exists(oldrl.datafile)
67 if copydata:
68 util.copyfile(olddata, newdata)
69
70 if not (
71 unencodedname.endswith(b'00changelog.i')
72 or unencodedname.endswith(b'00manifest.i')
73 ):
74 destrepo.svfs.fncache.add(unencodedname)
75 if copydata:
76 destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')
77
78
79 UPGRADE_CHANGELOG = b"changelog"
80 UPGRADE_MANIFEST = b"manifest"
81 UPGRADE_FILELOGS = b"all-filelogs"
82
83 UPGRADE_ALL_REVLOGS = frozenset(
84 [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS]
85 )
86
87
88 def getsidedatacompanion(srcrepo, dstrepo):
89 sidedatacompanion = None
90 removedreqs = srcrepo.requirements - dstrepo.requirements
91 addedreqs = dstrepo.requirements - srcrepo.requirements
92 if requirements.SIDEDATA_REQUIREMENT in removedreqs:
93
94 def sidedatacompanion(rl, rev):
95 rl = getattr(rl, '_revlog', rl)
96 if rl.flags(rev) & revlog.REVIDX_SIDEDATA:
97 return True, (), {}, 0, 0
98 return False, (), {}, 0, 0
99
100 elif requirements.COPIESSDC_REQUIREMENT in addedreqs:
101 sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo)
102 elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
103 sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
104 return sidedatacompanion
105
106
107 def matchrevlog(revlogfilter, entry):
108 """check if a revlog is selected for cloning.
109
110 In other words, are there any updates which need to be done on revlog
111 or it can be blindly copied.
112
113 The store entry is checked against the passed filter"""
114 if entry.endswith(b'00changelog.i'):
115 return UPGRADE_CHANGELOG in revlogfilter
116 elif entry.endswith(b'00manifest.i'):
117 return UPGRADE_MANIFEST in revlogfilter
118 return UPGRADE_FILELOGS in revlogfilter
119
120
121 def _clonerevlogs(
122 ui,
123 srcrepo,
124 dstrepo,
125 tr,
126 deltareuse,
127 forcedeltabothparents,
128 revlogs=UPGRADE_ALL_REVLOGS,
129 ):
130 """Copy revlogs between 2 repos."""
131 revcount = 0
132 srcsize = 0
133 srcrawsize = 0
134 dstsize = 0
135 fcount = 0
136 frevcount = 0
137 fsrcsize = 0
138 frawsize = 0
139 fdstsize = 0
140 mcount = 0
141 mrevcount = 0
142 msrcsize = 0
143 mrawsize = 0
144 mdstsize = 0
145 crevcount = 0
146 csrcsize = 0
147 crawsize = 0
148 cdstsize = 0
149
150 alldatafiles = list(srcrepo.store.walk())
151
152 # Perform a pass to collect metadata. This validates we can open all
153 # source files and allows a unified progress bar to be displayed.
154 for unencoded, encoded, size in alldatafiles:
155 if unencoded.endswith(b'.d'):
156 continue
157
158 rl = _revlogfrompath(srcrepo, unencoded)
159
160 info = rl.storageinfo(
161 exclusivefiles=True,
162 revisionscount=True,
163 trackedsize=True,
164 storedsize=True,
165 )
166
167 revcount += info[b'revisionscount'] or 0
168 datasize = info[b'storedsize'] or 0
169 rawsize = info[b'trackedsize'] or 0
170
171 srcsize += datasize
172 srcrawsize += rawsize
173
174 # This is for the separate progress bars.
175 if isinstance(rl, changelog.changelog):
176 crevcount += len(rl)
177 csrcsize += datasize
178 crawsize += rawsize
179 elif isinstance(rl, manifest.manifestrevlog):
180 mcount += 1
181 mrevcount += len(rl)
182 msrcsize += datasize
183 mrawsize += rawsize
184 elif isinstance(rl, filelog.filelog):
185 fcount += 1
186 frevcount += len(rl)
187 fsrcsize += datasize
188 frawsize += rawsize
189 else:
190 error.ProgrammingError(b'unknown revlog type')
191
192 if not revcount:
193 return
194
195 ui.status(
196 _(
197 b'migrating %d total revisions (%d in filelogs, %d in manifests, '
198 b'%d in changelog)\n'
199 )
200 % (revcount, frevcount, mrevcount, crevcount)
201 )
202 ui.status(
203 _(b'migrating %s in store; %s tracked data\n')
204 % ((util.bytecount(srcsize), util.bytecount(srcrawsize)))
205 )
206
207 # Used to keep track of progress.
208 progress = None
209
210 def oncopiedrevision(rl, rev, node):
211 progress.increment()
212
213 sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
214
215 # Do the actual copying.
216 # FUTURE this operation can be farmed off to worker processes.
217 seen = set()
218 for unencoded, encoded, size in alldatafiles:
219 if unencoded.endswith(b'.d'):
220 continue
221
222 oldrl = _revlogfrompath(srcrepo, unencoded)
223
224 if isinstance(oldrl, changelog.changelog) and b'c' not in seen:
225 ui.status(
226 _(
227 b'finished migrating %d manifest revisions across %d '
228 b'manifests; change in size: %s\n'
229 )
230 % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize))
231 )
232
233 ui.status(
234 _(
235 b'migrating changelog containing %d revisions '
236 b'(%s in store; %s tracked data)\n'
237 )
238 % (
239 crevcount,
240 util.bytecount(csrcsize),
241 util.bytecount(crawsize),
242 )
243 )
244 seen.add(b'c')
245 progress = srcrepo.ui.makeprogress(
246 _(b'changelog revisions'), total=crevcount
247 )
248 elif isinstance(oldrl, manifest.manifestrevlog) and b'm' not in seen:
249 ui.status(
250 _(
251 b'finished migrating %d filelog revisions across %d '
252 b'filelogs; change in size: %s\n'
253 )
254 % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize))
255 )
256
257 ui.status(
258 _(
259 b'migrating %d manifests containing %d revisions '
260 b'(%s in store; %s tracked data)\n'
261 )
262 % (
263 mcount,
264 mrevcount,
265 util.bytecount(msrcsize),
266 util.bytecount(mrawsize),
267 )
268 )
269 seen.add(b'm')
270 if progress:
271 progress.complete()
272 progress = srcrepo.ui.makeprogress(
273 _(b'manifest revisions'), total=mrevcount
274 )
275 elif b'f' not in seen:
276 ui.status(
277 _(
278 b'migrating %d filelogs containing %d revisions '
279 b'(%s in store; %s tracked data)\n'
280 )
281 % (
282 fcount,
283 frevcount,
284 util.bytecount(fsrcsize),
285 util.bytecount(frawsize),
286 )
287 )
288 seen.add(b'f')
289 if progress:
290 progress.complete()
291 progress = srcrepo.ui.makeprogress(
292 _(b'file revisions'), total=frevcount
293 )
294
295 if matchrevlog(revlogs, unencoded):
296 ui.note(
297 _(b'cloning %d revisions from %s\n') % (len(oldrl), unencoded)
298 )
299 newrl = _revlogfrompath(dstrepo, unencoded)
300 oldrl.clone(
301 tr,
302 newrl,
303 addrevisioncb=oncopiedrevision,
304 deltareuse=deltareuse,
305 forcedeltabothparents=forcedeltabothparents,
306 sidedatacompanion=sidedatacompanion,
307 )
308 else:
309 msg = _(b'blindly copying %s containing %i revisions\n')
310 ui.note(msg % (unencoded, len(oldrl)))
311 _copyrevlog(tr, dstrepo, oldrl, unencoded)
312
313 newrl = _revlogfrompath(dstrepo, unencoded)
314
315 info = newrl.storageinfo(storedsize=True)
316 datasize = info[b'storedsize'] or 0
317
318 dstsize += datasize
319
320 if isinstance(newrl, changelog.changelog):
321 cdstsize += datasize
322 elif isinstance(newrl, manifest.manifestrevlog):
323 mdstsize += datasize
324 else:
325 fdstsize += datasize
326
327 progress.complete()
328
329 ui.status(
330 _(
331 b'finished migrating %d changelog revisions; change in size: '
332 b'%s\n'
333 )
334 % (crevcount, util.bytecount(cdstsize - csrcsize))
335 )
336
337 ui.status(
338 _(
339 b'finished migrating %d total revisions; total change in store '
340 b'size: %s\n'
341 )
342 % (revcount, util.bytecount(dstsize - srcsize))
343 )
344
345
346 def _filterstorefile(srcrepo, dstrepo, requirements, path, mode, st):
347 """Determine whether to copy a store file during upgrade.
348
349 This function is called when migrating store files from ``srcrepo`` to
350 ``dstrepo`` as part of upgrading a repository.
351
352 Args:
353 srcrepo: repo we are copying from
354 dstrepo: repo we are copying to
355 requirements: set of requirements for ``dstrepo``
356 path: store file being examined
357 mode: the ``ST_MODE`` file type of ``path``
358 st: ``stat`` data structure for ``path``
359
360 Function should return ``True`` if the file is to be copied.
361 """
362 # Skip revlogs.
363 if path.endswith((b'.i', b'.d', b'.n', b'.nd')):
364 return False
365 # Skip transaction related files.
366 if path.startswith(b'undo'):
367 return False
368 # Only copy regular files.
369 if mode != stat.S_IFREG:
370 return False
371 # Skip other skipped files.
372 if path in (b'lock', b'fncache'):
373 return False
374
375 return True
376
377
378 def _finishdatamigration(ui, srcrepo, dstrepo, requirements):
379 """Hook point for extensions to perform additional actions during upgrade.
380
381 This function is called after revlogs and store files have been copied but
382 before the new store is swapped into the original location.
383 """
384
385
386 def upgrade(
387 ui, srcrepo, dstrepo, requirements, actions, revlogs=UPGRADE_ALL_REVLOGS
388 ):
389 """Do the low-level work of upgrading a repository.
390
391 The upgrade is effectively performed as a copy between a source
392 repository and a temporary destination repository.
393
394 The source repository is unmodified for as long as possible so the
395 upgrade can abort at any time without causing loss of service for
396 readers and without corrupting the source repository.
397 """
398 assert srcrepo.currentwlock()
399 assert dstrepo.currentwlock()
400
401 ui.status(
402 _(
403 b'(it is safe to interrupt this process any time before '
404 b'data migration completes)\n'
405 )
406 )
407
408 if b're-delta-all' in actions:
409 deltareuse = revlog.revlog.DELTAREUSENEVER
410 elif b're-delta-parent' in actions:
411 deltareuse = revlog.revlog.DELTAREUSESAMEREVS
412 elif b're-delta-multibase' in actions:
413 deltareuse = revlog.revlog.DELTAREUSESAMEREVS
414 elif b're-delta-fulladd' in actions:
415 deltareuse = revlog.revlog.DELTAREUSEFULLADD
416 else:
417 deltareuse = revlog.revlog.DELTAREUSEALWAYS
418
419 with dstrepo.transaction(b'upgrade') as tr:
420 _clonerevlogs(
421 ui,
422 srcrepo,
423 dstrepo,
424 tr,
425 deltareuse,
426 b're-delta-multibase' in actions,
427 revlogs=revlogs,
428 )
429
430 # Now copy other files in the store directory.
431 # The sorted() makes execution deterministic.
432 for p, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)):
433 if not _filterstorefile(srcrepo, dstrepo, requirements, p, kind, st):
434 continue
435
436 srcrepo.ui.status(_(b'copying %s\n') % p)
437 src = srcrepo.store.rawvfs.join(p)
438 dst = dstrepo.store.rawvfs.join(p)
439 util.copyfile(src, dst, copystat=True)
440
441 _finishdatamigration(ui, srcrepo, dstrepo, requirements)
442
443 ui.status(_(b'data fully migrated to temporary repository\n'))
444
445 backuppath = pycompat.mkdtemp(prefix=b'upgradebackup.', dir=srcrepo.path)
446 backupvfs = vfsmod.vfs(backuppath)
447
448 # Make a backup of requires file first, as it is the first to be modified.
449 util.copyfile(srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires'))
450
451 # We install an arbitrary requirement that clients must not support
452 # as a mechanism to lock out new clients during the data swap. This is
453 # better than allowing a client to continue while the repository is in
454 # an inconsistent state.
455 ui.status(
456 _(
457 b'marking source repository as being upgraded; clients will be '
458 b'unable to read from repository\n'
459 )
460 )
461 scmutil.writereporequirements(
462 srcrepo, srcrepo.requirements | {b'upgradeinprogress'}
463 )
464
465 ui.status(_(b'starting in-place swap of repository data\n'))
466 ui.status(_(b'replaced files will be backed up at %s\n') % backuppath)
467
468 # Now swap in the new store directory. Doing it as a rename should make
469 # the operation nearly instantaneous and atomic (at least in well-behaved
470 # environments).
471 ui.status(_(b'replacing store...\n'))
472 tstart = util.timer()
473 util.rename(srcrepo.spath, backupvfs.join(b'store'))
474 util.rename(dstrepo.spath, srcrepo.spath)
475 elapsed = util.timer() - tstart
476 ui.status(
477 _(
478 b'store replacement complete; repository was inconsistent for '
479 b'%0.1fs\n'
480 )
481 % elapsed
482 )
483
484 # We first write the requirements file. Any new requirements will lock
485 # out legacy clients.
486 ui.status(
487 _(
488 b'finalizing requirements file and making repository readable '
489 b'again\n'
490 )
491 )
492 scmutil.writereporequirements(srcrepo, requirements)
493
494 # The lock file from the old store won't be removed because nothing has a
495 # reference to its new location. So clean it up manually. Alternatively, we
496 # could update srcrepo.svfs and other variables to point to the new
497 # location. This is simpler.
498 backupvfs.unlink(b'store/lock')
499
500 return backuppath