|
1 # upgrade.py - functions for in place upgrade of Mercurial repository |
|
2 # |
|
3 # Copyright (c) 2016-present, Gregory Szorc |
|
4 # |
|
5 # This software may be used and distributed according to the terms of the |
|
6 # GNU General Public License version 2 or any later version. |
|
7 |
|
8 from __future__ import absolute_import |
|
9 |
|
10 import stat |
|
11 |
|
12 from ..i18n import _ |
|
13 from ..pycompat import getattr |
|
14 from .. import ( |
|
15 changelog, |
|
16 error, |
|
17 filelog, |
|
18 manifest, |
|
19 metadata, |
|
20 pycompat, |
|
21 requirements, |
|
22 revlog, |
|
23 scmutil, |
|
24 util, |
|
25 vfs as vfsmod, |
|
26 ) |
|
27 |
|
28 |
|
29 def _revlogfrompath(repo, path): |
|
30 """Obtain a revlog from a repo path. |
|
31 |
|
32 An instance of the appropriate class is returned. |
|
33 """ |
|
34 if path == b'00changelog.i': |
|
35 return changelog.changelog(repo.svfs) |
|
36 elif path.endswith(b'00manifest.i'): |
|
37 mandir = path[: -len(b'00manifest.i')] |
|
38 return manifest.manifestrevlog(repo.svfs, tree=mandir) |
|
39 else: |
|
40 # reverse of "/".join(("data", path + ".i")) |
|
41 return filelog.filelog(repo.svfs, path[5:-2]) |
|
42 |
|
43 |
|
44 def _copyrevlog(tr, destrepo, oldrl, unencodedname): |
|
45 """copy all relevant files for `oldrl` into `destrepo` store |
|
46 |
|
47 Files are copied "as is" without any transformation. The copy is performed |
|
48 without extra checks. Callers are responsible for making sure the copied |
|
49 content is compatible with format of the destination repository. |
|
50 """ |
|
51 oldrl = getattr(oldrl, '_revlog', oldrl) |
|
52 newrl = _revlogfrompath(destrepo, unencodedname) |
|
53 newrl = getattr(newrl, '_revlog', newrl) |
|
54 |
|
55 oldvfs = oldrl.opener |
|
56 newvfs = newrl.opener |
|
57 oldindex = oldvfs.join(oldrl.indexfile) |
|
58 newindex = newvfs.join(newrl.indexfile) |
|
59 olddata = oldvfs.join(oldrl.datafile) |
|
60 newdata = newvfs.join(newrl.datafile) |
|
61 |
|
62 with newvfs(newrl.indexfile, b'w'): |
|
63 pass # create all the directories |
|
64 |
|
65 util.copyfile(oldindex, newindex) |
|
66 copydata = oldrl.opener.exists(oldrl.datafile) |
|
67 if copydata: |
|
68 util.copyfile(olddata, newdata) |
|
69 |
|
70 if not ( |
|
71 unencodedname.endswith(b'00changelog.i') |
|
72 or unencodedname.endswith(b'00manifest.i') |
|
73 ): |
|
74 destrepo.svfs.fncache.add(unencodedname) |
|
75 if copydata: |
|
76 destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d') |
|
77 |
|
78 |
|
79 UPGRADE_CHANGELOG = b"changelog" |
|
80 UPGRADE_MANIFEST = b"manifest" |
|
81 UPGRADE_FILELOGS = b"all-filelogs" |
|
82 |
|
83 UPGRADE_ALL_REVLOGS = frozenset( |
|
84 [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS] |
|
85 ) |
|
86 |
|
87 |
|
88 def getsidedatacompanion(srcrepo, dstrepo): |
|
89 sidedatacompanion = None |
|
90 removedreqs = srcrepo.requirements - dstrepo.requirements |
|
91 addedreqs = dstrepo.requirements - srcrepo.requirements |
|
92 if requirements.SIDEDATA_REQUIREMENT in removedreqs: |
|
93 |
|
94 def sidedatacompanion(rl, rev): |
|
95 rl = getattr(rl, '_revlog', rl) |
|
96 if rl.flags(rev) & revlog.REVIDX_SIDEDATA: |
|
97 return True, (), {}, 0, 0 |
|
98 return False, (), {}, 0, 0 |
|
99 |
|
100 elif requirements.COPIESSDC_REQUIREMENT in addedreqs: |
|
101 sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo) |
|
102 elif requirements.COPIESSDC_REQUIREMENT in removedreqs: |
|
103 sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo) |
|
104 return sidedatacompanion |
|
105 |
|
106 |
|
107 def matchrevlog(revlogfilter, entry): |
|
108 """check if a revlog is selected for cloning. |
|
109 |
|
110 In other words, are there any updates which need to be done on revlog |
|
111 or it can be blindly copied. |
|
112 |
|
113 The store entry is checked against the passed filter""" |
|
114 if entry.endswith(b'00changelog.i'): |
|
115 return UPGRADE_CHANGELOG in revlogfilter |
|
116 elif entry.endswith(b'00manifest.i'): |
|
117 return UPGRADE_MANIFEST in revlogfilter |
|
118 return UPGRADE_FILELOGS in revlogfilter |
|
119 |
|
120 |
|
121 def _clonerevlogs( |
|
122 ui, |
|
123 srcrepo, |
|
124 dstrepo, |
|
125 tr, |
|
126 deltareuse, |
|
127 forcedeltabothparents, |
|
128 revlogs=UPGRADE_ALL_REVLOGS, |
|
129 ): |
|
130 """Copy revlogs between 2 repos.""" |
|
131 revcount = 0 |
|
132 srcsize = 0 |
|
133 srcrawsize = 0 |
|
134 dstsize = 0 |
|
135 fcount = 0 |
|
136 frevcount = 0 |
|
137 fsrcsize = 0 |
|
138 frawsize = 0 |
|
139 fdstsize = 0 |
|
140 mcount = 0 |
|
141 mrevcount = 0 |
|
142 msrcsize = 0 |
|
143 mrawsize = 0 |
|
144 mdstsize = 0 |
|
145 crevcount = 0 |
|
146 csrcsize = 0 |
|
147 crawsize = 0 |
|
148 cdstsize = 0 |
|
149 |
|
150 alldatafiles = list(srcrepo.store.walk()) |
|
151 |
|
152 # Perform a pass to collect metadata. This validates we can open all |
|
153 # source files and allows a unified progress bar to be displayed. |
|
154 for unencoded, encoded, size in alldatafiles: |
|
155 if unencoded.endswith(b'.d'): |
|
156 continue |
|
157 |
|
158 rl = _revlogfrompath(srcrepo, unencoded) |
|
159 |
|
160 info = rl.storageinfo( |
|
161 exclusivefiles=True, |
|
162 revisionscount=True, |
|
163 trackedsize=True, |
|
164 storedsize=True, |
|
165 ) |
|
166 |
|
167 revcount += info[b'revisionscount'] or 0 |
|
168 datasize = info[b'storedsize'] or 0 |
|
169 rawsize = info[b'trackedsize'] or 0 |
|
170 |
|
171 srcsize += datasize |
|
172 srcrawsize += rawsize |
|
173 |
|
174 # This is for the separate progress bars. |
|
175 if isinstance(rl, changelog.changelog): |
|
176 crevcount += len(rl) |
|
177 csrcsize += datasize |
|
178 crawsize += rawsize |
|
179 elif isinstance(rl, manifest.manifestrevlog): |
|
180 mcount += 1 |
|
181 mrevcount += len(rl) |
|
182 msrcsize += datasize |
|
183 mrawsize += rawsize |
|
184 elif isinstance(rl, filelog.filelog): |
|
185 fcount += 1 |
|
186 frevcount += len(rl) |
|
187 fsrcsize += datasize |
|
188 frawsize += rawsize |
|
189 else: |
|
190 error.ProgrammingError(b'unknown revlog type') |
|
191 |
|
192 if not revcount: |
|
193 return |
|
194 |
|
195 ui.status( |
|
196 _( |
|
197 b'migrating %d total revisions (%d in filelogs, %d in manifests, ' |
|
198 b'%d in changelog)\n' |
|
199 ) |
|
200 % (revcount, frevcount, mrevcount, crevcount) |
|
201 ) |
|
202 ui.status( |
|
203 _(b'migrating %s in store; %s tracked data\n') |
|
204 % ((util.bytecount(srcsize), util.bytecount(srcrawsize))) |
|
205 ) |
|
206 |
|
207 # Used to keep track of progress. |
|
208 progress = None |
|
209 |
|
210 def oncopiedrevision(rl, rev, node): |
|
211 progress.increment() |
|
212 |
|
213 sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo) |
|
214 |
|
215 # Do the actual copying. |
|
216 # FUTURE this operation can be farmed off to worker processes. |
|
217 seen = set() |
|
218 for unencoded, encoded, size in alldatafiles: |
|
219 if unencoded.endswith(b'.d'): |
|
220 continue |
|
221 |
|
222 oldrl = _revlogfrompath(srcrepo, unencoded) |
|
223 |
|
224 if isinstance(oldrl, changelog.changelog) and b'c' not in seen: |
|
225 ui.status( |
|
226 _( |
|
227 b'finished migrating %d manifest revisions across %d ' |
|
228 b'manifests; change in size: %s\n' |
|
229 ) |
|
230 % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize)) |
|
231 ) |
|
232 |
|
233 ui.status( |
|
234 _( |
|
235 b'migrating changelog containing %d revisions ' |
|
236 b'(%s in store; %s tracked data)\n' |
|
237 ) |
|
238 % ( |
|
239 crevcount, |
|
240 util.bytecount(csrcsize), |
|
241 util.bytecount(crawsize), |
|
242 ) |
|
243 ) |
|
244 seen.add(b'c') |
|
245 progress = srcrepo.ui.makeprogress( |
|
246 _(b'changelog revisions'), total=crevcount |
|
247 ) |
|
248 elif isinstance(oldrl, manifest.manifestrevlog) and b'm' not in seen: |
|
249 ui.status( |
|
250 _( |
|
251 b'finished migrating %d filelog revisions across %d ' |
|
252 b'filelogs; change in size: %s\n' |
|
253 ) |
|
254 % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize)) |
|
255 ) |
|
256 |
|
257 ui.status( |
|
258 _( |
|
259 b'migrating %d manifests containing %d revisions ' |
|
260 b'(%s in store; %s tracked data)\n' |
|
261 ) |
|
262 % ( |
|
263 mcount, |
|
264 mrevcount, |
|
265 util.bytecount(msrcsize), |
|
266 util.bytecount(mrawsize), |
|
267 ) |
|
268 ) |
|
269 seen.add(b'm') |
|
270 if progress: |
|
271 progress.complete() |
|
272 progress = srcrepo.ui.makeprogress( |
|
273 _(b'manifest revisions'), total=mrevcount |
|
274 ) |
|
275 elif b'f' not in seen: |
|
276 ui.status( |
|
277 _( |
|
278 b'migrating %d filelogs containing %d revisions ' |
|
279 b'(%s in store; %s tracked data)\n' |
|
280 ) |
|
281 % ( |
|
282 fcount, |
|
283 frevcount, |
|
284 util.bytecount(fsrcsize), |
|
285 util.bytecount(frawsize), |
|
286 ) |
|
287 ) |
|
288 seen.add(b'f') |
|
289 if progress: |
|
290 progress.complete() |
|
291 progress = srcrepo.ui.makeprogress( |
|
292 _(b'file revisions'), total=frevcount |
|
293 ) |
|
294 |
|
295 if matchrevlog(revlogs, unencoded): |
|
296 ui.note( |
|
297 _(b'cloning %d revisions from %s\n') % (len(oldrl), unencoded) |
|
298 ) |
|
299 newrl = _revlogfrompath(dstrepo, unencoded) |
|
300 oldrl.clone( |
|
301 tr, |
|
302 newrl, |
|
303 addrevisioncb=oncopiedrevision, |
|
304 deltareuse=deltareuse, |
|
305 forcedeltabothparents=forcedeltabothparents, |
|
306 sidedatacompanion=sidedatacompanion, |
|
307 ) |
|
308 else: |
|
309 msg = _(b'blindly copying %s containing %i revisions\n') |
|
310 ui.note(msg % (unencoded, len(oldrl))) |
|
311 _copyrevlog(tr, dstrepo, oldrl, unencoded) |
|
312 |
|
313 newrl = _revlogfrompath(dstrepo, unencoded) |
|
314 |
|
315 info = newrl.storageinfo(storedsize=True) |
|
316 datasize = info[b'storedsize'] or 0 |
|
317 |
|
318 dstsize += datasize |
|
319 |
|
320 if isinstance(newrl, changelog.changelog): |
|
321 cdstsize += datasize |
|
322 elif isinstance(newrl, manifest.manifestrevlog): |
|
323 mdstsize += datasize |
|
324 else: |
|
325 fdstsize += datasize |
|
326 |
|
327 progress.complete() |
|
328 |
|
329 ui.status( |
|
330 _( |
|
331 b'finished migrating %d changelog revisions; change in size: ' |
|
332 b'%s\n' |
|
333 ) |
|
334 % (crevcount, util.bytecount(cdstsize - csrcsize)) |
|
335 ) |
|
336 |
|
337 ui.status( |
|
338 _( |
|
339 b'finished migrating %d total revisions; total change in store ' |
|
340 b'size: %s\n' |
|
341 ) |
|
342 % (revcount, util.bytecount(dstsize - srcsize)) |
|
343 ) |
|
344 |
|
345 |
|
346 def _filterstorefile(srcrepo, dstrepo, requirements, path, mode, st): |
|
347 """Determine whether to copy a store file during upgrade. |
|
348 |
|
349 This function is called when migrating store files from ``srcrepo`` to |
|
350 ``dstrepo`` as part of upgrading a repository. |
|
351 |
|
352 Args: |
|
353 srcrepo: repo we are copying from |
|
354 dstrepo: repo we are copying to |
|
355 requirements: set of requirements for ``dstrepo`` |
|
356 path: store file being examined |
|
357 mode: the ``ST_MODE`` file type of ``path`` |
|
358 st: ``stat`` data structure for ``path`` |
|
359 |
|
360 Function should return ``True`` if the file is to be copied. |
|
361 """ |
|
362 # Skip revlogs. |
|
363 if path.endswith((b'.i', b'.d', b'.n', b'.nd')): |
|
364 return False |
|
365 # Skip transaction related files. |
|
366 if path.startswith(b'undo'): |
|
367 return False |
|
368 # Only copy regular files. |
|
369 if mode != stat.S_IFREG: |
|
370 return False |
|
371 # Skip other skipped files. |
|
372 if path in (b'lock', b'fncache'): |
|
373 return False |
|
374 |
|
375 return True |
|
376 |
|
377 |
|
378 def _finishdatamigration(ui, srcrepo, dstrepo, requirements): |
|
379 """Hook point for extensions to perform additional actions during upgrade. |
|
380 |
|
381 This function is called after revlogs and store files have been copied but |
|
382 before the new store is swapped into the original location. |
|
383 """ |
|
384 |
|
385 |
|
386 def upgrade( |
|
387 ui, srcrepo, dstrepo, requirements, actions, revlogs=UPGRADE_ALL_REVLOGS |
|
388 ): |
|
389 """Do the low-level work of upgrading a repository. |
|
390 |
|
391 The upgrade is effectively performed as a copy between a source |
|
392 repository and a temporary destination repository. |
|
393 |
|
394 The source repository is unmodified for as long as possible so the |
|
395 upgrade can abort at any time without causing loss of service for |
|
396 readers and without corrupting the source repository. |
|
397 """ |
|
398 assert srcrepo.currentwlock() |
|
399 assert dstrepo.currentwlock() |
|
400 |
|
401 ui.status( |
|
402 _( |
|
403 b'(it is safe to interrupt this process any time before ' |
|
404 b'data migration completes)\n' |
|
405 ) |
|
406 ) |
|
407 |
|
408 if b're-delta-all' in actions: |
|
409 deltareuse = revlog.revlog.DELTAREUSENEVER |
|
410 elif b're-delta-parent' in actions: |
|
411 deltareuse = revlog.revlog.DELTAREUSESAMEREVS |
|
412 elif b're-delta-multibase' in actions: |
|
413 deltareuse = revlog.revlog.DELTAREUSESAMEREVS |
|
414 elif b're-delta-fulladd' in actions: |
|
415 deltareuse = revlog.revlog.DELTAREUSEFULLADD |
|
416 else: |
|
417 deltareuse = revlog.revlog.DELTAREUSEALWAYS |
|
418 |
|
419 with dstrepo.transaction(b'upgrade') as tr: |
|
420 _clonerevlogs( |
|
421 ui, |
|
422 srcrepo, |
|
423 dstrepo, |
|
424 tr, |
|
425 deltareuse, |
|
426 b're-delta-multibase' in actions, |
|
427 revlogs=revlogs, |
|
428 ) |
|
429 |
|
430 # Now copy other files in the store directory. |
|
431 # The sorted() makes execution deterministic. |
|
432 for p, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)): |
|
433 if not _filterstorefile(srcrepo, dstrepo, requirements, p, kind, st): |
|
434 continue |
|
435 |
|
436 srcrepo.ui.status(_(b'copying %s\n') % p) |
|
437 src = srcrepo.store.rawvfs.join(p) |
|
438 dst = dstrepo.store.rawvfs.join(p) |
|
439 util.copyfile(src, dst, copystat=True) |
|
440 |
|
441 _finishdatamigration(ui, srcrepo, dstrepo, requirements) |
|
442 |
|
443 ui.status(_(b'data fully migrated to temporary repository\n')) |
|
444 |
|
445 backuppath = pycompat.mkdtemp(prefix=b'upgradebackup.', dir=srcrepo.path) |
|
446 backupvfs = vfsmod.vfs(backuppath) |
|
447 |
|
448 # Make a backup of requires file first, as it is the first to be modified. |
|
449 util.copyfile(srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires')) |
|
450 |
|
451 # We install an arbitrary requirement that clients must not support |
|
452 # as a mechanism to lock out new clients during the data swap. This is |
|
453 # better than allowing a client to continue while the repository is in |
|
454 # an inconsistent state. |
|
455 ui.status( |
|
456 _( |
|
457 b'marking source repository as being upgraded; clients will be ' |
|
458 b'unable to read from repository\n' |
|
459 ) |
|
460 ) |
|
461 scmutil.writereporequirements( |
|
462 srcrepo, srcrepo.requirements | {b'upgradeinprogress'} |
|
463 ) |
|
464 |
|
465 ui.status(_(b'starting in-place swap of repository data\n')) |
|
466 ui.status(_(b'replaced files will be backed up at %s\n') % backuppath) |
|
467 |
|
468 # Now swap in the new store directory. Doing it as a rename should make |
|
469 # the operation nearly instantaneous and atomic (at least in well-behaved |
|
470 # environments). |
|
471 ui.status(_(b'replacing store...\n')) |
|
472 tstart = util.timer() |
|
473 util.rename(srcrepo.spath, backupvfs.join(b'store')) |
|
474 util.rename(dstrepo.spath, srcrepo.spath) |
|
475 elapsed = util.timer() - tstart |
|
476 ui.status( |
|
477 _( |
|
478 b'store replacement complete; repository was inconsistent for ' |
|
479 b'%0.1fs\n' |
|
480 ) |
|
481 % elapsed |
|
482 ) |
|
483 |
|
484 # We first write the requirements file. Any new requirements will lock |
|
485 # out legacy clients. |
|
486 ui.status( |
|
487 _( |
|
488 b'finalizing requirements file and making repository readable ' |
|
489 b'again\n' |
|
490 ) |
|
491 ) |
|
492 scmutil.writereporequirements(srcrepo, requirements) |
|
493 |
|
494 # The lock file from the old store won't be removed because nothing has a |
|
495 # reference to its new location. So clean it up manually. Alternatively, we |
|
496 # could update srcrepo.svfs and other variables to point to the new |
|
497 # location. This is simpler. |
|
498 backupvfs.unlink(b'store/lock') |
|
499 |
|
500 return backuppath |