Mercurial > public > mercurial-scm > hg

comparison mercurial/upgrade.py @ 46046:f105c49e89cd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
upgrade: split actual upgrade code away from the main module The main module is getting big and hard to follow. So we are splitting all the logic to actually run an upgrade in a sub module. It nicely highlight that there are very few actual call point to the code we just moved. Differential Revision: https://phab.mercurial-scm.org/D9476
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Tue, 01 Dec 2020 09:13:08 +0100
parents 6c960b708ac4
children 4b89cf08d8dc
comparison
equal deleted inserted replaced
46045:7905899c4f8f 46046:f105c49e89cd
5 # This software may be used and distributed according to the terms of the 5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version. 6 # GNU General Public License version 2 or any later version.
7 7
8 from __future__ import absolute_import 8 from __future__ import absolute_import
9 9
10 import stat
11
12 from .i18n import _ 10 from .i18n import _
13 from .pycompat import getattr
14 from . import ( 11 from . import (
15 changelog,
16 error, 12 error,
17 filelog,
18 hg, 13 hg,
19 localrepo, 14 localrepo,
20 manifest,
21 metadata,
22 pycompat, 15 pycompat,
23 requirements, 16 requirements,
24 revlog,
25 scmutil,
26 util, 17 util,
27 vfs as vfsmod, 18 )
19
20 from .upgrade_utils import (
21 engine as upgrade_engine,
28 ) 22 )
29 23
30 from .utils import compression 24 from .utils import compression
31 25
32 # list of requirements that request a clone of all revlog if added/removed 26 # list of requirements that request a clone of all revlog if added/removed
690 # e.g. adding generaldelta could schedule parent redeltas. 684 # e.g. adding generaldelta could schedule parent redeltas.
691 685
692 return newactions 686 return newactions
693 687
694 688
695 def _revlogfrompath(repo, path):
696 """Obtain a revlog from a repo path.
697
698 An instance of the appropriate class is returned.
699 """
700 if path == b'00changelog.i':
701 return changelog.changelog(repo.svfs)
702 elif path.endswith(b'00manifest.i'):
703 mandir = path[: -len(b'00manifest.i')]
704 return manifest.manifestrevlog(repo.svfs, tree=mandir)
705 else:
706 # reverse of "/".join(("data", path + ".i"))
707 return filelog.filelog(repo.svfs, path[5:-2])
708
709
710 def _copyrevlog(tr, destrepo, oldrl, unencodedname):
711 """copy all relevant files for `oldrl` into `destrepo` store
712
713 Files are copied "as is" without any transformation. The copy is performed
714 without extra checks. Callers are responsible for making sure the copied
715 content is compatible with format of the destination repository.
716 """
717 oldrl = getattr(oldrl, '_revlog', oldrl)
718 newrl = _revlogfrompath(destrepo, unencodedname)
719 newrl = getattr(newrl, '_revlog', newrl)
720
721 oldvfs = oldrl.opener
722 newvfs = newrl.opener
723 oldindex = oldvfs.join(oldrl.indexfile)
724 newindex = newvfs.join(newrl.indexfile)
725 olddata = oldvfs.join(oldrl.datafile)
726 newdata = newvfs.join(newrl.datafile)
727
728 with newvfs(newrl.indexfile, b'w'):
729 pass # create all the directories
730
731 util.copyfile(oldindex, newindex)
732 copydata = oldrl.opener.exists(oldrl.datafile)
733 if copydata:
734 util.copyfile(olddata, newdata)
735
736 if not (
737 unencodedname.endswith(b'00changelog.i')
738 or unencodedname.endswith(b'00manifest.i')
739 ):
740 destrepo.svfs.fncache.add(unencodedname)
741 if copydata:
742 destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')
743
744
745 UPGRADE_CHANGELOG = b"changelog"
746 UPGRADE_MANIFEST = b"manifest"
747 UPGRADE_FILELOGS = b"all-filelogs"
748
749 UPGRADE_ALL_REVLOGS = frozenset(
750 [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS]
751 )
752
753
754 def getsidedatacompanion(srcrepo, dstrepo):
755 sidedatacompanion = None
756 removedreqs = srcrepo.requirements - dstrepo.requirements
757 addedreqs = dstrepo.requirements - srcrepo.requirements
758 if requirements.SIDEDATA_REQUIREMENT in removedreqs:
759
760 def sidedatacompanion(rl, rev):
761 rl = getattr(rl, '_revlog', rl)
762 if rl.flags(rev) & revlog.REVIDX_SIDEDATA:
763 return True, (), {}, 0, 0
764 return False, (), {}, 0, 0
765
766 elif requirements.COPIESSDC_REQUIREMENT in addedreqs:
767 sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo)
768 elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
769 sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
770 return sidedatacompanion
771
772
773 def matchrevlog(revlogfilter, entry):
774 """check if a revlog is selected for cloning.
775
776 In other words, are there any updates which need to be done on revlog
777 or it can be blindly copied.
778
779 The store entry is checked against the passed filter"""
780 if entry.endswith(b'00changelog.i'):
781 return UPGRADE_CHANGELOG in revlogfilter
782 elif entry.endswith(b'00manifest.i'):
783 return UPGRADE_MANIFEST in revlogfilter
784 return UPGRADE_FILELOGS in revlogfilter
785
786
787 def _clonerevlogs(
788 ui,
789 srcrepo,
790 dstrepo,
791 tr,
792 deltareuse,
793 forcedeltabothparents,
794 revlogs=UPGRADE_ALL_REVLOGS,
795 ):
796 """Copy revlogs between 2 repos."""
797 revcount = 0
798 srcsize = 0
799 srcrawsize = 0
800 dstsize = 0
801 fcount = 0
802 frevcount = 0
803 fsrcsize = 0
804 frawsize = 0
805 fdstsize = 0
806 mcount = 0
807 mrevcount = 0
808 msrcsize = 0
809 mrawsize = 0
810 mdstsize = 0
811 crevcount = 0
812 csrcsize = 0
813 crawsize = 0
814 cdstsize = 0
815
816 alldatafiles = list(srcrepo.store.walk())
817
818 # Perform a pass to collect metadata. This validates we can open all
819 # source files and allows a unified progress bar to be displayed.
820 for unencoded, encoded, size in alldatafiles:
821 if unencoded.endswith(b'.d'):
822 continue
823
824 rl = _revlogfrompath(srcrepo, unencoded)
825
826 info = rl.storageinfo(
827 exclusivefiles=True,
828 revisionscount=True,
829 trackedsize=True,
830 storedsize=True,
831 )
832
833 revcount += info[b'revisionscount'] or 0
834 datasize = info[b'storedsize'] or 0
835 rawsize = info[b'trackedsize'] or 0
836
837 srcsize += datasize
838 srcrawsize += rawsize
839
840 # This is for the separate progress bars.
841 if isinstance(rl, changelog.changelog):
842 crevcount += len(rl)
843 csrcsize += datasize
844 crawsize += rawsize
845 elif isinstance(rl, manifest.manifestrevlog):
846 mcount += 1
847 mrevcount += len(rl)
848 msrcsize += datasize
849 mrawsize += rawsize
850 elif isinstance(rl, filelog.filelog):
851 fcount += 1
852 frevcount += len(rl)
853 fsrcsize += datasize
854 frawsize += rawsize
855 else:
856 error.ProgrammingError(b'unknown revlog type')
857
858 if not revcount:
859 return
860
861 ui.status(
862 _(
863 b'migrating %d total revisions (%d in filelogs, %d in manifests, '
864 b'%d in changelog)\n'
865 )
866 % (revcount, frevcount, mrevcount, crevcount)
867 )
868 ui.status(
869 _(b'migrating %s in store; %s tracked data\n')
870 % ((util.bytecount(srcsize), util.bytecount(srcrawsize)))
871 )
872
873 # Used to keep track of progress.
874 progress = None
875
876 def oncopiedrevision(rl, rev, node):
877 progress.increment()
878
879 sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
880
881 # Do the actual copying.
882 # FUTURE this operation can be farmed off to worker processes.
883 seen = set()
884 for unencoded, encoded, size in alldatafiles:
885 if unencoded.endswith(b'.d'):
886 continue
887
888 oldrl = _revlogfrompath(srcrepo, unencoded)
889
890 if isinstance(oldrl, changelog.changelog) and b'c' not in seen:
891 ui.status(
892 _(
893 b'finished migrating %d manifest revisions across %d '
894 b'manifests; change in size: %s\n'
895 )
896 % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize))
897 )
898
899 ui.status(
900 _(
901 b'migrating changelog containing %d revisions '
902 b'(%s in store; %s tracked data)\n'
903 )
904 % (
905 crevcount,
906 util.bytecount(csrcsize),
907 util.bytecount(crawsize),
908 )
909 )
910 seen.add(b'c')
911 progress = srcrepo.ui.makeprogress(
912 _(b'changelog revisions'), total=crevcount
913 )
914 elif isinstance(oldrl, manifest.manifestrevlog) and b'm' not in seen:
915 ui.status(
916 _(
917 b'finished migrating %d filelog revisions across %d '
918 b'filelogs; change in size: %s\n'
919 )
920 % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize))
921 )
922
923 ui.status(
924 _(
925 b'migrating %d manifests containing %d revisions '
926 b'(%s in store; %s tracked data)\n'
927 )
928 % (
929 mcount,
930 mrevcount,
931 util.bytecount(msrcsize),
932 util.bytecount(mrawsize),
933 )
934 )
935 seen.add(b'm')
936 if progress:
937 progress.complete()
938 progress = srcrepo.ui.makeprogress(
939 _(b'manifest revisions'), total=mrevcount
940 )
941 elif b'f' not in seen:
942 ui.status(
943 _(
944 b'migrating %d filelogs containing %d revisions '
945 b'(%s in store; %s tracked data)\n'
946 )
947 % (
948 fcount,
949 frevcount,
950 util.bytecount(fsrcsize),
951 util.bytecount(frawsize),
952 )
953 )
954 seen.add(b'f')
955 if progress:
956 progress.complete()
957 progress = srcrepo.ui.makeprogress(
958 _(b'file revisions'), total=frevcount
959 )
960
961 if matchrevlog(revlogs, unencoded):
962 ui.note(
963 _(b'cloning %d revisions from %s\n') % (len(oldrl), unencoded)
964 )
965 newrl = _revlogfrompath(dstrepo, unencoded)
966 oldrl.clone(
967 tr,
968 newrl,
969 addrevisioncb=oncopiedrevision,
970 deltareuse=deltareuse,
971 forcedeltabothparents=forcedeltabothparents,
972 sidedatacompanion=sidedatacompanion,
973 )
974 else:
975 msg = _(b'blindly copying %s containing %i revisions\n')
976 ui.note(msg % (unencoded, len(oldrl)))
977 _copyrevlog(tr, dstrepo, oldrl, unencoded)
978
979 newrl = _revlogfrompath(dstrepo, unencoded)
980
981 info = newrl.storageinfo(storedsize=True)
982 datasize = info[b'storedsize'] or 0
983
984 dstsize += datasize
985
986 if isinstance(newrl, changelog.changelog):
987 cdstsize += datasize
988 elif isinstance(newrl, manifest.manifestrevlog):
989 mdstsize += datasize
990 else:
991 fdstsize += datasize
992
993 progress.complete()
994
995 ui.status(
996 _(
997 b'finished migrating %d changelog revisions; change in size: '
998 b'%s\n'
999 )
1000 % (crevcount, util.bytecount(cdstsize - csrcsize))
1001 )
1002
1003 ui.status(
1004 _(
1005 b'finished migrating %d total revisions; total change in store '
1006 b'size: %s\n'
1007 )
1008 % (revcount, util.bytecount(dstsize - srcsize))
1009 )
1010
1011
1012 def _filterstorefile(srcrepo, dstrepo, requirements, path, mode, st):
1013 """Determine whether to copy a store file during upgrade.
1014
1015 This function is called when migrating store files from ``srcrepo`` to
1016 ``dstrepo`` as part of upgrading a repository.
1017
1018 Args:
1019 srcrepo: repo we are copying from
1020 dstrepo: repo we are copying to
1021 requirements: set of requirements for ``dstrepo``
1022 path: store file being examined
1023 mode: the ``ST_MODE`` file type of ``path``
1024 st: ``stat`` data structure for ``path``
1025
1026 Function should return ``True`` if the file is to be copied.
1027 """
1028 # Skip revlogs.
1029 if path.endswith((b'.i', b'.d', b'.n', b'.nd')):
1030 return False
1031 # Skip transaction related files.
1032 if path.startswith(b'undo'):
1033 return False
1034 # Only copy regular files.
1035 if mode != stat.S_IFREG:
1036 return False
1037 # Skip other skipped files.
1038 if path in (b'lock', b'fncache'):
1039 return False
1040
1041 return True
1042
1043
1044 def _finishdatamigration(ui, srcrepo, dstrepo, requirements):
1045 """Hook point for extensions to perform additional actions during upgrade.
1046
1047 This function is called after revlogs and store files have been copied but
1048 before the new store is swapped into the original location.
1049 """
1050
1051
1052 def _upgraderepo(
1053 ui, srcrepo, dstrepo, requirements, actions, revlogs=UPGRADE_ALL_REVLOGS
1054 ):
1055 """Do the low-level work of upgrading a repository.
1056
1057 The upgrade is effectively performed as a copy between a source
1058 repository and a temporary destination repository.
1059
1060 The source repository is unmodified for as long as possible so the
1061 upgrade can abort at any time without causing loss of service for
1062 readers and without corrupting the source repository.
1063 """
1064 assert srcrepo.currentwlock()
1065 assert dstrepo.currentwlock()
1066
1067 ui.status(
1068 _(
1069 b'(it is safe to interrupt this process any time before '
1070 b'data migration completes)\n'
1071 )
1072 )
1073
1074 if b're-delta-all' in actions:
1075 deltareuse = revlog.revlog.DELTAREUSENEVER
1076 elif b're-delta-parent' in actions:
1077 deltareuse = revlog.revlog.DELTAREUSESAMEREVS
1078 elif b're-delta-multibase' in actions:
1079 deltareuse = revlog.revlog.DELTAREUSESAMEREVS
1080 elif b're-delta-fulladd' in actions:
1081 deltareuse = revlog.revlog.DELTAREUSEFULLADD
1082 else:
1083 deltareuse = revlog.revlog.DELTAREUSEALWAYS
1084
1085 with dstrepo.transaction(b'upgrade') as tr:
1086 _clonerevlogs(
1087 ui,
1088 srcrepo,
1089 dstrepo,
1090 tr,
1091 deltareuse,
1092 b're-delta-multibase' in actions,
1093 revlogs=revlogs,
1094 )
1095
1096 # Now copy other files in the store directory.
1097 # The sorted() makes execution deterministic.
1098 for p, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)):
1099 if not _filterstorefile(srcrepo, dstrepo, requirements, p, kind, st):
1100 continue
1101
1102 srcrepo.ui.status(_(b'copying %s\n') % p)
1103 src = srcrepo.store.rawvfs.join(p)
1104 dst = dstrepo.store.rawvfs.join(p)
1105 util.copyfile(src, dst, copystat=True)
1106
1107 _finishdatamigration(ui, srcrepo, dstrepo, requirements)
1108
1109 ui.status(_(b'data fully migrated to temporary repository\n'))
1110
1111 backuppath = pycompat.mkdtemp(prefix=b'upgradebackup.', dir=srcrepo.path)
1112 backupvfs = vfsmod.vfs(backuppath)
1113
1114 # Make a backup of requires file first, as it is the first to be modified.
1115 util.copyfile(srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires'))
1116
1117 # We install an arbitrary requirement that clients must not support
1118 # as a mechanism to lock out new clients during the data swap. This is
1119 # better than allowing a client to continue while the repository is in
1120 # an inconsistent state.
1121 ui.status(
1122 _(
1123 b'marking source repository as being upgraded; clients will be '
1124 b'unable to read from repository\n'
1125 )
1126 )
1127 scmutil.writereporequirements(
1128 srcrepo, srcrepo.requirements | {b'upgradeinprogress'}
1129 )
1130
1131 ui.status(_(b'starting in-place swap of repository data\n'))
1132 ui.status(_(b'replaced files will be backed up at %s\n') % backuppath)
1133
1134 # Now swap in the new store directory. Doing it as a rename should make
1135 # the operation nearly instantaneous and atomic (at least in well-behaved
1136 # environments).
1137 ui.status(_(b'replacing store...\n'))
1138 tstart = util.timer()
1139 util.rename(srcrepo.spath, backupvfs.join(b'store'))
1140 util.rename(dstrepo.spath, srcrepo.spath)
1141 elapsed = util.timer() - tstart
1142 ui.status(
1143 _(
1144 b'store replacement complete; repository was inconsistent for '
1145 b'%0.1fs\n'
1146 )
1147 % elapsed
1148 )
1149
1150 # We first write the requirements file. Any new requirements will lock
1151 # out legacy clients.
1152 ui.status(
1153 _(
1154 b'finalizing requirements file and making repository readable '
1155 b'again\n'
1156 )
1157 )
1158 scmutil.writereporequirements(srcrepo, requirements)
1159
1160 # The lock file from the old store won't be removed because nothing has a
1161 # reference to its new location. So clean it up manually. Alternatively, we
1162 # could update srcrepo.svfs and other variables to point to the new
1163 # location. This is simpler.
1164 backupvfs.unlink(b'store/lock')
1165
1166 return backuppath
1167
1168
1169 def upgraderepo( 689 def upgraderepo(
1170 ui, 690 ui,
1171 repo, 691 repo,
1172 run=False, 692 run=False,
1173 optimize=None, 693 optimize=None,
1180 if optimize is None: 700 if optimize is None:
1181 optimize = [] 701 optimize = []
1182 optimize = {legacy_opts_map.get(o, o) for o in optimize} 702 optimize = {legacy_opts_map.get(o, o) for o in optimize}
1183 repo = repo.unfiltered() 703 repo = repo.unfiltered()
1184 704
1185 revlogs = set(UPGRADE_ALL_REVLOGS) 705 revlogs = set(upgrade_engine.UPGRADE_ALL_REVLOGS)
1186 specentries = ( 706 specentries = (
1187 (UPGRADE_CHANGELOG, changelog), 707 (upgrade_engine.UPGRADE_CHANGELOG, changelog),
1188 (UPGRADE_MANIFEST, manifest), 708 (upgrade_engine.UPGRADE_MANIFEST, manifest),
1189 (UPGRADE_FILELOGS, filelogs), 709 (upgrade_engine.UPGRADE_FILELOGS, filelogs),
1190 ) 710 )
1191 specified = [(y, x) for (y, x) in specentries if x is not None] 711 specified = [(y, x) for (y, x) in specentries if x is not None]
1192 if specified: 712 if specified:
1193 # we have some limitation on revlogs to be recloned 713 # we have some limitation on revlogs to be recloned
1194 if any(x for y, x in specified): 714 if any(x for y, x in specified):
1285 ) 805 )
1286 806
1287 removedreqs = repo.requirements - newreqs 807 removedreqs = repo.requirements - newreqs
1288 addedreqs = newreqs - repo.requirements 808 addedreqs = newreqs - repo.requirements
1289 809
1290 if revlogs != UPGRADE_ALL_REVLOGS: 810 if revlogs != upgrade_engine.UPGRADE_ALL_REVLOGS:
1291 incompatible = RECLONES_REQUIREMENTS & (removedreqs | addedreqs) 811 incompatible = RECLONES_REQUIREMENTS & (removedreqs | addedreqs)
1292 if incompatible: 812 if incompatible:
1293 msg = _( 813 msg = _(
1294 b'ignoring revlogs selection flags, format requirements ' 814 b'ignoring revlogs selection flags, format requirements '
1295 b'change: %s\n' 815 b'change: %s\n'
1296 ) 816 )
1297 ui.warn(msg % b', '.join(sorted(incompatible))) 817 ui.warn(msg % b', '.join(sorted(incompatible)))
1298 revlogs = UPGRADE_ALL_REVLOGS 818 revlogs = upgrade_engine.UPGRADE_ALL_REVLOGS
1299 819
1300 def write_labeled(l, label): 820 def write_labeled(l, label):
1301 first = True 821 first = True
1302 for r in sorted(l): 822 for r in sorted(l):
1303 if not first: 823 if not first:
1445 # clone ui without using ui.copy because repo.ui is protected 965 # clone ui without using ui.copy because repo.ui is protected
1446 repoui = repo.ui.__class__(repo.ui) 966 repoui = repo.ui.__class__(repo.ui)
1447 dstrepo = hg.repository(repoui, path=tmppath, create=True) 967 dstrepo = hg.repository(repoui, path=tmppath, create=True)
1448 968
1449 with dstrepo.wlock(), dstrepo.lock(): 969 with dstrepo.wlock(), dstrepo.lock():
1450 backuppath = _upgraderepo( 970 backuppath = upgrade_engine.upgrade(
1451 ui, repo, dstrepo, newreqs, upgradeactions, revlogs=revlogs 971 ui, repo, dstrepo, newreqs, upgradeactions, revlogs=revlogs
1452 ) 972 )
1453 if not (backup or backuppath is None): 973 if not (backup or backuppath is None):
1454 ui.status( 974 ui.status(
1455 _(b'removing old repository content%s\n') % backuppath 975 _(b'removing old repository content%s\n') % backuppath