mercurial-scm/hg-stable: mercurial/revlog.py comparison

comparison mercurial/revlog.py @ 50316:87f0155d68aa stable

revlog: improve the robustness of the splitting process The previous "in-place" splitting, preserving the splitting on transaction failure had a couple of issue in case of transaction rollback: - a race windows that could still lead to a crash and data loss - it corrupted the `fncache`. So instead, we use a new approach that we summarized as "we do a backup of the inline revlog pre-split, and we restore this in case of failure". To make readers live easier, we don't overwrite the inline index file until transaction finalization. (once the transaction get into its finalization phase, it is not expected to rollback, unless some crash happens). To do so, we write the index of the split index in a temporary file that we use until transaction finalization. We also keep a backup of the initial inline file to be able to rollback the split if needed. As a result, transaction rollback cancel the split and no longer corrupt fncache. We also no longer have a small inconsistency windows where the transaction could be unrecoverable.

author	Pierre-Yves David <pierre-yves.david@octobus.net>
date	Mon, 20 Mar 2023 11:52:17 +0100
parents	9854a9adc466
children	f952be90b051

comparison

equal deleted inserted replaced

-:cf6e1d535602
+:87f0155d68aa
 censorable=False,
 upperboundcomp=None,
 persistentnodemap=False,
 concurrencychecker=None,
 trypending=False,
+try_split=False,
 canonical_parent_order=True,
 ):
 """
 create a revlog object
 self._datafile = None
 self._sidedatafile = None
 self._nodemap_file = None
 self.postfix = postfix
 self._trypending = trypending
+self._try_split = try_split
 self.opener = opener
 if persistentnodemap:
 self._nodemap_file = nodemaputil.get_nodemap_file(self)
 assert target[0] in ALL_KINDS
 if self.postfix is not None:
 entry_point = b'%s.i.%s' % (self.radix, self.postfix)
 elif self._trypending and self.opener.exists(b'%s.i.a' % self.radix):
 entry_point = b'%s.i.a' % self.radix
+elif self._try_split and self.opener.exists(b'%s.i.s' % self.radix):
+entry_point = b'%s.i.s' % self.radix
 else:
 entry_point = b'%s.i' % self.radix
 if docket is not None:
 self._docket = docket
 except error.RevlogError:
 if self._censorable and storageutil.iscensoredtext(text):
 raise error.CensoredNodeError(self.display_id, node, text)
 raise
-def _enforceinlinesize(self, tr):
+def _enforceinlinesize(self, tr, side_write=True):
 """Check if the revlog is too big for inline and convert if so.
 This should be called after revisions are added to the revlog. If the
 revlog has grown too large to be an inline revlog, it will convert it
 to use multiple index and data files.
 troffset = tr.findoffset(self._indexfile)
 if troffset is None:
 raise error.RevlogError(
 _(b"%s not found in the transaction") % self._indexfile
 )
-trindex = None
+if troffset:
+tr.addbackup(self._indexfile, for_offset=True)
 tr.add(self._datafile, 0)
 existing_handles = False
 if self._writinghandles is not None:
 existing_handles = True
 self._writinghandles = None
 self._segmentfile.writing_handle = None
 # No need to deal with sidedata writing handle as it is only
 # relevant with revlog-v2 which is never inline, not reaching
 # this code
+if side_write:
+old_index_file_path = self._indexfile
+new_index_file_path = self._indexfile + b'.s'
+opener = self.opener
+fncache = getattr(opener, 'fncache', None)
+if fncache is not None:
+fncache.addignore(new_index_file_path)
+# the "split" index replace the real index when the transaction is finalized
+def finalize_callback(tr):
+opener.rename(
+new_index_file_path,
+old_index_file_path,
+checkambig=True,
+)
+tr.registertmp(new_index_file_path)
+if self.target[1] is not None:
+finalize_id = b'000-revlog-split-%d-%s' % self.target
+else:
+finalize_id = b'000-revlog-split-%d' % self.target[0]
+tr.addfinalize(finalize_id, finalize_callback)
 new_dfh = self._datafp(b'w+')
 new_dfh.truncate(0)  # drop any potentially existing data
 try:
 with self._indexfp() as read_ifh:
 for r in self:
 new_dfh.write(self._getsegmentforrevs(r, r, df=read_ifh)[1])
-if (
-trindex is None
-and troffset
-<= self.start(r) + r * self.index.entry_size
-):
-trindex = r
 new_dfh.flush()
-if trindex is None:
+if side_write:
-trindex = 0
+self._indexfile = new_index_file_path
 with self.__index_new_fp() as fp:
 self._format_flags &= ~FLAG_INLINE_DATA
 self._inline = False
 for i in self:
 e = self.index.entry_binary(i)
 e = header + e
 fp.write(e)
 if self._docket is not None:
 self._docket.index_end = fp.tell()
-# There is a small transactional race here. If the rename of
+# If we don't use side-write, the temp file replace the real
-# the index fails, we should remove the datafile. It is more
+# index when we exit the context manager
-# important to ensure that the data file is not truncated
-# when the index is replaced as otherwise data is lost.
-tr.replace(self._datafile, self.start(trindex))
-# the temp file replace the real index when we exit the context
-# manager
-tr.replace(self._indexfile, trindex * self.index.entry_size)
 nodemaputil.setup_persistent_nodemap(tr, self)
 self._segmentfile = randomaccessfile.randomaccessfile(
 self.opener,
 self._datafile,
 self._chunkcachesize,

Mercurial > public > mercurial-scm > hg-stable

comparison mercurial/revlog.py @ 50316:87f0155d68aa stable