mercurial-scm/hg: mercurial/revlog.py comparison

comparison mercurial/revlog.py @ 46709:3d740058b467

sidedata: move to new sidedata storage in revlogv2 The current (experimental) sidedata system uses flagprocessors to signify the presence and store/retrieve sidedata from the raw revlog data. This proved to be quite fragile from an exchange perspective and a lot more complex than simply having a dedicated space in the new revlog format. This change does not handle exchange (ironically), so the test for amend - that uses a bundle - is broken. This functionality is split into the next patches. Differential Revision: https://phab.mercurial-scm.org/D9993

author	Rapha?l Gom?s <rgomes@octobus.net>
date	Mon, 18 Jan 2021 11:44:51 +0100
parents	913485776542
children	4cd214c9948d

comparison

equal deleted inserted replaced

-:358737abeeef
+:3d740058b467
 _maxinline = 131072
 _chunksize = 1048576
 # Flag processors for REVIDX_ELLIPSIS.
 def ellipsisreadprocessor(rl, text):
-return text, False, {}
+return text, False
-def ellipsiswriteprocessor(rl, text, sidedata):
+def ellipsiswriteprocessor(rl, text):
 return text, False
 def ellipsisrawprocessor(rl, text):
 return False
 if b'maxdeltachainspan' in opts:
 self._maxdeltachainspan = opts[b'maxdeltachainspan']
 if self._mmaplargeindex and b'mmapindexthreshold' in opts:
 mmapindexthreshold = opts[b'mmapindexthreshold']
 self.hassidedata = bool(opts.get(b'side-data', False))
-if self.hassidedata:
-self._flagprocessors[REVIDX_SIDEDATA] = sidedatautil.processors
 self._sparserevlog = bool(opts.get(b'sparse-revlog', False))
 withsparseread = bool(opts.get(b'with-sparse-read', False))
 # sparse-revlog forces sparse-read
 self._withsparseread = self._sparserevlog or withsparseread
 if b'sparse-read-density-threshold' in opts:
 return self.index[rev][0] & 0xFFFF
 def length(self, rev):
 return self.index[rev][1]
+def sidedata_length(self, rev):
+if self.version & 0xFFFF != REVLOGV2:
+return 0
+return self.index[rev][9]
 def rawsize(self, rev):
 """return the length of the uncompressed text for a given revision"""
 l = self.index[rev][2]
 if l >= 0:
 return l
 raise
 # Derived from index values.
 def end(self, rev):
-return self.start(rev) + self.length(rev)
+return self.start(rev) + self.length(rev) + self.sidedata_length(rev)
 def parents(self, node):
 i = self.index
 d = i[self.rev(node)]
 return i[d[5]][7], i[d[6]][7]  # map revisions to nodes inline
 if raw:
 return text, flagutil.processflagsraw(self, text, flags)
 elif operation == b'read':
 return flagutil.processflagsread(self, text, flags)
 else:  # write operation
-return flagutil.processflagswrite(self, text, flags, None)
+return flagutil.processflagswrite(self, text, flags)
 def revision(self, nodeorrev, _df=None, raw=False):
 """return an uncompressed revision of a given node or revision
 number.
 # ``rawtext`` is the text as stored inside the revlog. Might be the
 # revision or might need to be processed to retrieve the revision.
 rev, rawtext, validated = self._rawtext(node, rev, _df=_df)
+if self.version & 0xFFFF == REVLOGV2:
+if rev is None:
+rev = self.rev(node)
+sidedata = self._sidedata(rev)
+else:
+sidedata = {}
 if raw and validated:
 # if we don't want to process the raw text and that raw
 # text is cached, we can exit early.
-return rawtext, {}
+return rawtext, sidedata
 if rev is None:
 rev = self.rev(node)
 # the revlog's flag for this revision
 # (usually alter its state or content)
 flags = self.flags(rev)
 if validated and flags == REVIDX_DEFAULT_FLAGS:
 # no extra flags set, no flag processor runs, text = rawtext
-return rawtext, {}
+return rawtext, sidedata
-sidedata = {}
 if raw:
 validatehash = flagutil.processflagsraw(self, rawtext, flags)
 text = rawtext
 else:
-try:
+r = flagutil.processflagsread(self, rawtext, flags)
-r = flagutil.processflagsread(self, rawtext, flags)
+text, validatehash = r
-except error.SidedataHashError as exc:
-msg = _(b"integrity check failed on %s:%s sidedata key %d")
-msg %= (self.indexfile, pycompat.bytestr(rev), exc.sidedatakey)
-raise error.RevlogError(msg)
-text, validatehash, sidedata = r
 if validatehash:
 self.checkhash(text, node, rev=rev)
 if not validated:
 self._revisioncache = (node, rev, rawtext)
 bins = bins[1:]
 rawtext = mdiff.patches(basetext, bins)
 del basetext  # let us have a chance to free memory early
 return (rev, rawtext, False)
+def _sidedata(self, rev):
+"""Return the sidedata for a given revision number."""
+index_entry = self.index[rev]
+sidedata_offset = index_entry[8]
+sidedata_size = index_entry[9]
+if self._inline:
+sidedata_offset += self._io.size * (1 + rev)
+if sidedata_size == 0:
+return {}
+segment = self._getsegment(sidedata_offset, sidedata_size)
+sidedata = sidedatautil.deserialize_sidedata(segment)
+return sidedata
 def rawdata(self, nodeorrev, _df=None):
 """return an uncompressed raw data of a given node or revision number.
 _df - an existing file handle to read from. (internal-only)
 _(b"attempted to add linkrev -1 to %s") % self.indexfile
 )
 if sidedata is None:
 sidedata = {}
-flags = flags & ~REVIDX_SIDEDATA
 elif not self.hassidedata:
 raise error.ProgrammingError(
 _(b"trying to add sidedata to a revlog who don't support them")
 )
-else:
-flags |= REVIDX_SIDEDATA
 if flags:
 node = node or self.hash(text, p1, p2)
-rawtext, validatehash = flagutil.processflagswrite(
+rawtext, validatehash = flagutil.processflagswrite(self, text, flags)
-self, text, flags, sidedata=sidedata
-)
 # If the flag processor modifies the revision data, ignore any provided
 # cachedelta.
 if rawtext != text:
 cachedelta = None
 p2,
 node,
 flags,
 cachedelta=cachedelta,
 deltacomputer=deltacomputer,
+sidedata=sidedata,
 )
 def addrawrevision(
 self,
 rawtext,
 p2,
 node,
 flags,
 cachedelta=None,
 deltacomputer=None,
+sidedata=None,
 ):
 """add a raw revision with known flags, node and parents
 useful when reusing a revision not stored in this revlog (ex: received
 over wire, or read from an external bundle).
 """
 flags,
 cachedelta,
 ifh,
 dfh,
 deltacomputer=deltacomputer,
+sidedata=sidedata,
 )
 finally:
 if dfh:
 dfh.close()
 ifh.close()
 cachedelta,
 ifh,
 dfh,
 alwayscache=False,
 deltacomputer=None,
+sidedata=None,
 ):
 """internal function to add revisions to the log
 see addrevision for argument descriptions.
 revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
 deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
+if sidedata:
+serialized_sidedata = sidedatautil.serialize_sidedata(sidedata)
+sidedata_offset = offset + deltainfo.deltalen
+else:
+serialized_sidedata = b""
+# Don't store the offset if the sidedata is empty, that way
+# we can easily detect empty sidedata and they will be no different
+# than ones we manually add.
+sidedata_offset = 0
 e = (
 offset_type(offset, flags),
 deltainfo.deltalen,
 textlen,
 deltainfo.base,
 link,
 p1r,
 p2r,
 node,
-0,
+sidedata_offset,
-0,
+len(serialized_sidedata),
 )
 if self.version & 0xFFFF != REVLOGV2:
 e = e[:8]
 self.index.append(e)
 entry = self._io.packentry(e, self.node, self.version, curr)
 self._writeentry(
-transaction, ifh, dfh, entry, deltainfo.data, link, offset
+transaction,
+ifh,
+dfh,
+entry,
+deltainfo.data,
+link,
+offset,
+serialized_sidedata,
 )
 rawtext = btext[0]
 if alwayscache and rawtext is None:
 if type(rawtext) == bytes:  # only accept immutable objects
 self._revisioncache = (node, curr, rawtext)
 self._chainbasecache[curr] = deltainfo.chainbase
 return curr
-def _writeentry(self, transaction, ifh, dfh, entry, data, link, offset):
+def _writeentry(
+self, transaction, ifh, dfh, entry, data, link, offset, sidedata
+):
 # Files opened in a+ mode have inconsistent behavior on various
 # platforms. Windows requires that a file positioning call be made
 # when the file handle transitions between reads and writes. See
 # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
 # platforms, Python or the platform itself can be buggy. Some versions
 transaction.add(self.datafile, offset)
 transaction.add(self.indexfile, curr * len(entry))
 if data[0]:
 dfh.write(data[0])
 dfh.write(data[1])
+if sidedata:
+dfh.write(sidedata)
 ifh.write(entry)
 else:
 offset += curr * self._io.size
 transaction.add(self.indexfile, offset)
 ifh.write(entry)
 ifh.write(data[0])
 ifh.write(data[1])
+if sidedata:
+ifh.write(sidedata)
 self._enforceinlinesize(transaction, ifh)
 nodemaputil.setup_persistent_nodemap(transaction, self)
 def addgroup(
 self,

Mercurial > public > mercurial-scm > hg

comparison mercurial/revlog.py @ 46709:3d740058b467