mercurial-scm/hg: mercurial/revlog.py comparison

comparison mercurial/revlog.py @ 52163:7346f93be7a4

revlog: add the glue to use the Rust `InnerRevlog` from Python The performance of this has been looked at for quite some time, and some workflows are actually quite a bit faster than with the Python + C code. However, we are still (up to 20%) slower in some crucial places like cloning certain repos, log, cat, which makes this an incomplete rewrite. This is mostly due to the high amount of overhead in Python <-> Rust FFI, especially around the VFS code. A future patch series will rewrite the VFS code in pure Rust, which should hopefully get us up to par with current perfomance, if not better in all important cases. This is a "save state" of sorts, as this is a ton of code, and I don't want to pile up even more things in a single review. Continuing to try to match the current performance will take an extremely long time, if it's not impossible, without the aforementioned VFS work.

author	Rapha?l Gom?s <rgomes@octobus.net>
date	Wed, 19 Jun 2024 19:10:49 +0200
parents	13815c9decd4
children	72bc29f01570

comparison

equal deleted inserted replaced

-:13815c9decd4
+:7346f93be7a4
 from __future__ import annotations
 import binascii
 import collections
 import contextlib
-import functools
 import io
 import os
 import struct
 import typing
 import weakref
 # Force pytype to use the non-vendored package
 if typing.TYPE_CHECKING:
 # noinspection PyPackageRequirements
 import attr
+from .pure.parsers import BaseIndexObject
 from . import (
 ancestor,
 dagop,
 error,
 feature_config,
 chunk_cache,
 default_compression_header,
 ):
 self.opener = opener
-self.index = index
+self.index: BaseIndexObject = index
 self.index_file = index_file
 self.data_file = data_file
 self.sidedata_file = sidedata_file
 self.inline = inline
 ``stoprev`` was hit.
 """
 generaldelta = self.delta_config.general_delta
 # Try C implementation.
 try:
-return self.index.deltachain(rev, stoprev, generaldelta)
+return self.index.deltachain(
+rev, stoprev, generaldelta
+)  # pytype: disable=attribute-error
 except AttributeError:
 pass
 chain = []
 msg = b"not delay or divert found on this revlog"
 raise error.ProgrammingError(msg)
 return self.canonical_index_file
+if typing.TYPE_CHECKING:
+# Tell Pytype what kind of object we expect
+ProxyBase = BaseIndexObject
+else:
+ProxyBase = object
+class RustIndexProxy(ProxyBase):
+"""Wrapper around the Rust index to fake having direct access to the index.
+Rust enforces xor mutability (one mutable reference XOR 1..n non-mutable),
+so we can't expose the index from Rust directly, since the `InnerRevlog`
+already has ownership of the index. This object redirects all calls to the
+index through the Rust-backed `InnerRevlog` glue which defines all
+necessary forwarding methods.
+"""
+def __init__(self, inner):
+# Do not rename as it's being used to access the index from Rust
+self.inner = inner
+# TODO possibly write all index methods manually to save on overhead?
+def __getattr__(self, name):
+return getattr(self.inner, f"_index_{name}")
+# Magic methods need to be defined explicitely
+def __len__(self):
+return self.inner._index___len__()
+def __getitem__(self, key):
+return self.inner._index___getitem__(key)
+def __contains__(self, key):
+return self.inner._index___contains__(key)
+def __delitem__(self, key):
+return self.inner._index___delitem__(key)
+class RustVFSWrapper:
+"""Used to wrap a Python VFS to pass it to Rust to lower the overhead of
+calling back multiple times into Python.
+"""
+def __init__(self, inner):
+self.inner = inner
+def __call__(
+self,
+path: bytes,
+mode: bytes = b"rb",
+atomictemp=False,
+checkambig=False,
+):
+fd = self.inner.__call__(
+path=path, mode=mode, atomictemp=atomictemp, checkambig=checkambig
+)
+# Information that Rust needs to get ownership of the file that's
+# being opened.
+return (os.dup(fd.fileno()), fd._tempname if atomictemp else None)
+def __getattr__(self, name):
+return getattr(self.inner, name)
 class revlog:
 """
 the underlying revision storage object
 A revlog consists of two parts, an index and the revision data.
 self._nodemap_file = None
 self.postfix = postfix
 self._trypending = trypending
 self._try_split = try_split
 self._may_inline = may_inline
+self.uses_rust = False
 self.opener = opener
 if persistentnodemap:
 self._nodemap_file = nodemaputil.get_nodemap_file(self)
 assert target[0] in ALL_KINDS
 self.delta_config.upper_bound_comp = upperboundcomp
 # Maps rev to chain base rev.
 self._chainbasecache = util.lrucachedict(100)
-self.index = None
+self.index: Optional[BaseIndexObject] = None
 self._docket = None
 self._nodemap_docket = None
 # Mapping of partial identifiers to full nodes.
 self._pcache = {}
 # custom flags.
 self._flagprocessors = dict(flagutil.flagprocessors)
 # prevent nesting of addgroup
 self._adding_group = None
-chunk_cache = self._loadindex()
+index, chunk_cache = self._loadindex()
-self._load_inner(chunk_cache)
+self._load_inner(index, chunk_cache)
 self._concurrencychecker = concurrencychecker
 def _init_opts(self):
 """process options (from above/config) to setup associated default revlog mode
 and force_nodemap
 and parse_index_v1_nodemap is not None
 )
 use_rust_index = False
-if rustrevlog is not None and self._nodemap_file is not None:
+rust_applicable = self._nodemap_file is not None
+rust_applicable = rust_applicable or self.target[0] == KIND_FILELOG
+rust_applicable = rust_applicable and getattr(
+self.opener, "rust_compatible", True
+)
+if rustrevlog is not None and rust_applicable:
 # we would like to use the rust_index in all case, especially
 # because it is necessary for AncestorsIterator and LazyAncestors
 # since the 6.7 cycle.
 #
 # However, the performance impact of inconditionnaly building the
 # nodemap is currently a problem for non-persistent nodemap
 # repository.
 use_rust_index = True
+if self._format_version != REVLOGV1:
+use_rust_index = False
 self._parse_index = parse_index_v1
 if self._format_version == REVLOGV0:
 self._parse_index = revlogv0.parse_index_v0
 elif self._format_version == REVLOGV2:
 self._parse_index = parse_index_v2
 elif self._format_version == CHANGELOGV2:
 self._parse_index = parse_index_cl_v2
 elif devel_nodemap:
 self._parse_index = parse_index_v1_nodemap
-elif use_rust_index:
-self._parse_index = functools.partial(
+if use_rust_index:
-parse_index_v1_rust, default_header=new_header
+# Let the Rust code parse its own index
-)
+index, chunkcache = (index_data, None)
-try:
+self.uses_rust = True
-d = self._parse_index(index_data, self._inline)
+else:
-index, chunkcache = d
+try:
-use_nodemap = (
+d = self._parse_index(index_data, self._inline)
-not self._inline
+index, chunkcache = d
-and self._nodemap_file is not None
+self._register_nodemap_info(index)
-and hasattr(index, 'update_nodemap_data')
+except (ValueError, IndexError):
-)
+raise error.RevlogError(
-if use_nodemap:
+_(b"index %s is corrupted") % self.display_id
-nodemap_data = nodemaputil.persisted_data(self)
+)
-if nodemap_data is not None:
-docket = nodemap_data[0]
-if (
-len(d[0]) > docket.tip_rev
-and d[0][docket.tip_rev][7] == docket.tip_node
-):
-# no changelog tampering
-self._nodemap_docket = docket
-index.update_nodemap_data(*nodemap_data)
-except (ValueError, IndexError):
-raise error.RevlogError(
-_(b"index %s is corrupted") % self.display_id
-)
-self.index = index
 # revnum -> (chain-length, sum-delta-length)
 self._chaininfocache = util.lrucachedict(500)
-return chunkcache
+return index, chunkcache
-def _load_inner(self, chunk_cache):
+def _load_inner(self, index, chunk_cache):
 if self._docket is None:
 default_compression_header = None
 else:
 default_compression_header = self._docket.default_compression_header
-self._inner = _InnerRevlog(
+if self.uses_rust:
-opener=self.opener,
+self._inner = rustrevlog.InnerRevlog(
-index=self.index,
+opener=RustVFSWrapper(self.opener),
-index_file=self._indexfile,
+index_data=index,
-data_file=self._datafile,
+index_file=self._indexfile,
-sidedata_file=self._sidedatafile,
+data_file=self._datafile,
-inline=self._inline,
+sidedata_file=self._sidedatafile,
-data_config=self.data_config,
+inline=self._inline,
-delta_config=self.delta_config,
+data_config=self.data_config,
-feature_config=self.feature_config,
+delta_config=self.delta_config,
-chunk_cache=chunk_cache,
+feature_config=self.feature_config,
-default_compression_header=default_compression_header,
+chunk_cache=chunk_cache,
+default_compression_header=default_compression_header,
+revlog_type=self.target[0],
+)
+self.index = RustIndexProxy(self._inner)
+self._register_nodemap_info(self.index)
+self.uses_rust = True
+else:
+self._inner = _InnerRevlog(
+opener=self.opener,
+index=index,
+index_file=self._indexfile,
+data_file=self._datafile,
+sidedata_file=self._sidedatafile,
+inline=self._inline,
+data_config=self.data_config,
+delta_config=self.delta_config,
+feature_config=self.feature_config,
+chunk_cache=chunk_cache,
+default_compression_header=default_compression_header,
+)
+self.index = self._inner.index
+def _register_nodemap_info(self, index):
+use_nodemap = (
+not self._inline
+and self._nodemap_file is not None
+and hasattr(index, 'update_nodemap_data')
 )
+if use_nodemap:
+nodemap_data = nodemaputil.persisted_data(self)
+if nodemap_data is not None:
+docket = nodemap_data[0]
+if (
+len(index) > docket.tip_rev
+and index[docket.tip_rev][7] == docket.tip_node
+):
+# no changelog tampering
+self._nodemap_docket = docket
+index.update_nodemap_data(
+*nodemap_data
+)  # pytype: disable=attribute-error
 def get_revlog(self):
 """simple function to mirror API of other not-really-revlog API"""
 return self
 )
 if use_nodemap:
 nodemap_data = nodemaputil.persisted_data(self)
 if nodemap_data is not None:
 self._nodemap_docket = nodemap_data[0]
-self.index.update_nodemap_data(*nodemap_data)
+self.index.update_nodemap_data(
+*nodemap_data
+)  # pytype: disable=attribute-error
 def rev(self, node):
 """return the revision number associated with a <nodeid>"""
 try:
 return self.index.rev(node)
 return (orderedout, roots, heads)
 def headrevs(self, revs=None, stop_rev=None):
 if revs is None:
 return self.index.headrevs(None, stop_rev)
-assert stop_rev is None
 if rustdagop is not None and self.index.rust_ext_compat:
 return rustdagop.headrevs(self.index, revs)
 return dagop.headrevs(revs, self._uncheckedparentrevs)
 def headrevsdiff(self, start, stop):
 try:
-return self.index.headrevsdiff(start, stop)
+return self.index.headrevsdiff(
+start, stop
+)  # pytype: disable=attribute-error
 except AttributeError:
 return dagop.headrevsdiff(self._uncheckedparentrevs, start, stop)
 def computephases(self, roots):
-return self.index.computephasesmapsets(roots)
+return self.index.computephasesmapsets(
+roots
+)  # pytype: disable=attribute-error
 def _head_node_ids(self):
 try:
-return self.index.head_node_ids()
+return self.index.head_node_ids()  # pytype: disable=attribute-error
 except AttributeError:
 return [self.node(r) for r in self.headrevs()]
 def heads(self, start=None, stop=None):
 """return the list of all nodes that have no children
 return pycompat.maplist(self.node, ancs)
 def _commonancestorsheads(self, *revs):
 """calculate all the heads of the common ancestors of revs"""
 try:
-ancs = self.index.commonancestorsheads(*revs)
+ancs = self.index.commonancestorsheads(
+*revs
+)  # pytype: disable=attribute-error
 except (AttributeError, OverflowError):  # C implementation failed
 ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
 return ancs
 def isancestor(self, a, b):
 If includepath is True, return (<roots>::<heads>)."""
 try:
 return self.index.reachableroots2(
 minroot, heads, roots, includepath
-)
+)  # pytype: disable=attribute-error
 except AttributeError:
 return dagop._reachablerootspure(
 self.parentrevs, minroot, roots, heads, includepath
 )
 def ancestor(self, a, b):
 """calculate the "best" common ancestor of nodes a and b"""
 a, b = self.rev(a), self.rev(b)
 try:
-ancs = self.index.ancestors(a, b)
+ancs = self.index.ancestors(a, b)  # pytype: disable=attribute-error
 except (AttributeError, OverflowError):
 ancs = ancestor.ancestors(self.parentrevs, a, b)
 if ancs:
 # choose a consistent winner when there's a tie
 return min(map(self.node, ancs))
 def _partialmatch(self, id):
 # we don't care wdirfilenodeids as they should be always full hash
 maybewdir = self.nodeconstants.wdirhex.startswith(id)
 ambiguous = False
 try:
-partial = self.index.partialmatch(id)
+partial = self.index.partialmatch(
+id
+)  # pytype: disable=attribute-error
 if partial and self.hasnode(partial):
 if maybewdir:
 # single 'ff...' match in radix tree, ambiguous with wdir
 ambiguous = True
 else:
 if not maybewdir(prefix):
 return prefix
 if not getattr(self, 'filteredrevs', None):
 try:
-length = max(self.index.shortest(node), minlength)
+shortest = self.index.shortest(
+node
+)  # pytype: disable=attribute-error
+length = max(shortest, minlength)
 return disambiguate(hexnode, length)
 except error.RevlogError:
 if node != self.nodeconstants.wdirid:
 raise error.LookupError(
 node, self.display_id, _(b'no node')
 # rewrite the new index entries
 ifh.seek(startrev * self.index.entry_size)
 for i, e in enumerate(new_entries):
 rev = startrev + i
-self.index.replace_sidedata_info(rev, *e)
+self.index.replace_sidedata_info(
+rev, *e
+)  # pytype: disable=attribute-error
 packed = self.index.entry_binary(rev)
 if rev == 0 and self._docket is None:
 header = self._format_flags | self._format_version
 header = self.index.pack_header(header)
 packed = header + packed

Mercurial > public > mercurial-scm > hg

comparison mercurial/revlog.py @ 52163:7346f93be7a4