mercurial-scm/hg: mercurial/branchmap.py comparison

comparison mercurial/branchmap.py @ 51372:02e7d79edf62

branchmap: use mmap for faster revbranchcache loading A typical revbranchmap usage is: - load the entire revbranchmap file into memory - maybe do a few lookups - add a few bytes to it - write the addition to disk There's no reason to load the entire revbranchmap into memory. We can split it into a large immutable prefix and a mutable suffix, and then memorymap the prefix, thus saving all the useless loading. Benchmarking on some real-world pushes suggests that out of ~100s server-side push handling revbranchcache handling is responsible for: * ~7s with no change * ~1.3s with the change, without mmap * 0.04s with the change, with mmap

author	Arseniy Alekseyev <aalekseyev@janestreet.com>
date	Wed, 10 Jan 2024 18:58:42 +0000
parents	f4a0806081f2
children	40943970b7ae 7f7086a42b2b

comparison

equal deleted inserted replaced

-:54a75576287a
+:02e7d79edf62
 _rbcnodelen = 4
 _rbcbranchidxmask = 0x7FFFFFFF
 _rbccloseflag = 0x80000000
+class rbcrevs:
+"""a byte string consisting of an immutable prefix followed by a mutable suffix"""
+def __init__(self, revs):
+self._prefix = revs
+self._rest = bytearray()
+def __len__(self):
+return len(self._prefix) + len(self._rest)
+def unpack_record(self, rbcrevidx):
+if rbcrevidx < len(self._prefix):
+return unpack_from(_rbcrecfmt, util.buffer(self._prefix), rbcrevidx)
+else:
+return unpack_from(
+_rbcrecfmt,
+util.buffer(self._rest),
+rbcrevidx - len(self._prefix),
+)
+def make_mutable(self):
+if len(self._prefix) > 0:
+entirety = bytearray()
+entirety[:] = self._prefix
+entirety.extend(self._rest)
+self._rest = entirety
+self._prefix = bytearray()
+def truncate(self, pos):
+self.make_mutable()
+del self._rest[pos:]
+def pack_into(self, rbcrevidx, node, branchidx):
+if rbcrevidx < len(self._prefix):
+self.make_mutable()
+buf = self._rest
+start_offset = rbcrevidx - len(self._prefix)
+end_offset = start_offset + _rbcrecsize
+if len(self._rest) < end_offset:
+# bytearray doesn't allocate extra space at least in Python 3.7.
+# When multiple changesets are added in a row, precise resize would
+# result in quadratic complexity. Overallocate to compensate by
+# using the classic doubling technique for dynamic arrays instead.
+# If there was a gap in the map before, less space will be reserved.
+self._rest.extend(b'\0' * end_offset)
+return pack_into(
+_rbcrecfmt,
+buf,
+start_offset,
+node,
+branchidx,
+)
+def extend(self, extension):
+return self._rest.extend(extension)
+def slice(self, begin, end):
+if begin < len(self._prefix):
+acc = bytearray()
+acc[:] = self._prefix[begin:end]
+acc.extend(
+self._rest[begin - len(self._prefix) : end - len(self._prefix)]
+)
+return acc
+return self._rest[begin - len(self._prefix) : end - len(self._prefix)]
 class revbranchcache:
 """Persistent cache, mapping from revision number to branch name and close.
 This is a low level cache, independent of filtering.
 Branch names are stored in rbc-names in internal encoding separated by 0.
 def __init__(self, repo, readonly=True):
 assert repo.filtername is None
 self._repo = repo
 self._names = []  # branch names in local encoding with static index
-self._rbcrevs = bytearray()
+self._rbcrevs = rbcrevs(bytearray())
 self._rbcsnameslen = 0  # length of names read at _rbcsnameslen
 try:
 bndata = repo.cachevfs.read(_rbcnames)
 self._rbcsnameslen = len(bndata)  # for verification before writing
 if bndata:
 # don't try to use cache - fall back to the slow path
 self.branchinfo = self._branchinfo
 if self._names:
 try:
-data = repo.cachevfs.read(_rbcrevs)
+if repo.ui.configbool(b'format', b'mmap-revbranchcache'):
-self._rbcrevs[:] = data
+with repo.cachevfs(_rbcrevs) as fp:
+data = util.buffer(util.mmapread(fp))
+else:
+data = repo.cachevfs.read(_rbcrevs)
+self._rbcrevs = rbcrevs(data)
 except (IOError, OSError) as inst:
 repo.ui.debug(
 b"couldn't read revision branch cache: %s\n"
 % stringutil.forcebytestr(inst)
 )
 def _clear(self):
 self._rbcsnameslen = 0
 del self._names[:]
 self._rbcnamescount = 0
 self._rbcrevslen = len(self._repo.changelog)
-self._rbcrevs = bytearray(self._rbcrevslen * _rbcrecsize)
+self._rbcrevs = rbcrevs(bytearray(self._rbcrevslen * _rbcrecsize))
 util.clearcachedproperty(self, b'_namesreverse')
 @util.propertycache
 def _namesreverse(self):
 return {b: r for r, b in enumerate(self._names)}
 if len(self._rbcrevs) < rbcrevidx + _rbcrecsize:
 return self._branchinfo(rev)
 # fast path: extract data from cache, use it if node is matching
 reponode = changelog.node(rev)[:_rbcnodelen]
-cachenode, branchidx = unpack_from(
+cachenode, branchidx = self._rbcrevs.unpack_record(rbcrevidx)
-_rbcrecfmt, util.buffer(self._rbcrevs), rbcrevidx
-)
 close = bool(branchidx & _rbccloseflag)
 if close:
 branchidx &= _rbcbranchidxmask
 if cachenode == b'\0\0\0\0':
 pass
 self._repo.ui.debug(
 b"history modification detected - truncating "
 b"revision branch cache to revision %d\n" % rev
 )
 truncate = rbcrevidx + _rbcrecsize
-del self._rbcrevs[truncate:]
+self._rbcrevs.truncate(truncate)
 self._rbcrevslen = min(self._rbcrevslen, truncate)
 # fall back to slow path and make sure it will be written to disk
 return self._branchinfo(rev)
 def _setcachedata(self, rev, node, branchidx):
 """Writes the node's branch data to the in-memory cache data."""
 if rev == nullrev:
 return
 rbcrevidx = rev * _rbcrecsize
-requiredsize = rbcrevidx + _rbcrecsize
+self._rbcrevs.pack_into(rbcrevidx, node, branchidx)
-rbccur = len(self._rbcrevs)
-if rbccur < requiredsize:
-# bytearray doesn't allocate extra space at least in Python 3.7.
-# When multiple changesets are added in a row, precise resize would
-# result in quadratic complexity. Overallocate to compensate by
-# use the classic doubling technique for dynamic arrays instead.
-# If there was a gap in the map before, less space will be reserved.
-self._rbcrevs.extend(b'\0' * max(_rbcmininc, requiredsize))
-pack_into(_rbcrecfmt, self._rbcrevs, rbcrevidx, node, branchidx)
 self._rbcrevslen = min(self._rbcrevslen, rev)
 tr = self._repo.currenttransaction()
 if tr:
 tr.addfinalize(b'write-revbranchcache', self.write)
 if f.tell() != start:
 start = 0
 f.seek(start)
 f.truncate()
 end = revs * _rbcrecsize
-f.write(self._rbcrevs[start:end])
+f.write(self._rbcrevs.slice(start, end))
 self._rbcrevslen = revs

Mercurial > public > mercurial-scm > hg

comparison mercurial/branchmap.py @ 51372:02e7d79edf62