Mercurial > public > mercurial-scm > hg-stable

comparison mercurial/revlogutils/nodemap.py @ 44397:f7459da77f23

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
nodemap: introduce an option to use mmap to read the nodemap mapping The performance and memory benefit is much greater if we don't have to copy all the data in memory for each information. So we introduce an option (on by default) to read the data using mmap. This changeset is the last one definition the API for index support nodemap data. (they have to be able to use the mmaping). Below are some benchmark comparing the best we currently have in 5.3 with the final step of this series (using the persistent nodemap implementation in Rust). The benchmark run `hg perfindex` with various revset and the following variants: Before: * do not use the persistent nodemap * use the CPython implementation of the index for nodemap * use mmapping of the changelog index After: * use the MixedIndex Rust code, with the NodeTree object for nodemap access (still in review) * use the persistent nodemap data from disk * access the persistent nodemap data through mmap * use mmapping of the changelog index The persistent nodemap greatly speed up most operation on very large repositories. Some of the previously very fast lookup end up a bit slower because the persistent nodemap has to be setup. However the absolute slowdown is very small and won't matters in the big picture. Here are some numbers (in seconds) for the reference copy of mozilla-try: Revset Before After abs-change speedup -10000: 0.004622 0.005532 0.000910 ? 0.83 -10: 0.000050 0.000132 0.000082 ? 0.37 tip 0.000052 0.000085 0.000033 ? 0.61 0 + (-10000:) 0.028222 0.005337 -0.022885 ? 5.29 0 0.023521 0.000084 -0.023437 ? 280.01 (-10000:) + 0 0.235539 0.005308 -0.230231 ? 44.37 (-10:) + :9 0.232883 0.000180 -0.232703 ?1293.79 (-10000:) + (:99) 0.238735 0.005358 -0.233377 ? 44.55 :99 + (-10000:) 0.317942 0.005593 -0.312349 ? 56.84 :9 + (-10:) 0.313372 0.000179 -0.313193 ?1750.68 :9 0.316450 0.000143 -0.316307 ?2212.93 On smaller repositories, the cost of nodemap related operation is not as big, so the win is much more modest. Yet it helps shaving a handful of millisecond here and there. Here are some numbers (in seconds) for the reference copy of mercurial: Revset Before After abs-change speedup -10: 0.000065 0.000097 0.000032 ? 0.67 tip 0.000063 0.000078 0.000015 ? 0.80 0 0.000561 0.000079 -0.000482 ? 7.10 -10000: 0.004609 0.003648 -0.000961 ? 1.26 0 + (-10000:) 0.005023 0.003715 -0.001307 ? 1.35 (-10:) + :9 0.002187 0.000108 -0.002079 ?20.25 (-10000:) + 0 0.006252 0.003716 -0.002536 ? 1.68 (-10000:) + (:99) 0.006367 0.003707 -0.002660 ? 1.71 :9 + (-10:) 0.003846 0.000110 -0.003736 ?34.96 :9 0.003854 0.000099 -0.003755 ?38.92 :99 + (-10000:) 0.007644 0.003778 -0.003866 ? 2.02 Differential Revision: https://phab.mercurial-scm.org/D7894
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Tue, 11 Feb 2020 11:18:52 +0100
parents 6ecc34b31137
children 336ec75ed1ac
comparison
equal deleted inserted replaced
44396:c089a0947f3e 44397:f7459da77f23
6 # This software may be used and distributed according to the terms of the 6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version. 7 # GNU General Public License version 2 or any later version.
8 8
9 from __future__ import absolute_import 9 from __future__ import absolute_import
10 10
11 import errno
11 import os 12 import os
12 import re 13 import re
13 import struct 14 import struct
14 15
15 from .. import ( 16 from .. import (
43 docket.tip_rev = tip_rev 44 docket.tip_rev = tip_rev
44 docket.data_length = data_length 45 docket.data_length = data_length
45 docket.data_unused = data_unused 46 docket.data_unused = data_unused
46 47
47 filename = _rawdata_filepath(revlog, docket) 48 filename = _rawdata_filepath(revlog, docket)
48 data = revlog.opener.tryread(filename) 49 use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
50 try:
51 with revlog.opener(filename) as fd:
52 if use_mmap:
53 data = util.buffer(util.mmapread(fd, data_length))
54 else:
55 data = fd.read(data_length)
56 except OSError as e:
57 if e.errno != errno.ENOENT:
58 raise
49 if len(data) < data_length: 59 if len(data) < data_length:
50 return None 60 return None
51 elif len(data) > data_length:
52 data = data[:data_length]
53 return docket, data 61 return docket, data
54 62
55 63
56 def setup_persistent_nodemap(tr, revlog): 64 def setup_persistent_nodemap(tr, revlog):
57 """Install whatever is needed transaction side to persist a nodemap on disk 65 """Install whatever is needed transaction side to persist a nodemap on disk
79 msg = "calling persist nodemap on a revlog without the feature enableb" 87 msg = "calling persist nodemap on a revlog without the feature enableb"
80 raise error.ProgrammingError(msg) 88 raise error.ProgrammingError(msg)
81 89
82 can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental") 90 can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
83 ondisk_docket = revlog._nodemap_docket 91 ondisk_docket = revlog._nodemap_docket
92 feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
93 use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
84 94
85 data = None 95 data = None
86 # first attemp an incremental update of the data 96 # first attemp an incremental update of the data
87 if can_incremental and ondisk_docket is not None: 97 if can_incremental and ondisk_docket is not None:
88 target_docket = revlog._nodemap_docket.copy() 98 target_docket = revlog._nodemap_docket.copy()
95 data = None 105 data = None
96 else: 106 else:
97 datafile = _rawdata_filepath(revlog, target_docket) 107 datafile = _rawdata_filepath(revlog, target_docket)
98 # EXP-TODO: if this is a cache, this should use a cache vfs, not a 108 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
99 # store vfs 109 # store vfs
110 new_length = target_docket.data_length + len(data)
100 with revlog.opener(datafile, b'r+') as fd: 111 with revlog.opener(datafile, b'r+') as fd:
101 fd.seek(target_docket.data_length) 112 fd.seek(target_docket.data_length)
102 fd.write(data) 113 fd.write(data)
103 fd.seek(0) 114 if feed_data:
104 new_data = fd.read(target_docket.data_length + len(data)) 115 if use_mmap:
105 target_docket.data_length += len(data) 116 fd.seek(0)
117 new_data = fd.read(new_length)
118 else:
119 fd.flush()
120 new_data = util.buffer(util.mmapread(fd, new_length))
121 target_docket.data_length = new_length
106 target_docket.data_unused += data_changed_count 122 target_docket.data_unused += data_changed_count
107 123
108 if data is None: 124 if data is None:
109 # otherwise fallback to a full new export 125 # otherwise fallback to a full new export
110 target_docket = NodeMapDocket() 126 target_docket = NodeMapDocket()
113 data = revlog.index.nodemap_data_all() 129 data = revlog.index.nodemap_data_all()
114 else: 130 else:
115 data = persistent_data(revlog.index) 131 data = persistent_data(revlog.index)
116 # EXP-TODO: if this is a cache, this should use a cache vfs, not a 132 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
117 # store vfs 133 # store vfs
118 new_data = data 134 with revlog.opener(datafile, b'w+') as fd:
119 with revlog.opener(datafile, b'w') as fd:
120 fd.write(data) 135 fd.write(data)
136 if feed_data:
137 if use_mmap:
138 new_data = data
139 else:
140 fd.flush()
141 new_data = util.buffer(util.mmapread(fd, len(data)))
121 target_docket.data_length = len(data) 142 target_docket.data_length = len(data)
122 target_docket.tip_rev = revlog.tiprev() 143 target_docket.tip_rev = revlog.tiprev()
123 # EXP-TODO: if this is a cache, this should use a cache vfs, not a 144 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
124 # store vfs 145 # store vfs
125 with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp: 146 with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
126 fp.write(target_docket.serialize()) 147 fp.write(target_docket.serialize())
127 revlog._nodemap_docket = target_docket 148 revlog._nodemap_docket = target_docket
128 if util.safehasattr(revlog.index, "update_nodemap_data"): 149 if feed_data:
129 revlog.index.update_nodemap_data(target_docket, new_data) 150 revlog.index.update_nodemap_data(target_docket, new_data)
130 151
131 # EXP-TODO: if the transaction abort, we should remove the new data and 152 # EXP-TODO: if the transaction abort, we should remove the new data and
132 # reinstall the old one. 153 # reinstall the old one.
133 154