Mercurial > public > mercurial-scm > hg-stable

changeset 53028:89ab2459f62a stable 6.9.3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
stream-bundle: properly process requirements When the manifest bundle constain some known requirement that does not affect the stream clone, we used to crash. However since we know them and know they don't affect the stream clone, we can ignore them. Mozilla generated such buggy manifest bundle for a time which allowed us to catch this error. The issue was not caught until 961900fbd67c (released in 6.9.2) as the requirements information were ignored for stream-v2 until then. We fix the issue, refactor the code for robustness and adds more tests to better catch this kind of issue in the future.
author Pierre-Yves David <pierre-yves.david@octobus.net>
date Wed, 05 Mar 2025 11:41:45 +0100
parents 0570351db4e7
children 687f5c5e9370
files mercurial/bundlecaches.py mercurial/requirements.py mercurial/streamclone.py tests/test-persistent-nodemap.t tests/test-stream-bundle-v2.t
diffstat 5 files changed, 129 insertions(+), 33 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial/bundlecaches.py	Tue Feb 25 20:21:43 2025 -0500
+++ b/mercurial/bundlecaches.py	Wed Mar 05 11:41:45 2025 +0100
@@ -323,11 +323,18 @@
     # repo supports and error if the bundle isn't compatible.
     if b'requirements' in params:
         requirements = set(cast(bytes, params[b'requirements']).split(b','))
-        missingreqs = requirements - requirementsmod.STREAM_FIXED_REQUIREMENTS
-        if missingreqs:
+        relevant_reqs = (
+            requirements - requirementsmod.STREAM_IGNORABLE_REQUIREMENTS
+        )
+        # avoid cycle (not great for pytype)
+        from . import localrepo
+
+        supported_req = localrepo.gathersupportedrequirements(repo.ui)
+        missing_reqs = relevant_reqs - supported_req
+        if missing_reqs:
             raise error.UnsupportedBundleSpecification(
                 _(b'missing support for repository features: %s')
-                % b', '.join(sorted(missingreqs))
+                % b', '.join(sorted(missing_reqs))
             )
 
     # Compute contentopts based on the version
--- a/mercurial/requirements.py	Tue Feb 25 20:21:43 2025 -0500
+++ b/mercurial/requirements.py	Wed Mar 05 11:41:45 2025 +0100
@@ -105,22 +105,13 @@
     DIRSTATE_V2_REQUIREMENT,
 }
 
-# List of requirement that impact "stream-clone" (and hardlink clone) and
-# cannot be changed in such cases.
-#
-# requirements not in this list are safe to be altered during stream-clone.
+# List of requirement that do not impact "stream-clone" (and hardlink clone) and
+# can be ignored in such case.
 #
-# note: the list is currently inherited from previous code and miss some relevant requirement while containing some irrelevant ones.
-STREAM_FIXED_REQUIREMENTS = {
-    ARCHIVED_PHASE_REQUIREMENT,
-    BOOKMARKS_IN_STORE_REQUIREMENT,
-    CHANGELOGV2_REQUIREMENT,
-    COPIESSDC_REQUIREMENT,
-    GENERALDELTA_REQUIREMENT,
-    INTERNAL_PHASE_REQUIREMENT,
-    REVLOG_COMPRESSION_ZSTD,
-    REVLOGV1_REQUIREMENT,
-    REVLOGV2_REQUIREMENT,
-    SPARSEREVLOG_REQUIREMENT,
-    TREEMANIFEST_REQUIREMENT,
+# requirements not in this list safe to be altered during stream-clone.
+STREAM_IGNORABLE_REQUIREMENTS = WORKING_DIR_REQUIREMENTS | {
+    DOTENCODE_REQUIREMENT,  # abstracted by the vfs layer
+    FNCACHE_REQUIREMENT,  # abstracted by the vfs layer
+    STORE_REQUIREMENT,  # abstracted by the vfs layer
+    NODEMAP_REQUIREMENT,  # Have some special logic to handle that case
 }
--- a/mercurial/streamclone.py	Tue Feb 25 20:21:43 2025 -0500
+++ b/mercurial/streamclone.py	Wed Mar 05 11:41:45 2025 +0100
@@ -40,7 +40,7 @@
     configuration choice when possible.
     """
     requirements = set(default_requirements)
-    requirements -= requirementsmod.STREAM_FIXED_REQUIREMENTS
+    requirements & requirementsmod.STREAM_IGNORABLE_REQUIREMENTS
     requirements.update(streamed_requirements)
     return requirements
 
@@ -51,7 +51,7 @@
     This is used for advertising the stream options and to generate the actual
     stream content."""
     requiredformats = (
-        repo.requirements & requirementsmod.STREAM_FIXED_REQUIREMENTS
+        repo.requirements - requirementsmod.STREAM_IGNORABLE_REQUIREMENTS
     )
     return requiredformats
 
--- a/tests/test-persistent-nodemap.t	Tue Feb 25 20:21:43 2025 -0500
+++ b/tests/test-persistent-nodemap.t	Wed Mar 05 11:41:45 2025 +0100
@@ -913,9 +913,11 @@
 standard clone
 --------------
 
-The persistent nodemap should exist after a streaming clone
+The persistent nodemap should exist after a normal clone
 
   $ hg clone --pull --quiet -U test-repo standard-clone
+  $ hg debugformat -R standard-clone | grep persistent-nodemap
+  persistent-nodemap: yes
   $ ls -1 standard-clone/.hg/store/ | grep -E '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
@@ -929,6 +931,19 @@
   data-unused: 0
   data-unused: 0.000%
 
+standard clone, no nodemap requested
+-------------------------------------
+
+If persistent normal is requested to not be here, it should not exist after a normal clone
+
+  $ hg clone --pull --quiet -U test-repo standard-clone-no-nm \
+  >     --config format.use-persistent-nodemap=no
+  $ hg debugformat -R standard-clone-no-nm | grep persistent-nodemap
+  persistent-nodemap:  no
+  $ ls -1 standard-clone-no-nm/.hg/store/ | grep -E '00(changelog|manifest)(\.n|-.*\.nd)'
+  [1]
+  $ hg -R standard-clone-no-nm debugnodemap --metadata
+
 
 local clone
 ------------
@@ -936,6 +951,8 @@
 The persistent nodemap should exist after a streaming clone
 
   $ hg clone -U test-repo local-clone
+  $ hg debugformat -R local-clone | grep persistent-nodemap
+  persistent-nodemap: yes
   $ ls -1 local-clone/.hg/store/ | grep -E '00(changelog|manifest)(\.n|-.*\.nd)'
   00changelog-*.nd (glob)
   00changelog.n
@@ -949,6 +966,44 @@
   data-unused: 0
   data-unused: 0.000%
 
+stream clone
+------------
+
+  $ hg clone -U  --stream ssh://user@dummy/test-repo stream-clone --quiet
+  $ hg debugformat -R stream-clone | grep persistent-nodemap
+  persistent-nodemap: yes
+  $ ls -1 stream-clone/.hg/store/ | grep -E '00(changelog|manifest)(\.n|-.*\.nd)'
+  00changelog-*.nd (glob)
+  00changelog.n
+  00manifest-*.nd (glob)
+  00manifest.n
+  $ hg -R stream-clone debugnodemap --metadata
+  uid: * (glob)
+  tip-rev: 5005
+  tip-node: 90d5d3ba2fc47db50f712570487cb261a68c8ffe
+  data-length: 121088
+  data-unused: 0
+  data-unused: 0.000%
+
+stream requesting no persistent nodemap
+---------------------------------------
+
+Even if persistent nodemap affect the store, there is logic to stream clone
+without it.
+
+This helps client without supports for persistent nodemap.
+
+  $ hg clone -U --stream ssh://user@dummy/test-repo stream-clone-no-nm \
+  >     --config format.use-persistent-nodemap=no \
+  >     --config devel.persistent-nodemap=no \
+  >     --config revlog.persistent-nodemap.slow-path=no \
+  >     --quiet
+  $ hg debugformat -R stream-clone-no-nm | grep persistent-nodemap
+  persistent-nodemap:  no
+  $ ls -1 stream-clone-no-nm/.hg/store/ | grep -E '00(changelog|manifest)(\.n|-.*\.nd)'
+  [1]
+  $ hg -R stream-clone-no-nm debugnodemap --metadata
+
 Test various corruption case
 ============================
 
--- a/tests/test-stream-bundle-v2.t	Tue Feb 25 20:21:43 2025 -0500
+++ b/tests/test-stream-bundle-v2.t	Wed Mar 05 11:41:45 2025 +0100
@@ -97,20 +97,22 @@
   $ "$PYTHON" $TESTDIR/dumbhttp.py -p $HGPORT1 --pid http.pid
   $ cat http.pid >> $DAEMON_PIDS
 
+  $ cd ..
+
+
+Requirements filtering
+======================
+
+
+Unknown requirements
+--------------------
+
 Stream bundle spec with unknown requirements should be filtered out
 
-#if stream-v2
-  $ cat > .hg/clonebundles.manifest << EOF
-  > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=none-v2;stream=v2;requirements%3Drevlogv42
+  $ cat > main/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=none-v2;stream=$stream_version;requirements%3Drevlogv42
   > EOF
-#endif
-#if stream-v3
-  $ cat > .hg/clonebundles.manifest << EOF
-  > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=none-v2;stream=v3-exp;requirements%3Drevlogv42
-  > EOF
-#endif
 
-  $ cd ..
 
   $ hg clone -U http://localhost:$HGPORT stream-clone-unsupported-requirements
   no compatible clone bundles available on server; falling back to regular clone
@@ -122,7 +124,48 @@
   added 5 changesets with 5 changes to 5 files
   new changesets 426bada5c675:9bc730a19041 (5 drafts)
 
+known requirements
+------------------
+
+Stream bundle spec with known requirements should be filtered out
+
+
+
+  $ cat > main/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=none-v2;stream=$stream_version;requirements%3Dsparserevlog,revlogv1
+  > EOF
+
+  $ hg clone -U http://localhost:$HGPORT stream-clone-supported-requirements
+  applying clone bundle from http://localhost:$HGPORT1/bundle.hg
+  * to transfer* (glob)
+  transferred * in * seconds (*/sec) (glob)
+  finished applying clone bundle
+  searching for changes
+  no changes found
+
+
+known but irrelevant requirements
+---------------------------------
+
+As fncache and dotencode are abstracted by the vfs, they don't actually matters for streamclone
+
+  $ cat > main/.hg/clonebundles.manifest << EOF
+  > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=none-v2;stream=$stream_version;requirements%3Dshare-safe
+  > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=none-v2;stream=$stream_version;requirements%3Dshare-safe,fncache,dotencode
+  > EOF
+
+  $ hg clone -U http://localhost:$HGPORT stream-clone-ignorable-requirements
+  applying clone bundle from http://localhost:$HGPORT1/bundle.hg
+  * to transfer* (glob)
+  transferred * in * seconds (*/sec) (glob)
+  finished applying clone bundle
+  searching for changes
+  no changes found
+
+
+
 Test that we can apply the bundle as a stream clone bundle
+==========================================================
 
   $ cat > main/.hg/clonebundles.manifest << EOF
   > http://localhost:$HGPORT1/bundle.hg BUNDLESPEC=`hg debugbundle --spec main/bundle.hg`