tests/testlib/ext-sidedata.py
author Rapha?l Gom?s <rgomes@octobus.net>
Fri, 19 Feb 2021 11:24:50 +0100
changeset 46718 ba8e508a8e69
parent 46709 3d740058b467
child 47012 d55b71393907
permissions -rw-r--r--
sidedata-exchange: rewrite sidedata on-the-fly whenever possible When a A exchanges with B, the difference of their supported sidedata categories is made, and the responsibility is always with the client to generated it: - If A pushes to B and B requires category `foo` that A does not have, A will need to generate it when sending it to B. - If A pulls from B and A needs category `foo`, it will generate `foo` before the end of the transaction. - Any category that is not required is removed. If peers are not compatible, abort. It is forbidden to rewrite sidedata for a rev that already has sidedata, since that would introduce unreachable (garbage) data in the data file, something we're not prepared for yet. Differential Revision: https://phab.mercurial-scm.org/D10032
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     1
# ext-sidedata.py - small extension to test the sidedata logic
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     2
#
46718
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
     3
# Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net>
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     4
#
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     5
# This software may be used and distributed according to the terms of the
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     6
# GNU General Public License version 2 or any later version.
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     7
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     8
from __future__ import absolute_import
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
     9
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    10
import hashlib
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    11
import struct
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    12
46113
59fa3890d40a node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents: 46046
diff changeset
    13
from mercurial.node import (
59fa3890d40a node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents: 46046
diff changeset
    14
    nullid,
59fa3890d40a node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents: 46046
diff changeset
    15
    nullrev,
59fa3890d40a node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents: 46046
diff changeset
    16
)
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    17
from mercurial import (
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    18
    extensions,
45373
d7dcc75a3eae localrepo: move requirements constant to requirements module
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43134
diff changeset
    19
    requirements,
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    20
    revlog,
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    21
)
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    22
46046
f105c49e89cd upgrade: split actual upgrade code away from the main module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 45735
diff changeset
    23
from mercurial.upgrade_utils import engine as upgrade_engine
f105c49e89cd upgrade: split actual upgrade code away from the main module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 45735
diff changeset
    24
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    25
from mercurial.revlogutils import sidedata
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    26
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    27
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    28
def wrapaddrevision(
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    29
    orig, self, text, transaction, link, p1, p2, *args, **kwargs
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    30
):
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    31
    if kwargs.get('sidedata') is None:
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    32
        kwargs['sidedata'] = {}
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    33
    sd = kwargs['sidedata']
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    34
    ## let's store some arbitrary data just for testing
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    35
    # text length
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    36
    sd[sidedata.SD_TEST1] = struct.pack('>I', len(text))
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    37
    # and sha2 hashes
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    38
    sha256 = hashlib.sha256(text).digest()
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    39
    sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    40
    return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    41
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    42
46709
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    43
def wrap_revisiondata(orig, self, nodeorrev, *args, **kwargs):
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    44
    text, sd = orig(self, nodeorrev, *args, **kwargs)
43134
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    45
    if getattr(self, 'sidedatanocheck', False):
46709
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    46
        return text, sd
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    47
    if self.version & 0xFFFF != 2:
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    48
        return text, sd
46113
59fa3890d40a node: import symbols explicitly
Joerg Sonnenberger <joerg@bec.de>
parents: 46046
diff changeset
    49
    if nodeorrev != nullrev and nodeorrev != nullid:
46718
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    50
        cat1 = sd.get(sidedata.SD_TEST1)
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    51
        if cat1 is not None and len(text) != struct.unpack('>I', cat1)[0]:
43042
03e769278ef3 sidedata: check that the sidedata safely roundtrip
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43040
diff changeset
    52
            raise RuntimeError('text size mismatch')
46718
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    53
        expected = sd.get(sidedata.SD_TEST2)
43042
03e769278ef3 sidedata: check that the sidedata safely roundtrip
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43040
diff changeset
    54
        got = hashlib.sha256(text).digest()
46718
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    55
        if expected is not None and got != expected:
43042
03e769278ef3 sidedata: check that the sidedata safely roundtrip
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43040
diff changeset
    56
            raise RuntimeError('sha256 mismatch')
46709
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    57
    return text, sd
43042
03e769278ef3 sidedata: check that the sidedata safely roundtrip
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43040
diff changeset
    58
43076
2372284d9457 formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents: 43042
diff changeset
    59
43134
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    60
def wrapgetsidedatacompanion(orig, srcrepo, dstrepo):
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    61
    sidedatacompanion = orig(srcrepo, dstrepo)
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    62
    addedreqs = dstrepo.requirements - srcrepo.requirements
45373
d7dcc75a3eae localrepo: move requirements constant to requirements module
Pulkit Goyal <7895pulkit@gmail.com>
parents: 43134
diff changeset
    63
    if requirements.SIDEDATA_REQUIREMENT in addedreqs:
43134
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    64
        assert sidedatacompanion is None  # deal with composition later
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    65
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    66
        def sidedatacompanion(revlog, rev):
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    67
            update = {}
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    68
            revlog.sidedatanocheck = True
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    69
            try:
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    70
                text = revlog.revision(rev)
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    71
            finally:
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    72
                del revlog.sidedatanocheck
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    73
            ## let's store some arbitrary data just for testing
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    74
            # text length
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    75
            update[sidedata.SD_TEST1] = struct.pack('>I', len(text))
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    76
            # and sha2 hashes
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    77
            sha256 = hashlib.sha256(text).digest()
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    78
            update[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
45735
edf4fa06df94 upgrade: allow sidedata upgrade to modify revision flag
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 45373
diff changeset
    79
            return False, (), update, 0, 0
43134
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    80
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    81
    return sidedatacompanion
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    82
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    83
43040
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    84
def extsetup(ui):
ba4072c0a911 sidedata: test we can successfully write sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents:
diff changeset
    85
    extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)
46709
3d740058b467 sidedata: move to new sidedata storage in revlogv2
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46113
diff changeset
    86
    extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata)
43134
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    87
    extensions.wrapfunction(
46046
f105c49e89cd upgrade: split actual upgrade code away from the main module
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 45735
diff changeset
    88
        upgrade_engine, 'getsidedatacompanion', wrapgetsidedatacompanion
43134
75ad8af9c95e upgrade: allow upgrade to repository using sidedata
Pierre-Yves David <pierre-yves.david@octobus.net>
parents: 43076
diff changeset
    89
    )
46718
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    90
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    91
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    92
def reposetup(ui, repo):
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    93
    # We don't register sidedata computers because we don't care within these
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    94
    # tests
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    95
    repo.register_wanted_sidedata(sidedata.SD_TEST1)
ba8e508a8e69 sidedata-exchange: rewrite sidedata on-the-fly whenever possible
Rapha?l Gom?s <rgomes@octobus.net>
parents: 46709
diff changeset
    96
    repo.register_wanted_sidedata(sidedata.SD_TEST2)