# revlog.py - storage back-end for mercurial

# coding: utf8

#

# Copyright 2005-2007 Olivia Mackall <olivia@selenic.com>

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""Storage back-end for Mercurial.

This provides efficient delta storage with O(1) retrieve and append

and O(changes) merge between branches.

"""

import binascii

import collections

import contextlib

import io

import os

import struct

import weakref

import zlib

# import stuff from node for others to import from revlog

from .node import (

    bin,

    hex,

    nullrev,

    sha1nodeconstants,

    short,

    wdirrev,

)

from .i18n import _

from .revlogutils.constants import (

    ALL_KINDS,

    CHANGELOGV2,

    COMP_MODE_DEFAULT,

    COMP_MODE_INLINE,

    COMP_MODE_PLAIN,

    DELTA_BASE_REUSE_NO,

    DELTA_BASE_REUSE_TRY,

    ENTRY_RANK,

    FEATURES_BY_VERSION,

    FLAG_GENERALDELTA,

    FLAG_INLINE_DATA,

    INDEX_HEADER,

    KIND_CHANGELOG,

    KIND_FILELOG,

    RANK_UNKNOWN,

    REVLOGV0,

    REVLOGV1,

    REVLOGV1_FLAGS,

    REVLOGV2,

    REVLOGV2_FLAGS,

    REVLOG_DEFAULT_FLAGS,

    REVLOG_DEFAULT_FORMAT,

    REVLOG_DEFAULT_VERSION,

    SUPPORTED_FLAGS,

)

from .revlogutils.flagutil import (

    REVIDX_DEFAULT_FLAGS,

    REVIDX_ELLIPSIS,

    REVIDX_EXTSTORED,

    REVIDX_FLAGS_ORDER,

    REVIDX_HASCOPIESINFO,

    REVIDX_ISCENSORED,

    REVIDX_RAWTEXT_CHANGING_FLAGS,

)

from .thirdparty import attr

from . import (

    ancestor,

    dagop,

    error,

    mdiff,

    policy,

    pycompat,

    revlogutils,

    templatefilters,

    util,

)

from .interfaces import (

    repository,

    util as interfaceutil,

)

from .revlogutils import (

    deltas as deltautil,

    docket as docketutil,

    flagutil,

    nodemap as nodemaputil,

    randomaccessfile,

    revlogv0,

    rewrite,

    sidedata as sidedatautil,

)

from .utils import (

    storageutil,

    stringutil,

)

# blanked usage of all the name to prevent pyflakes constraints

# We need these name available in the module for extensions.

REVLOGV0

REVLOGV1

REVLOGV2

CHANGELOGV2

FLAG_INLINE_DATA

FLAG_GENERALDELTA

REVLOG_DEFAULT_FLAGS

REVLOG_DEFAULT_FORMAT

REVLOG_DEFAULT_VERSION

REVLOGV1_FLAGS

REVLOGV2_FLAGS

REVIDX_ISCENSORED

REVIDX_ELLIPSIS

REVIDX_HASCOPIESINFO

REVIDX_EXTSTORED

REVIDX_DEFAULT_FLAGS

REVIDX_FLAGS_ORDER

REVIDX_RAWTEXT_CHANGING_FLAGS

parsers = policy.importmod('parsers')

rustancestor = policy.importrust('ancestor')

rustdagop = policy.importrust('dagop')

rustrevlog = policy.importrust('revlog')

# Aliased for performance.

_zlibdecompress = zlib.decompress

# max size of inline data embedded into a revlog

_maxinline = 131072

# Flag processors for REVIDX_ELLIPSIS.

def ellipsisreadprocessor(rl, text):

    return text, False

def ellipsiswriteprocessor(rl, text):

    return text, False

def ellipsisrawprocessor(rl, text):

    return False

ellipsisprocessor = (

    ellipsisreadprocessor,

    ellipsiswriteprocessor,

    ellipsisrawprocessor,

)

def _verify_revision(rl, skipflags, state, node):

    """Verify the integrity of the given revlog ``node`` while providing a hook

    point for extensions to influence the operation."""

    if skipflags:

        state[b'skipread'].add(node)

    else:

        # Side-effect: read content and verify hash.

        rl.revision(node)

# True if a fast implementation for persistent-nodemap is available

#

# We also consider we have a "fast" implementation in "pure" python because

# people using pure don't really have performance consideration (and a

# wheelbarrow of other slowness source)

HAS_FAST_PERSISTENT_NODEMAP = rustrevlog is not None or hasattr(

    parsers, 'BaseIndexObject'

)

@interfaceutil.implementer(repository.irevisiondelta)

@attr.s(slots=True)

class revlogrevisiondelta:

    node = attr.ib()

    p1node = attr.ib()

    p2node = attr.ib()

    basenode = attr.ib()

    flags = attr.ib()

    baserevisionsize = attr.ib()

    revision = attr.ib()

    delta = attr.ib()

    sidedata = attr.ib()

    protocol_flags = attr.ib()

    linknode = attr.ib(default=None)

@interfaceutil.implementer(repository.iverifyproblem)

@attr.s(frozen=True)

class revlogproblem:

    warning = attr.ib(default=None)

    error = attr.ib(default=None)

    node = attr.ib(default=None)

def parse_index_v1(data, inline):

    # call the C implementation to parse the index data

    index, cache = parsers.parse_index2(data, inline)

    return index, cache

def parse_index_v2(data, inline):

    # call the C implementation to parse the index data

    index, cache = parsers.parse_index2(data, inline, format=REVLOGV2)

    return index, cache

def parse_index_cl_v2(data, inline):

    # call the C implementation to parse the index data

    index, cache = parsers.parse_index2(data, inline, format=CHANGELOGV2)

    return index, cache

if hasattr(parsers, 'parse_index_devel_nodemap'):

    def parse_index_v1_nodemap(data, inline):

        index, cache = parsers.parse_index_devel_nodemap(data, inline)

        return index, cache

else:

    parse_index_v1_nodemap = None

def parse_index_v1_mixed(data, inline):

    index, cache = parse_index_v1(data, inline)

    return rustrevlog.MixedIndex(index), cache

# corresponds to uncompressed length of indexformatng (2 gigs, 4-byte

# signed integer)

_maxentrysize = 0x7FFFFFFF

FILE_TOO_SHORT_MSG = _(

    b'cannot read from revlog %s;'

    b'  expected %d bytes from offset %d, data size is %d'

)

hexdigits = b'0123456789abcdefABCDEF'

class _Config:

    def copy(self):

        return self.__class__(**self.__dict__)

@attr.s()

class FeatureConfig(_Config):

    """Hold configuration values about the available revlog features"""

    # the default compression engine

    compression_engine = attr.ib(default=b'zlib')

    # compression engines options

    compression_engine_options = attr.ib(default=attr.Factory(dict))

    # can we use censor on this revlog

    censorable = attr.ib(default=False)

    # does this revlog use the "side data" feature

    has_side_data = attr.ib(default=False)

    # might remove rank configuration once the computation has no impact

    compute_rank = attr.ib(default=False)

    # parent order is supposed to be semantically irrelevant, so we

    # normally resort parents to ensure that the first parent is non-null,

    # if there is a non-null parent at all.

    # filelog abuses the parent order as flag to mark some instances of

    # meta-encoded files, so allow it to disable this behavior.

    canonical_parent_order = attr.ib(default=False)

    # can ellipsis commit be used

    enable_ellipsis = attr.ib(default=False)

    def copy(self):

        new = super().copy()

        new.compression_engine_options = self.compression_engine_options.copy()

        return new

@attr.s()

class DataConfig(_Config):

    """Hold configuration value about how the revlog data are read"""

    # should we try to open the "pending" version of the revlog

    try_pending = attr.ib(default=False)

    # should we try to open the "splitted" version of the revlog

    try_split = attr.ib(default=False)

    #  When True, indexfile should be opened with checkambig=True at writing,

    #  to avoid file stat ambiguity.

    check_ambig = attr.ib(default=False)

    # If true, use mmap instead of reading to deal with large index

    mmap_large_index = attr.ib(default=False)

    # how much data is large

    mmap_index_threshold = attr.ib(default=None)

    # How much data to read and cache into the raw revlog data cache.

    chunk_cache_size = attr.ib(default=65536)

    # Allow sparse reading of the revlog data

    with_sparse_read = attr.ib(default=False)

    # minimal density of a sparse read chunk

    sr_density_threshold = attr.ib(default=0.50)

    # minimal size of data we skip when performing sparse read

    sr_min_gap_size = attr.ib(default=262144)

    # are delta encoded against arbitrary bases.

    generaldelta = attr.ib(default=False)

@attr.s()

class DeltaConfig(_Config):

    """Hold configuration value about how new delta are computed

    Some attributes are duplicated from DataConfig to help havign each object

    self contained.

    """

    # can delta be encoded against arbitrary bases.

    general_delta = attr.ib(default=False)

    # Allow sparse writing of the revlog data

    sparse_revlog = attr.ib(default=False)

    # maximum length of a delta chain

    max_chain_len = attr.ib(default=None)

    # Maximum distance between delta chain base start and end

    max_deltachain_span = attr.ib(default=-1)

    # If `upper_bound_comp` is not None, this is the expected maximal gain from

    # compression for the data content.

    upper_bound_comp = attr.ib(default=None)

    # Should we try a delta against both parent

    delta_both_parents = attr.ib(default=True)

    # Test delta base candidate group by chunk of this maximal size.

    candidate_group_chunk_size = attr.ib(default=0)

    # Should we display debug information about delta computation

    debug_delta = attr.ib(default=False)

    # trust incoming delta by default

    lazy_delta = attr.ib(default=True)

    # trust the base of incoming delta by default

    lazy_delta_base = attr.ib(default=False)

class _InnerRevlog:

    """An inner layer of the revlog object

    That layer exist to be able to delegate some operation to Rust, its

    boundaries are arbitrary and based on what we can delegate to Rust.

    """

    def __init__(

        self,

        opener,

        index,

        index_file,

        data_file,

        sidedata_file,

        inline,

        data_config,

        chunk_cache,

    ):

        self.opener = opener

        self.index = index

        self.__index_file = index_file

        self.data_file = data_file

        self.sidedata_file = sidedata_file

        self.inline = inline

        self.data_config = data_config

        # index

        # 3-tuple of file handles being used for active writing.

        self._writinghandles = None

        self._segmentfile = randomaccessfile.randomaccessfile(

            self.opener,

            (self.index_file if self.inline else self.data_file),

            self.data_config.chunk_cache_size,

            chunk_cache,

        )

        self._segmentfile_sidedata = randomaccessfile.randomaccessfile(

            self.opener,

            self.sidedata_file,

            self.data_config.chunk_cache_size,

        )

    @property

    def index_file(self):

        return self.__index_file

    @index_file.setter

    def index_file(self, new_index_file):

        self.__index_file = new_index_file

        if self.inline:

            self._segmentfile.filename = new_index_file

    # Derived from index values.

    def start(self, rev):

        """the offset of the data chunk for this revision"""

        return int(self.index[rev][0] >> 16)

    def length(self, rev):

        """the length of the data chunk for this revision"""

        return self.index[rev][1]

    def end(self, rev):

        """the end of the data chunk for this revision"""

        return self.start(rev) + self.length(rev)

    @contextlib.contextmanager

    def reading(self):

        """Context manager that keeps data and sidedata files open for reading"""

        if len(self.index) == 0:

            yield  # nothing to be read

        else:

            with self._segmentfile.reading():

                with self._segmentfile_sidedata.reading():

                    yield

    @property

    def is_writing(self):

        """True is a writing context is open"""

        return self._writinghandles is not None

    @contextlib.contextmanager

    def writing(self, transaction, data_end=None, sidedata_end=None):

        """Open the revlog files for writing

        Add content to a revlog should be done within such context.

        """

        if self.is_writing:

            yield

        else:

            ifh = dfh = sdfh = None

            try:

                r = len(self.index)

                # opening the data file.

                dsize = 0

                if r:

                    dsize = self.end(r - 1)

                dfh = None

                if not self.inline:

                    try:

                        dfh = self.opener(self.data_file, mode=b"r+")

                        if data_end is None:

                            dfh.seek(0, os.SEEK_END)

                        else:

                            dfh.seek(data_end, os.SEEK_SET)

                    except FileNotFoundError:

                        dfh = self.opener(self.data_file, mode=b"w+")

                    transaction.add(self.data_file, dsize)

                if self.sidedata_file is not None:

                    assert sidedata_end is not None

                    # revlog-v2 does not inline, help Pytype

                    assert dfh is not None

                    try:

                        sdfh = self.opener(self.sidedata_file, mode=b"r+")

                        dfh.seek(sidedata_end, os.SEEK_SET)

                    except FileNotFoundError:

                        sdfh = self.opener(self.sidedata_file, mode=b"w+")

                    transaction.add(self.sidedata_file, sidedata_end)

                # opening the index file.

                isize = r * self.index.entry_size

                ifh = self.__index_write_fp()

                if self.inline:

                    transaction.add(self.index_file, dsize + isize)

                else:

                    transaction.add(self.index_file, isize)

                # exposing all file handle for writing.

                self._writinghandles = (ifh, dfh, sdfh)

                self._segmentfile.writing_handle = ifh if self.inline else dfh

                self._segmentfile_sidedata.writing_handle = sdfh

                yield

            finally:

                self._writinghandles = None

                self._segmentfile.writing_handle = None

                self._segmentfile_sidedata.writing_handle = None

                if dfh is not None:

                    dfh.close()

                if sdfh is not None:

                    sdfh.close()

                # closing the index file last to avoid exposing referent to

                # potential unflushed data content.

                if ifh is not None:

                    ifh.close()

    def __index_write_fp(self, index_end=None):

        """internal method to open the index file for writing