# revlog.py - storage back-end for mercurial

#

# Copyright 2005-2007 Matt Mackall <mpm@selenic.com>

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

"""Storage back-end for Mercurial.

This provides efficient delta storage with O(1) retrieve and append

and O(changes) merge between branches.

"""

from __future__ import absolute_import

import collections

import errno

import hashlib

import os

import struct

import zlib

# import stuff from node for others to import from revlog

from .node import (

    bin,

    hex,

    nullid,

    nullrev,

)

from .i18n import _

from . import (

    ancestor,

    error,

    mdiff,

    parsers,

    templatefilters,

    util,

)

_pack = struct.pack

_unpack = struct.unpack

_compress = zlib.compress

_decompress = zlib.decompress

# revlog header flags

REVLOGV0 = 0

REVLOGNG = 1

REVLOGNGINLINEDATA = (1 << 16)

REVLOGGENERALDELTA = (1 << 17)

REVLOG_DEFAULT_FLAGS = REVLOGNGINLINEDATA

REVLOG_DEFAULT_FORMAT = REVLOGNG

REVLOG_DEFAULT_VERSION = REVLOG_DEFAULT_FORMAT | REVLOG_DEFAULT_FLAGS

REVLOGNG_FLAGS = REVLOGNGINLINEDATA | REVLOGGENERALDELTA

# revlog index flags

REVIDX_ISCENSORED = (1 << 15) # revision has censor metadata, must be verified

REVIDX_DEFAULT_FLAGS = 0

# max size of revlog with inline data

_maxinline = 131072

_chunksize = 1048576

RevlogError = error.RevlogError

LookupError = error.LookupError

CensoredNodeError = error.CensoredNodeError

def getoffset(q):

    return int(q >> 16)

def gettype(q):

    return int(q & 0xFFFF)

def offset_type(offset, type):

    if (type & ~REVIDX_KNOWN_FLAGS) != 0:

        raise ValueError('unknown revlog index flags')

    return long(long(offset) << 16 | type)

_nullhash = hashlib.sha1(nullid)

def hash(text, p1, p2):

    """generate a hash from the given text and its parent hashes

    This hash combines both the current file contents and its history

    in a manner that makes it easy to distinguish nodes with the same

    content in the revision graph.

    """

    # As of now, if one of the parent node is null, p2 is null

    if p2 == nullid:

        # deep copy of a hash is faster than creating one

        s = _nullhash.copy()

        s.update(p1)

    else:

        # none of the parent nodes are nullid

        l = [p1, p2]

        l.sort()

        s = hashlib.sha1(l[0])

        s.update(l[1])

    s.update(text)

    return s.digest()

def decompress(bin):

    """ decompress the given input """

    if not bin:

        return bin

    t = bin[0]

    if t == '\0':

        return bin

    if t == 'x':

        try:

            return _decompress(bin)

        except zlib.error as e:

            raise RevlogError(_("revlog decompress error: %s") % str(e))

    if t == 'u':

        return util.buffer(bin, 1)

    raise RevlogError(_("unknown compression type %r") % t)

# index v0:

#  4 bytes: offset

#  4 bytes: compressed length

#  4 bytes: base rev

#  4 bytes: link rev

# 20 bytes: parent 1 nodeid

# 20 bytes: parent 2 nodeid

# 20 bytes: nodeid

indexformatv0 = ">4l20s20s20s"

class revlogoldio(object):

    def __init__(self):

        self.size = struct.calcsize(indexformatv0)

    def parseindex(self, data, inline):

        s = self.size

        index = []

        nodemap = {nullid: nullrev}

        n = off = 0

        l = len(data)

        while off + s <= l:

            cur = data[off:off + s]

            off += s

            e = _unpack(indexformatv0, cur)

            # transform to revlogv1 format

            e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],

                  nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])

            index.append(e2)

            nodemap[e[6]] = n

            n += 1

        # add the magic null revision at -1

        index.append((0, 0, 0, -1, -1, -1, -1, nullid))

        return index, nodemap, None

    def packentry(self, entry, node, version, rev):

        if gettype(entry[0]):

            raise RevlogError(_("index entry flags need RevlogNG"))

        e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],

              node(entry[5]), node(entry[6]), entry[7])

        return _pack(indexformatv0, *e2)

# index ng:

#  6 bytes: offset

#  2 bytes: flags

#  4 bytes: compressed length

#  4 bytes: uncompressed length

#  4 bytes: base rev

#  4 bytes: link rev

#  4 bytes: parent 1 rev

#  4 bytes: parent 2 rev

# 32 bytes: nodeid

indexformatng = ">Qiiiiii20s12x"

versionformat = ">I"

# corresponds to uncompressed length of indexformatng (2 gigs, 4-byte

# signed integer)

_maxentrysize = 0x7fffffff

class revlogio(object):

    def __init__(self):

        self.size = struct.calcsize(indexformatng)

    def parseindex(self, data, inline):

        # call the C implementation to parse the index data

        index, cache = parsers.parse_index2(data, inline)

        return index, getattr(index, 'nodemap', None), cache

    def packentry(self, entry, node, version, rev):

        p = _pack(indexformatng, *entry)

        if rev == 0:

            p = _pack(versionformat, version) + p[4:]

        return p

class revlog(object):

    """

    the underlying revision storage object

    A revlog consists of two parts, an index and the revision data.

    The index is a file with a fixed record size containing

    information on each revision, including its nodeid (hash), the

    nodeids of its parents, the position and offset of its data within

    the data file, and the revision it's based on. Finally, each entry

    contains a linkrev entry that can serve as a pointer to external

    data.

    The revision data itself is a linear collection of data chunks.

    Each chunk represents a revision and is usually represented as a

    delta against the previous chunk. To bound lookup time, runs of

    deltas are limited to about 2 times the length of the original

    version data. This makes retrieval of a version proportional to

    its size, or O(1) relative to the number of revisions.

    Both pieces of the revlog are written to in an append-only

    fashion, which means we never need to rewrite a file to insert or

    remove data, and can use some simple techniques to avoid the need

    for locking while reading.

    If checkambig, indexfile is opened with checkambig=True at

    writing, to avoid file stat ambiguity.

    """

    def __init__(self, opener, indexfile, checkambig=False):

        """

        create a revlog object

        opener is a function that abstracts the file opening operation

        and can be used to implement COW semantics or the like.

        """

        self.indexfile = indexfile

        self.datafile = indexfile[:-2] + ".d"

        self.opener = opener

        #  When True, indexfile is opened with checkambig=True at writing, to

        #  avoid file stat ambiguity.

        self._checkambig = checkambig

        # 3-tuple of (node, rev, text) for a raw revision.

        self._cache = None

        # Maps rev to chain base rev.

        self._chainbasecache = util.lrucachedict(100)

        # 2-tuple of (offset, data) of raw data from the revlog at an offset.

        self._chunkcache = (0, '')

        # How much data to read and cache into the raw revlog data cache.

        self._chunkcachesize = 65536

        self._maxchainlen = None

        self._aggressivemergedeltas = False

        self.index = []

        # Mapping of partial identifiers to full nodes.

        self._pcache = {}

        # Mapping of revision integer to full node.

        self._nodecache = {nullid: nullrev}

        self._nodepos = None

        v = REVLOG_DEFAULT_VERSION

        opts = getattr(opener, 'options', None)

        if opts is not None:

            if 'revlogv1' in opts:

                if 'generaldelta' in opts:

                    v |= REVLOGGENERALDELTA

            else:

                v = 0

            if 'chunkcachesize' in opts:

                self._chunkcachesize = opts['chunkcachesize']

            if 'maxchainlen' in opts:

                self._maxchainlen = opts['maxchainlen']

            if 'aggressivemergedeltas' in opts:

                self._aggressivemergedeltas = opts['aggressivemergedeltas']

            self._lazydeltabase = bool(opts.get('lazydeltabase', False))

        if self._chunkcachesize <= 0:

            raise RevlogError(_('revlog chunk cache size %r is not greater '

                                'than 0') % self._chunkcachesize)

        elif self._chunkcachesize & (self._chunkcachesize - 1):

            raise RevlogError(_('revlog chunk cache size %r is not a power '

                                'of 2') % self._chunkcachesize)

        indexdata = ''

        self._initempty = True

        try:

            f = self.opener(self.indexfile)

            indexdata = f.read()

            f.close()

            if len(indexdata) > 0:

                v = struct.unpack(versionformat, indexdata[:4])[0]

                self._initempty = False

        except IOError as inst:

            if inst.errno != errno.ENOENT:

                raise

        self.version = v

        self._inline = v & REVLOGNGINLINEDATA

        self._generaldelta = v & REVLOGGENERALDELTA

        flags = v & ~0xFFFF

        fmt = v & 0xFFFF

        if fmt == REVLOGV0 and flags:

            raise RevlogError(_("index %s unknown flags %#04x for format v0")

                              % (self.indexfile, flags >> 16))

        elif fmt == REVLOGNG and flags & ~REVLOGNG_FLAGS:

            raise RevlogError(_("index %s unknown flags %#04x for revlogng")

                              % (self.indexfile, flags >> 16))

        elif fmt > REVLOGNG:

            raise RevlogError(_("index %s unknown format %d")

                              % (self.indexfile, fmt))

        self.storedeltachains = True

        self._io = revlogio()

        if self.version == REVLOGV0:

            self._io = revlogoldio()

        try:

            d = self._io.parseindex(indexdata, self._inline)

        except (ValueError, IndexError):

            raise RevlogError(_("index %s is corrupted") % (self.indexfile))

        self.index, nodemap, self._chunkcache = d

        if nodemap is not None:

            self.nodemap = self._nodecache = nodemap

        if not self._chunkcache:

            self._chunkclear()

        # revnum -> (chain-length, sum-delta-length)

        self._chaininfocache = {}

    def tip(self):

        return self.node(len(self.index) - 2)

    def __contains__(self, rev):

        return 0 <= rev < len(self)

    def __len__(self):

        return len(self.index) - 1

    def __iter__(self):

        return iter(xrange(len(self)))

    def revs(self, start=0, stop=None):

        """iterate over all rev in this revlog (from start to stop)"""

        step = 1

        if stop is not None:

            if start > stop:

                step = -1

            stop += step

        else:

            stop = len(self)

        return xrange(start, stop, step)

    @util.propertycache

    def nodemap(self):

        self.rev(self.node(0))

        return self._nodecache

    def hasnode(self, node):

        try:

            self.rev(node)

            return True

        except KeyError:

            return False

    def clearcaches(self):

        self._cache = None

        self._chainbasecache.clear()

        self._chunkcache = (0, '')

        self._pcache = {}

        try:

            self._nodecache.clearcaches()

        except AttributeError:

            self._nodecache = {nullid: nullrev}

            self._nodepos = None

    def rev(self, node):

        try:

            return self._nodecache[node]

        except TypeError:

            raise

        except RevlogError:

            # parsers.c radix tree lookup failed

            raise LookupError(node, self.indexfile, _('no node'))

        except KeyError:

            # pure python cache lookup failed

            n = self._nodecache

            i = self.index

            p = self._nodepos

            if p is None:

                p = len(i) - 2

            for r in xrange(p, -1, -1):

                v = i[r][7]

                n[v] = r

                if v == node:

                    self._nodepos = r - 1

                    return r

            raise LookupError(node, self.indexfile, _('no node'))

    # Accessors for index entries.

    # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes

    # are flags.

    def start(self, rev):

        return int(self.index[rev][0] >> 16)

    def flags(self, rev):

        return self.index[rev][0] & 0xFFFF

    def length(self, rev):

        return self.index[rev][1]

    def rawsize(self, rev):

        """return the length of the uncompressed text for a given revision"""

        l = self.index[rev][2]

        if l >= 0:

            return l

        t = self.revision(self.node(rev))

        return len(t)

    size = rawsize

    def chainbase(self, rev):

        base = self._chainbasecache.get(rev)

        if base is not None:

            return base

        index = self.index

        base = index[rev][3]

        while base != rev:

            rev = base

            base = index[rev][3]

        self._chainbasecache[rev] = base

        return base

    def linkrev(self, rev):

        return self.index[rev][4]

    def parentrevs(self, rev):

        return self.index[rev][5:7]

    def node(self, rev):

        return self.index[rev][7]

    # Derived from index values.

    def end(self, rev):

        return self.start(rev) + self.length(rev)

    def parents(self, node):

        i = self.index

        d = i[self.rev(node)]

        return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline

    def chainlen(self, rev):

        return self._chaininfo(rev)[0]

    def _chaininfo(self, rev):