/*

 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.

 * All rights reserved.

 *

 * This source code is licensed under both the BSD-style license (found in the

 * LICENSE file in the root directory of this source tree) and the GPLv2 (found

 * in the COPYING file in the root directory of this source tree).

 * You may select, at your option, one of the above-listed licenses.

 */

/*-**************************************

*  Tuning parameters

****************************************/

#define MINRATIO 4   /* minimum nb of apparition to be selected in dictionary */

#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20)

#define ZDICT_MIN_SAMPLES_SIZE (ZDICT_CONTENTSIZE_MIN * MINRATIO)

/*-**************************************

*  Compiler Options

****************************************/

/* Unix Large Files support (>4GB) */

#define _FILE_OFFSET_BITS 64

#if (defined(__sun__) && (!defined(__LP64__)))   /* Sun Solaris 32-bits requires specific definitions */

#  define _LARGEFILE_SOURCE

#elif ! defined(__LP64__)                        /* No point defining Large file for 64 bit */

#  define _LARGEFILE64_SOURCE

#endif

/*-*************************************

*  Dependencies

***************************************/

#include <stdlib.h>        /* malloc, free */

#include <string.h>        /* memset */

#include <stdio.h>         /* fprintf, fopen, ftello64 */

#include <time.h>          /* clock */

#include "mem.h"           /* read */

#include "fse.h"           /* FSE_normalizeCount, FSE_writeNCount */

#define HUF_STATIC_LINKING_ONLY

#include "huf.h"           /* HUF_buildCTable, HUF_writeCTable */

#include "zstd_internal.h" /* includes zstd.h */

#include "xxhash.h"        /* XXH64 */

#include "divsufsort.h"

#ifndef ZDICT_STATIC_LINKING_ONLY

#  define ZDICT_STATIC_LINKING_ONLY

#endif

#include "zdict.h"

/*-*************************************

*  Constants

***************************************/

#define KB *(1 <<10)

#define MB *(1 <<20)

#define GB *(1U<<30)

#define DICTLISTSIZE_DEFAULT 10000

#define NOISELENGTH 32

static const int g_compressionLevel_default = 3;

static const U32 g_selectivity_default = 9;

/*-*************************************

*  Console display

***************************************/

#define DISPLAY(...)         { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }

#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); }    /* 0 : no display;   1: errors;   2: default;  3: details;  4: debug */

static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }

static void ZDICT_printHex(const void* ptr, size_t length)

{

    const BYTE* const b = (const BYTE*)ptr;

    size_t u;

    for (u=0; u<length; u++) {

        BYTE c = b[u];

        if (c<32 || c>126) c = '.';   /* non-printable char */

        DISPLAY("%c", c);

    }

}

/*-********************************************************

*  Helper functions

**********************************************************/

unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); }

const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }

unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)

{

    if (dictSize < 8) return 0;

    if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0;

    return MEM_readLE32((const char*)dictBuffer + 4);

}

/*-********************************************************

*  Dictionary training functions

**********************************************************/

static unsigned ZDICT_NbCommonBytes (size_t val)

{

    if (MEM_isLittleEndian()) {

        if (MEM_64bits()) {

#       if defined(_MSC_VER) && defined(_WIN64)

            unsigned long r = 0;

            _BitScanForward64( &r, (U64)val );

            return (unsigned)(r>>3);

#       elif defined(__GNUC__) && (__GNUC__ >= 3)

            return (__builtin_ctzll((U64)val) >> 3);

#       else

            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };

            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];

#       endif

        } else { /* 32 bits */

#       if defined(_MSC_VER)

            unsigned long r=0;

            _BitScanForward( &r, (U32)val );

            return (unsigned)(r>>3);

#       elif defined(__GNUC__) && (__GNUC__ >= 3)

            return (__builtin_ctz((U32)val) >> 3);

#       else

            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };

            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];

#       endif

        }

    } else {  /* Big Endian CPU */

        if (MEM_64bits()) {

#       if defined(_MSC_VER) && defined(_WIN64)

            unsigned long r = 0;

            _BitScanReverse64( &r, val );

            return (unsigned)(r>>3);

#       elif defined(__GNUC__) && (__GNUC__ >= 3)

            return (__builtin_clzll(val) >> 3);

#       else

            unsigned r;

            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */

            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }

            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }

            r += (!val);

            return r;

#       endif

        } else { /* 32 bits */

#       if defined(_MSC_VER)

            unsigned long r = 0;

            _BitScanReverse( &r, (unsigned long)val );

            return (unsigned)(r>>3);

#       elif defined(__GNUC__) && (__GNUC__ >= 3)

            return (__builtin_clz((U32)val) >> 3);

#       else

            unsigned r;

            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }

            r += (!val);

            return r;

#       endif

    }   }

}

/*! ZDICT_count() :

    Count the nb of common bytes between 2 pointers.

    Note : this function presumes end of buffer followed by noisy guard band.

*/

static size_t ZDICT_count(const void* pIn, const void* pMatch)

{

    const char* const pStart = (const char*)pIn;

    for (;;) {

        size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);

        if (!diff) {

            pIn = (const char*)pIn+sizeof(size_t);

            pMatch = (const char*)pMatch+sizeof(size_t);

            continue;

        }

        pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);

        return (size_t)((const char*)pIn - pStart);

    }

}

typedef struct {

    U32 pos;

    U32 length;

    U32 savings;

} dictItem;

static void ZDICT_initDictItem(dictItem* d)

{

    d->pos = 1;

    d->length = 0;

    d->savings = (U32)(-1);

}

#define LLIMIT 64          /* heuristic determined experimentally */

#define MINMATCHLENGTH 7   /* heuristic determined experimentally */

static dictItem ZDICT_analyzePos(

                       BYTE* doneMarks,

                       const int* suffix, U32 start,

                       const void* buffer, U32 minRatio, U32 notificationLevel)

{

    U32 lengthList[LLIMIT] = {0};

    U32 cumulLength[LLIMIT] = {0};

    U32 savings[LLIMIT] = {0};

    const BYTE* b = (const BYTE*)buffer;

    size_t maxLength = LLIMIT;

    size_t pos = suffix[start];

    U32 end = start;

    dictItem solution;

    /* init */

    memset(&solution, 0, sizeof(solution));

    doneMarks[pos] = 1;

    /* trivial repetition cases */

    if ( (MEM_read16(b+pos+0) == MEM_read16(b+pos+2))

       ||(MEM_read16(b+pos+1) == MEM_read16(b+pos+3))

       ||(MEM_read16(b+pos+2) == MEM_read16(b+pos+4)) ) {

        /* skip and mark segment */

        U16 const pattern16 = MEM_read16(b+pos+4);

        U32 u, patternEnd = 6;

        while (MEM_read16(b+pos+patternEnd) == pattern16) patternEnd+=2 ;

        if (b[pos+patternEnd] == b[pos+patternEnd-1]) patternEnd++;

        for (u=1; u<patternEnd; u++)

            doneMarks[pos+u] = 1;

        return solution;

    }

    /* look forward */

    {   size_t length;

        do {

            end++;

            length = ZDICT_count(b + pos, b + suffix[end]);

        } while (length >= MINMATCHLENGTH);

    }

    /* look backward */

    {   size_t length;

        do {

            length = ZDICT_count(b + pos, b + *(suffix+start-1));

            if (length >=MINMATCHLENGTH) start--;

        } while(length >= MINMATCHLENGTH);

    }

    /* exit if not found a minimum nb of repetitions */

    if (end-start < minRatio) {

        U32 idx;

        for(idx=start; idx<end; idx++)

            doneMarks[suffix[idx]] = 1;

        return solution;

    }

    {   int i;

        U32 searchLength;

        U32 refinedStart = start;

        U32 refinedEnd = end;

        DISPLAYLEVEL(4, "\n");

        DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u  ", (U32)(end-start), MINMATCHLENGTH, (U32)pos);

        DISPLAYLEVEL(4, "\n");

        for (searchLength = MINMATCHLENGTH ; ; searchLength++) {

            BYTE currentChar = 0;

            U32 currentCount = 0;

            U32 currentID = refinedStart;

            U32 id;

            U32 selectedCount = 0;

            U32 selectedID = currentID;

            for (id =refinedStart; id < refinedEnd; id++) {

                if (b[suffix[id] + searchLength] != currentChar) {

                    if (currentCount > selectedCount) {

                        selectedCount = currentCount;

                        selectedID = currentID;

                    }

                    currentID = id;

                    currentChar = b[ suffix[id] + searchLength];

                    currentCount = 0;

                }

                currentCount ++;

            }

            if (currentCount > selectedCount) {  /* for last */

                selectedCount = currentCount;

                selectedID = currentID;

            }

            if (selectedCount < minRatio)

                break;

            refinedStart = selectedID;

            refinedEnd = refinedStart + selectedCount;

        }

        /* evaluate gain based on new ref */

        start = refinedStart;

        pos = suffix[refinedStart];

        end = start;

        memset(lengthList, 0, sizeof(lengthList));

        /* look forward */

        {   size_t length;

            do {

                end++;

                length = ZDICT_count(b + pos, b + suffix[end]);

                if (length >= LLIMIT) length = LLIMIT-1;

                lengthList[length]++;

            } while (length >=MINMATCHLENGTH);

        }

        /* look backward */

        {   size_t length = MINMATCHLENGTH;

            while ((length >= MINMATCHLENGTH) & (start > 0)) {

                length = ZDICT_count(b + pos, b + suffix[start - 1]);

                if (length >= LLIMIT) length = LLIMIT - 1;

                lengthList[length]++;

                if (length >= MINMATCHLENGTH) start--;

            }

        }

        /* largest useful length */

        memset(cumulLength, 0, sizeof(cumulLength));

        cumulLength[maxLength-1] = lengthList[maxLength-1];

        for (i=(int)(maxLength-2); i>=0; i--)

            cumulLength[i] = cumulLength[i+1] + lengthList[i];

        for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break;

        maxLength = i;

        /* reduce maxLength in case of final into repetitive data */

        {   U32 l = (U32)maxLength;

            BYTE const c = b[pos + maxLength-1];

            while (b[pos+l-2]==c) l--;

            maxLength = l;

        }

        if (maxLength < MINMATCHLENGTH) return solution;   /* skip : no long-enough solution */

        /* calculate savings */

        savings[5] = 0;

        for (i=MINMATCHLENGTH; i<=(int)maxLength; i++)

            savings[i] = savings[i-1] + (lengthList[i] * (i-3));

        DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f)  \n",

                     (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength);

        solution.pos = (U32)pos;

        solution.length = (U32)maxLength;

        solution.savings = savings[maxLength];

        /* mark positions done */

        {   U32 id;

            for (id=start; id<end; id++) {

                U32 p, pEnd, length;

                U32 const testedPos = suffix[id];

                if (testedPos == pos)

                    length = solution.length;

                else {

                    length = (U32)ZDICT_count(b+pos, b+testedPos);

                    if (length > solution.length) length = solution.length;

                }

                pEnd = (U32)(testedPos + length);

                for (p=testedPos; p<pEnd; p++)

                    doneMarks[p] = 1;

    }   }   }

    return solution;

}

static int isIncluded(const void* in, const void* container, size_t length)

{

    const char* const ip = (const char*) in;

    const char* const into = (const char*) container;

    size_t u;

    for (u=0; u<length; u++) {  /* works because end of buffer is a noisy guard band */

        if (ip[u] != into[u]) break;

    }

    return u==length;

}

/*! ZDICT_tryMerge() :

    check if dictItem can be merged, do it if possible

    @return : id of destination elt, 0 if not merged

*/

static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const void* buffer)

{

    const U32 tableSize = table->pos;

    const U32 eltEnd = elt.pos + elt.length;

    const char* const buf = (const char*) buffer;

    /* tail overlap */

    U32 u; for (u=1; u<tableSize; u++) {

        if (u==eltNbToSkip) continue;

        if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) {  /* overlap, existing > new */

            /* append */

            U32 const addedLength = table[u].pos - elt.pos;

            table[u].length += addedLength;

            table[u].pos = elt.pos;

            table[u].savings += elt.savings * addedLength / elt.length;   /* rough approx */

            table[u].savings += elt.length / 8;    /* rough approx bonus */

            elt = table[u];

            /* sort : improve rank */

            while ((u>1) && (table[u-1].savings < elt.savings))

            table[u] = table[u-1], u--;

            table[u] = elt;

            return u;

    }   }

    /* front overlap */

    for (u=1; u<tableSize; u++) {

        if (u==eltNbToSkip) continue;

        if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) {  /* overlap, existing < new */

            /* append */

            int const addedLength = (int)eltEnd - (table[u].pos + table[u].length);

            table[u].savings += elt.length / 8;    /* rough approx bonus */

            if (addedLength > 0) {   /* otherwise, elt fully included into existing */

                table[u].length += addedLength;

                table[u].savings += elt.savings * addedLength / elt.length;   /* rough approx */

            }

            /* sort : improve rank */

            elt = table[u];

            while ((u>1) && (table[u-1].savings < elt.savings))

                table[u] = table[u-1], u--;

            table[u] = elt;

            return u;

        }

        if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) {

            if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) {

                size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 );

                table[u].pos = elt.pos;

                table[u].savings += (U32)(elt.savings * addedLength / elt.length);

                table[u].length = MIN(elt.length, table[u].length + 1);

                return u;

            }

        }

    }

    return 0;

}

static void ZDICT_removeDictItem(dictItem* table, U32 id)

{

    /* convention : table[0].pos stores nb of elts */

    U32 const max = table[0].pos;

    U32 u;

    if (!id) return;   /* protection, should never happen */

    for (u=id; u<max-1; u++)

        table[u] = table[u+1];

    table->pos--;

}

static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt, const void* buffer)

{

    /* merge if possible */

    U32 mergeId = ZDICT_tryMerge(table, elt, 0, buffer);

    if (mergeId) {

        U32 newMerge = 1;

        while (newMerge) {

            newMerge = ZDICT_tryMerge(table, table[mergeId], mergeId, buffer);

            if (newMerge) ZDICT_removeDictItem(table, mergeId);

            mergeId = newMerge;

        }

        return;

    }

    /* insert */

    {   U32 current;

        U32 nextElt = table->pos;

        if (nextElt >= maxSize) nextElt = maxSize-1;

        current = nextElt-1;

        while (table[current].savings < elt.savings) {

            table[current+1] = table[current];

            current--;

        }

        table[current+1] = elt;

        table->pos = nextElt+1;

    }

}

static U32 ZDICT_dictSize(const dictItem* dictList)

{

    U32 u, dictSize = 0;

    for (u=1; u<dictList[0].pos; u++)

        dictSize += dictList[u].length;

    return dictSize;

}

static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,

                            const void* const buffer, size_t bufferSize,   /* buffer must end with noisy guard band */

                            const size_t* fileSizes, unsigned nbFiles,

                            U32 minRatio, U32 notificationLevel)

{

    int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));

    int* const suffix = suffix0+1;

    U32* reverseSuffix = (U32*)malloc((bufferSize)*sizeof(*reverseSuffix));

    BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks));   /* +16 for overflow security */

    U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));

    size_t result = 0;

    clock_t displayClock = 0;

    clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;

#   define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \

            if (ZDICT_clockSpan(displayClock) > refreshRate)  \

            { displayClock = clock(); DISPLAY(__VA_ARGS__); \

            if (notificationLevel>=4) fflush(stderr); } }

    /* init */

    DISPLAYLEVEL(2, "\r%70s\r", "");   /* clean display line */

    if (!suffix0 || !reverseSuffix || !doneMarks || !filePos) {

        result = ERROR(memory_allocation);