#!/usr/bin/env python

#

# run-tests.py - Run a set of tests on Mercurial

#

# Copyright 2006 Matt Mackall <mpm@selenic.com>

#

# This software may be used and distributed according to the terms of the

# GNU General Public License version 2 or any later version.

# Modifying this script is tricky because it has many modes:

#   - serial (default) vs parallel (-jN, N > 1)

#   - no coverage (default) vs coverage (-c, -C, -s)

#   - temp install (default) vs specific hg script (--with-hg, --local)

#   - tests are a mix of shell scripts and Python scripts

#

# If you change this script, it is recommended that you ensure you

# haven't broken it by running it in various modes with a representative

# sample of test scripts.  For example:

#

#  1) serial, no coverage, temp install:

#      ./run-tests.py test-s*

#  2) serial, no coverage, local hg:

#      ./run-tests.py --local test-s*

#  3) serial, coverage, temp install:

#      ./run-tests.py -c test-s*

#  4) serial, coverage, local hg:

#      ./run-tests.py -c --local test-s*      # unsupported

#  5) parallel, no coverage, temp install:

#      ./run-tests.py -j2 test-s*

#  6) parallel, no coverage, local hg:

#      ./run-tests.py -j2 --local test-s*

#  7) parallel, coverage, temp install:

#      ./run-tests.py -j2 -c test-s*          # currently broken

#  8) parallel, coverage, local install:

#      ./run-tests.py -j2 -c --local test-s*  # unsupported (and broken)

#  9) parallel, custom tmp dir:

#      ./run-tests.py -j2 --tmpdir /tmp/myhgtests

#  10) parallel, pure, tests that call run-tests:

#      ./run-tests.py --pure `grep -l run-tests.py *.t`

#

# (You could use any subset of the tests: test-s* happens to match

# enough that it's worth doing parallel runs, few enough that it

# completes fairly quickly, includes both shell and Python scripts, and

# includes some scripts that run daemon processes.)

from __future__ import print_function

from distutils import version

import difflib

import errno

import json

import optparse

import os

import shutil

import subprocess

import signal

import socket

import sys

import tempfile

import time

import random

import re

import threading

import killdaemons as killmod

try:

    import Queue as queue

except ImportError:

    import queue

from xml.dom import minidom

import unittest

if os.environ.get('RTUNICODEPEDANTRY', False):

    try:

        reload(sys)

        sys.setdefaultencoding("undefined")

    except NameError:

        pass

osenvironb = getattr(os, 'environb', os.environ)

processlock = threading.Lock()

if sys.version_info > (3, 5, 0):

    PYTHON3 = True

    xrange = range # we use xrange in one place, and we'd rather not use range

    def _bytespath(p):

        return p.encode('utf-8')

    def _strpath(p):

        return p.decode('utf-8')

elif sys.version_info >= (3, 0, 0):

    print('%s is only supported on Python 3.5+ and 2.6-2.7, not %s' %

          (sys.argv[0], '.'.join(str(v) for v in sys.version_info[:3])))

    sys.exit(70) # EX_SOFTWARE from `man 3 sysexit`

else:

    PYTHON3 = False

    # In python 2.x, path operations are generally done using

    # bytestrings by default, so we don't have to do any extra

    # fiddling there. We define the wrapper functions anyway just to

    # help keep code consistent between platforms.

    def _bytespath(p):

        return p

    _strpath = _bytespath

# For Windows support

wifexited = getattr(os, "WIFEXITED", lambda x: False)

def checkportisavailable(port):

    """return true if a port seems free to bind on localhost"""

    try:

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        s.bind(('localhost', port))

        s.close()

        return True

    except socket.error as exc:

        if not exc.errno == errno.EADDRINUSE:

            raise

        return False

closefds = os.name == 'posix'

def Popen4(cmd, wd, timeout, env=None):

    processlock.acquire()

    p = subprocess.Popen(cmd, shell=True, bufsize=-1, cwd=wd, env=env,

                         close_fds=closefds,

                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,

                         stderr=subprocess.STDOUT)

    processlock.release()

    p.fromchild = p.stdout

    p.tochild = p.stdin

    p.childerr = p.stderr

    p.timeout = False

    if timeout:

        def t():

            start = time.time()

            while time.time() - start < timeout and p.returncode is None:

                time.sleep(.1)

            p.timeout = True

            if p.returncode is None:

                terminate(p)

        threading.Thread(target=t).start()

    return p

PYTHON = _bytespath(sys.executable.replace('\\', '/'))

IMPL_PATH = b'PYTHONPATH'

if 'java' in sys.platform:

    IMPL_PATH = b'JYTHONPATH'

defaults = {

    'jobs': ('HGTEST_JOBS', 1),

    'timeout': ('HGTEST_TIMEOUT', 180),

    'slowtimeout': ('HGTEST_SLOWTIMEOUT', 500),

    'port': ('HGTEST_PORT', 20059),

    'shell': ('HGTEST_SHELL', 'sh'),

}

def canonpath(path):

    return os.path.realpath(os.path.expanduser(path))

def parselistfiles(files, listtype, warn=True):

    entries = dict()

    for filename in files:

        try:

            path = os.path.expanduser(os.path.expandvars(filename))

            f = open(path, "rb")

        except IOError as err:

            if err.errno != errno.ENOENT:

                raise

            if warn:

                print("warning: no such %s file: %s" % (listtype, filename))

            continue

        for line in f.readlines():

            line = line.split(b'#', 1)[0].strip()

            if line:

                entries[line] = filename

        f.close()

    return entries

def getparser():

    """Obtain the OptionParser used by the CLI."""

    parser = optparse.OptionParser("%prog [options] [tests]")

    # keep these sorted

    parser.add_option("--blacklist", action="append",

        help="skip tests listed in the specified blacklist file")

    parser.add_option("--whitelist", action="append",

        help="always run tests listed in the specified whitelist file")

    parser.add_option("--changed", type="string",

        help="run tests that are changed in parent rev or working directory")

    parser.add_option("-C", "--annotate", action="store_true",

        help="output files annotated with coverage")

    parser.add_option("-c", "--cover", action="store_true",

        help="print a test coverage report")

    parser.add_option("-d", "--debug", action="store_true",

        help="debug mode: write output of test scripts to console"

             " rather than capturing and diffing it (disables timeout)")

    parser.add_option("-f", "--first", action="store_true",

        help="exit on the first test failure")

    parser.add_option("-H", "--htmlcov", action="store_true",

        help="create an HTML report of the coverage of the files")

    parser.add_option("-i", "--interactive", action="store_true",

        help="prompt to accept changed output")

    parser.add_option("-j", "--jobs", type="int",

        help="number of jobs to run in parallel"

             " (default: $%s or %d)" % defaults['jobs'])

    parser.add_option("--keep-tmpdir", action="store_true",

        help="keep temporary directory after running tests")

    parser.add_option("-k", "--keywords",

        help="run tests matching keywords")

    parser.add_option("-l", "--local", action="store_true",

        help="shortcut for --with-hg=<testdir>/../hg")

    parser.add_option("--loop", action="store_true",

        help="loop tests repeatedly")

    parser.add_option("--runs-per-test", type="int", dest="runs_per_test",

        help="run each test N times (default=1)", default=1)

    parser.add_option("-n", "--nodiff", action="store_true",

        help="skip showing test changes")

    parser.add_option("-p", "--port", type="int",

        help="port on which servers should listen"

             " (default: $%s or %d)" % defaults['port'])

    parser.add_option("--compiler", type="string",

        help="compiler to build with")

    parser.add_option("--pure", action="store_true",

        help="use pure Python code instead of C extensions")

    parser.add_option("-R", "--restart", action="store_true",

        help="restart at last error")

    parser.add_option("-r", "--retest", action="store_true",

        help="retest failed tests")

    parser.add_option("-S", "--noskips", action="store_true",

        help="don't report skip tests verbosely")

    parser.add_option("--shell", type="string",

        help="shell to use (default: $%s or %s)" % defaults['shell'])

    parser.add_option("-t", "--timeout", type="int",

        help="kill errant tests after TIMEOUT seconds"

             " (default: $%s or %d)" % defaults['timeout'])

    parser.add_option("--slowtimeout", type="int",

        help="kill errant slow tests after SLOWTIMEOUT seconds"

             " (default: $%s or %d)" % defaults['slowtimeout'])

    parser.add_option("--time", action="store_true",

        help="time how long each test takes")

    parser.add_option("--json", action="store_true",

                      help="store test result data in 'report.json' file")

    parser.add_option("--tmpdir", type="string",

        help="run tests in the given temporary directory"

             " (implies --keep-tmpdir)")

    parser.add_option("-v", "--verbose", action="store_true",

        help="output verbose messages")

    parser.add_option("--xunit", type="string",

                      help="record xunit results at specified path")

    parser.add_option("--view", type="string",

        help="external diff viewer")

    parser.add_option("--with-hg", type="string",

        metavar="HG",

        help="test using specified hg script rather than a "

             "temporary installation")

    parser.add_option("--chg", action="store_true",

                      help="install and use chg wrapper in place of hg")

    parser.add_option("--with-chg", metavar="CHG",

                      help="use specified chg wrapper in place of hg")

    parser.add_option("-3", "--py3k-warnings", action="store_true",

        help="enable Py3k warnings on Python 2.6+")

    # This option should be deleted once test-check-py3-compat.t and other

    # Python 3 tests run with Python 3.

    parser.add_option("--with-python3", metavar="PYTHON3",

                      help="Python 3 interpreter (if running under Python 2)"

                           " (TEMPORARY)")

    parser.add_option('--extra-config-opt', action="append",

                      help='set the given config opt in the test hgrc')

    parser.add_option('--random', action="store_true",

                      help='run tests in random order')

    parser.add_option('--profile-runner', action='store_true',

                      help='run statprof on run-tests')

    parser.add_option('--allow-slow-tests', action='store_true',

                      help='allow extremely slow tests')

    parser.add_option('--showchannels', action='store_true',

                      help='show scheduling channels')

    parser.add_option('--known-good-rev', type="string",

                      metavar="known_good_rev",

                      help=("Automatically bisect any failures using this "

                            "revision as a known-good revision."))

    for option, (envvar, default) in defaults.items():

        defaults[option] = type(default)(os.environ.get(envvar, default))

    parser.set_defaults(**defaults)

    return parser

def parseargs(args, parser):

    """Parse arguments with our OptionParser and validate results."""

    (options, args) = parser.parse_args(args)

    # jython is always pure

    if 'java' in sys.platform or '__pypy__' in sys.modules:

        options.pure = True

    if options.with_hg:

        options.with_hg = canonpath(_bytespath(options.with_hg))

        if not (os.path.isfile(options.with_hg) and

                os.access(options.with_hg, os.X_OK)):

            parser.error('--with-hg must specify an executable hg script')

        if not os.path.basename(options.with_hg) == b'hg':

            sys.stderr.write('warning: --with-hg should specify an hg script\n')

    if options.local:

        testdir = os.path.dirname(_bytespath(canonpath(sys.argv[0])))

        hgbin = os.path.join(os.path.dirname(testdir), b'hg')

        if os.name != 'nt' and not os.access(hgbin, os.X_OK):

            parser.error('--local specified, but %r not found or not executable'

                         % hgbin)

        options.with_hg = hgbin

    if (options.chg or options.with_chg) and os.name == 'nt':

        parser.error('chg does not work on %s' % os.name)

    if options.with_chg:

        options.chg = False  # no installation to temporary location

        options.with_chg = canonpath(_bytespath(options.with_chg))

        if not (os.path.isfile(options.with_chg) and

                os.access(options.with_chg, os.X_OK)):

            parser.error('--with-chg must specify a chg executable')

    if options.chg and options.with_hg:

        # chg shares installation location with hg

        parser.error('--chg does not work when --with-hg is specified '

                     '(use --with-chg instead)')

    options.anycoverage = options.cover or options.annotate or options.htmlcov

    if options.anycoverage:

        try:

            import coverage

            covver = version.StrictVersion(coverage.__version__).version

            if covver < (3, 3):

                parser.error('coverage options require coverage 3.3 or later')

        except ImportError:

            parser.error('coverage options now require the coverage package')

    if options.anycoverage and options.local:

        # this needs some path mangling somewhere, I guess

        parser.error("sorry, coverage options do not work when --local "

                     "is specified")

    if options.anycoverage and options.with_hg:

        parser.error("sorry, coverage options do not work when --with-hg "

                     "is specified")

    global verbose

    if options.verbose:

        verbose = ''

    if options.tmpdir:

        options.tmpdir = canonpath(options.tmpdir)

    if options.jobs < 1:

        parser.error('--jobs must be positive')

    if options.interactive and options.debug:

        parser.error("-i/--interactive and -d/--debug are incompatible")

    if options.debug:

        if options.timeout != defaults['timeout']:

            sys.stderr.write(

                'warning: --timeout option ignored with --debug\n')

        if options.slowtimeout != defaults['slowtimeout']:

            sys.stderr.write(

                'warning: --slowtimeout option ignored with --debug\n')

        options.timeout = 0

        options.slowtimeout = 0

    if options.py3k_warnings:

        if PYTHON3:

            parser.error(

                '--py3k-warnings can only be used on Python 2.6 and 2.7')

    if options.with_python3:

        if PYTHON3:

            parser.error('--with-python3 cannot be used when executing with '

                         'Python 3')

        options.with_python3 = canonpath(options.with_python3)

        # Verify Python3 executable is acceptable.

        proc = subprocess.Popen([options.with_python3, b'--version'],

                                stdout=subprocess.PIPE,

                                stderr=subprocess.STDOUT)

        out, _err = proc.communicate()

        ret = proc.wait()

        if ret != 0:

            parser.error('could not determine version of python 3')

        if not out.startswith('Python '):

            parser.error('unexpected output from python3 --version: %s' %

                         out)

        vers = version.LooseVersion(out[len('Python '):])

        if vers < version.LooseVersion('3.5.0'):

            parser.error('--with-python3 version must be 3.5.0 or greater; '

                         'got %s' % out)

    if options.blacklist:

        options.blacklist = parselistfiles(options.blacklist, 'blacklist')

    if options.whitelist:

        options.whitelisted = parselistfiles(options.whitelist, 'whitelist')

    else:

        options.whitelisted = {}

    if options.showchannels:

        options.nodiff = True

    return (options, args)

def rename(src, dst):

    """Like os.rename(), trade atomicity and opened files friendliness

    for existing destination support.

    """

    shutil.copy(src, dst)

    os.remove(src)

_unified_diff = difflib.unified_diff

if PYTHON3:

    import functools

    _unified_diff = functools.partial(difflib.diff_bytes, difflib.unified_diff)

def getdiff(expected, output, ref, err):

    servefail = False

    lines = []

    for line in _unified_diff(expected, output, ref, err):

        if line.startswith(b'+++') or line.startswith(b'---'):

            line = line.replace(b'\\', b'/')

            if line.endswith(b' \n'):

                line = line[:-2] + b'\n'

        lines.append(line)

        if not servefail and line.startswith(

                             b'+  abort: child process failed to start'):

            servefail = True

    return servefail, lines

verbose = False

def vlog(*msg):

    """Log only when in verbose mode."""

    if verbose is False:

        return

    return log(*msg)

# Bytes that break XML even in a CDATA block: control characters 0-31

# sans \t, \n and \r

CDATA_EVIL = re.compile(br"[\000-\010\013\014\016-\037]")

def cdatasafe(data):

    """Make a string safe to include in a CDATA block.

    Certain control characters are illegal in a CDATA block, and

    there's no way to include a ]]> in a CDATA either. This function

    replaces illegal bytes with ? and adds a space between the ]] so

    that it won't break the CDATA block.

    """

    return CDATA_EVIL.sub(b'?', data).replace(b']]>', b'] ]>')

def log(*msg):

    """Log something to stdout.

    Arguments are strings to print.

    """

    with iolock:

        if verbose:

            print(verbose, end=' ')

        for m in msg:

            print(m, end=' ')

        print()

        sys.stdout.flush()

def terminate(proc):

    """Terminate subprocess (with fallback for Python versions < 2.6)"""

    vlog('# Terminating process %d' % proc.pid)

    try:

        getattr(proc, 'terminate', lambda : os.kill(proc.pid, signal.SIGTERM))()

    except OSError:

        pass

def killdaemons(pidfile):

    return killmod.killdaemons(pidfile, tryhard=False, remove=True,

                               logfn=vlog)

class Test(unittest.TestCase):

    """Encapsulates a single, runnable test.