contrib/automation/hgautomation/linux.py
changeset 43076 2372284d9457
parent 43018 d1d919f679f7
child 43235 252ec4dbc1b4
equal deleted inserted replaced
43075:57875cf423c9 43076:2372284d9457
    11 import pathlib
    11 import pathlib
    12 import shlex
    12 import shlex
    13 import subprocess
    13 import subprocess
    14 import tempfile
    14 import tempfile
    15 
    15 
    16 from .ssh import (
    16 from .ssh import exec_command
    17     exec_command,
       
    18 )
       
    19 
    17 
    20 
    18 
    21 # Linux distributions that are supported.
    19 # Linux distributions that are supported.
    22 DISTROS = {
    20 DISTROS = {
    23     'debian9',
    21     'debian9',
    60     ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
    58     ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
    61     ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
    59     ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
    62 done
    60 done
    63 
    61 
    64 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
    62 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
    65 '''.lstrip().replace('\r\n', '\n')
    63 '''.lstrip().replace(
       
    64     '\r\n', '\n'
       
    65 )
    66 
    66 
    67 
    67 
    68 INSTALL_RUST = r'''
    68 INSTALL_RUST = r'''
    69 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
    69 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
    70 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
    70 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
    85 
    85 
    86 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
    86 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
    87 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
    87 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
    88 
    88 
    89 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
    89 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
    90 '''.lstrip().replace('\r\n', '\n')
    90 '''.lstrip().replace(
    91 
    91     '\r\n', '\n'
    92 
    92 )
    93 BOOTSTRAP_DEBIAN = r'''
    93 
       
    94 
       
    95 BOOTSTRAP_DEBIAN = (
       
    96     r'''
    94 #!/bin/bash
    97 #!/bin/bash
    95 
    98 
    96 set -ex
    99 set -ex
    97 
   100 
    98 DISTRO=`grep DISTRIB_ID /etc/lsb-release  | awk -F= '{{print $2}}'`
   101 DISTRO=`grep DISTRIB_ID /etc/lsb-release  | awk -F= '{{print $2}}'`
   321 [phases]
   324 [phases]
   322 publish = false
   325 publish = false
   323 EOF
   326 EOF
   324 
   327 
   325 sudo chown -R hg:hg /hgdev
   328 sudo chown -R hg:hg /hgdev
   326 '''.lstrip().format(
   329 '''.lstrip()
   327     install_rust=INSTALL_RUST,
   330     .format(
   328     install_pythons=INSTALL_PYTHONS,
   331         install_rust=INSTALL_RUST,
   329     bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV
   332         install_pythons=INSTALL_PYTHONS,
   330 ).replace('\r\n', '\n')
   333         bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
       
   334     )
       
   335     .replace('\r\n', '\n')
       
   336 )
   331 
   337 
   332 
   338 
   333 # Prepares /hgdev for operations.
   339 # Prepares /hgdev for operations.
   334 PREPARE_HGDEV = '''
   340 PREPARE_HGDEV = '''
   335 #!/bin/bash
   341 #!/bin/bash
   407 sudo chown hg:hg /hgwork
   413 sudo chown hg:hg /hgwork
   408 mkdir /hgwork/tmp
   414 mkdir /hgwork/tmp
   409 chown hg:hg /hgwork/tmp
   415 chown hg:hg /hgwork/tmp
   410 
   416 
   411 rsync -a /hgdev/src /hgwork/
   417 rsync -a /hgdev/src /hgwork/
   412 '''.lstrip().replace('\r\n', '\n')
   418 '''.lstrip().replace(
       
   419     '\r\n', '\n'
       
   420 )
   413 
   421 
   414 
   422 
   415 HG_UPDATE_CLEAN = '''
   423 HG_UPDATE_CLEAN = '''
   416 set -ex
   424 set -ex
   417 
   425 
   419 
   427 
   420 cd /hgwork/src
   428 cd /hgwork/src
   421 ${HG} --config extensions.purge= purge --all
   429 ${HG} --config extensions.purge= purge --all
   422 ${HG} update -C $1
   430 ${HG} update -C $1
   423 ${HG} log -r .
   431 ${HG} log -r .
   424 '''.lstrip().replace('\r\n', '\n')
   432 '''.lstrip().replace(
       
   433     '\r\n', '\n'
       
   434 )
   425 
   435 
   426 
   436 
   427 def prepare_exec_environment(ssh_client, filesystem='default'):
   437 def prepare_exec_environment(ssh_client, filesystem='default'):
   428     """Prepare an EC2 instance to execute things.
   438     """Prepare an EC2 instance to execute things.
   429 
   439 
   454         print(line, end='')
   464         print(line, end='')
   455 
   465 
   456     res = chan.recv_exit_status()
   466     res = chan.recv_exit_status()
   457 
   467 
   458     if res:
   468     if res:
   459         raise Exception('non-0 exit code updating working directory; %d'
   469         raise Exception('non-0 exit code updating working directory; %d' % res)
   460                         % res)
   470 
   461 
   471 
   462 
   472 def synchronize_hg(
   463 def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None):
   473     source_path: pathlib.Path, ec2_instance, revision: str = None
       
   474 ):
   464     """Synchronize a local Mercurial source path to remote EC2 instance."""
   475     """Synchronize a local Mercurial source path to remote EC2 instance."""
   465 
   476 
   466     with tempfile.TemporaryDirectory() as temp_dir:
   477     with tempfile.TemporaryDirectory() as temp_dir:
   467         temp_dir = pathlib.Path(temp_dir)
   478         temp_dir = pathlib.Path(temp_dir)
   468 
   479 
   480             fh.write('  StrictHostKeyChecking no\n')
   491             fh.write('  StrictHostKeyChecking no\n')
   481             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
   492             fh.write('  UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
   482             fh.write('  IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
   493             fh.write('  IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
   483 
   494 
   484         if not (source_path / '.hg').is_dir():
   495         if not (source_path / '.hg').is_dir():
   485             raise Exception('%s is not a Mercurial repository; synchronization '
   496             raise Exception(
   486                             'not yet supported' % source_path)
   497                 '%s is not a Mercurial repository; synchronization '
       
   498                 'not yet supported' % source_path
       
   499             )
   487 
   500 
   488         env = dict(os.environ)
   501         env = dict(os.environ)
   489         env['HGPLAIN'] = '1'
   502         env['HGPLAIN'] = '1'
   490         env['HGENCODING'] = 'utf-8'
   503         env['HGENCODING'] = 'utf-8'
   491 
   504 
   492         hg_bin = source_path / 'hg'
   505         hg_bin = source_path / 'hg'
   493 
   506 
   494         res = subprocess.run(
   507         res = subprocess.run(
   495             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
   508             ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
   496             cwd=str(source_path), env=env, check=True, capture_output=True)
   509             cwd=str(source_path),
       
   510             env=env,
       
   511             check=True,
       
   512             capture_output=True,
       
   513         )
   497 
   514 
   498         full_revision = res.stdout.decode('ascii')
   515         full_revision = res.stdout.decode('ascii')
   499 
   516 
   500         args = [
   517         args = [
   501             'python2.7', str(hg_bin),
   518             'python2.7',
   502             '--config', 'ui.ssh=ssh -F %s' % ssh_config,
   519             str(hg_bin),
   503             '--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
   520             '--config',
       
   521             'ui.ssh=ssh -F %s' % ssh_config,
       
   522             '--config',
       
   523             'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
   504             # Also ensure .hgtags changes are present so auto version
   524             # Also ensure .hgtags changes are present so auto version
   505             # calculation works.
   525             # calculation works.
   506             'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)',
   526             'push',
       
   527             '-f',
       
   528             '-r',
       
   529             full_revision,
       
   530             '-r',
       
   531             'file(.hgtags)',
   507             'ssh://%s//hgwork/src' % public_ip,
   532             'ssh://%s//hgwork/src' % public_ip,
   508         ]
   533         ]
   509 
   534 
   510         res = subprocess.run(args, cwd=str(source_path), env=env)
   535         res = subprocess.run(args, cwd=str(source_path), env=env)
   511 
   536 
   520         with sftp.open('/hgdev/hgup', 'wb') as fh:
   545         with sftp.open('/hgdev/hgup', 'wb') as fh:
   521             fh.write(HG_UPDATE_CLEAN)
   546             fh.write(HG_UPDATE_CLEAN)
   522             fh.chmod(0o0700)
   547             fh.chmod(0o0700)
   523 
   548 
   524         chan, stdin, stdout = exec_command(
   549         chan, stdin, stdout = exec_command(
   525             ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision)
   550             ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
       
   551         )
   526         stdin.close()
   552         stdin.close()
   527 
   553 
   528         for line in stdout:
   554         for line in stdout:
   529             print(line, end='')
   555             print(line, end='')
   530 
   556 
   531         res = chan.recv_exit_status()
   557         res = chan.recv_exit_status()
   532 
   558 
   533         if res:
   559         if res:
   534             raise Exception('non-0 exit code updating working directory; %d'
   560             raise Exception(
   535                             % res)
   561                 'non-0 exit code updating working directory; %d' % res
       
   562             )
   536 
   563 
   537 
   564 
   538 def run_tests(ssh_client, python_version, test_flags=None):
   565 def run_tests(ssh_client, python_version, test_flags=None):
   539     """Run tests on a remote Linux machine via an SSH client."""
   566     """Run tests on a remote Linux machine via an SSH client."""
   540     test_flags = test_flags or []
   567     test_flags = test_flags or []
   552 
   579 
   553     test_flags = ' '.join(shlex.quote(a) for a in test_flags)
   580     test_flags = ' '.join(shlex.quote(a) for a in test_flags)
   554 
   581 
   555     command = (
   582     command = (
   556         '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
   583         '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
   557         'cd /hgwork/src/tests && %s run-tests.py %s"' % (
   584         'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
   558             python, test_flags))
   585     )
   559 
   586 
   560     chan, stdin, stdout = exec_command(ssh_client, command)
   587     chan, stdin, stdout = exec_command(ssh_client, command)
   561 
   588 
   562     stdin.close()
   589     stdin.close()
   563 
   590