Mercurial > public > mercurial-scm > hg-stable

comparison contrib/automation/hgautomation/linux.py @ 43076:2372284d9457

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
formatting: blacken the codebase This is using my patch to black (https://github.com/psf/black/pull/826) so we don't un-wrap collection literals. Done with: hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S # skip-blame mass-reformatting only # no-check-commit reformats foo_bar functions Differential Revision: https://phab.mercurial-scm.org/D6971
author Augie Fackler <augie@google.com>
date Sun, 06 Oct 2019 09:45:02 -0400
parents d1d919f679f7
children 252ec4dbc1b4
comparison
equal deleted inserted replaced
43075:57875cf423c9 43076:2372284d9457
11 import pathlib 11 import pathlib
12 import shlex 12 import shlex
13 import subprocess 13 import subprocess
14 import tempfile 14 import tempfile
15 15
16 from .ssh import ( 16 from .ssh import exec_command
17 exec_command,
18 )
19 17
20 18
21 # Linux distributions that are supported. 19 # Linux distributions that are supported.
22 DISTROS = { 20 DISTROS = {
23 'debian9', 21 'debian9',
60 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py 58 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
61 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt 59 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
62 done 60 done
63 61
64 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system 62 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
65 '''.lstrip().replace('\r\n', '\n') 63 '''.lstrip().replace(
64 '\r\n', '\n'
65 )
66 66
67 67
68 INSTALL_RUST = r''' 68 INSTALL_RUST = r'''
69 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076 69 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
70 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init 70 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
85 85
86 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL} 86 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
87 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check - 87 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
88 88
89 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL} 89 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
90 '''.lstrip().replace('\r\n', '\n') 90 '''.lstrip().replace(
91 91 '\r\n', '\n'
92 92 )
93 BOOTSTRAP_DEBIAN = r''' 93
94
95 BOOTSTRAP_DEBIAN = (
96 r'''
94 #!/bin/bash 97 #!/bin/bash
95 98
96 set -ex 99 set -ex
97 100
98 DISTRO=`grep DISTRIB_ID /etc/lsb-release | awk -F= '{{print $2}}'` 101 DISTRO=`grep DISTRIB_ID /etc/lsb-release | awk -F= '{{print $2}}'`
321 [phases] 324 [phases]
322 publish = false 325 publish = false
323 EOF 326 EOF
324 327
325 sudo chown -R hg:hg /hgdev 328 sudo chown -R hg:hg /hgdev
326 '''.lstrip().format( 329 '''.lstrip()
327 install_rust=INSTALL_RUST, 330 .format(
328 install_pythons=INSTALL_PYTHONS, 331 install_rust=INSTALL_RUST,
329 bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV 332 install_pythons=INSTALL_PYTHONS,
330 ).replace('\r\n', '\n') 333 bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
334 )
335 .replace('\r\n', '\n')
336 )
331 337
332 338
333 # Prepares /hgdev for operations. 339 # Prepares /hgdev for operations.
334 PREPARE_HGDEV = ''' 340 PREPARE_HGDEV = '''
335 #!/bin/bash 341 #!/bin/bash
407 sudo chown hg:hg /hgwork 413 sudo chown hg:hg /hgwork
408 mkdir /hgwork/tmp 414 mkdir /hgwork/tmp
409 chown hg:hg /hgwork/tmp 415 chown hg:hg /hgwork/tmp
410 416
411 rsync -a /hgdev/src /hgwork/ 417 rsync -a /hgdev/src /hgwork/
412 '''.lstrip().replace('\r\n', '\n') 418 '''.lstrip().replace(
419 '\r\n', '\n'
420 )
413 421
414 422
415 HG_UPDATE_CLEAN = ''' 423 HG_UPDATE_CLEAN = '''
416 set -ex 424 set -ex
417 425
419 427
420 cd /hgwork/src 428 cd /hgwork/src
421 ${HG} --config extensions.purge= purge --all 429 ${HG} --config extensions.purge= purge --all
422 ${HG} update -C $1 430 ${HG} update -C $1
423 ${HG} log -r . 431 ${HG} log -r .
424 '''.lstrip().replace('\r\n', '\n') 432 '''.lstrip().replace(
433 '\r\n', '\n'
434 )
425 435
426 436
427 def prepare_exec_environment(ssh_client, filesystem='default'): 437 def prepare_exec_environment(ssh_client, filesystem='default'):
428 """Prepare an EC2 instance to execute things. 438 """Prepare an EC2 instance to execute things.
429 439
454 print(line, end='') 464 print(line, end='')
455 465
456 res = chan.recv_exit_status() 466 res = chan.recv_exit_status()
457 467
458 if res: 468 if res:
459 raise Exception('non-0 exit code updating working directory; %d' 469 raise Exception('non-0 exit code updating working directory; %d' % res)
460 % res) 470
461 471
462 472 def synchronize_hg(
463 def synchronize_hg(source_path: pathlib.Path, ec2_instance, revision: str=None): 473 source_path: pathlib.Path, ec2_instance, revision: str = None
474 ):
464 """Synchronize a local Mercurial source path to remote EC2 instance.""" 475 """Synchronize a local Mercurial source path to remote EC2 instance."""
465 476
466 with tempfile.TemporaryDirectory() as temp_dir: 477 with tempfile.TemporaryDirectory() as temp_dir:
467 temp_dir = pathlib.Path(temp_dir) 478 temp_dir = pathlib.Path(temp_dir)
468 479
480 fh.write(' StrictHostKeyChecking no\n') 491 fh.write(' StrictHostKeyChecking no\n')
481 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts')) 492 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
482 fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path) 493 fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
483 494
484 if not (source_path / '.hg').is_dir(): 495 if not (source_path / '.hg').is_dir():
485 raise Exception('%s is not a Mercurial repository; synchronization ' 496 raise Exception(
486 'not yet supported' % source_path) 497 '%s is not a Mercurial repository; synchronization '
498 'not yet supported' % source_path
499 )
487 500
488 env = dict(os.environ) 501 env = dict(os.environ)
489 env['HGPLAIN'] = '1' 502 env['HGPLAIN'] = '1'
490 env['HGENCODING'] = 'utf-8' 503 env['HGENCODING'] = 'utf-8'
491 504
492 hg_bin = source_path / 'hg' 505 hg_bin = source_path / 'hg'
493 506
494 res = subprocess.run( 507 res = subprocess.run(
495 ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'], 508 ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
496 cwd=str(source_path), env=env, check=True, capture_output=True) 509 cwd=str(source_path),
510 env=env,
511 check=True,
512 capture_output=True,
513 )
497 514
498 full_revision = res.stdout.decode('ascii') 515 full_revision = res.stdout.decode('ascii')
499 516
500 args = [ 517 args = [
501 'python2.7', str(hg_bin), 518 'python2.7',
502 '--config', 'ui.ssh=ssh -F %s' % ssh_config, 519 str(hg_bin),
503 '--config', 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg', 520 '--config',
521 'ui.ssh=ssh -F %s' % ssh_config,
522 '--config',
523 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
504 # Also ensure .hgtags changes are present so auto version 524 # Also ensure .hgtags changes are present so auto version
505 # calculation works. 525 # calculation works.
506 'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)', 526 'push',
527 '-f',
528 '-r',
529 full_revision,
530 '-r',
531 'file(.hgtags)',
507 'ssh://%s//hgwork/src' % public_ip, 532 'ssh://%s//hgwork/src' % public_ip,
508 ] 533 ]
509 534
510 res = subprocess.run(args, cwd=str(source_path), env=env) 535 res = subprocess.run(args, cwd=str(source_path), env=env)
511 536
520 with sftp.open('/hgdev/hgup', 'wb') as fh: 545 with sftp.open('/hgdev/hgup', 'wb') as fh:
521 fh.write(HG_UPDATE_CLEAN) 546 fh.write(HG_UPDATE_CLEAN)
522 fh.chmod(0o0700) 547 fh.chmod(0o0700)
523 548
524 chan, stdin, stdout = exec_command( 549 chan, stdin, stdout = exec_command(
525 ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision) 550 ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
551 )
526 stdin.close() 552 stdin.close()
527 553
528 for line in stdout: 554 for line in stdout:
529 print(line, end='') 555 print(line, end='')
530 556
531 res = chan.recv_exit_status() 557 res = chan.recv_exit_status()
532 558
533 if res: 559 if res:
534 raise Exception('non-0 exit code updating working directory; %d' 560 raise Exception(
535 % res) 561 'non-0 exit code updating working directory; %d' % res
562 )
536 563
537 564
538 def run_tests(ssh_client, python_version, test_flags=None): 565 def run_tests(ssh_client, python_version, test_flags=None):
539 """Run tests on a remote Linux machine via an SSH client.""" 566 """Run tests on a remote Linux machine via an SSH client."""
540 test_flags = test_flags or [] 567 test_flags = test_flags or []
552 579
553 test_flags = ' '.join(shlex.quote(a) for a in test_flags) 580 test_flags = ' '.join(shlex.quote(a) for a in test_flags)
554 581
555 command = ( 582 command = (
556 '/bin/sh -c "export TMPDIR=/hgwork/tmp; ' 583 '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
557 'cd /hgwork/src/tests && %s run-tests.py %s"' % ( 584 'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
558 python, test_flags)) 585 )
559 586
560 chan, stdin, stdout = exec_command(ssh_client, command) 587 chan, stdin, stdout = exec_command(ssh_client, command)
561 588
562 stdin.close() 589 stdin.close()
563 590