diff -r 195dcc10b3d7 -r 65b3ef162b39 contrib/automation/hgautomation/cli.py --- a/contrib/automation/hgautomation/cli.py Tue Apr 23 21:57:32 2019 -0700 +++ b/contrib/automation/hgautomation/cli.py Sat Apr 27 11:48:26 2019 -0700 @@ -8,12 +8,15 @@ # no-check-code because Python 3 native. import argparse +import concurrent.futures as futures import os import pathlib +import time from . import ( aws, HGAutomation, + linux, windows, ) @@ -22,6 +25,33 @@ DIST_PATH = SOURCE_ROOT / 'dist' +def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None, + parallel=False): + c = hga.aws_connection(aws_region) + + if distros: + distros = distros.split(',') + else: + distros = sorted(linux.DISTROS) + + # TODO There is a wonky interaction involving KeyboardInterrupt whereby + # the context manager that is supposed to terminate the temporary EC2 + # instance doesn't run. Until we fix this, make parallel building opt-in + # so we don't orphan instances. + if parallel: + fs = [] + + with futures.ThreadPoolExecutor(len(distros)) as e: + for distro in distros: + fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro)) + + for f in fs: + f.result() + else: + for distro in distros: + aws.ensure_linux_dev_ami(c, distro=distro) + + def bootstrap_windows_dev(hga: HGAutomation, aws_region): c = hga.aws_connection(aws_region) image = aws.ensure_windows_dev_ami(c) @@ -107,6 +137,37 @@ aws.remove_resources(c) +def run_tests_linux(hga: HGAutomation, aws_region, instance_type, + python_version, test_flags, distro, filesystem): + c = hga.aws_connection(aws_region) + image = aws.ensure_linux_dev_ami(c, distro=distro) + + t_start = time.time() + + ensure_extra_volume = filesystem not in ('default', 'tmpfs') + + with aws.temporary_linux_dev_instances( + c, image, instance_type, + ensure_extra_volume=ensure_extra_volume) as insts: + + instance = insts[0] + + linux.prepare_exec_environment(instance.ssh_client, + filesystem=filesystem) + linux.synchronize_hg(SOURCE_ROOT, instance, '.') + t_prepared = time.time() + linux.run_tests(instance.ssh_client, python_version, + test_flags) + t_done = time.time() + + t_setup = t_prepared - t_start + t_all = t_done - t_start + + print( + 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' + % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)) + + def run_tests_windows(hga: HGAutomation, aws_region, instance_type, python_version, arch, test_flags): c = hga.aws_connection(aws_region) @@ -138,6 +199,21 @@ subparsers = parser.add_subparsers() sp = subparsers.add_parser( + 'bootstrap-linux-dev', + help='Bootstrap Linux development environments', + ) + sp.add_argument( + '--distros', + help='Comma delimited list of distros to bootstrap', + ) + sp.add_argument( + '--parallel', + action='store_true', + help='Generate AMIs in parallel (not CTRL-c safe)' + ) + sp.set_defaults(func=bootstrap_linux_dev) + + sp = subparsers.add_parser( 'bootstrap-windows-dev', help='Bootstrap the Windows development environment', ) @@ -233,6 +309,41 @@ sp.set_defaults(func=purge_ec2_resources) sp = subparsers.add_parser( + 'run-tests-linux', + help='Run tests on Linux', + ) + sp.add_argument( + '--distro', + help='Linux distribution to run tests on', + choices=linux.DISTROS, + default='debian9', + ) + sp.add_argument( + '--filesystem', + help='Filesystem type to use', + choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'}, + default='default', + ) + sp.add_argument( + '--instance-type', + help='EC2 instance type to use', + default='c5.9xlarge', + ) + sp.add_argument( + '--python-version', + help='Python version to use', + choices={'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8', + 'pypy', 'pypy3.5', 'pypy3.6'}, + default='system2', + ) + sp.add_argument( + 'test_flags', + help='Extra command line flags to pass to run-tests.py', + nargs='*', + ) + sp.set_defaults(func=run_tests_linux) + + sp = subparsers.add_parser( 'run-tests-windows', help='Run tests on Windows', )