6 # GNU General Public License version 2 or any later version. |
6 # GNU General Public License version 2 or any later version. |
7 |
7 |
8 # no-check-code because Python 3 native. |
8 # no-check-code because Python 3 native. |
9 |
9 |
10 import argparse |
10 import argparse |
|
11 import concurrent.futures as futures |
11 import os |
12 import os |
12 import pathlib |
13 import pathlib |
|
14 import time |
13 |
15 |
14 from . import ( |
16 from . import ( |
15 aws, |
17 aws, |
16 HGAutomation, |
18 HGAutomation, |
|
19 linux, |
17 windows, |
20 windows, |
18 ) |
21 ) |
19 |
22 |
20 |
23 |
21 SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent |
24 SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent |
22 DIST_PATH = SOURCE_ROOT / 'dist' |
25 DIST_PATH = SOURCE_ROOT / 'dist' |
|
26 |
|
27 |
|
28 def bootstrap_linux_dev(hga: HGAutomation, aws_region, distros=None, |
|
29 parallel=False): |
|
30 c = hga.aws_connection(aws_region) |
|
31 |
|
32 if distros: |
|
33 distros = distros.split(',') |
|
34 else: |
|
35 distros = sorted(linux.DISTROS) |
|
36 |
|
37 # TODO There is a wonky interaction involving KeyboardInterrupt whereby |
|
38 # the context manager that is supposed to terminate the temporary EC2 |
|
39 # instance doesn't run. Until we fix this, make parallel building opt-in |
|
40 # so we don't orphan instances. |
|
41 if parallel: |
|
42 fs = [] |
|
43 |
|
44 with futures.ThreadPoolExecutor(len(distros)) as e: |
|
45 for distro in distros: |
|
46 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro)) |
|
47 |
|
48 for f in fs: |
|
49 f.result() |
|
50 else: |
|
51 for distro in distros: |
|
52 aws.ensure_linux_dev_ami(c, distro=distro) |
23 |
53 |
24 |
54 |
25 def bootstrap_windows_dev(hga: HGAutomation, aws_region): |
55 def bootstrap_windows_dev(hga: HGAutomation, aws_region): |
26 c = hga.aws_connection(aws_region) |
56 c = hga.aws_connection(aws_region) |
27 image = aws.ensure_windows_dev_ami(c) |
57 image = aws.ensure_windows_dev_ami(c) |
105 def purge_ec2_resources(hga: HGAutomation, aws_region): |
135 def purge_ec2_resources(hga: HGAutomation, aws_region): |
106 c = hga.aws_connection(aws_region, ensure_ec2_state=False) |
136 c = hga.aws_connection(aws_region, ensure_ec2_state=False) |
107 aws.remove_resources(c) |
137 aws.remove_resources(c) |
108 |
138 |
109 |
139 |
|
140 def run_tests_linux(hga: HGAutomation, aws_region, instance_type, |
|
141 python_version, test_flags, distro, filesystem): |
|
142 c = hga.aws_connection(aws_region) |
|
143 image = aws.ensure_linux_dev_ami(c, distro=distro) |
|
144 |
|
145 t_start = time.time() |
|
146 |
|
147 ensure_extra_volume = filesystem not in ('default', 'tmpfs') |
|
148 |
|
149 with aws.temporary_linux_dev_instances( |
|
150 c, image, instance_type, |
|
151 ensure_extra_volume=ensure_extra_volume) as insts: |
|
152 |
|
153 instance = insts[0] |
|
154 |
|
155 linux.prepare_exec_environment(instance.ssh_client, |
|
156 filesystem=filesystem) |
|
157 linux.synchronize_hg(SOURCE_ROOT, instance, '.') |
|
158 t_prepared = time.time() |
|
159 linux.run_tests(instance.ssh_client, python_version, |
|
160 test_flags) |
|
161 t_done = time.time() |
|
162 |
|
163 t_setup = t_prepared - t_start |
|
164 t_all = t_done - t_start |
|
165 |
|
166 print( |
|
167 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%' |
|
168 % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)) |
|
169 |
|
170 |
110 def run_tests_windows(hga: HGAutomation, aws_region, instance_type, |
171 def run_tests_windows(hga: HGAutomation, aws_region, instance_type, |
111 python_version, arch, test_flags): |
172 python_version, arch, test_flags): |
112 c = hga.aws_connection(aws_region) |
173 c = hga.aws_connection(aws_region) |
113 image = aws.ensure_windows_dev_ami(c) |
174 image = aws.ensure_windows_dev_ami(c) |
114 |
175 |
134 help='AWS region to use', |
195 help='AWS region to use', |
135 default='us-west-1', |
196 default='us-west-1', |
136 ) |
197 ) |
137 |
198 |
138 subparsers = parser.add_subparsers() |
199 subparsers = parser.add_subparsers() |
|
200 |
|
201 sp = subparsers.add_parser( |
|
202 'bootstrap-linux-dev', |
|
203 help='Bootstrap Linux development environments', |
|
204 ) |
|
205 sp.add_argument( |
|
206 '--distros', |
|
207 help='Comma delimited list of distros to bootstrap', |
|
208 ) |
|
209 sp.add_argument( |
|
210 '--parallel', |
|
211 action='store_true', |
|
212 help='Generate AMIs in parallel (not CTRL-c safe)' |
|
213 ) |
|
214 sp.set_defaults(func=bootstrap_linux_dev) |
139 |
215 |
140 sp = subparsers.add_parser( |
216 sp = subparsers.add_parser( |
141 'bootstrap-windows-dev', |
217 'bootstrap-windows-dev', |
142 help='Bootstrap the Windows development environment', |
218 help='Bootstrap the Windows development environment', |
143 ) |
219 ) |
231 help='Purge all EC2 resources managed by us', |
307 help='Purge all EC2 resources managed by us', |
232 ) |
308 ) |
233 sp.set_defaults(func=purge_ec2_resources) |
309 sp.set_defaults(func=purge_ec2_resources) |
234 |
310 |
235 sp = subparsers.add_parser( |
311 sp = subparsers.add_parser( |
|
312 'run-tests-linux', |
|
313 help='Run tests on Linux', |
|
314 ) |
|
315 sp.add_argument( |
|
316 '--distro', |
|
317 help='Linux distribution to run tests on', |
|
318 choices=linux.DISTROS, |
|
319 default='debian9', |
|
320 ) |
|
321 sp.add_argument( |
|
322 '--filesystem', |
|
323 help='Filesystem type to use', |
|
324 choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'}, |
|
325 default='default', |
|
326 ) |
|
327 sp.add_argument( |
|
328 '--instance-type', |
|
329 help='EC2 instance type to use', |
|
330 default='c5.9xlarge', |
|
331 ) |
|
332 sp.add_argument( |
|
333 '--python-version', |
|
334 help='Python version to use', |
|
335 choices={'system2', 'system3', '2.7', '3.5', '3.6', '3.7', '3.8', |
|
336 'pypy', 'pypy3.5', 'pypy3.6'}, |
|
337 default='system2', |
|
338 ) |
|
339 sp.add_argument( |
|
340 'test_flags', |
|
341 help='Extra command line flags to pass to run-tests.py', |
|
342 nargs='*', |
|
343 ) |
|
344 sp.set_defaults(func=run_tests_linux) |
|
345 |
|
346 sp = subparsers.add_parser( |
236 'run-tests-windows', |
347 'run-tests-windows', |
237 help='Run tests on Windows', |
348 help='Run tests on Windows', |
238 ) |
349 ) |
239 sp.add_argument( |
350 sp.add_argument( |
240 '--instance-type', |
351 '--instance-type', |