Mercurial > public > mercurial-scm > hg

comparison contrib/automation/hgautomation/aws.py @ 42284:195dcc10b3d7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
automation: move image operations to own functions An upcoming commit will need this functionality with slightly different values and it is enough code to not want to duplicate. Let's refactor into standalone functions so it can be reused. Differential Revision: https://phab.mercurial-scm.org/D6318
author Gregory Szorc <gregory.szorc@gmail.com>
date Tue, 23 Apr 2019 21:57:32 -0700
parents e570106beda1
children 65b3ef162b39
comparison
equal deleted inserted replaced
42283:d137a3d5ad41 42284:195dcc10b3d7
400 for role in sorted(wanted - have): 400 for role in sorted(wanted - have):
401 print('adding role %s to %s' % (role, profile.name)) 401 print('adding role %s to %s' % (role, profile.name))
402 profile.add_role(RoleName=role) 402 profile.add_role(RoleName=role)
403 403
404 404
405 def find_windows_server_2019_image(ec2resource): 405 def find_image(ec2resource, owner_id, name):
406 """Find the Amazon published Windows Server 2019 base image.""" 406 """Find an AMI by its owner ID and name."""
407 407
408 images = ec2resource.images.filter( 408 images = ec2resource.images.filter(
409 Filters=[ 409 Filters=[
410 { 410 {
411 'Name': 'owner-alias', 411 'Name': 'owner-id',
412 'Values': ['amazon'], 412 'Values': [owner_id],
413 }, 413 },
414 { 414 {
415 'Name': 'state', 415 'Name': 'state',
416 'Values': ['available'], 416 'Values': ['available'],
417 }, 417 },
419 'Name': 'image-type', 419 'Name': 'image-type',
420 'Values': ['machine'], 420 'Values': ['machine'],
421 }, 421 },
422 { 422 {
423 'Name': 'name', 423 'Name': 'name',
424 'Values': ['Windows_Server-2019-English-Full-Base-2019.02.13'], 424 'Values': [name],
425 }, 425 },
426 ]) 426 ])
427 427
428 for image in images: 428 for image in images:
429 return image 429 return image
430 430
431 raise Exception('unable to find Windows Server 2019 image') 431 raise Exception('unable to find image for %s' % name)
432 432
433 433
434 def ensure_security_groups(ec2resource, prefix='hg-'): 434 def ensure_security_groups(ec2resource, prefix='hg-'):
435 """Ensure all necessary Mercurial security groups are present. 435 """Ensure all necessary Mercurial security groups are present.
436 436
682 instance.winrm_client = client 682 instance.winrm_client = client
683 683
684 yield instances 684 yield instances
685 685
686 686
687 def resolve_fingerprint(fingerprint):
688 fingerprint = json.dumps(fingerprint, sort_keys=True)
689 return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
690
691
692 def find_and_reconcile_image(ec2resource, name, fingerprint):
693 """Attempt to find an existing EC2 AMI with a name and fingerprint.
694
695 If an image with the specified fingerprint is found, it is returned.
696 Otherwise None is returned.
697
698 Existing images for the specified name that don't have the specified
699 fingerprint or are missing required metadata or deleted.
700 """
701 # Find existing AMIs with this name and delete the ones that are invalid.
702 # Store a reference to a good image so it can be returned one the
703 # image state is reconciled.
704 images = ec2resource.images.filter(
705 Filters=[{'Name': 'name', 'Values': [name]}])
706
707 existing_image = None
708
709 for image in images:
710 if image.tags is None:
711 print('image %s for %s lacks required tags; removing' % (
712 image.id, image.name))
713 remove_ami(ec2resource, image)
714 else:
715 tags = {t['Key']: t['Value'] for t in image.tags}
716
717 if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
718 existing_image = image
719 else:
720 print('image %s for %s has wrong fingerprint; removing' % (
721 image.id, image.name))
722 remove_ami(ec2resource, image)
723
724 return existing_image
725
726
727 def create_ami_from_instance(ec2client, instance, name, description,
728 fingerprint):
729 """Create an AMI from a running instance.
730
731 Returns the ``ec2resource.Image`` representing the created AMI.
732 """
733 instance.stop()
734
735 ec2client.get_waiter('instance_stopped').wait(
736 InstanceIds=[instance.id],
737 WaiterConfig={
738 'Delay': 5,
739 })
740 print('%s is stopped' % instance.id)
741
742 image = instance.create_image(
743 Name=name,
744 Description=description,
745 )
746
747 image.create_tags(Tags=[
748 {
749 'Key': 'HGIMAGEFINGERPRINT',
750 'Value': fingerprint,
751 },
752 ])
753
754 print('waiting for image %s' % image.id)
755
756 ec2client.get_waiter('image_available').wait(
757 ImageIds=[image.id],
758 )
759
760 print('image %s available as %s' % (image.id, image.name))
761
762 return image
763
764
687 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'): 765 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'):
688 """Ensure Windows Development AMI is available and up-to-date. 766 """Ensure Windows Development AMI is available and up-to-date.
689 767
690 If necessary, a modern AMI will be built by starting a temporary EC2 768 If necessary, a modern AMI will be built by starting a temporary EC2
691 instance and bootstrapping it. 769 instance and bootstrapping it.
699 ec2client = c.ec2client 777 ec2client = c.ec2client
700 ec2resource = c.ec2resource 778 ec2resource = c.ec2resource
701 ssmclient = c.session.client('ssm') 779 ssmclient = c.session.client('ssm')
702 780
703 name = '%s%s' % (prefix, 'windows-dev') 781 name = '%s%s' % (prefix, 'windows-dev')
782
783 image = find_image(ec2resource,
784 '801119661308',
785 'Windows_Server-2019-English-Full-Base-2019.02.13')
704 786
705 config = { 787 config = {
706 'BlockDeviceMappings': [ 788 'BlockDeviceMappings': [
707 { 789 {
708 'DeviceName': '/dev/sda1', 790 'DeviceName': '/dev/sda1',
711 'VolumeSize': 32, 793 'VolumeSize': 32,
712 'VolumeType': 'gp2', 794 'VolumeType': 'gp2',
713 }, 795 },
714 } 796 }
715 ], 797 ],
716 'ImageId': find_windows_server_2019_image(ec2resource).id, 798 'ImageId': image.id,
717 'InstanceInitiatedShutdownBehavior': 'stop', 799 'InstanceInitiatedShutdownBehavior': 'stop',
718 'InstanceType': 't3.medium', 800 'InstanceType': 't3.medium',
719 'KeyName': '%sautomation' % prefix, 801 'KeyName': '%sautomation' % prefix,
720 'MaxCount': 1, 802 'MaxCount': 1,
721 'MinCount': 1, 803 'MinCount': 1,
746 commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true') 828 commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true')
747 commands.append('Set-MpPreference -DisableRealtimeMonitoring $false') 829 commands.append('Set-MpPreference -DisableRealtimeMonitoring $false')
748 830
749 # Compute a deterministic fingerprint to determine whether image needs 831 # Compute a deterministic fingerprint to determine whether image needs
750 # to be regenerated. 832 # to be regenerated.
751 fingerprint = { 833 fingerprint = resolve_fingerprint({
752 'instance_config': config, 834 'instance_config': config,
753 'user_data': WINDOWS_USER_DATA, 835 'user_data': WINDOWS_USER_DATA,
754 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, 836 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
755 'bootstrap_commands': commands, 837 'bootstrap_commands': commands,
756 } 838 })
757 839
758 fingerprint = json.dumps(fingerprint, sort_keys=True) 840 existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
759 fingerprint = hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
760
761 # Find existing AMIs with this name and delete the ones that are invalid.
762 # Store a reference to a good image so it can be returned one the
763 # image state is reconciled.
764 images = ec2resource.images.filter(
765 Filters=[{'Name': 'name', 'Values': [name]}])
766
767 existing_image = None
768
769 for image in images:
770 if image.tags is None:
771 print('image %s for %s lacks required tags; removing' % (
772 image.id, image.name))
773 remove_ami(ec2resource, image)
774 else:
775 tags = {t['Key']: t['Value'] for t in image.tags}
776
777 if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
778 existing_image = image
779 else:
780 print('image %s for %s has wrong fingerprint; removing' % (
781 image.id, image.name))
782 remove_ami(ec2resource, image)
783 841
784 if existing_image: 842 if existing_image:
785 return existing_image 843 return existing_image
786 844
787 print('no suitable Windows development image found; creating one...') 845 print('no suitable Windows development image found; creating one...')
837 895
838 print('bootstrapping instance...') 896 print('bootstrapping instance...')
839 run_powershell(instance.winrm_client, '\n'.join(commands)) 897 run_powershell(instance.winrm_client, '\n'.join(commands))
840 898
841 print('bootstrap completed; stopping %s to create image' % instance.id) 899 print('bootstrap completed; stopping %s to create image' % instance.id)
842 instance.stop() 900 return create_ami_from_instance(ec2client, instance, name,
843 901 'Mercurial Windows development environment',
844 ec2client.get_waiter('instance_stopped').wait( 902 fingerprint)
845 InstanceIds=[instance.id],
846 WaiterConfig={
847 'Delay': 5,
848 })
849 print('%s is stopped' % instance.id)
850
851 image = instance.create_image(
852 Name=name,
853 Description='Mercurial Windows development environment',
854 )
855
856 image.create_tags(Tags=[
857 {
858 'Key': 'HGIMAGEFINGERPRINT',
859 'Value': fingerprint,
860 },
861 ])
862
863 print('waiting for image %s' % image.id)
864
865 ec2client.get_waiter('image_available').wait(
866 ImageIds=[image.id],
867 )
868
869 print('image %s available as %s' % (image.id, image.name))
870
871 return image
872 903
873 904
874 @contextlib.contextmanager 905 @contextlib.contextmanager
875 def temporary_windows_dev_instances(c: AWSConnection, image, instance_type, 906 def temporary_windows_dev_instances(c: AWSConnection, image, instance_type,
876 prefix='hg-', disable_antivirus=False): 907 prefix='hg-', disable_antivirus=False):