This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6286

automation: wait for instance profiles and roles
ClosedPublic

Authored by indygreg on Apr 19 2019, 10:41 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG8dc22a209420: automation: wait for instance profiles and roles
Summary

Otherwise there is a race condition between creating the resources
and us attempting to use them / them becoming available.

The role waiter API was recently introduced, so we had to upgrade
the boto3 package to get it. Other packages were also updated
to latest versions just because.

Even with this change, I still run into issues with the IAM instance
profile not being available when we attempt to create an EC2 instance
using a just-created profile. I'm not sure what's going on. Possibly
a bug on Amazon's end. But the new behavior is "more correct."

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Apr 19 2019, 10:41 AM
indygreg updated this revision to Diff 14933.Apr 27 2019, 2:51 PM
Closed by commit rHG8dc22a209420: automation: wait for instance profiles and roles (authored by indygreg). · Explain WhyMay 15 2019, 11:46 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 14865

contrib/automation/hgautomation/aws.py

self.ec2resource = self.session.resource('ec2') self.ec2resource = self.session.resource('ec2')
self.iamclient = self.session.client('iam') self.iamclient = self.session.client('iam')
self.iamresource = self.session.resource('iam') self.iamresource = self.session.resource('iam')
self.security_groups = {} self.security_groups = {}
if ensure_ec2_state: if ensure_ec2_state:
ensure_key_pairs(automation.state_path, self.ec2resource) ensure_key_pairs(automation.state_path, self.ec2resource)
self.security_groups = ensure_security_groups(self.ec2resource) self.security_groups = ensure_security_groups(self.ec2resource)
ensure_iam_state(self.iamresource) ensure_iam_state(self.iamclient, self.iamresource)
def key_pair_path_private(self, name): def key_pair_path_private(self, name):
"""Path to a key pair private key file.""" """Path to a key pair private key file."""
return self.local_state_path / 'keys' / ('keypair-%s' % name) return self.local_state_path / 'keys' / ('keypair-%s' % name)
def key_pair_path_public(self, name): def key_pair_path_public(self, name):
return self.local_state_path / 'keys' / ('keypair-%s.pub' % name) return self.local_state_path / 'keys' / ('keypair-%s.pub' % name)
print('removing role %s from instance profile %s' % (role.name, print('removing role %s from instance profile %s' % (role.name,
profile.name)) profile.name))
profile.remove_role(RoleName=role.name) profile.remove_role(RoleName=role.name)
print('deleting instance profile %s' % profile.name) print('deleting instance profile %s' % profile.name)
profile.delete() profile.delete()
def ensure_iam_state(iamresource, prefix='hg-'): def ensure_iam_state(iamclient, iamresource, prefix='hg-'):
"""Ensure IAM state is in sync with our canonical definition.""" """Ensure IAM state is in sync with our canonical definition."""
remote_profiles = {} remote_profiles = {}
for profile in iamresource.instance_profiles.all(): for profile in iamresource.instance_profiles.all():
if profile.name.startswith(prefix): if profile.name.startswith(prefix):
remote_profiles[profile.name[len(prefix):]] = profile remote_profiles[profile.name[len(prefix):]] = profile
for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)): for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)):
actual = '%s%s' % (prefix, name) actual = '%s%s' % (prefix, name)
print('creating IAM instance profile %s' % actual) print('creating IAM instance profile %s' % actual)
profile = iamresource.create_instance_profile( profile = iamresource.create_instance_profile(
InstanceProfileName=actual) InstanceProfileName=actual)
remote_profiles[name] = profile remote_profiles[name] = profile
waiter = iamclient.get_waiter('instance_profile_exists')
waiter.wait(InstanceProfileName=actual)
print('IAM instance profile %s is available' % actual)
for name in sorted(set(IAM_ROLES) - set(remote_roles)): for name in sorted(set(IAM_ROLES) - set(remote_roles)):
entry = IAM_ROLES[name] entry = IAM_ROLES[name]
actual = '%s%s' % (prefix, name) actual = '%s%s' % (prefix, name)
print('creating IAM role %s' % actual) print('creating IAM role %s' % actual)
role = iamresource.create_role( role = iamresource.create_role(
RoleName=actual, RoleName=actual,
Description=entry['description'], Description=entry['description'],
AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT, AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
) )
waiter = iamclient.get_waiter('role_exists')
waiter.wait(RoleName=actual)
print('IAM role %s is available' % actual)
remote_roles[name] = role remote_roles[name] = role
for arn in entry['policy_arns']: for arn in entry['policy_arns']:
print('attaching policy %s to %s' % (arn, role.name)) print('attaching policy %s to %s' % (arn, role.name))
role.attach_policy(PolicyArn=arn) role.attach_policy(PolicyArn=arn)
# Now reconcile state of profiles. # Now reconcile state of profiles.
for name, meta in sorted(IAM_INSTANCE_PROFILES.items()): for name, meta in sorted(IAM_INSTANCE_PROFILES.items()):

contrib/automation/requirements.txt

# #
# This file is autogenerated by pip-compile # This file is autogenerated by pip-compile
# To update, run: # To update, run:
# #
# pip-compile -U --generate-hashes --output-file contrib/automation/requirements.txt contrib/automation/requirements.txt.in # pip-compile -U --generate-hashes --output-file contrib/automation/requirements.txt contrib/automation/requirements.txt.in
# #
asn1crypto==0.24.0 \ asn1crypto==0.24.0 \
--hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
--hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \ --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \
# via cryptography # via cryptography
boto3==1.9.111 \ boto3==1.9.133 \
--hash=sha256:06414c75d1f62af7d04fd652b38d1e4fd3cfd6b35bad978466af88e2aaecd00d \ --hash=sha256:9fedb9ca5f5c32672f633db18d82b33bb8792506d6298602d356c16889ce504e \
--hash=sha256:f3b77dff382374773d02411fa47ee408f4f503aeebd837fd9dc9ed8635bc5e8e --hash=sha256:e63831daad499191811851e6c6016b1fc710fc2d92ddc221288b26f6bdea8f74
botocore==1.12.111 \ botocore==1.12.133 \
--hash=sha256:6af473c52d5e3e7ff82de5334e9fee96b2d5ec2df5d78bc00cd9937e2573a7a8 \ --hash=sha256:3b5968fc897b590c2b57fd6105b52ba8bdf5eb2100b7e181d4c17c7c05a2f83a \
--hash=sha256:9f5123c7be704b17aeacae99b5842ab17bda1f799dd29134de8c70e0a50a45d7 \ --hash=sha256:cc270cc2c282b2679f44bd1de011a270c4d8b5364afc2f705152ca187821d4eb \
# via boto3, s3transfer # via boto3, s3transfer
certifi==2019.3.9 \ certifi==2019.3.9 \
--hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \ --hash=sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5 \
--hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \ --hash=sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae \
# via requests # via requests
cffi==1.12.2 \ cffi==1.12.2 \
--hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \ --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \
--hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \ --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \
idna==2.8 \ idna==2.8 \
--hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
--hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \ --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
# via requests # via requests
jmespath==0.9.4 \ jmespath==0.9.4 \
--hash=sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6 \ --hash=sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6 \
--hash=sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c \ --hash=sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c \
# via boto3, botocore # via boto3, botocore
ntlm-auth==1.2.0 \ ntlm-auth==1.3.0 \
--hash=sha256:7bc02a3fbdfee7275d3dc20fce8028ed8eb6d32364637f28be9e9ae9160c6d5c \ --hash=sha256:bb2fd03c665f0f62c5f65695b62dcdb07fb7a45df6ebc86c770be2054d6902dd \
--hash=sha256:9b13eaf88f16a831637d75236a93d60c0049536715aafbf8190ba58a590b023e \ --hash=sha256:ce5b4483ed761f341a538a426a71a52e5a9cf5fd834ebef1d2090f9eef14b3f8 \
# via pypsrp # via pypsrp
pycparser==2.19 \ pycparser==2.19 \
--hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
# via cffi # via cffi
pypsrp==0.3.1 \ pypsrp==0.3.1 \
--hash=sha256:309853380fe086090a03cc6662a778ee69b1cae355ae4a932859034fd76e9d0b \ --hash=sha256:309853380fe086090a03cc6662a778ee69b1cae355ae4a932859034fd76e9d0b \
--hash=sha256:90f946254f547dc3493cea8493c819ab87e152a755797c93aa2668678ba8ae85 --hash=sha256:90f946254f547dc3493cea8493c819ab87e152a755797c93aa2668678ba8ae85
python-dateutil==2.8.0 \ python-dateutil==2.8.0 \
--hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \ --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
--hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \ --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
# via botocore # via botocore
requests==2.21.0 \ requests==2.21.0 \
--hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \ --hash=sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e \
--hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \ --hash=sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b \
# via pypsrp # via pypsrp
s3transfer==0.2.0 \ s3transfer==0.2.0 \
--hash=sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e \ --hash=sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e \
--hash=sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021 \ --hash=sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021 \
# via boto3 # via boto3
six==1.12.0 \ six==1.12.0 \
--hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
--hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
# via cryptography, pypsrp, python-dateutil # via cryptography, pypsrp, python-dateutil
urllib3==1.24.1 \ urllib3==1.24.2 \
--hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \
--hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \ --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3 \
# via botocore, requests # via botocore, requests