diff --git a/contrib/ci/lambda_functions/ci.py b/contrib/ci/lambda_functions/ci.py --- a/contrib/ci/lambda_functions/ci.py +++ b/contrib/ci/lambda_functions/ci.py @@ -116,6 +116,7 @@ state = event['detail']['state'] print('received %s for %s' % (state, instance_id)) + ec2_client = boto3.client('ec2') ec2 = boto3.resource('ec2') dynamodb = boto3.resource('dynamodb') @@ -132,7 +133,7 @@ job_table = dynamodb.Table(os.environ['DYNAMODB_JOB_TABLE']) - react_to_instance_state_change(job_table, instance, state) + react_to_instance_state_change(ec2_client, job_table, instance, state) def handle_try_server_upload(event, context): @@ -645,7 +646,7 @@ ) -def react_to_instance_state_change(job_table, instance, state): +def react_to_instance_state_change(ec2, job_table, instance, state): """React to a CI worker instance state change.""" now = decimal.Decimal(time.time()) @@ -690,17 +691,32 @@ # New instance/job seen. Record that. if state == 'pending': print('recording running state for job %s' % job_id) + + # Try to record the cost to running this instance. + hourly_cost = None + + if instance.spot_instance_request_id: + spot_instance_requests = ec2.describe_spot_instance_requests( + SpotInstanceRequestIds=[instance.spot_instance_request_id], + )['SpotInstanceRequests'] + + if spot_instance_requests: + hourly_cost = decimal.Decimal( + spot_instance_requests[0]['ActualBlockHourlyPrice']) + job_table.update_item( Key={'job_id': job_id}, UpdateExpression=( 'set execution_state = :state, ' 'instance_id = :instance_id, ' + 'instance_hourly_cost = :hourly_cost, ' 'start_time = :start_time, ' 'exit_clean = :exit_clean' ), ExpressionAttributeValues={ ':state': 'running', ':instance_id': instance.instance_id, + ':hourly_cost': hourly_cost, ':start_time': now, ':exit_clean': False, }, diff --git a/contrib/ci/lambda_functions/web.py b/contrib/ci/lambda_functions/web.py --- a/contrib/ci/lambda_functions/web.py +++ b/contrib/ci/lambda_functions/web.py @@ -115,6 +115,7 @@ 'Scheduled At', 'Start Delay', 'Execution Time', + 'Cost', 'Total Tests', 'Passed', 'Failed', @@ -136,14 +137,28 @@ start_time = datetime.datetime.utcfromtimestamp(job_info['start_time']) start_delay = '%ds' % (start_time - schedule_time).total_seconds() else: + start_time = None start_delay = 'n/a' if 'end_time' in job_info: end_time = datetime.datetime.utcfromtimestamp(job_info['end_time']) execution_time = '%ds' % (end_time - start_time).total_seconds() + + instance_time = (end_time - start_time).total_seconds() else: execution_time = 'n/a' + if start_time is not None: + instance_time = (datetime.datetime.utcnow() - start_time).total_seconds() + else: + instance_time = None + + if 'instance_hourly_cost' in job_info and instance_time is not None: + total_cost = float(job_info['instance_hourly_cost'] )/ 3600.0 * instance_time + total_cost = '$%.3f' % total_cost + else: + total_cost = 'n/a' + if 'test_count' in job_info: test_count = '%d' % job_info['test_count'] else: @@ -207,6 +222,7 @@ '%s' % schedule_time.isoformat(), '%s' % start_delay, '%s' % execution_entry, + '%s' % e(total_cost), '%s' % test_count, '%s' % pass_count, '%s' % fail_entry, diff --git a/contrib/ci/terraform/job_executor.tf b/contrib/ci/terraform/job_executor.tf --- a/contrib/ci/terraform/job_executor.tf +++ b/contrib/ci/terraform/job_executor.tf @@ -142,6 +142,7 @@ "ec2:CreateTags", "ec2:DescribeInstanceAttribute", "ec2:DescribeInstances", + "ec2:DescribeSpotInstanceRequests", ] resources = ["*"] }