Created
April 16, 2020 18:47
-
-
Save jspalink/28627b39916f9d983fe77449f3d8f5ad to your computer and use it in GitHub Desktop.
List EC2-Instances
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Generate a list of EC2 instances with the information that I most need to see, including estimated monthly prices. | |
Add a line to your .bash_profile like this: | |
alias list='/usr/local/bin/python3 /path/to/ec2_instances.py' | |
This assumes that you have aws cli installed... | |
""" | |
import boto3 | |
import argparse | |
import json | |
import datetime | |
import time | |
import tempfile | |
import os | |
import pickle | |
from collections import defaultdict | |
tmpdir = tempfile.gettempdir() | |
aws_region_map = { | |
'ca-central-1': 'Canada (Central)', | |
'ap-northeast-3': 'Asia Pacific (Osaka-Local)', | |
'us-east-1': 'US East (N. Virginia)', | |
'ap-northeast-2': 'Asia Pacific (Seoul)', | |
'us-gov-west-1': 'AWS GovCloud (US)', | |
'us-east-2': 'US East (Ohio)', | |
'ap-northeast-1': 'Asia Pacific (Tokyo)', | |
'ap-south-1': 'Asia Pacific (Mumbai)', | |
'ap-southeast-2': 'Asia Pacific (Sydney)', | |
'ap-southeast-1': 'Asia Pacific (Singapore)', | |
'sa-east-1': 'South America (Sao Paulo)', | |
'us-west-2': 'US West (Oregon)', | |
'eu-west-1': 'EU (Ireland)', | |
'eu-west-3': 'EU (Paris)', | |
'eu-west-2': 'EU (London)', | |
'us-west-1': 'US West (N. California)', | |
'eu-central-1': 'EU (Frankfurt)' | |
} | |
ebs_name_map = { | |
'standard': 'Magnetic', | |
'gp2': 'General Purpose', | |
'io1': 'Provisioned IOPS', | |
'st1': 'Throughput Optimized HDD', | |
'sc1': 'Cold HDD' | |
} | |
def build_pricing_defaults(region='us-east-1'): | |
pricing_client = boto3.client('pricing', region_name=region) | |
ebs_pricing = get_ebs_pricing(client=pricing_client, region=region) | |
ondemand_pricing = get_ondemand_pricing(client=pricing_client, region=region) | |
spot_pricing = get_spot_pricing(region=region) | |
file_path = build_pricing_path('aws_ebs_prices', region) | |
if not pricing_file_is_good(file_path): | |
with open(file_path, 'wb') as f: | |
pickle.dump(ebs_pricing, f) | |
file_path = build_pricing_path('aws_spot_prices', region) | |
if not pricing_file_is_good(file_path): | |
with open(file_path, 'wb') as f: | |
pickle.dump(spot_pricing, f) | |
file_path = build_pricing_path('aws_ondemand_prices', region) | |
if not pricing_file_is_good(file_path): | |
with open(file_path, 'wb') as f: | |
pickle.dump(ondemand_pricing, f) | |
return | |
def build_pricing_path(n, region='us-east-1'): | |
return os.path.abspath(os.path.join(tmpdir, '{}-{}'.format(n, region))) | |
def pricing_file_is_good(file_path, ttl=604800): | |
return os.path.exists(file_path) and os.path.getctime(file_path) > (time.time() - 604800) | |
def get_existing(file_path, ttl=604800): | |
if pricing_file_is_good(file_path, ttl): | |
return pickle.load(open(file_path, 'rb')) | |
def get_ebs_pricing(client=None, region='us-east-1', *args, **kwargs): | |
""" | |
Returns a pricing dictionary for EBS pricing for this region | |
""" | |
price_dictionary = get_existing(build_pricing_path('aws_ebs_prices', region)) | |
if price_dictionary: | |
return price_dictionary | |
if not client: | |
client = boto3.client('pricing', region_name=region) | |
resolved_region = aws_region_map.get(region) | |
price_dictionary = dict() | |
for ebs_code in ebs_name_map: | |
response = client.get_products(ServiceCode='AmazonEC2', Filters=[ | |
{'Type': 'TERM_MATCH', 'Field': 'volumeType', 'Value': ebs_name_map[ebs_code]}, | |
{'Type': 'TERM_MATCH', 'Field': 'location', 'Value': resolved_region} | |
]) | |
for result in response['PriceList']: | |
json_result = json.loads(result) | |
for json_result_level_1 in json_result['terms']['OnDemand'].values(): | |
for json_result_level_2 in json_result_level_1['priceDimensions'].values(): | |
for price_value in json_result_level_2['pricePerUnit'].values(): | |
continue | |
price_dictionary[ebs_code] = float(price_value) | |
return price_dictionary | |
def get_spot_pricing(client=None, region='us-east-1', instance_types=None, *args, **kwargs): | |
""" | |
Return spot pricing for the specified instance sizes | |
""" | |
price_dictionary = get_existing(build_pricing_path('aws_spot_prices', region)) | |
if price_dictionary: | |
return price_dictionary | |
if not client: | |
client = boto3.client('ec2', region_name=region) | |
price_dictionary = defaultdict(dict) | |
next_token = '' | |
while True: | |
if instance_types: | |
response=client.describe_spot_price_history(ProductDescriptions=['Linux/UNIX (Amazon VPC)'], StartTime=datetime.datetime.now(), InstanceTypes=instance_types, NextToken=next_token) | |
else: | |
response=client.describe_spot_price_history(ProductDescriptions=['Linux/UNIX (Amazon VPC)'], StartTime=datetime.datetime.now(), NextToken=next_token) | |
for x in response['SpotPriceHistory']: | |
price_dictionary[x['InstanceType']][x['AvailabilityZone']] = float(x['SpotPrice']) | |
next_token = response.get('NextToken') | |
if not next_token: | |
break | |
return price_dictionary | |
def get_ondemand_pricing(client=None, region='us-east-1', instance_types=None, *args, **kwargs): | |
price_dictionary = get_existing(build_pricing_path('aws_ondemand_prices', region)) | |
if price_dictionary: | |
return price_dictionary | |
if not client: | |
client = boto3.client('pricing', region_name=region) | |
resolved_region = aws_region_map.get(region) | |
prices = dict() | |
next_token = '' | |
while True: | |
response = client.get_products( | |
ServiceCode='AmazonEC2', | |
Filters=[ | |
{'Type': 'TERM_MATCH', 'Field': 'operatingSystem', 'Value': 'Linux'}, | |
{'Type': 'TERM_MATCH', 'Field': 'tenancy', 'Value': 'Shared'}, | |
{'Type': 'TERM_MATCH', 'Field': 'location', 'Value': resolved_region}, | |
{'Type': 'TERM_MATCH', 'Field': 'capacitystatus', 'Value': 'Used'}, | |
{'Type': 'TERM_MATCH', 'Field': 'preInstalledSw', 'Value': 'NA'} | |
], | |
NextToken=next_token | |
) | |
for x in response['PriceList']: | |
p = json.loads(x) | |
instance_type = p['product']['attributes']['instanceType'] | |
instance_cost = 0.00 | |
for terms in p['terms']['OnDemand'].values(): | |
for price_dimensions in terms['priceDimensions'].values(): | |
instance_cost = float(price_dimensions['pricePerUnit'].get('USD', 0)) | |
prices[instance_type] = instance_cost | |
next_token = response.get('NextToken') | |
if not next_token: | |
break | |
return prices | |
def get_value_from_tags(tags, key): | |
if not tags: | |
return | |
for tag in tags: | |
if tag.get('Key').lower() == key.lower(): | |
return tag.get('Value') | |
def get_instance_name(i): | |
return get_instance_name_from_tags(i.tags) | |
def get_instance_name_from_tags(tags): | |
return get_value_from_tags(tags, 'name') | |
def get_division(i): | |
return get_division_from_tags(i.tags) | |
def get_division_from_tags(tags): | |
return get_value_from_tags(tags, 'division') | |
def get_owner(i): | |
return get_owner_from_tags(i.tags) | |
def get_owner_from_tags(tags): | |
return get_value_from_tags(tags, 'owner') | |
def get_projects(i): | |
return get_projects_from_tags(i.tags) | |
def get_projects_from_tags(tags): | |
project = get_value_from_tags(tags, 'project') | |
if not project: | |
project = 'unspecified' | |
return [p.strip() for p in project.split(',')] | |
def get_start_date(i): | |
if i.state.get('Name') in ('running', 'stopping', 'starting'): | |
return i.launch_time.strftime('%Y-%m-%d %H:%M') | |
return '' | |
def get_volume_size(i): | |
return sum((v.size for v in i.volumes.iterator())) | |
def get_volume_cost(ebs_prices, volumes): | |
return sum(v.size * ebs_prices[v.volume_type] for v in volumes) | |
def is_spot(i): | |
return i.instance_lifecycle == 'spot' | |
def gather_instances(instances, filter=None, states=None, owners=None, projects=None, unowned=None, region='us-east-1', *args, **kwargs): | |
filter = filter or "" | |
filters = [] | |
if filter: | |
filters.append({'Name':'tag:Name', 'Values':['*{}*'.format(filter)]}) | |
if owners: | |
values = [] | |
for o in owners: | |
values.append('*{}*'.format(o)) | |
filters.append({'Name':'tag:owner', 'Values':values}) | |
if projects: | |
values = [] | |
for p in projects: | |
values.append('*{}*'.format(p)) | |
filters.append({'Name':'tag:project', 'Values':values}) | |
if filters: | |
instance_list = list(instances.filter(Filters=filters)) | |
else: | |
instance_list = list(instances.all()) | |
instance_types = {i.instance_type for i in instance_list} | |
ebs_prices = get_ebs_pricing(region=region) | |
spot_prices = get_spot_pricing(instance_types=list(instance_types), region=region) | |
ond_prices = get_ondemand_pricing(instance_types=list(instance_types), region=region) | |
for i in instance_list: | |
tags = i.tags | |
id = i.id | |
name = get_instance_name_from_tags(tags) or '' | |
owner = get_owner_from_tags(tags) or '' | |
division = get_division_from_tags(tags) or None | |
projects = get_projects_from_tags(tags) or list() | |
status = i.state.get('Name') | |
volumes = list(i.volumes.all()) | |
volume_size = sum((v.size for v in volumes)) | |
volume_cost = get_volume_cost(ebs_prices, volumes) | |
availability_zone = i.placement.get('AvailabilityZone') | |
if states and status not in states: | |
continue | |
if unowned and owner: | |
continue | |
public_ip = i.public_ip_address or '' | |
private_ip = i.private_ip_address or '' | |
start_date = get_start_date(i) | |
instance_type = i.instance_type or '' | |
spot_instance = is_spot(i) | |
instance_price = 0 | |
if spot_instance: | |
instance_price = spot_prices[instance_type][availability_zone] * 730 # 8760 hours (in a year) / 12 | |
else: | |
instance_price = ond_prices[instance_type] * 730 # 8760 hours (in a year) / 12 | |
instance_total = volume_cost + instance_price | |
yield (id, instance_type, name.lower(), status, public_ip, private_ip, start_date, owner, spot_instance, volume_size, volume_cost, instance_price, instance_total), (owner, division, projects) | |
def print_instance_header(i): | |
print("Instances:") | |
print("–" * 203) | |
print("{!s:20} {!s:12} {!s:40} {!s:7} {!s:15} {!s:15} {!s:17} {!s:12} {!s:5} {!s:>9} {!s:>9} {!s:>9} {!s:>9}".format(*i)) | |
def print_instance(i): | |
#print(i) | |
print("{!s:20} {!s:12} {!s:40} {!s:7} {!s:15} {!s:15} {!s:17} {!s:12} {!s:5} {!s:>9} {:9.2f} {:9.2f} {:9.2f}".format(*i)) | |
def print_summary_costs(title, cost_dict): | |
print(title) | |
print("––––––––––––––––––––––––––––––––––") | |
costs = sorted(cost_dict.items(), key=lambda x: x[0]) | |
for a,b in costs: | |
print(" {!s:20}: {:9.2f}".format(a, b)) | |
print() | |
def main(filter=None, states=None, region='us-east-1', owners=None, projects=None, unowned=None, *args, **kwargs): | |
ec2 = boto3.resource('ec2', region) | |
build_pricing_defaults(region=region) | |
states = states or ('pending', 'running', 'shutting-down', 'terminated', 'stopping', 'stopped') | |
owners = owners or None | |
instances = sorted(gather_instances(ec2.instances, filter=filter, states=states, owners=owners, projects=projects, unowned=unowned, region=region), key=lambda x: (x[0][2], x[0][6])) | |
print_instance_header(('id', 'type', 'name', 'state', 'public ip', 'private ip', 'launched', 'owner', 'spot', 'disk (gb)', 'disk $/m', '$/month', 'total')) | |
project_costs = defaultdict(float) | |
owner_costs = defaultdict(float) | |
division_costs = defaultdict(float) | |
for i, (owner, division, projects) in instances: | |
print_instance(i) | |
for p in projects: | |
project_costs[p] += i[12] | |
owner_costs[owner or 'unspecified'] += i[12] | |
division_costs[division or 'unspecified'] += i[12] | |
disk_total = sum(i[10] for i, x in instances) | |
instance_total = sum(i[11] for i, x in instances) | |
total_cost = sum(i[12] for i, x in instances) | |
print("{:>181.2f} {:9.2f} {:9.2f}".format(disk_total, instance_total, total_cost)) | |
print() | |
print_summary_costs('Monthly Cost Summary By Owner', owner_costs) | |
print_summary_costs('Monthly Cost Summary By Division', division_costs) | |
print_summary_costs('Monthly Cost Summary By Project', project_costs) | |
return True | |
if __name__ == '__main__': | |
parser = argparse.ArgumentParser(description='List EC2 Instances') | |
parser.add_argument("filter", nargs='?', default=None, help="Filter by name or id") | |
parser.add_argument("-s", "--state", dest="state", default='running', nargs="*", help="Return instances in this state") | |
parser.add_argument("-r", "--region", dest="region", default="us-east-1", help="Return instances in this region") | |
parser.add_argument("-o", "--owners", dest="owners", default=None, nargs="*", help="Filter based on owners") | |
parser.add_argument("-p", "--projects", dest="projects", default=None, nargs="*", help="Filter based on projects") | |
parser.add_argument("-u", "--unowned", dest="unowned", action='store_true', help="Return instances that do not have a declared owner") | |
parser.add_argument("-t", "--tempdir", dest="temp", default=tempfile.gettempdir(), help="Temp directory to store pricing lists") | |
options = parser.parse_args() | |
tmpdir = options.temp | |
main(filter=options.filter, states=options.state, region=options.region, owners=options.owners, projects=options.projects, unowned=options.unowned) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment