aws-instance-types/update_files.py at master · CloudVE/aws-instance-types · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
import json
import requests
import sys

import cachetools

from cloudbridge.factory import CloudProviderFactory
from cloudbridge.factory import ProviderList

aws = CloudProviderFactory().create_provider(ProviderList.AWS, {})

info_json_url = 'https://cdn.rawgit.com/powdahound/ec2instances' \
                '.info/master/www/instances.json'

vmtypes_dirpath = 'vmtypes/'


@cachetools.cached(cachetools.TTLCache(maxsize=1, ttl=24*3600))
def get_all_info_from_json_file():
    print("Getting all VM Types")
    r = requests.get(info_json_url)
    print("Successfully reached: " + info_json_url)
    data = r.json()
    # keeping entire dict to not invalidate "extra_data" property in
    # CloudBridge
    vm_types = {}
    for entry in data:
        vm_types[entry['instance_type']] = entry
    return vm_types


def get_info_for_vm_type(vm_type_id):
    all_info = get_all_info_from_json_file()
    vmtype_info = all_info.get(vm_type_id, None)
    if vmtype_info and vmtype_info.get("pricing", None):
    	# Removing pricing information to make files smaller
    	vmtype_info.pop('pricing')
    return vmtype_info

def get_all_region_names():
    return [region['RegionName'] for region in
            aws.ec2_conn.meta.client.describe_regions().get('Regions')]


def get_all_zones_in_region(region_name):
    aws._ec2_conn = aws._connect_ec2_region(region_name)
    allz = (aws.ec2_conn.meta.client
               .describe_availability_zones(Filters=[{'Name': 'region-name',
                                                      'Values':[region_name]}])
               .get('AvailabilityZones'))
    return [zone.get('ZoneName') for zone in allz]


def list_vm_types_in_zone(zone_name):
    result = (aws.ec2_conn.meta.client
                 .describe_reserved_instances_offerings(
                     AvailabilityZone=zone_name))
    ids = list(set(offering.get('InstanceType') for offering
                   in result.get('ReservedInstancesOfferings')))
    # Stop if 10 pages in a row did not add any VM
    num = len(ids)
    count_same = 0
    while result.get('NextToken') and count_same < 50:
        result = (aws.ec2_conn.meta.client
                     .describe_reserved_instances_offerings(
                         AvailabilityZone=zone_name,
                         NextToken=result.get('NextToken')))
        for offering in result.get('ReservedInstancesOfferings'):
            vm_type_id = offering.get('InstanceType')
            if vm_type_id not in ids:
                ids.append(vm_type_id)
        if len(ids) == num:
            count_same += 1
        else:
            count_same = 0
            num = len(ids)
    # Removing instances that we don't have information about from the
    # json file. eg: r5.metal, m5d.metal, z1d.metal
    print("Total {} vmtypes after gathering all pages".format(len(ids)))
    vm_types = []
    for type_id in ids:
        type_dict = get_info_for_vm_type(type_id)
        if type_dict:
            vm_types.append(type_dict)
    return vm_types


def update_all_files():
    regions = get_all_region_names()
    print("Regions for which zones will be updated:")
    print(regions)
    for region in regions:
        zones = get_all_zones_in_region(region)
        for zone in zones:
            print("Zone currently being updated:")
            print(zone)
            with open(vmtypes_dirpath + zone + '.json', 'w') as file:
                file.write(json.dumps((list_vm_types_in_zone(zone))))


# def update_files_for_second_third():
#     regions = get_all_region_names()
#     regions = regions[int(len(regions)/3):2*int(len(regions)/3)]
#     print("Regions for which zones will be updated:")
#     print(regions)
#     for region in regions:
#         zones = get_all_zones_in_region(region)
#         for zone in zones:
#             print("Zone currently being updated:")
#             print(zone)
#             with open(vmtypes_dirpath + zone + '.json', 'w') as file:
#                 file.write(json.dumps((list_vm_types_in_zone(zone))))


# def update_files_for_last_third():
#     regions = get_all_region_names()
#     regions = regions[2*int(len(regions)/3):]
#     print("Regions for which zones will be updated:")
#     print(regions)
#     for region in regions:
#         zones = get_all_zones_in_region(region)
#         for zone in zones:
#             print("Zone currently being updated:")
#             print(zone)
#             with open(vmtypes_dirpath + zone + '.json', 'w') as file:
#                 file.write(json.dumps((list_vm_types_in_zone(zone))))


def update_files_test():
    regions = get_all_region_names()
    regions = regions[0:1]
    print("Regions for which zones will be updated:")
    print(regions)
    for region in regions:
        zones = get_all_zones_in_region(region)
        for zone in zones[0:1]:
            print("Zone currently being updated:")
            print(zone)
            with open(vmtypes_dirpath + zone + '.json', 'w') as file:
                file.write(json.dumps((list_vm_types_in_zone(zone))))


def main():
    part = sys.argv[1]
    if part == 'test':
        update_files_test()
    elif part == 'all':
        update_all_files()
    else:
        msg = 'The script takes a string argument.\n' \
              'Accepted values are: [all, test]\n' \
              'Example: python3 update_files.py first\n'
        print(msg)


if __name__ == '__main__':
    main()
    print("Done :)")