2 # Copyright (C) The Arvados Authors. All rights reserved.
4 # SPDX-License-Identifier: AGPL-3.0
6 from __future__ import absolute_import, print_function
10 import libcloud.compute.base as cloud_base
11 import libcloud.compute.providers as cloud_provider
12 import libcloud.compute.types as cloud_types
13 from libcloud.compute.drivers import ec2 as cloud_ec2
15 from . import BaseComputeNodeDriver
16 from .. import arvados_node_fqdn
18 ### Monkeypatch libcloud to support AWS' new SecurityGroup API.
19 # These classes can be removed when libcloud support specifying
20 # security groups with the SecurityGroupId parameter.
21 class ANMEC2Connection(cloud_ec2.EC2Connection):
22 def request(self, *args, **kwargs):
23 params = kwargs.get('params')
24 if (params is not None) and (params.get('Action') == 'RunInstances'):
25 for key in params.keys():
26 if key.startswith('SecurityGroup.'):
27 new_key = key.replace('Group.', 'GroupId.', 1)
28 params[new_key] = params.pop(key).id
29 kwargs['params'] = params
30 return super(ANMEC2Connection, self).request(*args, **kwargs)
33 class ANMEC2NodeDriver(cloud_ec2.EC2NodeDriver):
34 connectionCls = ANMEC2Connection
37 class ComputeNodeDriver(BaseComputeNodeDriver):
38 """Compute node driver wrapper for EC2.
40 This translates cloud driver requests to EC2's specific parameters.
42 DEFAULT_DRIVER = ANMEC2NodeDriver
46 def __init__(self, auth_kwargs, list_kwargs, create_kwargs,
47 driver_class=DEFAULT_DRIVER):
48 # We need full lists of keys up front because these loops modify
49 # dictionaries in-place.
50 for key in list_kwargs.keys():
51 list_kwargs[key.replace('_', ':')] = list_kwargs.pop(key)
52 self.tags = {key[4:]: value
53 for key, value in list_kwargs.iteritems()
54 if key.startswith('tag:')}
55 super(ComputeNodeDriver, self).__init__(
56 auth_kwargs, {'ex_filters': list_kwargs}, create_kwargs,
59 def _init_image_id(self, image_id):
60 return 'image', self.search_for(image_id, 'list_images')
62 def _init_security_groups(self, group_names):
63 return 'ex_security_groups', [
64 self.search_for(gname.strip(), 'ex_get_security_groups')
65 for gname in group_names.split(',')]
67 def _init_subnet_id(self, subnet_id):
68 return 'ex_subnet', self.search_for(subnet_id, 'ex_list_subnets')
70 create_cloud_name = staticmethod(arvados_node_fqdn)
72 def arvados_create_kwargs(self, size, arvados_node):
73 kw = {'name': self.create_cloud_name(arvados_node),
74 'ex_userdata': self._make_ping_url(arvados_node)}
75 # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
76 scratch = int(size.scratch / 1000) + 1
77 if scratch > size.disk:
78 volsize = scratch - size.disk
80 # Must be 1-16384 for General Purpose SSD (gp2) devices
81 # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_EbsBlockDevice.html
82 self._logger.warning("Requested EBS volume size %d is too large, capping size request to 16384 GB", volsize)
84 kw["ex_blockdevicemappings"] = [{
85 "DeviceName": "/dev/xvdt",
87 "DeleteOnTermination": True,
88 "VolumeSize": volsize,
93 def post_create_node(self, cloud_node):
94 self.real.ex_create_tags(cloud_node, self.tags)
96 def sync_node(self, cloud_node, arvados_node):
97 self.real.ex_create_tags(cloud_node,
98 {'Name': arvados_node_fqdn(arvados_node)})
100 def list_nodes(self):
101 # Need to populate Node.size
102 nodes = super(ComputeNodeDriver, self).list_nodes()
105 n.size = self.sizes[n.extra["instance_type"]]
109 def node_fqdn(cls, node):
113 def node_start_time(cls, node):
114 time_str = node.extra['launch_time'].split('.', 2)[0] + 'UTC'
115 return time.mktime(time.strptime(
116 time_str,'%Y-%m-%dT%H:%M:%S%Z')) - time.timezone
119 def node_id(cls, node):