Merge branch '8784-dir-listings'

[arvados.git] / services / nodemanager / arvnodeman / computenode / driver / ec2.py

diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index 255a948a6c3aa0ee2c17ae1581685d282c341813..1c1b51b829377aa2daa001165a852aa152a93bce 100644 (file)
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
 
 from __future__ import absolute_import, print_function
 
@@ -52,19 +55,10 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         super(ComputeNodeDriver, self).__init__(
             auth_kwargs, {'ex_filters': list_kwargs}, create_kwargs,
             driver_class)
-        for key in self.create_kwargs.keys():
-            init_method = getattr(self, '_init_' + key, None)
-            if init_method is not None:
-                new_pair = init_method(self.create_kwargs.pop(key))
-                if new_pair is not None:
-                    self.create_kwargs[new_pair[0]] = new_pair[1]
 
     def _init_image_id(self, image_id):
         return 'image', self.search_for(image_id, 'list_images')
 
-    def _init_ping_host(self, ping_host):
-        self.ping_host = ping_host
-
     def _init_security_groups(self, group_names):
         return 'ex_security_groups', [
             self.search_for(gname.strip(), 'ex_get_security_groups')
@@ -73,20 +67,28 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
     def _init_subnet_id(self, subnet_id):
         return 'ex_subnet', self.search_for(subnet_id, 'ex_list_subnets')
 
-    def _init_ssh_key(self, filename):
-        with open(filename) as ssh_file:
-            key = cloud_base.NodeAuthSSHKey(ssh_file.read())
-        return 'auth', key
-
-    def arvados_create_kwargs(self, arvados_node):
-        result = {'name': arvados_node_fqdn(arvados_node)}
-        ping_secret = arvados_node['info'].get('ping_secret')
-        if ping_secret is not None:
-            ping_url = ('https://{}/arvados/v1/nodes/{}/ping?ping_secret={}'.
-                        format(self.ping_host, arvados_node['uuid'],
-                               ping_secret))
-            result['ex_userdata'] = ping_url
-        return result
+    create_cloud_name = staticmethod(arvados_node_fqdn)
+
+    def arvados_create_kwargs(self, size, arvados_node):
+        kw = {'name': self.create_cloud_name(arvados_node),
+                'ex_userdata': self._make_ping_url(arvados_node)}
+        # libcloud/ec2 disk sizes are in GB, Arvados/SLURM "scratch" value is in MB
+        scratch = int(size.scratch / 1000) + 1
+        if scratch > size.disk:
+            volsize = scratch - size.disk
+            if volsize > 16384:
+                # Must be 1-16384 for General Purpose SSD (gp2) devices
+                # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_EbsBlockDevice.html
+                self._logger.warning("Requested EBS volume size %d is too large, capping size request to 16384 GB", volsize)
+                volsize = 16384
+            kw["ex_blockdevicemappings"] = [{
+                "DeviceName": "/dev/xvdt",
+                "Ebs": {
+                    "DeleteOnTermination": True,
+                    "VolumeSize": volsize,
+                    "VolumeType": "gp2"
+                }}]
+        return kw
 
     def post_create_node(self, cloud_node):
         self.real.ex_create_tags(cloud_node, self.tags)
@@ -95,8 +97,24 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         self.real.ex_create_tags(cloud_node,
                                  {'Name': arvados_node_fqdn(arvados_node)})
 
+    def list_nodes(self):
+        # Need to populate Node.size
+        nodes = super(ComputeNodeDriver, self).list_nodes()
+        for n in nodes:
+            if not n.size:
+                n.size = self.sizes[n.extra["instance_type"]]
+        return nodes
+
+    @classmethod
+    def node_fqdn(cls, node):
+        return node.name
+
     @classmethod
     def node_start_time(cls, node):
         time_str = node.extra['launch_time'].split('.', 2)[0] + 'UTC'
         return time.mktime(time.strptime(
                 time_str,'%Y-%m-%dT%H:%M:%S%Z')) - time.timezone
+
+    @classmethod
+    def node_id(cls, node):
+        return node.id