From: Tim Pierce <twp@curoverse.com>
Date: Wed, 21 Jan 2015 18:06:35 +0000 (-0500)
Subject: 4138: general GCE fixes
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/2b6cfad3ac09667655e5e5a745cffe97b215ab4b?ds=sidebyside

4138: general GCE fixes

* JSON credential file
** GCE credentials are delivered as a JSON string (and the key is formatted as a multi-line RSA private key). Let the GCE config file specify a path to the JSON credential file for simplicity.
* Accept NodeSizes addressed by id or name
** In EC2, NodeSizes are identified by the 'id' field.  In GCE they are identified by the 'name' field.  Allow the Node Manager config module to accept either.
---

diff --git a/services/nodemanager/arvnodeman/computenode/driver/gce.py b/services/nodemanager/arvnodeman/computenode/driver/gce.py
index a4fd57deee..125a0909c4 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/gce.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/gce.py
@@ -38,7 +38,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
                     self.create_kwargs[new_pair[0]] = new_pair[1]
 
     def _init_image_id(self, image_id):
-        return 'image', self.search_for(image_id, 'list_images')
+        return 'image', image_id
 
     def _init_ping_host(self, ping_host):
         self.ping_host = ping_host
@@ -47,7 +47,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
         self.service_accounts = json.loads(service_accounts_str)
 
     def _init_network_id(self, subnet_id):
-        return 'ex_network', self.search_for(subnet_id, 'ex_list_networks')
+        return 'ex_network', subnet_id
 
     def _init_ssh_key(self, filename):
         with open(filename) as ssh_file:
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index e15f0233b9..f386653e98 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function
 
 import ConfigParser
 import importlib
+import json
 import logging
 import ssl
 import sys
@@ -98,6 +99,13 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
         module = importlib.import_module('arvnodeman.computenode.driver.' +
                                          self.get('Cloud', 'provider'))
         auth_kwargs = self.get_section('Cloud Credentials')
+        # GCE credentials are delivered in a JSON file.
+        if 'json_credential_file' in auth_kwargs:
+            with open(auth_kwargs['json_credential_file']) as jf:
+                json_creds = json.load(jf)
+            auth_kwargs['user_id'] = json_creds['client_email']
+            auth_kwargs['key'] = json_creds['private_key']
+
         if 'timeout' in auth_kwargs:
             auth_kwargs['timeout'] = int(auth_kwargs['timeout'])
         return module.ComputeNodeDriver(auth_kwargs,
@@ -105,14 +113,26 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
                                         self.get_section('Cloud Create'))
 
     def node_sizes(self, all_sizes):
+        """Finds all acceptable NodeSizes for our installation.
+
+        Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
+        returned by libcloud that matches a size listed in our config file.
+        """
+
         size_kwargs = {}
         for sec_name in self.sections():
             sec_words = sec_name.split(None, 2)
             if sec_words[0] != 'Size':
                 continue
             size_kwargs[sec_words[1]] = self.get_section(sec_name, int)
-        return [(size, size_kwargs[size.id]) for size in all_sizes
-                if size.id in size_kwargs]
+        # EC2 node sizes are identified by id. GCE sizes are identified by name.
+        matching_sizes = []
+        for size in all_sizes:
+            if size.id in size_kwargs:
+                matching_sizes.append((size, size_kwargs[size.id]))
+            elif size.name in size_kwargs:
+                matching_sizes.append((size, size_kwargs[size.name]))
+        return matching_sizes
 
     def shutdown_windows(self):
         return [int(n)
diff --git a/services/nodemanager/doc/gce.example.cfg b/services/nodemanager/doc/gce.example.cfg
index adc0300ec1..11289bc564 100644
--- a/services/nodemanager/doc/gce.example.cfg
+++ b/services/nodemanager/doc/gce.example.cfg
@@ -79,9 +79,8 @@ provider = gce
 shutdown_windows = 54, 5, 1
 
 [Cloud Credentials]
-user_id = USERID
-key = SECRET_KEY
-project = project_name
+json_credential_file = /path/to/credential_file.json
+project = project-id-from-google-cloud-dashboard
 timeout = 60
 
 # Optional settings. For full documentation see
@@ -103,9 +102,11 @@ ping_host = hostname:port
 # A file path for an SSH key that can log in to the compute node.
 # ssh_key = path
 
-# The GCE IDs of the image and network compute nodes should use.
-image_id = idstring
-network_id = idstring
+# The GCE image name and network zone name to use when creating new nodes.
+# * Valid image aliases: https://cloud.google.com/sdk/gcloud/reference/compute/instances/create
+# * Valid network zones: https://cloud.google.com/compute/docs/zones
+image_id = debian-7
+network_id = us-central1-a
 
 # JSON string of service account authorizations for this cluster.
 # See http://libcloud.readthedocs.org/en/latest/compute/drivers/gce.html#specifying-service-account-scopes
@@ -120,6 +121,9 @@ network_id = idstring
 # The Size fields are interpreted the same way as with a libcloud NodeSize:
 # http://libcloud.readthedocs.org/en/latest/compute/api.html#libcloud.compute.base.NodeSize
 #
+# See https://cloud.google.com/compute/docs/machine-types for a list
+# of known machine types that may be used as a Size parameter.
+#
 # Each size section MUST define the number of cores are available in this
 # size class (since libcloud does not provide any consistent API for exposing
 # this setting).