* JSON credential file
** GCE credentials are delivered as a JSON string (and the key is formatted as a multi-line RSA private key). Let the GCE config file specify a path to the JSON credential file for simplicity.
* Accept NodeSizes addressed by id or name
** In EC2, NodeSizes are identified by the 'id' field. In GCE they are identified by the 'name' field. Allow the Node Manager config module to accept either.
self.create_kwargs[new_pair[0]] = new_pair[1]
def _init_image_id(self, image_id):
self.create_kwargs[new_pair[0]] = new_pair[1]
def _init_image_id(self, image_id):
- return 'image', self.search_for(image_id, 'list_images')
+ return 'image', image_id
def _init_ping_host(self, ping_host):
self.ping_host = ping_host
def _init_ping_host(self, ping_host):
self.ping_host = ping_host
self.service_accounts = json.loads(service_accounts_str)
def _init_network_id(self, subnet_id):
self.service_accounts = json.loads(service_accounts_str)
def _init_network_id(self, subnet_id):
- return 'ex_network', self.search_for(subnet_id, 'ex_list_networks')
+ return 'ex_network', subnet_id
def _init_ssh_key(self, filename):
with open(filename) as ssh_file:
def _init_ssh_key(self, filename):
with open(filename) as ssh_file:
import ConfigParser
import importlib
import ConfigParser
import importlib
import logging
import ssl
import sys
import logging
import ssl
import sys
module = importlib.import_module('arvnodeman.computenode.driver.' +
self.get('Cloud', 'provider'))
auth_kwargs = self.get_section('Cloud Credentials')
module = importlib.import_module('arvnodeman.computenode.driver.' +
self.get('Cloud', 'provider'))
auth_kwargs = self.get_section('Cloud Credentials')
+ # GCE credentials are delivered in a JSON file.
+ if 'json_credential_file' in auth_kwargs:
+ with open(auth_kwargs['json_credential_file']) as jf:
+ json_creds = json.load(jf)
+ auth_kwargs['user_id'] = json_creds['client_email']
+ auth_kwargs['key'] = json_creds['private_key']
+
if 'timeout' in auth_kwargs:
auth_kwargs['timeout'] = int(auth_kwargs['timeout'])
return module.ComputeNodeDriver(auth_kwargs,
if 'timeout' in auth_kwargs:
auth_kwargs['timeout'] = int(auth_kwargs['timeout'])
return module.ComputeNodeDriver(auth_kwargs,
self.get_section('Cloud Create'))
def node_sizes(self, all_sizes):
self.get_section('Cloud Create'))
def node_sizes(self, all_sizes):
+ """Finds all acceptable NodeSizes for our installation.
+
+ Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
+ returned by libcloud that matches a size listed in our config file.
+ """
+
size_kwargs = {}
for sec_name in self.sections():
sec_words = sec_name.split(None, 2)
if sec_words[0] != 'Size':
continue
size_kwargs[sec_words[1]] = self.get_section(sec_name, int)
size_kwargs = {}
for sec_name in self.sections():
sec_words = sec_name.split(None, 2)
if sec_words[0] != 'Size':
continue
size_kwargs[sec_words[1]] = self.get_section(sec_name, int)
- return [(size, size_kwargs[size.id]) for size in all_sizes
- if size.id in size_kwargs]
+ # EC2 node sizes are identified by id. GCE sizes are identified by name.
+ matching_sizes = []
+ for size in all_sizes:
+ if size.id in size_kwargs:
+ matching_sizes.append((size, size_kwargs[size.id]))
+ elif size.name in size_kwargs:
+ matching_sizes.append((size, size_kwargs[size.name]))
+ return matching_sizes
def shutdown_windows(self):
return [int(n)
def shutdown_windows(self):
return [int(n)
shutdown_windows = 54, 5, 1
[Cloud Credentials]
shutdown_windows = 54, 5, 1
[Cloud Credentials]
-user_id = USERID
-key = SECRET_KEY
-project = project_name
+json_credential_file = /path/to/credential_file.json
+project = project-id-from-google-cloud-dashboard
timeout = 60
# Optional settings. For full documentation see
timeout = 60
# Optional settings. For full documentation see
# A file path for an SSH key that can log in to the compute node.
# ssh_key = path
# A file path for an SSH key that can log in to the compute node.
# ssh_key = path
-# The GCE IDs of the image and network compute nodes should use.
-image_id = idstring
-network_id = idstring
+# The GCE image name and network zone name to use when creating new nodes.
+# * Valid image aliases: https://cloud.google.com/sdk/gcloud/reference/compute/instances/create
+# * Valid network zones: https://cloud.google.com/compute/docs/zones
+image_id = debian-7
+network_id = us-central1-a
# JSON string of service account authorizations for this cluster.
# See http://libcloud.readthedocs.org/en/latest/compute/drivers/gce.html#specifying-service-account-scopes
# JSON string of service account authorizations for this cluster.
# See http://libcloud.readthedocs.org/en/latest/compute/drivers/gce.html#specifying-service-account-scopes
# The Size fields are interpreted the same way as with a libcloud NodeSize:
# http://libcloud.readthedocs.org/en/latest/compute/api.html#libcloud.compute.base.NodeSize
#
# The Size fields are interpreted the same way as with a libcloud NodeSize:
# http://libcloud.readthedocs.org/en/latest/compute/api.html#libcloud.compute.base.NodeSize
#
+# See https://cloud.google.com/compute/docs/machine-types for a list
+# of known machine types that may be used as a Size parameter.
+#
# Each size section MUST define the number of cores are available in this
# size class (since libcloud does not provide any consistent API for exposing
# this setting).
# Each size section MUST define the number of cores are available in this
# size class (since libcloud does not provide any consistent API for exposing
# this setting).
projects / arvados.git / commitdiff