Merge branch '1880-check-duplicate-public-key'

[arvados.git] / sdk / python / arvados / __init__.py

diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index a7b3b3259d98776371890a3cb3bf56b7e50265bd..dacdba851af3a99baaa5cdda21f105cf9354d3ae 100644 (file)
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -21,7 +21,25 @@ import threading
 import apiclient
 import apiclient.discovery
 
-if 'ARVADOS_DEBUG' in os.environ:
+# Arvados configuration settings are taken from $HOME/.config/arvados.
+# Environment variables override settings in the config file.
+#
+class ArvadosConfig(dict):
+    def __init__(self, config_file):
+        dict.__init__(self)
+        if os.path.exists(config_file):
+            with open(config_file, "r") as f:
+                for config_line in f:
+                    var, val = config_line.rstrip().split('=', 2)
+                    self[var] = val
+        for var in os.environ:
+            if var.startswith('ARVADOS_'):
+                self[var] = os.environ[var]
+
+
+config = ArvadosConfig(os.environ['HOME'] + '/.config/arvados')
+
+if 'ARVADOS_DEBUG' in config:
     logging.basicConfig(level=logging.DEBUG)
 
 EMPTY_BLOCK_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
@@ -45,10 +63,11 @@ class errors:
 class CredentialsFromEnv(object):
     @staticmethod
     def http_request(self, uri, **kwargs):
+        global config
         from httplib import BadStatusLine
         if 'headers' not in kwargs:
             kwargs['headers'] = {}
-        kwargs['headers']['Authorization'] = 'OAuth2 %s' % os.environ['ARVADOS_API_TOKEN']
+        kwargs['headers']['Authorization'] = 'OAuth2 %s' % config.get('ARVADOS_API_TOKEN', 'ARVADOS_API_TOKEN_not_set')
         try:
             return self.orig_http_request(uri, **kwargs)
         except BadStatusLine:
@@ -111,7 +130,7 @@ def _cast_objects_too(value, schema_type):
 apiclient.discovery._cast = _cast_objects_too
 
 def api(version=None):
-    global services
+    global services, config
     if not services.get(version):
         apiVersion = version
         if not version:
@@ -119,10 +138,10 @@ def api(version=None):
             logging.info("Using default API version. " +
                          "Call arvados.api('%s') instead." %
                          apiVersion)
-        if 'ARVADOS_API_HOST' not in os.environ:
+        if 'ARVADOS_API_HOST' not in config:
             raise Exception("ARVADOS_API_HOST is not set. Aborting.")
         url = ('https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' %
-               os.environ['ARVADOS_API_HOST'])
+               config['ARVADOS_API_HOST'])
         credentials = CredentialsFromEnv()
 
         # Use system's CA certificates (if we find them) instead of httplib2's
@@ -133,7 +152,7 @@ def api(version=None):
         http = httplib2.Http(ca_certs=ca_certs)
         http = credentials.authorize(http)
         if re.match(r'(?i)^(true|1|yes)$',
-                    os.environ.get('ARVADOS_API_HOST_INSECURE', '')):
+                    config.get('ARVADOS_API_HOST_INSECURE', 'no')):
             http.disable_ssl_certificate_validation=True
         services[version] = apiclient.discovery.build(
             'arvados', apiVersion, http=http, discoveryServiceUrl=url)
@@ -585,12 +604,12 @@ class StreamReader(object):
 
         for tok in self._tokens:
             if self._stream_name == None:
-                self._stream_name = tok
+                self._stream_name = tok.replace('\\040', ' ')
             elif re.search(r'^[0-9a-f]{32}(\+\S+)*$', tok):
                 self.data_locators += [tok]
             elif re.search(r'^\d+:\d+:\S+', tok):
                 pos, size, name = tok.split(':',2)
-                self.files += [[int(pos), int(size), name]]
+                self.files += [[int(pos), int(size), name.replace('\\040', ' ')]]
             else:
                 raise errors.SyntaxError("Invalid manifest format")
 
@@ -781,8 +800,7 @@ class CollectionWriter(object):
         self.finish_current_file()
         self.set_current_file_name(newfilename)
     def set_current_file_name(self, newfilename):
-        newfilename = re.sub(r' ', '\\\\040', newfilename)
-        if re.search(r'[ \t\n]', newfilename):
+        if re.search(r'[\t\n]', newfilename):
             raise errors.AssertionError(
                 "Manifest filenames cannot contain whitespace: %s" %
                 newfilename)
@@ -807,7 +825,7 @@ class CollectionWriter(object):
         self.finish_current_stream()
         self.set_current_stream_name(newstreamname)
     def set_current_stream_name(self, newstreamname):
-        if re.search(r'[ \t\n]', newstreamname):
+        if re.search(r'[\t\n]', newstreamname):
             raise errors.AssertionError(
                 "Manifest stream names cannot contain whitespace")
         self._current_stream_name = '.' if newstreamname=='' else newstreamname
@@ -842,11 +860,11 @@ class CollectionWriter(object):
         for stream in self._finished_streams:
             if not re.search(r'^\.(/.*)?$', stream[0]):
                 manifest += './'
-            manifest += stream[0]
+            manifest += stream[0].replace(' ', '\\040')
             for locator in stream[1]:
                 manifest += " %s" % locator
             for sfile in stream[2]:
-                manifest += " %d:%d:%s" % (sfile[0], sfile[1], sfile[2])
+                manifest += " %d:%d:%s" % (sfile[0], sfile[1], sfile[2].replace(' ', '\\040'))
             manifest += "\n"
         return manifest
     def data_locators(self):
@@ -924,6 +942,7 @@ class KeepClient(object):
             super(KeepClient.KeepWriterThread, self).__init__()
             self.args = kwargs
         def run(self):
+            global config
             with self.args['thread_limiter'] as limiter:
                 if not limiter.shall_i_proceed():
                     # My turn arrived, but the job has been done without
@@ -935,7 +954,7 @@ class KeepClient(object):
                                self.args['service_root']))
                 h = httplib2.Http()
                 url = self.args['service_root'] + self.args['data_hash']
-                api_token = os.environ['ARVADOS_API_TOKEN']
+                api_token = config['ARVADOS_API_TOKEN']
                 headers = {'Authorization': "OAuth2 %s" % api_token}
                 try:
                     resp, content = h.request(url.encode('utf-8'), 'PUT',
@@ -995,6 +1014,7 @@ class KeepClient(object):
         return pseq
 
     def get(self, locator):
+        global config
         if re.search(r',', locator):
             return ''.join(self.get(x) for x in locator.split(','))
         if 'KEEP_LOCAL_STORE' in os.environ:
@@ -1003,7 +1023,7 @@ class KeepClient(object):
         for service_root in self.shuffled_service_roots(expect_hash):
             h = httplib2.Http()
             url = service_root + expect_hash
-            api_token = os.environ['ARVADOS_API_TOKEN']
+            api_token = config['ARVADOS_API_TOKEN']
             headers = {'Authorization': "OAuth2 %s" % api_token,
                        'Accept': 'application/octet-stream'}
             try: