Merge branch '16100-mime-types'

[arvados.git] / sdk / python / arvados / util.py

diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 2ac6ab9129af16edfac4670452c7194761332274..9e0a3178305068c4edec716c57467221f98af562 100644 (file)
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -1,7 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import division
+from builtins import range
+
 import fcntl
 import hashlib
 import httplib2
 import os
+import random
 import re
 import subprocess
 import errno
@@ -11,6 +19,9 @@ import arvados
 from arvados.collection import CollectionReader
 
 HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+CR_UNCOMMITTED = 'Uncommitted'
+CR_COMMITTED = 'Committed'
+CR_FINAL = 'Final'
 
 keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+\d+(\+\S+)*')
 signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+\d+(\+\S+)*\+A\S+(\+\S+)*')
@@ -20,6 +31,8 @@ collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
 group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
 user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
 link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
 manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+\d+)(\+\S+)*)+( +\d+:\d+:\S+)+$)+', flags=re.MULTILINE)
 
 def clear_tmpdir(path=None):
@@ -124,7 +137,7 @@ def tarball_extract(tarball, path):
                 raise arvados.errors.CommandFailedError(
                     "tar exited %d" % p.returncode)
         os.symlink(tarball, os.path.join(path, '.locator'))
-    tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+    tld_extracts = [f for f in os.listdir(path) if f != '.locator']
     lockfile.close()
     if len(tld_extracts) == 1:
         return os.path.join(path, tld_extracts[0])
@@ -190,7 +203,7 @@ def zipball_extract(zipball, path):
                     "unzip exited %d" % p.returncode)
             os.unlink(zip_filename)
         os.symlink(zipball, os.path.join(path, '.locator'))
-    tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+    tld_extracts = [f for f in os.listdir(path) if f != '.locator']
     lockfile.close()
     if len(tld_extracts) == 1:
         return os.path.join(path, tld_extracts[0])
@@ -364,10 +377,10 @@ def is_hex(s, *length_args):
 
 def list_all(fn, num_retries=0, **kwargs):
     # Default limit to (effectively) api server's MAX_LIMIT
-    kwargs.setdefault('limit', sys.maxint)
+    kwargs.setdefault('limit', sys.maxsize)
     items = []
     offset = 0
-    items_available = sys.maxint
+    items_available = sys.maxsize
     while len(items) < items_available:
         c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
         items += c['items']
@@ -383,6 +396,8 @@ def ca_certs_path(fallback=httplib2.CA_CERTS):
     it returns the value of `fallback` (httplib2's CA certs by default).
     """
     for ca_certs_path in [
+        # Arvados specific:
+        '/etc/arvados/ca-certificates.crt',
         # Debian:
         '/etc/ssl/certs/ca-certificates.crt',
         # Red Hat:
@@ -391,3 +406,24 @@ def ca_certs_path(fallback=httplib2.CA_CERTS):
         if os.path.exists(ca_certs_path):
             return ca_certs_path
     return fallback
+
+def new_request_id():
+    rid = "req-"
+    # 2**104 > 36**20 > 2**103
+    n = random.getrandbits(104)
+    for _ in range(20):
+        c = n % 36
+        if c < 10:
+            rid += chr(c+ord('0'))
+        else:
+            rid += chr(c+ord('a')-10)
+        n = n // 36
+    return rid
+
+def get_config_once(svc):
+    if not svc._rootDesc.get('resources')['configs']:
+        # Old API server version, no config export endpoint
+        return {}
+    if not hasattr(svc, '_cached_config'):
+        svc._cached_config = svc.configs().get().execute()
+    return svc._cached_config