projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
15061: Add documentation for using arv-federation-migrate
[arvados.git]
/
sdk
/
python
/
arvados
/
util.py
diff --git
a/sdk/python/arvados/util.py
b/sdk/python/arvados/util.py
index aaf20945595e8b5e32fe74520465a761fd539e5e..66da2d12af2082689179bbf1b89dd6285c1edbd9 100644
(file)
--- a/
sdk/python/arvados/util.py
+++ b/
sdk/python/arvados/util.py
@@
-1,7
+1,15
@@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import division
+from builtins import range
+
import fcntl
import hashlib
import httplib2
import os
import fcntl
import hashlib
import httplib2
import os
+import random
import re
import subprocess
import errno
import re
import subprocess
import errno
@@
-20,6
+28,8
@@
collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+\d+)(\+\S+)*)+( +\d+:\d+:\S+)+$)+', flags=re.MULTILINE)
def clear_tmpdir(path=None):
manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+\d+)(\+\S+)*)+( +\d+:\d+:\S+)+$)+', flags=re.MULTILINE)
def clear_tmpdir(path=None):
@@
-45,7
+55,7
@@
def run_command(execargs, **kwargs):
p = subprocess.Popen(execargs, **kwargs)
stdoutdata, stderrdata = p.communicate(None)
if p.returncode != 0:
p = subprocess.Popen(execargs, **kwargs)
stdoutdata, stderrdata = p.communicate(None)
if p.returncode != 0:
- raise errors.CommandFailedError(
+ raise
arvados.
errors.CommandFailedError(
"run_command %s exit %d:\n%s" %
(execargs, p.returncode, stderrdata))
return stdoutdata, stderrdata
"run_command %s exit %d:\n%s" %
(execargs, p.returncode, stderrdata))
return stdoutdata, stderrdata
@@
-110,7
+120,7
@@
def tarball_extract(tarball, path):
elif re.search('\.tar$', f.name()):
p = tar_extractor(path, '')
else:
elif re.search('\.tar$', f.name()):
p = tar_extractor(path, '')
else:
- raise errors.AssertionError(
+ raise
arvados.
errors.AssertionError(
"tarball_extract cannot handle filename %s" % f.name())
while True:
buf = f.read(2**20)
"tarball_extract cannot handle filename %s" % f.name())
while True:
buf = f.read(2**20)
@@
-121,10
+131,10
@@
def tarball_extract(tarball, path):
p.wait()
if p.returncode != 0:
lockfile.close()
p.wait()
if p.returncode != 0:
lockfile.close()
- raise errors.CommandFailedError(
+ raise
arvados.
errors.CommandFailedError(
"tar exited %d" % p.returncode)
os.symlink(tarball, os.path.join(path, '.locator'))
"tar exited %d" % p.returncode)
os.symlink(tarball, os.path.join(path, '.locator'))
- tld_extracts =
filter(lambda f: f != '.locator', os.listdir(path))
+ tld_extracts =
[f for f in os.listdir(path) if f != '.locator']
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
@@
-165,7
+175,7
@@
def zipball_extract(zipball, path):
for f in CollectionReader(zipball).all_files():
if not re.search('\.zip$', f.name()):
for f in CollectionReader(zipball).all_files():
if not re.search('\.zip$', f.name()):
- raise errors.NotImplementedError(
+ raise
arvados.
errors.NotImplementedError(
"zipball_extract cannot handle filename %s" % f.name())
zip_filename = os.path.join(path, os.path.basename(f.name()))
zip_file = open(zip_filename, 'wb')
"zipball_extract cannot handle filename %s" % f.name())
zip_filename = os.path.join(path, os.path.basename(f.name()))
zip_file = open(zip_filename, 'wb')
@@
-186,11
+196,11
@@
def zipball_extract(zipball, path):
p.wait()
if p.returncode != 0:
lockfile.close()
p.wait()
if p.returncode != 0:
lockfile.close()
- raise errors.CommandFailedError(
+ raise
arvados.
errors.CommandFailedError(
"unzip exited %d" % p.returncode)
os.unlink(zip_filename)
os.symlink(zipball, os.path.join(path, '.locator'))
"unzip exited %d" % p.returncode)
os.unlink(zip_filename)
os.symlink(zipball, os.path.join(path, '.locator'))
- tld_extracts =
filter(lambda f: f != '.locator', os.listdir(path))
+ tld_extracts =
[f for f in os.listdir(path) if f != '.locator']
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
@@
-250,7
+260,7
@@
def collection_extract(collection, path, files=[], decompress=True):
outfile.write(buf)
outfile.close()
if len(files_got) < len(files):
outfile.write(buf)
outfile.close()
if len(files_got) < len(files):
- raise errors.AssertionError(
+ raise
arvados.
errors.AssertionError(
"Wanted files %s but only got %s from %s" %
(files, files_got,
[z.name() for z in CollectionReader(collection).all_files()]))
"Wanted files %s but only got %s from %s" %
(files, files_got,
[z.name() for z in CollectionReader(collection).all_files()]))
@@
-305,7
+315,7
@@
def stream_extract(stream, path, files=[], decompress=True):
outfile.write(buf)
outfile.close()
if len(files_got) < len(files):
outfile.write(buf)
outfile.close()
if len(files_got) < len(files):
- raise errors.AssertionError(
+ raise
arvados.
errors.AssertionError(
"Wanted files %s but only got %s from %s" %
(files, files_got, [z.name() for z in stream.all_files()]))
lockfile.close()
"Wanted files %s but only got %s from %s" %
(files, files_got, [z.name() for z in stream.all_files()]))
lockfile.close()
@@
-352,8
+362,8
@@
def is_hex(s, *length_args):
"""
num_length_args = len(length_args)
if num_length_args > 2:
"""
num_length_args = len(length_args)
if num_length_args > 2:
- raise
errors.ArgumentError("is_hex accepts up to 3 arguments ({} given)"
-
.format(1 + num_length_args))
+ raise
arvados.errors.ArgumentError(
+
"is_hex accepts up to 3 arguments ({} given)"
.format(1 + num_length_args))
elif num_length_args == 2:
good_len = (length_args[0] <= len(s) <= length_args[1])
elif num_length_args == 1:
elif num_length_args == 2:
good_len = (length_args[0] <= len(s) <= length_args[1])
elif num_length_args == 1:
@@
-363,9
+373,11
@@
def is_hex(s, *length_args):
return bool(good_len and HEX_RE.match(s))
def list_all(fn, num_retries=0, **kwargs):
return bool(good_len and HEX_RE.match(s))
def list_all(fn, num_retries=0, **kwargs):
+ # Default limit to (effectively) api server's MAX_LIMIT
+ kwargs.setdefault('limit', sys.maxsize)
items = []
offset = 0
items = []
offset = 0
- items_available = sys.max
int
+ items_available = sys.max
size
while len(items) < items_available:
c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
items += c['items']
while len(items) < items_available:
c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
items += c['items']
@@
-381,6
+393,8
@@
def ca_certs_path(fallback=httplib2.CA_CERTS):
it returns the value of `fallback` (httplib2's CA certs by default).
"""
for ca_certs_path in [
it returns the value of `fallback` (httplib2's CA certs by default).
"""
for ca_certs_path in [
+ # Arvados specific:
+ '/etc/arvados/ca-certificates.crt',
# Debian:
'/etc/ssl/certs/ca-certificates.crt',
# Red Hat:
# Debian:
'/etc/ssl/certs/ca-certificates.crt',
# Red Hat:
@@
-389,3
+403,16
@@
def ca_certs_path(fallback=httplib2.CA_CERTS):
if os.path.exists(ca_certs_path):
return ca_certs_path
return fallback
if os.path.exists(ca_certs_path):
return ca_certs_path
return fallback
+
+def new_request_id():
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid