<pre><code>pub rsa2048 2010-11-15 [SC]
B2DA 2991 656E B4A5 0314 CA2B 5716 5911 1078 ECD7
uid [ unknown] Arvados Automatic Signing Key <sysadmin@arvados.org>
-uid [ unknown] Curoverse, Inc Automatic Signing Key <sysadmin@curoverse.com>
sub rsa2048 2010-11-15 [E]
</code></pre>
</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+<notextile>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install curl gnupg2</span>
+# <span class="userinput">curl https://apt.arvados.org/pubkey.gpg -o /etc/apt/trusted.gpg.d/arvados.asc</span>
+</code></pre>
+</notextile>
+The Arvados package signing GPG key is also available via the keyservers, though they can be unreliable. To retrieve the signing key via keyserver.ubuntu.com:
<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends install gnupg</span>
-# <span class="userinput">/usr/bin/apt-key adv --keyserver pgp.mit.edu --recv 1078ECD7</span>
-</code></pre>
+<pre><code># <span class="userinput">/usr/bin/apt-key adv --keyserver keyserver.ubuntu.com --recv 1078ECD7</code></pre>
</notextile>
h2(#introduction). Introduction
-The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":https://pkg.go.dev/git.arvados.org/arvados.git/services/keep-web for more detail.
h2(#dns). Configure DNS
name=Arvados
baseurl=http://rpm.arvados.org/CentOS/$releasever/os/$basearch/
gpgcheck=1
-gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-curoverse
+gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-arvados
</code></pre>
</notextile>
-{% include 'install_redhat_key' %}
+{% include 'gpg_key_fingerprint' %}
h3(#debian). Debian and Ubuntu
{% include 'install_debian_key' %}
+{% include 'gpg_key_fingerprint' %}
+
As root, add the Arvados package repository to your sources. This command depends on your OS vendor and version:
table(table table-bordered table-condensed).
table(table table-bordered table-condensed).
|_. Option |_. Description |
-|==--help==| This list of options|
-|==--parent-group-uuid==| UUID of group to own all the externally synchronized groups|
-|==--user-id== | Identifier to use in looking up user. One of 'email' or 'username' (Default: 'email')|
-|==--verbose==| Log informational messages (Default: False)|
-|==--version==| Print version and exit|
+|==--help==|This list of options|
+|==--case-insensitive==|Uses case-insensitive username matching|
+|==--parent-group-uuid==|UUID of group to own all the externally synchronized groups|
+|==--user-id==|Identifier to use in looking up user. One of 'email' or 'username' (Default: 'email')|
+|==--verbose==|Log informational messages (Default: False)|
+|==--version==|Print version and exit|
h2. Examples
env := make([]string, 0, len(e.spec.Env))
for k, v := range e.spec.Env {
if k == "HOME" {
- // $HOME is a special case
+ // $HOME is a special case on Singularity 3.5,
+ // but is just a normal variable on Singularity 3.6+
+ // I think this will work with both
args = append(args, "--home="+v)
- } else {
- env = append(env, "SINGULARITYENV_"+k+"="+v)
}
+ env = append(env, "SINGULARITYENV_"+k+"="+v)
}
args = append(args, e.imageFilename)
s.required_ruby_version = '>= 2.1.0'
s.add_runtime_dependency 'arvados', '>= 1.4.1.20190320201707'
# Our google-api-client dependency used to be < 0.9, but that could be
- # satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
- s.add_runtime_dependency 'arvados-google-api-client', '~> 0.6', '>= 0.6.3', '<0.8.9'
+ # satisfied by the buggy 0.9.pre*, cf. https://dev.arvados.org/issues/9213
+ # We need at least version 0.8.7.3, cf. https://dev.arvados.org/issues/15673
+ s.add_runtime_dependency('arvados-google-api-client', '>= 0.8.7.3', '< 0.8.9')
s.add_runtime_dependency 'activesupport', '>= 3.2.13', '< 5.3'
s.add_runtime_dependency 'json', '>= 1.7.7', '<3'
s.add_runtime_dependency 'optimist', '~> 3.0'
s.add_runtime_dependency 'oj', '< 3.10.9'
s.add_runtime_dependency 'curb', '~> 0.8'
s.add_runtime_dependency 'launchy', '< 2.5'
- # arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
- s.add_dependency('faraday', '< 0.16')
s.homepage =
'https://arvados.org'
end
api_client.users().current().execute()
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
+ executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4, stdout=stdout)
except WorkflowException as e:
logger.error(e, exc_info=(sys.exc_info()[1] if arvargs.debug else False))
return 1
if self.output_ttl < 0:
raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
- storage_class_req, _ = self.get_requirement("http://arvados.org/cwl#OutputStorageClass")
- if storage_class_req and storage_class_req.get("intermediateStorageClass"):
- container_request["output_storage_classes"] = aslist(storage_class_req["intermediateStorageClass"])
- else:
- container_request["output_storage_classes"] = runtimeContext.intermediate_storage_classes.strip().split(",")
+
+ if self.arvrunner.api._rootDesc["revision"] >= "20210628":
+ storage_class_req, _ = self.get_requirement("http://arvados.org/cwl#OutputStorageClass")
+ if storage_class_req and storage_class_req.get("intermediateStorageClass"):
+ container_request["output_storage_classes"] = aslist(storage_class_req["intermediateStorageClass"])
+ else:
+ container_request["output_storage_classes"] = runtimeContext.intermediate_storage_classes.strip().split(",")
if self.timelimit is not None and self.timelimit > 0:
scheduling_parameters["max_run_time"] = self.timelimit
if runtimeContext.debug:
command.append("--debug")
- if runtimeContext.storage_classes != "default":
+ if runtimeContext.storage_classes != "default" and runtimeContext.storage_classes:
command.append("--storage-classes=" + runtimeContext.storage_classes)
- if runtimeContext.intermediate_storage_classes != "default":
+ if runtimeContext.intermediate_storage_classes != "default" and runtimeContext.intermediate_storage_classes:
command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
if self.on_error:
if not images:
# Fetch Docker image if necessary.
try:
- cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
+ result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
force_pull, tmp_outdir_prefix)
+ if not result:
+ raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
except OSError as e:
raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
if not docker_req:
self.hints.append({"class": "DockerRequirement",
- "dockerImageId": "arvados/jobs:"+__version__})
+ "dockerPull": "arvados/jobs:"+__version__})
self.arvrunner = arvrunner
arvargs=None,
keep_client=None,
num_retries=4,
- thread_count=4):
+ thread_count=4,
+ stdout=sys.stdout):
if arvargs is None:
arvargs = argparse.Namespace()
self.should_estimate_cache_size = True
self.fs_access = None
self.secret_store = None
+ self.stdout = stdout
if keep_client is not None:
self.keep_client = keep_client
if runtimeContext.submit_request_uuid and self.work_api != "containers":
raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
- default_storage_classes = ",".join([k for k,v in self.api.config()["StorageClasses"].items() if v.get("Default") is True])
+ default_storage_classes = ",".join([k for k,v in self.api.config().get("StorageClasses", {"default": {"Default": True}}).items() if v.get("Default") is True])
if runtimeContext.storage_classes == "default":
runtimeContext.storage_classes = default_storage_classes
if runtimeContext.intermediate_storage_classes == "default":
if existing_uuid or runtimeContext.create_workflow:
# Create a pipeline template or workflow record and exit.
if self.work_api == "containers":
- return (upload_workflow(self, tool, job_order,
+ uuid = upload_workflow(self, tool, job_order,
self.project_uuid,
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
merged_map=merged_map,
- submit_runner_image=runtimeContext.submit_runner_image),
- "success")
+ submit_runner_image=runtimeContext.submit_runner_image)
+ self.stdout.write(uuid + "\n")
+ return (None, "success")
self.apply_reqs(job_order, tool)
if runtimeContext.submit and not runtimeContext.wait:
runnerjob = next(jobiter)
runnerjob.run(runtimeContext)
- return (runnerjob.uuid, "success")
+ self.stdout.write(runnerjob.uuid+"\n")
+ return (None, "success")
current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
if current_container:
outdir="", # type: Text
tmpdir="", # type: Text
stagedir="", # type: Text
- cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion") or metadata.get("cwlVersion")
+ cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion") or metadata.get("cwlVersion"),
+ container_engine="docker"
)
def search_schemadef(name, reqs):
elif isinstance(pattern, dict):
specs.append(pattern)
elif isinstance(pattern, str):
- specs.append({"pattern": pattern})
+ specs.append({"pattern": pattern, "required": sf.get("required")})
else:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Expression must return list, object, string or null")
for i, sf in enumerate(specs):
if isinstance(sf, dict):
if sf.get("class") == "File":
- pattern = sf["basename"]
+ pattern = None
+ sfpath = sf["location"]
+ required = True
else:
pattern = sf["pattern"]
required = sf.get("required")
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Expression must return list, object, string or null")
- sfpath = substitute(primary["location"], pattern)
+ if pattern is not None:
+ sfpath = substitute(primary["location"], pattern)
+
required = builder.do_eval(required, context=primary)
if fsaccess.exists(sfpath):
- found.append({"location": sfpath, "class": "File"})
+ if pattern is not None:
+ found.append({"location": sfpath, "class": "File"})
+ else:
+ found.append(sf)
elif required:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Required secondary file '%s' does not exist" % sfpath)
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20210816212154',
- 'schema-salad==8.2.20210902094147',
+ 'cwltool==3.1.20210922203925',
+ 'schema-salad==8.2.20210918131710',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0',
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
stubs.api = mock.MagicMock()
stubs.api._rootDesc = get_rootDesc()
stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
+ stubs.api._rootDesc["revision"] = "20210628"
stubs.api.users().current().execute.return_value = {
"uuid": stubs.fake_user_uuid,
"enableReuse": False,
},
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
"cwltool": "http://commonwl.org/cwltool#"
}
def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
final_output_c = arvados.collection.Collection()
make_output.return_value = ({},final_output_c)
+ stubs.api.config().get.return_value = {"default": {"Default": True}}
def set_final_output(job_order, output_callback, runtimeContext):
output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
"keep_cache": 512
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
"content": {
"$graph": [
{
- "$namespaces": {
- "cwltool": "http://commonwl.org/cwltool#"
- },
"arguments": [
"md5sum",
"example.conf"
]
}
],
+ "$namespaces": {
+ "cwltool": "http://commonwl.org/cwltool#"
+ },
"cwlVersion": "v1.0"
},
"kind": "json"
],
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#"
}
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
hints:
- class: arv:RunInSingleContainer
- class: DockerRequirement
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
run:
class: Workflow
id: mysub
]
}
],
+ "$namespaces": {
+ "arv": "http://arvados.org/cwl#"
+ },
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
import os
import re
import socket
+import ssl
import sys
import time
import types
def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
- if (self.max_request_size and
- kwargs.get('body') and
- self.max_request_size < len(kwargs['body'])):
- raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
-
- if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
- headers['X-External-Client'] = '1'
-
- headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
if not headers.get('X-Request-Id'):
headers['X-Request-Id'] = self._request_id()
+ try:
+ if (self.max_request_size and
+ kwargs.get('body') and
+ self.max_request_size < len(kwargs['body'])):
+ raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- retryable = method in [
- 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
- retry_count = self._retry_count if retryable else 0
-
- if (not retryable and
- time.time() - self._last_request_time > self._max_keepalive_idle):
- # High probability of failure due to connection atrophy. Make
- # sure this request [re]opens a new connection by closing and
- # forgetting all cached connections first.
- for conn in self.connections.values():
- conn.close()
- self.connections.clear()
-
- delay = self._retry_delay_initial
- for _ in range(retry_count):
- self._last_request_time = time.time()
- try:
- return self.orig_http_request(uri, method, headers=headers, **kwargs)
- except http.client.HTTPException:
- _logger.debug("Retrying API request in %d s after HTTP error",
- delay, exc_info=True)
- except socket.error:
- # This is the one case where httplib2 doesn't close the
- # underlying connection first. Close all open
- # connections, expecting this object only has the one
- # connection to the API server. This is safe because
- # httplib2 reopens connections when needed.
- _logger.debug("Retrying API request in %d s after socket error",
- delay, exc_info=True)
+ if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
+ headers['X-External-Client'] = '1'
+
+ headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+
+ retryable = method in [
+ 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
+ retry_count = self._retry_count if retryable else 0
+
+ if (not retryable and
+ time.time() - self._last_request_time > self._max_keepalive_idle):
+ # High probability of failure due to connection atrophy. Make
+ # sure this request [re]opens a new connection by closing and
+ # forgetting all cached connections first.
for conn in self.connections.values():
conn.close()
- except httplib2.SSLHandshakeError as e:
- # Intercept and re-raise with a better error message.
- raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e))
+ self.connections.clear()
+
+ delay = self._retry_delay_initial
+ for _ in range(retry_count):
+ self._last_request_time = time.time()
+ try:
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ except http.client.HTTPException:
+ _logger.debug("[%s] Retrying API request in %d s after HTTP error",
+ headers['X-Request-Id'], delay, exc_info=True)
+ except ssl.SSLCertVerificationError as e:
+ raise ssl.SSLCertVerificationError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None
+ except socket.error:
+ # This is the one case where httplib2 doesn't close the
+ # underlying connection first. Close all open
+ # connections, expecting this object only has the one
+ # connection to the API server. This is safe because
+ # httplib2 reopens connections when needed.
+ _logger.debug("[%s] Retrying API request in %d s after socket error",
+ headers['X-Request-Id'], delay, exc_info=True)
+ for conn in self.connections.values():
+ conn.close()
+
+ time.sleep(delay)
+ delay = delay * self._retry_delay_backoff
- time.sleep(delay)
- delay = delay * self._retry_delay_backoff
-
- self._last_request_time = time.time()
- return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ self._last_request_time = time.time()
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ except Exception as e:
+ # Prepend "[request_id] " to the error message, which we
+ # assume is the first string argument passed to the exception
+ # constructor.
+ for i in range(len(e.args or ())):
+ if type(e.args[i]) == type(""):
+ e.args = e.args[:i] + ("[{}] {}".format(headers['X-Request-Id'], e.args[i]),) + e.args[i+1:]
+ raise type(e)(*e.args)
+ raise
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
try:
self._populate()
- except (IOError, errors.SyntaxError) as e:
- raise errors.ArgumentError("Error processing manifest text: %s", e)
+ except errors.SyntaxError as e:
+ raise errors.ArgumentError("Error processing manifest text: %s", str(e)) from None
def storage_classes_desired(self):
return self._storage_classes_desired or []
self.find_or_create(os.path.join(stream_name, name[:-2]), COLLECTION)
else:
filepath = os.path.join(stream_name, name)
- afile = self.find_or_create(filepath, FILE)
+ try:
+ afile = self.find_or_create(filepath, FILE)
+ except IOError as e:
+ if e.errno == errno.ENOTDIR:
+ raise errors.SyntaxError("Dir part of %s conflicts with file of the same name.", filepath) from None
+ else:
+ raise e from None
if isinstance(afile, ArvadosFile):
afile.add_segment(blocks, pos, size)
else:
self.get_counter.add(1)
+ request_id = (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id())
+ if headers is None:
+ headers = {}
+ headers['X-Request-Id'] = request_id
+
slot = None
blob = None
try:
self.misses_counter.add(1)
- if headers is None:
- headers = {}
- headers['X-Request-Id'] = (request_id or
- (hasattr(self, 'api_client') and self.api_client.request_id) or
- arvados.util.new_request_id())
-
# If the locator has hints specifying a prefix (indicating a
# remote keepproxy) or the UUID of a local gateway service,
# read data from the indicated service(s) instead of the usual
for key in sorted_roots)
if not roots_map:
raise arvados.errors.KeepReadError(
- "failed to read {}: no Keep services available ({})".format(
- loc_s, loop.last_result()))
+ "[{}] failed to read {}: no Keep services available ({})".format(
+ request_id, loc_s, loop.last_result()))
elif not_founds == len(sorted_roots):
raise arvados.errors.NotFoundError(
- "{} not found".format(loc_s), service_errors)
+ "[{}] {} not found".format(request_id, loc_s), service_errors)
else:
raise arvados.errors.KeepReadError(
- "failed to read {} after {}".format(loc_s, loop.attempts_str()), service_errors, label="service")
+ "[{}] failed to read {} after {}".format(request_id, loc_s, loop.attempts_str()), service_errors, label="service")
@retry.retry_method
def put(self, data, copies=2, num_retries=None, request_id=None, classes=None):
return loc_s
locator = KeepLocator(loc_s)
+ request_id = (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id())
headers = {
- 'X-Request-Id': (request_id or
- (hasattr(self, 'api_client') and self.api_client.request_id) or
- arvados.util.new_request_id()),
+ 'X-Request-Id': request_id,
'X-Keep-Desired-Replicas': str(copies),
}
roots_map = {}
return writer_pool.response()
if not roots_map:
raise arvados.errors.KeepWriteError(
- "failed to write {}: no Keep services available ({})".format(
- data_hash, loop.last_result()))
+ "[{}] failed to write {}: no Keep services available ({})".format(
+ request_id, data_hash, loop.last_result()))
else:
service_errors = ((key, roots_map[key].last_result()['error'])
for key in sorted_roots
if roots_map[key].last_result()['error'])
raise arvados.errors.KeepWriteError(
- "failed to write {} after {} (wanted {} copies but wrote {})".format(
- data_hash, loop.attempts_str(), (copies, classes), writer_pool.done()), service_errors, label="service")
+ "[{}] failed to write {} after {} (wanted {} copies but wrote {})".format(
+ request_id, data_hash, loop.attempts_str(), (copies, classes), writer_pool.done()), service_errors, label="service")
def local_store_put(self, data, copies=1, num_retries=None, classes=[]):
"""A stub for put().
for msg in ["Bad UUID format", "Bad output format"]:
self.assertIn(msg, err_s)
+ @mock.patch('time.sleep')
+ def test_exceptions_include_request_id(self, sleep):
+ api = arvados.api('v1')
+ api.request_id='fake-request-id'
+ api._http.orig_http_request = mock.MagicMock()
+ api._http.orig_http_request.side_effect = socket.error('mock error')
+ caught = None
+ try:
+ api.users().current().execute()
+ except Exception as e:
+ caught = e
+ self.assertRegex(str(caught), r'fake-request-id')
+
def test_exceptions_without_errors_have_basic_info(self):
mock_responses = {
'arvados.humans.delete': (
self.keep_client.head(self.locator)
self.assertAutomaticRequestId(mock.responses[0])
+ def test_request_id_in_exception(self):
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepReadError, self.test_id):
+ self.keep_client.head(self.locator, request_id=self.test_id)
+
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepReadError, r'req-[a-z0-9]{20}'):
+ self.keep_client.get(self.locator)
+
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepWriteError, self.test_id):
+ self.keep_client.put(self.data, request_id=self.test_id)
+
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepWriteError, r'req-[a-z0-9]{20}'):
+ self.keep_client.put(self.data)
+
def assertAutomaticRequestId(self, resp):
hdr = [x for x in resp.getopt(pycurl.HTTPHEADER)
if x.startswith('X-Request-Id: ')][0]
s.add_dependency('activesupport', '>= 3')
s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
# Our google-api-client dependency used to be < 0.9, but that could be
- # satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
- s.add_dependency('arvados-google-api-client', '>= 0.7', '< 0.8.9')
+ # satisfied by the buggy 0.9.pre*, cf. https://dev.arvados.org/issues/9213
+ # We need at least version 0.8.7.3, cf. https://dev.arvados.org/issues/15673
+ s.add_dependency('arvados-google-api-client', '>= 0.8.7.3', '< 0.8.9')
# work around undeclared dependency on i18n in some activesupport 3.x.x:
s.add_dependency('i18n', '~> 0')
s.add_dependency('json', '>= 1.7.7', '<3')
- # arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
- s.add_dependency('faraday', '< 0.16')
+ # Avoid warning on Ruby 2.7, cf. https://dev.arvados.org/issues/18247
+ s.add_dependency('faraday', '>= 0.17.4')
s.add_runtime_dependency('jwt', '<2', '>= 0.1.5')
s.homepage =
'https://arvados.org'
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20210503",
+ revision: "20210628",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
RUN mkdir -p /etc/apt/sources.list.d && \
echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \
apt-get update && \
- apt-get -yq --no-install-recommends install docker-ce=5:19.03.13~3-0~debian-buster && \
+ apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \
apt-get clean
# Set UTF-8 locale
mkdir -p /tmp/crunch0 /tmp/crunch1
chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
+ # singularity needs to be owned by root and suid
+ chown root /var/lib/arvados/bin/singularity \
+ /var/lib/arvados/etc/singularity/singularity.conf \
+ /var/lib/arvados/etc/singularity/capability.json \
+ /var/lib/arvados/etc/singularity/ecl.toml
+ chmod u+s /var/lib/arvados/bin/singularity
+
echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
cat <<EOF > /etc/profile.d/paths.sh
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-# singularity can use suid
-chown root /var/lib/arvados/bin/singularity \
- /var/lib/arvados/etc/singularity/singularity.conf \
- /var/lib/arvados/etc/singularity/capability.json \
- /var/lib/arvados/etc/singularity/ecl.toml
-chmod u+s /var/lib/arvados/bin/singularity
-
-exec /usr/local/lib/arvbox/runsu.sh $0-service $1
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
projects / arvados.git / commitdiff