"Rebalance and garbage-collect data blocks stored in Arvados Keep"
package_go_binary services/keepproxy keepproxy \
"Make a Keep cluster accessible to clients that are not on the LAN"
-package_go_binary services/keepstore keepstore \
+package_go_binary cmd/arvados-server keepstore \
"Keep storage daemon, accessible to clients on the LAN"
package_go_binary services/keep-web keep-web \
"Static web hosting service for user data stored in Arvados Keep"
do_install services/api
do_install services/arv-git-httpd go
do_install services/keepproxy go
- do_install services/keepstore go
do_install services/keep-web go
- do_install services/ws go
}
install_all() {
"git.arvados.org/arvados.git/lib/install"
"git.arvados.org/arvados.git/lib/lsf"
"git.arvados.org/arvados.git/lib/recovercollection"
+ "git.arvados.org/arvados.git/services/keepstore"
"git.arvados.org/arvados.git/services/ws"
)
"dispatch-lsf": lsf.DispatchCommand,
"install": install.Command,
"init": install.InitCommand,
+ "keepstore": keepstore.Command,
"recover-collection": recovercollection.Command,
"ws": ws.Command,
})
Description=Arvados Keep Storage Daemon
Documentation=https://doc.arvados.org/
After=network.target
+AssertPathExists=/etc/arvados/config.yml
# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
StartLimitIntervalSec=0
<pre><code>pub rsa2048 2010-11-15 [SC]
B2DA 2991 656E B4A5 0314 CA2B 5716 5911 1078 ECD7
uid [ unknown] Arvados Automatic Signing Key <sysadmin@arvados.org>
-uid [ unknown] Curoverse, Inc Automatic Signing Key <sysadmin@curoverse.com>
sub rsa2048 2010-11-15 [E]
</code></pre>
</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+<notextile>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install curl gnupg2</span>
+# <span class="userinput">curl https://apt.arvados.org/pubkey.gpg -o /etc/apt/trusted.gpg.d/arvados.asc</span>
+</code></pre>
+</notextile>
+The Arvados package signing GPG key is also available via the keyservers, though they can be unreliable. To retrieve the signing key via keyserver.ubuntu.com:
<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends install gnupg</span>
-# <span class="userinput">/usr/bin/apt-key adv --keyserver pgp.mit.edu --recv 1078ECD7</span>
-</code></pre>
+<pre><code># <span class="userinput">/usr/bin/apt-key adv --keyserver keyserver.ubuntu.com --recv 1078ECD7</code></pre>
</notextile>
h3. Debian and Ubuntu
-Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5, which is supported by Arvados.
+Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5 or newer, which is sufficient for Arvados.
<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends install ruby ruby-dev bundler</span></code></pre>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install ruby ruby-dev</span></code></pre>
</notextile>
h2(#rvm). Option 2: Install with RVM
apt-get --no-install-recommends install gpg curl
</pre>
-h3. Install RVM
+h3. Install RVM, Ruby and Bundler
<notextile>
<pre><code># <span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.5
</span></code></pre></notextile>
+This command installs the latest Ruby 2.5.x release, as well as the @gem@ and @bundle@ commands.
+
To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
-<pre><code><span class="userinput">. /usr/local/rvm/scripts/rvm
+<pre><code><span class="userinput">source /usr/local/rvm/scripts/rvm
</span></code></pre></notextile>
Alternately you can use @rvm-exec@ (the first parameter is the ruby version to use, or "default"), for example:
<notextile>
-<pre><code><span class="userinput">rvm-exec default rails console
+<pre><code><span class="userinput">rvm-exec default ruby -v
</span></code></pre></notextile>
-Finally, install Bundler:
-
-<notextile>
-<pre><code>~$ <span class="userinput">gem install bundler</span>
-</code></pre></notextile>
-
h2(#fromsource). Option 3: Install from source
-Install prerequisites for Debian 10:
+Install prerequisites for Debian 10, Ubuntu 18.04 and Ubuntu 20.04:
<notextile>
<pre><code><span class="userinput">sudo apt-get install \
make automake libtool bison sqlite-devel tar
</span></code></pre></notextile>
-Install prerequisites for Ubuntu 16.04:
-
-<notextile>
-<pre><code><span class="userinput">sudo apt-get install \
- bison build-essential gettext libcurl3 \
- libcurl3-openssl-dev libpcre3-dev libreadline-dev \
- libssl-dev libxslt1.1 zlib1g-dev
-</span></code></pre></notextile>
-
Build and install Ruby:
<notextile>
<pre><code><span class="userinput">mkdir -p ~/src
cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.5.tar.gz | tar xz
-cd ruby-2.5.5
+curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.8.tar.gz | tar xz
+cd ruby-2.5.8
./configure --disable-install-rdoc
make
sudo make install
+# Make sure the post install script can find the gem and ruby executables
+sudo ln -s /usr/local/bin/gem /usr/bin/gem
+sudo ln -s /usr/local/bin/ruby /usr/bin/ruby
+# Install bundler
sudo -i gem install bundler</span>
</code></pre></notextile>
arvados-dispatch-lsf is only relevant for on premises clusters that will spool jobs to LSF. Skip this section if you are installing a cloud cluster.
{% include 'notebox_end' %}
-Containers can be dispatched to an LSF cluster. The dispatcher sends work to the cluster using LSF's @bsub@ command, so it works in a variety of LSF configurations.
-
-*LSF support is currently considered experimental.*
+h2(#overview). Overview
-Limitations include:
-* Arvados container priority is not propagated to LSF job priority. This can cause inefficient use of compute resources, and even deadlock if there are fewer compute nodes than concurrent Arvados workflows.
-* Combining LSF with docker may not work, depending on LSF configuration and user/group IDs (if LSF only sets up the configured user's primary group ID when executing the crunch-run process on a compute node, it may not have permission to connect to the docker daemon).
+Containers can be dispatched to an LSF cluster. The dispatcher sends work to the cluster using LSF's @bsub@ command, so it works in a variety of LSF configurations.
In order to run containers, you must choose a user that has permission to set up FUSE mounts and run Singularity/Docker containers on each compute node. This install guide refers to this user as the @crunch@ user. We recommend you create this user on each compute node with the same UID and GID, and add it to the @fuse@ and @docker@ system groups to grant it the necessary permissions. However, you can run the dispatcher under any account with sufficient permissions across the cluster.
Set up all of your compute nodes "as you would for a SLURM cluster":../crunch2-slurm/install-compute-node.html.
+*Current limitations*:
+* Arvados container priority is not propagated to LSF job priority. This can cause inefficient use of compute resources, and even deadlock if there are fewer compute nodes than concurrent Arvados workflows.
+* Combining LSF with docker may not work, depending on LSF configuration and user/group IDs (if LSF only sets up the configured user's primary group ID when executing the crunch-run process on a compute node, it may not have permission to connect to the docker daemon).
h2(#update-config). Update config.yml
Review the following configuration parameters and adjust as needed.
-
h3(#BsubSudoUser). Containers.LSF.BsubSudoUser
arvados-dispatch-lsf uses @sudo@ to execute @bsub@, for example @sudo -E -u crunch bsub [...]@. This means the @crunch@ account must exist on the hosts where LSF jobs run ("execution hosts"), as well as on the host where you are installing the Arvados LSF dispatcher (the "submission host"). To use a user account other than @crunch@, configure @BsubSudoUser@:
h2(#introduction). Introduction
-The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":https://pkg.go.dev/git.arvados.org/arvados.git/services/keep-web for more detail.
h2(#dns). Configure DNS
name=Arvados
baseurl=http://rpm.arvados.org/CentOS/$releasever/os/$basearch/
gpgcheck=1
-gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-curoverse
+gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-arvados
</code></pre>
</notextile>
-{% include 'install_redhat_key' %}
+{% include 'gpg_key_fingerprint' %}
h3(#debian). Debian and Ubuntu
{% include 'install_debian_key' %}
+{% include 'gpg_key_fingerprint' %}
+
As root, add the Arvados package repository to your sources. This command depends on your OS vendor and version:
table(table table-bordered table-condensed).
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+h2(#overview). Overview
+
Arvados can be configured to use "Singularity":https://sylabs.io/singularity/ instead of Docker to execute containers on cloud nodes or a SLURM/LSF cluster. Singularity may be preferable due to its simpler installation and lack of long-running daemon process and special system users/groups.
-Please note:
-* *Singularity support is currently considered experimental.*
+*Current limitations*:
* Even when using the singularity runtime, users' container images are expected to be saved in Docker format using @arv keep docker@. Arvados converts the Docker image to Singularity format (@.sif@) at runtime as needed. Specifying a @.sif@ file as an image when submitting a container request is not yet supported.
* Singularity does not limit the amount of memory available in a container. Each container will have access to all memory on the host where it runs, unless memory use is restricted by SLURM/LSF.
* Programs running in containers may behave differently due to differences between Singularity and Docker.
** The root (image) filesystem is read-only in a Singularity container. Programs that attempt to write outside a designated output or temporary directory are likely to fail.
** The Docker ENTRYPOINT instruction is ignored.
-* Arvados is currently tested with Singularity version 3.5.2.
+* Arvados is tested with Singularity version 3.7.4. Other versions may not work.
+
+h2(#configuration). Configuration
-To use singularity, first make sure "Singularity is installed":https://sylabs.io/guides/3.5/user-guide/quick_start.html on your cloud worker image or SLURM/LSF compute nodes as applicable. Note @squashfs-tools@ is required.
+To use singularity, first make sure "Singularity is installed":https://sylabs.io/guides/3.7/user-guide/quick_start.html on your cloud worker image or SLURM/LSF compute nodes as applicable. Note @squashfs-tools@ is required.
<notextile>
<pre><code>$ <span class="userinput">singularity version</span>
-3.5.2
+3.7.4
$ <span class="userinput">mksquashfs -version</span>
mksquashfs version 4.3-git (2014/06/09)
[...]
Then update @Containers.RuntimeEngine@ in your cluster configuration:
<notextile>
-<pre><code> # Container runtime: "docker" (default) or "singularity" (experimental)
+<pre><code> # Container runtime: "docker" (default) or "singularity"
RuntimeEngine: singularity
</code></pre>
</notextile>
table(table table-bordered table-condensed).
|_. Option |_. Description |
-|==--help==| This list of options|
-|==--parent-group-uuid==| UUID of group to own all the externally synchronized groups|
-|==--user-id== | Identifier to use in looking up user. One of 'email' or 'username' (Default: 'email')|
-|==--verbose==| Log informational messages (Default: False)|
-|==--version==| Print version and exit|
+|==--help==|This list of options|
+|==--case-insensitive==|Uses case-insensitive username matching|
+|==--parent-group-uuid==|UUID of group to own all the externally synchronized groups|
+|==--user-id==|Identifier to use in looking up user. One of 'email' or 'username' (Default: 'email')|
+|==--verbose==|Log informational messages (Default: False)|
+|==--version==|Print version and exit|
h2. Examples
runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
- runGoProgram{src: "services/keepstore", svc: super.cluster.Services.Keepstore},
+ runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
installPassenger{src: "services/api"},
AcceptAccessTokenScope: ""
PAM:
- # (Experimental) Use PAM to authenticate users.
+ # Use PAM to authenticate users.
Enable: false
# PAM service name. PAM will apply the policy in the
UsePreemptibleInstances: false
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
- # (experimental) cloud dispatcher for executing containers on
- # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
+ # cloud dispatcher for executing containers on worker VMs.
+ # Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
# and ends with "\n-----END RSA PRIVATE KEY-----\n".
DispatchPrivateKey: ""
# Minimum time between two attempts to run the same container
MinRetryPeriod: 0s
- # Container runtime: "docker" (default) or "singularity" (experimental)
+ # Container runtime: "docker" (default) or "singularity"
RuntimeEngine: docker
Logging:
GitInternalDir: /var/lib/arvados/internal.git
CloudVMs:
- # Enable the cloud scheduler (experimental).
+ # Enable the cloud scheduler.
Enable: false
# Name/number of port where workers' SSH services listen.
AcceptAccessTokenScope: ""
PAM:
- # (Experimental) Use PAM to authenticate users.
+ # Use PAM to authenticate users.
Enable: false
# PAM service name. PAM will apply the policy in the
UsePreemptibleInstances: false
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
- # (experimental) cloud dispatcher for executing containers on
- # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
+ # cloud dispatcher for executing containers on worker VMs.
+ # Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
# and ends with "\n-----END RSA PRIVATE KEY-----\n".
DispatchPrivateKey: ""
# Minimum time between two attempts to run the same container
MinRetryPeriod: 0s
- # Container runtime: "docker" (default) or "singularity" (experimental)
+ # Container runtime: "docker" (default) or "singularity"
RuntimeEngine: docker
Logging:
GitInternalDir: /var/lib/arvados/internal.git
CloudVMs:
- # Enable the cloud scheduler (experimental).
+ # Enable the cloud scheduler.
Enable: false
# Name/number of port where workers' SSH services listen.
}
func (e *singularityExecutor) Start() error {
- args := []string{"singularity", "exec", "--containall", "--no-home", "--cleanenv", "--pwd", e.spec.WorkingDir}
+ args := []string{"singularity", "exec", "--containall", "--cleanenv", "--pwd", e.spec.WorkingDir}
if !e.spec.EnableNetwork {
args = append(args, "--net", "--network=none")
}
sort.Strings(binds)
for _, path := range binds {
mount := e.spec.BindMounts[path]
- args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
+ if path == e.spec.Env["HOME"] {
+ // Singularity treates $HOME as special case
+ args = append(args, "--home", mount.HostPath+":"+path)
+ } else {
+ args = append(args, "--bind", mount.HostPath+":"+path+":"+readonlyflag[mount.ReadOnly])
+ }
}
// This is for singularity 3.5.2. There are some behaviors
env := make([]string, 0, len(e.spec.Env))
for k, v := range e.spec.Env {
if k == "HOME" {
- // $HOME is a special case
- args = append(args, "--home="+v)
- } else {
- env = append(env, "SINGULARITYENV_"+k+"="+v)
+ // Singularity treates $HOME as special case, this is handled
+ // with --home above
+ continue
}
+ env = append(env, "SINGULARITYENV_"+k+"="+v)
}
args = append(args, e.imageFilename)
s.required_ruby_version = '>= 2.1.0'
s.add_runtime_dependency 'arvados', '>= 1.4.1.20190320201707'
# Our google-api-client dependency used to be < 0.9, but that could be
- # satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
- s.add_runtime_dependency 'arvados-google-api-client', '~> 0.6', '>= 0.6.3', '<0.8.9'
+ # satisfied by the buggy 0.9.pre*, cf. https://dev.arvados.org/issues/9213
+ # We need at least version 0.8.7.3, cf. https://dev.arvados.org/issues/15673
+ s.add_runtime_dependency('arvados-google-api-client', '>= 0.8.7.3', '< 0.8.9')
s.add_runtime_dependency 'activesupport', '>= 3.2.13', '< 5.3'
s.add_runtime_dependency 'json', '>= 1.7.7', '<3'
s.add_runtime_dependency 'optimist', '~> 3.0'
s.add_runtime_dependency 'oj', '< 3.10.9'
s.add_runtime_dependency 'curb', '~> 0.8'
s.add_runtime_dependency 'launchy', '< 2.5'
- # arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
- s.add_dependency('faraday', '< 0.16')
s.homepage =
'https://arvados.org'
end
api_client.users().current().execute()
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
+ executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4, stdout=stdout)
except WorkflowException as e:
logger.error(e, exc_info=(sys.exc_info()[1] if arvargs.debug else False))
return 1
if self.output_ttl < 0:
raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
- storage_class_req, _ = self.get_requirement("http://arvados.org/cwl#OutputStorageClass")
- if storage_class_req and storage_class_req.get("intermediateStorageClass"):
- container_request["output_storage_classes"] = aslist(storage_class_req["intermediateStorageClass"])
- else:
- container_request["output_storage_classes"] = runtimeContext.intermediate_storage_classes.strip().split(",")
+
+ if self.arvrunner.api._rootDesc["revision"] >= "20210628":
+ storage_class_req, _ = self.get_requirement("http://arvados.org/cwl#OutputStorageClass")
+ if storage_class_req and storage_class_req.get("intermediateStorageClass"):
+ container_request["output_storage_classes"] = aslist(storage_class_req["intermediateStorageClass"])
+ else:
+ container_request["output_storage_classes"] = runtimeContext.intermediate_storage_classes.strip().split(",")
if self.timelimit is not None and self.timelimit > 0:
scheduling_parameters["max_run_time"] = self.timelimit
if runtimeContext.debug:
command.append("--debug")
- if runtimeContext.storage_classes != "default":
+ if runtimeContext.storage_classes != "default" and runtimeContext.storage_classes:
command.append("--storage-classes=" + runtimeContext.storage_classes)
- if runtimeContext.intermediate_storage_classes != "default":
+ if runtimeContext.intermediate_storage_classes != "default" and runtimeContext.intermediate_storage_classes:
command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes)
if self.on_error:
if not images:
# Fetch Docker image if necessary.
try:
- cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
+ result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
force_pull, tmp_outdir_prefix)
+ if not result:
+ raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
except OSError as e:
raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
if not docker_req:
self.hints.append({"class": "DockerRequirement",
- "dockerImageId": "arvados/jobs:"+__version__})
+ "dockerPull": "arvados/jobs:"+__version__})
self.arvrunner = arvrunner
arvargs=None,
keep_client=None,
num_retries=4,
- thread_count=4):
+ thread_count=4,
+ stdout=sys.stdout):
if arvargs is None:
arvargs = argparse.Namespace()
self.should_estimate_cache_size = True
self.fs_access = None
self.secret_store = None
+ self.stdout = stdout
if keep_client is not None:
self.keep_client = keep_client
if runtimeContext.submit_request_uuid and self.work_api != "containers":
raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
- default_storage_classes = ",".join([k for k,v in self.api.config()["StorageClasses"].items() if v.get("Default") is True])
+ default_storage_classes = ",".join([k for k,v in self.api.config().get("StorageClasses", {"default": {"Default": True}}).items() if v.get("Default") is True])
if runtimeContext.storage_classes == "default":
runtimeContext.storage_classes = default_storage_classes
if runtimeContext.intermediate_storage_classes == "default":
if existing_uuid or runtimeContext.create_workflow:
# Create a pipeline template or workflow record and exit.
if self.work_api == "containers":
- return (upload_workflow(self, tool, job_order,
+ uuid = upload_workflow(self, tool, job_order,
self.project_uuid,
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
merged_map=merged_map,
- submit_runner_image=runtimeContext.submit_runner_image),
- "success")
+ submit_runner_image=runtimeContext.submit_runner_image)
+ self.stdout.write(uuid + "\n")
+ return (None, "success")
self.apply_reqs(job_order, tool)
if runtimeContext.submit and not runtimeContext.wait:
runnerjob = next(jobiter)
runnerjob.run(runtimeContext)
- return (runnerjob.uuid, "success")
+ self.stdout.write(runnerjob.uuid+"\n")
+ return (None, "success")
current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
if current_container:
outdir="", # type: Text
tmpdir="", # type: Text
stagedir="", # type: Text
- cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion") or metadata.get("cwlVersion")
+ cwlVersion=metadata.get("http://commonwl.org/cwltool#original_cwlVersion") or metadata.get("cwlVersion"),
+ container_engine="docker"
)
def search_schemadef(name, reqs):
elif isinstance(pattern, dict):
specs.append(pattern)
elif isinstance(pattern, str):
- specs.append({"pattern": pattern})
+ specs.append({"pattern": pattern, "required": sf.get("required")})
else:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Expression must return list, object, string or null")
for i, sf in enumerate(specs):
if isinstance(sf, dict):
if sf.get("class") == "File":
- pattern = sf["basename"]
+ pattern = None
+ sfpath = sf["location"]
+ required = True
else:
pattern = sf["pattern"]
required = sf.get("required")
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Expression must return list, object, string or null")
- sfpath = substitute(primary["location"], pattern)
+ if pattern is not None:
+ sfpath = substitute(primary["location"], pattern)
+
required = builder.do_eval(required, context=primary)
if fsaccess.exists(sfpath):
- found.append({"location": sfpath, "class": "File"})
+ if pattern is not None:
+ found.append({"location": sfpath, "class": "File"})
+ else:
+ found.append(sf)
elif required:
raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
"Required secondary file '%s' does not exist" % sfpath)
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20210816212154',
- 'schema-salad==8.2.20210902094147',
+ 'cwltool==3.1.20210922203925',
+ 'schema-salad==8.2.20210918131710',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0',
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
stubs.api = mock.MagicMock()
stubs.api._rootDesc = get_rootDesc()
stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
+ stubs.api._rootDesc["revision"] = "20210628"
stubs.api.users().current().execute.return_value = {
"uuid": stubs.fake_user_uuid,
"enableReuse": False,
},
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
"cwltool": "http://commonwl.org/cwltool#"
}
def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
final_output_c = arvados.collection.Collection()
make_output.return_value = ({},final_output_c)
+ stubs.api.config().get.return_value = {"default": {"Default": True}}
def set_final_output(job_order, output_callback, runtimeContext):
output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success")
"keep_cache": 512
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
"content": {
"$graph": [
{
- "$namespaces": {
- "cwltool": "http://commonwl.org/cwltool#"
- },
"arguments": [
"md5sum",
"example.conf"
]
}
],
+ "$namespaces": {
+ "cwltool": "http://commonwl.org/cwltool#"
+ },
"cwlVersion": "v1.0"
},
"kind": "json"
],
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
"arv": "http://arvados.org/cwl#"
}
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
StepInputExpressionRequirement: {}
hints:
DockerRequirement:
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
steps:
substep:
in:
hints:
- class: arv:RunInSingleContainer
- class: DockerRequirement
- dockerPull: arvados/jobs:1.4.0.20190604172024
+ dockerPull: arvados/jobs:2.2.2
run:
class: Workflow
id: mysub
]
}
],
+ "$namespaces": {
+ "arv": "http://arvados.org/cwl#"
+ },
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
_, err = s.fs.OpenFile("/fg3/A Subproject", 0, 0)
c.Assert(err, check.Not(check.IsNil))
+
+ // An 'exists' 'arvados#collection' filter means only collections with certain properties should be returned.
+ s.fs.MountProject("fg4", fixtureAFilterGroupFourUUID)
+
+ _, err = s.fs.Stat("/fg4/collection with list property with odd values")
+ c.Assert(err, check.IsNil)
+
+ _, err = s.fs.Stat("/fg4/collection with list property with even values")
+ c.Assert(err, check.IsNil)
+
+ // A 'contains' 'arvados#collection' filter means only collections with certain properties should be returned.
+ s.fs.MountProject("fg5", fixtureAFilterGroupFiveUUID)
+
+ _, err = s.fs.Stat("/fg5/collection with list property with odd values")
+ c.Assert(err, check.IsNil)
+
+ _, err = s.fs.Stat("/fg5/collection with list property with string value")
+ c.Assert(err, check.IsNil)
+
+ _, err = s.fs.Stat("/fg5/collection with prop2 5")
+ c.Assert(err, check.Not(check.IsNil))
+
+ _, err = s.fs.Stat("/fg5/collection with list property with even values")
+ c.Assert(err, check.Not(check.IsNil))
}
func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
+ fixtureAFilterGroupFourUUID = "zzzzz-j7d0g-filtergroupfour"
+ fixtureAFilterGroupFiveUUID = "zzzzz-j7d0g-filtergroupfive"
fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
import os
import re
import socket
+import ssl
import sys
import time
import types
def _intercept_http_request(self, uri, method="GET", headers={}, **kwargs):
- if (self.max_request_size and
- kwargs.get('body') and
- self.max_request_size < len(kwargs['body'])):
- raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
-
- if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
- headers['X-External-Client'] = '1'
-
- headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
if not headers.get('X-Request-Id'):
headers['X-Request-Id'] = self._request_id()
+ try:
+ if (self.max_request_size and
+ kwargs.get('body') and
+ self.max_request_size < len(kwargs['body'])):
+ raise apiclient_errors.MediaUploadSizeError("Request size %i bytes exceeds published limit of %i bytes" % (len(kwargs['body']), self.max_request_size))
- retryable = method in [
- 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
- retry_count = self._retry_count if retryable else 0
-
- if (not retryable and
- time.time() - self._last_request_time > self._max_keepalive_idle):
- # High probability of failure due to connection atrophy. Make
- # sure this request [re]opens a new connection by closing and
- # forgetting all cached connections first.
- for conn in self.connections.values():
- conn.close()
- self.connections.clear()
-
- delay = self._retry_delay_initial
- for _ in range(retry_count):
- self._last_request_time = time.time()
- try:
- return self.orig_http_request(uri, method, headers=headers, **kwargs)
- except http.client.HTTPException:
- _logger.debug("Retrying API request in %d s after HTTP error",
- delay, exc_info=True)
- except socket.error:
- # This is the one case where httplib2 doesn't close the
- # underlying connection first. Close all open
- # connections, expecting this object only has the one
- # connection to the API server. This is safe because
- # httplib2 reopens connections when needed.
- _logger.debug("Retrying API request in %d s after socket error",
- delay, exc_info=True)
+ if config.get("ARVADOS_EXTERNAL_CLIENT", "") == "true":
+ headers['X-External-Client'] = '1'
+
+ headers['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
+
+ retryable = method in [
+ 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
+ retry_count = self._retry_count if retryable else 0
+
+ if (not retryable and
+ time.time() - self._last_request_time > self._max_keepalive_idle):
+ # High probability of failure due to connection atrophy. Make
+ # sure this request [re]opens a new connection by closing and
+ # forgetting all cached connections first.
for conn in self.connections.values():
conn.close()
- except httplib2.SSLHandshakeError as e:
- # Intercept and re-raise with a better error message.
- raise httplib2.SSLHandshakeError("Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e))
+ self.connections.clear()
+
+ delay = self._retry_delay_initial
+ for _ in range(retry_count):
+ self._last_request_time = time.time()
+ try:
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ except http.client.HTTPException:
+ _logger.debug("[%s] Retrying API request in %d s after HTTP error",
+ headers['X-Request-Id'], delay, exc_info=True)
+ except ssl.SSLCertVerificationError as e:
+ raise ssl.SSLCertVerificationError(e.args[0], "Could not connect to %s\n%s\nPossible causes: remote SSL/TLS certificate expired, or was issued by an untrusted certificate authority." % (uri, e)) from None
+ except socket.error:
+ # This is the one case where httplib2 doesn't close the
+ # underlying connection first. Close all open
+ # connections, expecting this object only has the one
+ # connection to the API server. This is safe because
+ # httplib2 reopens connections when needed.
+ _logger.debug("[%s] Retrying API request in %d s after socket error",
+ headers['X-Request-Id'], delay, exc_info=True)
+ for conn in self.connections.values():
+ conn.close()
+
+ time.sleep(delay)
+ delay = delay * self._retry_delay_backoff
- time.sleep(delay)
- delay = delay * self._retry_delay_backoff
-
- self._last_request_time = time.time()
- return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ self._last_request_time = time.time()
+ return self.orig_http_request(uri, method, headers=headers, **kwargs)
+ except Exception as e:
+ # Prepend "[request_id] " to the error message, which we
+ # assume is the first string argument passed to the exception
+ # constructor.
+ for i in range(len(e.args or ())):
+ if type(e.args[i]) == type(""):
+ e.args = e.args[:i] + ("[{}] {}".format(headers['X-Request-Id'], e.args[i]),) + e.args[i+1:]
+ raise type(e)(*e.args)
+ raise
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
try:
self._populate()
- except (IOError, errors.SyntaxError) as e:
- raise errors.ArgumentError("Error processing manifest text: %s", e)
+ except errors.SyntaxError as e:
+ raise errors.ArgumentError("Error processing manifest text: %s", str(e)) from None
def storage_classes_desired(self):
return self._storage_classes_desired or []
self.find_or_create(os.path.join(stream_name, name[:-2]), COLLECTION)
else:
filepath = os.path.join(stream_name, name)
- afile = self.find_or_create(filepath, FILE)
+ try:
+ afile = self.find_or_create(filepath, FILE)
+ except IOError as e:
+ if e.errno == errno.ENOTDIR:
+ raise errors.SyntaxError("Dir part of %s conflicts with file of the same name.", filepath) from None
+ else:
+ raise e from None
if isinstance(afile, ArvadosFile):
afile.add_segment(blocks, pos, size)
else:
self.get_counter.add(1)
+ request_id = (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id())
+ if headers is None:
+ headers = {}
+ headers['X-Request-Id'] = request_id
+
slot = None
blob = None
try:
self.misses_counter.add(1)
- if headers is None:
- headers = {}
- headers['X-Request-Id'] = (request_id or
- (hasattr(self, 'api_client') and self.api_client.request_id) or
- arvados.util.new_request_id())
-
# If the locator has hints specifying a prefix (indicating a
# remote keepproxy) or the UUID of a local gateway service,
# read data from the indicated service(s) instead of the usual
for key in sorted_roots)
if not roots_map:
raise arvados.errors.KeepReadError(
- "failed to read {}: no Keep services available ({})".format(
- loc_s, loop.last_result()))
+ "[{}] failed to read {}: no Keep services available ({})".format(
+ request_id, loc_s, loop.last_result()))
elif not_founds == len(sorted_roots):
raise arvados.errors.NotFoundError(
- "{} not found".format(loc_s), service_errors)
+ "[{}] {} not found".format(request_id, loc_s), service_errors)
else:
raise arvados.errors.KeepReadError(
- "failed to read {} after {}".format(loc_s, loop.attempts_str()), service_errors, label="service")
+ "[{}] failed to read {} after {}".format(request_id, loc_s, loop.attempts_str()), service_errors, label="service")
@retry.retry_method
def put(self, data, copies=2, num_retries=None, request_id=None, classes=None):
return loc_s
locator = KeepLocator(loc_s)
+ request_id = (request_id or
+ (hasattr(self, 'api_client') and self.api_client.request_id) or
+ arvados.util.new_request_id())
headers = {
- 'X-Request-Id': (request_id or
- (hasattr(self, 'api_client') and self.api_client.request_id) or
- arvados.util.new_request_id()),
+ 'X-Request-Id': request_id,
'X-Keep-Desired-Replicas': str(copies),
}
roots_map = {}
return writer_pool.response()
if not roots_map:
raise arvados.errors.KeepWriteError(
- "failed to write {}: no Keep services available ({})".format(
- data_hash, loop.last_result()))
+ "[{}] failed to write {}: no Keep services available ({})".format(
+ request_id, data_hash, loop.last_result()))
else:
service_errors = ((key, roots_map[key].last_result()['error'])
for key in sorted_roots
if roots_map[key].last_result()['error'])
raise arvados.errors.KeepWriteError(
- "failed to write {} after {} (wanted {} copies but wrote {})".format(
- data_hash, loop.attempts_str(), (copies, classes), writer_pool.done()), service_errors, label="service")
+ "[{}] failed to write {} after {} (wanted {} copies but wrote {})".format(
+ request_id, data_hash, loop.attempts_str(), (copies, classes), writer_pool.done()), service_errors, label="service")
def local_store_put(self, data, copies=1, num_retries=None, classes=[]):
"""A stub for put().
confdata['Clusters']['zzzzz']['Collections']['BlobSigning'] = blob_signing
with open(conf, 'w') as f:
yaml.safe_dump(confdata, f)
- keep_cmd = ["keepstore", "-config", conf]
+ keep_cmd = ["arvados-server", "keepstore", "-config", conf]
with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf:
with open('/dev/null') as _stdin:
for msg in ["Bad UUID format", "Bad output format"]:
self.assertIn(msg, err_s)
+ @mock.patch('time.sleep')
+ def test_exceptions_include_request_id(self, sleep):
+ api = arvados.api('v1')
+ api.request_id='fake-request-id'
+ api._http.orig_http_request = mock.MagicMock()
+ api._http.orig_http_request.side_effect = socket.error('mock error')
+ caught = None
+ try:
+ api.users().current().execute()
+ except Exception as e:
+ caught = e
+ self.assertRegex(str(caught), r'fake-request-id')
+
def test_exceptions_without_errors_have_basic_info(self):
mock_responses = {
'arvados.humans.delete': (
self.keep_client.head(self.locator)
self.assertAutomaticRequestId(mock.responses[0])
+ def test_request_id_in_exception(self):
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepReadError, self.test_id):
+ self.keep_client.head(self.locator, request_id=self.test_id)
+
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepReadError, r'req-[a-z0-9]{20}'):
+ self.keep_client.get(self.locator)
+
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepWriteError, self.test_id):
+ self.keep_client.put(self.data, request_id=self.test_id)
+
+ with tutil.mock_keep_responses(b'', 400, 400, 400) as mock:
+ with self.assertRaisesRegex(arvados.errors.KeepWriteError, r'req-[a-z0-9]{20}'):
+ self.keep_client.put(self.data)
+
def assertAutomaticRequestId(self, resp):
hdr = [x for x in resp.getopt(pycurl.HTTPHEADER)
if x.startswith('X-Request-Id: ')][0]
s.add_dependency('activesupport', '>= 3')
s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
# Our google-api-client dependency used to be < 0.9, but that could be
- # satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
- s.add_dependency('arvados-google-api-client', '>= 0.7', '< 0.8.9')
+ # satisfied by the buggy 0.9.pre*, cf. https://dev.arvados.org/issues/9213
+ # We need at least version 0.8.7.3, cf. https://dev.arvados.org/issues/15673
+ s.add_dependency('arvados-google-api-client', '>= 0.8.7.3', '< 0.8.9')
# work around undeclared dependency on i18n in some activesupport 3.x.x:
s.add_dependency('i18n', '~> 0')
s.add_dependency('json', '>= 1.7.7', '<3')
- # arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
- s.add_dependency('faraday', '< 0.16')
+ # Avoid warning on Ruby 2.7, cf. https://dev.arvados.org/issues/18247
+ s.add_dependency('faraday', '>= 0.17.4')
s.add_runtime_dependency('jwt', '<2', '>= 0.1.5')
s.homepage =
'https://arvados.org'
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20210503",
+ revision: "20210628",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
errors.add :properties, "when filter operator is 'is_a', attribute must be 'uuid'"
return
end
- if ! ["=","<","<=",">",">=","!=","like","ilike","in","not in","is_a"].include?(filter[1].downcase)
- errors.add :properties, "filter operator is not valid (must be =,<,<=,>,>=,!=,like,ilike,in,not in,is_a)"
+ if ! ["=","<","<=",">",">=","!=","like","ilike","in","not in","is_a","exists","contains"].include?(filter[1].downcase)
+ errors.add :properties, "filter operator is not valid (must be =,<,<=,>,>=,!=,like,ilike,in,not in,is_a,exists,contains)"
return
end
end
properties:
filters: [["uuid", "is_a", "arvados#collection"]]
+afiltergroup4:
+ uuid: zzzzz-j7d0g-filtergroupfour
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2014-04-21 15:37:48 -0400
+ updated_at: 2014-04-21 15:37:48 -0400
+ name: A filter group with an exists collections filter
+ group_class: filter
+ properties:
+ filters: [["collections.properties.listprop","exists",true],["uuid", "is_a", "arvados#collection"]]
+
+afiltergroup5:
+ uuid: zzzzz-j7d0g-filtergroupfive
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2014-04-21 15:37:48 -0400
+ updated_at: 2014-04-21 15:37:48 -0400
+ name: A filter group with a contains collections filter
+ group_class: filter
+ properties:
+ filters: [["collections.properties.listprop","contains","elem1"],["uuid", "is_a", "arvados#collection"]]
+
future_project_viewing_group:
uuid: zzzzz-j7d0g-futrprojviewgrp
owner_uuid: zzzzz-tpzed-000000000000000
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"sync"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
Command = service.Command(arvados.ServiceNameKeepstore, newHandlerOrErrorHandler)
)
-func main() {
- os.Exit(runCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
-}
-
func runCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
args, ok := convertKeepstoreFlagsToServiceFlags(args, ctxlog.FromContext(context.Background()))
if !ok {
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"io"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"gopkg.in/check.v1"
// The HTTP handlers are responsible for enforcing permission policy,
// so these tests must exercise all possible permission permutations.
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"container/list"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"time"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"fmt"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
type MockMutex struct {
AllowLock chan struct{}
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"time"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"strconv"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bufio"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"sync"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"encoding/json"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"errors"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"container/list"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"context"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"bytes"
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
/* A WorkQueue is an asynchronous thread-safe queue manager. It
provides a channel from which items can be read off the queue, and
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepstore
import (
"container/list"
echo "Status: running"
echo "Container IP: $(getip)"
echo "Published host: $(gethost)"
+ echo "Workbench: https://$(gethost)"
else
echo "Status: not running"
fi
RUN mkdir -p /etc/apt/sources.list.d && \
echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \
apt-get update && \
- apt-get -yq --no-install-recommends install docker-ce=5:19.03.13~3-0~debian-buster && \
+ apt-get -yq --no-install-recommends install docker-ce=5:20.10.6~3-0~debian-buster && \
apt-get clean
# Set UTF-8 locale
RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
-RUN /var/lib/arvbox/service/crunch-dispatch-local/run --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run --only-deps
RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
mkdir -p /tmp/crunch0 /tmp/crunch1
chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
+ # singularity needs to be owned by root and suid
+ chown root /var/lib/arvados/bin/singularity \
+ /var/lib/arvados/etc/singularity/singularity.conf \
+ /var/lib/arvados/etc/singularity/capability.json \
+ /var/lib/arvados/etc/singularity/ecl.toml
+ chmod u+s /var/lib/arvados/bin/singularity
+
echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
cat <<EOF > /etc/profile.d/paths.sh
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-# singularity can use suid
-chown root /var/lib/arvados/bin/singularity \
- /var/lib/arvados/etc/singularity/singularity.conf \
- /var/lib/arvados/etc/singularity/capability.json \
- /var/lib/arvados/etc/singularity/ecl.toml
-chmod u+s /var/lib/arvados/bin/singularity
-
-exec /usr/local/lib/arvbox/runsu.sh $0-service $1
--- /dev/null
+/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
- postgresql12-contrib
{%- else %}
+ use_upstream_repo: false
pkgs_extra:
- postgresql-contrib
{%- endif %}
- openssl
- ca-certificates
+# Remove the RANDFILE parameter in openssl.cnf as it makes openssl fail in Ubuntu 18.04
+# Saving and restoring the rng state is not necessary anymore in the openssl 1.1.1
+# random generator, cf
+# https://github.com/openssl/openssl/issues/7754
+#
+arvados_test_salt_states_examples_single_host_snakeoil_certs_file_comment_etc_openssl_conf:
+ file.comment:
+ - name: /etc/ssl/openssl.cnf
+ - regex: ^RANDFILE.*
+ - onlyif: grep -q ^RANDFILE /etc/ssl/openssl.cnf
+ - require_in:
+ - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
+
arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
# Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
cmd.run:
export ARVADOS_API_TOKEN="${user_api_token}"
echo "Running test CWL workflow"
-cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
+cwl-runner --debug hasher-workflow.cwl hasher-workflow-job.yml
projects / arvados.git / commitdiff