*by-sa-3.0.txt
*COPYING
doc/fonts/*
+doc/user/cwl/federated/*
*/docker_image
docker/jobs/apt.arvados.org.list
*/en.bootstrap.yml
*/script/rails
sdk/cwl/tests/input/blorp.txt
sdk/cwl/tests/tool/blub.txt
+sdk/cwl/tests/federation/data/*
sdk/go/manifest/testdata/*_manifest
sdk/java/.classpath
sdk/java/pom.xml
sdk/R/ArvadosR.Rproj
*.Rd
lib/dispatchcloud/test/sshkey_*
+*.asc
multi_json (~> 1.0)
websocket-driver (>= 0.2.0)
public_suffix (3.0.2)
- rack (1.6.10)
+ rack (1.6.11)
rack-mini-profiler (0.10.7)
rack (>= 1.2.0)
rack-test (0.6.3)
owner_uuid: @object.uuid
}
})
- redirect_to root_url(api_token: resp[:api_token])
+ redirect_to root_url(api_token: "v2/#{resp[:uuid]}/#{resp[:api_token]}")
end
def home
attrs['state'] = "Uncommitted"
# required
- attrs['command'] = ["arvados-cwl-runner",
- "--local",
- "--api=containers",
- "--project-uuid=#{params['work_unit']['owner_uuid']}",
- "/var/lib/cwl/workflow.json#main",
- "/var/lib/cwl/cwl.input.json"]
attrs['container_image'] = "arvados/jobs"
attrs['cwd'] = "/var/spool/cwl"
attrs['output_path'] = "/var/spool/cwl"
"API" => true
}
+ keep_cache = 256
input_defaults = {}
if wf_json
main = get_cwl_main(wf_json)
if hint[:ramMin]
runtime_constraints["ram"] = hint[:ramMin] * 1024 * 1024
end
+ if hint[:keep_cache]
+ keep_cache = hint[:keep_cache]
+ end
end
end
end
end
+ attrs['command'] = ["arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--project-uuid=#{params['work_unit']['owner_uuid']}",
+ "--collection-keep-cache=#{keep_cache}",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"]
+
# mounts
mounts = {
"/var/lib/cwl/cwl.input.json" => {
end
def render_markup(markup)
- sanitize(raw(RedCloth.new(markup.to_s).to_html(:refs_arvados, :textile))) if markup
+ allowed_tags = Rails::Html::Sanitizer.white_list_sanitizer.allowed_tags + %w(table tbody th tr td col colgroup caption thead tfoot)
+ sanitize(raw(RedCloth.new(markup.to_s).to_html(:refs_arvados, :textile)), tags: allowed_tags) if markup
end
def human_readable_bytes_html(n)
assert_includes @response.body, 'Textile description with unsafe script tag alert("Hello there").'
end
+ # Tests #14519
+ test "textile table on description renders as table html markup" do
+ use_token :active
+ project = api_fixture('groups')['aproject']
+ textile_table = <<EOT
+table(table table-striped table-condensed).
+|_. First Header |_. Second Header |
+|Content Cell |Content Cell |
+|Content Cell |Content Cell |
+EOT
+ found = Group.find(project['uuid'])
+ found.description = textile_table
+ found.save!
+ get(:show, {id: project['uuid']}, session_for(:active))
+ assert_includes @response.body, '<th>First Header'
+ assert_includes @response.body, '<td>Content Cell'
+ end
+
test "find a project and edit description to textile description with link to object" do
project = api_fixture('groups')['aproject']
use_token :active
assert_match /\/users\/welcome/, @response.redirect_url
end
+ test "'log in as user' feature uses a v2 token" do
+ post :sudo, {
+ id: api_fixture('users')['active']['uuid']
+ }, session_for('admin_trustedclient')
+ assert_response :redirect
+ assert_match /api_token=v2%2F/, @response.redirect_url
+ end
+
test "request shell access" do
user = api_fixture('users')['spectator']
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pyyaml|3.12|2|python|amd64
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rdflib|4.2.2|2|python|all
debian8,debian9,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|mistune|0.7.3|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|mistune|0.8.1|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|typing|3.6.4|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|avro|1.8.1|2|python|all
debian8,debian9,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
debian8,debian9,centos7|six|1.10.0|2|python3|all
debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all
-ubuntu1404|requests|2.4.3|2|python|all
+debian8,ubuntu1404,centos7|requests|2.6.1|2|python|all
centos7|contextlib2|0.5.4|2|python|all
centos7|isodate|0.5.4|2|python|all
centos7|python-daemon|2.1.2|1|python|all
centos7|psutil|5.0.1|0|python|all
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1
debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all
-all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
+all|ruamel.yaml|0.15.77|1|python|amd64|--python-setup-py-arguments --single-version-externally-managed --depends 'python-ruamel.ordereddict >= 0.4.9'
all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
all|junit-xml|1.8|3|python|all
all|rdflib-jsonld|0.4.0|2|python|all
#
# SPDX-License-Identifier: AGPL-3.0
+set -e
+
+arvados-cwl-runner --version
+
exec python <<EOF
import arvados_cwl
print "arvados-cwl-runner version", arvados_cwl.__version__
# clean up the docker build environment
cd "$WORKSPACE"
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
+ ARVADOS_BUILDING_VERSION="$version_tag"
+ ARVADOS_BUILDING_ITERATION="1"
+fi
+
python_sdk_ts=$(cd sdk/python && timestamp_from_git)
cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ python_sdk_version="${python_sdk_version}-2"
+else
+ python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+fi
+
+cwl_runner_version_orig=$cwl_runner_version
+
+if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ cwl_runner_version="${cwl_runner_version}-4"
+else
+ cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+fi
+
cd docker/jobs
docker build $NOCACHE \
- --build-arg python_sdk_version=${python_sdk_version}-2 \
- --build-arg cwl_runner_version=${cwl_runner_version}-4 \
- -t arvados/jobs:$cwl_runner_version .
+ --build-arg python_sdk_version=${python_sdk_version} \
+ --build-arg cwl_runner_version=${cwl_runner_version} \
+ -t arvados/jobs:$cwl_runner_version_orig .
ECODE=$?
FORCE=-f
fi
if ! [[ -z "$version_tag" ]]; then
- docker tag $FORCE arvados/jobs:$cwl_runner_version arvados/jobs:"$version_tag"
+ docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag"
else
- docker tag $FORCE arvados/jobs:$cwl_runner_version arvados/jobs:latest
+ docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:latest
fi
ECODE=$?
if ! [[ -z "$version_tag" ]]; then
docker_push arvados/jobs:"$version_tag"
else
- docker_push arvados/jobs:$cwl_runner_version
+ docker_push arvados/jobs:$cwl_runner_version_orig
docker_push arvados/jobs:latest
fi
title "upload arvados images finished (`timer`)"
package_go_binary tools/keep-exercise keep-exercise \
"Performance testing tool for Arvados Keep"
+
+# we need explicit debian_revision values in the dependencies for ruamel.yaml, because we have a package iteration
+# greater than zero. So we parse setup.py, get the ruamel.yaml dependencies, tell fpm not to automatically include
+# them in the package being built, and re-add them manually with an appropriate debian_revision value.
+# See #14552 for the reason for this (nasty) workaround. We use ${ruamel_depends[@]} in a few places further down
+# in this script.
+# Ward, 2018-11-28
+IFS=', ' read -r -a deps <<< `grep ruamel.yaml $WORKSPACE/sdk/python/setup.py |cut -f 3 -dl |sed -e "s/'//g"`
+declare -a ruamel_depends=()
+for i in ${deps[@]}; do
+ i=`echo "$i" | sed -e 's!\([0-9]\)! \1!'`
+ if [[ $i =~ .*\>.* ]]; then
+ ruamel_depends+=(--depends "python-ruamel.yaml $i-1")
+ elif [[ $i =~ .*\<.* ]]; then
+ ruamel_depends+=(--depends "python-ruamel.yaml $i-9")
+ else
+ echo "Encountered ruamel dependency that I can't parse. Aborting..."
+ exit 1
+ fi
+done
+
+
# The Python SDK
# Please resist the temptation to add --no-python-fix-name to the fpm call here
# (which would remove the python- prefix from the package name), because this
arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)}
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+
+ fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
fi
# cwl-runner
fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" "${iterargs[@]}"
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python 2
+test_package_presence python-cwltool "$cwltoolversion" python 3
if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion --iteration 2
+ fpm_build cwltool "" "" python $cwltoolversion --iteration 3 --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
fi
# The PAM module
fi
declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- ARVADOS_BUILDING_VERSION="$(git describe --abbrev=0).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
+ ARVADOS_BUILDING_VERSION="$(git tag -l |sort -V -r |head -n1).$(date -ud "@$git_ts" +%Y%m%d%H%M%S)"
echo "$ARVADOS_BUILDING_VERSION"
}
- Working with data sets:
- user/tutorials/tutorial-keep.html.textile.liquid
- user/tutorials/tutorial-keep-get.html.textile.liquid
- - user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
- - user/tutorials/tutorial-keep-mount.html.textile.liquid
+ - user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
+ - user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
+ - user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
- user/topics/keep.html.textile.liquid
+ - user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
- user/topics/arv-copy.html.textile.liquid
- user/topics/storage-classes.html.textile.liquid
- user/topics/collection-versioning.html.textile.liquid
- Develop an Arvados workflow:
- user/tutorials/intro-crunch.html.textile.liquid
- user/tutorials/writing-cwl-workflow.html.textile.liquid
+ - user/cwl/federated-workflows.html.textile.liquid
- user/cwl/cwl-style.html.textile.liquid
- user/cwl/cwl-extensions.html.textile.liquid
- user/topics/arv-docker.html.textile.liquid
--- /dev/null
+../user/cwl/federated/federated.cwl
\ No newline at end of file
--- /dev/null
+../user/cwl/federated/shards.yml
\ No newline at end of file
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
+h3. v1.3.0 (2018-12-05)
+
+This release includes several database migrations, which will be executed automatically as part of the API server upgrade. On large Arvados installations, these migrations will take a while. We've seen the upgrade take 30 minutes or more on installations with a lot of collections.
+
+The @arvados-controller@ component now requires the /etc/arvados/config.yml file to be present. See <a href="{{ site.baseurl }}/install/install-controller.html#configuration">the @arvados-controller@ installation instructions</a>.
+
+h3. v1.2.1 (2018-11-26)
+
+There are no special upgrade notes for this release.
+
h3. v1.2.0 (2018-09-05)
h4. Regenerate Postgres table statistics
Arguments specifying a *Location* of "query" are incorporated into the query portion of the URI or request body. For example, @/arvados/v1/resource_type?count=none@.
-Certain method calls on certain object types support "federation":{{site.baseurl}}/architecture/federation.html , that is, the ability to operate on objects owned by different clusters. API pages for specific object types list which federated operations are supported for that type (if any) in the "Methods" section.
+Certain method calls on certain object types support "federation":{{site.baseurl}}/architecture/federation.html , that is, the ability to operate on objects owned by different clusters. API pages for specific object types list which federated operations are supported for that type (if any) in the "Methods" section. Methods which implicitly include a cluster id (such as @GET@ on a specific uuid, using the uuid prefix) will be directed to the appropriate cluster. Methods that don't implicitly include the cluster id (such as @create@) use the @cluster_id@ query parameter to specify which cluster to direct the request.
h2. create
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |
-|{resource_type}|object|Name is the singular form of the resource type, e.g., for the "collections" resource, this argument is "collection"|body||
-|{cluster_id}|string|Optional, the cluster on which to create the object if not the current cluster.|query||
+|{resource_type}|object|Name is the singular form of the resource type, e.g., for the "collections" resource, this argument is "collection"|body|
+|{cluster_id}|string|Optional, the cluster on which to create the object if not the current cluster.|query|
h2. delete
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
{background:#ccffcc}.|container_request|object|Container request resource.|request body||
-
+|cluster_id|string|The federated cluster to submit the container request.|query||
The request body must include the required attributes command, container_image, cwd, and output_path. It can also inlcude other attributes such as environment, mounts, and runtime_constraints.
<div class="row">
<div class="col-sm-6">
<p><strong>What is Arvados</strong>
- <p><a href="https://arvados.org/">Arvados</a> enables you to quickly begin using cloud computing resources in your data science work. It allows you to track your methods and datasets, share them securely, and easily re-run analyses.
+ <p><a href="https://arvados.org/">Arvados</a> is a platform for managing compute and storage for cloud and HPC clusters. It allows you to track your methods and datasets, share them securely, and easily re-run analyses. It also make it possible to run analysis across multiple clusters (HPC, cloud, or hybrid) with <a href="{{site.baseurl}}/user/cwl/federated-workflows.html">Federated Multi-Cluster Workflows</a>.
</p>
<a name="Support"></a>
</code></pre>
</notextile>
-h3. Configure arvados-controller
+h3(#configuration). Configure arvados-controller
Create the cluster configuration file @/etc/arvados/config.yml@ using the following template.
proxy_pass http://keep-web;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ client_max_body_size 0;
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
}
}
</pre></notextile>
}
server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name keep.<span class="userinput">uuid_prefix</span>.your.domain;
+ listen <span class="userinput">[your public IP address]</span>:443 ssl;
+ server_name keep.<span class="userinput">uuid_prefix</span>.your.domain;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
- proxy_set_header X-Real-IP $remote_addr;
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
- ssl on;
- ssl_certificate /etc/nginx/keep.<span class="userinput">uuid_prefix</span>.your.domain-ssl.crt;
- ssl_certificate_key /etc/nginx/keep.<span class="userinput">uuid_prefix</span>.your.domain-ssl.key;
+ ssl on;
+ ssl_certificate /etc/nginx/keep.<span class="userinput">uuid_prefix</span>.your.domain-ssl.crt;
+ ssl_certificate_key /etc/nginx/keep.<span class="userinput">uuid_prefix</span>.your.domain-ssl.key;
# Clients need to be able to upload blocks of data up to 64MiB in size.
- client_max_body_size 64m;
+ client_max_body_size 64m;
location / {
- proxy_pass http://keepproxy;
+ proxy_pass http://keepproxy;
}
}
</pre></notextile>
h3. Usage
-Please refer to the "Mounting Keep as a filesystem":{{site.baseurl}}/user/tutorials/tutorial-keep-mount.html tutorial for more information.
\ No newline at end of file
+Please refer to the "Accessing Keep from GNU/Linux":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-gnu-linux.html tutorial for more information.
arv:WorkflowRunnerResources:
ramMin: 2048
coresMin: 2
+ keep_cache: 512
+ arv:ClusterTarget:
+ cluster_id: clsr1
+ project_uuid: clsr1-j7d0g-qxc4jcji7n4lafx
</pre>
The one exception to this is @arv:APIRequirement@, see note below.
|_. Field |_. Type |_. Description |
|ramMin|int|RAM, in mebibytes, to reserve for the arvados-cwl-runner process. Default 1 GiB|
|coresMin|int|Number of cores to reserve to the arvados-cwl-runner process. Default 1 core.|
+|keep_cache|int|Size of collection metadata cache for the workflow runner, in MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request.|
+
+h2(#clustertarget). arv:ClusterTarget
+
+Specify which Arvados cluster should execute a container or subworkflow, and the parent project for the container request.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|cluster_id|string|The five-character alphanumeric cluster id (uuid prefix) where a container or subworkflow will execute. May be an expression.|
+|project_uuid|string|The uuid of the project which will own container request and output of the container. May be an expression.|
+
+h2. arv:dockerCollectionPDH
+
+This is an optional extension field appearing on the standard @DockerRequirement@. It specifies the portable data hash of the Arvados collection containing the Docker image. If present, it takes precedence over @dockerPull@ or @dockerImageId@.
+
+<pre>
+requirements:
+ DockerRequirement:
+ dockerPull: "debian:8"
+ arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
+</pre>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+# "*Command line options*":#options
+# "*Specify workflow and output names*":#names
+# "*Submit a workflow without waiting for the result*":#nowait
+# "*Control a workflow locally*":#local
+# "*Automatically delete intermediate outputs*":#delete
+# "*Run workflow on a remote federated cluster*":#federation
+
+h3(#options). Command line options
+
The following command line options are available for @arvados-cwl-runner@:
table(table table-bordered table-condensed).
|_. Option |_. Description |
|==--basedir== BASEDIR| Base directory used to resolve relative references in the input, default to directory of input object file or current directory (if inputs piped/provided on command line).|
+|==--eval-timeout EVAL_TIMEOUT==|Time to wait for a Javascript expression to evaluate before giving an error, default 20s.|
+|==--print-dot==| Print workflow visualization in graphviz format and exit|
|==--version==| Print version and exit|
|==--validate==| Validate CWL document only.|
|==--verbose==| Default logging|
|==--quiet==| Only print warnings and errors.|
|==--debug==| Print even more logging|
+|==--metrics==| Print timing metrics|
|==--tool-help==| Print command line help for tool|
-|==--enable-reuse==|Enable job reuse (default)|
-|==--disable-reuse==|Disable job reuse (always run new jobs).|
+|==--enable-reuse==| Enable job or container reuse (default)|
+|==--disable-reuse==| Disable job or container reuse|
|==--project-uuid UUID==| Project that will own the workflow jobs, if not provided, will go to home project.|
|==--output-name OUTPUT_NAME==|Name to use for collection that stores the final output.|
|==--output-tags OUTPUT_TAGS==|Tags for the final output collection separated by commas, e.g., =='--output-tags tag0,tag1,tag2'==.|
|==--ignore-docker-for-reuse==|Ignore Docker image version when deciding whether to reuse past jobs.|
-|==--submit==| Submit workflow to run on Arvados.|
-|==--local==| Control workflow from local host (submits jobs to Arvados).|
-|==--create-template==| (Deprecated) synonym for ==--create-workflow.==|
-|==--create-workflow==| Create an Arvados workflow (if using the 'containers' API) or pipeline template (if using the 'jobs' API). See ==--api==.|
+|==--submit==| Submit workflow runner to Arvados to manage the workflow (default).|
+|==--local==| Run workflow on local host (still submits jobs to Arvados).|
+|==--create-template==| (Deprecated) synonym for --create-workflow.|
+|==--create-workflow==| Create an Arvados workflow (if using the 'containers' API) or pipeline template (if using the 'jobs' API). See --api.|
|==--update-workflow== UUID|Update an existing Arvados workflow or pipeline template with the given UUID.|
|==--wait==| After submitting workflow runner job, wait for completion.|
|==--no-wait==| Submit workflow runner job and exit.|
-|==--api== WORK_API| Select work submission API, one of 'jobs' or 'containers'. Default is 'jobs' if that API is available, otherwise 'containers'.|
+|==--log-timestamps==| Prefix logging lines with timestamp|
+|==--no-log-timestamps==| No timestamp on logging lines|
+|==--api== {jobs,containers}|Select work submission API. Default is 'jobs' if that API is available, otherwise 'containers'.|
|==--compute-checksum==| Compute checksum of contents while collecting outputs|
|==--submit-runner-ram== SUBMIT_RUNNER_RAM|RAM (in MiB) required for the workflow runner job (default 1024)|
-|==--submit-runner-image== SUBMIT_RUNNER_IMAGE|Docker image for workflow runner job, default arvados/jobs|
-|==--name== NAME| Name to use for workflow execution instance.|
-|==--on-error {stop,continue}==|Desired workflow behavior when a step fails. One of 'stop' or 'continue'. Default is 'continue'.|
-|==--enable-dev==| Enable loading and running development versions of CWL spec.|
+|==--submit-runner-image== SUBMIT_RUNNER_IMAGE|Docker image for workflow runner job|
+|==--always-submit-runner==|When invoked with --submit --wait, always submit a runner to manage the workflow, even when only running a single CommandLineTool|
+|==--submit-request-uuid== UUID|Update and commit to supplied container request instead of creating a new one (containers API only).|
+|==--submit-runner-cluster== CLUSTER_ID|Submit workflow runner to a remote cluster (containers API only)|
+|==--name NAME==|Name to use for workflow execution instance.|
+|==--on-error== {stop,continue}|Desired workflow behavior when a step fails. One of 'stop' (do not submit any more steps) or 'continue' (may submit other steps that are not downstream from the error). Default is 'continue'.|
+|==--enable-dev==|Enable loading and running development versions of CWL spec.|
+|==--storage-classes== STORAGE_CLASSES|Specify comma separated list of storage classes to be used when saving workflow output to Keep.|
|==--intermediate-output-ttl== N|If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).|
-|==--trash-intermediate==| Immediately trash intermediate outputs on workflow success.|
+|==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower, containers api only)|
+|==--thread-count== THREAD_COUNT|Number of threads to use for job submit and output collection.|
+|==--http-timeout== HTTP_TIMEOUT|API request timeout in seconds. Default is 300 seconds (5 minutes).|
+|==--trash-intermediate==|Immediately trash intermediate outputs on workflow success.|
|==--no-trash-intermediate==|Do not trash intermediate outputs (default).|
-h3. Specify workflow and output names
+h3(#names). Specify workflow and output names
Use the @--name@ and @--output-name@ options to specify the name of the workflow and name of the output collection.
</code></pre>
</notextile>
-h3. Submit a workflow with no waiting
+h3(#nowait). Submit a workflow without waiting for the result
To submit a workflow and exit immediately, use the @--no-wait@ option. This will submit the workflow to Arvados, print out the UUID of the job that was submitted to standard output, and exit.
</code></pre>
</notextile>
-h3. Control a workflow locally
+h3(#local). Control a workflow locally
To run a workflow with local control, use @--local@. This means that the host where you run @arvados-cwl-runner@ will be responsible for submitting jobs, however, the jobs themselves will still run on the Arvados cluster. With @--local@, if you interrupt @arvados-cwl-runner@ or log out, the workflow will be terminated.
</code></pre>
</notextile>
-h3. Automatically delete intermediate outputs
+h3(#delete). Automatically delete intermediate outputs
Use the @--intermediate-output-ttl@ and @--trash-intermediate@ options to specify how long intermediate outputs should be kept (in seconds) and whether to trash them immediately upon successful workflow completion.
Using @--trash-intermediate@ without @--intermediate-output-ttl@ means that intermediate files will be trashed on successful completion, but will remain on workflow failure.
Using @--intermediate-output-ttl@ without @--trash-intermediate@ means that intermediate files will be trashed only after the TTL expires (regardless of workflow success or failure).
+
+h3(#federation). Run workflow on a remote federated cluster
+
+By default, the workflow runner will run on the local (home) cluster. Using @--submit-runner-cluster@ you can specify that the runner should be submitted to a remote federated cluster. When doing this, @--project-uuid@ should specify a project on that cluster. Steps making up the workflow will be submitted to the remote federated cluster by default, but the behavior of @arv:ClusterTarget@ is unchanged. Note: when using this option, any resources that need to be uploaded in order to run the workflow (such as files or Docker images) will be uploaded to the local (home) cluster, and streamed to the federated cluster on demand.
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.2" width="210mm" height="148mm" viewBox="0 0 21000 14800" preserveAspectRatio="xMidYMid" fill-rule="evenodd" stroke-width="28.222" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" xmlns:ooo="http://xml.openoffice.org/svg/export" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:presentation="http://sun.com/xmlns/staroffice/presentation" xmlns:smil="http://www.w3.org/2001/SMIL20/" xmlns:anim="urn:oasis:names:tc:opendocument:xmlns:animation:1.0" xml:space="preserve">
+ <defs class="ClipPathGroup">
+ <clipPath id="presentation_clip_path" clipPathUnits="userSpaceOnUse">
+ <rect x="0" y="0" width="21000" height="14800"/>
+ </clipPath>
+ <clipPath id="presentation_clip_path_shrink" clipPathUnits="userSpaceOnUse">
+ <rect x="21" y="14" width="20958" height="14771"/>
+ </clipPath>
+ </defs>
+ <defs>
+ <font id="EmbeddedFont_1" horiz-adv-x="2048">
+ <font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="normal" font-style="normal" ascent="1852" descent="423"/>
+ <missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
+ <glyph unicode="y" horiz-adv-x="1059" d="M 604,1 C 579,-64 553,-123 527,-175 500,-227 471,-272 438,-309 405,-346 369,-374 329,-394 289,-413 243,-423 191,-423 168,-423 147,-423 128,-423 109,-423 88,-420 67,-414 L 67,-279 C 80,-282 94,-284 110,-284 126,-284 140,-284 151,-284 204,-284 253,-264 298,-225 343,-186 383,-124 417,-38 L 434,5 5,1082 197,1082 425,484 C 432,466 440,442 451,412 461,382 471,352 482,322 492,292 501,265 509,241 517,217 522,202 523,196 525,203 530,218 538,240 545,261 554,285 564,312 573,339 583,366 593,393 603,420 611,444 618,464 L 830,1082 1020,1082 604,1 Z"/>
+ <glyph unicode="w" horiz-adv-x="1535" d="M 1174,0 L 965,0 792,698 C 787,716 781,738 776,765 770,792 764,818 759,843 752,872 746,903 740,934 734,904 728,874 721,845 716,820 710,793 704,766 697,739 691,715 686,694 L 508,0 300,0 -3,1082 175,1082 358,347 C 363,332 367,313 372,291 377,268 381,246 386,225 391,200 396,175 401,149 406,174 412,199 418,223 423,244 429,265 434,286 439,307 444,325 448,339 L 644,1082 837,1082 1026,339 C 1031,322 1036,302 1041,280 1046,258 1051,237 1056,218 1061,195 1067,172 1072,149 1077,174 1083,199 1088,223 1093,244 1098,265 1103,288 1108,310 1112,330 1117,347 L 1308,1082 1484,1082 1174,0 Z"/>
+ <glyph unicode="u" horiz-adv-x="901" d="M 314,1082 L 314,396 C 314,343 318,299 326,264 333,229 346,200 363,179 380,157 403,142 432,133 460,124 495,119 537,119 580,119 618,127 653,142 687,157 716,178 741,207 765,235 784,270 797,312 810,353 817,401 817,455 L 817,1082 997,1082 997,228 C 997,205 997,181 998,156 998,131 998,107 999,85 1000,62 1000,43 1001,27 1002,11 1002,3 1003,3 L 833,3 C 832,6 832,15 831,30 830,44 830,61 829,79 828,98 827,117 826,136 825,156 825,172 825,185 L 822,185 C 805,154 786,125 765,100 744,75 720,53 693,36 666,18 634,4 599,-6 564,-15 523,-20 476,-20 416,-20 364,-13 321,2 278,17 242,39 214,70 186,101 166,140 153,188 140,236 133,294 133,361 L 133,1082 314,1082 Z"/>
+ <glyph unicode="t" horiz-adv-x="531" d="M 554,8 C 527,1 499,-5 471,-10 442,-14 409,-16 372,-16 228,-16 156,66 156,229 L 156,951 31,951 31,1082 163,1082 216,1324 336,1324 336,1082 536,1082 536,951 336,951 336,268 C 336,216 345,180 362,159 379,138 408,127 450,127 467,127 484,128 501,131 517,134 535,137 554,141 L 554,8 Z"/>
+ <glyph unicode="s" horiz-adv-x="927" d="M 950,299 C 950,248 940,203 921,164 901,124 872,91 835,64 798,37 752,16 698,2 643,-13 581,-20 511,-20 448,-20 392,-15 342,-6 291,4 247,20 209,41 171,62 139,91 114,126 88,161 69,203 57,254 L 216,285 C 231,227 263,185 311,158 359,131 426,117 511,117 550,117 585,120 618,125 650,130 678,140 701,153 724,166 743,183 756,205 769,226 775,253 775,285 775,318 767,345 752,366 737,387 715,404 688,418 661,432 628,444 589,455 550,465 507,476 460,489 417,500 374,513 331,527 288,541 250,560 216,583 181,606 153,634 132,668 111,702 100,745 100,796 100,895 135,970 206,1022 276,1073 378,1099 513,1099 632,1099 727,1078 798,1036 868,994 912,927 931,834 L 769,814 C 763,842 752,866 736,885 720,904 701,919 678,931 655,942 630,951 602,956 573,961 544,963 513,963 432,963 372,951 333,926 294,901 275,864 275,814 275,785 282,761 297,742 311,723 331,707 357,694 382,681 413,669 449,660 485,650 525,640 568,629 597,622 626,614 656,606 686,597 715,587 744,576 772,564 799,550 824,535 849,519 870,500 889,478 908,456 923,430 934,401 945,372 950,338 950,299 Z"/>
+ <glyph unicode="r" horiz-adv-x="556" d="M 142,0 L 142,830 C 142,853 142,876 142,900 141,923 141,946 140,968 139,990 139,1011 138,1030 137,1049 137,1067 136,1082 L 306,1082 C 307,1067 308,1049 309,1030 310,1010 311,990 312,969 313,948 313,929 314,910 314,891 314,874 314,861 L 318,861 C 331,902 344,938 359,969 373,999 390,1024 409,1044 428,1063 451,1078 478,1088 505,1097 537,1102 575,1102 590,1102 604,1101 617,1099 630,1096 641,1094 648,1092 L 648,927 C 636,930 622,933 606,935 590,936 572,937 552,937 511,937 476,928 447,909 418,890 394,865 376,832 357,799 344,759 335,714 326,668 322,618 322,564 L 322,0 142,0 Z"/>
+ <glyph unicode="o" horiz-adv-x="980" d="M 1053,542 C 1053,353 1011,212 928,119 845,26 724,-20 565,-20 490,-20 422,-9 363,14 304,37 254,71 213,118 172,165 140,223 119,294 97,364 86,447 86,542 86,915 248,1102 571,1102 655,1102 728,1090 789,1067 850,1044 900,1009 939,962 978,915 1006,857 1025,787 1044,717 1053,635 1053,542 Z M 864,542 C 864,626 858,695 845,750 832,805 813,848 788,881 763,914 732,937 696,950 660,963 619,969 574,969 528,969 487,962 450,949 413,935 381,912 355,879 329,846 309,802 296,747 282,692 275,624 275,542 275,458 282,389 297,334 312,279 332,235 358,202 383,169 414,146 449,133 484,120 522,113 563,113 609,113 651,120 688,133 725,146 757,168 783,201 809,234 829,278 843,333 857,388 864,458 864,542 Z"/>
+ <glyph unicode="n" horiz-adv-x="900" d="M 825,0 L 825,686 C 825,739 821,783 814,818 806,853 793,882 776,904 759,925 736,941 708,950 679,959 644,963 602,963 559,963 521,956 487,941 452,926 423,904 399,876 374,847 355,812 342,771 329,729 322,681 322,627 L 322,0 142,0 142,853 C 142,876 142,900 142,925 141,950 141,974 140,996 139,1019 139,1038 138,1054 137,1070 137,1078 136,1078 L 306,1078 C 307,1075 307,1066 308,1052 309,1037 310,1021 311,1002 312,984 312,965 313,945 314,926 314,910 314,897 L 317,897 C 334,928 353,957 374,982 395,1007 419,1029 446,1047 473,1064 505,1078 540,1088 575,1097 616,1102 663,1102 723,1102 775,1095 818,1080 861,1065 897,1043 925,1012 953,981 974,942 987,894 1000,845 1006,788 1006,721 L 1006,0 825,0 Z"/>
+ <glyph unicode="l" horiz-adv-x="187" d="M 138,0 L 138,1484 318,1484 318,0 138,0 Z"/>
+ <glyph unicode="k" horiz-adv-x="927" d="M 816,0 L 450,494 318,385 318,0 138,0 138,1484 318,1484 318,557 793,1082 1004,1082 565,617 1027,0 816,0 Z"/>
+ <glyph unicode="i" horiz-adv-x="187" d="M 137,1312 L 137,1484 317,1484 317,1312 137,1312 Z M 137,0 L 137,1082 317,1082 317,0 137,0 Z"/>
+ <glyph unicode="f" horiz-adv-x="557" d="M 361,951 L 361,0 181,0 181,951 29,951 29,1082 181,1082 181,1204 C 181,1243 185,1280 192,1314 199,1347 213,1377 233,1402 252,1427 279,1446 313,1461 347,1475 391,1482 445,1482 466,1482 489,1481 512,1479 535,1477 555,1474 572,1470 L 572,1333 C 561,1335 548,1337 533,1339 518,1340 504,1341 492,1341 465,1341 444,1337 427,1330 410,1323 396,1312 387,1299 377,1285 370,1268 367,1248 363,1228 361,1205 361,1179 L 361,1082 572,1082 572,951 361,951 Z"/>
+ <glyph unicode="e" horiz-adv-x="980" d="M 276,503 C 276,446 282,394 294,347 305,299 323,258 348,224 372,189 403,163 441,144 479,125 525,115 578,115 656,115 719,131 766,162 813,193 844,233 861,281 L 1019,236 C 1008,206 992,176 972,146 951,115 924,88 890,64 856,39 814,19 763,4 712,-12 650,-20 578,-20 418,-20 296,28 213,123 129,218 87,360 87,548 87,649 100,735 125,806 150,876 185,933 229,977 273,1021 324,1053 383,1073 442,1092 504,1102 571,1102 662,1102 738,1087 799,1058 860,1029 909,988 946,937 983,885 1009,824 1025,754 1040,684 1048,608 1048,527 L 1048,503 276,503 Z M 862,641 C 852,755 823,838 775,891 727,943 658,969 568,969 538,969 507,964 474,955 441,945 410,928 382,903 354,878 330,845 311,803 292,760 281,706 278,641 L 862,641 Z"/>
+ <glyph unicode="a" horiz-adv-x="1060" d="M 414,-20 C 305,-20 224,9 169,66 114,124 87,203 87,303 87,375 101,434 128,480 155,526 190,562 234,588 277,614 327,632 383,642 439,652 496,657 554,657 L 797,657 797,717 C 797,762 792,800 783,832 774,863 759,889 740,908 721,928 697,942 668,951 639,960 604,965 565,965 530,965 499,963 471,958 443,953 419,944 398,931 377,918 361,900 348,878 335,855 327,827 323,793 L 135,810 C 142,853 154,892 173,928 192,963 218,994 253,1020 287,1046 330,1066 382,1081 433,1095 496,1102 569,1102 705,1102 807,1071 876,1009 945,946 979,856 979,738 L 979,272 C 979,219 986,179 1000,152 1014,125 1041,111 1080,111 1090,111 1100,112 1110,113 1120,114 1130,116 1139,118 L 1139,6 C 1116,1 1094,-3 1072,-6 1049,-9 1025,-10 1000,-10 966,-10 937,-5 913,4 888,13 868,26 853,45 838,63 826,86 818,113 810,140 805,171 803,207 L 797,207 C 778,172 757,141 734,113 711,85 684,61 653,42 622,22 588,7 549,-4 510,-15 465,-20 414,-20 Z M 455,115 C 512,115 563,125 606,146 649,167 684,194 713,226 741,259 762,294 776,332 790,371 797,408 797,443 L 797,531 600,531 C 556,531 514,528 475,522 435,517 400,506 370,489 340,472 316,449 299,418 281,388 272,349 272,300 272,241 288,195 320,163 351,131 396,115 455,115 Z"/>
+ <glyph unicode="W" horiz-adv-x="1906" d="M 1511,0 L 1283,0 1039,895 C 1032,920 1024,950 1016,985 1007,1020 1000,1053 993,1084 985,1121 977,1158 969,1196 960,1157 952,1120 944,1083 937,1051 929,1018 921,984 913,950 905,920 898,895 L 652,0 424,0 9,1409 208,1409 461,514 C 472,472 483,430 494,389 504,348 513,311 520,278 529,239 537,203 544,168 554,214 564,259 575,304 580,323 584,342 589,363 594,384 599,404 604,424 609,444 614,463 619,482 624,500 628,517 632,532 L 877,1409 1060,1409 1305,532 C 1309,517 1314,500 1319,482 1324,463 1329,444 1334,425 1339,405 1343,385 1348,364 1353,343 1357,324 1362,305 1373,260 1383,215 1393,168 1394,168 1397,180 1402,203 1407,226 1414,254 1422,289 1430,324 1439,361 1449,402 1458,442 1468,479 1478,514 L 1727,1409 1926,1409 1511,0 Z"/>
+ <glyph unicode="J" horiz-adv-x="848" d="M 457,-20 C 343,-20 250,10 177,69 104,128 55,222 32,350 L 219,381 C 226,338 237,301 252,270 267,239 286,213 307,193 328,173 352,158 378,149 404,140 431,135 458,135 527,135 582,159 622,207 662,254 682,324 682,416 L 682,1253 411,1253 411,1409 872,1409 872,420 C 872,353 863,292 844,238 825,184 798,138 763,100 727,61 683,32 632,11 581,-10 522,-20 457,-20 Z"/>
+ <glyph unicode="C" horiz-adv-x="1297" d="M 792,1274 C 712,1274 641,1261 580,1234 518,1207 466,1169 425,1120 383,1071 351,1011 330,942 309,873 298,796 298,711 298,626 310,549 333,479 356,408 389,348 432,297 475,246 527,207 590,179 652,151 722,137 800,137 855,137 905,144 950,159 995,173 1035,193 1072,219 1108,245 1140,276 1169,312 1198,347 1223,387 1245,430 L 1401,352 C 1376,299 1344,250 1307,205 1270,160 1226,120 1176,87 1125,54 1068,28 1005,9 941,-10 870,-20 791,-20 677,-20 577,-2 492,35 406,71 334,122 277,187 219,252 176,329 147,418 118,507 104,605 104,711 104,821 119,920 150,1009 180,1098 224,1173 283,1236 341,1298 413,1346 498,1380 583,1413 681,1430 790,1430 940,1430 1065,1401 1166,1342 1267,1283 1341,1196 1388,1081 L 1207,1021 C 1194,1054 1176,1086 1153,1117 1130,1147 1102,1174 1068,1197 1034,1220 994,1239 949,1253 903,1267 851,1274 792,1274 Z"/>
+ <glyph unicode="A" horiz-adv-x="1350" d="M 1167,0 L 1006,412 364,412 202,0 4,0 579,1409 796,1409 1362,0 1167,0 Z M 768,1026 C 757,1053 747,1080 738,1107 728,1134 719,1159 712,1182 705,1204 699,1223 694,1238 689,1253 686,1262 685,1265 684,1262 681,1252 676,1237 671,1222 665,1203 658,1180 650,1157 641,1132 632,1105 622,1078 612,1051 602,1024 L 422,561 949,561 768,1026 Z"/>
+ <glyph unicode="3" horiz-adv-x="980" d="M 1049,389 C 1049,324 1039,267 1018,216 997,165 966,123 926,88 885,53 835,26 776,8 716,-11 648,-20 571,-20 484,-20 410,-9 351,13 291,34 242,63 203,99 164,134 135,175 116,221 97,266 84,313 78,362 L 264,379 C 269,342 279,308 294,277 308,246 327,220 352,198 377,176 407,159 443,147 479,135 522,129 571,129 662,129 733,151 785,196 836,241 862,307 862,395 862,447 851,489 828,521 805,552 776,577 742,595 707,612 670,624 630,630 589,636 552,639 518,639 L 416,639 416,795 514,795 C 548,795 583,799 620,806 657,813 690,825 721,844 751,862 776,887 796,918 815,949 825,989 825,1038 825,1113 803,1173 759,1217 714,1260 648,1282 561,1282 482,1282 418,1262 369,1221 320,1180 291,1123 283,1049 L 102,1063 C 109,1125 126,1179 153,1225 180,1271 214,1309 255,1340 296,1370 342,1393 395,1408 448,1423 504,1430 563,1430 642,1430 709,1420 766,1401 823,1381 869,1354 905,1321 941,1287 968,1247 985,1202 1002,1157 1010,1108 1010,1057 1010,1016 1004,977 993,941 982,905 964,873 940,844 916,815 886,791 849,770 812,749 767,734 715,723 L 715,719 C 772,713 821,700 863,681 905,661 940,636 967,607 994,578 1015,544 1029,507 1042,470 1049,430 1049,389 Z"/>
+ <glyph unicode="2" horiz-adv-x="927" d="M 103,0 L 103,127 C 137,205 179,274 228,334 277,393 328,447 382,496 436,544 490,589 543,630 596,671 643,713 686,754 729,795 763,839 790,884 816,929 829,981 829,1038 829,1078 823,1113 811,1144 799,1174 782,1199 759,1220 736,1241 709,1256 678,1267 646,1277 611,1282 572,1282 536,1282 502,1277 471,1267 439,1257 411,1242 386,1222 361,1202 341,1177 326,1148 310,1118 300,1083 295,1044 L 111,1061 C 117,1112 131,1159 153,1204 175,1249 205,1288 244,1322 283,1355 329,1382 384,1401 438,1420 501,1430 572,1430 642,1430 704,1422 759,1405 814,1388 860,1364 898,1331 935,1298 964,1258 984,1210 1004,1162 1014,1107 1014,1044 1014,997 1006,952 989,909 972,866 949,826 921,787 892,748 859,711 822,675 785,639 746,604 705,570 664,535 623,501 582,468 541,434 502,400 466,366 429,332 397,298 368,263 339,228 317,191 301,153 L 1036,153 1036,0 103,0 Z"/>
+ <glyph unicode="1" horiz-adv-x="874" d="M 156,0 L 156,153 515,153 515,1237 197,1010 197,1180 530,1409 696,1409 696,153 1039,153 1039,0 156,0 Z"/>
+ <glyph unicode=" " horiz-adv-x="556"/>
+ </font>
+ </defs>
+ <defs>
+ <font id="EmbeddedFont_2" horiz-adv-x="2048">
+ <font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="normal" font-style="italic" ascent="1852" descent="423"/>
+ <missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
+ <glyph unicode="w" horiz-adv-x="1510" d="M 1068,0 L 859,0 822,698 C 821,711 821,731 820,757 819,783 819,809 818,836 817,867 817,900 816,934 804,904 792,874 780,845 769,820 758,793 747,766 735,739 724,715 715,694 L 402,0 194,0 102,1082 280,1082 320,347 C 321,339 321,326 322,308 323,289 323,270 324,250 325,229 325,210 326,191 327,172 327,158 327,149 337,173 347,197 357,220 366,240 375,261 384,283 393,305 401,324 408,339 L 749,1082 942,1082 986,339 C 988,303 990,268 991,235 992,202 992,173 992,149 1002,173 1012,197 1023,220 1032,240 1042,261 1052,284 1061,307 1070,328 1079,347 L 1413,1082 1589,1082 1068,0 Z"/>
+ <glyph unicode="u" horiz-adv-x="1060" d="M 415,1082 L 289,437 C 284,411 279,385 276,358 273,331 271,307 271,287 271,234 285,193 313,164 341,135 387,120 450,120 493,120 533,129 571,146 608,163 642,187 673,218 704,249 730,286 752,330 773,373 789,422 800,476 L 918,1082 1098,1082 932,228 C 927,205 923,181 919,156 914,131 910,107 907,85 903,62 900,43 898,27 895,11 894,3 893,3 L 723,3 C 723,6 724,15 726,30 728,44 731,61 734,79 737,98 740,117 743,136 746,156 748,172 751,185 L 748,185 C 725,154 702,125 678,100 654,75 628,53 599,36 570,18 538,4 503,-5 468,-14 428,-19 383,-19 284,-19 210,5 161,54 111,103 86,173 86,265 86,289 88,316 93,346 97,376 102,404 107,429 L 234,1082 415,1082 Z"/>
+ <glyph unicode="t" horiz-adv-x="530" d="M 448,4 C 423,-2 396,-7 367,-13 338,-17 307,-20 275,-20 218,-20 174,-3 142,31 109,65 93,110 93,166 93,187 95,210 98,235 101,259 105,279 108,296 L 234,951 109,951 135,1082 262,1082 367,1324 487,1324 440,1082 640,1082 614,951 414,951 289,306 C 286,293 284,276 281,257 278,238 277,222 277,211 277,183 284,161 298,146 312,131 335,123 367,123 384,123 401,124 416,127 431,129 448,132 467,137 L 448,4 Z"/>
+ <glyph unicode="s" horiz-adv-x="980" d="M 907,317 C 907,260 896,211 873,169 850,126 818,91 777,63 735,35 684,14 625,1 566,-13 499,-20 425,-20 363,-20 309,-15 262,-4 215,7 175,22 142,43 108,63 80,88 58,119 35,149 18,184 5,223 L 152,279 C 162,252 175,229 191,208 206,187 226,169 249,155 272,140 299,129 331,122 362,115 399,111 441,111 484,111 523,115 559,122 594,129 625,140 651,155 676,170 696,190 711,214 725,238 732,267 732,301 732,328 726,351 713,370 700,389 683,405 660,420 637,434 609,447 576,460 543,472 506,484 465,497 422,511 381,526 342,543 303,560 268,580 239,603 209,626 185,654 168,686 150,717 141,754 141,797 141,852 153,898 177,937 200,975 232,1006 273,1030 313,1054 360,1072 414,1083 467,1094 524,1099 584,1099 639,1099 689,1094 734,1085 779,1076 819,1061 853,1041 887,1020 915,994 937,962 959,929 974,890 982,844 L 819,819 C 804,872 777,910 736,933 695,956 641,968 572,968 537,968 504,965 473,960 442,955 414,946 391,934 368,922 349,906 336,887 322,868 315,844 315,817 315,790 321,767 334,749 347,730 365,714 388,700 411,686 438,674 471,663 503,652 539,640 579,627 617,615 656,601 695,585 734,569 769,549 800,526 831,502 857,473 877,440 897,406 907,365 907,317 Z"/>
+ <glyph unicode="r" horiz-adv-x="742" d="M 718,938 C 707,941 693,944 678,947 662,950 645,951 628,951 585,951 547,939 513,914 479,889 449,858 424,820 398,782 377,740 360,695 343,649 331,605 324,564 L 214,0 34,0 196,830 C 201,853 205,877 209,900 213,923 217,946 221,968 224,990 228,1011 231,1031 234,1050 237,1067 239,1082 L 409,1082 C 407,1067 405,1050 402,1030 399,1010 395,990 392,969 389,948 386,929 383,910 380,891 377,874 374,861 L 378,861 C 399,902 419,938 440,969 460,999 481,1024 503,1044 525,1063 549,1078 574,1088 599,1097 626,1102 656,1102 663,1102 671,1102 680,1101 689,1100 698,1098 707,1097 716,1096 724,1094 732,1093 740,1091 746,1089 751,1088 L 718,938 Z"/>
+ <glyph unicode="p" horiz-adv-x="1138" d="M 554,-20 C 472,-20 405,-3 354,32 302,67 265,115 244,178 L 239,178 C 239,177 238,170 237,159 236,147 234,132 231,115 228,98 225,79 222,58 218,37 214,17 210,-2 L 128,-425 -51,-425 198,864 C 203,891 208,916 212,940 216,964 220,986 223,1005 226,1025 228,1042 230,1056 231,1070 232,1077 233,1077 L 400,1077 C 400,1072 400,1063 399,1052 398,1040 397,1027 396,1013 394,998 392,983 390,967 388,950 386,935 383,921 L 387,921 C 411,952 436,979 461,1002 486,1025 512,1044 541,1059 569,1074 599,1085 632,1092 665,1099 701,1102 741,1102 794,1102 842,1094 883,1077 924,1060 959,1037 987,1006 1015,975 1036,938 1051,895 1066,851 1073,802 1073,748 1073,715 1072,678 1069,639 1066,599 1060,558 1052,516 1034,421 1010,340 981,273 952,205 916,149 875,106 834,63 786,31 733,11 680,-10 620,-20 554,-20 Z M 689,963 C 646,963 606,957 568,944 529,931 494,910 461,879 428,848 400,806 375,753 350,700 329,634 314,554 301,489 295,430 295,377 295,334 301,297 312,264 323,231 340,203 361,181 382,158 407,141 437,130 466,119 499,113 535,113 576,113 614,119 647,132 680,144 711,165 738,196 765,226 788,267 809,318 830,369 847,433 862,510 877,591 885,659 885,716 885,798 869,860 838,901 807,942 757,963 689,963 Z"/>
+ <glyph unicode="o" horiz-adv-x="1007" d="M 1074,683 C 1074,648 1072,614 1068,579 1064,544 1057,506 1048,467 1028,379 1000,304 965,242 929,180 887,130 839,91 791,52 738,24 679,7 620,-11 558,-20 491,-20 427,-20 369,-10 317,10 265,29 221,58 184,96 147,133 118,179 98,234 77,288 67,350 67,419 68,450 70,483 73,516 76,549 81,584 89,620 108,704 135,776 169,837 203,897 243,947 290,986 337,1025 390,1054 449,1073 508,1092 572,1101 642,1101 713,1101 775,1092 829,1073 882,1054 927,1027 964,991 1000,955 1027,911 1046,860 1065,808 1074,749 1074,683 Z M 888,683 C 888,734 882,778 871,814 860,850 843,880 822,903 800,926 774,942 743,953 712,964 678,969 640,969 605,969 569,965 534,957 498,948 464,931 432,906 399,881 370,845 343,798 316,751 294,689 276,612 267,575 261,541 258,508 254,475 252,444 252,416 252,361 258,315 271,276 284,237 301,206 324,182 346,158 372,141 403,130 433,119 466,113 502,113 538,113 574,117 609,125 644,133 677,150 708,176 739,201 768,238 795,285 821,332 843,395 861,473 870,513 877,550 881,583 884,616 887,650 888,683 Z"/>
+ <glyph unicode="n" horiz-adv-x="1033" d="M 717,0 L 843,645 C 848,671 853,698 856,725 859,752 861,775 861,795 861,848 847,889 819,918 791,947 745,962 682,962 639,962 599,954 562,937 524,920 490,896 459,865 428,834 402,796 381,753 359,709 343,660 332,606 L 214,0 34,0 200,853 C 205,876 209,900 214,925 218,950 222,974 226,996 229,1019 232,1038 235,1054 237,1070 238,1078 239,1078 L 409,1078 C 409,1075 408,1066 406,1052 404,1037 402,1021 399,1002 396,984 393,965 390,945 387,926 384,910 381,897 L 384,897 C 407,928 430,957 454,982 478,1007 505,1029 534,1047 563,1064 595,1078 630,1087 665,1096 704,1101 749,1101 848,1101 922,1077 972,1028 1021,979 1046,909 1046,817 1046,793 1044,766 1040,736 1035,706 1030,678 1025,653 L 898,0 717,0 Z"/>
+ <glyph unicode="m" horiz-adv-x="1589" d="M 660,0 L 784,634 C 787,647 790,662 793,678 796,694 798,710 801,726 803,742 805,757 807,772 808,786 809,798 809,808 809,858 796,896 771,923 746,949 704,962 647,962 609,962 573,954 539,937 504,920 473,896 446,865 419,834 395,796 375,752 355,707 340,658 331,604 L 213,0 34,0 200,853 C 205,876 209,900 214,925 218,950 222,974 226,996 229,1019 232,1038 235,1054 237,1070 238,1078 239,1078 L 409,1078 C 409,1075 408,1066 406,1052 404,1037 402,1021 399,1002 396,984 393,965 390,945 387,926 384,910 381,897 L 384,897 C 404,928 425,957 446,982 467,1007 491,1029 516,1047 541,1064 570,1078 601,1087 632,1096 667,1101 706,1101 787,1101 851,1081 898,1042 945,1002 974,944 983,869 1004,902 1026,933 1049,961 1072,989 1097,1014 1125,1035 1152,1056 1183,1072 1217,1084 1250,1095 1288,1101 1331,1101 1421,1101 1490,1077 1539,1028 1587,979 1611,909 1611,817 1611,793 1609,766 1605,736 1600,706 1595,678 1590,653 L 1463,0 1285,0 1409,634 C 1412,647 1415,662 1418,678 1421,694 1423,710 1426,726 1428,742 1430,757 1432,772 1433,786 1434,798 1434,808 1434,858 1421,896 1396,923 1371,949 1329,962 1272,962 1234,962 1198,954 1164,937 1129,920 1098,897 1071,866 1044,835 1020,798 1000,754 980,710 965,661 956,607 L 838,0 660,0 Z"/>
+ <glyph unicode="l" horiz-adv-x="504" d="M 33,0 L 321,1484 501,1484 212,0 33,0 Z"/>
+ <glyph unicode="k" horiz-adv-x="1113" d="M 721,0 L 453,502 285,378 213,0 34,0 322,1484 502,1484 323,567 527,757 888,1082 1110,1082 580,617 916,0 721,0 Z"/>
+ <glyph unicode="i" horiz-adv-x="478" d="M 287,1312 L 321,1484 501,1484 467,1312 287,1312 Z M 33,0 L 243,1082 423,1082 212,0 33,0 Z"/>
+ <glyph unicode="h" horiz-adv-x="1033" d="M 383,897 C 406,928 429,957 453,982 477,1007 504,1029 533,1047 562,1064 594,1078 629,1087 664,1096 703,1101 748,1101 847,1101 921,1077 971,1028 1020,979 1045,909 1045,817 1045,793 1043,766 1039,736 1034,706 1029,678 1024,653 L 897,0 716,0 842,645 C 847,671 852,698 855,725 858,752 860,775 860,795 860,848 846,889 818,918 790,947 744,962 681,962 638,962 598,954 561,937 523,920 489,896 458,865 427,834 401,796 380,753 358,709 342,660 331,606 L 213,0 34,0 322,1484 502,1484 427,1098 C 423,1076 419,1054 414,1032 409,1010 404,990 399,972 394,953 390,937 387,924 384,911 381,902 380,897 L 383,897 Z"/>
+ <glyph unicode="f" horiz-adv-x="663" d="M 434,951 L 249,0 69,0 254,951 102,951 128,1082 280,1082 303,1204 C 311,1243 321,1280 334,1314 347,1348 365,1378 389,1403 412,1428 443,1448 480,1463 517,1477 565,1484 622,1484 643,1484 665,1483 688,1481 710,1479 729,1476 746,1472 L 720,1335 C 714,1336 707,1337 700,1338 692,1339 684,1340 675,1341 666,1342 658,1342 650,1342 642,1342 635,1342 629,1342 604,1342 583,1338 566,1331 549,1324 535,1313 524,1299 513,1285 504,1268 497,1248 490,1228 484,1205 479,1179 L 460,1082 671,1082 645,951 434,951 Z"/>
+ <glyph unicode="e" horiz-adv-x="980" d="M 256,503 C 253,484 251,466 250,447 249,428 248,409 247,390 247,301 269,233 314,186 358,139 425,115 514,115 551,115 585,120 616,130 647,139 675,152 700,169 725,185 747,204 766,226 785,247 800,270 813,294 L 951,231 C 934,201 914,171 890,142 866,112 836,85 801,61 765,37 722,18 672,3 622,-12 562,-20 493,-20 426,-20 367,-10 314,9 261,28 217,55 181,92 144,128 117,172 98,225 79,278 69,338 69,405 69,510 83,606 112,692 140,778 179,851 230,912 280,973 339,1020 408,1053 476,1086 550,1102 630,1102 703,1102 767,1092 821,1073 875,1054 920,1027 956,992 992,957 1019,916 1037,868 1054,819 1063,766 1063,708 1063,694 1063,679 1062,662 1061,645 1059,628 1057,610 1055,592 1053,574 1050,556 1047,537 1043,520 1039,503 L 256,503 Z M 880,641 C 881,654 882,667 883,679 884,690 884,702 884,713 884,757 878,795 866,828 854,860 837,887 815,908 793,929 767,944 736,954 705,964 671,969 634,969 602,969 568,964 533,955 498,945 464,928 432,903 399,878 370,845 343,803 316,760 295,706 280,641 L 880,641 Z"/>
+ <glyph unicode="d" horiz-adv-x="1166" d="M 401,-21 C 348,-21 300,-13 259,4 218,21 183,44 155,75 127,106 106,143 91,187 76,230 69,279 69,333 69,363 71,399 74,440 77,481 82,523 90,565 108,660 132,741 161,809 190,876 226,932 267,975 308,1018 356,1050 409,1071 462,1091 522,1101 588,1101 670,1101 737,1084 789,1049 840,1014 877,966 898,903 L 903,903 C 904,910 906,921 909,936 912,951 915,968 918,985 921,1002 923,1018 926,1033 929,1048 930,1059 931,1065 L 1013,1484 1193,1484 948,219 C 943,193 938,168 934,143 929,119 925,97 922,77 919,57 916,40 914,26 912,11 911,4 910,4 L 738,4 C 738,17 740,38 744,66 747,95 752,126 759,160 L 754,160 C 730,129 706,102 681,79 656,56 629,38 601,23 573,8 543,-3 510,-11 477,-17 441,-21 401,-21 Z M 453,118 C 496,118 536,124 575,137 613,150 648,172 681,203 714,234 743,275 768,328 793,381 813,447 828,527 841,592 847,651 847,704 847,747 841,785 830,818 819,851 803,878 782,901 761,923 735,940 706,951 676,962 643,968 607,968 566,968 529,962 496,950 462,937 432,916 405,886 378,855 354,815 334,764 313,713 295,648 280,571 265,490 257,422 257,365 257,283 273,221 304,180 335,139 385,118 453,118 Z"/>
+ <glyph unicode="c" horiz-adv-x="927" d="M 469,122 C 506,122 540,128 570,139 600,150 627,165 650,185 673,205 694,229 712,258 730,286 745,317 758,352 L 914,303 C 895,253 873,208 846,169 819,129 787,95 750,67 713,39 670,18 623,3 576,-12 523,-20 465,-20 396,-20 337,-10 287,11 236,32 195,61 163,98 130,135 106,178 91,229 75,280 67,335 67,395 67,422 68,451 71,482 73,513 77,544 83,574 98,648 117,712 140,767 163,822 188,869 217,908 245,947 276,979 309,1004 342,1029 376,1049 411,1064 446,1078 481,1088 518,1094 554,1099 590,1102 625,1102 684,1102 737,1094 782,1079 827,1064 865,1042 896,1014 927,986 952,953 970,914 987,875 998,831 1001,784 L 824,759 C 822,789 816,816 807,841 798,866 785,887 768,905 751,922 730,936 705,946 680,956 652,961 619,961 573,961 532,954 495,941 458,928 426,906 397,876 368,846 343,807 322,759 301,710 284,651 270,581 264,549 259,515 256,480 253,445 251,414 251,389 251,304 268,239 303,192 337,145 392,122 469,122 Z"/>
+ <glyph unicode="b" horiz-adv-x="1060" d="M 744,1102 C 797,1102 845,1094 886,1077 927,1060 962,1037 990,1006 1018,975 1039,938 1054,895 1069,851 1076,802 1076,748 1076,715 1075,678 1072,639 1069,599 1063,558 1055,516 1037,421 1013,340 984,273 955,205 919,149 878,106 837,63 789,31 736,11 683,-10 623,-20 557,-20 475,-20 408,-3 357,32 306,67 269,115 248,178 L 245,178 C 242,160 238,142 233,122 228,102 224,83 220,66 215,48 212,33 209,21 206,8 203,2 202,2 L 29,2 C 31,8 34,18 37,32 40,47 44,64 49,84 53,104 58,126 63,150 68,174 73,199 78,225 L 323,1484 503,1484 420,1061 C 417,1042 413,1023 409,1006 404,989 400,974 397,961 393,946 389,933 386,921 L 390,921 C 414,952 439,979 464,1002 489,1025 515,1044 544,1059 572,1074 602,1085 635,1092 668,1099 704,1102 744,1102 Z M 692,963 C 649,963 609,957 571,944 532,931 497,910 464,879 431,848 403,806 378,753 353,700 332,634 317,554 304,489 298,430 298,377 298,334 304,297 315,264 326,231 343,203 364,181 385,158 410,141 440,130 469,119 502,113 538,113 579,113 617,119 650,132 683,144 714,165 741,196 768,226 791,267 812,318 833,369 850,433 865,510 880,591 888,659 888,716 888,798 872,860 841,901 810,942 760,963 692,963 Z"/>
+ <glyph unicode="a" horiz-adv-x="1033" d="M 1055,6 C 1036,1 1015,-2 993,-6 970,-8 948,-10 927,-10 865,-10 820,3 792,29 763,54 749,92 749,143 749,153 750,164 751,176 752,187 753,198 754,207 L 748,207 C 725,172 701,140 676,112 651,84 623,60 593,41 562,21 528,6 491,-5 454,-15 410,-20 361,-20 309,-20 264,-12 225,5 186,22 153,44 126,72 99,100 79,131 66,168 53,204 46,241 46,279 46,333 54,380 70,419 85,459 107,493 134,521 161,549 192,572 229,589 265,607 304,621 345,631 386,641 428,648 472,652 516,656 559,658 601,658 L 833,658 840,694 C 843,711 846,727 849,743 851,758 852,772 852,786 852,847 834,892 799,921 764,950 715,965 652,965 619,965 589,963 561,958 532,953 507,944 485,931 462,918 443,900 426,878 409,855 395,827 384,793 L 206,822 C 219,863 236,901 258,936 280,970 309,999 345,1024 381,1049 425,1068 477,1082 528,1095 590,1102 662,1102 721,1102 774,1095 820,1080 866,1065 905,1045 936,1019 967,993 991,962 1008,926 1024,890 1032,850 1032,807 1032,786 1030,762 1027,733 1023,704 1018,676 1013,650 L 939,272 C 936,257 933,242 931,227 929,212 928,197 928,184 928,159 935,141 948,129 961,117 981,111 1009,111 1019,111 1029,112 1040,113 1050,114 1060,116 1069,118 L 1055,6 Z M 809,530 L 610,530 C 583,530 556,530 527,530 498,530 470,527 443,520 415,514 389,505 364,495 339,484 317,469 298,451 279,432 265,410 254,383 243,357 237,325 237,288 237,266 241,245 248,225 255,204 265,186 280,170 295,154 313,141 335,132 356,122 382,117 411,117 469,117 520,127 563,147 606,166 643,191 674,220 705,248 729,280 747,314 764,347 776,379 782,407 L 809,530 Z"/>
+ <glyph unicode="U" horiz-adv-x="1377" d="M 654,-20 C 585,-20 520,-11 459,7 398,25 344,53 299,90 254,127 218,174 192,231 166,288 153,354 153,431 153,445 154,461 155,480 156,498 158,516 161,535 163,554 165,572 168,590 171,607 173,622 176,635 L 326,1409 517,1409 355,566 C 350,542 346,517 343,492 340,466 338,443 338,423 338,374 346,331 363,295 380,259 403,229 432,206 461,182 496,164 537,153 578,141 622,135 670,135 728,135 782,142 832,157 881,172 926,195 966,227 1005,259 1039,301 1068,353 1096,404 1117,467 1131,541 L 1299,1409 1489,1409 1319,530 C 1300,436 1272,355 1234,286 1195,217 1148,159 1091,114 1034,69 969,35 896,13 823,-9 742,-20 654,-20 Z"/>
+ <glyph unicode="I" horiz-adv-x="478" d="M 81,0 L 355,1409 546,1409 272,0 81,0 Z"/>
+ <glyph unicode="3" horiz-adv-x="1059" d="M 566,795 C 590,795 616,796 644,799 671,802 699,807 726,814 753,821 778,831 803,844 828,857 849,875 868,896 887,917 902,942 913,973 924,1003 930,1039 930,1081 930,1110 925,1137 916,1162 906,1187 892,1208 873,1226 854,1243 830,1257 803,1267 776,1277 744,1282 708,1282 629,1282 561,1262 504,1221 447,1180 407,1123 384,1049 L 206,1063 C 245,1187 309,1279 398,1340 487,1400 593,1430 718,1430 780,1430 836,1422 886,1407 935,1391 978,1368 1013,1339 1048,1309 1074,1273 1093,1231 1112,1188 1121,1140 1121,1086 1121,1034 1113,987 1096,945 1079,902 1054,865 1022,834 990,803 951,777 906,758 861,738 810,724 753,717 L 752,713 C 839,696 907,661 956,609 1005,556 1029,489 1029,407 1029,349 1019,294 998,242 977,190 945,145 904,106 862,67 810,37 747,14 684,-9 610,-20 526,-20 450,-20 384,-9 328,13 272,34 225,62 187,97 148,132 118,170 96,213 73,255 57,297 48,338 L 212,386 C 220,355 233,325 250,295 267,264 289,237 316,212 343,187 375,167 412,152 449,137 492,129 541,129 590,129 634,136 671,150 708,163 740,182 765,207 790,231 809,260 822,294 835,327 841,364 841,404 841,444 834,479 820,508 806,537 787,562 762,581 737,600 707,615 673,625 639,634 602,639 562,639 L 438,639 468,795 566,795 Z"/>
+ <glyph unicode="2" horiz-adv-x="1112" d="M -12,0 L 12,127 C 49,189 90,244 135,293 180,342 228,386 277,426 326,465 377,501 428,534 479,567 528,598 576,629 623,659 668,689 710,719 751,749 788,781 819,815 850,849 875,886 893,927 911,967 920,1012 920,1063 920,1097 914,1128 903,1155 892,1182 876,1205 856,1224 835,1243 811,1257 783,1267 754,1277 723,1282 689,1282 616,1282 553,1262 499,1223 445,1183 406,1123 381,1044 L 211,1081 C 227,1132 249,1178 276,1221 303,1264 337,1301 377,1332 417,1363 464,1387 518,1404 571,1421 632,1430 700,1430 759,1430 814,1422 864,1405 914,1388 957,1365 994,1334 1030,1303 1058,1266 1079,1223 1099,1180 1109,1131 1109,1078 1109,1021 1099,969 1080,921 1060,873 1033,829 1000,788 967,747 928,708 884,672 840,636 794,601 745,568 696,535 647,502 596,470 545,438 497,405 450,372 403,339 360,304 321,269 282,233 249,194 222,153 L 949,153 920,0 -12,0 Z"/>
+ <glyph unicode="1" horiz-adv-x="953" d="M 53,0 L 83,153 442,153 650,1223 289,1000 324,1180 701,1409 867,1409 623,153 966,153 936,0 53,0 Z"/>
+ <glyph unicode="," horiz-adv-x="292" d="M 299,51 C 292,16 285,-16 276,-46 267,-74 256,-101 245,-127 234,-151 221,-175 207,-197 193,-219 177,-241 160,-262 L 37,-262 C 75,-219 107,-175 132,-131 157,-87 173,-43 182,0 L 94,0 136,219 331,219 299,51 Z"/>
+ <glyph unicode=" " horiz-adv-x="556"/>
+ </font>
+ </defs>
+ <defs class="TextShapeIndex">
+ <g ooo:slide="id1" ooo:id-list="id3 id4 id5 id6 id7 id8 id9 id10 id11 id12 id13 id14 id15 id16 id17 id18"/>
+ </defs>
+ <defs class="EmbeddedBulletChars">
+ <g id="bullet-char-template(57356)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 580,1141 L 1163,571 580,0 -4,571 580,1141 Z"/>
+ </g>
+ <g id="bullet-char-template(57354)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 8,1128 L 1137,1128 1137,0 8,0 8,1128 Z"/>
+ </g>
+ <g id="bullet-char-template(10146)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 174,0 L 602,739 174,1481 1456,739 174,0 Z M 1358,739 L 309,1346 659,739 1358,739 Z"/>
+ </g>
+ <g id="bullet-char-template(10132)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 2015,739 L 1276,0 717,0 1260,543 174,543 174,936 1260,936 717,1481 1274,1481 2015,739 Z"/>
+ </g>
+ <g id="bullet-char-template(10007)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 0,-2 C -7,14 -16,27 -25,37 L 356,567 C 262,823 215,952 215,954 215,979 228,992 255,992 264,992 276,990 289,987 310,991 331,999 354,1012 L 381,999 492,748 772,1049 836,1024 860,1049 C 881,1039 901,1025 922,1006 886,937 835,863 770,784 769,783 710,716 594,584 L 774,223 C 774,196 753,168 711,139 L 727,119 C 717,90 699,76 672,76 641,76 570,178 457,381 L 164,-76 C 142,-110 111,-127 72,-127 30,-127 9,-110 8,-76 1,-67 -2,-52 -2,-32 -2,-23 -1,-13 0,-2 Z"/>
+ </g>
+ <g id="bullet-char-template(10004)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 285,-33 C 182,-33 111,30 74,156 52,228 41,333 41,471 41,549 55,616 82,672 116,743 169,778 240,778 293,778 328,747 346,684 L 369,508 C 377,444 397,411 428,410 L 1163,1116 C 1174,1127 1196,1133 1229,1133 1271,1133 1292,1118 1292,1087 L 1292,965 C 1292,929 1282,901 1262,881 L 442,47 C 390,-6 338,-33 285,-33 Z"/>
+ </g>
+ <g id="bullet-char-template(9679)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 813,0 C 632,0 489,54 383,161 276,268 223,411 223,592 223,773 276,916 383,1023 489,1130 632,1184 813,1184 992,1184 1136,1130 1245,1023 1353,916 1407,772 1407,592 1407,412 1353,268 1245,161 1136,54 992,0 813,0 Z"/>
+ </g>
+ <g id="bullet-char-template(8226)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 346,457 C 273,457 209,483 155,535 101,586 74,649 74,723 74,796 101,859 155,911 209,963 273,989 346,989 419,989 480,963 531,910 582,859 608,796 608,723 608,648 583,586 532,535 482,483 420,457 346,457 Z"/>
+ </g>
+ <g id="bullet-char-template(8211)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M -4,459 L 1135,459 1135,606 -4,606 -4,459 Z"/>
+ </g>
+ <g id="bullet-char-template(61548)" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 173,740 C 173,903 231,1043 346,1159 462,1274 601,1332 765,1332 928,1332 1067,1274 1183,1159 1299,1043 1357,903 1357,740 1357,577 1299,437 1183,322 1067,206 928,148 765,148 601,148 462,206 346,322 231,437 173,577 173,740 Z"/>
+ </g>
+ </defs>
+ <defs class="TextEmbeddedBitmaps"/>
+ <g>
+ <g id="id2" class="Master_Slide">
+ <g id="bg-id2" class="Background"/>
+ <g id="bo-id2" class="BackgroundObjects"/>
+ </g>
+ </g>
+ <g class="SlideGroup">
+ <g>
+ <g id="container-id1">
+ <g id="id1" class="Slide" clip-path="url(#presentation_clip_path)">
+ <g class="Page">
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id3">
+ <rect class="BoundingBox" stroke="none" fill="none" x="1761" y="2523" width="5845" height="9147"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4683,11668 L 1762,11668 1762,2524 7604,2524 7604,11668 4683,11668 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="2012" y="3225"><tspan fill="rgb(0,0,0)" stroke="none">Cluster 1</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id4">
+ <rect class="BoundingBox" stroke="none" fill="none" x="2015" y="3920" width="5083" height="1019"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 4556,4937 L 2016,4937 2016,3921 7096,3921 7096,4937 4556,4937 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4556,4937 L 2016,4937 2016,3921 7096,3921 7096,4937 4556,4937 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="2261" y="4650"><tspan fill="rgb(0,0,0)" stroke="none">Workflow runner</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id5">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7857" y="4555" width="5845" height="3559"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 10779,8112 L 7858,8112 7858,4556 13700,4556 13700,8112 10779,8112 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="8108" y="5257"><tspan fill="rgb(0,0,0)" stroke="none">Cluster 2</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id6">
+ <rect class="BoundingBox" stroke="none" fill="none" x="13953" y="4555" width="5845" height="3559"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 16875,8112 L 13954,8112 13954,4556 19796,4556 19796,8112 16875,8112 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="14204" y="5257"><tspan fill="rgb(0,0,0)" stroke="none">Cluster 3</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id7">
+ <rect class="BoundingBox" stroke="none" fill="none" x="2015" y="6460" width="5083" height="1019"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 4556,7477 L 2016,7477 2016,6461 7096,6461 7096,7477 4556,7477 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4556,7477 L 2016,7477 2016,6461 7096,6461 7096,7477 4556,7477 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="2446" y="7190"><tspan fill="rgb(0,0,0)" stroke="none">Analysis task 1</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id8">
+ <rect class="BoundingBox" stroke="none" fill="none" x="8237" y="6460" width="4956" height="1019"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 10715,7477 L 8238,7477 8238,6461 13191,6461 13191,7477 10715,7477 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 10715,7477 L 8238,7477 8238,6461 13191,6461 13191,7477 10715,7477 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="8604" y="7190"><tspan fill="rgb(0,0,0)" stroke="none">Analysis task 2</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id9">
+ <rect class="BoundingBox" stroke="none" fill="none" x="14206" y="6460" width="4956" height="1019"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 16684,7477 L 14207,7477 14207,6461 19160,6461 19160,7477 16684,7477 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 16684,7477 L 14207,7477 14207,6461 19160,6461 19160,7477 16684,7477 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="14573" y="7190"><tspan fill="rgb(0,0,0)" stroke="none">Analysis task 3</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id10">
+ <rect class="BoundingBox" stroke="none" fill="none" x="2015" y="10270" width="5083" height="1019"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 4556,11287 L 2016,11287 2016,10271 7096,10271 7096,11287 4556,11287 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4556,11287 L 2016,11287 2016,10271 7096,10271 7096,11287 4556,11287 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="2954" y="11000"><tspan fill="rgb(0,0,0)" stroke="none">Join results</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id11">
+ <rect class="BoundingBox" stroke="none" fill="none" x="4555" y="4936" width="6311" height="1526"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 4556,4937 L 4556,5699 10715,5699 10715,6031"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 10715,6461 L 10865,6011 10565,6011 10715,6461 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id12">
+ <rect class="BoundingBox" stroke="none" fill="none" x="4555" y="4936" width="12280" height="1526"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 4556,4937 L 4556,5699 16684,5699 16684,6031"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 16684,6461 L 16834,6011 16534,6011 16684,6461 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id13">
+ <rect class="BoundingBox" stroke="none" fill="none" x="4406" y="4936" width="301" height="1526"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 4556,4937 L 4556,6031"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 4556,6461 L 4706,6011 4406,6011 4556,6461 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id14">
+ <rect class="BoundingBox" stroke="none" fill="none" x="4406" y="7476" width="6311" height="2796"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 10715,7477 L 10715,8874 4556,8874 4556,9841"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 4556,10271 L 4706,9821 4406,9821 4556,10271 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id15">
+ <rect class="BoundingBox" stroke="none" fill="none" x="4406" y="7476" width="12280" height="2796"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 16684,7477 L 16684,8874 4556,8874 4556,9841"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 4556,10271 L 4706,9821 4406,9821 4556,10271 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id16">
+ <rect class="BoundingBox" stroke="none" fill="none" x="4406" y="7476" width="301" height="2796"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 4556,7477 L 4556,9841"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 4556,10271 L 4706,9821 4406,9821 4556,10271 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id17">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7857" y="3286" width="7748" height="1017"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-style="italic" font-weight="400"><tspan class="TextPosition" x="8107" y="3718"><tspan fill="rgb(0,0,0)" stroke="none">User from cluster 1 is able to authenticate </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-style="italic" font-weight="400"><tspan class="TextPosition" x="8107" y="4112"><tspan fill="rgb(0,0,0)" stroke="none">to clusters 2 and 3 and submit work</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id18">
+ <rect class="BoundingBox" stroke="none" fill="none" x="7730" y="9127" width="8048" height="1144"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-style="italic" font-weight="400"><tspan class="TextPosition" x="7980" y="9623"><tspan fill="rgb(0,0,0)" stroke="none">Input and output data for workflow steps is </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="353px" font-style="italic" font-weight="400"><tspan class="TextPosition" x="7980" y="10017"><tspan fill="rgb(0,0,0)" stroke="none">streamed to, from or between clusters on demand</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+</svg>
\ No newline at end of file
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Federated Multi-Cluster Workflows
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+To support running analysis on geographically dispersed data (avoiding expensive data transfers by sending the computation to the data), and "hybrid cloud" configurations where an on-premise cluster can expand its capabilities by delegating work to a cloud-hosted cluster, Arvados supports federated workflows. In a federated workflow, different steps of a workflow may execute on different clusters. Arvados manages data transfer and delegation of credentials, so that all that is required is adding "arv:ClusterTarget":cwl-extensions.html#clustertarget hints to your existing workflow.
+
+!(full-width)federated-workflow.svg!
+
+For more information, visit the "architecture":{{site.baseurl}}/architecture/federation.html and "admin":{{site.baseurl}}/admin/federation.html sections about Arvados federation.
+
+h2. Get the example files
+
+The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/curoverse/arvados/tree/master/doc/user/cwl/federated or "see below":#fed-example
+
+<notextile>
+<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados</span>
+~$ <span class="userinput">cd arvados/doc/user/cwl/federated</span>
+</code></pre>
+</notextile>
+
+h2. Run example
+
+{% include 'notebox_begin' %}
+
+At this time, remote steps of a workflow on Workbench are not displayed. As a workaround, you can find the UUIDs of the remote steps in the live logs of the workflow runner (the "Logs" tab). You may visit the remote cluster's workbench and enter the UUID into the search box to view the details of the remote step. This will be fixed in a future version of workbench.
+
+{% include 'notebox_end' %}
+
+Run it like any other workflow:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-cwl-runner federated.cwl shards.cwl</span>
+</code></pre>
+</notextile>
+
+You can also "run a workflow on a remote federated cluster":cwl-run-options.html#federation .
+
+h2(#fed-example). Federated scatter/gather example
+
+In this following example, an analysis task is executed on three different clusters with different data, then the results are combined to produce the final output.
+
+{% codeblock as yaml %}
+{% include 'federated_cwl' %}
+{% endcodeblock %}
+
+Example input document:
+
+{% codeblock as yaml %}
+{% include 'shards_yml' %}
+{% endcodeblock %}
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp:
+ type: File[]
+ inputBinding: {}
+outputs:
+ joined: stdout
+stdout: joined.txt
+baseCommand: cat
--- /dev/null
+#
+# Demonstrate Arvados federation features. This performs a parallel
+# scatter over some arbitrary number of files and federated clusters,
+# then joins the results.
+#
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ # When using Arvados extensions to CWL, must declare the 'arv' namespace
+ arv: "http://arvados.org/cwl#"
+
+requirements:
+ InlineJavascriptRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+
+ DockerRequirement:
+ # Replace this with your own Docker container
+ dockerPull: arvados/jobs
+
+ # Define a record type so we can conveniently associate the input
+ # file, the cluster on which the file lives, and the project on that
+ # cluster that will own the container requests and intermediate
+ # outputs.
+ SchemaDefRequirement:
+ types:
+ - name: FileOnCluster
+ type: record
+ fields:
+ file: File
+ cluster: string
+ project: string
+
+inputs:
+ # Expect an array of FileOnCluster records (defined above)
+ # as our input.
+ shards:
+ type:
+ type: array
+ items: FileOnCluster
+
+outputs:
+ # Will produce an output file with the results of the distributed
+ # analysis jobs joined together.
+ joined:
+ type: File
+ outputSource: gather-results/joined
+
+steps:
+ distributed-analysis:
+ in:
+ # Take "shards" array as input, we scatter over it below.
+ shard: shards
+
+ # Use an expression to extract the "file" field to assign to the
+ # "inp" parameter of the tool.
+ inp: {valueFrom: $(inputs.shard.file)}
+
+ # Scatter over shards, this means creating a parallel job for each
+ # element in the "shards" array. Expressions are evaluated for
+ # each element.
+ scatter: shard
+
+ # Specify the cluster target for this job. This means each
+ # separate scatter job will execute on the cluster that was
+ # specified in the "cluster" field.
+ #
+ # Arvados handles streaming data between clusters, for example,
+ # the Docker image containing the code for a particular tool will
+ # be fetched on demand, as long as it is available somewhere in
+ # the federation.
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.shard.cluster)
+ project_uuid: $(inputs.shard.project)
+
+ out: [out]
+ run: md5sum.cwl
+
+ # Collect the results of the distributed step and join them into a
+ # single output file. Arvados handles streaming inputs,
+ # intermediate results, and outputs between clusters on demand.
+ gather-results:
+ in:
+ inp: distributed-analysis/out
+ out: [joined]
+ run: cat.cwl
--- /dev/null
+file-on-clsr1.dat
--- /dev/null
+file-on-clsr2.dat
--- /dev/null
+file-on-clsr3.dat
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+inputs:
+ inp:
+ type: File
+outputs:
+ out:
+ type: File
+ outputBinding:
+ glob: out.txt
+stdin: $(inputs.inp.path)
+stdout: out.txt
+arguments: ["md5sum", "-"]
--- /dev/null
+shards:
+ - cluster: clsr1
+ project: clsr1-j7d0g-qxc4jcji7n4lafx
+ file:
+ class: File
+ location: keep:485df2c5cec3207a32f49c42f1cdcca9+61/file-on-clsr1.dat
+
+ - cluster: clsr2
+ project: clsr2-j7d0g-ivdrm1hyym21vkq
+ file:
+ class: File
+ location: keep:ae6e9c3e9bfa52a0122ecb489d8198ff+61/file-on-clsr2.dat
+
+ - cluster: clsr3
+ project: clsr3-j7d0g-e3njz2s53lyb0ka
+ file:
+ class: File
+ location: keep:0b43a0ef9ea592d5d7b299978dfa8643+61/file-on-clsr3.dat
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This guide provides a reference for using Arvados to solve big data bioinformatics problems, including:
+This guide provides a reference for using Arvados to solve scientific big data problems, including:
* Robust storage of very large files, such as whole genome sequences, using the "Arvados Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html content-addressable cluster file system.
-* Running compute-intensive genomic analysis pipelines, such as alignment and variant calls using the "Arvados Crunch":{{site.baseurl}}/user/tutorials/intro-crunch.html cluster compute engine.
-* Accessing, organizing, and sharing data, pipelines and results using the "Arvados Workbench":{{site.baseurl}}/user/getting_started/workbench.html web application.
+* Running compute-intensive scientific analysis pipelines, such as genomic alignment and variant calls using the "Arvados Crunch":{{site.baseurl}}/user/tutorials/intro-crunch.html cluster compute engine.
+* Accessing, organizing, and sharing data, workflows and results using the "Arvados Workbench":{{site.baseurl}}/user/getting_started/workbench.html web application.
+* Running an analysis using multiple clusters (HPC, cloud, or hybrid) with "Federated Multi-Cluster Workflows":{{site.baseurl}}/user/cwl/federated-workflows.html .
The examples in this guide use the public Arvados instance located at <a href="{{site.arvados_workbench_host}}/" target="_blank">{{site.arvados_workbench_host}}</a>. If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
# Using a secure shell client such as SSH or PuTTY to log on to a remote server
# Using the Unix command line shell, Bash
# Viewing and editing files using a unix text editor such as vi, Emacs, or nano
-# Programming in Python
# Revision control using Git
We also recommend you read the "Arvados Platform Overview":https://dev.arvados.org/projects/arvados/wiki#Platform-Overview for an introduction and background information about Arvados.
</code></pre>
</notextile>
+h3. Federated downloads
+
+If your cluster is "configured to be part of a federation":{{site.baseurl}}/admin/federation.html you can also download collections hosted on other clusters (with appropriate permissions).
+
+If you request a collection by portable data hash, it will first search the home cluster, then search federated clusters.
+
+You may also request a collection by UUID. In this case, it will contact the cluster named in the UUID prefix (in this example, @qr1hi@).
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv-get qr1hi-4zz18-fw6dnjxtkvzdewt/ .</span>
+</code></pre>
+</notextile>
+
h2(#download-using-workbench). Downloading using Workbench
You can also download Arvados data collections using the Workbench.
---
layout: default
navsection: userguide
-title: "Mounting Keep as a filesystem"
+title: "Accessing Keep from GNU/Linux"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This tutoral describes how to access Arvados collections using traditional filesystem tools by mounting Keep as a file system using @arv-mount@.
+This tutoral describes how to access Arvados collections on GNU/Linux using traditional filesystem tools by mounting Keep as a file system using @arv-mount@.
{% include 'tutorial_expectations' %}
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Accessing Keep from OS X"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+OS X users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
+
+h3. Browsing Keep (read-only)
+
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.uuid_prefix.your.domain/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+
+This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
+
+h3. Accessing a specific collection in Keep (read-write)
+
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.uuid_prefix.your.domain/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+
+This collection is now accessible read/write.
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Accessing Keep from Windows"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Windows users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
+
+h3. Browsing Keep (read-only)
+
+Use the 'Map network drive' functionality, and enter @https://collections.uuid_prefix.your.domain/@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid Arvados token in the @Password@ field.
+
+This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
+
+h3. Accessing a specific collection in Keep (read-write)
+
+Use the 'Map network drive' functionality, and enter @https://collections.uuid_prefix.your.domain/c=your-collection-uuid@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid token in the @Password@ field.
+
+This collection is now accessible read/write.
# apt.arvados.org
-deb http://apt.arvados.org/ jessie main
deb http://apt.arvados.org/ jessie-dev main
exit
end
-git_latest_tag = `git describe --abbrev=0`
+git_latest_tag = `git tag -l |sort -V -r |head -n1`
git_latest_tag = git_latest_tag.encode('utf-8').strip
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H .`.chomp.split(":")
git_timestamp = Time.at(git_timestamp.to_i).utc
import logging
import os
import sys
-import threading
-import hashlib
-import copy
-import json
import re
-from functools import partial
import pkg_resources # part of setuptools
-import Queue
-import time
-import signal
-import thread
-from cwltool.errors import WorkflowException
+from schema_salad.sourceline import SourceLine
+import schema_salad.validate as validate
import cwltool.main
import cwltool.workflow
import cwltool.process
-from schema_salad.sourceline import SourceLine
-import schema_salad.validate as validate
import cwltool.argparser
+from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
import arvados
import arvados.config
from arvados.keep import KeepClient
from arvados.errors import ApiError
import arvados.commands._util as arv_cmd
+from arvados.api import OrderedJsonModel
-from .arvcontainer import ArvadosContainer, RunnerContainer
-from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
-from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
-from .arvtool import ArvadosCommandTool
-from .arvworkflow import ArvadosWorkflow, upload_workflow
-from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache
from .perf import Perf
-from .pathmapper import NoFollowPathMapper
-from .task_queue import TaskQueue
-from .context import ArvLoadingContext, ArvRuntimeContext
-from .util import get_current_container
from ._version import __version__
+from .executor import ArvCwlExecutor
-from cwltool.pack import pack
-from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
-from cwltool.command_line_tool import compute_checksums
-
-from arvados.api import OrderedJsonModel
+# These arn't used directly in this file but
+# other code expects to import them from here
+from .arvcontainer import ArvadosContainer
+from .arvjob import ArvadosJob
+from .arvtool import ArvadosCommandTool
+from .fsaccess import CollectionFsAccess, CollectionCache, CollectionFetcher
+from .util import get_current_container
+from .executor import RuntimeStatusLoggingHandler, DEFAULT_PRIORITY
+from .arvworkflow import ArvadosWorkflow
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
'%(asctime)s %(name)s %(levelname)s: %(message)s',
'%Y-%m-%d %H:%M:%S'))
-DEFAULT_PRIORITY = 500
-
-class RuntimeStatusLoggingHandler(logging.Handler):
- """
- Intercepts logging calls and report them as runtime statuses on runner
- containers.
- """
- def __init__(self, runtime_status_update_func):
- super(RuntimeStatusLoggingHandler, self).__init__()
- self.runtime_status_update = runtime_status_update_func
-
- def emit(self, record):
- kind = None
- if record.levelno >= logging.ERROR:
- kind = 'error'
- elif record.levelno >= logging.WARNING:
- kind = 'warning'
- if kind is not None:
- log_msg = record.getMessage()
- if '\n' in log_msg:
- # If the logged message is multi-line, use its first line as status
- # and the rest as detail.
- status, detail = log_msg.split('\n', 1)
- self.runtime_status_update(
- kind,
- "%s: %s" % (record.name, status),
- detail
- )
- else:
- self.runtime_status_update(
- kind,
- "%s: %s" % (record.name, record.getMessage())
- )
-
-class ArvCwlRunner(object):
- """Execute a CWL tool or workflow, submit work (using either jobs or
- containers API), wait for them to complete, and report output.
-
- """
-
- def __init__(self, api_client,
- arvargs=None,
- keep_client=None,
- num_retries=4,
- thread_count=4):
-
- if arvargs is None:
- arvargs = argparse.Namespace()
- arvargs.work_api = None
- arvargs.output_name = None
- arvargs.output_tags = None
- arvargs.thread_count = 1
-
- self.api = api_client
- self.processes = {}
- self.workflow_eval_lock = threading.Condition(threading.RLock())
- self.final_output = None
- self.final_status = None
- self.num_retries = num_retries
- self.uuid = None
- self.stop_polling = threading.Event()
- self.poll_api = None
- self.pipeline = None
- self.final_output_collection = None
- self.output_name = arvargs.output_name
- self.output_tags = arvargs.output_tags
- self.project_uuid = None
- self.intermediate_output_ttl = 0
- self.intermediate_output_collections = []
- self.trash_intermediate = False
- self.thread_count = arvargs.thread_count
- self.poll_interval = 12
- self.loadingContext = None
-
- if keep_client is not None:
- self.keep_client = keep_client
- else:
- self.keep_client = arvados.keep.KeepClient(api_client=self.api, num_retries=self.num_retries)
-
- self.collection_cache = CollectionCache(self.api, self.keep_client, self.num_retries)
-
- self.fetcher_constructor = partial(CollectionFetcher,
- api_client=self.api,
- fs_access=CollectionFsAccess("", collection_cache=self.collection_cache),
- num_retries=self.num_retries)
-
- self.work_api = None
- expected_api = ["jobs", "containers"]
- for api in expected_api:
- try:
- methods = self.api._rootDesc.get('resources')[api]['methods']
- if ('httpMethod' in methods['create'] and
- (arvargs.work_api == api or arvargs.work_api is None)):
- self.work_api = api
- break
- except KeyError:
- pass
-
- if not self.work_api:
- if arvargs.work_api is None:
- raise Exception("No supported APIs")
- else:
- raise Exception("Unsupported API '%s', expected one of %s" % (arvargs.work_api, expected_api))
-
- if self.work_api == "jobs":
- logger.warn("""
-*******************************
-Using the deprecated 'jobs' API.
-
-To get rid of this warning:
-
-Users: read about migrating at
-http://doc.arvados.org/user/cwl/cwl-style.html#migrate
-and use the option --api=containers
-
-Admins: configure the cluster to disable the 'jobs' API as described at:
-http://doc.arvados.org/install/install-api-server.html#disable_api_methods
-*******************************""")
-
- self.loadingContext = ArvLoadingContext(vars(arvargs))
- self.loadingContext.fetcher_constructor = self.fetcher_constructor
- self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
- self.loadingContext.construct_tool_object = self.arv_make_tool
-
- # Add a custom logging handler to the root logger for runtime status reporting
- # if running inside a container
- if get_current_container(self.api, self.num_retries, logger):
- root_logger = logging.getLogger('')
- handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
- root_logger.addHandler(handler)
-
- def arv_make_tool(self, toolpath_object, loadingContext):
- if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
- return ArvadosCommandTool(self, toolpath_object, loadingContext)
- elif "class" in toolpath_object and toolpath_object["class"] == "Workflow":
- return ArvadosWorkflow(self, toolpath_object, loadingContext)
- else:
- return cwltool.workflow.default_make_tool(toolpath_object, loadingContext)
-
- def output_callback(self, out, processStatus):
- with self.workflow_eval_lock:
- if processStatus == "success":
- logger.info("Overall process status is %s", processStatus)
- state = "Complete"
- else:
- logger.error("Overall process status is %s", processStatus)
- state = "Failed"
- if self.pipeline:
- self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
- body={"state": state}).execute(num_retries=self.num_retries)
- self.final_status = processStatus
- self.final_output = out
- self.workflow_eval_lock.notifyAll()
-
-
- def start_run(self, runnable, runtimeContext):
- self.task_queue.add(partial(runnable.run, runtimeContext))
-
- def process_submitted(self, container):
- with self.workflow_eval_lock:
- self.processes[container.uuid] = container
-
- def process_done(self, uuid, record):
- with self.workflow_eval_lock:
- j = self.processes[uuid]
- logger.info("%s %s is %s", self.label(j), uuid, record["state"])
- self.task_queue.add(partial(j.done, record))
- del self.processes[uuid]
-
- def runtime_status_update(self, kind, message, detail=None):
- """
- Updates the runtime_status field on the runner container.
- Called when there's a need to report errors, warnings or just
- activity statuses, for example in the RuntimeStatusLoggingHandler.
- """
- with self.workflow_eval_lock:
- current = get_current_container(self.api, self.num_retries, logger)
- if current is None:
- return
- runtime_status = current.get('runtime_status', {})
- # In case of status being an error, only report the first one.
- if kind == 'error':
- if not runtime_status.get('error'):
- runtime_status.update({
- 'error': message
- })
- if detail is not None:
- runtime_status.update({
- 'errorDetail': detail
- })
- # Further errors are only mentioned as a count.
- else:
- # Get anything before an optional 'and N more' string.
- try:
- error_msg = re.match(
- r'^(.*?)(?=\s*\(and \d+ more\)|$)', runtime_status.get('error')).groups()[0]
- more_failures = re.match(
- r'.*\(and (\d+) more\)', runtime_status.get('error'))
- except TypeError:
- # Ignore tests stubbing errors
- return
- if more_failures:
- failure_qty = int(more_failures.groups()[0])
- runtime_status.update({
- 'error': "%s (and %d more)" % (error_msg, failure_qty+1)
- })
- else:
- runtime_status.update({
- 'error': "%s (and 1 more)" % error_msg
- })
- elif kind in ['warning', 'activity']:
- # Record the last warning/activity status without regard of
- # previous occurences.
- runtime_status.update({
- kind: message
- })
- if detail is not None:
- runtime_status.update({
- kind+"Detail": detail
- })
- else:
- # Ignore any other status kind
- return
- try:
- self.api.containers().update(uuid=current['uuid'],
- body={
- 'runtime_status': runtime_status,
- }).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.info("Couldn't update runtime_status: %s", e)
-
- def wrapped_callback(self, cb, obj, st):
- with self.workflow_eval_lock:
- cb(obj, st)
- self.workflow_eval_lock.notifyAll()
-
- def get_wrapped_callback(self, cb):
- return partial(self.wrapped_callback, cb)
-
- def on_message(self, event):
- if event.get("object_uuid") in self.processes and event["event_type"] == "update":
- uuid = event["object_uuid"]
- if event["properties"]["new_attributes"]["state"] == "Running":
- with self.workflow_eval_lock:
- j = self.processes[uuid]
- if j.running is False:
- j.running = True
- j.update_pipeline_component(event["properties"]["new_attributes"])
- logger.info("%s %s is Running", self.label(j), uuid)
- elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled", "Final"):
- self.process_done(uuid, event["properties"]["new_attributes"])
-
- def label(self, obj):
- return "[%s %s]" % (self.work_api[0:-1], obj.name)
-
- def poll_states(self):
- """Poll status of jobs or containers listed in the processes dict.
-
- Runs in a separate thread.
- """
-
- try:
- remain_wait = self.poll_interval
- while True:
- if remain_wait > 0:
- self.stop_polling.wait(remain_wait)
- if self.stop_polling.is_set():
- break
- with self.workflow_eval_lock:
- keys = list(self.processes.keys())
- if not keys:
- remain_wait = self.poll_interval
- continue
-
- begin_poll = time.time()
- if self.work_api == "containers":
- table = self.poll_api.container_requests()
- elif self.work_api == "jobs":
- table = self.poll_api.jobs()
-
- try:
- proc_states = table.list(filters=[["uuid", "in", keys]]).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.warn("Error checking states on API server: %s", e)
- remain_wait = self.poll_interval
- continue
-
- for p in proc_states["items"]:
- self.on_message({
- "object_uuid": p["uuid"],
- "event_type": "update",
- "properties": {
- "new_attributes": p
- }
- })
- finish_poll = time.time()
- remain_wait = self.poll_interval - (finish_poll - begin_poll)
- except:
- logger.exception("Fatal error in state polling thread.")
- with self.workflow_eval_lock:
- self.processes.clear()
- self.workflow_eval_lock.notifyAll()
- finally:
- self.stop_polling.set()
-
- def add_intermediate_output(self, uuid):
- if uuid:
- self.intermediate_output_collections.append(uuid)
-
- def trash_intermediate_output(self):
- logger.info("Cleaning up intermediate output collections")
- for i in self.intermediate_output_collections:
- try:
- self.api.collections().delete(uuid=i).execute(num_retries=self.num_retries)
- except:
- logger.warn("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
- if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
- break
-
- def check_features(self, obj):
- if isinstance(obj, dict):
- if obj.get("writable") and self.work_api != "containers":
- raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
- if obj.get("class") == "DockerRequirement":
- if obj.get("dockerOutputDirectory"):
- if self.work_api != "containers":
- raise SourceLine(obj, "dockerOutputDirectory", UnsupportedRequirement).makeError(
- "Option 'dockerOutputDirectory' of DockerRequirement not supported with --api=jobs.")
- if not obj.get("dockerOutputDirectory").startswith('/'):
- raise SourceLine(obj, "dockerOutputDirectory", validate.ValidationException).makeError(
- "Option 'dockerOutputDirectory' must be an absolute path.")
- if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
- raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
- for v in obj.itervalues():
- self.check_features(v)
- elif isinstance(obj, list):
- for i,v in enumerate(obj):
- with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
- self.check_features(v)
-
- def make_output_collection(self, name, storage_classes, tagsString, outputObj):
- outputObj = copy.deepcopy(outputObj)
-
- files = []
- def capture(fileobj):
- files.append(fileobj)
-
- adjustDirObjs(outputObj, capture)
- adjustFileObjs(outputObj, capture)
-
- generatemapper = NoFollowPathMapper(files, "", "", separateDirs=False)
-
- final = arvados.collection.Collection(api_client=self.api,
- keep_client=self.keep_client,
- num_retries=self.num_retries)
-
- for k,v in generatemapper.items():
- if k.startswith("_:"):
- if v.type == "Directory":
- continue
- if v.type == "CreateFile":
- with final.open(v.target, "wb") as f:
- f.write(v.resolved.encode("utf-8"))
- continue
-
- if not k.startswith("keep:"):
- raise Exception("Output source is not in keep or a literal")
- sp = k.split("/")
- srccollection = sp[0][5:]
- try:
- reader = self.collection_cache.get(srccollection)
- srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
- final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
- except arvados.errors.ArgumentError as e:
- logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
- raise
- except IOError as e:
- logger.warn("While preparing output collection: %s", e)
-
- def rewrite(fileobj):
- fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
- for k in ("listing", "contents", "nameext", "nameroot", "dirname"):
- if k in fileobj:
- del fileobj[k]
-
- adjustDirObjs(outputObj, rewrite)
- adjustFileObjs(outputObj, rewrite)
-
- with final.open("cwl.output.json", "w") as f:
- json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
-
- final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
-
- logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
- final.api_response()["name"],
- final.manifest_locator())
-
- final_uuid = final.manifest_locator()
- tags = tagsString.split(',')
- for tag in tags:
- self.api.links().create(body={
- "head_uuid": final_uuid, "link_class": "tag", "name": tag
- }).execute(num_retries=self.num_retries)
-
- def finalcollection(fileobj):
- fileobj["location"] = "keep:%s/%s" % (final.portable_data_hash(), fileobj["location"])
-
- adjustDirObjs(outputObj, finalcollection)
- adjustFileObjs(outputObj, finalcollection)
-
- return (outputObj, final)
-
- def set_crunch_output(self):
- if self.work_api == "containers":
- current = get_current_container(self.api, self.num_retries, logger)
- if current is None:
- return
- try:
- self.api.containers().update(uuid=current['uuid'],
- body={
- 'output': self.final_output_collection.portable_data_hash(),
- }).execute(num_retries=self.num_retries)
- self.api.collections().update(uuid=self.final_output_collection.manifest_locator(),
- body={
- 'is_trashed': True
- }).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.info("Setting container output: %s", e)
- elif self.work_api == "jobs" and "TASK_UUID" in os.environ:
- self.api.job_tasks().update(uuid=os.environ["TASK_UUID"],
- body={
- 'output': self.final_output_collection.portable_data_hash(),
- 'success': self.final_status == "success",
- 'progress':1.0
- }).execute(num_retries=self.num_retries)
-
- def arv_executor(self, tool, job_order, runtimeContext, logger=None):
- self.debug = runtimeContext.debug
-
- tool.visit(self.check_features)
-
- self.project_uuid = runtimeContext.project_uuid
- self.pipeline = None
- self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
- self.secret_store = runtimeContext.secret_store
-
- self.trash_intermediate = runtimeContext.trash_intermediate
- if self.trash_intermediate and self.work_api != "containers":
- raise Exception("--trash-intermediate is only supported with --api=containers.")
-
- self.intermediate_output_ttl = runtimeContext.intermediate_output_ttl
- if self.intermediate_output_ttl and self.work_api != "containers":
- raise Exception("--intermediate-output-ttl is only supported with --api=containers.")
- if self.intermediate_output_ttl < 0:
- raise Exception("Invalid value %d for --intermediate-output-ttl, cannot be less than zero" % self.intermediate_output_ttl)
-
- if runtimeContext.submit_request_uuid and self.work_api != "containers":
- raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
-
- if not runtimeContext.name:
- runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
-
- # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
- # Also uploads docker images.
- merged_map = upload_workflow_deps(self, tool)
-
- # Reload tool object which may have been updated by
- # upload_workflow_deps
- # Don't validate this time because it will just print redundant errors.
- loadingContext = self.loadingContext.copy()
- loadingContext.loader = tool.doc_loader
- loadingContext.avsc_names = tool.doc_schema
- loadingContext.metadata = tool.metadata
- loadingContext.do_validate = False
-
- tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
- loadingContext)
-
- # Upload local file references in the job order.
- job_order = upload_job_order(self, "%s input" % runtimeContext.name,
- tool, job_order)
-
- existing_uuid = runtimeContext.update_workflow
- if existing_uuid or runtimeContext.create_workflow:
- # Create a pipeline template or workflow record and exit.
- if self.work_api == "jobs":
- tmpl = RunnerTemplate(self, tool, job_order,
- runtimeContext.enable_reuse,
- uuid=existing_uuid,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- merged_map=merged_map)
- tmpl.save()
- # cwltool.main will write our return value to stdout.
- return (tmpl.uuid, "success")
- elif self.work_api == "containers":
- return (upload_workflow(self, tool, job_order,
- self.project_uuid,
- uuid=existing_uuid,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- merged_map=merged_map),
- "success")
-
- self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse
- self.eval_timeout = runtimeContext.eval_timeout
-
- runtimeContext = runtimeContext.copy()
- runtimeContext.use_container = True
- runtimeContext.tmpdir_prefix = "tmp"
- runtimeContext.work_api = self.work_api
-
- if self.work_api == "containers":
- if self.ignore_docker_for_reuse:
- raise Exception("--ignore-docker-for-reuse not supported with containers API.")
- runtimeContext.outdir = "/var/spool/cwl"
- runtimeContext.docker_outdir = "/var/spool/cwl"
- runtimeContext.tmpdir = "/tmp"
- runtimeContext.docker_tmpdir = "/tmp"
- elif self.work_api == "jobs":
- if runtimeContext.priority != DEFAULT_PRIORITY:
- raise Exception("--priority not implemented for jobs API.")
- runtimeContext.outdir = "$(task.outdir)"
- runtimeContext.docker_outdir = "$(task.outdir)"
- runtimeContext.tmpdir = "$(task.tmpdir)"
-
- if runtimeContext.priority < 1 or runtimeContext.priority > 1000:
- raise Exception("--priority must be in the range 1..1000.")
-
- runnerjob = None
- if runtimeContext.submit:
- # Submit a runner job to run the workflow for us.
- if self.work_api == "containers":
- if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait:
- runtimeContext.runnerjob = tool.tool["id"]
- runnerjob = tool.job(job_order,
- self.output_callback,
- runtimeContext).next()
- else:
- runnerjob = RunnerContainer(self, tool, job_order, runtimeContext.enable_reuse,
- self.output_name,
- self.output_tags,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- on_error=runtimeContext.on_error,
- submit_runner_image=runtimeContext.submit_runner_image,
- intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
- merged_map=merged_map,
- priority=runtimeContext.priority,
- secret_store=self.secret_store)
- elif self.work_api == "jobs":
- runnerjob = RunnerJob(self, tool, job_order, runtimeContext.enable_reuse,
- self.output_name,
- self.output_tags,
- submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
- on_error=runtimeContext.on_error,
- submit_runner_image=runtimeContext.submit_runner_image,
- merged_map=merged_map)
- elif runtimeContext.cwl_runner_job is None and self.work_api == "jobs":
- # Create pipeline for local run
- self.pipeline = self.api.pipeline_instances().create(
- body={
- "owner_uuid": self.project_uuid,
- "name": runtimeContext.name if runtimeContext.name else shortname(tool.tool["id"]),
- "components": {},
- "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
- logger.info("Pipeline instance %s", self.pipeline["uuid"])
-
- if runnerjob and not runtimeContext.wait:
- submitargs = runtimeContext.copy()
- submitargs.submit = False
- runnerjob.run(submitargs)
- return (runnerjob.uuid, "success")
-
- self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout)
- self.polling_thread = threading.Thread(target=self.poll_states)
- self.polling_thread.start()
-
- self.task_queue = TaskQueue(self.workflow_eval_lock, self.thread_count)
-
- if runnerjob:
- jobiter = iter((runnerjob,))
- else:
- if runtimeContext.cwl_runner_job is not None:
- self.uuid = runtimeContext.cwl_runner_job.get('uuid')
- jobiter = tool.job(job_order,
- self.output_callback,
- runtimeContext)
-
- try:
- self.workflow_eval_lock.acquire()
- # Holds the lock while this code runs and releases it when
- # it is safe to do so in self.workflow_eval_lock.wait(),
- # at which point on_message can update job state and
- # process output callbacks.
-
- loopperf = Perf(metrics, "jobiter")
- loopperf.__enter__()
- for runnable in jobiter:
- loopperf.__exit__()
-
- if self.stop_polling.is_set():
- break
-
- if self.task_queue.error is not None:
- raise self.task_queue.error
-
- if runnable:
- with Perf(metrics, "run"):
- self.start_run(runnable, runtimeContext)
- else:
- if (self.task_queue.in_flight + len(self.processes)) > 0:
- self.workflow_eval_lock.wait(3)
- else:
- logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
- break
- loopperf.__enter__()
- loopperf.__exit__()
-
- while (self.task_queue.in_flight + len(self.processes)) > 0:
- if self.task_queue.error is not None:
- raise self.task_queue.error
- self.workflow_eval_lock.wait(3)
-
- except UnsupportedRequirement:
- raise
- except:
- if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
- logger.error("Interrupted, workflow will be cancelled")
- else:
- logger.error("Execution failed: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
- if self.pipeline:
- self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
- body={"state": "Failed"}).execute(num_retries=self.num_retries)
- if runnerjob and runnerjob.uuid and self.work_api == "containers":
- self.api.container_requests().update(uuid=runnerjob.uuid,
- body={"priority": "0"}).execute(num_retries=self.num_retries)
- finally:
- self.workflow_eval_lock.release()
- self.task_queue.drain()
- self.stop_polling.set()
- self.polling_thread.join()
- self.task_queue.join()
-
- if self.final_status == "UnsupportedRequirement":
- raise UnsupportedRequirement("Check log for details.")
-
- if self.final_output is None:
- raise WorkflowException("Workflow did not return a result.")
-
- if runtimeContext.submit and isinstance(runnerjob, Runner):
- logger.info("Final output collection %s", runnerjob.final_output)
- else:
- if self.output_name is None:
- self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
- if self.output_tags is None:
- self.output_tags = ""
-
- storage_classes = runtimeContext.storage_classes.strip().split(",")
- self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
- self.set_crunch_output()
-
- if runtimeContext.compute_checksum:
- adjustDirObjs(self.final_output, partial(get_listing, self.fs_access))
- adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
-
- if self.trash_intermediate and self.final_status == "success":
- self.trash_intermediate_output()
-
- return (self.final_output, self.final_status)
-
-
def versionstring():
"""Print version string of key packages for provenance and debugging."""
help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
default=None)
- parser.add_argument("--submit-request-uuid", type=str,
+ parser.add_argument("--always-submit-runner", action="store_true",
+ help="When invoked with --submit --wait, always submit a runner to manage the workflow, even when only running a single CommandLineTool",
+ default=False)
+
+ exgroup = parser.add_mutually_exclusive_group()
+ exgroup.add_argument("--submit-request-uuid", type=str,
+ default=None,
+ help="Update and commit to supplied container request instead of creating a new one (containers API only).",
+ metavar="UUID")
+ exgroup.add_argument("--submit-runner-cluster", type=str,
+ help="Submit workflow runner to a remote cluster (containers API only)",
+ default=None,
+ metavar="CLUSTER_ID")
+
+ parser.add_argument("--collection-cache-size", type=int,
default=None,
- help="Update and commit supplied container request instead of creating a new one (containers API only).")
+ help="Collection cache size (in MiB, default 256).")
parser.add_argument("--name", type=str,
help="Name to use for workflow execution instance.",
default=None)
- parser.add_argument("--on-error", type=str,
- help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. "
- "Default is 'continue'.", default="continue", choices=("stop", "continue"))
+ parser.add_argument("--on-error",
+ help="Desired workflow behavior when a step fails. One of 'stop' (do not submit any more steps) or "
+ "'continue' (may submit other steps that are not downstream from the error). Default is 'continue'.",
+ default="continue", choices=("stop", "continue"))
parser.add_argument("--enable-dev", action="store_true",
help="Enable loading and running development versions "
help=argparse.SUPPRESS)
parser.add_argument("--thread-count", type=int,
- default=4, help="Number of threads to use for job submit and output collection.")
+ default=1, help="Number of threads to use for job submit and output collection.")
parser.add_argument("--http-timeout", type=int,
default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
"http://arvados.org/cwl#APIRequirement",
"http://commonwl.org/cwltool#LoadListingRequirement",
"http://arvados.org/cwl#IntermediateOutput",
- "http://arvados.org/cwl#ReuseRequirement"
+ "http://arvados.org/cwl#ReuseRequirement",
+ "http://arvados.org/cwl#ClusterTarget"
])
def exit_signal_handler(sigcode, frame):
add_arv_hints()
+ for key, val in cwltool.argparser.get_default_args().items():
+ if not hasattr(arvargs, key):
+ setattr(arvargs, key, val)
+
try:
if api_client is None:
api_client = arvados.safeapi.ThreadSafeApiCache(
api_client.users().current().execute()
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
- runner = ArvCwlRunner(api_client, arvargs, keep_client=keep_client, num_retries=4)
+ executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
except Exception as e:
logger.error(e)
return 1
else:
arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
- for key, val in cwltool.argparser.get_default_args().items():
- if not hasattr(arvargs, key):
- setattr(arvargs, key, val)
-
- runtimeContext = ArvRuntimeContext(vars(arvargs))
- runtimeContext.make_fs_access = partial(CollectionFsAccess,
- collection_cache=runner.collection_cache)
- runtimeContext.http_timeout = arvargs.http_timeout
-
return cwltool.main.main(args=arvargs,
stdout=stdout,
stderr=stderr,
- executor=runner.arv_executor,
+ executor=executor.arv_executor,
versionfunc=versionstring,
job_order_object=job_order_object,
logger_handler=arvados.log_handler,
custom_schema_callback=add_arv_hints,
- loadingContext=runner.loadingContext,
- runtimeContext=runtimeContext)
+ loadingContext=executor.loadingContext,
+ runtimeContext=executor.runtimeContext)
coresMin:
type: int?
doc: Minimum cores allocated to cwl-runner
- jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
\ No newline at end of file
+ jsonldPredicate: "https://w3id.org/cwl/cwl#ResourceRequirement/coresMin"
+ keep_cache:
+ type: int?
+ doc: |
+ Size of collection metadata cache for the workflow runner, in
+ MiB. Default 256 MiB. Will be added on to the RAM request
+ when determining node size to request.
+ jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+
+- name: ClusterTarget
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify where a workflow step should run
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:ClusterTarget'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ cluster_id:
+ type: string?
+ doc: The cluster to run the container
+ project_uuid:
+ type: string?
+ doc: The project that will own the container requests and intermediate collections
import uuid
import math
-from arvados_cwl.util import get_current_container, get_intermediate_collection_info
+import arvados_cwl.util
import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
class ArvadosContainer(JobBase):
"""Submit and manage a Crunch container request for executing a CWL CommandLineTool."""
- def __init__(self, runner,
+ def __init__(self, runner, job_runtime,
builder, # type: Builder
joborder, # type: Dict[Text, Union[Dict[Text, Any], List, Text]]
make_path_mapper, # type: Callable[..., PathMapper]
):
super(ArvadosContainer, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name)
self.arvrunner = runner
+ self.job_runtime = job_runtime
self.running = False
self.uuid = None
# ArvadosContainer object by CommandLineTool.job() before
# run() is called.
+ runtimeContext = self.job_runtime
+
container_request = {
"command": self.command_line,
"name": self.name,
}
runtime_constraints = {}
- if self.arvrunner.project_uuid:
- container_request["owner_uuid"] = self.arvrunner.project_uuid
+ if runtimeContext.project_uuid:
+ container_request["owner_uuid"] = runtimeContext.project_uuid
if self.arvrunner.secret_store.has_secret(self.command_line):
raise WorkflowException("Secret material leaked on command line, only file literals may contain secrets")
keepemptydirs(vwd)
if not runtimeContext.current_container:
- runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ runtimeContext.current_container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = arvados_cwl.util.get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
- owner_uuid=self.arvrunner.project_uuid,
+ owner_uuid=runtimeContext.project_uuid,
ensure_unique_name=True,
trash_at=info["trash_at"],
properties=info["properties"])
docker_req = {"dockerImageId": "arvados/jobs"}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
- docker_req,
- runtimeContext.pull_image,
- self.arvrunner.project_uuid)
+ docker_req,
+ runtimeContext.pull_image,
+ runtimeContext.project_uuid)
api_req, _ = self.get_requirement("http://arvados.org/cwl#APIRequirement")
if api_req:
if self.timelimit is not None:
scheduling_parameters["max_run_time"] = self.timelimit
+ extra_submit_params = {}
+ if runtimeContext.submit_runner_cluster:
+ extra_submit_params["cluster_id"] = runtimeContext.submit_runner_cluster
+
container_request["output_name"] = "Output for step %s" % (self.name)
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
if runtimeContext.submit_request_uuid:
response = self.arvrunner.api.container_requests().update(
uuid=runtimeContext.submit_request_uuid,
- body=container_request
+ body=container_request,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
else:
response = self.arvrunner.api.container_requests().create(
- body=container_request
+ body=container_request,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
"secret_mounts": secret_mounts,
"runtime_constraints": {
"vcpus": math.ceil(self.submit_runner_cores),
- "ram": math.ceil(1024*1024 * self.submit_runner_ram),
+ "ram": 1024*1024 * (math.ceil(self.submit_runner_ram) + math.ceil(self.collection_cache_size)),
"API": True
},
"use_existing": self.enable_reuse,
# --eval-timeout is the timeout for javascript invocation
# --parallel-task-count is the number of threads to use for job submission
# --enable/disable-reuse sets desired job reuse
+ # --collection-cache-size sets aside memory to store collections
command = ["arvados-cwl-runner",
"--local",
"--api=containers",
"--disable-validate",
"--eval-timeout=%s" % self.arvrunner.eval_timeout,
"--thread-count=%s" % self.arvrunner.thread_count,
- "--enable-reuse" if self.enable_reuse else "--disable-reuse"]
+ "--enable-reuse" if self.enable_reuse else "--disable-reuse",
+ "--collection-cache-size=%s" % self.collection_cache_size]
if self.output_name:
command.append("--output-name=" + self.output_name)
if self.arvrunner.project_uuid:
job_spec["owner_uuid"] = self.arvrunner.project_uuid
+ extra_submit_params = {}
+ if runtimeContext.submit_runner_cluster:
+ extra_submit_params["cluster_id"] = runtimeContext.submit_runner_cluster
+
if runtimeContext.submit_request_uuid:
response = self.arvrunner.api.container_requests().update(
uuid=runtimeContext.submit_request_uuid,
- body=job_spec
+ body=job_spec,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
else:
response = self.arvrunner.api.container_requests().create(
- body=job_spec
+ body=job_spec,
+ **extra_submit_params
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
+ if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
+ return dockerRequirement["http://arvados.org/cwl#dockerCollectionPDH"]
+
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement = copy.deepcopy(dockerRequirement)
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
global cached_lookups_lock
with cached_lookups_lock:
if dockerRequirement["dockerImageId"] in cached_lookups:
- return dockerRequirement["dockerImageId"]
+ return cached_lookups[dockerRequirement["dockerImageId"]]
with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
sp = dockerRequirement["dockerImageId"].split(":")
if not images:
raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
+ pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
+
with cached_lookups_lock:
- cached_lookups[dockerRequirement["dockerImageId"]] = True
+ cached_lookups[dockerRequirement["dockerImageId"]] = pdh
- return dockerRequirement["dockerImageId"]
+ return pdh
def arv_docker_clear_cache():
global cached_lookups
from schema_salad.sourceline import SourceLine
-from arvados_cwl.util import get_current_container, get_intermediate_collection_info
+import arvados_cwl.util
import ruamel.yaml as yaml
import arvados.collection
from .perf import Perf
from . import done
from ._version import __version__
+from .util import get_intermediate_collection_info
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
if vwd:
with Perf(metrics, "generatefiles.save_new %s" % self.name):
- if not runtimeContext.current_container:
- runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+ info = get_intermediate_collection_info(self.name, None, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
ensure_unique_name=True,
#
# SPDX-License-Identifier: Apache-2.0
-from cwltool.command_line_tool import CommandLineTool
+from cwltool.command_line_tool import CommandLineTool, ExpressionTool
+from cwltool.builder import Builder
from .arvjob import ArvadosJob
from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
from functools import partial
+from schema_salad.sourceline import SourceLine
+from cwltool.errors import WorkflowException
+
+def validate_cluster_target(arvrunner, runtimeContext):
+ if (runtimeContext.submit_runner_cluster and
+ runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and
+ runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]):
+ raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster,
+ ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys())))
+def set_cluster_target(tool, arvrunner, builder, runtimeContext):
+ cluster_target_req = None
+ for field in ("hints", "requirements"):
+ if field not in tool:
+ continue
+ for item in tool[field]:
+ if item["class"] == "http://arvados.org/cwl#ClusterTarget":
+ cluster_target_req = item
+
+ if cluster_target_req is None:
+ return runtimeContext
+
+ with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug):
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster
+ runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid
+ validate_cluster_target(arvrunner, runtimeContext)
+
+ return runtimeContext
+
+def make_builder(joborder, hints, requirements, runtimeContext):
+ return Builder(
+ job=joborder,
+ files=[], # type: List[Dict[Text, Text]]
+ bindings=[], # type: List[Dict[Text, Any]]
+ schemaDefs={}, # type: Dict[Text, Dict[Text, Any]]
+ names=None, # type: Names
+ requirements=requirements, # type: List[Dict[Text, Any]]
+ hints=hints, # type: List[Dict[Text, Any]]
+ resources={}, # type: Dict[str, int]
+ mutation_manager=None, # type: Optional[MutationManager]
+ formatgraph=None, # type: Optional[Graph]
+ make_fs_access=None, # type: Type[StdFsAccess]
+ fs_access=None, # type: StdFsAccess
+ job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any]
+ timeout=runtimeContext.eval_timeout, # type: float
+ debug=runtimeContext.debug, # type: bool
+ js_console=runtimeContext.js_console, # type: bool
+ force_docker_pull=runtimeContext.force_docker_pull, # type: bool
+ loadListing="", # type: Text
+ outdir="", # type: Text
+ tmpdir="", # type: Text
+ stagedir="", # type: Text
+ )
class ArvadosCommandTool(CommandLineTool):
"""Wrap cwltool CommandLineTool to override selected methods."""
def make_job_runner(self, runtimeContext):
if runtimeContext.work_api == "containers":
- return partial(ArvadosContainer, self.arvrunner)
+ return partial(ArvadosContainer, self.arvrunner, runtimeContext)
elif runtimeContext.work_api == "jobs":
return partial(ArvadosJob, self.arvrunner)
else:
"$(task.keep)/%s/%s")
def job(self, joborder, output_callback, runtimeContext):
-
- # Workaround for #13365
- builderargs = runtimeContext.copy()
- builderargs.toplevel = True
- builderargs.tmp_outdir_prefix = ""
- builder = self._init_job(joborder, builderargs)
- joborder = builder.job
-
- runtimeContext = runtimeContext.copy()
+ builder = make_builder(joborder, self.hints, self.requirements, runtimeContext)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
if runtimeContext.work_api == "containers":
dockerReq, is_req = self.get_requirement("DockerRequirement")
runtimeContext.tmpdir = "$(task.tmpdir)"
runtimeContext.docker_tmpdir = "$(task.tmpdir)"
return super(ArvadosCommandTool, self).job(joborder, output_callback, runtimeContext)
+
+class ArvadosExpressionTool(ExpressionTool):
+ def __init__(self, arvrunner, toolpath_object, loadingContext):
+ super(ArvadosExpressionTool, self).__init__(toolpath_object, loadingContext)
+ self.arvrunner = arvrunner
+
+ def job(self,
+ job_order, # type: Mapping[Text, Text]
+ output_callback, # type: Callable[[Any, Any], Any]
+ runtimeContext # type: RuntimeContext
+ ):
+ return super(ArvadosExpressionTool, self).job(job_order, self.arvrunner.get_wrapped_callback(output_callback), runtimeContext)
from cwltool.pack import pack
from cwltool.load_tool import fetch_document
from cwltool.process import shortname
-from cwltool.workflow import Workflow, WorkflowException
+from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
-from cwltool.builder import Builder
from cwltool.context import LoadingContext
import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
from .pathmapper import ArvPathMapper, trim_listing
-from .arvtool import ArvadosCommandTool
+from .arvtool import ArvadosCommandTool, set_cluster_target, make_builder
from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
overall_res_req["class"] = "ResourceRequirement"
return cmap(overall_res_req)
+class ArvadosWorkflowStep(WorkflowStep):
+ def __init__(self,
+ toolpath_object, # type: Dict[Text, Any]
+ pos, # type: int
+ loadingContext, # type: LoadingContext
+ arvrunner,
+ *argc,
+ **argv
+ ): # type: (...) -> None
+
+ super(ArvadosWorkflowStep, self).__init__(toolpath_object, pos, loadingContext, *argc, **argv)
+ self.tool["class"] = "WorkflowStep"
+ self.arvrunner = arvrunner
+
+ def job(self, joborder, output_callback, runtimeContext):
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.toplevel = True # Preserve behavior for #13365
+
+ builder = make_builder({shortname(k): v for k,v in joborder.items()}, self.hints, self.requirements, runtimeContext)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
+ return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
+
+
class ArvadosWorkflow(Workflow):
"""Wrap cwltool Workflow to override selected methods."""
def __init__(self, arvrunner, toolpath_object, loadingContext):
- super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
self.arvrunner = arvrunner
self.wf_pdh = None
self.dynamic_resource_req = []
self.static_resource_req = []
self.wf_reffiles = []
self.loadingContext = loadingContext
+ super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
+ self.cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
def job(self, joborder, output_callback, runtimeContext):
+
+ builder = make_builder(joborder, self.hints, self.requirements, runtimeContext)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
+
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
- if req:
- with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
- if "id" not in self.tool:
- raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
- document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
+ if not req:
+ return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
- discover_secondary_files(self.tool["inputs"], joborder)
+ # RunInSingleContainer is true
+
+ with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
+ if "id" not in self.tool:
+ raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
+ document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
+
+ discover_secondary_files(self.tool["inputs"], joborder)
+
+ with Perf(metrics, "subworkflow upload_deps"):
+ upload_dependencies(self.arvrunner,
+ os.path.basename(joborder.get("id", "#")),
+ document_loader,
+ joborder,
+ joborder.get("id", "#"),
+ False)
+
+ if self.wf_pdh is None:
+ workflowobj["requirements"] = dedup_reqs(self.requirements)
+ workflowobj["hints"] = dedup_reqs(self.hints)
+
+ packed = pack(document_loader, workflowobj, uri, self.metadata)
+
+ def visit(item):
+ for t in ("hints", "requirements"):
+ if t not in item:
+ continue
+ for req in item[t]:
+ if req["class"] == "ResourceRequirement":
+ dyn = False
+ for k in max_res_pars + sum_res_pars:
+ if k in req:
+ if isinstance(req[k], basestring):
+ if item["id"] == "#main":
+ # only the top-level requirements/hints may contain expressions
+ self.dynamic_resource_req.append(req)
+ dyn = True
+ break
+ else:
+ with SourceLine(req, k, WorkflowException):
+ raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
+ if not dyn:
+ self.static_resource_req.append(req)
+ if req["class"] == "DockerRequirement":
+ if "http://arvados.org/cwl#dockerCollectionPDH" in req:
+ del req["http://arvados.org/cwl#dockerCollectionPDH"]
+
+ visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
+
+ if self.static_resource_req:
+ self.static_resource_req = [get_overall_res_req(self.static_resource_req)]
- with Perf(metrics, "subworkflow upload_deps"):
upload_dependencies(self.arvrunner,
- os.path.basename(joborder.get("id", "#")),
+ runtimeContext.name,
document_loader,
- joborder,
- joborder.get("id", "#"),
+ packed,
+ uri,
False)
- if self.wf_pdh is None:
- workflowobj["requirements"] = dedup_reqs(self.requirements)
- workflowobj["hints"] = dedup_reqs(self.hints)
-
- packed = pack(document_loader, workflowobj, uri, self.metadata)
-
- builder = Builder(joborder,
- requirements=workflowobj["requirements"],
- hints=workflowobj["hints"],
- resources={})
-
- def visit(item):
- for t in ("hints", "requirements"):
- if t not in item:
- continue
- for req in item[t]:
- if req["class"] == "ResourceRequirement":
- dyn = False
- for k in max_res_pars + sum_res_pars:
- if k in req:
- if isinstance(req[k], basestring):
- if item["id"] == "#main":
- # only the top-level requirements/hints may contain expressions
- self.dynamic_resource_req.append(req)
- dyn = True
- break
- else:
- with SourceLine(req, k, WorkflowException):
- raise WorkflowException("Non-top-level ResourceRequirement in single container cannot have expressions")
- if not dyn:
- self.static_resource_req.append(req)
-
- visit_class(packed["$graph"], ("Workflow", "CommandLineTool"), visit)
-
- if self.static_resource_req:
- self.static_resource_req = [get_overall_res_req(self.static_resource_req)]
-
- upload_dependencies(self.arvrunner,
- runtimeContext.name,
- document_loader,
- packed,
- uri,
- False)
-
- # Discover files/directories referenced by the
- # workflow (mainly "default" values)
- visit_class(packed, ("File", "Directory"), self.wf_reffiles.append)
-
-
- if self.dynamic_resource_req:
- builder = Builder(joborder,
- requirements=self.requirements,
- hints=self.hints,
- resources={})
-
- # Evaluate dynamic resource requirements using current builder
- rs = copy.copy(self.static_resource_req)
- for dyn_rs in self.dynamic_resource_req:
- eval_req = {"class": "ResourceRequirement"}
- for a in max_res_pars + sum_res_pars:
- if a in dyn_rs:
- eval_req[a] = builder.do_eval(dyn_rs[a])
- rs.append(eval_req)
- job_res_reqs = [get_overall_res_req(rs)]
- else:
- job_res_reqs = self.static_resource_req
-
- with Perf(metrics, "subworkflow adjust"):
- joborder_resolved = copy.deepcopy(joborder)
- joborder_keepmount = copy.deepcopy(joborder)
-
- reffiles = []
- visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
-
- mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, runtimeContext.basedir,
- "/keep/%s",
- "/keep/%s/%s")
-
- # For containers API, we need to make sure any extra
- # referenced files (ie referenced by the workflow but
- # not in the inputs) are included in the mounts.
- if self.wf_reffiles:
- runtimeContext = runtimeContext.copy()
- runtimeContext.extra_reffiles = copy.deepcopy(self.wf_reffiles)
-
- def keepmount(obj):
- remove_redundant_fields(obj)
- with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
- if "location" not in obj:
- raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
- with SourceLine(obj, "location", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
- if obj["location"].startswith("keep:"):
- obj["location"] = mapper.mapper(obj["location"]).target
- if "listing" in obj:
- del obj["listing"]
- elif obj["location"].startswith("_:"):
- del obj["location"]
- else:
- raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
-
- visit_class(joborder_keepmount, ("File", "Directory"), keepmount)
-
- def resolved(obj):
- if obj["location"].startswith("keep:"):
- obj["location"] = mapper.mapper(obj["location"]).resolved
-
- visit_class(joborder_resolved, ("File", "Directory"), resolved)
-
- if self.wf_pdh is None:
- adjustFileObjs(packed, keepmount)
- adjustDirObjs(packed, keepmount)
- self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
-
- wf_runner = cmap({
- "class": "CommandLineTool",
- "baseCommand": "cwltool",
- "inputs": self.tool["inputs"],
- "outputs": self.tool["outputs"],
- "stdout": "cwl.output.json",
- "requirements": self.requirements+job_res_reqs+[
- {"class": "InlineJavascriptRequirement"},
- {
- "class": "InitialWorkDirRequirement",
- "listing": [{
- "entryname": "workflow.cwl",
- "entry": '$({"class": "File", "location": "keep:%s/workflow.cwl"})' % self.wf_pdh
- }, {
- "entryname": "cwl.input.yml",
- "entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
- }]
- }],
- "hints": self.hints,
- "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
- "id": "#"
- })
- return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+ # Discover files/directories referenced by the
+ # workflow (mainly "default" values)
+ visit_class(packed, ("File", "Directory"), self.wf_reffiles.append)
+
+
+ if self.dynamic_resource_req:
+ # Evaluate dynamic resource requirements using current builder
+ rs = copy.copy(self.static_resource_req)
+ for dyn_rs in self.dynamic_resource_req:
+ eval_req = {"class": "ResourceRequirement"}
+ for a in max_res_pars + sum_res_pars:
+ if a in dyn_rs:
+ eval_req[a] = builder.do_eval(dyn_rs[a])
+ rs.append(eval_req)
+ job_res_reqs = [get_overall_res_req(rs)]
else:
- return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
+ job_res_reqs = self.static_resource_req
+
+ with Perf(metrics, "subworkflow adjust"):
+ joborder_resolved = copy.deepcopy(joborder)
+ joborder_keepmount = copy.deepcopy(joborder)
+
+ reffiles = []
+ visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
+
+ mapper = ArvPathMapper(self.arvrunner, reffiles+self.wf_reffiles, runtimeContext.basedir,
+ "/keep/%s",
+ "/keep/%s/%s")
+
+ # For containers API, we need to make sure any extra
+ # referenced files (ie referenced by the workflow but
+ # not in the inputs) are included in the mounts.
+ if self.wf_reffiles:
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.extra_reffiles = copy.deepcopy(self.wf_reffiles)
+
+ def keepmount(obj):
+ remove_redundant_fields(obj)
+ with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
+ if "location" not in obj:
+ raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
+ with SourceLine(obj, "location", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
+ if obj["location"].startswith("keep:"):
+ obj["location"] = mapper.mapper(obj["location"]).target
+ if "listing" in obj:
+ del obj["listing"]
+ elif obj["location"].startswith("_:"):
+ del obj["location"]
+ else:
+ raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
+
+ visit_class(joborder_keepmount, ("File", "Directory"), keepmount)
+
+ def resolved(obj):
+ if obj["location"].startswith("keep:"):
+ obj["location"] = mapper.mapper(obj["location"]).resolved
+
+ visit_class(joborder_resolved, ("File", "Directory"), resolved)
+
+ if self.wf_pdh is None:
+ adjustFileObjs(packed, keepmount)
+ adjustDirObjs(packed, keepmount)
+ self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
+
+ wf_runner = cmap({
+ "class": "CommandLineTool",
+ "baseCommand": "cwltool",
+ "inputs": self.tool["inputs"],
+ "outputs": self.tool["outputs"],
+ "stdout": "cwl.output.json",
+ "requirements": self.requirements+job_res_reqs+[
+ {"class": "InlineJavascriptRequirement"},
+ {
+ "class": "InitialWorkDirRequirement",
+ "listing": [{
+ "entryname": "workflow.cwl",
+ "entry": '$({"class": "File", "location": "keep:%s/workflow.cwl"})' % self.wf_pdh
+ }, {
+ "entryname": "cwl.input.yml",
+ "entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
+ }]
+ }],
+ "hints": self.hints,
+ "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
+ "id": "#"
+ })
+ return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+
+ def make_workflow_step(self,
+ toolpath_object, # type: Dict[Text, Any]
+ pos, # type: int
+ loadingContext, # type: LoadingContext
+ *argc,
+ **argv
+ ):
+ # (...) -> WorkflowStep
+ return ArvadosWorkflowStep(toolpath_object, pos, loadingContext, self.arvrunner, *argc, **argv)
# SPDX-License-Identifier: Apache-2.0
from cwltool.context import LoadingContext, RuntimeContext
+from collections import namedtuple
class ArvLoadingContext(LoadingContext):
def __init__(self, kwargs=None):
self.storage_classes = "default"
self.current_container = None
self.http_timeout = 300
+ self.submit_runner_cluster = None
+ self.cluster_target_id = 0
+ self.always_submit_runner = False
+ self.collection_cache_size = 256
super(ArvRuntimeContext, self).__init__(kwargs)
+
+ if self.submit_request_uuid:
+ self.submit_runner_cluster = self.submit_request_uuid[0:5]
arvargs.output_tags = output_tags
arvargs.thread_count = 1
- runner = arvados_cwl.ArvCwlRunner(api_client=arvados.safeapi.ThreadSafeApiCache(
+ runner = arvados_cwl.ArvCwlExecutor(api_client=arvados.safeapi.ThreadSafeApiCache(
api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}),
arvargs=arvargs)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import logging
+import os
+import sys
+import threading
+import copy
+import json
+import re
+from functools import partial
+import time
+
+from cwltool.errors import WorkflowException
+import cwltool.workflow
+from schema_salad.sourceline import SourceLine
+import schema_salad.validate as validate
+
+import arvados
+import arvados.config
+from arvados.keep import KeepClient
+from arvados.errors import ApiError
+
+import arvados_cwl.util
+from .arvcontainer import RunnerContainer
+from .arvjob import RunnerJob, RunnerTemplate
+from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
+from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool
+from .arvworkflow import ArvadosWorkflow, upload_workflow
+from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
+from .perf import Perf
+from .pathmapper import NoFollowPathMapper
+from .task_queue import TaskQueue
+from .context import ArvLoadingContext, ArvRuntimeContext
+from ._version import __version__
+
+from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
+from cwltool.command_line_tool import compute_checksums
+
+logger = logging.getLogger('arvados.cwl-runner')
+metrics = logging.getLogger('arvados.cwl-runner.metrics')
+
+DEFAULT_PRIORITY = 500
+
+class RuntimeStatusLoggingHandler(logging.Handler):
+ """
+ Intercepts logging calls and report them as runtime statuses on runner
+ containers.
+ """
+ def __init__(self, runtime_status_update_func):
+ super(RuntimeStatusLoggingHandler, self).__init__()
+ self.runtime_status_update = runtime_status_update_func
+
+ def emit(self, record):
+ kind = None
+ if record.levelno >= logging.ERROR:
+ kind = 'error'
+ elif record.levelno >= logging.WARNING:
+ kind = 'warning'
+ if kind is not None:
+ log_msg = record.getMessage()
+ if '\n' in log_msg:
+ # If the logged message is multi-line, use its first line as status
+ # and the rest as detail.
+ status, detail = log_msg.split('\n', 1)
+ self.runtime_status_update(
+ kind,
+ "%s: %s" % (record.name, status),
+ detail
+ )
+ else:
+ self.runtime_status_update(
+ kind,
+ "%s: %s" % (record.name, record.getMessage())
+ )
+
+class ArvCwlExecutor(object):
+ """Execute a CWL tool or workflow, submit work (using either jobs or
+ containers API), wait for them to complete, and report output.
+
+ """
+
+ def __init__(self, api_client,
+ arvargs=None,
+ keep_client=None,
+ num_retries=4,
+ thread_count=4):
+
+ if arvargs is None:
+ arvargs = argparse.Namespace()
+ arvargs.work_api = None
+ arvargs.output_name = None
+ arvargs.output_tags = None
+ arvargs.thread_count = 1
+ arvargs.collection_cache_size = None
+
+ self.api = api_client
+ self.processes = {}
+ self.workflow_eval_lock = threading.Condition(threading.RLock())
+ self.final_output = None
+ self.final_status = None
+ self.num_retries = num_retries
+ self.uuid = None
+ self.stop_polling = threading.Event()
+ self.poll_api = None
+ self.pipeline = None
+ self.final_output_collection = None
+ self.output_name = arvargs.output_name
+ self.output_tags = arvargs.output_tags
+ self.project_uuid = None
+ self.intermediate_output_ttl = 0
+ self.intermediate_output_collections = []
+ self.trash_intermediate = False
+ self.thread_count = arvargs.thread_count
+ self.poll_interval = 12
+ self.loadingContext = None
+ self.should_estimate_cache_size = True
+
+ if keep_client is not None:
+ self.keep_client = keep_client
+ else:
+ self.keep_client = arvados.keep.KeepClient(api_client=self.api, num_retries=self.num_retries)
+
+ if arvargs.collection_cache_size:
+ collection_cache_size = arvargs.collection_cache_size*1024*1024
+ self.should_estimate_cache_size = False
+ else:
+ collection_cache_size = 256*1024*1024
+
+ self.collection_cache = CollectionCache(self.api, self.keep_client, self.num_retries,
+ cap=collection_cache_size)
+
+ self.fetcher_constructor = partial(CollectionFetcher,
+ api_client=self.api,
+ fs_access=CollectionFsAccess("", collection_cache=self.collection_cache),
+ num_retries=self.num_retries)
+
+ self.work_api = None
+ expected_api = ["jobs", "containers"]
+ for api in expected_api:
+ try:
+ methods = self.api._rootDesc.get('resources')[api]['methods']
+ if ('httpMethod' in methods['create'] and
+ (arvargs.work_api == api or arvargs.work_api is None)):
+ self.work_api = api
+ break
+ except KeyError:
+ pass
+
+ if not self.work_api:
+ if arvargs.work_api is None:
+ raise Exception("No supported APIs")
+ else:
+ raise Exception("Unsupported API '%s', expected one of %s" % (arvargs.work_api, expected_api))
+
+ if self.work_api == "jobs":
+ logger.warn("""
+*******************************
+Using the deprecated 'jobs' API.
+
+To get rid of this warning:
+
+Users: read about migrating at
+http://doc.arvados.org/user/cwl/cwl-style.html#migrate
+and use the option --api=containers
+
+Admins: configure the cluster to disable the 'jobs' API as described at:
+http://doc.arvados.org/install/install-api-server.html#disable_api_methods
+*******************************""")
+
+ self.loadingContext = ArvLoadingContext(vars(arvargs))
+ self.loadingContext.fetcher_constructor = self.fetcher_constructor
+ self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
+ self.loadingContext.construct_tool_object = self.arv_make_tool
+
+ # Add a custom logging handler to the root logger for runtime status reporting
+ # if running inside a container
+ if arvados_cwl.util.get_current_container(self.api, self.num_retries, logger):
+ root_logger = logging.getLogger('')
+ handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
+ root_logger.addHandler(handler)
+
+ self.runtimeContext = ArvRuntimeContext(vars(arvargs))
+ self.runtimeContext.make_fs_access = partial(CollectionFsAccess,
+ collection_cache=self.collection_cache)
+
+ validate_cluster_target(self, self.runtimeContext)
+
+
+ def arv_make_tool(self, toolpath_object, loadingContext):
+ if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
+ return ArvadosCommandTool(self, toolpath_object, loadingContext)
+ elif "class" in toolpath_object and toolpath_object["class"] == "Workflow":
+ return ArvadosWorkflow(self, toolpath_object, loadingContext)
+ elif "class" in toolpath_object and toolpath_object["class"] == "ExpressionTool":
+ return ArvadosExpressionTool(self, toolpath_object, loadingContext)
+ else:
+ raise Exception("Unknown tool %s" % toolpath_object.get("class"))
+
+ def output_callback(self, out, processStatus):
+ with self.workflow_eval_lock:
+ if processStatus == "success":
+ logger.info("Overall process status is %s", processStatus)
+ state = "Complete"
+ else:
+ logger.error("Overall process status is %s", processStatus)
+ state = "Failed"
+ if self.pipeline:
+ self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
+ body={"state": state}).execute(num_retries=self.num_retries)
+ self.final_status = processStatus
+ self.final_output = out
+ self.workflow_eval_lock.notifyAll()
+
+
+ def start_run(self, runnable, runtimeContext):
+ self.task_queue.add(partial(runnable.run, runtimeContext),
+ self.workflow_eval_lock, self.stop_polling)
+
+ def process_submitted(self, container):
+ with self.workflow_eval_lock:
+ self.processes[container.uuid] = container
+
+ def process_done(self, uuid, record):
+ with self.workflow_eval_lock:
+ j = self.processes[uuid]
+ logger.info("%s %s is %s", self.label(j), uuid, record["state"])
+ self.task_queue.add(partial(j.done, record),
+ self.workflow_eval_lock, self.stop_polling)
+ del self.processes[uuid]
+
+ def runtime_status_update(self, kind, message, detail=None):
+ """
+ Updates the runtime_status field on the runner container.
+ Called when there's a need to report errors, warnings or just
+ activity statuses, for example in the RuntimeStatusLoggingHandler.
+ """
+ with self.workflow_eval_lock:
+ current = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
+ if current is None:
+ return
+ runtime_status = current.get('runtime_status', {})
+ # In case of status being an error, only report the first one.
+ if kind == 'error':
+ if not runtime_status.get('error'):
+ runtime_status.update({
+ 'error': message
+ })
+ if detail is not None:
+ runtime_status.update({
+ 'errorDetail': detail
+ })
+ # Further errors are only mentioned as a count.
+ else:
+ # Get anything before an optional 'and N more' string.
+ try:
+ error_msg = re.match(
+ r'^(.*?)(?=\s*\(and \d+ more\)|$)', runtime_status.get('error')).groups()[0]
+ more_failures = re.match(
+ r'.*\(and (\d+) more\)', runtime_status.get('error'))
+ except TypeError:
+ # Ignore tests stubbing errors
+ return
+ if more_failures:
+ failure_qty = int(more_failures.groups()[0])
+ runtime_status.update({
+ 'error': "%s (and %d more)" % (error_msg, failure_qty+1)
+ })
+ else:
+ runtime_status.update({
+ 'error': "%s (and 1 more)" % error_msg
+ })
+ elif kind in ['warning', 'activity']:
+ # Record the last warning/activity status without regard of
+ # previous occurences.
+ runtime_status.update({
+ kind: message
+ })
+ if detail is not None:
+ runtime_status.update({
+ kind+"Detail": detail
+ })
+ else:
+ # Ignore any other status kind
+ return
+ try:
+ self.api.containers().update(uuid=current['uuid'],
+ body={
+ 'runtime_status': runtime_status,
+ }).execute(num_retries=self.num_retries)
+ except Exception as e:
+ logger.info("Couldn't update runtime_status: %s", e)
+
+ def wrapped_callback(self, cb, obj, st):
+ with self.workflow_eval_lock:
+ cb(obj, st)
+ self.workflow_eval_lock.notifyAll()
+
+ def get_wrapped_callback(self, cb):
+ return partial(self.wrapped_callback, cb)
+
+ def on_message(self, event):
+ if event.get("object_uuid") in self.processes and event["event_type"] == "update":
+ uuid = event["object_uuid"]
+ if event["properties"]["new_attributes"]["state"] == "Running":
+ with self.workflow_eval_lock:
+ j = self.processes[uuid]
+ if j.running is False:
+ j.running = True
+ j.update_pipeline_component(event["properties"]["new_attributes"])
+ logger.info("%s %s is Running", self.label(j), uuid)
+ elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled", "Final"):
+ self.process_done(uuid, event["properties"]["new_attributes"])
+
+ def label(self, obj):
+ return "[%s %s]" % (self.work_api[0:-1], obj.name)
+
+ def poll_states(self):
+ """Poll status of jobs or containers listed in the processes dict.
+
+ Runs in a separate thread.
+ """
+
+ try:
+ remain_wait = self.poll_interval
+ while True:
+ if remain_wait > 0:
+ self.stop_polling.wait(remain_wait)
+ if self.stop_polling.is_set():
+ break
+ with self.workflow_eval_lock:
+ keys = list(self.processes.keys())
+ if not keys:
+ remain_wait = self.poll_interval
+ continue
+
+ begin_poll = time.time()
+ if self.work_api == "containers":
+ table = self.poll_api.container_requests()
+ elif self.work_api == "jobs":
+ table = self.poll_api.jobs()
+
+ pageSize = self.poll_api._rootDesc.get('maxItemsPerResponse', 1000)
+
+ while keys:
+ page = keys[:pageSize]
+ keys = keys[pageSize:]
+ try:
+ proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries)
+ except Exception as e:
+ logger.warn("Error checking states on API server: %s", e)
+ remain_wait = self.poll_interval
+ continue
+
+ for p in proc_states["items"]:
+ self.on_message({
+ "object_uuid": p["uuid"],
+ "event_type": "update",
+ "properties": {
+ "new_attributes": p
+ }
+ })
+ finish_poll = time.time()
+ remain_wait = self.poll_interval - (finish_poll - begin_poll)
+ except:
+ logger.exception("Fatal error in state polling thread.")
+ with self.workflow_eval_lock:
+ self.processes.clear()
+ self.workflow_eval_lock.notifyAll()
+ finally:
+ self.stop_polling.set()
+
+ def add_intermediate_output(self, uuid):
+ if uuid:
+ self.intermediate_output_collections.append(uuid)
+
+ def trash_intermediate_output(self):
+ logger.info("Cleaning up intermediate output collections")
+ for i in self.intermediate_output_collections:
+ try:
+ self.api.collections().delete(uuid=i).execute(num_retries=self.num_retries)
+ except:
+ logger.warn("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
+ if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
+ break
+
+ def check_features(self, obj):
+ if isinstance(obj, dict):
+ if obj.get("writable") and self.work_api != "containers":
+ raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
+ if obj.get("class") == "DockerRequirement":
+ if obj.get("dockerOutputDirectory"):
+ if self.work_api != "containers":
+ raise SourceLine(obj, "dockerOutputDirectory", UnsupportedRequirement).makeError(
+ "Option 'dockerOutputDirectory' of DockerRequirement not supported with --api=jobs.")
+ if not obj.get("dockerOutputDirectory").startswith('/'):
+ raise SourceLine(obj, "dockerOutputDirectory", validate.ValidationException).makeError(
+ "Option 'dockerOutputDirectory' must be an absolute path.")
+ if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
+ raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
+ for v in obj.itervalues():
+ self.check_features(v)
+ elif isinstance(obj, list):
+ for i,v in enumerate(obj):
+ with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
+ self.check_features(v)
+
+ def make_output_collection(self, name, storage_classes, tagsString, outputObj):
+ outputObj = copy.deepcopy(outputObj)
+
+ files = []
+ def capture(fileobj):
+ files.append(fileobj)
+
+ adjustDirObjs(outputObj, capture)
+ adjustFileObjs(outputObj, capture)
+
+ generatemapper = NoFollowPathMapper(files, "", "", separateDirs=False)
+
+ final = arvados.collection.Collection(api_client=self.api,
+ keep_client=self.keep_client,
+ num_retries=self.num_retries)
+
+ for k,v in generatemapper.items():
+ if k.startswith("_:"):
+ if v.type == "Directory":
+ continue
+ if v.type == "CreateFile":
+ with final.open(v.target, "wb") as f:
+ f.write(v.resolved.encode("utf-8"))
+ continue
+
+ if not k.startswith("keep:"):
+ raise Exception("Output source is not in keep or a literal")
+ sp = k.split("/")
+ srccollection = sp[0][5:]
+ try:
+ reader = self.collection_cache.get(srccollection)
+ srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
+ final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
+ except arvados.errors.ArgumentError as e:
+ logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
+ raise
+ except IOError as e:
+ logger.warn("While preparing output collection: %s", e)
+
+ def rewrite(fileobj):
+ fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
+ for k in ("listing", "contents", "nameext", "nameroot", "dirname"):
+ if k in fileobj:
+ del fileobj[k]
+
+ adjustDirObjs(outputObj, rewrite)
+ adjustFileObjs(outputObj, rewrite)
+
+ with final.open("cwl.output.json", "w") as f:
+ json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
+
+ final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
+
+ logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
+ final.api_response()["name"],
+ final.manifest_locator())
+
+ final_uuid = final.manifest_locator()
+ tags = tagsString.split(',')
+ for tag in tags:
+ self.api.links().create(body={
+ "head_uuid": final_uuid, "link_class": "tag", "name": tag
+ }).execute(num_retries=self.num_retries)
+
+ def finalcollection(fileobj):
+ fileobj["location"] = "keep:%s/%s" % (final.portable_data_hash(), fileobj["location"])
+
+ adjustDirObjs(outputObj, finalcollection)
+ adjustFileObjs(outputObj, finalcollection)
+
+ return (outputObj, final)
+
+ def set_crunch_output(self):
+ if self.work_api == "containers":
+ current = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
+ if current is None:
+ return
+ try:
+ self.api.containers().update(uuid=current['uuid'],
+ body={
+ 'output': self.final_output_collection.portable_data_hash(),
+ }).execute(num_retries=self.num_retries)
+ self.api.collections().update(uuid=self.final_output_collection.manifest_locator(),
+ body={
+ 'is_trashed': True
+ }).execute(num_retries=self.num_retries)
+ except Exception as e:
+ logger.info("Setting container output: %s", e)
+ elif self.work_api == "jobs" and "TASK_UUID" in os.environ:
+ self.api.job_tasks().update(uuid=os.environ["TASK_UUID"],
+ body={
+ 'output': self.final_output_collection.portable_data_hash(),
+ 'success': self.final_status == "success",
+ 'progress':1.0
+ }).execute(num_retries=self.num_retries)
+
+ def arv_executor(self, tool, job_order, runtimeContext, logger=None):
+ self.debug = runtimeContext.debug
+
+ tool.visit(self.check_features)
+
+ self.project_uuid = runtimeContext.project_uuid
+ self.pipeline = None
+ self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
+ self.secret_store = runtimeContext.secret_store
+
+ self.trash_intermediate = runtimeContext.trash_intermediate
+ if self.trash_intermediate and self.work_api != "containers":
+ raise Exception("--trash-intermediate is only supported with --api=containers.")
+
+ self.intermediate_output_ttl = runtimeContext.intermediate_output_ttl
+ if self.intermediate_output_ttl and self.work_api != "containers":
+ raise Exception("--intermediate-output-ttl is only supported with --api=containers.")
+ if self.intermediate_output_ttl < 0:
+ raise Exception("Invalid value %d for --intermediate-output-ttl, cannot be less than zero" % self.intermediate_output_ttl)
+
+ if runtimeContext.submit_request_uuid and self.work_api != "containers":
+ raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api))
+
+ if not runtimeContext.name:
+ runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
+
+ # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
+ # Also uploads docker images.
+ merged_map = upload_workflow_deps(self, tool)
+
+ # Reload tool object which may have been updated by
+ # upload_workflow_deps
+ # Don't validate this time because it will just print redundant errors.
+ loadingContext = self.loadingContext.copy()
+ loadingContext.loader = tool.doc_loader
+ loadingContext.avsc_names = tool.doc_schema
+ loadingContext.metadata = tool.metadata
+ loadingContext.do_validate = False
+
+ tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
+ loadingContext)
+
+ # Upload local file references in the job order.
+ job_order = upload_job_order(self, "%s input" % runtimeContext.name,
+ tool, job_order)
+
+ existing_uuid = runtimeContext.update_workflow
+ if existing_uuid or runtimeContext.create_workflow:
+ # Create a pipeline template or workflow record and exit.
+ if self.work_api == "jobs":
+ tmpl = RunnerTemplate(self, tool, job_order,
+ runtimeContext.enable_reuse,
+ uuid=existing_uuid,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ merged_map=merged_map)
+ tmpl.save()
+ # cwltool.main will write our return value to stdout.
+ return (tmpl.uuid, "success")
+ elif self.work_api == "containers":
+ return (upload_workflow(self, tool, job_order,
+ self.project_uuid,
+ uuid=existing_uuid,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ merged_map=merged_map),
+ "success")
+
+ self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse
+ self.eval_timeout = runtimeContext.eval_timeout
+
+ runtimeContext = runtimeContext.copy()
+ runtimeContext.use_container = True
+ runtimeContext.tmpdir_prefix = "tmp"
+ runtimeContext.work_api = self.work_api
+
+ if self.work_api == "containers":
+ if self.ignore_docker_for_reuse:
+ raise Exception("--ignore-docker-for-reuse not supported with containers API.")
+ runtimeContext.outdir = "/var/spool/cwl"
+ runtimeContext.docker_outdir = "/var/spool/cwl"
+ runtimeContext.tmpdir = "/tmp"
+ runtimeContext.docker_tmpdir = "/tmp"
+ elif self.work_api == "jobs":
+ if runtimeContext.priority != DEFAULT_PRIORITY:
+ raise Exception("--priority not implemented for jobs API.")
+ runtimeContext.outdir = "$(task.outdir)"
+ runtimeContext.docker_outdir = "$(task.outdir)"
+ runtimeContext.tmpdir = "$(task.tmpdir)"
+
+ if runtimeContext.priority < 1 or runtimeContext.priority > 1000:
+ raise Exception("--priority must be in the range 1..1000.")
+
+ if self.should_estimate_cache_size:
+ visited = set()
+ estimated_size = [0]
+ def estimate_collection_cache(obj):
+ if obj.get("location", "").startswith("keep:"):
+ m = pdh_size.match(obj["location"][5:])
+ if m and m.group(1) not in visited:
+ visited.add(m.group(1))
+ estimated_size[0] += int(m.group(2))
+ visit_class(job_order, ("File", "Directory"), estimate_collection_cache)
+ runtimeContext.collection_cache_size = max(((estimated_size[0]*192) / (1024*1024))+1, 256)
+ self.collection_cache.set_cap(runtimeContext.collection_cache_size*1024*1024)
+
+ logger.info("Using collection cache size %s MiB", runtimeContext.collection_cache_size)
+
+ runnerjob = None
+ if runtimeContext.submit:
+ # Submit a runner job to run the workflow for us.
+ if self.work_api == "containers":
+ if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
+ runtimeContext.runnerjob = tool.tool["id"]
+ runnerjob = tool.job(job_order,
+ self.output_callback,
+ runtimeContext).next()
+ else:
+ runnerjob = RunnerContainer(self, tool, job_order, runtimeContext.enable_reuse,
+ self.output_name,
+ self.output_tags,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
+ intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
+ merged_map=merged_map,
+ priority=runtimeContext.priority,
+ secret_store=self.secret_store,
+ collection_cache_size=runtimeContext.collection_cache_size,
+ collection_cache_is_default=self.should_estimate_cache_size)
+ elif self.work_api == "jobs":
+ runnerjob = RunnerJob(self, tool, job_order, runtimeContext.enable_reuse,
+ self.output_name,
+ self.output_tags,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
+ merged_map=merged_map)
+ elif runtimeContext.cwl_runner_job is None and self.work_api == "jobs":
+ # Create pipeline for local run
+ self.pipeline = self.api.pipeline_instances().create(
+ body={
+ "owner_uuid": self.project_uuid,
+ "name": runtimeContext.name if runtimeContext.name else shortname(tool.tool["id"]),
+ "components": {},
+ "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
+ logger.info("Pipeline instance %s", self.pipeline["uuid"])
+
+ if runnerjob and not runtimeContext.wait:
+ submitargs = runtimeContext.copy()
+ submitargs.submit = False
+ runnerjob.run(submitargs)
+ return (runnerjob.uuid, "success")
+
+ current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger)
+ if current_container:
+ logger.info("Running inside container %s", current_container.get("uuid"))
+
+ self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout)
+ self.polling_thread = threading.Thread(target=self.poll_states)
+ self.polling_thread.start()
+
+ self.task_queue = TaskQueue(self.workflow_eval_lock, self.thread_count)
+
+ try:
+ self.workflow_eval_lock.acquire()
+ if runnerjob:
+ jobiter = iter((runnerjob,))
+ else:
+ if runtimeContext.cwl_runner_job is not None:
+ self.uuid = runtimeContext.cwl_runner_job.get('uuid')
+ jobiter = tool.job(job_order,
+ self.output_callback,
+ runtimeContext)
+
+ # Holds the lock while this code runs and releases it when
+ # it is safe to do so in self.workflow_eval_lock.wait(),
+ # at which point on_message can update job state and
+ # process output callbacks.
+
+ loopperf = Perf(metrics, "jobiter")
+ loopperf.__enter__()
+ for runnable in jobiter:
+ loopperf.__exit__()
+
+ if self.stop_polling.is_set():
+ break
+
+ if self.task_queue.error is not None:
+ raise self.task_queue.error
+
+ if runnable:
+ with Perf(metrics, "run"):
+ self.start_run(runnable, runtimeContext)
+ else:
+ if (self.task_queue.in_flight + len(self.processes)) > 0:
+ self.workflow_eval_lock.wait(3)
+ else:
+ logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
+ break
+
+ if self.stop_polling.is_set():
+ break
+
+ loopperf.__enter__()
+ loopperf.__exit__()
+
+ while (self.task_queue.in_flight + len(self.processes)) > 0:
+ if self.task_queue.error is not None:
+ raise self.task_queue.error
+ self.workflow_eval_lock.wait(3)
+
+ except UnsupportedRequirement:
+ raise
+ except:
+ if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
+ logger.error("Interrupted, workflow will be cancelled")
+ else:
+ logger.error("Execution failed:\n%s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
+ if self.pipeline:
+ self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
+ body={"state": "Failed"}).execute(num_retries=self.num_retries)
+ if runnerjob and runnerjob.uuid and self.work_api == "containers":
+ self.api.container_requests().update(uuid=runnerjob.uuid,
+ body={"priority": "0"}).execute(num_retries=self.num_retries)
+ finally:
+ self.workflow_eval_lock.release()
+ self.task_queue.drain()
+ self.stop_polling.set()
+ self.polling_thread.join()
+ self.task_queue.join()
+
+ if self.final_status == "UnsupportedRequirement":
+ raise UnsupportedRequirement("Check log for details.")
+
+ if self.final_output is None:
+ raise WorkflowException("Workflow did not return a result.")
+
+ if runtimeContext.submit and isinstance(runnerjob, Runner):
+ logger.info("Final output collection %s", runnerjob.final_output)
+ else:
+ if self.output_name is None:
+ self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
+ if self.output_tags is None:
+ self.output_tags = ""
+
+ storage_classes = runtimeContext.storage_classes.strip().split(",")
+ self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
+ self.set_crunch_output()
+
+ if runtimeContext.compute_checksum:
+ adjustDirObjs(self.final_output, partial(get_listing, self.fs_access))
+ adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
+
+ if self.trash_intermediate and self.final_status == "success":
+ self.trash_intermediate_output()
+
+ return (self.final_output, self.final_status)
logger = logging.getLogger('arvados.cwl-runner')
+pdh_size = re.compile(r'([0-9a-f]{32})\+(\d+)(\+\S+)*')
+
class CollectionCache(object):
def __init__(self, api_client, keep_client, num_retries,
cap=256*1024*1024,
self.cap = cap
self.min_entries = min_entries
- def cap_cache(self):
- if self.total > self.cap:
- # ordered list iterates from oldest to newest
- for pdh, v in self.collections.items():
- if self.total < self.cap or len(self.collections) < self.min_entries:
- break
- # cut it loose
- logger.debug("Evicting collection reader %s from cache", pdh)
- del self.collections[pdh]
- self.total -= v[1]
+ def set_cap(self, cap):
+ self.cap = cap
+
+ def cap_cache(self, required):
+ # ordered dict iterates from oldest to newest
+ for pdh, v in self.collections.items():
+ available = self.cap - self.total
+ if available >= required or len(self.collections) < self.min_entries:
+ return
+ # cut it loose
+ logger.debug("Evicting collection reader %s from cache (cap %s total %s required %s)", pdh, self.cap, self.total, required)
+ del self.collections[pdh]
+ self.total -= v[1]
def get(self, pdh):
with self.lock:
if pdh not in self.collections:
+ m = pdh_size.match(pdh)
+ if m:
+ self.cap_cache(int(m.group(2)) * 128)
logger.debug("Creating collection reader for %s", pdh)
cr = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
keep_client=self.keep_client,
sz = len(cr.manifest_text()) * 128
self.collections[pdh] = (cr, sz)
self.total += sz
- self.cap_cache()
else:
cr, sz = self.collections[pdh]
# bump it to the back
return super(CollectionFetcher, self).urljoin(base_url, url)
+ schemes = [u"file", u"http", u"https", u"mailto", u"keep", u"arvwf"]
+
+ def supported_schemes(self): # type: () -> List[Text]
+ return self.schemes
+
+
workflow_uuid_pattern = re.compile(r'[a-z0-9]{5}-7fd4e-[a-z0-9]{15}')
pipeline_template_uuid_pattern = re.compile(r'[a-z0-9]{5}-p5p6p-[a-z0-9]{15}')
import os
import urllib
-from arvados_cwl.util import get_current_container, get_intermediate_collection_info
+import arvados_cwl.util
import arvados.commands.run
import arvados.collection
for l in srcobj.get("listing", []):
self.addentry(l, c, ".", remap)
- container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
+ container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = arvados_cwl.util.get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
num_retries=self.arvrunner.num_retries )
self.addentry(srcobj, c, ".", remap)
- container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
+ container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = arvados_cwl.util.get_intermediate_collection_info(None, container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
import arvados.collection
import ruamel.yaml as yaml
-from .arvdocker import arv_docker_get_image
+import arvados_cwl.arvdocker
from .pathmapper import ArvPathMapper, trim_listing
from ._version import __version__
from . import done
# TODO: can be supported by containers API, but not jobs API.
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
- arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
else:
- arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
v["location"] = merged_map[cur_id].resolved[v["location"]]
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
+ if v.get("class") == "DockerRequirement":
+ v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, arvrunner.project_uuid)
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
try:
- arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
- return img
+
def upload_workflow_collection(arvrunner, name, packed):
collection = arvados.collection.Collection(api_client=arvrunner.api,
output_name, output_tags, submit_runner_ram=0,
name=None, on_error=None, submit_runner_image=None,
intermediate_output_ttl=0, merged_map=None,
- priority=None, secret_store=None):
+ priority=None, secret_store=None,
+ collection_cache_size=256,
+ collection_cache_is_default=True):
self.arvrunner = runner
self.tool = tool
self.job_order = job_order
self.submit_runner_cores = 1
self.submit_runner_ram = 1024 # defaut 1 GiB
+ self.collection_cache_size = collection_cache_size
runner_resource_req, _ = self.tool.get_requirement("http://arvados.org/cwl#WorkflowRunnerResources")
if runner_resource_req:
self.submit_runner_cores = runner_resource_req["coresMin"]
if runner_resource_req.get("ramMin"):
self.submit_runner_ram = runner_resource_req["ramMin"]
+ if runner_resource_req.get("keep_cache") and collection_cache_is_default:
+ self.collection_cache_size = runner_resource_req["keep_cache"]
if submit_runner_ram:
# Command line / initializer overrides default and/or spec from workflow
class TaskQueue(object):
def __init__(self, lock, thread_count):
self.thread_count = thread_count
- self.task_queue = Queue.Queue()
+ self.task_queue = Queue.Queue(maxsize=self.thread_count)
self.task_queue_threads = []
self.lock = lock
self.in_flight = 0
t.start()
def task_queue_func(self):
+ while True:
+ task = self.task_queue.get()
+ if task is None:
+ return
+ try:
+ task()
+ except Exception as e:
+ logger.exception("Unhandled exception running task")
+ self.error = e
- while True:
- task = self.task_queue.get()
- if task is None:
- return
- try:
- task()
- except Exception as e:
- logger.exception("Unhandled exception running task")
- self.error = e
-
- with self.lock:
- self.in_flight -= 1
-
- def add(self, task):
- with self.lock:
- if self.thread_count > 1:
+ with self.lock:
+ self.in_flight -= 1
+
+ def add(self, task, unlock, check_done):
+ if self.thread_count > 1:
+ with self.lock:
self.in_flight += 1
- self.task_queue.put(task)
- else:
- task()
+ else:
+ task()
+ return
+
+ while True:
+ try:
+ unlock.release()
+ if check_done.is_set():
+ return
+ self.task_queue.put(task, block=True, timeout=3)
+ return
+ except Queue.Full:
+ pass
+ finally:
+ unlock.acquire()
+
def drain(self):
try:
SETUP_DIR = os.path.dirname(__file__) or '.'
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def choose_version_from():
sdk_ts = subprocess.check_output(
from source package), leave it alone.
"""
def git_latest_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag(self):
gitinfo = subprocess.check_output(
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180806194258',
- 'schema-salad==2.7.20180719125426',
+ 'cwltool==1.0.20181116032456',
+ 'schema-salad==2.7.20181116024232',
'typing >= 3.6.4',
- # Need to limit ruamel.yaml version to 0.15.26 because of bug
- # https://bitbucket.org/ruamel/yaml/issues/227/regression-parsing-flow-mapping
- 'ruamel.yaml >=0.13.11, <= 0.15.26',
- 'arvados-python-client>=1.1.4.20180607143841',
+ 'ruamel.yaml >=0.15.54, <=0.15.77',
+ 'arvados-python-client>=1.2.1.20181130020805',
'setuptools',
'ciso8601 >=1.0.6, <2.0.0',
'subprocess32>=3.5.1',
arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz
fi
-exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum
+exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum --api=containers
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+Arvados federated workflow testing
+
+Requires cwltool 1.0.20181109150732 or later
+
+Create main-test.json:
+
+{
+ "acr": "/path/to/arvados-cwl-runner",
+ "arvado_api_host_insecure": false,
+ "arvados_api_hosts": [
+ "c97qk.arvadosapi.com",
+ "4xphq.arvadosapi.com",
+ "9tee4.arvadosapi.com"
+ ],
+ "arvados_api_token": "...",
+ "arvados_cluster_ids": [
+ "c97qk",
+ "4xphq",
+ "9tee4"
+ ]
+}
+
+Or create an arvbox test cluster:
+
+$ cwltool --enable-ext arvbox-make-federation.cwl --arvbox_base ~/.arvbox/ --in_acr /path/to/arvados-cwl-runner > main-test.json
+
+
+Run tests:
+
+$ cwltool main.cwl main-test.json
+
+
+List test cases:
+
+$ cwltool --print-targets main.cwl
+
+
+Run a specific test case:
+
+$ cwltool -t twostep-remote-copy-to-home main.cwl main-test.json
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+inputs:
+ containers:
+ type: string[]
+ default: [fedbox1, fedbox2, fedbox3]
+ arvbox_base: Directory
+ in_acr: string?
+ insecure:
+ type: boolean
+ default: true
+outputs:
+ arvados_api_token:
+ type: string
+ outputSource: setup-user/test_user_token
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/container_host
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/cluster_id
+ acr:
+ type: string?
+ outputSource: in_acr
+ arvado_api_host_insecure:
+ type: boolean
+ outputSource: insecure
+steps:
+ mkdir:
+ in:
+ containers: containers
+ arvbox_base: arvbox_base
+ out: [arvbox_data]
+ run: arvbox/mkdir.cwl
+ start:
+ in:
+ container_name: containers
+ arvbox_data: mkdir/arvbox_data
+ out: [cluster_id, container_host, arvbox_data_out, superuser_token]
+ scatter: [container_name, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox/start.cwl
+ fed-config:
+ in:
+ container_name: containers
+ this_cluster_id: start/cluster_id
+ cluster_ids: start/cluster_id
+ cluster_hosts: start/container_host
+ arvbox_data: start/arvbox_data_out
+ out: []
+ scatter: [container_name, this_cluster_id, arvbox_data]
+ scatterMethod: dotproduct
+ run: arvbox/fed-config.cwl
+ setup-user:
+ in:
+ container_host: {source: start/container_host, valueFrom: "$(self[0])"}
+ superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"}
+ out: [test_user_uuid, test_user_token]
+ run: arvbox/setup-user.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+ this_cluster_id: string
+ cluster_ids: string[]
+ cluster_hosts: string[]
+ arvbox_data: Directory
+outputs:
+ arvbox_data_out:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: cluster_config.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = {
+ "Host": inputs.cluster_hosts[i],
+ "Proxy": true,
+ "Insecure": true
+ };
+ }
+ var r = {"Clusters": {}};
+ r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
+ return JSON.stringify(r);
+ }
+ - entryname: application.yml.override
+ entry: >-
+ ${
+ var remoteClusters = {};
+ for (var i = 0; i < inputs.cluster_ids.length; i++) {
+ remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i];
+ }
+ return JSON.stringify({"development": {"remote_hosts": remoteClusters}});
+ }
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ ShellCommandRequirement: {}
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+ docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
+ arvbox sv restart api
+ arvbox sv restart controller
+ arvbox sv restart keepstore0
+ arvbox sv restart keepstore1
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ containers:
+ type:
+ type: array
+ items: string
+ inputBinding:
+ position: 3
+ valueFrom: |
+ ${
+ return "base/"+self;
+ }
+ arvbox_base: Directory
+outputs:
+ arvbox_data:
+ type: Directory[]
+ outputBinding:
+ glob: |
+ ${
+ var r = [];
+ for (var i = 0; i < inputs.containers.length; i++) {
+ r.push("base/"+inputs.containers[i]);
+ }
+ return r;
+ }
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_base)
+ entryname: base
+ writable: true
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+arguments:
+ - mkdir
+ - "-p"
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.container_host)
+ ARVADOS_API_TOKEN: $(inputs.superuser_token)
+ ARVADOS_API_HOST_INSECURE: "true"
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ DockerRequirement:
+ dockerPull: arvados/jobs
+inputs:
+ container_host: string
+ superuser_token: string
+ make_user_script:
+ type: File
+ default:
+ class: File
+ location: setup_user.py
+outputs:
+ test_user_uuid: string
+ test_user_token: string
+arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import arvados.errors
+import time
+import json
+
+while True:
+ try:
+ api = arvados.api()
+ break
+ except arvados.errors.ApiError:
+ time.sleep(2)
+
+existing = api.users().list(filters=[["email", "=", "test@example.com"],
+ ["is_active", "=", True]], limit=1).execute()
+if existing["items"]:
+ u = existing["items"][0]
+else:
+ u = api.users().create(body={
+ 'first_name': 'Test',
+ 'last_name': 'User',
+ 'email': 'test@example.com',
+ 'is_admin': False
+ }).execute()
+ api.users().activate(uuid=u["uuid"]).execute()
+
+tok = api.api_client_authorizations().create(body={
+ "api_client_authorization": {
+ "owner_uuid": u["uuid"]
+ }
+}).execute()
+
+with open("cwl.output.json", "w") as f:
+ json.dump({
+ "test_user_uuid": u["uuid"],
+ "test_user_token": "v2/%s/%s" % (tok["uuid"], tok["api_token"])
+ }, f)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+ arvbox_data: Directory
+outputs:
+ cluster_id:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Cluster id: ")) {
+ return sp[i].substr(12);
+ }
+ }
+ }
+ container_host:
+ type: string
+ outputBinding:
+ glob: status.txt
+ loadContents: true
+ outputEval: |
+ ${
+ var sp = self[0].contents.split("\n");
+ for (var i = 0; i < sp.length; i++) {
+ if (sp[i].startsWith("Container IP: ")) {
+ return sp[i].substr(14)+":8000";
+ }
+ }
+ }
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+ arvbox_data_out:
+ type: Directory
+ outputBinding:
+ outputEval: $(inputs.arvbox_data)
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+ ARVBOX_DATA: $(inputs.arvbox_data.path)
+ ShellCommandRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.arvbox_data)
+ entryname: $(inputs.container_name)
+ writable: true
+ cwltool:InplaceUpdateRequirement:
+ inplaceUpdate: true
+ InlineJavascriptRequirement: {}
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ set -e
+ arvbox start dev
+ arvbox status > status.txt
+ arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ container_name: string
+outputs: []
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container_name)
+arguments: [arvbox, stop]
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:base-case
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ inp:
+ type: File[]
+ inputBinding: {}
+outputs:
+ joined: stdout
+stdout: joined.txt
+baseCommand: cat
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:hint-on-tool
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ out: [hash]
+ run: md5sum-tool-hint.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:hint-on-wf
+hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ out: [hash]
+ run: md5sum.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+inputs:
+ inp: File
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputBinding:
+ glob: out.txt
+stdin: $(inputs.inp.path)
+stdout: out.txt
+arguments: ["md5sum", "-"]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+inputs:
+ inp:
+ type: File
+outputs:
+ hash:
+ type: File
+ outputBinding:
+ glob: out.txt
+stdin: $(inputs.inp.path)
+stdout: out.txt
+arguments: ["md5sum", "-"]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:remote-case
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ ShellCommandRequirement: {}
+inputs:
+ inp:
+ type: File
+outputs:
+ original:
+ type: File
+ outputBinding:
+ glob: $(inputs.inp.basename)
+ revhash:
+ type: stdout
+stdout: rev-$(inputs.inp.basename)
+arguments:
+ - shellQuote: false
+ valueFrom: |
+ ln -s $(inputs.inp.path) $(inputs.inp.basename) &&
+ rev $(inputs.inp.basename)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+inputs:
+ inp:
+ type: File
+outputs:
+ revhash:
+ type: File
+ outputBinding:
+ glob: out.txt
+stdout: out.txt
+arguments: [rev, $(inputs.inp)]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:runner-home-step-remote
+inputs:
+ inp: File
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ out: [hash]
+ run: md5sum.cwl
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:runner-remote-step-home
+inputs:
+ inp: File
+ runOnCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: runOnCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:scatter-gather
+ ScatterFeatureRequirement: {}
+inputs:
+ shards: File[]
+ clusters: string[]
+outputs:
+ joined:
+ type: File
+ outputSource: cat/joined
+steps:
+ md5sum:
+ in:
+ inp: shards
+ runOnCluster: clusters
+ scatter: [inp, runOnCluster]
+ scatterMethod: dotproduct
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
+ cat:
+ in:
+ inp: md5sum/hash
+ out: [joined]
+ run: cat.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:threestep-remote
+ ScatterFeatureRequirement: {}
+inputs:
+ inp: File
+ clusterA: string
+ clusterB: string
+ clusterC: string
+outputs:
+ revhash:
+ type: File
+ outputSource: revC/revhash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: clusterA
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
+ revB:
+ in:
+ inp: md5sum/hash
+ runOnCluster: clusterB
+ out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: rev-input-to-output.cwl
+ revC:
+ in:
+ inp: revB/revhash
+ runOnCluster: clusterC
+ out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: rev-input-to-output.cwl
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-both-remote
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: rev.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-home-to-remote
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: rev.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-remote-copy-to-home
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: rev-input-to-output.cwl
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/fed-test:twostep-remote-to-home
+inputs:
+ inp:
+ type: File
+ inputBinding: {}
+ md5sumCluster: string
+ revCluster: string
+outputs:
+ hash:
+ type: File
+ outputSource: md5sum/hash
+steps:
+ md5sum:
+ in:
+ inp: inp
+ runOnCluster: md5sumCluster
+ out: [hash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: md5sum.cwl
+ rev:
+ in:
+ inp: md5sum/hash
+ runOnCluster: revCluster
+ out: [revhash]
+ hints:
+ arv:ClusterTarget:
+ cluster_id: $(inputs.runOnCluster)
+ run: rev.cwl
--- /dev/null
+Call me base-case. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me hint-on-tool. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me hint-on-wf. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me remote-case. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me runner-home-step-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me runner-remote-step-home. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me scatter-gather-s1. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me scatter-gather-s2. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me scatter-gather-s3. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me threestep-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me twostep-both-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me twostep-home-to-remote. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me twostep-remote-copy-to-home. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+Call me twostep-remote-to-home. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: config.json
+ entry: |-
+ ${
+ return JSON.stringify({
+ check_collections: inputs.check_collections
+ });
+ }
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+ InlineJavascriptRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure: boolean
+ arvados_api_host: string
+ check_collections: string[]
+ preparescript:
+ type: File
+ default:
+ class: File
+ location: check_exist.py
+ inputBinding:
+ position: 1
+outputs:
+ success:
+ type: boolean
+ outputBinding:
+ glob: success
+ loadContents: true
+ outputEval: $(self[0].contents=="true")
+baseCommand: python2
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import json
+
+api = arvados.api()
+
+with open("config.json") as f:
+ config = json.load(f)
+
+success = True
+for c in config["check_collections"]:
+ try:
+ api.collections().get(uuid=c).execute()
+ except Exception as e:
+ print("Checking for %s got exception %s" % (c, e))
+ success = False
+
+with open("success", "w") as f:
+ if success:
+ f.write("true")
+ else:
+ f.write("false")
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ testcase: string
+outputs:
+ imagename:
+ type: string
+ outputBinding:
+ outputEval: $(inputs.testcase)
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: Dockerfile
+ entry: |-
+ FROM debian@sha256:0a5fcee6f52d5170f557ee2447d7a10a5bdcf715dd7f0250be0b678c556a501b
+ LABEL org.arvados.testcase="$(inputs.testcase)"
+arguments: [docker, build, -t, $(inputs.testcase), "."]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ - entryname: config.json
+ entry: |-
+ ${
+ return JSON.stringify({
+ arvados_cluster_ids: inputs.arvados_cluster_ids,
+ scrub_images: [inputs.scrub_image],
+ scrub_collections: inputs.scrub_collections
+ });
+ }
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+ InlineJavascriptRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure: boolean
+ arvados_api_host: string
+ arvados_cluster_ids: string[]
+ wf: File
+ obj: Any
+ scrub_image: string
+ scrub_collections: string[]
+ preparescript:
+ type: File
+ default:
+ class: File
+ location: prepare.py
+ inputBinding:
+ position: 1
+outputs:
+ done:
+ type: boolean
+ outputBinding:
+ outputEval: $(true)
+baseCommand: python2
\ No newline at end of file
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import json
+
+api = arvados.api()
+
+with open("config.json") as f:
+ config = json.load(f)
+
+scrub_collections = set(config["scrub_collections"])
+
+for cluster_id in config["arvados_cluster_ids"]:
+ images = []
+ for scrub_image in config["scrub_images"]:
+ sp = scrub_image.split(":")
+ image_name = sp[0]
+ image_tag = sp[1] if len(sp) > 1 else "latest"
+ images.append('{}:{}'.format(image_name, image_tag))
+
+ search_links = api.links().list(
+ filters=[['link_class', '=', 'docker_image_repo+tag'],
+ ['name', 'in', images]],
+ cluster_id=cluster_id).execute()
+
+ head_uuids = [lk["head_uuid"] for lk in search_links["items"]]
+ cols = api.collections().list(filters=[["uuid", "in", head_uuids]],
+ cluster_id=cluster_id).execute()
+ for c in cols["items"]:
+ scrub_collections.add(c["portable_data_hash"])
+ for lk in search_links["items"]:
+ api.links().delete(uuid=lk["uuid"]).execute()
+
+for cluster_id in config["arvados_cluster_ids"]:
+ matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]],
+ select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute()
+ for m in matches["items"]:
+ api.collections().delete(uuid=m["uuid"]).execute()
+ print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"]))
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ acr:
+ type: string?
+ default: arvados-cwl-runner
+ inputBinding:
+ position: 1
+ arvados_api_host: string
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: boolean
+ default: false
+ runner_cluster:
+ type: string?
+ inputBinding:
+ prefix: --submit-runner-cluster
+ position: 2
+ wf:
+ type: File
+ inputBinding:
+ position: 3
+ obj: Any
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: input.json
+ entry: $(JSON.stringify(inputs.obj))
+ EnvVarRequirement:
+ envDef:
+ ARVADOS_API_HOST: $(inputs.arvados_api_host)
+ ARVADOS_API_TOKEN: $(inputs.arvados_api_token)
+ ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure)
+ InlineJavascriptRequirement: {}
+outputs:
+ out:
+ type: Any
+ outputBinding:
+ glob: output.json
+ loadContents: true
+ #outputEval: $(JSON.parse(self[0].contents))
+ outputEval: $(self[0].contents)
+stdout: output.json
+arguments:
+ - valueFrom: --disable-reuse
+ position: 2
+ - valueFrom: --always-submit-runner
+ position: 2
+ - valueFrom: --api=containers
+ position: 2
+ - valueFrom: input.json
+ position: 4
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: boolean
+ default: false
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
+ wf: File
+ obj: Any
+ scrub_image: string
+ scrub_collections: string[]
+ runner_cluster: string?
+outputs:
+ out:
+ type: Any
+ outputSource: run-acr/out
+ success:
+ type: boolean
+ outputSource: check-result/success
+steps:
+ dockerbuild:
+ in:
+ testcase: scrub_image
+ out: [imagename]
+ run: dockerbuild.cwl
+ prepare:
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ arvados_cluster_ids: arvados_cluster_ids
+ wf: wf
+ obj: obj
+ scrub_image: scrub_image
+ scrub_collections: scrub_collections
+ out: [done]
+ run: prepare.cwl
+ run-acr:
+ in:
+ prepare: prepare/done
+ image-ready: dockerbuild/imagename
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ runner_cluster: runner_cluster
+ acr: acr
+ wf: wf
+ obj: obj
+ out: [out]
+ run: run-acr.cwl
+ check-result:
+ in:
+ acr-done: run-acr/out
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"}
+ check_collections: scrub_collections
+ out: [success]
+ run: check-exist.cwl
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+hints:
+ cwltool:Secrets:
+ secrets: [arvados_api_token]
+requirements:
+ StepInputExpressionRequirement: {}
+ InlineJavascriptRequirement: {}
+ SubworkflowFeatureRequirement: {}
+inputs:
+ arvados_api_token: string
+ arvado_api_host_insecure:
+ type: boolean
+ default: false
+ arvados_api_hosts: string[]
+ arvados_cluster_ids: string[]
+ acr: string?
+ testcases:
+ type: string[]
+ default:
+ - base-case
+ - runner-home-step-remote
+ - runner-remote-step-home
+outputs:
+ base-case-success:
+ type: Any
+ outputSource: base-case/success
+ runner-home-step-remote-success:
+ type: Any
+ outputSource: runner-home-step-remote/success
+ runner-remote-step-home-success:
+ type: Any
+ outputSource: runner-remote-step-home/success
+ remote-case-success:
+ type: Any
+ outputSource: remote-case/success
+ twostep-home-to-remote-success:
+ type: Any
+ outputSource: twostep-home-to-remote/success
+ twostep-remote-to-home-success:
+ type: Any
+ outputSource: twostep-remote-to-home/success
+ twostep-both-remote-success:
+ type: Any
+ outputSource: twostep-both-remote/success
+ twostep-remote-copy-to-home-success:
+ type: Any
+ outputSource: twostep-remote-copy-to-home/success
+ scatter-gather-success:
+ type: Any
+ outputSource: scatter-gather/success
+ threestep-remote-success:
+ type: Any
+ outputSource: threestep-remote/success
+ hint-on-wf-success:
+ type: Any
+ outputSource: hint-on-wf/success
+ hint-on-tool-success:
+ type: Any
+ outputSource: hint-on-tool/success
+
+steps:
+ base-case:
+ doc: |
+ Base case (no federation), single step workflow with both the
+ runner and step on the same cluster.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/base-case.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/base-case-input.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ scrub_image: {default: "arvados/fed-test:base-case"}
+ scrub_collections:
+ default:
+ - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection
+ - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection
+ - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ runner-home-step-remote:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster. ClusterTarget hint is on the workflow step.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/runner-home-step-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/runner-home-step-remote-input.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:runner-home-step-remote"}
+ scrub_collections:
+ default:
+ - 3bc373e38751fe13dcbd62778d583242+81 # input collection
+ - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection
+ - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ runner-remote-step-home:
+ doc: |
+ Single step workflow with the runner on the remote cluster and the
+ step on the home cluster.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/runner-remote-step-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/runner-remote-step-home-input.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
+ scrub_image: {default: "arvados/fed-test:runner-remote-step-home"}
+ scrub_collections:
+ default:
+ - 25fe10d8e8530329a738de69d9bc8ab5+81 # input collection
+ - 7f052d1a04b851b6f73fba77c7802e1d+51 # md5sum output collection
+ - ecb639201f454b6493757f5117f540df+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ remote-case:
+ doc: |
+ Single step workflow with both the runner and the step on the
+ remote cluster.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/remote-case.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/remote-case-input.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
+ scrub_image: {default: "arvados/fed-test:remote-case"}
+ scrub_collections:
+ default:
+ - fccd49fdef8e452295f718208abafd88+69 # input collection
+ - 58c0e8ea6b148134ef8577ee11307eec+51 # md5sum output collection
+ - 1fd679c5ab64c123b9764024dbf560f0+112 # final output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ twostep-home-to-remote:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the home cluster, the second step is on the remote
+ cluster.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-home-to-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-home-to-remote.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[0];
+ self["revCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-home-to-remote"}
+ scrub_collections:
+ default:
+ - 268a54947fb75115cfe05bb54cc62c30+74 # input collection
+ - 400f03b8c5d2dc3dcb513a21b626ef88+51 # md5sum output collection
+ - 3738166916ca5f6f6ad12bf7e06b4a21+51 # rev output collection
+ - bc37c17a37aa25229e5de1339b27fbcc+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ twostep-remote-to-home:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the remote cluster, the second step is on the home
+ cluster.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-remote-to-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-remote-to-home.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-remote-to-home"}
+ scrub_collections:
+ default:
+ - cce89b9f7b6e163978144051ce5f071a+74 # input collection
+ - 0c358c3af63644c6343766feff1b7238+51 # md5sum output collection
+ - 33fb7d512bf21f04847eca58cea46e74+51 # rev output collection
+ - 912e04aa3db04aba008cf5cd46c277b2+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ twostep-both-remote:
+ doc: |
+ Two step workflow. The runner is on the home cluster, both steps are
+ on the remote cluster.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-both-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-both-remote.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-both-remote"}
+ scrub_collections:
+ default:
+ - 3c5e39939cf197d304ac1eac20841238+71 # input collection
+ - 3edb99aa607731593969cdab663d65b4+51 # md5sum output collection
+ - a91625b7139e60fe61a88cae42fbee13+51 # rev output collection
+ - ddfa58a81953dad08436d571615dd584+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ twostep-remote-copy-to-home:
+ doc: |
+ Two step workflow. The runner is on the home cluster, the first
+ step is on the remote cluster, the second step is on the home
+ cluster, and propagates its input file directly from input to
+ output by symlinking the input file in the output directory.
+ Tests that crunch-run will copy blocks from remote to local
+ when preparing output collection.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/twostep-remote-copy-to-home.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev-input-to-output.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/twostep-remote-copy-to-home.txt
+ valueFrom: |-
+ ${
+ self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
+ self["revCluster"] = inputs.arvados_cluster_ids[0];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:twostep-remote-copy-to-home"}
+ scrub_collections:
+ default:
+ - 538887bc29a3098bf79abdb8536d17bd+79 # input collection
+ - 14da0e0d52d7ab2945427074b275e9ee+51 # md5sum output collection
+ - 2d3a4a840077390a0d7788f169eaba89+112 # rev output collection
+ - 2d3a4a840077390a0d7788f169eaba89+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ scatter-gather:
+ doc: ""
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/scatter-gather.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/cat.cwl
+ obj:
+ default:
+ shards:
+ - class: File
+ location: data/scatter-gather-s1.txt
+ - class: File
+ location: data/scatter-gather-s2.txt
+ - class: File
+ location: data/scatter-gather-s3.txt
+ valueFrom: |-
+ ${
+ self["clusters"] = inputs.arvados_cluster_ids;
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:scatter-gather"}
+ scrub_collections:
+ default:
+ - 99cc18329bce1b4a5fe6c4cf60477668+209 # input collection
+ - 2e570e844e03c7027baad148642d726f+51 # s1 md5sum output collection
+ - 61c88ee7811d0b849b5c06376eb065a6+51 # s2 md5sum output collection
+ - 85aaf18d638045fe609e025d3a319b2a+51 # s3 md5sum output collection
+ - ec44bcba77e65128f1a8f843d881ede4+56 # cat output collection
+ - 89de265942800ae36549109969940363+117 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ threestep-remote:
+ doc: ""
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/threestep-remote.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ - class: File
+ location: cases/rev-input-to-output.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/threestep-remote.txt
+ valueFrom: |-
+ ${
+ self["clusterA"] = inputs.arvados_cluster_ids[0];
+ self["clusterB"] = inputs.arvados_cluster_ids[1];
+ self["clusterC"] = inputs.arvados_cluster_ids[2];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:threestep-remote"}
+ scrub_collections:
+ default:
+ - 9fbf33e62876357fe134f619865cc5a5+68 # input collection
+ - 210c5f2a716f6689b04316acd4928c10+51 # md5sum output collection
+ - 3abea7506269d5ebf61fb17c78bbd2af+105 # revB output
+ - 9e1b3acb28949759ad07e4c9740bbaa5+113 # revC output
+ - 8c86dbec7de7948871b5e168ede417e1+120 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ hint-on-wf:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster. ClusterTarget hint is at the workflow level.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/hint-on-wf.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/hint-on-wf.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:hint-on-wf"}
+ scrub_collections:
+ default:
+ - 862433f328041b2525c90b1dc3c462fd+62 # input collection
+ - 9a68b0b9720977faba8a28e75a4398b7+51 # md5sum output collection
+ - 6a601cddb36ee2f766783b1aa9ff8d66+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
+
+ hint-on-tool:
+ doc: |
+ Single step workflow with the runner on the home cluster and the
+ step on the remote cluster. ClusterTarget hint is at the tool level.
+ in:
+ arvados_api_token: arvados_api_token
+ arvado_api_host_insecure: arvado_api_host_insecure
+ arvados_api_hosts: arvados_api_hosts
+ arvados_cluster_ids: arvados_cluster_ids
+ acr: acr
+ wf:
+ default:
+ class: File
+ location: cases/hint-on-tool.cwl
+ secondaryFiles:
+ - class: File
+ location: cases/md5sum-tool-hint.cwl
+ obj:
+ default:
+ inp:
+ class: File
+ location: data/hint-on-tool.txt
+ valueFrom: |-
+ ${
+ self["runOnCluster"] = inputs.arvados_cluster_ids[1];
+ return self;
+ }
+ runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
+ scrub_image: {default: "arvados/fed-test:hint-on-tool"}
+ scrub_collections:
+ default:
+ - 6803004a4f8db9f8d1d54f6229851599+64 # input collection
+ - cacb0d56235564b5ff485c5b31215ab5+51 # md5sum output collection
+ - 2b50af43fdd84a9e906be2d54b92cddf+112 # runner output json
+ out: [out, success]
+ run: framework/testcase.cwl
import arvados_cwl
import arvados_cwl.context
+import arvados_cwl.util
from arvados_cwl.arvdocker import arv_docker_clear_cache
import copy
import arvados.config
"make_fs_access": make_fs_access,
"tmpdir": "/tmp",
"enable_reuse": enable_reuse,
- "priority": 500})
+ "priority": 500,
+ "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ })
return loadingContext, runtimeContext
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
def test_resource_requirements(self, keepdocker):
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
runner.secret_store = cwltool.secrets.SecretStore()
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 7200,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {
def test_initial_work_dir(self, collection_mock, keepdocker):
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
runner = mock.MagicMock()
runner.api = api
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
col().open.return_value = []
+ loadingContext, runtimeContext = self.helper(runner)
+
arvjob = arvados_cwl.ArvadosContainer(runner,
+ runtimeContext,
mock.MagicMock(),
{},
None,
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
- @mock.patch("arvados_cwl.get_current_container")
+ @mock.patch("arvados_cwl.util.get_current_container")
@mock.patch("arvados.collection.CollectionReader")
@mock.patch("arvados.collection.Collection")
def test_child_failure(self, col, reader, gcc_mock):
# Set up runner with mocked runtime_status_update()
self.assertFalse(gcc_mock.called)
runtime_status_update = mock.MagicMock()
- arvados_cwl.ArvCwlRunner.runtime_status_update = runtime_status_update
- runner = arvados_cwl.ArvCwlRunner(api)
+ arvados_cwl.ArvCwlExecutor.runtime_status_update = runtime_status_update
+ runner = arvados_cwl.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
- # Make sure ArvCwlRunner thinks it's running inside a container so it
+ # Make sure ArvCwlExecutor thinks it's running inside a container so it
# adds the logging handler that will call runtime_status_update() mock
gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
self.assertTrue(gcc_mock.called)
handlerClasses = [h.__class__ for h in root_logger.handlers]
self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
col().open.return_value = []
+ loadingContext, runtimeContext = self.helper(runner)
+
arvjob = arvados_cwl.ArvadosContainer(runner,
+ runtimeContext,
mock.MagicMock(),
{},
None,
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999994+99',
'command': ['ls', '/var/spool/cwl'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'output_path': '/var/spool/cwl',
'output_ttl': 0,
- 'container_image': 'arvados/jobs',
+ 'container_image': '99999999999999999999999999999993+99',
'command': ['md5sum', 'example.conf'],
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
arv_docker_clear_cache()
runner = mock.MagicMock()
- runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
runner.secret_store = cwltool.secrets.SecretStore()
cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
cr().manifest_text.return_value = 'x' * 524289
self.assertEqual(0, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*1, cache.total)
- c2 = cache.get("99999999999999999999999999999992+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c2 = cache.get("99999999999999999999999999999992+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*2, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*2, cache.total)
- c3 = cache.get("99999999999999999999999999999993+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c3 = cache.get("99999999999999999999999999999993+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*3, cache.total)
- c4 = cache.get("99999999999999999999999999999994+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c4 = cache.get("99999999999999999999999999999994+524289")
+ self.assertIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*3, cache.total)
- c5 = cache.get("99999999999999999999999999999995+99")
- self.assertNotIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c5 = cache.get("99999999999999999999999999999995+524289")
+ self.assertNotIn("99999999999999999999999999999991+524289", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
self.assertEqual((524289*128)*3, cache.total)
cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
cr().manifest_text.return_value = 'x' * 524287
self.assertEqual(0, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*1, cache.total)
- c2 = cache.get("99999999999999999999999999999992+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c2 = cache.get("99999999999999999999999999999992+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*2, cache.total)
- c1 = cache.get("99999999999999999999999999999991+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c1 = cache.get("99999999999999999999999999999991+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*2, cache.total)
- c3 = cache.get("99999999999999999999999999999993+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c3 = cache.get("99999999999999999999999999999993+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*3, cache.total)
- c4 = cache.get("99999999999999999999999999999994+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertIn("99999999999999999999999999999992+99", cache.collections)
+ c4 = cache.get("99999999999999999999999999999994+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*4, cache.total)
- c5 = cache.get("99999999999999999999999999999995+99")
- self.assertIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c5 = cache.get("99999999999999999999999999999995+524287")
+ self.assertIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*4, cache.total)
- c6 = cache.get("99999999999999999999999999999996+99")
- self.assertNotIn("99999999999999999999999999999991+99", cache.collections)
- self.assertNotIn("99999999999999999999999999999992+99", cache.collections)
+ c6 = cache.get("99999999999999999999999999999996+524287")
+ self.assertNotIn("99999999999999999999999999999991+524287", cache.collections)
+ self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
self.assertEqual((524287*128)*4, cache.total)
import arvados
import arvados_cwl
+import arvados_cwl.executor
import cwltool.process
from arvados.errors import ApiError
from schema_salad.ref_resolver import Loader
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
- runner = arvados_cwl.ArvCwlRunner(api)
+ runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'jobs')
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
api = mock.MagicMock()
api._rootDesc = get_rootDesc()
- runner = arvados_cwl.ArvCwlRunner(api)
+ runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'jobs')
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
api = mock.MagicMock()
api._rootDesc = copy.deepcopy(get_rootDesc())
del api._rootDesc.get('resources')['jobs']['methods']['create']
- runner = arvados_cwl.ArvCwlRunner(api)
+ runner = arvados_cwl.executor.ArvCwlExecutor(api)
self.assertEqual(runner.work_api, 'containers')
import arvados
import arvados_cwl
+import arvados_cwl.executor
from .mock_discovery import get_rootDesc
class TestMakeOutput(unittest.TestCase):
@mock.patch("arvados.collection.CollectionReader")
def test_make_output_collection(self, reader, col):
keep_client = mock.MagicMock()
- runner = arvados_cwl.ArvCwlRunner(self.api, keep_client=keep_client)
+ runner = arvados_cwl.executor.ArvCwlExecutor(self.api, keep_client=keep_client)
runner.project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
final = mock.MagicMock()
import arvados.keep
import arvados.collection
import arvados_cwl
+import arvados_cwl.executor
from cwltool.pathmapper import MapperEnt
from .mock_discovery import get_rootDesc
def test_keepref(self):
"""Test direct keep references."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
p = ArvPathMapper(arvrunner, [{
"class": "File",
def test_upload(self, statfile, upl):
"""Test pathmapper uploading files."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)", raiseOSError=False):
st = arvados.commands.run.UploadFile("", "tests/hw.py")
@mock.patch("arvados.commands.run.statfile")
def test_statfile(self, statfile, upl):
"""Test pathmapper handling ArvFile references."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
# An ArvFile object returned from arvados.commands.run.statfile means the file is located on a
# keep mount, so we can construct a direct reference directly without upload.
@mock.patch("os.stat")
def test_missing_file(self, stat):
"""Test pathmapper handling missing references."""
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ arvrunner = arvados_cwl.executor.ArvCwlExecutor(self.api)
stat.side_effect = OSError(2, "No such file or directory")
import arvados
import arvados.collection
import arvados_cwl
+import arvados_cwl.executor
import arvados_cwl.runner
import arvados.keep
keep_client2.put.side_effect = putstub
stubs.keep_client = keep_client2
- stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ stubs.docker_images = {
+ "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
+ "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
+ "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
+ "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
+ }
+ def kd(a, b, image_name=None, image_tag=None):
+ return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
+ stubs.keepdocker.side_effect = kd
+
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
def collection_createstub(created_collections, body, ensure_unique_name=None):
mt = body["manifest_text"]
- uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
+ uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
created_collections[uuid] = {
"uuid": uuid,
"uuid": "",
"portable_data_hash": "99999999999999999999999999999994+99",
"manifest_text": ". 99999999999999999999999999999994+99 0:0:expect_arvworkflow.cwl"
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd3": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd3",
+ "portable_data_hash": "999999999999999999999999999999d3+99",
+ "manifest_text": ""
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd4": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd4",
+ "portable_data_hash": "999999999999999999999999999999d4+99",
+ "manifest_text": ""
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd5": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd5",
+ "portable_data_hash": "999999999999999999999999999999d5+99",
+ "manifest_text": ""
+ },
+ "zzzzz-4zz18-zzzzzzzzzzzzzd6": {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzd6",
+ "portable_data_hash": "999999999999999999999999999999d6+99",
+ "manifest_text": ""
}
}
stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
}
stubs.expect_job_spec = {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': '999999999999999999999999999999d3+99',
'min_ram_mb_per_node': 1024
},
'script_parameters': {
}],
'class': 'Directory'
},
- 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
+ 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
},
'repository': 'arvados',
'script_version': 'master',
'owner_uuid': None,
"components": {
"cwl-runner": {
- 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
+ 'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
'script_parameters': {
'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
'x': {"value": {
'size': 0
}
]}},
- 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
+ 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
'arv:debug': True,
'arv:enable_reuse': True,
'arv:on_error': 'continue'
'state': 'Committed',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
- 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'container_image': '999999999999999999999999999999d3+99',
'output_path': '/var/spool/cwl',
'cwd': '/var/spool/cwl',
'runtime_constraints': {
'API': True,
'vcpus': 1,
- 'ram': 1024*1024*1024
+ 'ram': (1024+256)*1024*1024
},
'use_existing': True,
'properties': {},
class TestSubmit(unittest.TestCase):
- @mock.patch("arvados_cwl.runner.arv_docker_get_image")
+ @mock.patch("arvados_cwl.arvdocker.arv_docker_get_image")
@mock.patch("time.sleep")
@stubs
def test_submit(self, stubs, tm, arvdock):
+ def get_image(api_client, dockerRequirement, pull_image, project_uuid):
+ if dockerRequirement["dockerPull"] == 'arvados/jobs:'+arvados_cwl.__version__:
+ return '999999999999999999999999999999d3+99'
+ elif dockerRequirement["dockerPull"] == "debian:8":
+ return '999999999999999999999999999999d4+99'
+ arvdock.side_effect = get_image
+
capture_stdout = cStringIO.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=jobs", "--debug",
}), ensure_unique_name=False),
mock.call(body=JsonDiffMatcher({
'manifest_text':
- '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
+ ". 68089141fbf7e020ac90a9d6a575bc8f+1312 0:1312:workflow.cwl\n",
'replication_desired': None,
'name': 'submit_wf.cwl',
}), ensure_unique_name=True) ])
arvdock.assert_has_calls([
mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
+ mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8", 'http://arvados.org/cwl#dockerCollectionPDH': '999999999999999999999999999999d4+99'}, True, None),
mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
])
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--disable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--disable-reuse', "--collection-cache-size=256",
+ '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--disable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
expect_container["name"] = "submit_wf_no_reuse.cwl"
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=stop',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256",
+ '--debug', '--on-error=stop',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.api.container_requests().create.assert_called_with(
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256",
"--output-name="+output_name, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["output_name"] = output_name
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', "--debug",
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256", "--debug",
"--storage-classes=foo", '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@mock.patch("arvados_cwl.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
- @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
@stubs
def test_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
def set_final_output(job_order, output_callback, runtimeContext):
@mock.patch("arvados_cwl.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
- @mock.patch("arvados_cwl.ArvCwlRunner.make_output_collection", return_value = (None, None))
+ @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection", return_value = (None, None))
@stubs
def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq):
def set_final_output(job_order, output_callback, runtimeContext):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256", '--debug',
+ '--on-error=continue',
"--intermediate-output-ttl=3600",
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256",
+ '--debug', '--on-error=continue',
"--trash-intermediate",
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256",
"--output-tags="+output_tags, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
logging.exception("")
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
+ expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
}, 'state': 'Committed',
'output_path': '/var/spool/cwl',
'name': 'expect_arvworkflow.cwl#main',
- 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'container_image': '999999999999999999999999999999d3+99',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
'runtime_constraints': {
'API': True,
'vcpus': 1,
- 'ram': 1073741824
+ 'ram': 1342177280
},
'use_existing': True,
'properties': {},
'id': '#submit_tool.cwl/x'}
],
'requirements': [
- {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
+ {
+ 'dockerPull': 'debian:8',
+ 'class': 'DockerRequirement',
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ }
],
'id': '#submit_tool.cwl',
'outputs': [],
}, 'state': 'Committed',
'output_path': '/var/spool/cwl',
'name': 'a test workflow',
- 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'container_image': "999999999999999999999999999999d3+99",
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=20', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
'runtime_constraints': {
'API': True,
'vcpus': 1,
- 'ram': 1073741824
+ 'ram': 1342177280
},
'use_existing': True,
'properties': {
expect_container["owner_uuid"] = project_uuid
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- "--eval-timeout=20", "--thread-count=4",
- '--enable-reuse', '--debug', '--on-error=continue',
+ "--eval-timeout=20", "--thread-count=1",
+ '--enable-reuse', "--collection-cache-size=256", '--debug',
+ '--on-error=continue',
'--project-uuid='+project_uuid,
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
- '--eval-timeout=60.0', '--thread-count=4',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--eval-timeout=60.0', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=256",
+ '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.api.container_requests().create.assert_called_with(
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_container_collection_cache(self, stubs):
+ project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=500",
+ '--debug', '--on-error=continue',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+ expect_container["runtime_constraints"]["ram"] = (1024+500)*1024*1024
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
@stubs
def test_submit_container_thread_count(self, stubs):
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate',
'--eval-timeout=20', '--thread-count=20',
- '--enable-reuse', '--debug', '--on-error=continue',
+ '--enable-reuse', "--collection-cache-size=256",
+ '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.api.container_requests().create.assert_called_with(
except:
logging.exception("")
- stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
+ stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
stubs.api.pipeline_instances().create.assert_called_with(
except:
logging.exception("")
- stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
+ stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
expect_container["runtime_constraints"] = {
"API": True,
"vcpus": 2,
- "ram": 2000 * 2**20
+ "ram": (2000+512) * 2**20
}
expect_container["name"] = "submit_wf_runner_resources.cwl"
expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
{
"class": "http://arvados.org/cwl#WorkflowRunnerResources",
"coresMin": 2,
- "ramMin": 2000
+ "ramMin": 2000,
+ "keep_cache": 512
}
]
expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
"arv": "http://arvados.org/cwl#",
}
+ expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
+ '--no-log-timestamps', '--disable-validate',
+ '--eval-timeout=20', '--thread-count=1',
+ '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
+ '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ def tearDown(self):
+ arvados_cwl.arvdocker.arv_docker_clear_cache()
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
@mock.patch("arvados.api")
def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
+ arvados_cwl.arvdocker.arv_docker_clear_cache()
+
arvrunner = mock.MagicMock()
arvrunner.project_uuid = ""
api.return_value = mock.MagicMock()
"properties": ""
}], "items_available": 1, "offset": 0},)
arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
+ arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "portable_data_hash": "9999999999999999999999999999999b+99"}
+ self.assertEqual("9999999999999999999999999999999b+99",
arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+
@stubs
def test_submit_secrets(self, stubs):
capture_stdout = cStringIO.StringIO()
"--no-log-timestamps",
"--disable-validate",
"--eval-timeout=20",
- '--thread-count=4',
+ '--thread-count=1',
"--enable-reuse",
+ "--collection-cache-size=256",
'--debug',
"--on-error=continue",
"/var/lib/cwl/workflow.json#main",
"/var/lib/cwl/cwl.input.json"
],
- "container_image": "arvados/jobs:"+arvados_cwl.__version__,
+ "container_image": "999999999999999999999999999999d3+99",
"cwd": "/var/spool/cwl",
"mounts": {
"/var/lib/cwl/cwl.input.json": {
"hints": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8"
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
},
{
"class": "http://commonwl.org/cwltool#Secrets",
"properties": {},
"runtime_constraints": {
"API": True,
- "ram": 1073741824,
+ "ram": 1342177280,
"vcpus": 1
},
"secret_mounts": {
logging.exception("")
stubs.api.container_requests().update.assert_called_with(
- uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec))
+ uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec), cluster_id="zzzzz")
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+ @stubs
+ def test_submit_container_cluster_id(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @stubs
+ def test_submit_validate_cluster_id(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 1)
+
+
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
"components": {
"inputs_test.cwl": {
'runtime_constraints': {
- 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'docker_image': '999999999999999999999999999999d3+99',
'min_ram_mb_per_node': 1024
},
'script_parameters': {
'cwl:tool':
- '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
+ 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main',
'optionalFloatInput': None,
'fileInput': {
'type': 'File',
params = expect_template[
"components"]["inputs_test.cwl"]["script_parameters"]
params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
- params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
+ params["cwl:tool"] = 'a2de777156fb700f1363b1f2e370adca+60/workflow.cwl#main'
params["floatInput"]["value"] = 1.234
params["boolInput"]["value"] = True
class TestTaskQueue(unittest.TestCase):
def test_tq(self):
tq = TaskQueue(threading.Lock(), 2)
+ try:
+ self.assertIsNone(tq.error)
- self.assertIsNone(tq.error)
-
- tq.add(success_task)
- tq.add(success_task)
- tq.add(success_task)
- tq.add(success_task)
+ unlock = threading.Lock()
+ unlock.acquire()
+ check_done = threading.Event()
- tq.join()
+ tq.add(success_task, unlock, check_done)
+ tq.add(success_task, unlock, check_done)
+ tq.add(success_task, unlock, check_done)
+ tq.add(success_task, unlock, check_done)
+ finally:
+ tq.join()
self.assertIsNone(tq.error)
def test_tq_error(self):
tq = TaskQueue(threading.Lock(), 2)
-
- self.assertIsNone(tq.error)
-
- tq.add(success_task)
- tq.add(success_task)
- tq.add(fail_task)
- tq.add(success_task)
-
- tq.join()
+ try:
+ self.assertIsNone(tq.error)
+
+ unlock = threading.Lock()
+ unlock.acquire()
+ check_done = threading.Event()
+
+ tq.add(success_task, unlock, check_done)
+ tq.add(success_task, unlock, check_done)
+ tq.add(fail_task, unlock, check_done)
+ tq.add(success_task, unlock, check_done)
+ finally:
+ tq.join()
self.assertIsNotNone(tq.error)
"requirements": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8"
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
}
]
},
requirements:
- class: DockerRequirement
dockerPull: debian:8
+ 'http://arvados.org/cwl#dockerCollectionPDH': 999999999999999999999999999999d4+99
inputs:
- id: '#submit_tool.cwl/x'
type: File
arv:WorkflowRunnerResources:
ramMin: 2000
coresMin: 2
+ keep_cache: 512
inputs:
- id: x
type: File
RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
-RUN pip install -U setuptools six
+RUN pip install -U setuptools six requests
ARG sdk
ARG runner
import re
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag():
gitinfo = subprocess.check_output(
import re
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag():
gitinfo = subprocess.check_output(
from source package), leave it alone.
"""
def git_latest_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag(self):
gitinfo = subprocess.check_output(
'google-api-python-client >=1.6.2, <1.7',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.13.11, <= 0.15.26',
+ 'ruamel.yaml >=0.15.54, <=0.15.77',
'setuptools',
'ws4py >=0.4.2',
'subprocess32 >=3.5.1',
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_redirect off;
+
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
}
}
upstream keep-web {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_redirect off;
+
+ client_max_body_size 0;
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
}
}
server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
+ client_max_body_size 0;
+ proxy_http_version 1.1;
+ proxy_request_buffering off;
+
# Unlike other proxy sections, here we need to override the
# requested Host header and use proxy_redirect because of the
# way the test suite orchestrates services. Keep-web's "download
exit
end
-git_latest_tag = `git describe --abbrev=0`
+git_latest_tag = `git tag -l |sort -V -r |head -n1`
git_latest_tag = git_latest_tag.encode('utf-8').strip
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H .`.chomp.split(":")
git_timestamp = Time.at(git_timestamp.to_i).utc
protected_attributes (1.1.4)
activemodel (>= 4.0.1, < 5.0)
public_suffix (3.0.2)
- rack (1.6.10)
+ rack (1.6.11)
rack-test (0.6.3)
rack (>= 1.0)
rails (4.2.10)
defaultTrashLifetime: Rails.application.config.default_trash_lifetime,
blobSignatureTtl: Rails.application.config.blob_signature_ttl,
maxRequestSize: Rails.application.config.max_request_size,
+ maxItemsPerResponse: Rails.application.config.max_items_per_response,
dockerImageFormats: Rails.application.config.docker_image_formats,
crunchLogBytesPerEvent: Rails.application.config.crunch_log_bytes_per_event,
crunchLogSecondsBetweenEvents: Rails.application.config.crunch_log_seconds_between_events,
if !include_trash
if sql_table != "api_client_authorizations"
# Only include records where the owner is not trashed
- sql_conds = "NOT EXISTS(SELECT 1 FROM #{PERMISSION_VIEW} "+
- "WHERE trashed = 1 AND "+
- "(#{sql_table}.owner_uuid = target_uuid)) #{exclude_trashed_records}"
+ sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
+ "WHERE trashed = 1) #{exclude_trashed_records}"
end
end
else
# see issue 13208 for details.
# Match a direct read permission link from the user to the record uuid
- direct_check = "EXISTS(SELECT 1 FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND target_uuid = #{sql_table}.uuid)"
+ direct_check = "#{sql_table}.uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
+ "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check})"
# Match a read permission link from the user to the record's owner_uuid
owner_check = ""
if sql_table != "api_client_authorizations" and sql_table != "groups" then
- owner_check = "OR EXISTS(SELECT 1 FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND target_uuid = #{sql_table}.owner_uuid AND target_owner_uuid IS NOT NULL) "
+ owner_check = "OR #{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
+ "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND target_owner_uuid IS NOT NULL) "
end
links_cond = ""
cast = serialized_attributes[column] ? '::text' : ''
"coalesce(#{column}#{cast},'')"
end
- "to_tsvector('english', #{parts.join(" || ' ' || ")})"
+ "to_tsvector('english', substr(#{parts.join(" || ' ' || ")}, 0, 8000))"
end
def self.apply_filters query, filters
candidates = candidates.where_serialized(:runtime_constraints, resolve_runtime_constraints(attrs[:runtime_constraints]), md5: true)
log_reuse_info(candidates) { "after filtering on runtime_constraints #{attrs[:runtime_constraints].inspect}" }
- candidates = candidates.where('runtime_user_uuid = ? or (runtime_user_uuid is NULL and runtime_auth_scopes is NULL)',
- attrs[:runtime_user_uuid])
- log_reuse_info(candidates) { "after filtering on runtime_user_uuid #{attrs[:runtime_user_uuid].inspect}" }
-
- candidates = candidates.where('runtime_auth_scopes = ? or (runtime_user_uuid is NULL and runtime_auth_scopes is NULL)',
- SafeJSON.dump(attrs[:runtime_auth_scopes].sort))
- log_reuse_info(candidates) { "after filtering on runtime_auth_scopes #{attrs[:runtime_auth_scopes].inspect}" }
-
log_reuse_info { "checking for state=Complete with readable output and log..." }
select_readable_pdh = Collection.
end
0.upto(6).each do |suffix_len|
pattern = "%s%s" % [quoted_name, "_" * suffix_len]
- self.class.
+ self.class.unscoped.
where("username like '#{pattern}'").
select(:username).
order('username asc').
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class ReplaceFullTextIndexes < ActiveRecord::Migration
+ def up
+ FullTextSearchIndexes.new.up
+ end
+
+ def down
+ end
+end
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_full_text_search_idx ON public.collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || COALESCE(file_names, (''::character varying)::text))));
+CREATE INDEX collections_full_text_search_idx ON public.collections USING gin (to_tsvector('english'::regconfig, substr((((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || COALESCE(file_names, ''::text)), 0, 1000000)));
--
-- Name: container_requests_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX container_requests_full_text_search_idx ON public.container_requests USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text)) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(requesting_container_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(container_uuid, ''::character varying))::text) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(container_image, ''::character varying))::text) || ' '::text) || COALESCE(environment, ''::text)) || ' '::text) || (COALESCE(cwd, ''::character varying))::text) || ' '::text) || COALESCE(command, ''::text)) || ' '::text) || (COALESCE(output_path, ''::character varying))::text) || ' '::text) || COALESCE(filters, ''::text)) || ' '::text) || COALESCE(scheduling_parameters, ''::text)) || ' '::text) || (COALESCE(output_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output_name, ''::character varying))::text)));
+CREATE INDEX container_requests_full_text_search_idx ON public.container_requests USING gin (to_tsvector('english'::regconfig, substr((((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text)) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(requesting_container_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(container_uuid, ''::character varying))::text) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(container_image, ''::character varying))::text) || ' '::text) || COALESCE(environment, ''::text)) || ' '::text) || (COALESCE(cwd, ''::character varying))::text) || ' '::text) || COALESCE(command, ''::text)) || ' '::text) || (COALESCE(output_path, ''::character varying))::text) || ' '::text) || COALESCE(filters, ''::text)) || ' '::text) || COALESCE(scheduling_parameters, ''::text)) || ' '::text) || (COALESCE(output_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output_name, ''::character varying))::text), 0, 1000000)));
--
-- Name: groups_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX groups_full_text_search_idx ON public.groups USING gin (to_tsvector('english'::regconfig, (((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(group_class, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text))));
+CREATE INDEX groups_full_text_search_idx ON public.groups USING gin (to_tsvector('english'::regconfig, substr((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(group_class, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)), 0, 1000000)));
--
-- Name: jobs_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX jobs_full_text_search_idx ON public.jobs USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(submit_id, ''::character varying))::text) || ' '::text) || (COALESCE(script, ''::character varying))::text) || ' '::text) || (COALESCE(script_version, ''::character varying))::text) || ' '::text) || COALESCE(script_parameters, ''::text)) || ' '::text) || (COALESCE(cancelled_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(cancelled_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output, ''::character varying))::text) || ' '::text) || (COALESCE(is_locked_by_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log, ''::character varying))::text) || ' '::text) || COALESCE(tasks_summary, ''::text)) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(repository, ''::character varying))::text) || ' '::text) || (COALESCE(supplied_script_version, ''::character varying))::text) || ' '::text) || (COALESCE(docker_image_locator, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(arvados_sdk_version, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text))));
+CREATE INDEX jobs_full_text_search_idx ON public.jobs USING gin (to_tsvector('english'::regconfig, substr((((((((((((((((((((((((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(submit_id, ''::character varying))::text) || ' '::text) || (COALESCE(script, ''::character varying))::text) || ' '::text) || (COALESCE(script_version, ''::character varying))::text) || ' '::text) || COALESCE(script_parameters, ''::text)) || ' '::text) || (COALESCE(cancelled_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(cancelled_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(output, ''::character varying))::text) || ' '::text) || (COALESCE(is_locked_by_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(log, ''::character varying))::text) || ' '::text) || COALESCE(tasks_summary, ''::text)) || ' '::text) || COALESCE(runtime_constraints, ''::text)) || ' '::text) || (COALESCE(repository, ''::character varying))::text) || ' '::text) || (COALESCE(supplied_script_version, ''::character varying))::text) || ' '::text) || (COALESCE(docker_image_locator, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || (COALESCE(arvados_sdk_version, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)), 0, 1000000)));
--
-- Name: pipeline_instances_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX pipeline_instances_full_text_search_idx ON public.pipeline_instances USING gin (to_tsvector('english'::regconfig, (((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(pipeline_template_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || COALESCE(components_summary, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
+CREATE INDEX pipeline_instances_full_text_search_idx ON public.pipeline_instances USING gin (to_tsvector('english'::regconfig, substr((((((((((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(pipeline_template_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(state, ''::character varying))::text) || ' '::text) || COALESCE(components_summary, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text), 0, 1000000)));
--
-- Name: pipeline_templates_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX pipeline_templates_full_text_search_idx ON public.pipeline_templates USING gin (to_tsvector('english'::regconfig, (((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text)));
+CREATE INDEX pipeline_templates_full_text_search_idx ON public.pipeline_templates USING gin (to_tsvector('english'::regconfig, substr((((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(components, ''::text)) || ' '::text) || (COALESCE(description, ''::character varying))::text), 0, 1000000)));
--
-- Name: workflows_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX workflows_full_text_search_idx ON public.workflows USING gin (to_tsvector('english'::regconfig, (((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text))));
+CREATE INDEX workflows_full_text_search_idx ON public.workflows USING gin (to_tsvector('english'::regconfig, substr((((((((((((COALESCE(uuid, ''::character varying))::text || ' '::text) || (COALESCE(owner_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || COALESCE(description, ''::text)), 0, 1000000)));
--
INSERT INTO schema_migrations (version) VALUES ('20180915155335');
+INSERT INTO schema_migrations (version) VALUES ('20180917200000');
+
INSERT INTO schema_migrations (version) VALUES ('20180917205609');
INSERT INTO schema_migrations (version) VALUES ('20180919001158');
c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:active).token}))
assert_equal Container::Queued, c1.state
reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
- assert_nil reused
+ # See #14584
+ assert_equal c1.uuid, reused.uuid
end
test "find_reusable method with nil runtime_token, then runtime_token with different user" do
c1, _ = minimal_new(common_attrs.merge({runtime_token: nil}))
assert_equal Container::Queued, c1.state
reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
- assert_nil reused
+ # See #14584
+ assert_equal c1.uuid, reused.uuid
end
test "find_reusable method with different runtime_token, different scope, same user" do
c1, _ = minimal_new(common_attrs.merge({runtime_token: api_client_authorizations(:runtime_token_limited_scope).token}))
assert_equal Container::Queued, c1.state
reused = Container.find_reusable(common_attrs.merge(runtime_token_attr(:container_runtime_token)))
- assert_nil reused
+ # See #14584
+ assert_equal c1.uuid, reused.uuid
end
test "Container running" do
import re
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag():
gitinfo = subprocess.check_output(
import re
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag():
gitinfo = subprocess.check_output(
"Depth", "Destination", "If", "Lock-Token", "Overwrite", "Timeout",
}, ", ")
writeMethod = map[string]bool{
- "COPY": true,
- "DELETE": true,
- "LOCK": true,
- "MKCOL": true,
- "MOVE": true,
- "PUT": true,
- "RMCOL": true,
- "UNLOCK": true,
+ "COPY": true,
+ "DELETE": true,
+ "LOCK": true,
+ "MKCOL": true,
+ "MOVE": true,
+ "PROPPATCH": true,
+ "PUT": true,
+ "RMCOL": true,
+ "UNLOCK": true,
}
webdavMethod = map[string]bool{
- "COPY": true,
- "DELETE": true,
- "LOCK": true,
- "MKCOL": true,
- "MOVE": true,
- "OPTIONS": true,
- "PROPFIND": true,
- "PUT": true,
- "RMCOL": true,
- "UNLOCK": true,
+ "COPY": true,
+ "DELETE": true,
+ "LOCK": true,
+ "MKCOL": true,
+ "MOVE": true,
+ "OPTIONS": true,
+ "PROPFIND": true,
+ "PROPPATCH": true,
+ "PUT": true,
+ "RMCOL": true,
+ "UNLOCK": true,
}
browserMethod = map[string]bool{
"GET": true,
return
}
w.Header().Set("Access-Control-Allow-Headers", corsAllowHeadersHeader)
- w.Header().Set("Access-Control-Allow-Methods", "COPY, DELETE, GET, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PUT, RMCOL")
+ w.Header().Set("Access-Control-Allow-Methods", "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Max-Age", "86400")
statusCode = http.StatusOK
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
- c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PUT, RMCOL")
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
// Check preflight for a disallowed request
exit
end
-git_latest_tag = `git describe --abbrev=0`
+git_latest_tag = `git tag -l |sort -V -r |head -n1`
git_latest_tag = git_latest_tag.encode('utf-8').strip
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H .`.chomp.split(":")
git_timestamp = Time.at(git_timestamp.to_i).utc
import re
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag():
gitinfo = subprocess.check_output(
--- /dev/null
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFWln24BEADrBl5p99uKh8+rpvqJ48u4eTtjeXAWbslJotmC/CakbNSqOb9o
+ddfzRvGVeJVERt/Q/mlvEqgnyTQy+e6oEYN2Y2kqXceUhXagThnqCoxcEJ3+KM4R
+mYdoe/BJ/J/6rHOjq7Omk24z2qB3RU1uAv57iY5VGw5p45uZB4C4pNNsBJXoCvPn
+TGAs/7IrekFZDDgVraPx/hdiwopQ8NltSfZCyu/jPpWFK28TR8yfVlzYFwibj5WK
+dHM7ZTqlA1tHIG+agyPf3Rae0jPMsHR6q+arXVwMccyOi+ULU0z8mHUJ3iEMIrpT
+X+80KaN/ZjibfsBOCjcfiJSB/acn4nxQQgNZigna32velafhQivsNREFeJpzENiG
+HOoyC6qVeOgKrRiKxzymj0FIMLru/iFF5pSWcBQB7PYlt8J0G80lAcPr6VCiN+4c
+NKv03SdvA69dCOj79PuO9IIvQsJXsSq96HB+TeEmmL+xSdpGtGdCJHHM1fDeCqkZ
+hT+RtBGQL2SEdWjxbF43oQopocT8cHvyX6Zaltn0svoGs+wX3Z/H6/8P5anog43U
+65c0A+64Jj00rNDr8j31izhtQMRo892kGeQAaaxg4Pz6HnS7hRC+cOMHUU4HA7iM
+zHrouAdYeTZeZEQOA7SxtCME9ZnGwe2grxPXh/U/80WJGkzLFNcTKdv+rwARAQAB
+tDdEb2NrZXIgUmVsZWFzZSBUb29sIChyZWxlYXNlZG9ja2VyKSA8ZG9ja2VyQGRv
+Y2tlci5jb20+iQGcBBABCgAGBQJaJYMKAAoJENNu5NUL+WcWfQML/RjicnhN0G28
++Hj3icn/SHYXg8VTHMX7aAuuClZh7GoXlvVlyN0cfRHTcFPkhv1LJ5/zFVwJxlIc
+xX0DlWbv5zlPQQQfNYH7mGCt3OS0QJGDpCM9Q6iw1EqC0CdtBDIZMGn7s9pnuq5C
+3kzer097BltvuXWI+BRMvVad2dhzuOQi76jyxhprTUL6Xwm7ytNSja5Xyigfc8HF
+rXhlQxnMEpWpTttY+En1SaTgGg7/4yB9jG7UqtdaVuAvWI69V+qzJcvgW6do5XwH
+b/5waezxOU033stXcRCYkhEenm+mXzcJYXt2avg1BYIQsZuubCBlpPtZkgWWLOf+
+eQR1Qcy9IdWQsfpH8DX6cEbeiC0xMImcuufI5KDHZQk7E7q8SDbDbk5Dam+2tRef
+eTB2A+MybVQnpsgCvEBNQ2TfcWsZ6uLHMBhesx/+rmyOnpJDTvvCLlkOMTUNPISf
+GJI0IHZFHUJ/+/uRfgIzG6dSqxQ0zHXOwGg4GbhjpQ5I+5Eg2BNRkYkCHAQQAQoA
+BgUCVsO73QAKCRBcs2HlUvsNEB8rD/4t+5uEsqDglXJ8m5dfL88ARHKeFQkW17x7
+zl7ctYHHFSFfP2iajSoAVfe5WN766TsoiHgfBE0HoLK8RRO7fxs9K7Czm6nyxB3Z
+p+YgSUZIS3wqc43jp8gd2dCCQelKIDv5rEFWHuQlyZersK9AJqIggS61ZQwJLcVY
+fUVnIdJdCmUV9haR7vIfrjNP88kqiInZWHy2t8uaB7HFPpxlNYuiJsA0w98rGQuY
+6fWlX71JnBEsgG+L73XAB0fm14QP0VvEB3njBZYlsO2do2B8rh5g51htslK5wqgC
+U61lfjnykSM8yRQbOHvPK7uYdmSF3UXqcP/gjmI9+C8s8UdnMa9rv8b8cFwpEjHu
+xeCmQKYQ/tcLOtRYZ1DIvzxETGH0xbrz6wpKuIMgY7d3xaWdjUf3ylvO0DnlXJ9Y
+r15fYndzDLPSlybIO0GrE+5grHntlSBbMa5BUNozaQ/iQBEUZ/RY+AKxy+U28JJB
+W2Wb0oun6+YdhmwgFyBoSFyp446Kz2P2A1+l/AGhzltc25Vsvwha+lRZfet464yY
+GoNBurTbQWS63JWYFoTkKXmWeS2789mQOQqka3wFXMDzVtXzmxSEbaler7lZbhTj
+wjAAJzp6kdNsPbde4lUIzt6FTdJm0Ivb47hMV4dWKEnFXrYjui0ppUH1RFUU6hyz
+IF8kfxDKO4kCHAQQAQoABgUCV0lgZQAKCRBcs2HlUvsNEHh9EACOm7QH2MGD7gI3
+0VMvapZz4Wfsbda58LFM7G5qPCt10zYfpf0dPJ7tHbHM8N9ENcI7tvH4dTfGsttt
+/uvX9PsiAml6kdfAGxoBRil+76NIHxFWsXSLVDd3hzcnRhc5njimwJa8SDBAp0kx
+v05BVWDvTbZb/b0jdgbqZk2oE0RK8S2Sp1bFkc6fl3pcJYFOQQmelOmXvPmyHOhd
+W2bLX9e1/IulzVf6zgi8dsj9IZ9aLKJY6Cz6VvJ85ML6mLGGwgNvJTLdWqntFFr0
+QqkdM8ZSp9ezWUKo28XGoxDAmo6ENNTLIZjuRlnj1Yr9mmwmf4mgucyqlU93XjCR
+y6u5bpuqoQONRPYCR/UKKk/qoGnYXnhX6AtUD+3JHvrV5mINkd/ad5eR5pviUGz+
+H/VeZqVhMbxxgkm3Gra9+bZ2pCCWboKtqIM7JtXYwks/dttkV5fTqBarJtWzcwO/
+Pv3DreTdnMoVNGzNk/84IeNmGww/iQ1Px0psVCKVPsKxr2RjNhVP7qdA0cTguFNX
+y+hx5Y/JYjSVnxIN74aLoDoeuoBhfYpOY+HiJTaM+pbLfoJr5WUPf/YUQ3qBvgG4
+WXiJUOAgsPmNY//n1MSMyhz1SvmhSXfqCVTb26IyVv0oA3UjLRcKjr18mHB5d9Fr
+NIGVHg8gJjRmXid5BZJZwKQ5niivjokCIgQQAQoADAUCV3uc0wWDB4YfgAAKCRAx
+uBWjAQZ0qe2DEACaq16AaJ2QKtOweqlGk92gQoJ2OCbIW15hW/1660u+X+2CQz8d
+nySXaq22AyBx4Do88b6d54D6TqScyObGJpGroHqAjvyh7v/t/V6oEwe34Ls2qUX2
+77lqfqsz3B0nW/aKZ2oH8ygM3tw0J5y4sAj5bMrxqcwuCs14Fds3v+K2mjsntZCu
+ztHB8mqZp/6v00d0vGGqcl6uVaS04cCQMNUkQ7tGMXlyAEIiH2ksU+/RJLaIqFtg
+klfP3Y7foAY15ymCSQPD9c81+xjbf0XNmBtDreL+rQVtesahU4Pp+Sc23iuXGdY2
+yF13wnGmScojNjM2BoUiffhFeyWBdOTgCFhOEhk0Y1zKrkNqDC0sDAj0B5vhQg/T
+10NLR2MerSk9+MJLHZqFrHXo5f59zUvte/JhtViP5TdO/Yd4ptoEcDspDKLv0FrN
+7xsP8Q6DmBz1doCe06PQS1Z1Sv4UToHRS2RXskUnDc8Cpuex5mDBQO+LV+tNToh4
+ZNcpj9lFHNuaA1qS15X3EVCySZaPyn2WRd6ZisCKtwopRmshVItTTcLmrxu+hHAF
+bVRVFRRSCE8JIZLkWwRyMrcxB2KLBYA+f2nCtD2rqiZ8K8Cr9J1qt2iu5yogCwA/
+ombzzYxWWrt/wD6ixJr5kZwBJZroHB7FkRBcTDIzDFYGBYmClACTvLuOnokCIgQS
+AQoADAUCWKy8/gWDB4YfgAAKCRAkW0txwCm5FmrGD/9lL31LQtn5wxwoZvfEKuMh
+KRw0FDUq59lQpqyMxp7lrZozFUqlH4MLTeEWbFle+R+UbUoVkBnZ/cSvVGwtRVaH
+wUeP9NAqBLtIqt4S0T2T0MW6Ug0DVH7V7uYuFktpv1xmIzcC4gV+LHhp95SPYbWr
+uVMi6ENIMZoEqW9uHOy6n2/nh76dR2NVJiZHt5LbG8YXM/Y+z3XsIenwKQ97YO7x
+yEaM7UdsQSqKVB0isTQXT2wxoA/pDvSyu7jpElD5dOtPPz3r0fQpcQKrq0IMjgcB
+u5X5tQ5uktmmdaAvIwLibUB9A+htFiFP4irSx//Lkn66RLjrSqwtMCsv7wbPvTfc
+fdpcmkR767t1VvjQWj9DBfOMjGJk9eiLkUSHYyQst6ELyVdutAIHRV2GQqfEKJzc
+cD3wKdbaOoABqRVr/ok5Oj0YKSrvk0lW3l8vS/TZXvQppSMdJuaTR8JDy6dGuoKt
+uyFDb0fKf1JU3+Gj3Yy2YEfqX0MjNQsck9pDV647UXXdzF9uh3cYVfPbl+xBYOU9
+d9qRcqMut50AVIxpUepGa4Iw7yOSRPCnPAMNAPSmAdJTaQcRWcUd9LOaZH+ZFLJZ
+mpbvS//jQpoBt++Ir8wl9ZJXICRJcvrQuhCjOSNLFzsNr/wyVLnGwmTjLWoJEA0p
+c0cYtLW6fSGknkvNA7e8LYkCMwQQAQgAHRYhBFI9KC2HD6c70cN9svEo88fgKodF
+BQJZ76NPAAoJEPEo88fgKodFYXwP+wW6F7UpNmKXaddu+aamLTe3uv8OSKUHQbRh
+By1oxfINI7iC+BZl9ycJip0S08JH0F+RZsi1H24+GcP9vGTDgu3z0NcOOD4mPpzM
+jSi2/hbGzh9C84pxRJVLAKrbqCz7YQ6JdNG4RUHW/r0QgKTnTlvikVx7n9QaPrVl
+PsVFU3xv5oQxUHpwNWyvpPGTDiycuaGKekodYhZ0vKzJzfyyaUTgfxvTVVj10jyi
+f+mSfY8YBHhDesgYF1d2CUEPth9z5KC/eDgY7KoWs8ZK6sVL3+tGrnqK/s6jqcsk
+J7Kt4c3k0jU56rUo8+jnu9yUHcBXAjtr1Vz/nwVfqmPzukIF1ZkMqdQqIRtvDyEC
+16yGngMpWEVM3/vIsi2/uUMuGvjEkEmqs2oLK1hf+Y0W6Avq+9fZUQUEk0e4wbpu
+RCqX5OjeQTEEXmAzoMsdAiwFvr1ul+eI/BPy+29OQ77hz3/dotdYYfs1JVkiFUhf
+PJwvpoUOXiA5V56wl3i5tkbRSLRSkLmiLTlCEfClHEK/wwLU4ZKuD5UpW8xL438l
+/Ycnsl7aumnofWoaEREBc1Xbnx9SZbrTT8VctW8XpMVIPxCwJCp/LqHtyEbnptnD
+7QoHtdWexFmQFUIlGaDiaL7nv0BD6RA/HwhVSxU3b3deKDYNpG9QnAzte8KXA9/s
+ejP18gCKiQI4BBMBAgAiBQJVpZ9uAhsvBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIX
+gAAKCRD3YiFXLFJgnbRfEAC9Uai7Rv20QIDlDogRzd+Vebg4ahyoUdj0CH+nAk40
+RIoq6G26u1e+sdgjpCa8jF6vrx+smpgd1HeJdmpahUX0XN3X9f9qU9oj9A4I1WDa
+lRWJh+tP5WNv2ySy6AwcP9QnjuBMRTnTK27pk1sEMg9oJHK5p+ts8hlSC4SluyMK
+H5NMVy9c+A9yqq9NF6M6d6/ehKfBFFLG9BX+XLBATvf1ZemGVHQusCQebTGv0C0V
+9yqtdPdRWVIEhHxyNHATaVYOafTj/EF0lDxLl6zDT6trRV5n9F1VCEh4Aal8L5Mx
+VPcIZVO7NHT2EkQgn8CvWjV3oKl2GopZF8V4XdJRl90U/WDv/6cmfI08GkzDYBHh
+S8ULWRFwGKobsSTyIvnbk4NtKdnTGyTJCQ8+6i52s+C54PiNgfj2ieNn6oOR7d+b
+NCcG1CdOYY+ZXVOcsjl73UYvtJrO0Rl/NpYERkZ5d/tzw4jZ6FCXgggA/Zxcjk6Y
+1ZvIm8Mt8wLRFH9Nww+FVsCtaCXJLP8DlJLASMD9rl5QS9Ku3u7ZNrr5HWXPHXIT
+X660jglyshch6CWeiUATqjIAzkEQom/kEnOrvJAtkypRJ59vYQOedZ1sFVELMXg2
+UCkD/FwojfnVtjzYaTCeGwFQeqzHmM241iuOmBYPeyTY5veF49aBJA1gEJOQTvBR
+8YkCOQQRAQgAIxYhBDlHZ/sRadXUayJzU3Es9wyw8WURBQJaajQrBYMHhh+AAAoJ
+EHEs9wyw8WURDyEP/iD903EcaiZP68IqUBsdHMxOaxnKZD9H2RTBaTjR6r9UjCOf
+bomXpVzL0dMZw1nHIE7u2VT++5wk+QvcN7epBgOWUb6tNcv3nI3vqMGRR+fKW15V
+J1sUwMOKGC4vlbLRVRWd2bb+oPZWeteOxNIqu/8DHDFHg3LtoYxWbrMYHhvd0ben
+B9GvwoqeBaqAeERKYCEoPZRB5O6ZHccX2HacjwFs4uYvIoRg4WI+ODXVHXCgOVZq
+yRuVAuQUjwkLbKL1vxJ01EWzWwRI6cY9mngFXNTHEkoxNyjzlfpn/YWheRiwpwg+
+ymDL4oj1KHNq06zNl38dZCd0rde3OFNuF904H6D+reYL50YA9lkL9mRtlaiYyo1J
+SOOjdr+qxuelfbLgDSeM75YVSiYiZZO8DWr2Cq/SNp47z4T4Il/yhQ6eAstZOIkF
+KQlBjr+ZtLdUu67sPdgPoT842IwSrRTrirEUd6cyADbRggPHrOoYEooBCrCgDYCM
+K1xxG9f6Q42yvL1zWKollibsvJF8MVwgkWfJJyhLYylmJ8osvX9LNdCJZErVrRTz
+wAM00crp/KIiIDCREEgE+5BiuGdM70gSuy3JXSs78JHA4l2tu1mDBrMxNR+C8lpj
+1pnLFHTfGYwHQSwKm42/JZqbePh6LKblUdS5Np1dl0tk5DDHBluRzhx16H7E
+=lwu7
+-----END PGP PUBLIC KEY BLOCK-----
VOLUME /var/log/nginx
VOLUME /etc/ssl/private
-RUN apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || \
- apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+ADD 58118E89F3A912897C070ADBF76221572C52609D.asc /tmp/
+RUN apt-key add --no-tty /tmp/58118E89F3A912897C070ADBF76221572C52609D.asc && \
+ rm -f /tmp/58118E89F3A912897C070ADBF76221572C52609D.asc
RUN mkdir -p /etc/apt/sources.list.d && \
echo deb https://apt.dockerproject.org/repo debian-stretch main > /etc/apt/sources.list.d/docker.list && \
import re
def git_latest_tag():
- gitinfo = subprocess.check_output(
- ['git', 'describe', '--abbrev=0']).strip()
- return str(gitinfo.decode('utf-8'))
+ gittags = subprocess.check_output(['git', 'tag', '-l']).split()
+ gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
+ return str(next(iter(gittags)).decode('utf-8'))
def git_timestamp_tag():
gitinfo = subprocess.check_output(
projects / arvados.git / commitdiff
