navsection: userguide
title: Arvados CWL Extensions
...
+ {% comment %}
+ Copyright (C) The Arvados Authors. All rights reserved.
+
+ SPDX-License-Identifier: CC-BY-SA-3.0
+ {% endcomment %}
Arvados provides several extensions to CWL for workflow optimization, site-specific configuration, and to enable access the Arvados API.
cwltool: "http://commonwl.org/cwltool#"
</pre>
-Arvados extensions should go into the @hints@ section, for example:
+For portability, Arvados extensions should go into the @hints@ section of your CWL file, for example:
<pre>
hints:
loadListing: shallow_listing
arv:IntermediateOutput:
outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
</pre>
+The one exception to this is @arv:APIRequirement@, see note below.
+
h2. arv:RunInSingleContainer
Indicates that a subworkflow should run in a single container and not be scheduled as separate steps.
|_. Field |_. Type |_. Description |
|outputTTL|int|If the value is greater than zero, consider intermediate output collections to be temporary and should be automatically trashed. Temporary collections will be trashed @outputTTL@ seconds after creation. A value of zero means intermediate output should be retained indefinitely (this is the default behavior).
Note: arvados-cwl-runner currently does not take workflow dependencies into account when setting the TTL on an intermediate output collection. If the TTL is too short, it is possible for a collection to be trashed before downstream steps that consume it are started. The recommended minimum value for TTL is the expected duration of the entire the workflow.|
+
+h2. arv:ReuseRequirement
+
+Enable/disable work reuse for current process. Default true (work reuse enabled).
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|enableReuse|boolean|Enable/disable work reuse for current process. Default true (work reuse enabled).|
#!/usr/bin/env python
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
# Implement cwl-runner interface for submitting and running work on Arvados, using
# either the Crunch jobs API or Crunch containers API.
"http://arvados.org/cwl#PartitionRequirement",
"http://arvados.org/cwl#APIRequirement",
"http://commonwl.org/cwltool#LoadListingRequirement",
- "http://arvados.org/cwl#IntermediateOutput"
+ "http://arvados.org/cwl#IntermediateOutput",
+ "http://arvados.org/cwl#ReuseRequirement"
])
def main(args, stdout, stderr, api_client=None, keep_client=None):
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
+
$base: "http://arvados.org/cwl#"
$namespaces:
cwl: "https://w3id.org/cwl/cwl#"
be trashed before downstream steps that consume it are started. The
recommended minimum value for TTL is the expected duration of the
entire the workflow.
+
+- name: ReuseRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Enable/disable work reuse for current process. Default true (work reuse enabled).
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:ReuseRequirement'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ - name: enableReuse
+ type: boolean
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
+
import logging
import json
import os
container_request["output_ttl"] = self.output_ttl
container_request["mounts"] = mounts
container_request["runtime_constraints"] = runtime_constraints
- container_request["use_existing"] = kwargs.get("enable_reuse", True)
container_request["scheduling_parameters"] = scheduling_parameters
+ enable_reuse = kwargs.get("enable_reuse", True)
+ if enable_reuse:
+ reuse_req, _ = get_feature(self, "http://arvados.org/cwl#ReuseRequirement")
+ if reuse_req:
+ enable_reuse = reuse_req["enableReuse"]
+ container_request["use_existing"] = enable_reuse
+
if kwargs.get("runnerjob", "").startswith("arvwf:"):
wfuuid = kwargs["runnerjob"][6:kwargs["runnerjob"].index("#")]
wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
+
import logging
import re
import copy
if not self.arvrunner.ignore_docker_for_reuse:
filters.append(["docker_image_locator", "in docker", runtime_constraints["docker_image"]])
+ enable_reuse = kwargs.get("enable_reuse", True)
+ if enable_reuse:
+ reuse_req, _ = get_feature(self, "http://arvados.org/cwl#ReuseRequirement")
+ if reuse_req:
+ enable_reuse = reuse_req["enableReuse"]
+
try:
with Perf(metrics, "create %s" % self.name):
response = self.arvrunner.api.jobs().create(
"runtime_constraints": runtime_constraints
},
filters=filters,
- find_or_create=kwargs.get("enable_reuse", True)
+ find_or_create=enable_reuse
).execute(num_retries=self.arvrunner.num_retries)
self.arvrunner.processes[response["uuid"]] = self
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
+
- job: dir-job.yml
output:
"outlist": {
}
tool: wf/listing_deep.cwl
doc: test deep directory listing
+
+- job: null
+ output: {}
+ tool: noreuse.cwl
+ doc: "Test arv:ReuseRequirement"
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
+
import arvados_cwl
from arvados_cwl.arvdocker import arv_docker_clear_cache
import logging
}, {
"class": "http://arvados.org/cwl#IntermediateOutput",
"outputTTL": 7200
+ }, {
+ "class": "http://arvados.org/cwl#ReuseRequirement",
+ "enableReuse": False
}],
"baseCommand": "ls"
})
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(enable_reuse=True)
call_args, call_kwargs = runner.api.container_requests().create.call_args
'keep_cache_ram': 536870912,
'API': True
},
- 'use_existing': True,
+ 'use_existing': False,
'priority': 1,
'mounts': {
'/tmp': {'kind': 'tmp',
+ # Copyright (C) The Arvados Authors. All rights reserved.
+ #
+ # SPDX-License-Identifier: Apache-2.0
+
import functools
import json
import logging
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
from .mock_discovery import get_rootDesc
- from .matcher import JsonDiffMatcher
+ from .matcher import JsonDiffMatcher, StripYAMLComments
if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
"outputDirType": "keep_output_dir"
}, {
"class": "http://arvados.org/cwl#APIRequirement",
+ },
+ {
+ "class": "http://arvados.org/cwl#ReuseRequirement",
+ "enableReuse": False
}],
"baseCommand": "ls"
}
make_fs_access=make_fs_access, loader=Loader({}))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
- j.run()
+ j.run(enable_reuse=True)
runner.api.jobs().create.assert_called_with(
body=JsonDiffMatcher({
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'keep_cache_mb_per_task': 512
}
}),
- find_or_create=True,
+ find_or_create=False,
filters=[['repository', '=', 'arvados'],
['script', '=', 'crunchrunner'],
['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
it.next().run()
with open("tests/wf/scatter2_subwf.cwl") as f:
- subwf = f.read()
+ subwf = StripYAMLComments(f.read())
runner.api.jobs().create.assert_called_with(
body=JsonDiffMatcher({
projects / arvados.git / commitdiff