type: string
doc: The property key
- name: propertyValue
- type: [cwl:Expression, Any]
+ type: [Any]
doc: The property value
_type: "@vocab"
processProperties:
type: PropertyDef[]
- jsonldPredicate:
- mapSubject: propertyName
- mapPredicate: propertyValue
+ #jsonldPredicate:
+ # mapSubject: propertyName
+ # mapPredicate: propertyValue
from .arvdocker import arv_docker_get_image
from . import done
-from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
+from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields, make_builder
from .fsaccess import CollectionFetcher
from .pathmapper import NoFollowPathMapper, trim_listing
from .perf import Perf
properties_req, _ = self.embedded_tool.get_requirement("http://arvados.org/cwl#ProcessProperties")
if properties_req:
builder = make_builder(self.job_order, self.embedded_tool.hints, self.embedded_tool.requirements, runtimeContext, self.embedded_tool.metadata)
+ print("ZZZZZZ", properties_req["processProperties"])
for pr in properties_req["processProperties"]:
- container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+ print("ZZZZZZ", pr)
+ container_req["properties"][pr["propertyName"]] = builder.do_eval(pr["propertyValue"])
# --local means execute the workflow instead of submitting a container request
# --api=containers means use the containers API
}))
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_setting_process_properties(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ tool = cmap({
+ "inputs": [
+ {"id": "x", "type": "string"}],
+ "outputs": [],
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://arvados.org/cwl#ProcessProperties",
+ "processProperties": [
+ {"propertyName": "foo",
+ "propertyValue": "bar"},
+ {"propertyName": "baz",
+ "propertyValue": "$(inputs.x)"},
+ {"propertyName": "quux",
+ "propertyValue": {
+ "q1": 1,
+ "q2": 2
+ }
+ }
+ ],
+ }
+ ]
+ })
+
+ loadingContext, runtimeContext = self.helper(runner, True)
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['ls', '/var/spool/cwl'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {
+ "baz": "blorp",
+ "foo": "bar",
+ "quux": {
+ "q1": 1,
+ "q2": 2
+ }
+ },
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
+
+
class TestWorkflow(unittest.TestCase):
def setUp(self):
cwltool.process._names = set()
finally:
cwltool_logger.removeHandler(stderr_logger)
+ @stubs
+ def test_submit_set_process_properties(self, stubs):
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_wf_process_properties.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ expect_container["name"] = "submit_wf_process_properties.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ {
+ "class": "http://arvados.org/cwl#ProcessProperties",
+ "processProperties": [
+ {"propertyName": "foo",
+ "propertyValue": "bar"},
+ {"propertyName": "baz",
+ "propertyValue": "$(inputs.x.basename)"},
+ {"propertyName": "quux",
+ "propertyValue": {
+ "q1": 1,
+ "q2": 2
+ }
+ }
+ ],
+ }
+ ]
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = {
+ "arv": "http://arvados.org/cwl#"
+ }
+
+ expect_container["properties"] = {
+ "baz": "blorp.txt",
+ "foo": "bar",
+ "quux": {
+ "q1": 1,
+ "q2": 2
+ }
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"