+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados_cwl
from arvados_cwl.arvdocker import arv_docker_clear_cache
import logging
import os
import functools
import cwltool.process
+import cwltool.secrets
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
"inputs": [],
"outputs": [],
"baseCommand": "ls",
- "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool"
})
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run(enable_reuse=enable_reuse)
+ j.run(enable_reuse=enable_reuse, priority=500)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
'ram': 1073741824
},
'use_existing': enable_reuse,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 1073741824
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
+ 'secret_mounts': {}
}))
# The test passes some fields in builder.resources
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 3600
+ runner.secret_store = cwltool.secrets.SecretStore()
+
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
}, {
"class": "http://arvados.org/cwl#IntermediateOutput",
"outputTTL": 7200
+ }, {
+ "class": "http://arvados.org/cwl#ReuseRequirement",
+ "enableReuse": False
}],
- "baseCommand": "ls"
+ "baseCommand": "ls",
+ "id": "#",
+ "class": "CommandLineTool"
})
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(enable_reuse=True, priority=500)
call_args, call_kwargs = runner.api.container_requests().create.call_args
'keep_cache_ram': 536870912,
'API': True
},
- 'use_existing': True,
- 'priority': 1,
+ 'use_existing': False,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 4194304000 },
'scheduling_parameters': {
'partitions': ['blurb']
},
- 'properties': {}
+ 'properties': {},
+ 'secret_mounts': {}
}
call_body = call_kwargs.get('body', None)
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
"location": "keep:99999999999999999999999999999995+99/subdir"
} ]
}],
- "baseCommand": "ls"
+ "baseCommand": "ls",
+ "id": "#",
+ "class": "CommandLineTool"
})
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(priority=500)
call_args, call_kwargs = runner.api.container_requests().create.call_args
'ram': 1073741824
},
'use_existing': True,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 1073741824 },
'cwd': '/var/spool/cwl',
'scheduling_parameters': {
},
- 'properties': {}
+ 'properties': {},
+ 'secret_mounts': {}
}
call_body = call_kwargs.get('body', None)
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
"stdout": "stdout.txt",
"stderr": "stderr.txt",
"stdin": "/keep/99999999999999999999999999999996+99/file.txt",
- "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool"
})
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool.formatgraph = None
for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(priority=500)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
'ram': 1073741824
},
'use_existing': True,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
'/tmp': {'kind': 'tmp',
"capacity": 1073741824 },
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
+ 'secret_mounts': {}
}))
@mock.patch("arvados.collection.Collection")
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
runner.api.containers().get().execute.return_value = {"state":"Complete",
"output": "abc+123",
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
runner.api.collections().get().execute.return_value = {
],
"outputs": [],
"baseCommand": "ls",
- "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool"
})
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
}
for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
make_fs_access=make_fs_access, tmpdir="/tmp"):
- j.run()
+ j.run(priority=500)
runner.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher({
'environment': {
'ram': 1073741824
},
'use_existing': True,
- 'priority': 1,
+ 'priority': 500,
'mounts': {
"/keep/99999999999999999999999999999994+44": {
"kind": "collection",
'cwd': '/var/spool/cwl',
'scheduling_parameters': {},
'properties': {},
+ 'secret_mounts': {}
+ }))
+
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_secrets(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ tool = cmap({"arguments": ["md5sum", "example.conf"],
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://commonwl.org/cwltool#Secrets",
+ "secrets": [
+ "#secret_job.cwl/pw"
+ ]
+ }
+ ],
+ "id": "#secret_job.cwl",
+ "inputs": [
+ {
+ "id": "#secret_job.cwl/pw",
+ "type": "string"
+ }
+ ],
+ "outputs": [
+ ],
+ "requirements": [
+ {
+ "class": "InitialWorkDirRequirement",
+ "listing": [
+ {
+ "entry": "username: user\npassword: $(inputs.pw)\n",
+ "entryname": "example.conf"
+ }
+ ]
+ }
+ ]})
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
+ basedir="", make_fs_access=make_fs_access, loader=Loader({}))
+ arvtool.formatgraph = None
+
+ job_order = {"pw": "blorp"}
+ runner.secret_store.store(["pw"], job_order)
+
+ for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_secrets",
+ make_fs_access=make_fs_access, tmpdir="/tmp"):
+ j.run(enable_reuse=True, priority=500)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_secrets',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': 'arvados/jobs',
+ 'command': ['md5sum', 'example.conf'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ "secret_mounts": {
+ "/var/spool/cwl/example.conf": {
+ "content": "username: user\npassword: blorp\n",
+ "kind": "text"
+ }
+ }
}))
projects / arvados.git / blobdiff