import copy
import io
+import itertools
import functools
import hashlib
import json
from .matcher import JsonDiffMatcher, StripYAMLComments
from .mock_discovery import get_rootDesc
-import ruamel.yaml as yaml
+import ruamel.yaml
_rootDesc = None
uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
"df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
- "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5"]
+ "df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
+ "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
+ "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9",
+ "df80736f-f14d-4b10-b2e3-03aa27f034c0", "df80736f-f14d-4b10-b2e3-03aa27f034c1",
+ "df80736f-f14d-4b10-b2e3-03aa27f034c2", "df80736f-f14d-4b10-b2e3-03aa27f034c3",
+ "df80736f-f14d-4b10-b2e3-03aa27f034c4", "df80736f-f14d-4b10-b2e3-03aa27f034c5",
+ "df80736f-f14d-4b10-b2e3-03aa27f034c6", "df80736f-f14d-4b10-b2e3-03aa27f034c7"]
determine_image_id.return_value = None
stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
- stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
- stubs.api.jobs().create().execute.return_value = {
- "uuid": stubs.expect_job_uuid,
- "state": "Queued",
- }
-
stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
stubs.api.container_requests().create().execute.return_value = {
"uuid": stubs.expect_container_request_uuid,
"state": "Queued"
}
- stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
- stubs.api.pipeline_templates().create().execute.return_value = {
- "uuid": stubs.expect_pipeline_template_uuid,
- }
- stubs.expect_job_spec = {
- 'runtime_constraints': {
- 'docker_image': '999999999999999999999999999999d3+99',
- 'min_ram_mb_per_node': 1024
- },
- 'script_parameters': {
- 'x': {
- 'basename': 'blorp.txt',
- 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
- 'class': 'File'
- },
- 'y': {
- 'basename': '99999999999999999999999999999998+99',
- 'location': 'keep:99999999999999999999999999999998+99',
- 'class': 'Directory'
- },
- 'z': {
- 'basename': 'anonymous',
- "listing": [{
- "basename": "renamed.txt",
- "class": "File",
- "location": "keep:99999999999999999999999999999998+99/file1.txt",
- "size": 0
- }],
- 'class': 'Directory'
- },
- 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main'
- },
- 'repository': 'arvados',
- 'script_version': 'master',
- 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
- 'script': 'cwl-runner'
- }
- stubs.pipeline_component = stubs.expect_job_spec.copy()
- stubs.expect_pipeline_instance = {
- 'name': 'submit_wf.cwl',
- 'state': 'RunningOnServer',
- 'owner_uuid': None,
- "components": {
- "cwl-runner": {
- 'runtime_constraints': {'docker_image': '999999999999999999999999999999d3+99', 'min_ram_mb_per_node': 1024},
- 'script_parameters': {
- 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
- 'x': {"value": {
- 'basename': 'blorp.txt',
- 'class': 'File',
- 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
- "size": 16
- }},
- 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
- 'listing': [
- {
- 'basename': 'renamed.txt',
- 'class': 'File', 'location':
- 'keep:99999999999999999999999999999998+99/file1.txt',
- 'size': 0
- }
- ]}},
- 'cwl:tool': '57ad063d64c60dbddc027791f0649211+60/workflow.cwl#main',
- 'arv:debug': True,
- 'arv:enable_reuse': True,
- 'arv:on_error': 'continue'
- },
- 'repository': 'arvados',
- 'script_version': 'master',
- 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
- 'script': 'cwl-runner',
- 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
- }
- }
- }
- stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
- stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
- stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
- stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
- stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
- "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
- "state": "Queued"
- }
- stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
- stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
-
cwd = os.getcwd()
- filepath = os.path.join(cwd, "tests/wf/submit_wf_packed.cwl")
+ filepath = os.path.join(cwd, "tests/wf/submit_wf_wrapper.cwl")
with open(filepath) as f:
- expect_packed_workflow = yaml.round_trip_load(f)
+ yaml = ruamel.yaml.YAML(typ='rt', pure=True)
+ expect_packed_workflow = yaml.load(f)
if wfpath is None:
wfpath = wfname
stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()}
+ step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
if wfname == wfpath:
container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"])
else:
container_name = wfname
+ expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name
+ expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name
+
stubs.expect_container_spec = {
'priority': 500,
'mounts': {
'vcpus': 1,
'ram': (1024+256)*1024*1024
},
- 'use_existing': False,
'properties': stubs.git_props,
+ 'use_existing': False,
'secret_mounts': {}
}
def setUp(self):
cwltool.process._names = set()
- arvados_cwl.arvdocker.arv_docker_clear_cache()
+ #arvados_cwl.arvdocker.arv_docker_clear_cache()
def tearDown(self):
root_logger = logging.getLogger('')
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]]
expect_container["use_existing"] = False
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
{
- "class": "http://arvados.org/cwl#ReuseRequirement",
+ "class": "WorkReuse",
"enableReuse": False,
},
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
- "arv": "http://arvados.org/cwl#",
- "cwltool": "http://commonwl.org/cwltool#"
- }
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "ramMin": 2048
+ }
+ ]
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
'kind': 'json',
'content': {
'cwlVersion': 'v1.0',
+ 'label': 'a test workflow',
'$graph': [
{
'id': '#main',
stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
+ stubs.expect_container_spec["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
+ {
+ "acrContainerImage": "999999999999999999999999999999d5+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ]
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
"vcpus": 2,
"ram": (2000+512) * 2**20
}
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
{
"class": "http://arvados.org/cwl#WorkflowRunnerResources",
+ "acrContainerImage": "999999999999999999999999999999d3+99",
"coresMin": 2,
"ramMin": 2000,
"keep_cache": 512
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
- "arv": "http://arvados.org/cwl#",
- }
+ #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
+ # "arv": "http://arvados.org/cwl#",
+ #}
expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@mock.patch("cwltool.docker.DockerCommandLineJob.get_image")
@mock.patch("arvados.api")
def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
- arvados_cwl.arvdocker.arv_docker_clear_cache()
+ #arvados_cwl.arvdocker.arv_docker_clear_cache()
arvrunner = mock.MagicMock()
arvrunner.project_uuid = ""
api.return_value = mock.MagicMock()
arvrunner.api = api.return_value
arvrunner.runtimeContext.match_local_docker = False
- arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
- "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "link_class": "docker_image_repo+tag",
- "name": "arvados/jobs:"+arvados_cwl.__version__,
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
- "link_class": "docker_image_hash",
- "name": "123456",
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "link_class": "docker_image_repo+tag",
- "name": "arvados/jobs:"+arvados_cwl.__version__,
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
- "link_class": "docker_image_hash",
- "name": "123456",
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
- )
+ arvrunner.api.links().list().execute.side_effect = itertools.cycle([
+ {"items": [{"created_at": "2023-08-25T12:34:56.123456Z",
+ "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "link_class": "docker_image_repo+tag",
+ "name": "arvados/jobs:"+arvados_cwl.__version__,
+ "owner_uuid": "",
+ "uuid": "zzzzz-o0j2j-arvadosjobsrepo",
+ "properties": {"image_timestamp": ""}}]},
+ {"items": []},
+ {"items": []},
+ {"items": [{"created_at": "2023-08-25T12:34:57.234567Z",
+ "head_uuid": "",
+ "link_class": "docker_image_hash",
+ "name": "123456",
+ "owner_uuid": "",
+ "uuid": "zzzzz-o0j2j-arvadosjobshash",
+ "properties": {"image_timestamp": ""}}]},
+ {"items": []},
+ {"items": []},
+ ])
find_one_image_hash.return_value = "123456"
- arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "owner_uuid": "",
- "manifest_text": "",
- "properties": ""
- }], "items_available": 1, "offset": 0},
- {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
- "owner_uuid": "",
- "manifest_text": "",
- "properties": ""
- }], "items_available": 1, "offset": 0})
+ arvrunner.api.collections().list().execute.side_effect = itertools.cycle([
+ {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
+ "owner_uuid": "",
+ "manifest_text": "",
+ "created_at": "2023-08-25T12:34:55.012345Z",
+ "properties": {}}]},
+ {"items": []},
+ {"items": []},
+ ])
arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
"portable_data_hash": "9999999999999999999999999999999b+99"}
@stubs()
def test_submit_secrets(self, stubs):
exited = arvados_cwl.main(
- ["--submit", "--no-wait", "--api=containers", "--debug",
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git",
"tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
'--thread-count=0',
"--enable-reuse",
"--collection-cache-size=256",
- '--output-name=Output from workflow secret_wf.cwl (%s)' % stubs.git_props["arv:gitDescribe"],
+ "--output-name=Output from workflow secret_wf.cwl",
"--debug",
"--on-error=continue",
"/var/lib/cwl/workflow.json#main",
"/var/lib/cwl/workflow.json": {
"content": {
"$graph": [
- {
- "arguments": [
- "md5sum",
- "example.conf"
- ],
- "class": "CommandLineTool",
- "hints": [
- {
- "class": "http://commonwl.org/cwltool#Secrets",
- "secrets": [
- "#secret_job.cwl/pw"
- ]
- }
- ],
- "id": "#secret_job.cwl",
- "inputs": [
- {
- "id": "#secret_job.cwl/pw",
- "type": "string"
- }
- ],
- "outputs": [
- {
- "id": "#secret_job.cwl/out",
- "type": "File",
- "outputBinding": {
- "glob": "hashed_example.txt"
- }
- }
- ],
- "stdout": "hashed_example.txt",
- "requirements": [
- {
- "class": "InitialWorkDirRequirement",
- "listing": [
- {
- "entry": "username: user\npassword: $(inputs.pw)\n",
- "entryname": "example.conf"
- }
- ]
- }
- ]
- },
{
"class": "Workflow",
"hints": [
{
- "class": "DockerRequirement",
- "dockerPull": "debian:buster-slim",
- "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ "class": "DockerRequirement",
+ "dockerPull": "debian:buster-slim",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
},
{
"class": "http://commonwl.org/cwltool#Secrets",
"secrets": [
"#main/pw"
]
+ },
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
}
],
"id": "#main",
"outputs": [
{
"id": "#main/out",
- "outputSource": "#main/step1/out",
+ "outputSource": "#main/step/out",
"type": "File"
}
],
+ "requirements": [
+ {
+ "class": "SubworkflowFeatureRequirement"
+ }
+ ],
"steps": [
{
- "id": "#main/step1",
+ "id": "#main/secret_wf.cwl",
"in": [
{
- "id": "#main/step1/pw",
+ "id": "#main/step/pw",
"source": "#main/pw"
}
],
+ "label": "secret_wf.cwl",
"out": [
- "#main/step1/out"
+ {"id": "#main/step/out"}
],
- "run": "#secret_job.cwl"
+ "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
}
]
}
],
- "$namespaces": {
- "cwltool": "http://commonwl.org/cwltool#"
- },
- "cwlVersion": "v1.0"
+ "cwlVersion": "v1.2"
},
"kind": "json"
},
"path": "/var/spool/cwl/cwl.output.json"
}
},
- "name": "secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
- "output_name": "Output from workflow secret_wf.cwl (%s)" % stubs.git_props["arv:gitDescribe"],
+ "name": "secret_wf.cwl",
+ "output_name": "Output from workflow secret_wf.cwl",
"output_path": "/var/spool/cwl",
"priority": 500,
- "properties": stubs.git_props,
+ "properties": {},
"runtime_constraints": {
"API": True,
"ram": 1342177280,
"use_existing": False
}
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"].update(stubs.git_info)
-
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
self.assertEqual(stubs.capture_stdout.getvalue(),
m.execute.return_value = {"items": []}
return m
stubs.api.collections().list.side_effect = list_side_effect
+ collectionReader().portable_data_hash.return_value = "99999999999999999999999999999998+99"
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
expect_container = copy.deepcopy(stubs.expect_container_spec)
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [
{
"class": "http://arvados.org/cwl#ProcessProperties",
"processProperties": [
}
}
],
+ },
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
- "arv": "http://arvados.org/cwl#"
- }
+ #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = {
+ # "arv": "http://arvados.org/cwl#"
+ #}
+
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
expect_container["properties"].update({
"baz": "blorp.txt",
def setUp(self):
cwltool.process._names = set()
- arvados_cwl.arvdocker.arv_docker_clear_cache()
+ #arvados_cwl.arvdocker.arv_docker_clear_cache()
def tearDown(self):
root_logger = logging.getLogger('')
self.assertEqual(stubs.capture_stdout.getvalue(),
stubs.expect_workflow_uuid + '\n')
self.assertEqual(exited, 0)
+
+ @stubs()
+ def test_create_map(self, stubs):
+ # test uploading a document that uses objects instead of arrays
+ # for certain fields like inputs and requirements.
+
+ project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+ stubs.api.groups().get().execute.return_value = {"group_class": "project"}
+
+ exited = arvados_cwl.main(
+ ["--create-workflow", "--debug",
+ "--api=containers",
+ "--project-uuid", project_uuid,
+ "--disable-git",
+ "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+ stubs.api.pipeline_templates().create.refute_called()
+ stubs.api.container_requests().create.refute_called()
+
+ expect_workflow = StripYAMLComments(
+ open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
+
+ body = {
+ "workflow": {
+ "owner_uuid": project_uuid,
+ "name": "submit_wf_map.cwl",
+ "description": "",
+ "definition": expect_workflow,
+ }
+ }
+ stubs.api.workflows().create.assert_called_with(
+ body=JsonDiffMatcher(body))
+
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_workflow_uuid + '\n')
+ self.assertEqual(exited, 0)
+
+
+class TestPrintKeepDeps(unittest.TestCase):
+ @stubs()
+ def test_print_keep_deps(self, stubs):
+ # test --print-keep-deps which is used by arv-copy
+
+ exited = arvados_cwl.main(
+ ["--print-keep-deps", "--debug",
+ "tests/wf/submit_wf_map.cwl"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
+ self.assertEqual(exited, 0)
projects / arvados.git / blobdiff