From 1c95615a417b6435dc86f80e3f02563196449611 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 4 Oct 2021 21:33:00 -0400 Subject: [PATCH] Fix CWL tests refs #18238 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/__init__.py | 2 +- sdk/cwl/arvados_cwl/arvcontainer.py | 4 ++-- sdk/cwl/arvados_cwl/executor.py | 14 +++++++++----- sdk/cwl/tests/test_container.py | 9 +++++++++ sdk/cwl/tests/test_submit.py | 14 ++++++++------ sdk/cwl/tests/wf/scatter2_subwf.cwl | 5 ++++- 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index ee636be371..71ef742e31 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -301,7 +301,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None, api_client.users().current().execute() if keep_client is None: keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4) - executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4) + executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4, stdout=stdout) except WorkflowException as e: logger.error(e, exc_info=(sys.exc_info()[1] if arvargs.debug else False)) return 1 diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 560e8979ec..ae3c668895 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -520,10 +520,10 @@ class RunnerContainer(Runner): if runtimeContext.debug: command.append("--debug") - if runtimeContext.storage_classes != "default": + if runtimeContext.storage_classes != "default" and runtimeContext.storage_classes: command.append("--storage-classes=" + runtimeContext.storage_classes) - if runtimeContext.intermediate_storage_classes != "default": + if runtimeContext.intermediate_storage_classes != "default" and runtimeContext.intermediate_storage_classes: command.append("--intermediate-storage-classes=" + runtimeContext.intermediate_storage_classes) if self.on_error: diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index db1db28162..aa19633d8c 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -99,7 +99,8 @@ class ArvCwlExecutor(object): arvargs=None, keep_client=None, num_retries=4, - thread_count=4): + thread_count=4, + stdout=sys.stdout): if arvargs is None: arvargs = argparse.Namespace() @@ -132,6 +133,7 @@ class ArvCwlExecutor(object): self.should_estimate_cache_size = True self.fs_access = None self.secret_store = None + self.stdout = stdout if keep_client is not None: self.keep_client = keep_client @@ -602,14 +604,15 @@ The 'jobs' API is no longer supported. if existing_uuid or runtimeContext.create_workflow: # Create a pipeline template or workflow record and exit. if self.work_api == "containers": - return (upload_workflow(self, tool, job_order, + uuid = upload_workflow(self, tool, job_order, self.project_uuid, uuid=existing_uuid, submit_runner_ram=runtimeContext.submit_runner_ram, name=runtimeContext.name, merged_map=merged_map, - submit_runner_image=runtimeContext.submit_runner_image), - "success") + submit_runner_image=runtimeContext.submit_runner_image) + self.stdout.write(uuid + "\n") + return (None, "success") self.apply_reqs(job_order, tool) @@ -679,7 +682,8 @@ The 'jobs' API is no longer supported. if runtimeContext.submit and not runtimeContext.wait: runnerjob = next(jobiter) runnerjob.run(runtimeContext) - return (runnerjob.uuid, "success") + self.stdout.write(runnerjob.uuid+"\n") + return (None, "success") current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) if current_container: diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 8a380ff80b..1a2bd112f3 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -112,6 +112,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -175,6 +176,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 3600 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -270,6 +272,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -398,6 +401,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -622,6 +626,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -712,6 +717,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -808,6 +814,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -851,6 +858,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -924,6 +932,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 8c0fcaf741..77f70851e8 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -87,6 +87,7 @@ def stubs(func): stubs.api = mock.MagicMock() stubs.api._rootDesc = get_rootDesc() stubs.api._rootDesc["uuidPrefix"] = "zzzzz" + stubs.api._rootDesc["revision"] = "20210628" stubs.api.users().current().execute.return_value = { "uuid": stubs.fake_user_uuid, @@ -446,7 +447,7 @@ class TestSubmit(unittest.TestCase): "enableReuse": False, }, ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = { + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { "arv": "http://arvados.org/cwl#", "cwltool": "http://commonwl.org/cwltool#" } @@ -572,6 +573,7 @@ class TestSubmit(unittest.TestCase): def test_default_storage_classes_correctly_propagate_to_make_output_collection(self, stubs, make_output, job, tq): final_output_c = arvados.collection.Collection() make_output.return_value = ({},final_output_c) + stubs.api.config().get.return_value = {"default": {"Default": True}} def set_final_output(job_order, output_callback, runtimeContext): output_callback("zzzzz-4zz18-zzzzzzzzzzzzzzzz", "success") @@ -1032,7 +1034,7 @@ class TestSubmit(unittest.TestCase): "keep_cache": 512 } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = { + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { "arv": "http://arvados.org/cwl#", } expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', @@ -1126,9 +1128,6 @@ class TestSubmit(unittest.TestCase): "content": { "$graph": [ { - "$namespaces": { - "cwltool": "http://commonwl.org/cwltool#" - }, "arguments": [ "md5sum", "example.conf" @@ -1217,6 +1216,9 @@ class TestSubmit(unittest.TestCase): ] } ], + "$namespaces": { + "cwltool": "http://commonwl.org/cwltool#" + }, "cwlVersion": "v1.0" }, "kind": "json" @@ -1445,7 +1447,7 @@ class TestSubmit(unittest.TestCase): ], } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["$namespaces"] = { + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { "arv": "http://arvados.org/cwl#" } diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl index c54e1707ff..218b0c5018 100644 --- a/sdk/cwl/tests/wf/scatter2_subwf.cwl +++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl @@ -86,5 +86,8 @@ ] } ], + "$namespaces": { + "arv": "http://arvados.org/cwl#" + }, "cwlVersion": "v1.0" -} \ No newline at end of file +} -- 2.30.2