From 50bd5e3f71a1eb18c7721fc44a934413463baa7c Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 12 Jan 2023 17:09:54 -0500 Subject: [PATCH] 19385: Fixing tests Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvworkflow.py | 13 +++-- sdk/cwl/arvados_cwl/runner.py | 7 ++- sdk/cwl/tests/test_submit.py | 84 ++++++++++++++++++++++-------- 3 files changed, 75 insertions(+), 29 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 299ad5c0d5..614530c3ea 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -216,7 +216,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid, prefix = firstfile[:n+1] - col = arvados.collection.Collection() + col = arvados.collection.Collection(api_client=arvRunner.api) for w in workflow_files | import_files: # 1. load YAML @@ -260,6 +260,11 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid, "arv:workflowMain": toolfile, } + if git_info: + for g in git_info: + p = g.split("#", 1)[1] + properties["arv:"+p] = git_info[g] + col.save_new(name=toolname, owner_uuid=arvRunner.project_uuid, ensure_unique_name=True, properties=properties) adjustDirObjs(job_order, trim_listing) @@ -292,9 +297,9 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid, hints.append(wf_runner_resources) # uncomment me - # wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, - # submit_runner_image or "arvados/jobs:"+__version__, - # runtimeContext) + wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, + submit_runner_image or "arvados/jobs:"+__version__, + runtimeContext) if submit_runner_ram: wf_runner_resources["ramMin"] = submit_runner_ram diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index c2c992d440..026eb30995 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -714,8 +714,8 @@ def upload_workflow_deps(arvrunner, tool, runtimeContext): # Ensure that Docker images needed by this workflow are available # commented out for testing only, uncomment me - #with Perf(metrics, "upload_docker"): - # upload_docker(arvrunner, tool, runtimeContext) + with Perf(metrics, "upload_docker"): + upload_docker(arvrunner, tool, runtimeContext) document_loader = tool.doc_loader @@ -818,6 +818,9 @@ class Runner(Process): reuse_req, _ = self.embedded_tool.get_requirement("http://arvados.org/cwl#ReuseRequirement") if reuse_req: enable_reuse = reuse_req["enableReuse"] + reuse_req, _ = self.embedded_tool.get_requirement("WorkReuse") + if reuse_req: + enable_reuse = reuse_req["enableReuse"] self.enable_reuse = enable_reuse self.uuid = None self.final_output = None diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 4fbca5c052..1280095a8f 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -200,14 +200,18 @@ def stubs(wfdetails=('submit_wf.cwl', None)): mocktool = mock.NonCallableMock(tool=gitinfo_workflow["$graph"][0], metadata=gitinfo_workflow) stubs.git_info = arvados_cwl.executor.ArvCwlExecutor.get_git_info(mocktool) - #expect_packed_workflow.update(stubs.git_info) + expect_packed_workflow.update(stubs.git_info) stubs.git_props = {"arv:"+k.split("#", 1)[1]: v for k,v in stubs.git_info.items()} - #if wfname == wfpath: - # container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"]) - #else: - container_name = wfname + step_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"]) + if wfname == wfpath: + container_name = "%s (%s)" % (wfpath, stubs.git_props["arv:gitDescribe"]) + else: + container_name = wfname + + expect_packed_workflow["$graph"][0]["steps"][0]["id"] = "#main/"+step_name + expect_packed_workflow["$graph"][0]["steps"][0]["label"] = container_name stubs.expect_container_spec = { 'priority': 500, @@ -267,11 +271,10 @@ def stubs(wfdetails=('submit_wf.cwl', None)): 'vcpus': 1, 'ram': (1024+256)*1024*1024 }, - 'properties': {}, + 'properties': stubs.git_props, 'use_existing': False, 'secret_mounts': {} } - #'properties': stubs.git_props, stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz" stubs.api.workflows().create().execute.return_value = { @@ -315,7 +318,7 @@ class TestSubmit(unittest.TestCase): @stubs() def test_submit_container(self, stubs): exited = arvados_cwl.main( - ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-git", + ["--submit", "--no-wait", "--api=containers", "--debug", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) @@ -324,7 +327,7 @@ class TestSubmit(unittest.TestCase): 'manifest_text': '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n', 'replication_desired': None, - 'name': 'submit_wf.cwl input (169f39d466a5438ac4a90e779bf750c7+53)', + 'name': 'submit_wf.cwl ('+ stubs.git_props["arv:gitDescribe"] +') input (169f39d466a5438ac4a90e779bf750c7+53)', }), ensure_unique_name=False), mock.call(body=JsonDiffMatcher({ 'manifest_text': @@ -389,16 +392,25 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ["--disable-reuse" if v == "--enable-reuse" else v for v in expect_container["command"]] expect_container["use_existing"] = False - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [ + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [ + { + "class": "LoadListingRequirement", + "loadListing": "deep_listing" + }, + { + "class": "NetworkAccess", + "networkAccess": True + }, { - "class": "http://arvados.org/cwl#ReuseRequirement", + "class": "WorkReuse", "enableReuse": False, }, + { + "acrContainerImage": "999999999999999999999999999999d3+99", + "class": "http://arvados.org/cwl#WorkflowRunnerResources" + } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { - "arv": "http://arvados.org/cwl#", - "cwltool": "http://commonwl.org/cwltool#" - } + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:40eac42ef5535aca47ca5e47b5786f58+137/wf/submit_wf_no_reuse.cwl" stubs.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(expect_container)) @@ -666,6 +678,8 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs() def test_submit_keepref(self, stubs, tm, reader): + raise Exception("broken") + with open("tests/wf/expect_arvworkflow.cwl") as f: reader().open().__enter__().read.return_value = f.read() @@ -991,18 +1005,28 @@ class TestSubmit(unittest.TestCase): "vcpus": 2, "ram": (2000+512) * 2**20 } - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [ + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [ + { + "class": "LoadListingRequirement", + "loadListing": "deep_listing" + }, + { + "class": "NetworkAccess", + "networkAccess": True + }, { "class": "http://arvados.org/cwl#WorkflowRunnerResources", + "acrContainerImage": "999999999999999999999999999999d3+99", "coresMin": 2, "ramMin": 2000, "keep_cache": 512 } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { - "arv": "http://arvados.org/cwl#", - } + #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { + # "arv": "http://arvados.org/cwl#", + #} expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]] + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:48136b8a3e0b5768ea179729309a365e+145/wf/submit_wf_runner_resources.cwl" stubs.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(expect_container)) @@ -1414,7 +1438,15 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][1]["hints"] = [ + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["hints"] = [ + { + "class": "LoadListingRequirement", + "loadListing": "deep_listing" + }, + { + "class": "NetworkAccess", + "networkAccess": True + }, { "class": "http://arvados.org/cwl#ProcessProperties", "processProperties": [ @@ -1429,11 +1461,17 @@ class TestSubmit(unittest.TestCase): } } ], + }, + { + "acrContainerImage": "999999999999999999999999999999d3+99", + "class": "http://arvados.org/cwl#WorkflowRunnerResources" } ] - expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { - "arv": "http://arvados.org/cwl#" - } + #expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$namespaces"] = { + # "arv": "http://arvados.org/cwl#" + #} + + expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:a068c0b781383b8a27ec5c04a355295b+147/wf/submit_wf_process_properties.cwl" expect_container["properties"].update({ "baz": "blorp.txt", -- 2.30.2