From 58ebceb34531be92f74f593d532276368a87a6de Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 12 May 2022 14:31:57 -0400 Subject: [PATCH] 19070: Fix --update-workflow Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvworkflow.py | 2 +- sdk/cwl/arvados_cwl/executor.py | 7 ++++--- sdk/cwl/arvados_cwl/runner.py | 8 ++++---- sdk/cwl/tests/scripts/download_all_data.sh | 4 ++-- sdk/cwl/tests/test_copy_deps.py | 24 +++++++++++++++------- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 8eb12913ad..51e7cd8b9e 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -307,7 +307,7 @@ class ArvadosWorkflow(Workflow): if self.wf_pdh is None: adjustFileObjs(packed, keepmount) adjustDirObjs(packed, keepmount) - self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed) + self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed, runtimeContext) self.loadingContext = self.loadingContext.copy() self.loadingContext.metadata = self.loadingContext.metadata.copy() diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 70bc0c4572..1759e4ac28 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -517,7 +517,6 @@ The 'jobs' API is no longer supported. updated_tool.visit(self.check_features) - self.project_uuid = runtimeContext.project_uuid self.pipeline = None self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir) self.secret_store = runtimeContext.secret_store @@ -558,7 +557,9 @@ The 'jobs' API is no longer supported. # gets uploaded goes into the same parent project, unless # an alternate --project-uuid was provided. existing_wf = self.api.workflows().get(uuid=runtimeContext.update_workflow).execute() - self.project_uuid = existing_wf["owner_uuid"] + runtimeContext.project_uuid = existing_wf["owner_uuid"] + + self.project_uuid = runtimeContext.project_uuid # Upload local file references in the job order. job_order = upload_job_order(self, "%s input" % runtimeContext.name, @@ -604,7 +605,7 @@ The 'jobs' API is no longer supported. # Create a pipeline template or workflow record and exit. if self.work_api == "containers": uuid = upload_workflow(self, tool, job_order, - self.project_uuid, + runtimeContext.project_uuid, runtimeContext, uuid=runtimeContext.update_workflow, submit_runner_ram=runtimeContext.submit_runner_ram, diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index f39c98d882..50b3bb94d8 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -671,7 +671,7 @@ def arvados_jobs_image(arvrunner, img, runtimeContext): raise Exception("Docker image %s is not available\n%s" % (img, e) ) -def upload_workflow_collection(arvrunner, name, packed): +def upload_workflow_collection(arvrunner, name, packed, runtimeContext): collection = arvados.collection.Collection(api_client=arvrunner.api, keep_client=arvrunner.keep_client, num_retries=arvrunner.num_retries) @@ -680,15 +680,15 @@ def upload_workflow_collection(arvrunner, name, packed): filters = [["portable_data_hash", "=", collection.portable_data_hash()], ["name", "like", name+"%"]] - if arvrunner.project_uuid: - filters.append(["owner_uuid", "=", arvrunner.project_uuid]) + if runtimeContext.project_uuid: + filters.append(["owner_uuid", "=", runtimeContext.project_uuid]) exists = arvrunner.api.collections().list(filters=filters).execute(num_retries=arvrunner.num_retries) if exists["items"]: logger.info("Using collection %s", exists["items"][0]["uuid"]) else: collection.save_new(name=name, - owner_uuid=arvrunner.project_uuid, + owner_uuid=runtimeContext.project_uuid, ensure_unique_name=True, num_retries=arvrunner.num_retries) logger.info("Uploaded to %s", collection.manifest_locator()) diff --git a/sdk/cwl/tests/scripts/download_all_data.sh b/sdk/cwl/tests/scripts/download_all_data.sh index d3a9d78762..7c769b5848 100755 --- a/sdk/cwl/tests/scripts/download_all_data.sh +++ b/sdk/cwl/tests/scripts/download_all_data.sh @@ -1,7 +1,7 @@ +#!/bin/sh + # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 -#!/bin/bash - echo bubble diff --git a/sdk/cwl/tests/test_copy_deps.py b/sdk/cwl/tests/test_copy_deps.py index 2b78db8b0b..853a7d3609 100644 --- a/sdk/cwl/tests/test_copy_deps.py +++ b/sdk/cwl/tests/test_copy_deps.py @@ -10,14 +10,14 @@ api = arvados.api() def check_contents(group, wf_uuid): contents = api.groups().contents(uuid=group["uuid"]).execute() if len(contents["items"]) != 3: - raise Exception("Expected 3 items") + raise Exception("Expected 3 items in "+group["uuid"]+" was "+len(contents["items"])) found = False for c in contents["items"]: if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid: found = True if not found: - raise Exception("Couldn't find workflow") + raise Exception("Couldn't find workflow in "+group["uuid"]) found = False for c in contents["items"]: @@ -42,7 +42,9 @@ def test_create(): raise Exception("Expected 0 items") # Create workflow, by default should also copy dependencies - wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]) + cmd = ["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"] + print(" ".join(cmd)) + wf_uuid = subprocess.check_output(cmd) wf_uuid = wf_uuid.decode("utf-8").strip() check_contents(group, wf_uuid) finally: @@ -57,7 +59,9 @@ def test_update(): raise Exception("Expected 0 items") # Create workflow, but with --no-copy-deps it shouldn't copy anything - wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]) + cmd = ["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"] + print(" ".join(cmd)) + wf_uuid = subprocess.check_output(cmd) wf_uuid = wf_uuid.decode("utf-8").strip() contents = api.groups().contents(uuid=group["uuid"]).execute() @@ -72,7 +76,9 @@ def test_update(): raise Exception("Couldn't find workflow") # Updating by default will copy missing items - wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]) + cmd = ["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"] + print(" ".join(cmd)) + wf_uuid = subprocess.check_output(cmd) wf_uuid = wf_uuid.decode("utf-8").strip() check_contents(group, wf_uuid) @@ -88,7 +94,9 @@ def test_execute(): raise Exception("Expected 0 items") # Execute workflow, shouldn't copy anything. - wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]) + cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"] + print(" ".join(cmd)) + wf_uuid = subprocess.check_output(cmd) wf_uuid = wf_uuid.decode("utf-8").strip() contents = api.groups().contents(uuid=group["uuid"]).execute() @@ -115,7 +123,9 @@ def test_execute(): raise Exception("Didn't expect to find jobs image dependency") # Execute workflow with --copy-deps - wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]) + cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"] + print(" ".join(cmd)) + wf_uuid = subprocess.check_output(cmd) wf_uuid = wf_uuid.decode("utf-8").strip() contents = api.groups().contents(uuid=group["uuid"]).execute() -- 2.30.2