From: Peter Amstutz Date: Wed, 4 May 2022 20:57:01 +0000 (-0400) Subject: 19070: Add integration test for copying dependencies X-Git-Tag: 2.5.0~179^2~5 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a4d6d061cb30cc2f77d4d64283ee998bb3e3a97b 19070: Add integration test for copying dependencies Fix python tests Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index ed57c6dae0..eace9f449f 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -544,10 +544,10 @@ The 'jobs' API is no longer supported. if not runtimeContext.name: runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"]) - if runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow): - runtimeContext.copy_deps = True + if self.runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow): + self.runtimeContext.copy_deps = True - if runtimeContext.update_workflow and self.project_uuid is None: + if self.runtimeContext.update_workflow and self.project_uuid is None: # If we are updating a workflow, make sure anything that # gets uploaded goes into the same parent project, unless # an alternate --project-uuid was provided. diff --git a/sdk/cwl/tests/19070-copy-deps.cwl b/sdk/cwl/tests/19070-copy-deps.cwl new file mode 100644 index 0000000000..b0d61700ef --- /dev/null +++ b/sdk/cwl/tests/19070-copy-deps.cwl @@ -0,0 +1,17 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.2 +class: CommandLineTool +baseCommand: echo +inputs: + message: + type: File + inputBinding: + position: 1 + default: + class: File + location: keep:d7514270f356df848477718d58308cc4+94/b + +outputs: [] diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh index 9cb5234cf0..1bbaa505e9 100755 --- a/sdk/cwl/tests/arvados-tests.sh +++ b/sdk/cwl/tests/arvados-tests.sh @@ -18,6 +18,15 @@ if ! arv-get 20850f01122e860fb878758ac1320877+71 > /dev/null ; then arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz fi +# Test for #18888 +# This is a standalone test because the bug was observed with this +# command line and was thought to be due to command line handling. arvados-cwl-runner 18888-download_def.cwl --scripts scripts/ +# Test for #19070 +# The most effective way to test this seemed to be to write an +# integration test to check for the expected behavior. +python test_copy_deps.py + +# Run integration tests exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum --api=containers diff --git a/sdk/cwl/tests/test_copy_deps.py b/sdk/cwl/tests/test_copy_deps.py new file mode 100644 index 0000000000..298a6800f8 --- /dev/null +++ b/sdk/cwl/tests/test_copy_deps.py @@ -0,0 +1,142 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import arvados +import subprocess + +api = arvados.api() + +def check_contents(group, wf_uuid): + contents = api.groups().contents(uuid=group["uuid"]).execute() + if len(contents["items"]) != 3: + raise Exception("Expected 3 items") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid: + found = True + if not found: + raise Exception("Couldn't find workflow") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94": + found = True + if not found: + raise Exception("Couldn't find collection dependency") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"): + found = True + if not found: + raise Exception("Couldn't find jobs image dependency") + + +def test_create(): + group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute() + try: + contents = api.groups().contents(uuid=group["uuid"]).execute() + if len(contents["items"]) != 0: + raise Exception("Expected 0 items") + + # Create workflow, by default should also copy dependencies + wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]) + wf_uuid = wf_uuid.decode("utf-8").strip() + check_contents(group, wf_uuid) + finally: + api.groups().delete(uuid=group["uuid"]).execute() + + +def test_update(): + group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute() + try: + contents = api.groups().contents(uuid=group["uuid"]).execute() + if len(contents["items"]) != 0: + raise Exception("Expected 0 items") + + # Create workflow, but with --no-copy-deps it shouldn't copy anything + wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]) + wf_uuid = wf_uuid.decode("utf-8").strip() + + contents = api.groups().contents(uuid=group["uuid"]).execute() + if len(contents["items"]) != 1: + raise Exception("Expected 1 items") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid: + found = True + if not found: + raise Exception("Couldn't find workflow") + + # Updating by default will copy missing items + wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]) + wf_uuid = wf_uuid.decode("utf-8").strip() + check_contents(group, wf_uuid) + + finally: + api.groups().delete(uuid=group["uuid"]).execute() + + +def test_execute(): + group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute() + try: + contents = api.groups().contents(uuid=group["uuid"]).execute() + if len(contents["items"]) != 0: + raise Exception("Expected 0 items") + + # Execute workflow, shouldn't copy anything. + wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]) + wf_uuid = wf_uuid.decode("utf-8").strip() + + contents = api.groups().contents(uuid=group["uuid"]).execute() + # container request + # final output collection + # container log + # step output collection + # container request log + if len(contents["items"]) != 5: + raise Exception("Expected 5 items") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94": + found = True + if found: + raise Exception("Didn't expect to find collection dependency") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"): + found = True + if found: + raise Exception("Didn't expect to find jobs image dependency") + + # Execute workflow with --copy-deps + wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]) + wf_uuid = wf_uuid.decode("utf-8").strip() + + contents = api.groups().contents(uuid=group["uuid"]).execute() + found = False + for c in contents["items"]: + if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94": + found = True + if not found: + raise Exception("Couldn't find collection dependency") + + found = False + for c in contents["items"]: + if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"): + found = True + if not found: + raise Exception("Couldn't find jobs image dependency") + + finally: + api.groups().delete(uuid=group["uuid"]).execute() + + +test_create() +test_update() +test_execute() diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py index fd3a69cae4..8fbfad4377 100644 --- a/sdk/python/tests/test_arv_keepdocker.py +++ b/sdk/python/tests/test_arv_keepdocker.py @@ -48,11 +48,13 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker): self.run_arv_keepdocker(['--version'], sys.stderr) self.assertVersionOutput(out, err) + @mock.patch('arvados.commands.keepdocker.list_images_in_arv', + return_value=[]) @mock.patch('arvados.commands.keepdocker.find_image_hashes', return_value=['abc123']) @mock.patch('arvados.commands.keepdocker.find_one_image_hash', return_value='abc123') - def test_image_format_compatibility(self, _1, _2): + def test_image_format_compatibility(self, _1, _2, _3): old_id = hashlib.sha256(b'old').hexdigest() new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest() for supported, img_id, expect_ok in [ @@ -152,11 +154,13 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker): self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr) find_image_mock.assert_called_with('[::1]/repo/img', 'latest') + @mock.patch('arvados.commands.keepdocker.list_images_in_arv', + return_value=[]) @mock.patch('arvados.commands.keepdocker.find_image_hashes', return_value=['abc123']) @mock.patch('arvados.commands.keepdocker.find_one_image_hash', return_value='abc123') - def test_collection_property_update(self, _1, _2): + def test_collection_property_update(self, _1, _2, _3): image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest() fakeDD = arvados.api('v1')._rootDesc fakeDD['dockerImageFormats'] = ['v2']