X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e3a2f5a80c1e4bd645e7e8bff27ef94f89ceae3e..aabf1ca0e99701550f9af785e9f1fee098b0020a:/sdk/cwl/tests/test_job.py diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py index 14e006eb19..faac1f77a6 100644 --- a/sdk/cwl/tests/test_job.py +++ b/sdk/cwl/tests/test_job.py @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + import functools import json import logging @@ -10,10 +14,11 @@ import StringIO import arvados import arvados_cwl import cwltool.process +from arvados.errors import ApiError from schema_salad.ref_resolver import Loader from schema_salad.sourceline import cmap from .mock_discovery import get_rootDesc -from .matcher import JsonDiffMatcher +from .matcher import JsonDiffMatcher, StripYAMLComments if not os.getenv('ARVADOS_DEBUG'): logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN) @@ -33,7 +38,15 @@ class TestJob(unittest.TestCase): document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] - runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"} + runner.api.collections().get().execute.return_value = {"portable_data_hash": "99999999999999999999999999999993+99"} + # Simulate reused job from another project so that we can check is a can_read + # link is added. + runner.api.jobs().create().execute.return_value = { + 'state': 'Complete' if enable_reuse else 'Queued', + 'owner_uuid': 'zzzzz-tpzed-yyyyyyyyyyyyyyy' if enable_reuse else 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'uuid': 'zzzzz-819sb-yyyyyyyyyyyyyyy', + 'output': None, + } tool = cmap({ "inputs": [], @@ -75,6 +88,24 @@ class TestJob(unittest.TestCase): ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'], ['docker_image_locator', 'in docker', 'arvados/jobs']] ) + if enable_reuse: + runner.api.links().create.assert_called_with( + body=JsonDiffMatcher({ + 'link_class': 'permission', + 'name': 'can_read', + "tail_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", + "head_uuid": "zzzzz-819sb-yyyyyyyyyyyyyyy", + }) + ) + # Simulate an API excepction when trying to create a + # sharing link on the job + runner.api.links().create.side_effect = ApiError( + mock.MagicMock(return_value={'status': 403}), + 'Permission denied') + j.run(enable_reuse=enable_reuse) + j.output_callback.assert_called_with({}, 'success') + else: + assert not runner.api.links().create.called # The test passes some fields in builder.resources # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @@ -106,6 +137,10 @@ class TestJob(unittest.TestCase): "outputDirType": "keep_output_dir" }, { "class": "http://arvados.org/cwl#APIRequirement", + }, + { + "class": "http://arvados.org/cwl#ReuseRequirement", + "enableReuse": False }], "baseCommand": "ls" } @@ -115,7 +150,7 @@ class TestJob(unittest.TestCase): make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access): - j.run() + j.run(enable_reuse=True) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher({ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', @@ -139,7 +174,7 @@ class TestJob(unittest.TestCase): 'keep_cache_mb_per_task': 512 } }), - find_or_create=True, + find_or_create=False, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'], @@ -271,9 +306,10 @@ class TestJob(unittest.TestCase): class TestWorkflow(unittest.TestCase): # The test passes no builder.resources # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.collection.CollectionReader") @mock.patch("arvados.collection.Collection") @mock.patch('arvados.commands.keepdocker.list_images_in_arv') - def test_run(self, list_images_in_arv, mockcollection): + def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader): arvados_cwl.add_arv_hints() api = mock.MagicMock() @@ -284,19 +320,25 @@ class TestWorkflow(unittest.TestCase): list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"} + runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]} runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access("")) + document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session) + document_loader.fetch_text = document_loader.fetcher.fetch_text + document_loader.check_exists = document_loader.fetcher.check_exists + tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl") metadata["cwlVersion"] = tool["cwlVersion"] mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118" - make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, - collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=document_loader, makeTool=runner.arv_make_tool, metadata=metadata) @@ -306,7 +348,7 @@ class TestWorkflow(unittest.TestCase): it.next().run() with open("tests/wf/scatter2_subwf.cwl") as f: - subwf = f.read() + subwf = StripYAMLComments(f.read()) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher({ @@ -340,6 +382,11 @@ class TestWorkflow(unittest.TestCase): mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)]) mockcollection().open().__enter__().write.assert_has_calls([mock.call( '''{ + "fileblub": { + "basename": "token.txt", + "class": "File", + "location": "/keep/99999999999999999999999999999999+118/token.txt" + }, "sleeptime": 5 }''')])