X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0f5b0542513b572959e39400bae42e69aeb1a7b6..HEAD:/sdk/cwl/tests/test_submit.py diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index d415be8856..bfc5fc8762 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -2,35 +2,21 @@ # # SPDX-License-Identifier: Apache-2.0 -from future import standard_library -standard_library.install_aliases() -from builtins import object -from builtins import str -from future.utils import viewvalues - import copy import io +import itertools import functools import hashlib import json import logging -import mock import sys import unittest import cwltool.process import re import os -from io import BytesIO - -# StringIO.StringIO and io.StringIO have different behavior write() is -# called with both python2 (byte) strings and unicode strings -# (specifically there's some logging in cwltool that causes trouble). -# This isn't a problem on python3 because all string are unicode. -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO +from io import BytesIO, StringIO +from unittest import mock import arvados import arvados.collection @@ -100,10 +86,7 @@ def stubs(wfdetails=('submit_wf.cwl', None)): stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz" stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz" - if sys.version_info[0] < 3: - stubs.capture_stdout = BytesIO() - else: - stubs.capture_stdout = StringIO() + stubs.capture_stdout = StringIO() stubs.api = mock.MagicMock() stubs.api._rootDesc = get_rootDesc() @@ -141,7 +124,7 @@ def stubs(wfdetails=('submit_wf.cwl', None)): return CollectionExecute(created_collections[uuid]) def collection_getstub(created_collections, uuid): - for v in viewvalues(created_collections): + for v in created_collections.values(): if uuid in (v["uuid"], v["portable_data_hash"]): return CollectionExecute(v) @@ -1047,43 +1030,37 @@ class TestSubmit(unittest.TestCase): api.return_value = mock.MagicMock() arvrunner.api = api.return_value arvrunner.runtimeContext.match_local_docker = False - arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "", - "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "link_class": "docker_image_repo+tag", - "name": "arvados/jobs:"+arvados_cwl.__version__, - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, - {"items": [{"created_at": "", - "head_uuid": "", - "link_class": "docker_image_hash", - "name": "123456", - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, - {"items": [{"created_at": "", - "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "link_class": "docker_image_repo+tag", - "name": "arvados/jobs:"+arvados_cwl.__version__, - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}, - {"items": [{"created_at": "", - "head_uuid": "", - "link_class": "docker_image_hash", - "name": "123456", - "owner_uuid": "", - "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} - ) + arvrunner.api.links().list().execute.side_effect = itertools.cycle([ + {"items": [{"created_at": "2023-08-25T12:34:56.123456Z", + "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "link_class": "docker_image_repo+tag", + "name": "arvados/jobs:"+arvados_cwl.__version__, + "owner_uuid": "", + "uuid": "zzzzz-o0j2j-arvadosjobsrepo", + "properties": {"image_timestamp": ""}}]}, + {"items": []}, + {"items": []}, + {"items": [{"created_at": "2023-08-25T12:34:57.234567Z", + "head_uuid": "", + "link_class": "docker_image_hash", + "name": "123456", + "owner_uuid": "", + "uuid": "zzzzz-o0j2j-arvadosjobshash", + "properties": {"image_timestamp": ""}}]}, + {"items": []}, + {"items": []}, + ]) find_one_image_hash.return_value = "123456" - arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "owner_uuid": "", - "manifest_text": "", - "properties": "" - }], "items_available": 1, "offset": 0}, - {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", - "owner_uuid": "", - "manifest_text": "", - "properties": "" - }], "items_available": 1, "offset": 0}) + arvrunner.api.collections().list().execute.side_effect = itertools.cycle([ + {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", + "owner_uuid": "", + "manifest_text": "", + "created_at": "2023-08-25T12:34:55.012345Z", + "properties": {}}]}, + {"items": []}, + {"items": []}, + ]) arvrunner.api.collections().create().execute.return_value = {"uuid": ""} arvrunner.api.collections().get().execute.return_value = {"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb", "portable_data_hash": "9999999999999999999999999999999b+99"} @@ -1185,7 +1162,7 @@ class TestSubmit(unittest.TestCase): "out": [ {"id": "#main/step/out"} ], - "run": "keep:7628e49da34b93de9f4baf08a6212817+247/secret_wf.cwl" + "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl" } ] } @@ -1742,3 +1719,55 @@ class TestCreateWorkflow(unittest.TestCase): self.assertEqual(stubs.capture_stdout.getvalue(), stubs.expect_workflow_uuid + '\n') self.assertEqual(exited, 0) + + @stubs() + def test_create_map(self, stubs): + # test uploading a document that uses objects instead of arrays + # for certain fields like inputs and requirements. + + project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' + stubs.api.groups().get().execute.return_value = {"group_class": "project"} + + exited = arvados_cwl.main( + ["--create-workflow", "--debug", + "--api=containers", + "--project-uuid", project_uuid, + "--disable-git", + "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api) + + stubs.api.pipeline_templates().create.refute_called() + stubs.api.container_requests().create.refute_called() + + expect_workflow = StripYAMLComments( + open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip()) + + body = { + "workflow": { + "owner_uuid": project_uuid, + "name": "submit_wf_map.cwl", + "description": "", + "definition": expect_workflow, + } + } + stubs.api.workflows().create.assert_called_with( + body=JsonDiffMatcher(body)) + + self.assertEqual(stubs.capture_stdout.getvalue(), + stubs.expect_workflow_uuid + '\n') + self.assertEqual(exited, 0) + + +class TestPrintKeepDeps(unittest.TestCase): + @stubs() + def test_print_keep_deps(self, stubs): + # test --print-keep-deps which is used by arv-copy + + exited = arvados_cwl.main( + ["--print-keep-deps", "--debug", + "tests/wf/submit_wf_map.cwl"], + stubs.capture_stdout, sys.stderr, api_client=stubs.api) + + self.assertEqual(stubs.capture_stdout.getvalue(), + '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n') + self.assertEqual(exited, 0)