8 from schema_salad.ref_resolver import Loader
10 from schema_salad.ref_resolver import Loader
12 if not os.getenv('ARVADOS_DEBUG'):
13 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
14 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
17 class TestContainer(unittest.TestCase):
19 # The test passes no builder.resources
20 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
21 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
22 def test_run(self, keepdocker):
23 runner = mock.MagicMock()
24 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
25 runner.ignore_docker_for_reuse = False
27 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
28 runner.api.collections().get().execute.return_value = {
29 "portable_data_hash": "99999999999999999999999999999993+99"}
31 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
37 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
39 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
40 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
41 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
42 arvtool.formatgraph = None
43 for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run",
44 make_fs_access=make_fs_access, tmpdir="/tmp"):
46 runner.api.container_requests().create.assert_called_with(
49 'HOME': '/var/spool/cwl',
53 'runtime_constraints': {
58 '/var/spool/cwl': {'kind': 'tmp'}
61 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
62 'output_path': '/var/spool/cwl',
63 'container_image': '99999999999999999999999999999993+99',
64 'command': ['ls', '/var/spool/cwl'],
65 'cwd': '/var/spool/cwl'
68 # The test passes some fields in builder.resources
69 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
70 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
71 def test_resource_requirements(self, keepdocker):
72 runner = mock.MagicMock()
73 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
74 runner.ignore_docker_for_reuse = False
75 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
77 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
78 runner.api.collections().get().execute.return_value = {
79 "portable_data_hash": "99999999999999999999999999999993+99"}
85 "class": "ResourceRequirement",
90 "class": "http://arvados.org/cwl#RuntimeConstraints",
93 "class": "http://arvados.org/cwl#APIRequirement",
97 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
98 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
99 avsc_names=avsc_names, make_fs_access=make_fs_access,
101 arvtool.formatgraph = None
102 for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
103 make_fs_access=make_fs_access, tmpdir="/tmp"):
106 runner.api.container_requests().create.assert_called_with(
109 'HOME': '/var/spool/cwl',
112 'name': 'test_resource_requirements',
113 'runtime_constraints': {
119 '/var/spool/cwl': {'kind': 'tmp'}
121 'state': 'Committed',
122 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
123 'output_path': '/var/spool/cwl',
124 'container_image': '99999999999999999999999999999993+99',
126 'cwd': '/var/spool/cwl'
129 @mock.patch("arvados.collection.Collection")
130 def test_done(self, col):
131 api = mock.MagicMock()
133 runner = mock.MagicMock()
135 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
136 runner.num_retries = 0
137 runner.ignore_docker_for_reuse = False
139 col().open.return_value = []
140 api.collections().list().execute.side_effect = ({"items": []},
141 {"items": [{"manifest_text": "XYZ"}]})
143 arvjob = arvados_cwl.ArvadosContainer(runner)
144 arvjob.name = "testjob"
145 arvjob.builder = mock.MagicMock()
146 arvjob.output_callback = mock.MagicMock()
147 arvjob.collect_outputs = mock.MagicMock()
148 arvjob.successCodes = [0]
149 arvjob.outdir = "/var/spool/cwl"
153 "output": "99999999999999999999999999999993+99",
154 "log": "99999999999999999999999999999994+99",
155 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
159 api.collections().list.assert_has_calls([
161 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
162 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
163 ['name', '=', 'Output 9999999 of testjob']]),
164 mock.call().execute(num_retries=0),
165 mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
166 select=['manifest_text']),
167 mock.call().execute(num_retries=0)])
169 api.collections().create.assert_called_with(
170 ensure_unique_name=True,
171 body={'portable_data_hash': '99999999999999999999999999999993+99',
172 'manifest_text': 'XYZ',
173 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
174 'name': 'Output 9999999 of testjob'})
176 @mock.patch("arvados.collection.Collection")
177 def test_done_use_existing_collection(self, col):
178 api = mock.MagicMock()
180 runner = mock.MagicMock()
182 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
183 runner.num_retries = 0
185 col().open.return_value = []
186 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
188 arvjob = arvados_cwl.ArvadosContainer(runner)
189 arvjob.name = "testjob"
190 arvjob.builder = mock.MagicMock()
191 arvjob.output_callback = mock.MagicMock()
192 arvjob.collect_outputs = mock.MagicMock()
193 arvjob.successCodes = [0]
194 arvjob.outdir = "/var/spool/cwl"
198 "output": "99999999999999999999999999999993+99",
199 "log": "99999999999999999999999999999994+99",
200 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
204 api.collections().list.assert_has_calls([
206 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
207 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
208 ['name', '=', 'Output 9999999 of testjob']]),
209 mock.call().execute(num_retries=0)])
211 self.assertFalse(api.collections().create.called)