10 import cwltool.process
11 from schema_salad.ref_resolver import Loader
13 if not os.getenv('ARVADOS_DEBUG'):
14 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
15 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
18 class TestJob(unittest.TestCase):
20 # The test passes no builder.resources
21 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
23 runner = mock.MagicMock()
24 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
25 runner.ignore_docker_for_reuse = False
26 runner.num_retries = 0
27 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
33 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
35 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
36 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
37 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
38 arvtool.formatgraph = None
39 for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
41 runner.api.jobs().create.assert_called_with(
43 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
44 'runtime_constraints': {},
45 'script_parameters': {
47 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
48 'command': ['ls', '$(task.outdir)']
51 'script_version': 'master',
52 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
53 'repository': 'arvados',
54 'script': 'crunchrunner',
55 'runtime_constraints': {
56 'docker_image': 'arvados/jobs',
57 'min_cores_per_node': 1,
58 'min_ram_mb_per_node': 1024,
59 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
63 filters=[['repository', '=', 'arvados'],
64 ['script', '=', 'crunchrunner'],
65 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
66 ['docker_image_locator', 'in docker', 'arvados/jobs']]
69 # The test passes some fields in builder.resources
70 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
71 def test_resource_requirements(self):
72 runner = mock.MagicMock()
73 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
74 runner.ignore_docker_for_reuse = False
75 runner.num_retries = 0
76 arvados_cwl.add_arv_hints()
78 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
85 "class": "ResourceRequirement",
90 "class": "http://arvados.org/cwl#RuntimeConstraints",
92 "outputDirType": "keep_output_dir"
94 "class": "http://arvados.org/cwl#APIRequirement",
98 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
99 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
100 make_fs_access=make_fs_access, loader=Loader({}))
101 arvtool.formatgraph = None
102 for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
104 runner.api.jobs().create.assert_called_with(
106 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
107 'runtime_constraints': {},
108 'script_parameters': {
110 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
111 'task.keepTmpOutput': True,
115 'script_version': 'master',
116 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
117 'repository': 'arvados',
118 'script': 'crunchrunner',
119 'runtime_constraints': {
120 'docker_image': 'arvados/jobs',
121 'min_cores_per_node': 3,
122 'min_ram_mb_per_node': 3000,
123 'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
124 'keep_cache_mb_per_task': 512
128 filters=[['repository', '=', 'arvados'],
129 ['script', '=', 'crunchrunner'],
130 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
131 ['docker_image_locator', 'in docker', 'arvados/jobs']])
133 @mock.patch("arvados.collection.CollectionReader")
134 def test_done(self, reader):
135 api = mock.MagicMock()
137 runner = mock.MagicMock()
139 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
140 runner.num_retries = 0
141 runner.ignore_docker_for_reuse = False
143 reader().open.return_value = []
144 api.collections().list().execute.side_effect = ({"items": []},
145 {"items": [{"manifest_text": "XYZ"}]})
147 arvjob = arvados_cwl.ArvadosJob(runner)
148 arvjob.name = "testjob"
149 arvjob.builder = mock.MagicMock()
150 arvjob.output_callback = mock.MagicMock()
151 arvjob.collect_outputs = mock.MagicMock()
155 "output": "99999999999999999999999999999993+99",
156 "log": "99999999999999999999999999999994+99",
157 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
160 api.collections().list.assert_has_calls([
162 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
163 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
164 ['name', '=', 'Output 9999999 of testjob']]),
165 mock.call().execute(num_retries=0),
166 mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
167 select=['manifest_text']),
168 mock.call().execute(num_retries=0)])
170 api.collections().create.assert_called_with(
171 ensure_unique_name=True,
172 body={'portable_data_hash': '99999999999999999999999999999993+99',
173 'manifest_text': 'XYZ',
174 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
175 'name': 'Output 9999999 of testjob'})
177 @mock.patch("arvados.collection.CollectionReader")
178 def test_done_use_existing_collection(self, reader):
179 api = mock.MagicMock()
181 runner = mock.MagicMock()
183 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
184 runner.num_retries = 0
186 reader().open.return_value = []
187 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
189 arvjob = arvados_cwl.ArvadosJob(runner)
190 arvjob.name = "testjob"
191 arvjob.builder = mock.MagicMock()
192 arvjob.output_callback = mock.MagicMock()
193 arvjob.collect_outputs = mock.MagicMock()
197 "output": "99999999999999999999999999999993+99",
198 "log": "99999999999999999999999999999994+99",
199 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
202 api.collections().list.assert_has_calls([
204 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
205 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
206 ['name', '=', 'Output 9999999 of testjob']]),
207 mock.call().execute(num_retries=0)])
209 self.assertFalse(api.collections().create.called)
212 class TestWorkflow(unittest.TestCase):
213 # The test passes no builder.resources
214 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
215 @mock.patch("arvados.collection.Collection")
216 def test_run(self, mockcollection):
217 arvados_cwl.add_arv_hints()
219 api = mock.MagicMock()
220 api._rootDesc = arvados.api('v1')._rootDesc
221 runner = arvados_cwl.ArvCwlRunner(api)
222 self.assertEqual(runner.work_api, 'jobs')
224 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
225 runner.ignore_docker_for_reuse = False
226 runner.num_retries = 0
227 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
229 tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
230 metadata["cwlVersion"] = tool["cwlVersion"]
232 mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
234 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
235 arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
236 basedir="", make_fs_access=make_fs_access, loader=document_loader,
237 makeTool=runner.arv_make_tool, metadata=metadata)
238 arvtool.formatgraph = None
239 it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
243 with open("tests/wf/scatter2_subwf.cwl") as f:
246 mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
247 mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
249 runner.api.jobs().create.assert_called_with(
251 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
252 'repository': 'arvados',
253 'script_version': 'master',
254 'script': 'crunchrunner',
255 'script_parameters': {
256 'tasks': [{'task.env': {
257 'HOME': '$(task.outdir)',
258 'TMPDIR': '$(task.tmpdir)'},
260 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
261 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
263 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
264 'task.stdout': 'cwl.output.json'}]},
265 'runtime_constraints': {
266 'min_scratch_mb_per_node': 2048,
267 'min_cores_per_node': 1,
268 'docker_image': 'arvados/jobs',
269 'min_ram_mb_per_node': 1024
271 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
272 filters=[['repository', '=', 'arvados'],
273 ['script', '=', 'crunchrunner'],
274 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
275 ['docker_image_locator', 'in docker', 'arvados/jobs']],
278 def test_default_work_api(self):
279 arvados_cwl.add_arv_hints()
281 api = mock.MagicMock()
282 api._rootDesc = arvados.api('v1')._rootDesc
283 del api._rootDesc.get('resources')['jobs']['methods']['create']
284 runner = arvados_cwl.ArvCwlRunner(api)
285 self.assertEqual(runner.work_api, 'containers')