10 from schema_salad.ref_resolver import Loader
12 if not os.getenv('ARVADOS_DEBUG'):
13 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
14 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
17 class TestJob(unittest.TestCase):
19 # The test passes no builder.resources
20 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
22 runner = mock.MagicMock()
23 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
24 runner.ignore_docker_for_reuse = False
25 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
31 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
33 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
34 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
35 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
36 arvtool.formatgraph = None
37 for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
39 runner.api.jobs().create.assert_called_with(
41 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
42 'runtime_constraints': {},
43 'script_parameters': {
45 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
46 'command': ['ls', '$(task.outdir)']
49 'script_version': 'master',
50 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
51 'repository': 'arvados',
52 'script': 'crunchrunner',
53 'runtime_constraints': {
54 'docker_image': 'arvados/jobs',
55 'min_cores_per_node': 1,
56 'min_ram_mb_per_node': 1024,
57 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
61 filters=[['repository', '=', 'arvados'],
62 ['script', '=', 'crunchrunner'],
63 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
64 ['docker_image_locator', 'in docker', 'arvados/jobs']]
67 # The test passes some fields in builder.resources
68 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
69 def test_resource_requirements(self):
70 runner = mock.MagicMock()
71 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
72 runner.ignore_docker_for_reuse = False
73 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
79 "class": "ResourceRequirement",
84 "class": "http://arvados.org/cwl#RuntimeConstraints",
87 "class": "http://arvados.org/cwl#APIRequirement",
91 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
92 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
93 make_fs_access=make_fs_access, loader=Loader({}))
94 arvtool.formatgraph = None
95 for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
97 runner.api.jobs().create.assert_called_with(
99 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
100 'runtime_constraints': {},
101 'script_parameters': {
103 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
107 'script_version': 'master',
108 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
109 'repository': 'arvados',
110 'script': 'crunchrunner',
111 'runtime_constraints': {
112 'docker_image': 'arvados/jobs',
113 'min_cores_per_node': 3,
114 'min_ram_mb_per_node': 3000,
115 'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
116 'keep_cache_mb_per_task': 512
120 filters=[['repository', '=', 'arvados'],
121 ['script', '=', 'crunchrunner'],
122 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
123 ['docker_image_locator', 'in docker', 'arvados/jobs']])
125 @mock.patch("arvados.collection.Collection")
126 def test_done(self, col):
127 api = mock.MagicMock()
129 runner = mock.MagicMock()
131 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
132 runner.num_retries = 0
133 runner.ignore_docker_for_reuse = False
135 col().open.return_value = []
136 api.collections().list().execute.side_effect = ({"items": []},
137 {"items": [{"manifest_text": "XYZ"}]})
139 arvjob = arvados_cwl.ArvadosJob(runner)
140 arvjob.name = "testjob"
141 arvjob.builder = mock.MagicMock()
142 arvjob.output_callback = mock.MagicMock()
143 arvjob.collect_outputs = mock.MagicMock()
147 "output": "99999999999999999999999999999993+99",
148 "log": "99999999999999999999999999999994+99",
149 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
152 api.collections().list.assert_has_calls([
154 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
155 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
156 ['name', '=', 'Output 9999999 of testjob']]),
157 mock.call().execute(num_retries=0),
158 mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
159 select=['manifest_text']),
160 mock.call().execute(num_retries=0)])
162 api.collections().create.assert_called_with(
163 ensure_unique_name=True,
164 body={'portable_data_hash': '99999999999999999999999999999993+99',
165 'manifest_text': 'XYZ',
166 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
167 'name': 'Output 9999999 of testjob'})
169 @mock.patch("arvados.collection.Collection")
170 def test_done_use_existing_collection(self, col):
171 api = mock.MagicMock()
173 runner = mock.MagicMock()
175 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
176 runner.num_retries = 0
178 col().open.return_value = []
179 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
181 arvjob = arvados_cwl.ArvadosJob(runner)
182 arvjob.name = "testjob"
183 arvjob.builder = mock.MagicMock()
184 arvjob.output_callback = mock.MagicMock()
185 arvjob.collect_outputs = mock.MagicMock()
189 "output": "99999999999999999999999999999993+99",
190 "log": "99999999999999999999999999999994+99",
191 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
194 api.collections().list.assert_has_calls([
196 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
197 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
198 ['name', '=', 'Output 9999999 of testjob']]),
199 mock.call().execute(num_retries=0)])
201 self.assertFalse(api.collections().create.called)
204 class TestWorkflow(unittest.TestCase):
205 # The test passes no builder.resources
206 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
207 @mock.patch("arvados.collection.Collection")
208 def test_run(self, mockcollection):
209 arvados_cwl.add_arv_hints()
211 runner = arvados_cwl.ArvCwlRunner(mock.MagicMock())
212 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
213 runner.ignore_docker_for_reuse = False
214 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
216 tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
217 metadata["cwlVersion"] = tool["cwlVersion"]
219 mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
221 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
222 arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
223 basedir="", make_fs_access=make_fs_access, loader=document_loader,
224 makeTool=runner.arv_make_tool, metadata=metadata)
225 arvtool.formatgraph = None
226 it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
230 with open("tests/wf/scatter2_subwf.cwl") as f:
233 mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
234 mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
236 runner.api.jobs().create.assert_called_with(
238 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
239 'repository': 'arvados',
240 'script_version': 'master',
241 'script': 'crunchrunner',
242 'script_parameters': {
243 'tasks': [{'task.env': {
244 'HOME': '$(task.outdir)',
245 'TMPDIR': '$(task.tmpdir)'},
247 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl',
248 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml'
250 'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
251 'task.stdout': 'cwl.output.json'}]},
252 'runtime_constraints': {
253 'min_scratch_mb_per_node': 2048,
254 'min_cores_per_node': 1,
255 'docker_image': 'arvados/jobs',
256 'min_ram_mb_per_node': 1024
258 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'},
259 filters=[['repository', '=', 'arvados'],
260 ['script', '=', 'crunchrunner'],
261 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
262 ['docker_image_locator', 'in docker', 'arvados/jobs']],