12 import arvados.collection
14 import arvados_cwl.runner
17 from .matcher import JsonDiffMatcher
18 from .mock_discovery import get_rootDesc
20 import ruamel.yaml as yaml
25 @functools.wraps(func)
26 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
27 @mock.patch("arvados.collection.KeepClient")
28 @mock.patch("arvados.keep.KeepClient")
29 @mock.patch("arvados.events.subscribe")
30 def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
35 stubs.keepdocker = keepdocker
37 def putstub(p, **kwargs):
38 return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
39 keep_client1().put.side_effect = putstub
40 keep_client1.put.side_effect = putstub
41 keep_client2().put.side_effect = putstub
42 keep_client2.put.side_effect = putstub
44 stubs.keep_client = keep_client2
45 stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
46 stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
48 stubs.api = mock.MagicMock()
49 stubs.api._rootDesc = get_rootDesc()
51 stubs.api.users().current().execute.return_value = {
52 "uuid": stubs.fake_user_uuid,
54 stubs.api.collections().list().execute.return_value = {"items": []}
56 class CollectionExecute(object):
57 def __init__(self, exe):
59 def execute(self, num_retries=None):
62 def collection_createstub(created_collections, body, ensure_unique_name=None):
63 mt = body["manifest_text"]
64 uuid = "zzzzz-4zz18-zzzzzzzzzzzzzz%d" % len(created_collections)
65 pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
66 created_collections[uuid] = {
68 "portable_data_hash": pdh,
71 return CollectionExecute(created_collections[uuid])
73 def collection_getstub(created_collections, uuid):
74 for v in created_collections.itervalues():
75 if uuid in (v["uuid"], v["portable_data_hash"]):
76 return CollectionExecute(v)
78 created_collections = {}
79 stubs.api.collections().create.side_effect = functools.partial(collection_createstub, created_collections)
80 stubs.api.collections().get.side_effect = functools.partial(collection_getstub, created_collections)
82 stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
83 stubs.api.jobs().create().execute.return_value = {
84 "uuid": stubs.expect_job_uuid,
88 stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
89 stubs.api.container_requests().create().execute.return_value = {
90 "uuid": stubs.expect_container_request_uuid,
91 "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
95 stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
96 stubs.api.pipeline_templates().create().execute.return_value = {
97 "uuid": stubs.expect_pipeline_template_uuid,
99 stubs.expect_job_spec = {
100 'runtime_constraints': {
101 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
102 'min_ram_mb_per_node': 1024
104 'script_parameters': {
106 'basename': 'blorp.txt',
107 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
111 'basename': '99999999999999999999999999999998+99',
112 'location': 'keep:99999999999999999999999999999998+99',
116 'basename': 'anonymous',
118 "basename": "renamed.txt",
120 "location": "keep:99999999999999999999999999999998+99/file1.txt"
124 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main'
126 'repository': 'arvados',
127 'script_version': 'master',
128 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
129 'script': 'cwl-runner'
131 stubs.pipeline_component = stubs.expect_job_spec.copy()
132 stubs.expect_pipeline_instance = {
133 'name': 'submit_wf.cwl',
134 'state': 'RunningOnServer',
138 'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
139 'script_parameters': {
140 'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
141 'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'}},
142 'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
144 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
146 'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
147 'arv:enable_reuse': True,
148 'arv:on_error': 'continue'
150 'repository': 'arvados',
151 'script_version': 'master',
152 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
153 'script': 'cwl-runner',
154 'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
158 stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
159 stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
160 stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
161 stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
162 stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
163 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
166 stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
167 stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
169 with open("tests/wf/submit_wf_packed.cwl") as f:
170 expect_packed_workflow = yaml.round_trip_load(f)
172 stubs.expect_container_spec = {
179 '/var/lib/cwl/workflow.json': {
180 'content': expect_packed_workflow,
184 'path': '/var/spool/cwl/cwl.output.json',
187 '/var/lib/cwl/cwl.input.json': {
190 'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
191 'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'},
192 'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
193 {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
199 'state': 'Committed',
201 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
202 '--enable-reuse', '--on-error=continue',
203 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
204 'name': 'submit_wf.cwl',
205 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
206 'output_path': '/var/spool/cwl',
207 'cwd': '/var/spool/cwl',
208 'runtime_constraints': {
211 'ram': 1024*1024*1024
216 stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
217 stubs.api.workflows().create().execute.return_value = {
218 "uuid": stubs.expect_workflow_uuid,
220 def update_mock(**kwargs):
221 stubs.updated_uuid = kwargs.get('uuid')
223 stubs.api.workflows().update.side_effect = update_mock
224 stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
225 "uuid": stubs.updated_uuid,
228 return func(self, stubs, *args, **kwargs)
232 class TestSubmit(unittest.TestCase):
233 @mock.patch("arvados_cwl.runner.arv_docker_get_image")
234 @mock.patch("time.sleep")
236 def test_submit(self, stubs, tm, arvdock):
237 capture_stdout = cStringIO.StringIO()
238 exited = arvados_cwl.main(
239 ["--submit", "--no-wait", "--api=jobs", "--debug",
240 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
241 capture_stdout, sys.stderr, api_client=stubs.api)
242 self.assertEqual(exited, 0)
244 stubs.api.collections().create.assert_has_calls([
245 mock.call(body=JsonDiffMatcher({
247 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
248 'replication_desired': None,
249 'name': 'submit_tool.cwl dependencies',
250 }), ensure_unique_name=True),
251 mock.call(body=JsonDiffMatcher({
253 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
254 'replication_desired': None,
255 'name': 'submit_wf.cwl input',
256 }), ensure_unique_name=True),
257 mock.call(body=JsonDiffMatcher({
259 '. 61df2ed9ee3eb7dd9b799e5ca35305fa+1217 0:1217:workflow.cwl\n',
260 'replication_desired': None,
261 'name': 'submit_wf.cwl',
262 }), ensure_unique_name=True) ])
264 arvdock.assert_has_calls([
265 mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
266 mock.call(stubs.api, {'dockerPull': 'arvados/jobs:'+arvados_cwl.__version__}, True, None)
269 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
270 stubs.api.pipeline_instances().create.assert_called_with(
271 body=JsonDiffMatcher(expect_pipeline))
272 self.assertEqual(capture_stdout.getvalue(),
273 stubs.expect_pipeline_uuid + '\n')
276 @mock.patch("time.sleep")
278 def test_submit_no_reuse(self, stubs, tm):
279 capture_stdout = cStringIO.StringIO()
280 exited = arvados_cwl.main(
281 ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
282 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
283 capture_stdout, sys.stderr, api_client=stubs.api)
284 self.assertEqual(exited, 0)
286 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
287 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
289 stubs.api.pipeline_instances().create.assert_called_with(
290 body=JsonDiffMatcher(expect_pipeline))
291 self.assertEqual(capture_stdout.getvalue(),
292 stubs.expect_pipeline_uuid + '\n')
294 @mock.patch("time.sleep")
296 def test_submit_on_error(self, stubs, tm):
297 capture_stdout = cStringIO.StringIO()
298 exited = arvados_cwl.main(
299 ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
300 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
301 capture_stdout, sys.stderr, api_client=stubs.api)
302 self.assertEqual(exited, 0)
304 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
305 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
307 stubs.api.pipeline_instances().create.assert_called_with(
308 body=JsonDiffMatcher(expect_pipeline))
309 self.assertEqual(capture_stdout.getvalue(),
310 stubs.expect_pipeline_uuid + '\n')
313 @mock.patch("time.sleep")
315 def test_submit_runner_ram(self, stubs, tm):
316 capture_stdout = cStringIO.StringIO()
317 exited = arvados_cwl.main(
318 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
319 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
320 capture_stdout, sys.stderr, api_client=stubs.api)
321 self.assertEqual(exited, 0)
323 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
324 expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
326 stubs.api.pipeline_instances().create.assert_called_with(
327 body=JsonDiffMatcher(expect_pipeline))
328 self.assertEqual(capture_stdout.getvalue(),
329 stubs.expect_pipeline_uuid + '\n')
332 @mock.patch("time.sleep")
334 def test_submit_invalid_runner_ram(self, stubs, tm):
335 capture_stdout = cStringIO.StringIO()
336 exited = arvados_cwl.main(
337 ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
338 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
339 capture_stdout, sys.stderr, api_client=stubs.api)
340 self.assertEqual(exited, 1)
342 @mock.patch("time.sleep")
344 def test_submit_output_name(self, stubs, tm):
345 output_name = "test_output_name"
347 capture_stdout = cStringIO.StringIO()
348 exited = arvados_cwl.main(
349 ["--submit", "--no-wait", "--debug", "--output-name", output_name,
350 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
351 capture_stdout, sys.stderr, api_client=stubs.api)
352 self.assertEqual(exited, 0)
354 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
355 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
357 stubs.api.pipeline_instances().create.assert_called_with(
358 body=JsonDiffMatcher(expect_pipeline))
359 self.assertEqual(capture_stdout.getvalue(),
360 stubs.expect_pipeline_uuid + '\n')
363 @mock.patch("time.sleep")
365 def test_submit_pipeline_name(self, stubs, tm):
366 capture_stdout = cStringIO.StringIO()
367 exited = arvados_cwl.main(
368 ["--submit", "--no-wait", "--debug", "--name=hello job 123",
369 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
370 capture_stdout, sys.stderr, api_client=stubs.api)
371 self.assertEqual(exited, 0)
373 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
374 expect_pipeline["name"] = "hello job 123"
376 stubs.api.pipeline_instances().create.assert_called_with(
377 body=JsonDiffMatcher(expect_pipeline))
378 self.assertEqual(capture_stdout.getvalue(),
379 stubs.expect_pipeline_uuid + '\n')
381 @mock.patch("time.sleep")
383 def test_submit_output_tags(self, stubs, tm):
384 output_tags = "tag0,tag1,tag2"
386 capture_stdout = cStringIO.StringIO()
387 exited = arvados_cwl.main(
388 ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
389 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
390 capture_stdout, sys.stderr, api_client=stubs.api)
391 self.assertEqual(exited, 0)
393 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
394 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
396 stubs.api.pipeline_instances().create.assert_called_with(
397 body=JsonDiffMatcher(expect_pipeline))
398 self.assertEqual(capture_stdout.getvalue(),
399 stubs.expect_pipeline_uuid + '\n')
401 @mock.patch("time.sleep")
403 def test_submit_with_project_uuid(self, stubs, tm):
404 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
406 exited = arvados_cwl.main(
407 ["--submit", "--no-wait",
408 "--project-uuid", project_uuid,
409 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
410 sys.stdout, sys.stderr, api_client=stubs.api)
411 self.assertEqual(exited, 0)
413 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
414 expect_pipeline["owner_uuid"] = project_uuid
415 stubs.api.pipeline_instances().create.assert_called_with(
416 body=JsonDiffMatcher(expect_pipeline))
419 def test_submit_container(self, stubs):
420 capture_stdout = cStringIO.StringIO()
422 exited = arvados_cwl.main(
423 ["--submit", "--no-wait", "--api=containers", "--debug",
424 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
425 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
426 self.assertEqual(exited, 0)
428 logging.exception("")
430 stubs.api.collections().create.assert_has_calls([
431 mock.call(body=JsonDiffMatcher({
433 '. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
434 'replication_desired': None,
435 'name': 'submit_tool.cwl dependencies',
436 }), ensure_unique_name=True),
437 mock.call(body=JsonDiffMatcher({
439 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
440 'replication_desired': None,
441 'name': 'submit_wf.cwl input',
442 }), ensure_unique_name=True)])
444 expect_container = copy.deepcopy(stubs.expect_container_spec)
445 stubs.api.container_requests().create.assert_called_with(
446 body=JsonDiffMatcher(expect_container))
447 self.assertEqual(capture_stdout.getvalue(),
448 stubs.expect_container_request_uuid + '\n')
451 def test_submit_container_no_reuse(self, stubs):
452 capture_stdout = cStringIO.StringIO()
454 exited = arvados_cwl.main(
455 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
456 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
457 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
458 self.assertEqual(exited, 0)
460 logging.exception("")
462 expect_container = copy.deepcopy(stubs.expect_container_spec)
463 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
464 '--disable-reuse', '--on-error=continue',
465 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
467 stubs.api.container_requests().create.assert_called_with(
468 body=JsonDiffMatcher(expect_container))
469 self.assertEqual(capture_stdout.getvalue(),
470 stubs.expect_container_request_uuid + '\n')
474 def test_submit_container_on_error(self, stubs):
475 capture_stdout = cStringIO.StringIO()
477 exited = arvados_cwl.main(
478 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
479 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
480 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
481 self.assertEqual(exited, 0)
483 logging.exception("")
485 expect_container = copy.deepcopy(stubs.expect_container_spec)
486 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
487 '--enable-reuse', '--on-error=stop',
488 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
490 stubs.api.container_requests().create.assert_called_with(
491 body=JsonDiffMatcher(expect_container))
492 self.assertEqual(capture_stdout.getvalue(),
493 stubs.expect_container_request_uuid + '\n')
496 def test_submit_container_output_name(self, stubs):
497 output_name = "test_output_name"
499 capture_stdout = cStringIO.StringIO()
501 exited = arvados_cwl.main(
502 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
503 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
504 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
505 self.assertEqual(exited, 0)
507 logging.exception("")
509 expect_container = copy.deepcopy(stubs.expect_container_spec)
510 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
511 "--output-name="+output_name, '--enable-reuse', '--on-error=continue',
512 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
513 expect_container["output_name"] = output_name
515 stubs.api.container_requests().create.assert_called_with(
516 body=JsonDiffMatcher(expect_container))
517 self.assertEqual(capture_stdout.getvalue(),
518 stubs.expect_container_request_uuid + '\n')
521 def test_submit_container_output_tags(self, stubs):
522 output_tags = "tag0,tag1,tag2"
524 capture_stdout = cStringIO.StringIO()
526 exited = arvados_cwl.main(
527 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
528 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
529 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
530 self.assertEqual(exited, 0)
532 logging.exception("")
534 expect_container = copy.deepcopy(stubs.expect_container_spec)
535 expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
536 "--output-tags="+output_tags, '--enable-reuse', '--on-error=continue',
537 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
539 stubs.api.container_requests().create.assert_called_with(
540 body=JsonDiffMatcher(expect_container))
541 self.assertEqual(capture_stdout.getvalue(),
542 stubs.expect_container_request_uuid + '\n')
545 def test_submit_container_runner_ram(self, stubs):
546 capture_stdout = cStringIO.StringIO()
548 exited = arvados_cwl.main(
549 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
550 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
551 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
552 self.assertEqual(exited, 0)
554 logging.exception("")
556 expect_container = copy.deepcopy(stubs.expect_container_spec)
557 expect_container["runtime_constraints"]["ram"] = 2048*1024*1024
559 stubs.api.container_requests().create.assert_called_with(
560 body=JsonDiffMatcher(expect_container))
561 self.assertEqual(capture_stdout.getvalue(),
562 stubs.expect_container_request_uuid + '\n')
564 @mock.patch("arvados.collection.CollectionReader")
565 @mock.patch("time.sleep")
567 def test_submit_file_keepref(self, stubs, tm, collectionReader):
568 capture_stdout = cStringIO.StringIO()
569 exited = arvados_cwl.main(
570 ["--submit", "--no-wait", "--api=containers", "--debug",
571 "tests/wf/submit_keepref_wf.cwl"],
572 capture_stdout, sys.stderr, api_client=stubs.api)
573 self.assertEqual(exited, 0)
576 @mock.patch("arvados.collection.CollectionReader")
577 @mock.patch("time.sleep")
579 def test_submit_keepref(self, stubs, tm, reader):
580 capture_stdout = cStringIO.StringIO()
582 with open("tests/wf/expect_arvworkflow.cwl") as f:
583 reader().open().__enter__().read.return_value = f.read()
585 exited = arvados_cwl.main(
586 ["--submit", "--no-wait", "--api=containers", "--debug",
587 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
588 capture_stdout, sys.stderr, api_client=stubs.api)
589 self.assertEqual(exited, 0)
599 'path': '/var/spool/cwl/cwl.output.json',
602 '/var/lib/cwl/workflow': {
603 'portable_data_hash': '99999999999999999999999999999994+99',
606 '/var/lib/cwl/cwl.input.json': {
612 }, 'state': 'Committed',
614 'output_path': '/var/spool/cwl',
615 'name': 'expect_arvworkflow.cwl#main',
616 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
617 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
618 '--enable-reuse', '--on-error=continue',
619 '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
620 'cwd': '/var/spool/cwl',
621 'runtime_constraints': {
629 stubs.api.container_requests().create.assert_called_with(
630 body=JsonDiffMatcher(expect_container))
631 self.assertEqual(capture_stdout.getvalue(),
632 stubs.expect_container_request_uuid + '\n')
635 @mock.patch("arvados.collection.CollectionReader")
636 @mock.patch("time.sleep")
638 def test_submit_jobs_keepref(self, stubs, tm, reader):
639 capture_stdout = cStringIO.StringIO()
641 with open("tests/wf/expect_arvworkflow.cwl") as f:
642 reader().open().__enter__().read.return_value = f.read()
644 exited = arvados_cwl.main(
645 ["--submit", "--no-wait", "--api=jobs", "--debug",
646 "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
647 capture_stdout, sys.stderr, api_client=stubs.api)
648 self.assertEqual(exited, 0)
650 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
651 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
652 del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["y"]
653 del expect_pipeline["components"]["cwl-runner"]["script_parameters"]["z"]
654 expect_pipeline["components"]["cwl-runner"]["script_parameters"]["cwl:tool"] = "99999999999999999999999999999994+99/expect_arvworkflow.cwl#main"
655 expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
656 stubs.api.pipeline_instances().create.assert_called_with(
657 body=JsonDiffMatcher(expect_pipeline))
659 @mock.patch("time.sleep")
661 def test_submit_arvworkflow(self, stubs, tm):
662 capture_stdout = cStringIO.StringIO()
664 with open("tests/wf/expect_arvworkflow.cwl") as f:
665 stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
667 exited = arvados_cwl.main(
668 ["--submit", "--no-wait", "--api=containers", "--debug",
669 "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
670 capture_stdout, sys.stderr, api_client=stubs.api)
671 self.assertEqual(exited, 0)
681 'path': '/var/spool/cwl/cwl.output.json',
684 '/var/lib/cwl/workflow.json': {
687 'cwlVersion': 'v1.0',
692 {'type': 'string', 'id': '#main/x'}
695 {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
696 'run': '#submit_tool.cwl',
706 'inputBinding': {'position': 1},
708 'id': '#submit_tool.cwl/x'}
711 {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
713 'id': '#submit_tool.cwl',
715 'baseCommand': 'cat',
716 'class': 'CommandLineTool'
721 '/var/lib/cwl/cwl.input.json': {
727 }, 'state': 'Committed',
729 'output_path': '/var/spool/cwl',
730 'name': 'a test workflow',
731 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
732 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
733 '--enable-reuse', '--on-error=continue',
734 '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
735 'cwd': '/var/spool/cwl',
736 'runtime_constraints': {
742 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
746 stubs.api.container_requests().create.assert_called_with(
747 body=JsonDiffMatcher(expect_container))
748 self.assertEqual(capture_stdout.getvalue(),
749 stubs.expect_container_request_uuid + '\n')
753 def test_submit_container_name(self, stubs):
754 capture_stdout = cStringIO.StringIO()
756 exited = arvados_cwl.main(
757 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
758 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
759 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
760 self.assertEqual(exited, 0)
762 logging.exception("")
764 expect_container = copy.deepcopy(stubs.expect_container_spec)
765 expect_container["name"] = "hello container 123"
767 stubs.api.container_requests().create.assert_called_with(
768 body=JsonDiffMatcher(expect_container))
769 self.assertEqual(capture_stdout.getvalue(),
770 stubs.expect_container_request_uuid + '\n')
774 def test_submit_job_runner_image(self, stubs):
775 capture_stdout = cStringIO.StringIO()
777 exited = arvados_cwl.main(
778 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
779 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
780 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
781 self.assertEqual(exited, 0)
783 logging.exception("")
785 stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "arvados/jobs:123"
787 expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
788 stubs.api.pipeline_instances().create.assert_called_with(
789 body=JsonDiffMatcher(expect_pipeline))
790 self.assertEqual(capture_stdout.getvalue(),
791 stubs.expect_pipeline_uuid + '\n')
794 def test_submit_container_runner_image(self, stubs):
795 capture_stdout = cStringIO.StringIO()
797 exited = arvados_cwl.main(
798 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
799 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
800 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
801 self.assertEqual(exited, 0)
803 logging.exception("")
805 stubs.expect_container_spec["container_image"] = "arvados/jobs:123"
807 expect_container = copy.deepcopy(stubs.expect_container_spec)
808 stubs.api.container_requests().create.assert_called_with(
809 body=JsonDiffMatcher(expect_container))
810 self.assertEqual(capture_stdout.getvalue(),
811 stubs.expect_container_request_uuid + '\n')
814 @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
815 @mock.patch("cwltool.docker.get_image")
816 @mock.patch("arvados.api")
817 def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
818 arvrunner = mock.MagicMock()
819 arvrunner.project_uuid = ""
820 api.return_value = mock.MagicMock()
821 arvrunner.api = api.return_value
822 arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
823 "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
824 "link_class": "docker_image_repo+tag",
825 "name": "arvados/jobs:"+arvados_cwl.__version__,
827 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
828 {"items": [{"created_at": "",
830 "link_class": "docker_image_hash",
833 "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
835 find_one_image_hash.return_value = "123456"
837 arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
841 }], "items_available": 1, "offset": 0},)
842 arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
843 self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
844 arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
846 class TestCreateTemplate(unittest.TestCase):
847 existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
849 def _adjust_script_params(self, expect_component):
850 expect_component['script_parameters']['x'] = {
854 'value': '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
856 expect_component['script_parameters']['y'] = {
857 'dataclass': 'Collection',
860 'value': '99999999999999999999999999999998+99',
862 expect_component['script_parameters']['z'] = {
863 'dataclass': 'Collection',
869 def test_create(self, stubs):
870 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
872 capture_stdout = cStringIO.StringIO()
874 exited = arvados_cwl.main(
875 ["--create-workflow", "--debug",
877 "--project-uuid", project_uuid,
878 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
879 capture_stdout, sys.stderr, api_client=stubs.api)
880 self.assertEqual(exited, 0)
882 stubs.api.pipeline_instances().create.refute_called()
883 stubs.api.jobs().create.refute_called()
885 expect_component = copy.deepcopy(stubs.expect_job_spec)
886 self._adjust_script_params(expect_component)
889 "submit_wf.cwl": expect_component,
891 "name": "submit_wf.cwl",
892 "owner_uuid": project_uuid,
894 stubs.api.pipeline_templates().create.assert_called_with(
895 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
897 self.assertEqual(capture_stdout.getvalue(),
898 stubs.expect_pipeline_template_uuid + '\n')
902 def test_create_name(self, stubs):
903 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
905 capture_stdout = cStringIO.StringIO()
907 exited = arvados_cwl.main(
908 ["--create-workflow", "--debug",
909 "--project-uuid", project_uuid,
911 "--name", "testing 123",
912 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
913 capture_stdout, sys.stderr, api_client=stubs.api)
914 self.assertEqual(exited, 0)
916 stubs.api.pipeline_instances().create.refute_called()
917 stubs.api.jobs().create.refute_called()
919 expect_component = copy.deepcopy(stubs.expect_job_spec)
920 self._adjust_script_params(expect_component)
923 "testing 123": expect_component,
925 "name": "testing 123",
926 "owner_uuid": project_uuid,
928 stubs.api.pipeline_templates().create.assert_called_with(
929 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
931 self.assertEqual(capture_stdout.getvalue(),
932 stubs.expect_pipeline_template_uuid + '\n')
936 def test_update_name(self, stubs):
937 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
939 capture_stdout = cStringIO.StringIO()
941 exited = arvados_cwl.main(
942 ["--update-workflow", self.existing_template_uuid,
944 "--project-uuid", project_uuid,
946 "--name", "testing 123",
947 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
948 capture_stdout, sys.stderr, api_client=stubs.api)
949 self.assertEqual(exited, 0)
951 stubs.api.pipeline_instances().create.refute_called()
952 stubs.api.jobs().create.refute_called()
954 expect_component = copy.deepcopy(stubs.expect_job_spec)
955 self._adjust_script_params(expect_component)
958 "testing 123": expect_component,
960 "name": "testing 123",
961 "owner_uuid": project_uuid,
963 stubs.api.pipeline_templates().create.refute_called()
964 stubs.api.pipeline_templates().update.assert_called_with(
965 body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
967 self.assertEqual(capture_stdout.getvalue(),
968 self.existing_template_uuid + '\n')
971 class TestCreateWorkflow(unittest.TestCase):
972 existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
973 expect_workflow = open("tests/wf/expect_packed.cwl").read()
976 def test_create(self, stubs):
977 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
979 capture_stdout = cStringIO.StringIO()
981 exited = arvados_cwl.main(
982 ["--create-workflow", "--debug",
984 "--project-uuid", project_uuid,
985 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
986 capture_stdout, sys.stderr, api_client=stubs.api)
987 self.assertEqual(exited, 0)
989 stubs.api.pipeline_templates().create.refute_called()
990 stubs.api.container_requests().create.refute_called()
994 "owner_uuid": project_uuid,
995 "name": "submit_wf.cwl",
997 "definition": self.expect_workflow,
1000 stubs.api.workflows().create.assert_called_with(
1001 body=JsonDiffMatcher(body))
1003 self.assertEqual(capture_stdout.getvalue(),
1004 stubs.expect_workflow_uuid + '\n')
1008 def test_create_name(self, stubs):
1009 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1011 capture_stdout = cStringIO.StringIO()
1013 exited = arvados_cwl.main(
1014 ["--create-workflow", "--debug",
1016 "--project-uuid", project_uuid,
1017 "--name", "testing 123",
1018 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1019 capture_stdout, sys.stderr, api_client=stubs.api)
1020 self.assertEqual(exited, 0)
1022 stubs.api.pipeline_templates().create.refute_called()
1023 stubs.api.container_requests().create.refute_called()
1027 "owner_uuid": project_uuid,
1028 "name": "testing 123",
1030 "definition": self.expect_workflow,
1033 stubs.api.workflows().create.assert_called_with(
1034 body=JsonDiffMatcher(body))
1036 self.assertEqual(capture_stdout.getvalue(),
1037 stubs.expect_workflow_uuid + '\n')
1040 def test_incompatible_api(self, stubs):
1041 capture_stderr = cStringIO.StringIO()
1042 logging.getLogger('arvados.cwl-runner').addHandler(
1043 logging.StreamHandler(capture_stderr))
1045 exited = arvados_cwl.main(
1046 ["--update-workflow", self.existing_workflow_uuid,
1049 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1050 sys.stderr, sys.stderr, api_client=stubs.api)
1051 self.assertEqual(exited, 1)
1052 self.assertRegexpMatches(
1053 capture_stderr.getvalue(),
1054 "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
1057 def test_update(self, stubs):
1058 capture_stdout = cStringIO.StringIO()
1060 exited = arvados_cwl.main(
1061 ["--update-workflow", self.existing_workflow_uuid,
1063 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1064 capture_stdout, sys.stderr, api_client=stubs.api)
1065 self.assertEqual(exited, 0)
1069 "name": "submit_wf.cwl",
1071 "definition": self.expect_workflow,
1074 stubs.api.workflows().update.assert_called_with(
1075 uuid=self.existing_workflow_uuid,
1076 body=JsonDiffMatcher(body))
1077 self.assertEqual(capture_stdout.getvalue(),
1078 self.existing_workflow_uuid + '\n')
1082 def test_update_name(self, stubs):
1083 capture_stdout = cStringIO.StringIO()
1085 exited = arvados_cwl.main(
1086 ["--update-workflow", self.existing_workflow_uuid,
1087 "--debug", "--name", "testing 123",
1088 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
1089 capture_stdout, sys.stderr, api_client=stubs.api)
1090 self.assertEqual(exited, 0)
1094 "name": "testing 123",
1096 "definition": self.expect_workflow,
1099 stubs.api.workflows().update.assert_called_with(
1100 uuid=self.existing_workflow_uuid,
1101 body=JsonDiffMatcher(body))
1102 self.assertEqual(capture_stdout.getvalue(),
1103 self.existing_workflow_uuid + '\n')
1107 def test_create_collection_per_tool(self, stubs):
1108 project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
1110 capture_stdout = cStringIO.StringIO()
1112 exited = arvados_cwl.main(
1113 ["--create-workflow", "--debug",
1115 "--project-uuid", project_uuid,
1116 "tests/collection_per_tool/collection_per_tool.cwl"],
1117 capture_stdout, sys.stderr, api_client=stubs.api)
1118 self.assertEqual(exited, 0)
1120 expect_workflow = open("tests/collection_per_tool/collection_per_tool_packed.cwl").read()
1124 "owner_uuid": project_uuid,
1125 "name": "collection_per_tool.cwl",
1127 "definition": expect_workflow,
1130 stubs.api.workflows().create.assert_called_with(
1131 body=JsonDiffMatcher(body))
1133 self.assertEqual(capture_stdout.getvalue(),
1134 stubs.expect_workflow_uuid + '\n')
1136 class TestTemplateInputs(unittest.TestCase):
1139 "inputs_test.cwl": {
1140 'runtime_constraints': {
1141 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1142 'min_ram_mb_per_node': 1024
1144 'script_parameters': {
1146 '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main',
1147 'optionalFloatInput': None,
1150 'dataclass': 'File',
1152 'title': "It's a file; we expect to find some characters in it.",
1153 'description': 'If there were anything further to say, it would be said here,\nor here.'
1157 'dataclass': 'number',
1159 'title': 'Floats like a duck',
1163 'optionalFloatInput': {
1164 'type': ['null', 'float'],
1165 'dataclass': 'number',
1170 'dataclass': 'boolean',
1172 'title': 'True or false?',
1175 'repository': 'arvados',
1176 'script_version': 'master',
1177 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1178 'script': 'cwl-runner',
1181 "name": "inputs_test.cwl",
1185 def test_inputs_empty(self, stubs):
1186 exited = arvados_cwl.main(
1187 ["--create-template",
1188 "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1189 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1190 self.assertEqual(exited, 0)
1192 stubs.api.pipeline_templates().create.assert_called_with(
1193 body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1196 def test_inputs(self, stubs):
1197 exited = arvados_cwl.main(
1198 ["--create-template",
1199 "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1200 cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1201 self.assertEqual(exited, 0)
1203 expect_template = copy.deepcopy(self.expect_template)
1204 params = expect_template[
1205 "components"]["inputs_test.cwl"]["script_parameters"]
1206 params["fileInput"]["value"] = '169f39d466a5438ac4a90e779bf750c7+53/blorp.txt'
1207 params["cwl:tool"] = '6c5ee1cd606088106d9f28367cde1e41+60/workflow.cwl#main'
1208 params["floatInput"]["value"] = 1.234
1209 params["boolInput"]["value"] = True
1211 stubs.api.pipeline_templates().create.assert_called_with(
1212 body=JsonDiffMatcher(expect_template), ensure_unique_name=True)