sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 _rootDesc = None
  21
  22 def stubs(func):
  23     @functools.wraps(func)
  24     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  25     @mock.patch("arvados.collection.KeepClient")
  26     @mock.patch("arvados.keep.KeepClient")
  27     @mock.patch("arvados.events.subscribe")
  28     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  29         class Stubs:
  30             pass
  31         stubs = Stubs()
  32         stubs.events = events
  33         stubs.keepdocker = keepdocker
  34
  35
  36         def putstub(p, **kwargs):
  37             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  38         keep_client1().put.side_effect = putstub
  39         keep_client1.put.side_effect = putstub
  40         keep_client2().put.side_effect = putstub
  41         keep_client2.put.side_effect = putstub
  42
  43         stubs.keep_client = keep_client2
  44         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  45         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  46
  47         stubs.api = mock.MagicMock()
  48         stubs.api._rootDesc = get_rootDesc()
  49
  50         stubs.api.users().current().execute.return_value = {
  51             "uuid": stubs.fake_user_uuid,
  52         }
  53         stubs.api.collections().list().execute.return_value = {"items": []}
  54         stubs.api.collections().create().execute.side_effect = ({
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  56             "portable_data_hash": "99999999999999999999999999999991+99",
  57             "manifest_text": ""
  58         }, {
  59             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  60             "portable_data_hash": "99999999999999999999999999999992+99",
  61             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  62         },
  63         {
  64             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  65             "portable_data_hash": "99999999999999999999999999999994+99",
  66             "manifest_text": ""
  67         },
  68         {
  69             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  70             "portable_data_hash": "99999999999999999999999999999995+99",
  71             "manifest_text": ""
  72         },
  73         {
  74             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  75             "portable_data_hash": "99999999999999999999999999999996+99",
  76             "manifest_text": ""
  77         }
  78         )
  79         stubs.api.collections().get().execute.return_value = {
  80             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 102                 'min_ram_mb_per_node': 1024
 103             },
 104             'script_parameters': {
 105                 'x': {
 106                     'basename': 'blorp.txt',
 107                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 108                     'class': 'File'
 109                 },
 110                 'y': {
 111                     'basename': '99999999999999999999999999999998+99',
 112                     'location': 'keep:99999999999999999999999999999998+99',
 113                     'class': 'Directory'
 114                 },
 115                 'z': {
 116                     'basename': 'anonymous',
 117                     "listing": [{
 118                         "basename": "renamed.txt",
 119                         "class": "File",
 120                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 121                     }],
 122                     'class': 'Directory'
 123                 },
 124                 'cwl:tool':
 125                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 126             },
 127             'repository': 'arvados',
 128             'script_version': arvados_cwl.__version__,
 129             'script': 'cwl-runner'
 130         }
 131         stubs.pipeline_component = stubs.expect_job_spec.copy()
 132         stubs.expect_pipeline_instance = {
 133             'name': 'submit_wf.cwl',
 134             'state': 'RunningOnServer',
 135             'owner_uuid': None,
 136             "components": {
 137                 "cwl-runner": {
 138                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 139                     'script_parameters': {
 140                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 141                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 142                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 143                               'listing': [
 144                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 145                               ]}},
 146                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
 147                         'arv:enable_reuse': True
 148                     },
 149                     'repository': 'arvados',
 150                     'script_version': arvados_cwl.__version__,
 151                     'script': 'cwl-runner',
 152                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 153                 }
 154             }
 155         }
 156         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 157         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 158         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 159         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 160         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 161             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 162             "state": "Queued"
 163         }
 164         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 165         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 166
 167         stubs.expect_container_spec = {
 168             'priority': 1,
 169             'mounts': {
 170                 '/var/spool/cwl': {
 171                     'writable': True,
 172                     'kind': 'collection'
 173                 },
 174                 '/var/lib/cwl/workflow': {
 175                     'portable_data_hash': '99999999999999999999999999999991+99',
 176                     'kind': 'collection'
 177                 },
 178                 'stdout': {
 179                     'path': '/var/spool/cwl/cwl.output.json',
 180                     'kind': 'file'
 181                 },
 182                 '/var/lib/cwl/job/cwl.input.json': {
 183                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 184                     'kind': 'collection'
 185                 }
 186             },
 187             'state': 'Committed',
 188             'owner_uuid': None,
 189             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 190             'name': 'submit_wf.cwl',
 191             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 192             'output_path': '/var/spool/cwl',
 193             'cwd': '/var/spool/cwl',
 194             'runtime_constraints': {
 195                 'API': True,
 196                 'vcpus': 1,
 197                 'ram': 1024*1024*1024
 198             }
 199         }
 200
 201         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 202         stubs.api.workflows().create().execute.return_value = {
 203             "uuid": stubs.expect_workflow_uuid,
 204         }
 205         def update_mock(**kwargs):
 206             stubs.updated_uuid = kwargs.get('uuid')
 207             return mock.DEFAULT
 208         stubs.api.workflows().update.side_effect = update_mock
 209         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 210             "uuid": stubs.updated_uuid,
 211         }
 212
 213         return func(self, stubs, *args, **kwargs)
 214     return wrapped
 215
 216
 217 class TestSubmit(unittest.TestCase):
 218     @mock.patch("time.sleep")
 219     @stubs
 220     def test_submit(self, stubs, tm):
 221         capture_stdout = cStringIO.StringIO()
 222         exited = arvados_cwl.main(
 223             ["--submit", "--no-wait", "--debug",
 224              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 225             capture_stdout, sys.stderr, api_client=stubs.api)
 226         self.assertEqual(exited, 0)
 227
 228         stubs.api.collections().create.assert_has_calls([
 229             mock.call(),
 230             mock.call(body={
 231                 'manifest_text':
 232                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 233                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 234                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 235                 'owner_uuid': None,
 236                 'name': 'submit_wf.cwl',
 237             }, ensure_unique_name=True),
 238             mock.call().execute(),
 239             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 240                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 241                             'replication_desired': None,
 242                             'name': 'New collection'
 243             }, ensure_unique_name=True),
 244             mock.call().execute(num_retries=4),
 245             mock.call(body={
 246                 'manifest_text':
 247                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 248                 'owner_uuid': None,
 249                 'name': '#',
 250             }, ensure_unique_name=True),
 251             mock.call().execute()])
 252
 253         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 254         stubs.api.pipeline_instances().create.assert_called_with(
 255             body=expect_pipeline)
 256         self.assertEqual(capture_stdout.getvalue(),
 257                          stubs.expect_pipeline_uuid + '\n')
 258
 259
 260     @mock.patch("time.sleep")
 261     @stubs
 262     def test_submit_no_reuse(self, stubs, tm):
 263         capture_stdout = cStringIO.StringIO()
 264         exited = arvados_cwl.main(
 265             ["--submit", "--no-wait", "--debug", "--disable-reuse",
 266              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 267             capture_stdout, sys.stderr, api_client=stubs.api)
 268         self.assertEqual(exited, 0)
 269
 270         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 271
 272         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 273         stubs.api.pipeline_instances().create.assert_called_with(
 274             body=expect_pipeline)
 275         self.assertEqual(capture_stdout.getvalue(),
 276                          stubs.expect_pipeline_uuid + '\n')
 277
 278
 279     @mock.patch("time.sleep")
 280     @stubs
 281     def test_submit_runner_ram(self, stubs, tm):
 282         capture_stdout = cStringIO.StringIO()
 283         exited = arvados_cwl.main(
 284             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 285              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 286             capture_stdout, sys.stderr, api_client=stubs.api)
 287         self.assertEqual(exited, 0)
 288
 289         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 290
 291         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 292         stubs.api.pipeline_instances().create.assert_called_with(
 293             body=expect_pipeline)
 294         self.assertEqual(capture_stdout.getvalue(),
 295                          stubs.expect_pipeline_uuid + '\n')
 296
 297
 298     @mock.patch("time.sleep")
 299     @stubs
 300     def test_submit_invalid_runner_ram(self, stubs, tm):
 301         capture_stdout = cStringIO.StringIO()
 302         exited = arvados_cwl.main(
 303             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 304              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 305             capture_stdout, sys.stderr, api_client=stubs.api)
 306         self.assertEqual(exited, 1)
 307
 308     @mock.patch("time.sleep")
 309     @stubs
 310     def test_submit_output_name(self, stubs, tm):
 311         output_name = "test_output_name"
 312
 313         capture_stdout = cStringIO.StringIO()
 314         exited = arvados_cwl.main(
 315             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 316              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 317             capture_stdout, sys.stderr, api_client=stubs.api)
 318         self.assertEqual(exited, 0)
 319
 320         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 321
 322         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 323         stubs.api.pipeline_instances().create.assert_called_with(
 324             body=expect_pipeline)
 325         self.assertEqual(capture_stdout.getvalue(),
 326                          stubs.expect_pipeline_uuid + '\n')
 327
 328     @mock.patch("time.sleep")
 329     @stubs
 330     def test_submit_output_tags(self, stubs, tm):
 331         output_tags = "tag0,tag1,tag2"
 332
 333         capture_stdout = cStringIO.StringIO()
 334         exited = arvados_cwl.main(
 335             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 336              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 337             capture_stdout, sys.stderr, api_client=stubs.api)
 338         self.assertEqual(exited, 0)
 339
 340         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 341
 342         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 343         stubs.api.pipeline_instances().create.assert_called_with(
 344             body=expect_pipeline)
 345         self.assertEqual(capture_stdout.getvalue(),
 346                          stubs.expect_pipeline_uuid + '\n')
 347
 348     @mock.patch("time.sleep")
 349     @stubs
 350     def test_submit_with_project_uuid(self, stubs, tm):
 351         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 352
 353         exited = arvados_cwl.main(
 354             ["--submit", "--no-wait",
 355              "--project-uuid", project_uuid,
 356              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 357             sys.stdout, sys.stderr, api_client=stubs.api)
 358         self.assertEqual(exited, 0)
 359
 360         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 361         expect_pipeline["owner_uuid"] = project_uuid
 362         stubs.api.pipeline_instances().create.assert_called_with(
 363             body=expect_pipeline)
 364
 365     @stubs
 366     def test_submit_container(self, stubs):
 367         capture_stdout = cStringIO.StringIO()
 368         try:
 369             exited = arvados_cwl.main(
 370                 ["--submit", "--no-wait", "--api=containers", "--debug",
 371                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 372                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 373             self.assertEqual(exited, 0)
 374         except:
 375             logging.exception("")
 376
 377         stubs.api.collections().create.assert_has_calls([
 378             mock.call(),
 379             mock.call(body={
 380                 'manifest_text':
 381                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 382                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 383                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 384                 'owner_uuid': None,
 385                 'name': 'submit_wf.cwl',
 386             }, ensure_unique_name=True),
 387             mock.call().execute(),
 388             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 389                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 390                             'name': 'New collection',
 391                             'replication_desired': None,
 392             }, ensure_unique_name=True),
 393             mock.call().execute(num_retries=4),
 394             mock.call(body={
 395                 'manifest_text':
 396                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 397                 'owner_uuid': None,
 398                 'name': '#',
 399             }, ensure_unique_name=True),
 400             mock.call().execute()])
 401
 402         expect_container = copy.deepcopy(stubs.expect_container_spec)
 403         stubs.api.container_requests().create.assert_called_with(
 404             body=expect_container)
 405         self.assertEqual(capture_stdout.getvalue(),
 406                          stubs.expect_container_request_uuid + '\n')
 407
 408     @stubs
 409     def test_submit_container_no_reuse(self, stubs):
 410         capture_stdout = cStringIO.StringIO()
 411         try:
 412             exited = arvados_cwl.main(
 413                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 414                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 415                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 416             self.assertEqual(exited, 0)
 417         except:
 418             logging.exception("")
 419
 420         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 421
 422         expect_container = copy.deepcopy(stubs.expect_container_spec)
 423         stubs.api.container_requests().create.assert_called_with(
 424             body=expect_container)
 425         self.assertEqual(capture_stdout.getvalue(),
 426                          stubs.expect_container_request_uuid + '\n')
 427
 428     @stubs
 429     def test_submit_container_output_name(self, stubs):
 430         output_name = "test_output_name"
 431
 432         capture_stdout = cStringIO.StringIO()
 433         try:
 434             exited = arvados_cwl.main(
 435                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 436                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 437                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 438             self.assertEqual(exited, 0)
 439         except:
 440             logging.exception("")
 441
 442         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-name="+output_name, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 443
 444         expect_container = copy.deepcopy(stubs.expect_container_spec)
 445         stubs.api.container_requests().create.assert_called_with(
 446             body=expect_container)
 447         self.assertEqual(capture_stdout.getvalue(),
 448                          stubs.expect_container_request_uuid + '\n')
 449
 450     @stubs
 451     def test_submit_container_output_tags(self, stubs):
 452         output_tags = "tag0,tag1,tag2"
 453
 454         capture_stdout = cStringIO.StringIO()
 455         try:
 456             exited = arvados_cwl.main(
 457                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 458                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 459                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 460             self.assertEqual(exited, 0)
 461         except:
 462             logging.exception("")
 463
 464         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-tags="+output_tags, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 465
 466         expect_container = copy.deepcopy(stubs.expect_container_spec)
 467         stubs.api.container_requests().create.assert_called_with(
 468             body=expect_container)
 469         self.assertEqual(capture_stdout.getvalue(),
 470                          stubs.expect_container_request_uuid + '\n')
 471
 472     @stubs
 473     def test_submit_container_runner_ram(self, stubs):
 474         capture_stdout = cStringIO.StringIO()
 475         try:
 476             exited = arvados_cwl.main(
 477                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 478                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 479                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 480             self.assertEqual(exited, 0)
 481         except:
 482             logging.exception("")
 483
 484         stubs.expect_container_spec["runtime_constraints"]["ram"] = 2048*1024*1024
 485
 486         expect_container = copy.deepcopy(stubs.expect_container_spec)
 487         stubs.api.container_requests().create.assert_called_with(
 488             body=expect_container)
 489         self.assertEqual(capture_stdout.getvalue(),
 490                          stubs.expect_container_request_uuid + '\n')
 491
 492     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 493     @mock.patch("cwltool.docker.get_image")
 494     @mock.patch("arvados.api")
 495     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 496         arvrunner = mock.MagicMock()
 497         arvrunner.project_uuid = ""
 498         api.return_value = mock.MagicMock()
 499         arvrunner.api = api.return_value
 500         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 501                                                             {"items": [], "items_available": 0, "offset": 0},
 502                                                             {"items": [], "items_available": 0, "offset": 0},
 503                                                             {"items": [{"created_at": "",
 504                                                                         "head_uuid": "",
 505                                                                         "link_class": "docker_image_hash",
 506                                                                         "name": "123456",
 507                                                                         "owner_uuid": "",
 508                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 509                                                             {"items": [], "items_available": 0, "offset": 0},
 510                                                             {"items": [{"created_at": "",
 511                                                                         "head_uuid": "",
 512                                                                         "link_class": "docker_image_repo+tag",
 513                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 514                                                                         "owner_uuid": "",
 515                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 516                                                             {"items": [{"created_at": "",
 517                                                                         "head_uuid": "",
 518                                                                         "link_class": "docker_image_hash",
 519                                                                         "name": "123456",
 520                                                                         "owner_uuid": "",
 521                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
 522         )
 523         find_one_image_hash.return_value = "123456"
 524
 525         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 526                                                                   {"items": [{"uuid": "",
 527                                                                               "owner_uuid": "",
 528                                                                               "manifest_text": "",
 529                                                                               "properties": ""
 530                                                                           }], "items_available": 1, "offset": 0},
 531                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
 532         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 533         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
 534
 535 class TestCreateTemplate(unittest.TestCase):
 536     @stubs
 537     def test_create(self, stubs):
 538         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 539
 540         capture_stdout = cStringIO.StringIO()
 541
 542         exited = arvados_cwl.main(
 543             ["--create-workflow", "--debug",
 544              "--project-uuid", project_uuid,
 545              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 546             capture_stdout, sys.stderr, api_client=stubs.api)
 547         self.assertEqual(exited, 0)
 548
 549         stubs.api.pipeline_instances().create.refute_called()
 550         stubs.api.jobs().create.refute_called()
 551
 552         expect_component = copy.deepcopy(stubs.expect_job_spec)
 553         expect_component['script_parameters']['x'] = {
 554             'dataclass': 'File',
 555             'required': True,
 556             'type': 'File',
 557             'value': '99999999999999999999999999999994+99/blorp.txt',
 558         }
 559         expect_component['script_parameters']['y'] = {
 560             'dataclass': 'Collection',
 561             'required': True,
 562             'type': 'Directory',
 563             'value': '99999999999999999999999999999998+99',
 564         }
 565         expect_component['script_parameters']['z'] = {
 566             'dataclass': 'Collection',
 567             'required': True,
 568             'type': 'Directory',
 569         }
 570         expect_template = {
 571             "components": {
 572                 "submit_wf.cwl": expect_component,
 573             },
 574             "name": "submit_wf.cwl",
 575             "owner_uuid": project_uuid,
 576         }
 577         stubs.api.pipeline_templates().create.assert_called_with(
 578             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 579
 580         self.assertEqual(capture_stdout.getvalue(),
 581                          stubs.expect_pipeline_template_uuid + '\n')
 582
 583
 584 class TestCreateWorkflow(unittest.TestCase):
 585     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 586     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 587
 588     @stubs
 589     def test_create(self, stubs):
 590         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 591
 592         capture_stdout = cStringIO.StringIO()
 593
 594         exited = arvados_cwl.main(
 595             ["--create-workflow", "--debug",
 596              "--api=containers",
 597              "--project-uuid", project_uuid,
 598              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 599             capture_stdout, sys.stderr, api_client=stubs.api)
 600         self.assertEqual(exited, 0)
 601
 602         stubs.api.pipeline_templates().create.refute_called()
 603         stubs.api.container_requests().create.refute_called()
 604
 605         body = {
 606             "workflow": {
 607                 "owner_uuid": project_uuid,
 608                 "name": "submit_wf.cwl",
 609                 "description": "",
 610                 "definition": self.expect_workflow,
 611             }
 612         }
 613         stubs.api.workflows().create.assert_called_with(
 614             body=JsonDiffMatcher(body))
 615
 616         self.assertEqual(capture_stdout.getvalue(),
 617                          stubs.expect_workflow_uuid + '\n')
 618
 619     @stubs
 620     def test_incompatible_api(self, stubs):
 621         capture_stderr = cStringIO.StringIO()
 622         logging.getLogger('arvados.cwl-runner').addHandler(
 623             logging.StreamHandler(capture_stderr))
 624
 625         exited = arvados_cwl.main(
 626             ["--update-workflow", self.existing_workflow_uuid,
 627              "--api=jobs",
 628              "--debug",
 629              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 630             sys.stderr, sys.stderr, api_client=stubs.api)
 631         self.assertEqual(exited, 1)
 632         self.assertRegexpMatches(
 633             capture_stderr.getvalue(),
 634             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
 635
 636     @stubs
 637     def test_update(self, stubs):
 638         capture_stdout = cStringIO.StringIO()
 639
 640         exited = arvados_cwl.main(
 641             ["--update-workflow", self.existing_workflow_uuid,
 642              "--debug",
 643              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 644             capture_stdout, sys.stderr, api_client=stubs.api)
 645         self.assertEqual(exited, 0)
 646
 647         body = {
 648             "workflow": {
 649                 "name": "submit_wf.cwl",
 650                 "description": "",
 651                 "definition": self.expect_workflow,
 652             }
 653         }
 654         stubs.api.workflows().update.assert_called_with(
 655             uuid=self.existing_workflow_uuid,
 656             body=JsonDiffMatcher(body))
 657         self.assertEqual(capture_stdout.getvalue(),
 658                          self.existing_workflow_uuid + '\n')
 659
 660
 661 class TestTemplateInputs(unittest.TestCase):
 662     expect_template = {
 663         "components": {
 664             "inputs_test.cwl": {
 665                 'runtime_constraints': {
 666                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 667                     'min_ram_mb_per_node': 1024
 668                 },
 669                 'script_parameters': {
 670                     'cwl:tool':
 671                     '99999999999999999999999999999991+99/'
 672                     'wf/inputs_test.cwl',
 673                     'optionalFloatInput': None,
 674                     'fileInput': {
 675                         'type': 'File',
 676                         'dataclass': 'File',
 677                         'required': True,
 678                         'title': "It's a file; we expect to find some characters in it.",
 679                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 680                     },
 681                     'floatInput': {
 682                         'type': 'float',
 683                         'dataclass': 'number',
 684                         'required': True,
 685                         'title': 'Floats like a duck',
 686                         'default': 0.1,
 687                         'value': 0.1,
 688                     },
 689                     'optionalFloatInput': {
 690                         'type': ['null', 'float'],
 691                         'dataclass': 'number',
 692                         'required': False,
 693                     },
 694                     'boolInput': {
 695                         'type': 'boolean',
 696                         'dataclass': 'boolean',
 697                         'required': True,
 698                         'title': 'True or false?',
 699                     },
 700                 },
 701                 'repository': 'arvados',
 702                 'script_version': arvados_cwl.__version__,
 703                 'script': 'cwl-runner',
 704             },
 705         },
 706         "name": "inputs_test.cwl",
 707     }
 708
 709     @stubs
 710     def test_inputs_empty(self, stubs):
 711         exited = arvados_cwl.main(
 712             ["--create-template", "--no-wait",
 713              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 714             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 715         self.assertEqual(exited, 0)
 716
 717         stubs.api.pipeline_templates().create.assert_called_with(
 718             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
 719
 720     @stubs
 721     def test_inputs(self, stubs):
 722         exited = arvados_cwl.main(
 723             ["--create-template", "--no-wait",
 724              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 725             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 726         self.assertEqual(exited, 0)
 727
 728         expect_template = copy.deepcopy(self.expect_template)
 729         params = expect_template[
 730             "components"]["inputs_test.cwl"]["script_parameters"]
 731         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 732         params["floatInput"]["value"] = 1.234
 733         params["boolInput"]["value"] = True
 734
 735         stubs.api.pipeline_templates().create.assert_called_with(
 736             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)