sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 _rootDesc = None
  21
  22 def stubs(func):
  23     @functools.wraps(func)
  24     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  25     @mock.patch("arvados.collection.KeepClient")
  26     @mock.patch("arvados.keep.KeepClient")
  27     @mock.patch("arvados.events.subscribe")
  28     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  29         class Stubs:
  30             pass
  31         stubs = Stubs()
  32         stubs.events = events
  33         stubs.keepdocker = keepdocker
  34
  35
  36         def putstub(p, **kwargs):
  37             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  38         keep_client1().put.side_effect = putstub
  39         keep_client1.put.side_effect = putstub
  40         keep_client2().put.side_effect = putstub
  41         keep_client2.put.side_effect = putstub
  42
  43         stubs.keep_client = keep_client2
  44         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  45         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  46
  47         stubs.api = mock.MagicMock()
  48         stubs.api._rootDesc = get_rootDesc()
  49
  50         stubs.api.users().current().execute.return_value = {
  51             "uuid": stubs.fake_user_uuid,
  52         }
  53         stubs.api.collections().list().execute.return_value = {"items": []}
  54         stubs.api.collections().create().execute.side_effect = ({
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  56             "portable_data_hash": "99999999999999999999999999999991+99",
  57             "manifest_text": ""
  58         }, {
  59             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  60             "portable_data_hash": "99999999999999999999999999999992+99",
  61             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  62         },
  63         {
  64             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  65             "portable_data_hash": "99999999999999999999999999999994+99",
  66             "manifest_text": ""
  67         },
  68         {
  69             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  70             "portable_data_hash": "99999999999999999999999999999995+99",
  71             "manifest_text": ""
  72         },
  73         {
  74             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  75             "portable_data_hash": "99999999999999999999999999999996+99",
  76             "manifest_text": ""
  77         }
  78         )
  79         stubs.api.collections().get().execute.return_value = {
  80             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 102                 'min_ram_mb_per_node': 1024
 103             },
 104             'script_parameters': {
 105                 'x': {
 106                     'basename': 'blorp.txt',
 107                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 108                     'class': 'File'
 109                 },
 110                 'y': {
 111                     'basename': '99999999999999999999999999999998+99',
 112                     'location': 'keep:99999999999999999999999999999998+99',
 113                     'class': 'Directory'
 114                 },
 115                 'z': {
 116                     'basename': 'anonymous',
 117                     "listing": [{
 118                         "basename": "renamed.txt",
 119                         "class": "File",
 120                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 121                     }],
 122                     'class': 'Directory'
 123                 },
 124                 'cwl:tool':
 125                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 126             },
 127             'repository': 'arvados',
 128             'script_version': 'master',
 129             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 130             'script': 'cwl-runner'
 131         }
 132         stubs.pipeline_component = stubs.expect_job_spec.copy()
 133         stubs.expect_pipeline_instance = {
 134             'name': 'submit_wf.cwl',
 135             'state': 'RunningOnServer',
 136             'owner_uuid': None,
 137             "components": {
 138                 "cwl-runner": {
 139                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 140                     'script_parameters': {
 141                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 142                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 143                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 144                               'listing': [
 145                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 146                               ]}},
 147                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
 148                         'arv:enable_reuse': True
 149                     },
 150                     'repository': 'arvados',
 151                     'script_version': 'master',
 152                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 153                     'script': 'cwl-runner',
 154                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 155                 }
 156             }
 157         }
 158         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 159         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 160         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 161         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 162         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 163             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 164             "state": "Queued"
 165         }
 166         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 167         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 168
 169         stubs.expect_container_spec = {
 170             'priority': 1,
 171             'mounts': {
 172                 '/var/spool/cwl': {
 173                     'writable': True,
 174                     'kind': 'collection'
 175                 },
 176                 '/var/lib/cwl/workflow': {
 177                     'portable_data_hash': '99999999999999999999999999999991+99',
 178                     'kind': 'collection'
 179                 },
 180                 'stdout': {
 181                     'path': '/var/spool/cwl/cwl.output.json',
 182                     'kind': 'file'
 183                 },
 184                 '/var/lib/cwl/cwl.input.json': {
 185                     'kind': 'json',
 186                     'content': {
 187                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 188                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999994+99/blorp.txt'},
 189                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 190                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 191                         ]}
 192                     },
 193                     'kind': 'json'
 194                 }
 195             },
 196             'state': 'Committed',
 197             'owner_uuid': None,
 198             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json'],
 199             'name': 'submit_wf.cwl',
 200             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 201             'output_path': '/var/spool/cwl',
 202             'cwd': '/var/spool/cwl',
 203             'runtime_constraints': {
 204                 'API': True,
 205                 'vcpus': 1,
 206                 'ram': 1024*1024*1024
 207             },
 208             "properties": {}
 209         }
 210
 211         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 212         stubs.api.workflows().create().execute.return_value = {
 213             "uuid": stubs.expect_workflow_uuid,
 214         }
 215         def update_mock(**kwargs):
 216             stubs.updated_uuid = kwargs.get('uuid')
 217             return mock.DEFAULT
 218         stubs.api.workflows().update.side_effect = update_mock
 219         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 220             "uuid": stubs.updated_uuid,
 221         }
 222
 223         return func(self, stubs, *args, **kwargs)
 224     return wrapped
 225
 226
 227 class TestSubmit(unittest.TestCase):
 228     @mock.patch("time.sleep")
 229     @stubs
 230     def test_submit(self, stubs, tm):
 231         capture_stdout = cStringIO.StringIO()
 232         exited = arvados_cwl.main(
 233             ["--submit", "--no-wait", "--debug",
 234              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 235             capture_stdout, sys.stderr, api_client=stubs.api)
 236         self.assertEqual(exited, 0)
 237
 238         stubs.api.collections().create.assert_has_calls([
 239             mock.call(),
 240             mock.call(body={
 241                 'manifest_text':
 242                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 243                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 244                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 245                 'owner_uuid': None,
 246                 'name': 'submit_wf.cwl',
 247             }, ensure_unique_name=True),
 248             mock.call().execute(),
 249             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 250                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 251                             'replication_desired': None,
 252                             'name': 'New collection'
 253             }, ensure_unique_name=True),
 254             mock.call().execute(num_retries=4),
 255             mock.call(body={
 256                 'manifest_text':
 257                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 258                 'owner_uuid': None,
 259                 'name': '#',
 260             }, ensure_unique_name=True),
 261             mock.call().execute()])
 262
 263         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 264         stubs.api.pipeline_instances().create.assert_called_with(
 265             body=expect_pipeline)
 266         self.assertEqual(capture_stdout.getvalue(),
 267                          stubs.expect_pipeline_uuid + '\n')
 268
 269
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit_no_reuse(self, stubs, tm):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--debug", "--disable-reuse",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 281
 282         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 283         stubs.api.pipeline_instances().create.assert_called_with(
 284             body=JsonDiffMatcher(expect_pipeline))
 285         self.assertEqual(capture_stdout.getvalue(),
 286                          stubs.expect_pipeline_uuid + '\n')
 287
 288
 289     @mock.patch("time.sleep")
 290     @stubs
 291     def test_submit_runner_ram(self, stubs, tm):
 292         capture_stdout = cStringIO.StringIO()
 293         exited = arvados_cwl.main(
 294             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 295              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 296             capture_stdout, sys.stderr, api_client=stubs.api)
 297         self.assertEqual(exited, 0)
 298
 299         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 300
 301         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 302         stubs.api.pipeline_instances().create.assert_called_with(
 303             body=JsonDiffMatcher(expect_pipeline))
 304         self.assertEqual(capture_stdout.getvalue(),
 305                          stubs.expect_pipeline_uuid + '\n')
 306
 307
 308     @mock.patch("time.sleep")
 309     @stubs
 310     def test_submit_invalid_runner_ram(self, stubs, tm):
 311         capture_stdout = cStringIO.StringIO()
 312         exited = arvados_cwl.main(
 313             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 314              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 315             capture_stdout, sys.stderr, api_client=stubs.api)
 316         self.assertEqual(exited, 1)
 317
 318     @mock.patch("time.sleep")
 319     @stubs
 320     def test_submit_output_name(self, stubs, tm):
 321         output_name = "test_output_name"
 322
 323         capture_stdout = cStringIO.StringIO()
 324         exited = arvados_cwl.main(
 325             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 326              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 327             capture_stdout, sys.stderr, api_client=stubs.api)
 328         self.assertEqual(exited, 0)
 329
 330         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 331
 332         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 333         stubs.api.pipeline_instances().create.assert_called_with(
 334             body=JsonDiffMatcher(expect_pipeline))
 335         self.assertEqual(capture_stdout.getvalue(),
 336                          stubs.expect_pipeline_uuid + '\n')
 337
 338
 339     @mock.patch("time.sleep")
 340     @stubs
 341     def test_submit_pipeline_name(self, stubs, tm):
 342         capture_stdout = cStringIO.StringIO()
 343         exited = arvados_cwl.main(
 344             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 345              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 346             capture_stdout, sys.stderr, api_client=stubs.api)
 347         self.assertEqual(exited, 0)
 348
 349         stubs.expect_pipeline_instance["name"] = "hello job 123"
 350
 351         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 352         stubs.api.pipeline_instances().create.assert_called_with(
 353             body=expect_pipeline)
 354         self.assertEqual(capture_stdout.getvalue(),
 355                          stubs.expect_pipeline_uuid + '\n')
 356
 357     @mock.patch("time.sleep")
 358     @stubs
 359     def test_submit_output_tags(self, stubs, tm):
 360         output_tags = "tag0,tag1,tag2"
 361
 362         capture_stdout = cStringIO.StringIO()
 363         exited = arvados_cwl.main(
 364             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 365              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 366             capture_stdout, sys.stderr, api_client=stubs.api)
 367         self.assertEqual(exited, 0)
 368
 369         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 370
 371         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 372         stubs.api.pipeline_instances().create.assert_called_with(
 373             body=JsonDiffMatcher(expect_pipeline))
 374         self.assertEqual(capture_stdout.getvalue(),
 375                          stubs.expect_pipeline_uuid + '\n')
 376
 377     @mock.patch("time.sleep")
 378     @stubs
 379     def test_submit_with_project_uuid(self, stubs, tm):
 380         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 381
 382         exited = arvados_cwl.main(
 383             ["--submit", "--no-wait",
 384              "--project-uuid", project_uuid,
 385              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 386             sys.stdout, sys.stderr, api_client=stubs.api)
 387         self.assertEqual(exited, 0)
 388
 389         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 390         expect_pipeline["owner_uuid"] = project_uuid
 391         stubs.api.pipeline_instances().create.assert_called_with(
 392             body=JsonDiffMatcher(expect_pipeline))
 393
 394     @stubs
 395     def test_submit_container(self, stubs):
 396         capture_stdout = cStringIO.StringIO()
 397         try:
 398             exited = arvados_cwl.main(
 399                 ["--submit", "--no-wait", "--api=containers", "--debug",
 400                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 401                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 402             self.assertEqual(exited, 0)
 403         except:
 404             logging.exception("")
 405
 406         stubs.api.collections().create.assert_has_calls([
 407             mock.call(),
 408             mock.call(body={
 409                 'manifest_text':
 410                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 411                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 412                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 413                 'owner_uuid': None,
 414                 'name': 'submit_wf.cwl',
 415             }, ensure_unique_name=True),
 416             mock.call().execute(),
 417             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 418                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 419                             'name': 'New collection',
 420                             'replication_desired': None,
 421             }, ensure_unique_name=True),
 422             mock.call().execute(num_retries=4),
 423             mock.call(body={
 424                 'manifest_text':
 425                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 426                 'owner_uuid': None,
 427                 'name': '#',
 428             }, ensure_unique_name=True),
 429             mock.call().execute()])
 430
 431         expect_container = copy.deepcopy(stubs.expect_container_spec)
 432         stubs.api.container_requests().create.assert_called_with(
 433             body=JsonDiffMatcher(expect_container))
 434         self.assertEqual(capture_stdout.getvalue(),
 435                          stubs.expect_container_request_uuid + '\n')
 436
 437     @stubs
 438     def test_submit_container_no_reuse(self, stubs):
 439         capture_stdout = cStringIO.StringIO()
 440         try:
 441             exited = arvados_cwl.main(
 442                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 443                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 444                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 445             self.assertEqual(exited, 0)
 446         except:
 447             logging.exception("")
 448
 449         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
 450
 451         expect_container = copy.deepcopy(stubs.expect_container_spec)
 452         stubs.api.container_requests().create.assert_called_with(
 453             body=JsonDiffMatcher(expect_container))
 454         self.assertEqual(capture_stdout.getvalue(),
 455                          stubs.expect_container_request_uuid + '\n')
 456
 457     @stubs
 458     def test_submit_container_output_name(self, stubs):
 459         output_name = "test_output_name"
 460
 461         capture_stdout = cStringIO.StringIO()
 462         try:
 463             exited = arvados_cwl.main(
 464                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 465                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 466                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 467             self.assertEqual(exited, 0)
 468         except:
 469             logging.exception("")
 470
 471         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-name="+output_name, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
 472
 473         expect_container = copy.deepcopy(stubs.expect_container_spec)
 474         stubs.api.container_requests().create.assert_called_with(
 475             body=JsonDiffMatcher(expect_container))
 476         self.assertEqual(capture_stdout.getvalue(),
 477                          stubs.expect_container_request_uuid + '\n')
 478
 479     @stubs
 480     def test_submit_container_output_tags(self, stubs):
 481         output_tags = "tag0,tag1,tag2"
 482
 483         capture_stdout = cStringIO.StringIO()
 484         try:
 485             exited = arvados_cwl.main(
 486                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 487                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 488                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 489             self.assertEqual(exited, 0)
 490         except:
 491             logging.exception("")
 492
 493         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-tags="+output_tags, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
 494
 495         expect_container = copy.deepcopy(stubs.expect_container_spec)
 496         stubs.api.container_requests().create.assert_called_with(
 497             body=JsonDiffMatcher(expect_container))
 498         self.assertEqual(capture_stdout.getvalue(),
 499                          stubs.expect_container_request_uuid + '\n')
 500
 501     @stubs
 502     def test_submit_container_runner_ram(self, stubs):
 503         capture_stdout = cStringIO.StringIO()
 504         try:
 505             exited = arvados_cwl.main(
 506                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 507                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 508                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 509             self.assertEqual(exited, 0)
 510         except:
 511             logging.exception("")
 512
 513         stubs.expect_container_spec["runtime_constraints"]["ram"] = 2048*1024*1024
 514
 515         expect_container = copy.deepcopy(stubs.expect_container_spec)
 516         stubs.api.container_requests().create.assert_called_with(
 517             body=JsonDiffMatcher(expect_container))
 518         self.assertEqual(capture_stdout.getvalue(),
 519                          stubs.expect_container_request_uuid + '\n')
 520
 521     @mock.patch("arvados.collection.CollectionReader")
 522     @mock.patch("time.sleep")
 523     @stubs
 524     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 525         capture_stdout = cStringIO.StringIO()
 526         exited = arvados_cwl.main(
 527             ["--submit", "--no-wait", "--api=containers", "--debug",
 528              "tests/wf/submit_keepref_wf.cwl"],
 529             capture_stdout, sys.stderr, api_client=stubs.api)
 530         self.assertEqual(exited, 0)
 531
 532
 533     @mock.patch("arvados.collection.CollectionReader")
 534     @mock.patch("time.sleep")
 535     @stubs
 536     def test_submit_keepref(self, stubs, tm, reader):
 537         capture_stdout = cStringIO.StringIO()
 538
 539         with open("tests/wf/expect_arvworkflow.cwl") as f:
 540             reader().open().__enter__().read.return_value = f.read()
 541
 542         exited = arvados_cwl.main(
 543             ["--submit", "--no-wait", "--api=containers", "--debug",
 544              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 545             capture_stdout, sys.stderr, api_client=stubs.api)
 546         self.assertEqual(exited, 0)
 547
 548         expect_container = {
 549             'priority': 1,
 550             'mounts': {
 551                 '/var/spool/cwl': {
 552                     'writable': True,
 553                     'kind': 'collection'
 554                 },
 555                 'stdout': {
 556                     'path': '/var/spool/cwl/cwl.output.json',
 557                     'kind': 'file'
 558                 },
 559                 '/var/lib/cwl/workflow': {
 560                     'portable_data_hash': '99999999999999999999999999999994+99',
 561                     'kind': 'collection'
 562                 },
 563                 '/var/lib/cwl/cwl.input.json': {
 564                     'content': {
 565                         'x': 'XxX'
 566                     },
 567                     'kind': 'json'
 568                 }
 569             }, 'state': 'Committed',
 570             'owner_uuid': None,
 571             'output_path': '/var/spool/cwl',
 572             'name': 'expect_arvworkflow.cwl#main',
 573             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 574             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 575             'cwd': '/var/spool/cwl',
 576             'runtime_constraints': {
 577                 'API': True,
 578                 'vcpus': 1,
 579                 'ram': 1073741824
 580             },
 581             "properties": {}
 582         }
 583
 584         stubs.api.container_requests().create.assert_called_with(
 585             body=JsonDiffMatcher(expect_container))
 586         self.assertEqual(capture_stdout.getvalue(),
 587                          stubs.expect_container_request_uuid + '\n')
 588
 589     @mock.patch("time.sleep")
 590     @stubs
 591     def test_submit_arvworkflow(self, stubs, tm):
 592         capture_stdout = cStringIO.StringIO()
 593
 594         with open("tests/wf/expect_arvworkflow.cwl") as f:
 595             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 596
 597         exited = arvados_cwl.main(
 598             ["--submit", "--no-wait", "--api=containers", "--debug",
 599              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 600             capture_stdout, sys.stderr, api_client=stubs.api)
 601         self.assertEqual(exited, 0)
 602
 603         expect_container = {
 604             'priority': 1,
 605             'mounts': {
 606                 '/var/spool/cwl': {
 607                     'writable': True,
 608                     'kind': 'collection'
 609                 },
 610                 'stdout': {
 611                     'path': '/var/spool/cwl/cwl.output.json',
 612                     'kind': 'file'
 613                 },
 614                 '/var/lib/cwl/workflow.json': {
 615                     'kind': 'json',
 616                     'json': {
 617                         'cwlVersion': 'v1.0',
 618                         '$graph': [
 619                             {
 620                                 'inputs': [
 621                                     {
 622                                         'inputBinding': {'position': 1},
 623                                         'type': 'string',
 624                                         'id': '#submit_tool.cwl/x'}
 625                                 ],
 626                                 'requirements': [
 627                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 628                                 ],
 629                                 'id': '#submit_tool.cwl',
 630                                 'outputs': [],
 631                                 'baseCommand': 'cat',
 632                                 'class': 'CommandLineTool'
 633                             }, {
 634                                 'id': '#main',
 635                                 'inputs': [
 636                                     {'type': 'string', 'id': '#main/x'}
 637                                 ],
 638                                 'steps': [
 639                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 640                                      'run': '#submit_tool.cwl',
 641                                      'id': '#main/step1',
 642                                      'out': []}
 643                                 ],
 644                                 'class': 'Workflow',
 645                                 'outputs': []
 646                             }
 647                         ]
 648                     }
 649                 },
 650                 '/var/lib/cwl/cwl.input.json': {
 651                     'content': {
 652                         'x': 'XxX'
 653                     },
 654                     'kind': 'json'
 655                 }
 656             }, 'state': 'Committed',
 657             'owner_uuid': None,
 658             'output_path': '/var/spool/cwl',
 659             'name': 'a test workflow',
 660             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 661             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 662             'cwd': '/var/spool/cwl',
 663             'runtime_constraints': {
 664                 'API': True,
 665                 'vcpus': 1,
 666                 'ram': 1073741824
 667             },
 668             "properties": {
 669                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 670             }
 671         }
 672
 673         stubs.api.container_requests().create.assert_called_with(
 674             body=JsonDiffMatcher(expect_container))
 675         self.assertEqual(capture_stdout.getvalue(),
 676                          stubs.expect_container_request_uuid + '\n')
 677
 678
 679     @stubs
 680     def test_submit_container_name(self, stubs):
 681         capture_stdout = cStringIO.StringIO()
 682         try:
 683             exited = arvados_cwl.main(
 684                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 685                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 686                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 687             self.assertEqual(exited, 0)
 688         except:
 689             logging.exception("")
 690
 691         stubs.expect_container_spec["name"] = "hello container 123"
 692
 693         expect_container = copy.deepcopy(stubs.expect_container_spec)
 694         stubs.api.container_requests().create.assert_called_with(
 695             body=expect_container)
 696         self.assertEqual(capture_stdout.getvalue(),
 697                          stubs.expect_container_request_uuid + '\n')
 698
 699
 700     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 701     @mock.patch("cwltool.docker.get_image")
 702     @mock.patch("arvados.api")
 703     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 704         arvrunner = mock.MagicMock()
 705         arvrunner.project_uuid = ""
 706         api.return_value = mock.MagicMock()
 707         arvrunner.api = api.return_value
 708         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 709                                                             {"items": [], "items_available": 0, "offset": 0},
 710                                                             {"items": [], "items_available": 0, "offset": 0},
 711                                                             {"items": [{"created_at": "",
 712                                                                         "head_uuid": "",
 713                                                                         "link_class": "docker_image_hash",
 714                                                                         "name": "123456",
 715                                                                         "owner_uuid": "",
 716                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 717                                                             {"items": [], "items_available": 0, "offset": 0},
 718                                                             {"items": [{"created_at": "",
 719                                                                         "head_uuid": "",
 720                                                                         "link_class": "docker_image_repo+tag",
 721                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 722                                                                         "owner_uuid": "",
 723                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 724                                                             {"items": [{"created_at": "",
 725                                                                         "head_uuid": "",
 726                                                                         "link_class": "docker_image_hash",
 727                                                                         "name": "123456",
 728                                                                         "owner_uuid": "",
 729                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
 730         )
 731         find_one_image_hash.return_value = "123456"
 732
 733         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 734                                                                   {"items": [{"uuid": "",
 735                                                                               "owner_uuid": "",
 736                                                                               "manifest_text": "",
 737                                                                               "properties": ""
 738                                                                           }], "items_available": 1, "offset": 0},
 739                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
 740         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 741         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
 742
 743 class TestCreateTemplate(unittest.TestCase):
 744     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 745
 746     def _adjust_script_params(self, expect_component):
 747         expect_component['script_parameters']['x'] = {
 748             'dataclass': 'File',
 749             'required': True,
 750             'type': 'File',
 751             'value': '99999999999999999999999999999994+99/blorp.txt',
 752         }
 753         expect_component['script_parameters']['y'] = {
 754             'dataclass': 'Collection',
 755             'required': True,
 756             'type': 'Directory',
 757             'value': '99999999999999999999999999999998+99',
 758         }
 759         expect_component['script_parameters']['z'] = {
 760             'dataclass': 'Collection',
 761             'required': True,
 762             'type': 'Directory',
 763         }
 764
 765     @stubs
 766     def test_create(self, stubs):
 767         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 768
 769         capture_stdout = cStringIO.StringIO()
 770
 771         exited = arvados_cwl.main(
 772             ["--create-workflow", "--debug",
 773              "--api=jobs",
 774              "--project-uuid", project_uuid,
 775              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 776             capture_stdout, sys.stderr, api_client=stubs.api)
 777         self.assertEqual(exited, 0)
 778
 779         stubs.api.pipeline_instances().create.refute_called()
 780         stubs.api.jobs().create.refute_called()
 781
 782         expect_component = copy.deepcopy(stubs.expect_job_spec)
 783         self._adjust_script_params(expect_component)
 784         expect_template = {
 785             "components": {
 786                 "submit_wf.cwl": expect_component,
 787             },
 788             "name": "submit_wf.cwl",
 789             "owner_uuid": project_uuid,
 790         }
 791         stubs.api.pipeline_templates().create.assert_called_with(
 792             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 793
 794         self.assertEqual(capture_stdout.getvalue(),
 795                          stubs.expect_pipeline_template_uuid + '\n')
 796
 797
 798     @stubs
 799     def test_create_name(self, stubs):
 800         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 801
 802         capture_stdout = cStringIO.StringIO()
 803
 804         exited = arvados_cwl.main(
 805             ["--create-workflow", "--debug",
 806              "--project-uuid", project_uuid,
 807              "--api=jobs",
 808              "--name", "testing 123",
 809              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 810             capture_stdout, sys.stderr, api_client=stubs.api)
 811         self.assertEqual(exited, 0)
 812
 813         stubs.api.pipeline_instances().create.refute_called()
 814         stubs.api.jobs().create.refute_called()
 815
 816         expect_component = copy.deepcopy(stubs.expect_job_spec)
 817         self._adjust_script_params(expect_component)
 818         expect_template = {
 819             "components": {
 820                 "testing 123": expect_component,
 821             },
 822             "name": "testing 123",
 823             "owner_uuid": project_uuid,
 824         }
 825         stubs.api.pipeline_templates().create.assert_called_with(
 826             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 827
 828         self.assertEqual(capture_stdout.getvalue(),
 829                          stubs.expect_pipeline_template_uuid + '\n')
 830
 831
 832     @stubs
 833     def test_update_name(self, stubs):
 834         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 835
 836         capture_stdout = cStringIO.StringIO()
 837
 838         exited = arvados_cwl.main(
 839             ["--update-workflow", self.existing_template_uuid,
 840              "--debug",
 841              "--project-uuid", project_uuid,
 842              "--api=jobs",
 843              "--name", "testing 123",
 844              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 845             capture_stdout, sys.stderr, api_client=stubs.api)
 846         self.assertEqual(exited, 0)
 847
 848         stubs.api.pipeline_instances().create.refute_called()
 849         stubs.api.jobs().create.refute_called()
 850
 851         expect_component = copy.deepcopy(stubs.expect_job_spec)
 852         self._adjust_script_params(expect_component)
 853         expect_template = {
 854             "components": {
 855                 "testing 123": expect_component,
 856             },
 857             "name": "testing 123",
 858             "owner_uuid": project_uuid,
 859         }
 860         stubs.api.pipeline_templates().create.refute_called()
 861         stubs.api.pipeline_templates().update.assert_called_with(
 862             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 863
 864         self.assertEqual(capture_stdout.getvalue(),
 865                          self.existing_template_uuid + '\n')
 866
 867
 868 class TestCreateWorkflow(unittest.TestCase):
 869     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 870     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 871
 872     @stubs
 873     def test_create(self, stubs):
 874         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 875
 876         capture_stdout = cStringIO.StringIO()
 877
 878         exited = arvados_cwl.main(
 879             ["--create-workflow", "--debug",
 880              "--api=containers",
 881              "--project-uuid", project_uuid,
 882              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 883             capture_stdout, sys.stderr, api_client=stubs.api)
 884         self.assertEqual(exited, 0)
 885
 886         stubs.api.pipeline_templates().create.refute_called()
 887         stubs.api.container_requests().create.refute_called()
 888
 889         body = {
 890             "workflow": {
 891                 "owner_uuid": project_uuid,
 892                 "name": "submit_wf.cwl",
 893                 "description": "",
 894                 "definition": self.expect_workflow,
 895             }
 896         }
 897         stubs.api.workflows().create.assert_called_with(
 898             body=JsonDiffMatcher(body))
 899
 900         self.assertEqual(capture_stdout.getvalue(),
 901                          stubs.expect_workflow_uuid + '\n')
 902
 903
 904     @stubs
 905     def test_create_name(self, stubs):
 906         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 907
 908         capture_stdout = cStringIO.StringIO()
 909
 910         exited = arvados_cwl.main(
 911             ["--create-workflow", "--debug",
 912              "--api=containers",
 913              "--project-uuid", project_uuid,
 914              "--name", "testing 123",
 915              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 916             capture_stdout, sys.stderr, api_client=stubs.api)
 917         self.assertEqual(exited, 0)
 918
 919         stubs.api.pipeline_templates().create.refute_called()
 920         stubs.api.container_requests().create.refute_called()
 921
 922         body = {
 923             "workflow": {
 924                 "owner_uuid": project_uuid,
 925                 "name": "testing 123",
 926                 "description": "",
 927                 "definition": self.expect_workflow,
 928             }
 929         }
 930         stubs.api.workflows().create.assert_called_with(
 931             body=JsonDiffMatcher(body))
 932
 933         self.assertEqual(capture_stdout.getvalue(),
 934                          stubs.expect_workflow_uuid + '\n')
 935
 936     @stubs
 937     def test_incompatible_api(self, stubs):
 938         capture_stderr = cStringIO.StringIO()
 939         logging.getLogger('arvados.cwl-runner').addHandler(
 940             logging.StreamHandler(capture_stderr))
 941
 942         exited = arvados_cwl.main(
 943             ["--update-workflow", self.existing_workflow_uuid,
 944              "--api=jobs",
 945              "--debug",
 946              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 947             sys.stderr, sys.stderr, api_client=stubs.api)
 948         self.assertEqual(exited, 1)
 949         self.assertRegexpMatches(
 950             capture_stderr.getvalue(),
 951             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
 952
 953     @stubs
 954     def test_update(self, stubs):
 955         capture_stdout = cStringIO.StringIO()
 956
 957         exited = arvados_cwl.main(
 958             ["--update-workflow", self.existing_workflow_uuid,
 959              "--debug",
 960              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 961             capture_stdout, sys.stderr, api_client=stubs.api)
 962         self.assertEqual(exited, 0)
 963
 964         body = {
 965             "workflow": {
 966                 "name": "submit_wf.cwl",
 967                 "description": "",
 968                 "definition": self.expect_workflow,
 969             }
 970         }
 971         stubs.api.workflows().update.assert_called_with(
 972             uuid=self.existing_workflow_uuid,
 973             body=JsonDiffMatcher(body))
 974         self.assertEqual(capture_stdout.getvalue(),
 975                          self.existing_workflow_uuid + '\n')
 976
 977
 978     @stubs
 979     def test_update_name(self, stubs):
 980         capture_stdout = cStringIO.StringIO()
 981
 982         exited = arvados_cwl.main(
 983             ["--update-workflow", self.existing_workflow_uuid,
 984              "--debug", "--name", "testing 123",
 985              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 986             capture_stdout, sys.stderr, api_client=stubs.api)
 987         self.assertEqual(exited, 0)
 988
 989         body = {
 990             "workflow": {
 991                 "name": "testing 123",
 992                 "description": "",
 993                 "definition": self.expect_workflow,
 994             }
 995         }
 996         stubs.api.workflows().update.assert_called_with(
 997             uuid=self.existing_workflow_uuid,
 998             body=JsonDiffMatcher(body))
 999         self.assertEqual(capture_stdout.getvalue(),
1000                          self.existing_workflow_uuid + '\n')
1001
1002
1003 class TestTemplateInputs(unittest.TestCase):
1004     expect_template = {
1005         "components": {
1006             "inputs_test.cwl": {
1007                 'runtime_constraints': {
1008                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1009                     'min_ram_mb_per_node': 1024
1010                 },
1011                 'script_parameters': {
1012                     'cwl:tool':
1013                     '99999999999999999999999999999991+99/'
1014                     'wf/inputs_test.cwl',
1015                     'optionalFloatInput': None,
1016                     'fileInput': {
1017                         'type': 'File',
1018                         'dataclass': 'File',
1019                         'required': True,
1020                         'title': "It's a file; we expect to find some characters in it.",
1021                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1022                     },
1023                     'floatInput': {
1024                         'type': 'float',
1025                         'dataclass': 'number',
1026                         'required': True,
1027                         'title': 'Floats like a duck',
1028                         'default': 0.1,
1029                         'value': 0.1,
1030                     },
1031                     'optionalFloatInput': {
1032                         'type': ['null', 'float'],
1033                         'dataclass': 'number',
1034                         'required': False,
1035                     },
1036                     'boolInput': {
1037                         'type': 'boolean',
1038                         'dataclass': 'boolean',
1039                         'required': True,
1040                         'title': 'True or false?',
1041                     },
1042                 },
1043                 'repository': 'arvados',
1044                 'script_version': 'master',
1045                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1046                 'script': 'cwl-runner',
1047             },
1048         },
1049         "name": "inputs_test.cwl",
1050     }
1051
1052     @stubs
1053     def test_inputs_empty(self, stubs):
1054         exited = arvados_cwl.main(
1055             ["--create-template",
1056              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1057             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1058         self.assertEqual(exited, 0)
1059
1060         stubs.api.pipeline_templates().create.assert_called_with(
1061             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1062
1063     @stubs
1064     def test_inputs(self, stubs):
1065         exited = arvados_cwl.main(
1066             ["--create-template",
1067              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1068             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1069         self.assertEqual(exited, 0)
1070
1071         expect_template = copy.deepcopy(self.expect_template)
1072         params = expect_template[
1073             "components"]["inputs_test.cwl"]["script_parameters"]
1074         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
1075         params["floatInput"]["value"] = 1.234
1076         params["boolInput"]["value"] = True
1077
1078         stubs.api.pipeline_templates().create.assert_called_with(
1079             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)