sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados_cwl.runner
  15 import arvados.keep
  16
  17 from .matcher import JsonDiffMatcher
  18 from .mock_discovery import get_rootDesc
  19
  20 _rootDesc = None
  21
  22 def stubs(func):
  23     @functools.wraps(func)
  24     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  25     @mock.patch("arvados.collection.KeepClient")
  26     @mock.patch("arvados.keep.KeepClient")
  27     @mock.patch("arvados.events.subscribe")
  28     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  29         class Stubs:
  30             pass
  31         stubs = Stubs()
  32         stubs.events = events
  33         stubs.keepdocker = keepdocker
  34
  35
  36         def putstub(p, **kwargs):
  37             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  38         keep_client1().put.side_effect = putstub
  39         keep_client1.put.side_effect = putstub
  40         keep_client2().put.side_effect = putstub
  41         keep_client2.put.side_effect = putstub
  42
  43         stubs.keep_client = keep_client2
  44         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  45         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  46
  47         stubs.api = mock.MagicMock()
  48         stubs.api._rootDesc = get_rootDesc()
  49
  50         stubs.api.users().current().execute.return_value = {
  51             "uuid": stubs.fake_user_uuid,
  52         }
  53         stubs.api.collections().list().execute.return_value = {"items": []}
  54         stubs.api.collections().create().execute.side_effect = ({
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  56             "portable_data_hash": "99999999999999999999999999999991+99",
  57             "manifest_text": ""
  58         }, {
  59             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  60             "portable_data_hash": "99999999999999999999999999999992+99",
  61             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  62         },
  63         {
  64             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  65             "portable_data_hash": "99999999999999999999999999999994+99",
  66             "manifest_text": ""
  67         },
  68         {
  69             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  70             "portable_data_hash": "99999999999999999999999999999995+99",
  71             "manifest_text": ""
  72         },
  73         {
  74             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  75             "portable_data_hash": "99999999999999999999999999999996+99",
  76             "manifest_text": ""
  77         }
  78         )
  79         stubs.api.collections().get().execute.return_value = {
  80             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  81
  82         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         stubs.api.jobs().create().execute.return_value = {
  84             "uuid": stubs.expect_job_uuid,
  85             "state": "Queued",
  86         }
  87
  88         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  89         stubs.api.container_requests().create().execute.return_value = {
  90             "uuid": stubs.expect_container_request_uuid,
  91             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  92             "state": "Queued"
  93         }
  94
  95         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  96         stubs.api.pipeline_templates().create().execute.return_value = {
  97             "uuid": stubs.expect_pipeline_template_uuid,
  98         }
  99         stubs.expect_job_spec = {
 100             'runtime_constraints': {
 101                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 102                 'min_ram_mb_per_node': 1024
 103             },
 104             'script_parameters': {
 105                 'x': {
 106                     'basename': 'blorp.txt',
 107                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 108                     'class': 'File'
 109                 },
 110                 'y': {
 111                     'basename': '99999999999999999999999999999998+99',
 112                     'location': 'keep:99999999999999999999999999999998+99',
 113                     'class': 'Directory'
 114                 },
 115                 'z': {
 116                     'basename': 'anonymous',
 117                     "listing": [{
 118                         "basename": "renamed.txt",
 119                         "class": "File",
 120                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 121                     }],
 122                     'class': 'Directory'
 123                 },
 124                 'cwl:tool':
 125                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 126             },
 127             'repository': 'arvados',
 128             'script_version': 'master',
 129             'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 130             'script': 'cwl-runner'
 131         }
 132         stubs.pipeline_component = stubs.expect_job_spec.copy()
 133         stubs.expect_pipeline_instance = {
 134             'name': 'submit_wf.cwl',
 135             'state': 'RunningOnServer',
 136             'owner_uuid': None,
 137             "components": {
 138                 "cwl-runner": {
 139                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__, 'min_ram_mb_per_node': 1024},
 140                     'script_parameters': {
 141                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 142                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 143                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 144                               'listing': [
 145                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 146                               ]}},
 147                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
 148                         'arv:enable_reuse': True
 149                     },
 150                     'repository': 'arvados',
 151                     'script_version': 'master',
 152                     'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
 153                     'script': 'cwl-runner',
 154                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 155                 }
 156             }
 157         }
 158         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 159         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 160         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 161         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 162         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 163             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 164             "state": "Queued"
 165         }
 166         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 167         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 168
 169         stubs.expect_container_spec = {
 170             'priority': 1,
 171             'mounts': {
 172                 '/var/spool/cwl': {
 173                     'writable': True,
 174                     'kind': 'collection'
 175                 },
 176                 '/var/lib/cwl/workflow': {
 177                     'portable_data_hash': '99999999999999999999999999999991+99',
 178                     'kind': 'collection'
 179                 },
 180                 'stdout': {
 181                     'path': '/var/spool/cwl/cwl.output.json',
 182                     'kind': 'file'
 183                 },
 184                 '/var/lib/cwl/cwl.input.json': {
 185                     'kind': 'json',
 186                     'content': {
 187                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 188                         'x': {'basename': u'blorp.txt', 'class': 'File', 'location': u'keep:99999999999999999999999999999994+99/blorp.txt'},
 189                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
 190                             {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 191                         ]}
 192                     },
 193                     'kind': 'json'
 194                 }
 195             },
 196             'state': 'Committed',
 197             'owner_uuid': None,
 198             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json'],
 199             'name': 'submit_wf.cwl',
 200             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 201             'output_path': '/var/spool/cwl',
 202             'cwd': '/var/spool/cwl',
 203             'runtime_constraints': {
 204                 'API': True,
 205                 'vcpus': 1,
 206                 'ram': 1024*1024*1024
 207             },
 208             "properties": {}
 209         }
 210
 211         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 212         stubs.api.workflows().create().execute.return_value = {
 213             "uuid": stubs.expect_workflow_uuid,
 214         }
 215         def update_mock(**kwargs):
 216             stubs.updated_uuid = kwargs.get('uuid')
 217             return mock.DEFAULT
 218         stubs.api.workflows().update.side_effect = update_mock
 219         stubs.api.workflows().update().execute.side_effect = lambda **kwargs: {
 220             "uuid": stubs.updated_uuid,
 221         }
 222
 223         return func(self, stubs, *args, **kwargs)
 224     return wrapped
 225
 226
 227 class TestSubmit(unittest.TestCase):
 228     @mock.patch("time.sleep")
 229     @stubs
 230     def test_submit(self, stubs, tm):
 231         capture_stdout = cStringIO.StringIO()
 232         exited = arvados_cwl.main(
 233             ["--submit", "--no-wait", "--debug",
 234              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 235             capture_stdout, sys.stderr, api_client=stubs.api)
 236         self.assertEqual(exited, 0)
 237
 238         stubs.api.collections().create.assert_has_calls([
 239             mock.call(),
 240             mock.call(body={
 241                 'manifest_text':
 242                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 243                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 244                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 245                 'owner_uuid': None,
 246                 'name': 'submit_wf.cwl',
 247             }, ensure_unique_name=True),
 248             mock.call().execute(),
 249             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 250                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 251                             'replication_desired': None,
 252                             'name': 'New collection'
 253             }, ensure_unique_name=True),
 254             mock.call().execute(num_retries=4),
 255             mock.call(body={
 256                 'manifest_text':
 257                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 258                 'owner_uuid': None,
 259                 'name': '#',
 260             }, ensure_unique_name=True),
 261             mock.call().execute()])
 262
 263         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 264         stubs.api.pipeline_instances().create.assert_called_with(
 265             body=expect_pipeline)
 266         self.assertEqual(capture_stdout.getvalue(),
 267                          stubs.expect_pipeline_uuid + '\n')
 268
 269
 270     @mock.patch("time.sleep")
 271     @stubs
 272     def test_submit_no_reuse(self, stubs, tm):
 273         capture_stdout = cStringIO.StringIO()
 274         exited = arvados_cwl.main(
 275             ["--submit", "--no-wait", "--debug", "--disable-reuse",
 276              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 277             capture_stdout, sys.stderr, api_client=stubs.api)
 278         self.assertEqual(exited, 0)
 279
 280         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 281
 282         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 283         stubs.api.pipeline_instances().create.assert_called_with(
 284             body=JsonDiffMatcher(expect_pipeline))
 285         self.assertEqual(capture_stdout.getvalue(),
 286                          stubs.expect_pipeline_uuid + '\n')
 287
 288
 289     @mock.patch("time.sleep")
 290     @stubs
 291     def test_submit_runner_ram(self, stubs, tm):
 292         capture_stdout = cStringIO.StringIO()
 293         exited = arvados_cwl.main(
 294             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
 295              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 296             capture_stdout, sys.stderr, api_client=stubs.api)
 297         self.assertEqual(exited, 0)
 298
 299         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 300
 301         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 302         stubs.api.pipeline_instances().create.assert_called_with(
 303             body=JsonDiffMatcher(expect_pipeline))
 304         self.assertEqual(capture_stdout.getvalue(),
 305                          stubs.expect_pipeline_uuid + '\n')
 306
 307
 308     @mock.patch("time.sleep")
 309     @stubs
 310     def test_submit_invalid_runner_ram(self, stubs, tm):
 311         capture_stdout = cStringIO.StringIO()
 312         exited = arvados_cwl.main(
 313             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
 314              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 315             capture_stdout, sys.stderr, api_client=stubs.api)
 316         self.assertEqual(exited, 1)
 317
 318     @mock.patch("time.sleep")
 319     @stubs
 320     def test_submit_output_name(self, stubs, tm):
 321         output_name = "test_output_name"
 322
 323         capture_stdout = cStringIO.StringIO()
 324         exited = arvados_cwl.main(
 325             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
 326              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 327             capture_stdout, sys.stderr, api_client=stubs.api)
 328         self.assertEqual(exited, 0)
 329
 330         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 331
 332         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 333         stubs.api.pipeline_instances().create.assert_called_with(
 334             body=JsonDiffMatcher(expect_pipeline))
 335         self.assertEqual(capture_stdout.getvalue(),
 336                          stubs.expect_pipeline_uuid + '\n')
 337
 338
 339     @mock.patch("time.sleep")
 340     @stubs
 341     def test_submit_pipeline_name(self, stubs, tm):
 342         capture_stdout = cStringIO.StringIO()
 343         exited = arvados_cwl.main(
 344             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
 345              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 346             capture_stdout, sys.stderr, api_client=stubs.api)
 347         self.assertEqual(exited, 0)
 348
 349         stubs.expect_pipeline_instance["name"] = "hello job 123"
 350
 351         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 352         stubs.api.pipeline_instances().create.assert_called_with(
 353             body=expect_pipeline)
 354         self.assertEqual(capture_stdout.getvalue(),
 355                          stubs.expect_pipeline_uuid + '\n')
 356
 357     @mock.patch("time.sleep")
 358     @stubs
 359     def test_submit_output_tags(self, stubs, tm):
 360         output_tags = "tag0,tag1,tag2"
 361
 362         capture_stdout = cStringIO.StringIO()
 363         exited = arvados_cwl.main(
 364             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
 365              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 366             capture_stdout, sys.stderr, api_client=stubs.api)
 367         self.assertEqual(exited, 0)
 368
 369         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:output_tags"] = output_tags
 370
 371         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 372         stubs.api.pipeline_instances().create.assert_called_with(
 373             body=JsonDiffMatcher(expect_pipeline))
 374         self.assertEqual(capture_stdout.getvalue(),
 375                          stubs.expect_pipeline_uuid + '\n')
 376
 377     @mock.patch("time.sleep")
 378     @stubs
 379     def test_submit_with_project_uuid(self, stubs, tm):
 380         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 381
 382         exited = arvados_cwl.main(
 383             ["--submit", "--no-wait",
 384              "--project-uuid", project_uuid,
 385              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 386             sys.stdout, sys.stderr, api_client=stubs.api)
 387         self.assertEqual(exited, 0)
 388
 389         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 390         expect_pipeline["owner_uuid"] = project_uuid
 391         stubs.api.pipeline_instances().create.assert_called_with(
 392             body=JsonDiffMatcher(expect_pipeline))
 393
 394     @stubs
 395     def test_submit_container(self, stubs):
 396         capture_stdout = cStringIO.StringIO()
 397         try:
 398             exited = arvados_cwl.main(
 399                 ["--submit", "--no-wait", "--api=containers", "--debug",
 400                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 401                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 402             self.assertEqual(exited, 0)
 403         except:
 404             logging.exception("")
 405
 406         stubs.api.collections().create.assert_has_calls([
 407             mock.call(),
 408             mock.call(body={
 409                 'manifest_text':
 410                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 411                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 412                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 413                 'owner_uuid': None,
 414                 'name': 'submit_wf.cwl',
 415             }, ensure_unique_name=True),
 416             mock.call().execute(),
 417             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 418                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 419                             'name': 'New collection',
 420                             'replication_desired': None,
 421             }, ensure_unique_name=True),
 422             mock.call().execute(num_retries=4),
 423             mock.call(body={
 424                 'manifest_text':
 425                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 426                 'owner_uuid': None,
 427                 'name': '#',
 428             }, ensure_unique_name=True),
 429             mock.call().execute()])
 430
 431         expect_container = copy.deepcopy(stubs.expect_container_spec)
 432         stubs.api.container_requests().create.assert_called_with(
 433             body=JsonDiffMatcher(expect_container))
 434         self.assertEqual(capture_stdout.getvalue(),
 435                          stubs.expect_container_request_uuid + '\n')
 436
 437     @stubs
 438     def test_submit_container_no_reuse(self, stubs):
 439         capture_stdout = cStringIO.StringIO()
 440         try:
 441             exited = arvados_cwl.main(
 442                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 443                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 444                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 445             self.assertEqual(exited, 0)
 446         except:
 447             logging.exception("")
 448
 449         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 450                                                   '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
 451
 452         expect_container = copy.deepcopy(stubs.expect_container_spec)
 453         stubs.api.container_requests().create.assert_called_with(
 454             body=JsonDiffMatcher(expect_container))
 455         self.assertEqual(capture_stdout.getvalue(),
 456                          stubs.expect_container_request_uuid + '\n')
 457
 458     @stubs
 459     def test_submit_container_output_name(self, stubs):
 460         output_name = "test_output_name"
 461
 462         capture_stdout = cStringIO.StringIO()
 463         try:
 464             exited = arvados_cwl.main(
 465                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
 466                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 467                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 468             self.assertEqual(exited, 0)
 469         except:
 470             logging.exception("")
 471
 472         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 473                                                   "--output-name="+output_name, '--enable-reuse',
 474                                                   '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
 475
 476         expect_container = copy.deepcopy(stubs.expect_container_spec)
 477         stubs.api.container_requests().create.assert_called_with(
 478             body=JsonDiffMatcher(expect_container))
 479         self.assertEqual(capture_stdout.getvalue(),
 480                          stubs.expect_container_request_uuid + '\n')
 481
 482     @stubs
 483     def test_submit_container_output_tags(self, stubs):
 484         output_tags = "tag0,tag1,tag2"
 485
 486         capture_stdout = cStringIO.StringIO()
 487         try:
 488             exited = arvados_cwl.main(
 489                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
 490                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 491                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 492             self.assertEqual(exited, 0)
 493         except:
 494             logging.exception("")
 495
 496         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 497                                                   "--output-tags="+output_tags, '--enable-reuse',
 498                                                   '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
 499
 500         expect_container = copy.deepcopy(stubs.expect_container_spec)
 501         stubs.api.container_requests().create.assert_called_with(
 502             body=JsonDiffMatcher(expect_container))
 503         self.assertEqual(capture_stdout.getvalue(),
 504                          stubs.expect_container_request_uuid + '\n')
 505
 506     @stubs
 507     def test_submit_container_runner_ram(self, stubs):
 508         capture_stdout = cStringIO.StringIO()
 509         try:
 510             exited = arvados_cwl.main(
 511                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
 512                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 513                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 514             self.assertEqual(exited, 0)
 515         except:
 516             logging.exception("")
 517
 518         stubs.expect_container_spec["runtime_constraints"]["ram"] = 2048*1024*1024
 519
 520         expect_container = copy.deepcopy(stubs.expect_container_spec)
 521         stubs.api.container_requests().create.assert_called_with(
 522             body=JsonDiffMatcher(expect_container))
 523         self.assertEqual(capture_stdout.getvalue(),
 524                          stubs.expect_container_request_uuid + '\n')
 525
 526     @mock.patch("arvados.collection.CollectionReader")
 527     @mock.patch("time.sleep")
 528     @stubs
 529     def test_submit_file_keepref(self, stubs, tm, collectionReader):
 530         capture_stdout = cStringIO.StringIO()
 531         exited = arvados_cwl.main(
 532             ["--submit", "--no-wait", "--api=containers", "--debug",
 533              "tests/wf/submit_keepref_wf.cwl"],
 534             capture_stdout, sys.stderr, api_client=stubs.api)
 535         self.assertEqual(exited, 0)
 536
 537
 538     @mock.patch("arvados.collection.CollectionReader")
 539     @mock.patch("time.sleep")
 540     @stubs
 541     def test_submit_keepref(self, stubs, tm, reader):
 542         capture_stdout = cStringIO.StringIO()
 543
 544         with open("tests/wf/expect_arvworkflow.cwl") as f:
 545             reader().open().__enter__().read.return_value = f.read()
 546
 547         exited = arvados_cwl.main(
 548             ["--submit", "--no-wait", "--api=containers", "--debug",
 549              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
 550             capture_stdout, sys.stderr, api_client=stubs.api)
 551         self.assertEqual(exited, 0)
 552
 553         expect_container = {
 554             'priority': 1,
 555             'mounts': {
 556                 '/var/spool/cwl': {
 557                     'writable': True,
 558                     'kind': 'collection'
 559                 },
 560                 'stdout': {
 561                     'path': '/var/spool/cwl/cwl.output.json',
 562                     'kind': 'file'
 563                 },
 564                 '/var/lib/cwl/workflow': {
 565                     'portable_data_hash': '99999999999999999999999999999994+99',
 566                     'kind': 'collection'
 567                 },
 568                 '/var/lib/cwl/cwl.input.json': {
 569                     'content': {
 570                         'x': 'XxX'
 571                     },
 572                     'kind': 'json'
 573                 }
 574             }, 'state': 'Committed',
 575             'owner_uuid': None,
 576             'output_path': '/var/spool/cwl',
 577             'name': 'expect_arvworkflow.cwl#main',
 578             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 579             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 580                         '--enable-reuse', '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
 581             'cwd': '/var/spool/cwl',
 582             'runtime_constraints': {
 583                 'API': True,
 584                 'vcpus': 1,
 585                 'ram': 1073741824
 586             },
 587             "properties": {}
 588         }
 589
 590         stubs.api.container_requests().create.assert_called_with(
 591             body=JsonDiffMatcher(expect_container))
 592         self.assertEqual(capture_stdout.getvalue(),
 593                          stubs.expect_container_request_uuid + '\n')
 594
 595     @mock.patch("time.sleep")
 596     @stubs
 597     def test_submit_arvworkflow(self, stubs, tm):
 598         capture_stdout = cStringIO.StringIO()
 599
 600         with open("tests/wf/expect_arvworkflow.cwl") as f:
 601             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 602
 603         exited = arvados_cwl.main(
 604             ["--submit", "--no-wait", "--api=containers", "--debug",
 605              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
 606             capture_stdout, sys.stderr, api_client=stubs.api)
 607         self.assertEqual(exited, 0)
 608
 609         expect_container = {
 610             'priority': 1,
 611             'mounts': {
 612                 '/var/spool/cwl': {
 613                     'writable': True,
 614                     'kind': 'collection'
 615                 },
 616                 'stdout': {
 617                     'path': '/var/spool/cwl/cwl.output.json',
 618                     'kind': 'file'
 619                 },
 620                 '/var/lib/cwl/workflow.json': {
 621                     'kind': 'json',
 622                     'json': {
 623                         'cwlVersion': 'v1.0',
 624                         '$graph': [
 625                             {
 626                                 'inputs': [
 627                                     {
 628                                         'inputBinding': {'position': 1},
 629                                         'type': 'string',
 630                                         'id': '#submit_tool.cwl/x'}
 631                                 ],
 632                                 'requirements': [
 633                                     {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
 634                                 ],
 635                                 'id': '#submit_tool.cwl',
 636                                 'outputs': [],
 637                                 'baseCommand': 'cat',
 638                                 'class': 'CommandLineTool'
 639                             }, {
 640                                 'id': '#main',
 641                                 'inputs': [
 642                                     {'type': 'string', 'id': '#main/x'}
 643                                 ],
 644                                 'steps': [
 645                                     {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
 646                                      'run': '#submit_tool.cwl',
 647                                      'id': '#main/step1',
 648                                      'out': []}
 649                                 ],
 650                                 'class': 'Workflow',
 651                                 'outputs': []
 652                             }
 653                         ]
 654                     }
 655                 },
 656                 '/var/lib/cwl/cwl.input.json': {
 657                     'content': {
 658                         'x': 'XxX'
 659                     },
 660                     'kind': 'json'
 661                 }
 662             }, 'state': 'Committed',
 663             'owner_uuid': None,
 664             'output_path': '/var/spool/cwl',
 665             'name': 'a test workflow',
 666             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 667             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
 668                         '--enable-reuse', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
 669             'cwd': '/var/spool/cwl',
 670             'runtime_constraints': {
 671                 'API': True,
 672                 'vcpus': 1,
 673                 'ram': 1073741824
 674             },
 675             "properties": {
 676                 "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
 677             }
 678         }
 679
 680         stubs.api.container_requests().create.assert_called_with(
 681             body=JsonDiffMatcher(expect_container))
 682         self.assertEqual(capture_stdout.getvalue(),
 683                          stubs.expect_container_request_uuid + '\n')
 684
 685
 686     @stubs
 687     def test_submit_container_name(self, stubs):
 688         capture_stdout = cStringIO.StringIO()
 689         try:
 690             exited = arvados_cwl.main(
 691                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
 692                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 693                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 694             self.assertEqual(exited, 0)
 695         except:
 696             logging.exception("")
 697
 698         stubs.expect_container_spec["name"] = "hello container 123"
 699
 700         expect_container = copy.deepcopy(stubs.expect_container_spec)
 701         stubs.api.container_requests().create.assert_called_with(
 702             body=expect_container)
 703         self.assertEqual(capture_stdout.getvalue(),
 704                          stubs.expect_container_request_uuid + '\n')
 705
 706
 707     @mock.patch("arvados.commands.keepdocker.find_one_image_hash")
 708     @mock.patch("cwltool.docker.get_image")
 709     @mock.patch("arvados.api")
 710     def test_arvados_jobs_image(self, api, get_image, find_one_image_hash):
 711         arvrunner = mock.MagicMock()
 712         arvrunner.project_uuid = ""
 713         api.return_value = mock.MagicMock()
 714         arvrunner.api = api.return_value
 715         arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 716                                                             {"items": [], "items_available": 0, "offset": 0},
 717                                                             {"items": [], "items_available": 0, "offset": 0},
 718                                                             {"items": [{"created_at": "",
 719                                                                         "head_uuid": "",
 720                                                                         "link_class": "docker_image_hash",
 721                                                                         "name": "123456",
 722                                                                         "owner_uuid": "",
 723                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 724                                                             {"items": [], "items_available": 0, "offset": 0},
 725                                                             {"items": [{"created_at": "",
 726                                                                         "head_uuid": "",
 727                                                                         "link_class": "docker_image_repo+tag",
 728                                                                         "name": "arvados/jobs:"+arvados_cwl.__version__,
 729                                                                         "owner_uuid": "",
 730                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
 731                                                             {"items": [{"created_at": "",
 732                                                                         "head_uuid": "",
 733                                                                         "link_class": "docker_image_hash",
 734                                                                         "name": "123456",
 735                                                                         "owner_uuid": "",
 736                                                                         "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}                                                            ,
 737         )
 738         find_one_image_hash.return_value = "123456"
 739
 740         arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
 741                                                                   {"items": [{"uuid": "",
 742                                                                               "owner_uuid": "",
 743                                                                               "manifest_text": "",
 744                                                                               "properties": ""
 745                                                                           }], "items_available": 1, "offset": 0},
 746                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
 747         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
 748         self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
 749
 750 class TestCreateTemplate(unittest.TestCase):
 751     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
 752
 753     def _adjust_script_params(self, expect_component):
 754         expect_component['script_parameters']['x'] = {
 755             'dataclass': 'File',
 756             'required': True,
 757             'type': 'File',
 758             'value': '99999999999999999999999999999994+99/blorp.txt',
 759         }
 760         expect_component['script_parameters']['y'] = {
 761             'dataclass': 'Collection',
 762             'required': True,
 763             'type': 'Directory',
 764             'value': '99999999999999999999999999999998+99',
 765         }
 766         expect_component['script_parameters']['z'] = {
 767             'dataclass': 'Collection',
 768             'required': True,
 769             'type': 'Directory',
 770         }
 771
 772     @stubs
 773     def test_create(self, stubs):
 774         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 775
 776         capture_stdout = cStringIO.StringIO()
 777
 778         exited = arvados_cwl.main(
 779             ["--create-workflow", "--debug",
 780              "--api=jobs",
 781              "--project-uuid", project_uuid,
 782              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 783             capture_stdout, sys.stderr, api_client=stubs.api)
 784         self.assertEqual(exited, 0)
 785
 786         stubs.api.pipeline_instances().create.refute_called()
 787         stubs.api.jobs().create.refute_called()
 788
 789         expect_component = copy.deepcopy(stubs.expect_job_spec)
 790         self._adjust_script_params(expect_component)
 791         expect_template = {
 792             "components": {
 793                 "submit_wf.cwl": expect_component,
 794             },
 795             "name": "submit_wf.cwl",
 796             "owner_uuid": project_uuid,
 797         }
 798         stubs.api.pipeline_templates().create.assert_called_with(
 799             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 800
 801         self.assertEqual(capture_stdout.getvalue(),
 802                          stubs.expect_pipeline_template_uuid + '\n')
 803
 804
 805     @stubs
 806     def test_create_name(self, stubs):
 807         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 808
 809         capture_stdout = cStringIO.StringIO()
 810
 811         exited = arvados_cwl.main(
 812             ["--create-workflow", "--debug",
 813              "--project-uuid", project_uuid,
 814              "--api=jobs",
 815              "--name", "testing 123",
 816              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 817             capture_stdout, sys.stderr, api_client=stubs.api)
 818         self.assertEqual(exited, 0)
 819
 820         stubs.api.pipeline_instances().create.refute_called()
 821         stubs.api.jobs().create.refute_called()
 822
 823         expect_component = copy.deepcopy(stubs.expect_job_spec)
 824         self._adjust_script_params(expect_component)
 825         expect_template = {
 826             "components": {
 827                 "testing 123": expect_component,
 828             },
 829             "name": "testing 123",
 830             "owner_uuid": project_uuid,
 831         }
 832         stubs.api.pipeline_templates().create.assert_called_with(
 833             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 834
 835         self.assertEqual(capture_stdout.getvalue(),
 836                          stubs.expect_pipeline_template_uuid + '\n')
 837
 838
 839     @stubs
 840     def test_update_name(self, stubs):
 841         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 842
 843         capture_stdout = cStringIO.StringIO()
 844
 845         exited = arvados_cwl.main(
 846             ["--update-workflow", self.existing_template_uuid,
 847              "--debug",
 848              "--project-uuid", project_uuid,
 849              "--api=jobs",
 850              "--name", "testing 123",
 851              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 852             capture_stdout, sys.stderr, api_client=stubs.api)
 853         self.assertEqual(exited, 0)
 854
 855         stubs.api.pipeline_instances().create.refute_called()
 856         stubs.api.jobs().create.refute_called()
 857
 858         expect_component = copy.deepcopy(stubs.expect_job_spec)
 859         self._adjust_script_params(expect_component)
 860         expect_template = {
 861             "components": {
 862                 "testing 123": expect_component,
 863             },
 864             "name": "testing 123",
 865             "owner_uuid": project_uuid,
 866         }
 867         stubs.api.pipeline_templates().create.refute_called()
 868         stubs.api.pipeline_templates().update.assert_called_with(
 869             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 870
 871         self.assertEqual(capture_stdout.getvalue(),
 872                          self.existing_template_uuid + '\n')
 873
 874
 875 class TestCreateWorkflow(unittest.TestCase):
 876     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
 877     expect_workflow = open("tests/wf/expect_packed.cwl").read()
 878
 879     @stubs
 880     def test_create(self, stubs):
 881         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 882
 883         capture_stdout = cStringIO.StringIO()
 884
 885         exited = arvados_cwl.main(
 886             ["--create-workflow", "--debug",
 887              "--api=containers",
 888              "--project-uuid", project_uuid,
 889              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 890             capture_stdout, sys.stderr, api_client=stubs.api)
 891         self.assertEqual(exited, 0)
 892
 893         stubs.api.pipeline_templates().create.refute_called()
 894         stubs.api.container_requests().create.refute_called()
 895
 896         body = {
 897             "workflow": {
 898                 "owner_uuid": project_uuid,
 899                 "name": "submit_wf.cwl",
 900                 "description": "",
 901                 "definition": self.expect_workflow,
 902             }
 903         }
 904         stubs.api.workflows().create.assert_called_with(
 905             body=JsonDiffMatcher(body))
 906
 907         self.assertEqual(capture_stdout.getvalue(),
 908                          stubs.expect_workflow_uuid + '\n')
 909
 910
 911     @stubs
 912     def test_create_name(self, stubs):
 913         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 914
 915         capture_stdout = cStringIO.StringIO()
 916
 917         exited = arvados_cwl.main(
 918             ["--create-workflow", "--debug",
 919              "--api=containers",
 920              "--project-uuid", project_uuid,
 921              "--name", "testing 123",
 922              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 923             capture_stdout, sys.stderr, api_client=stubs.api)
 924         self.assertEqual(exited, 0)
 925
 926         stubs.api.pipeline_templates().create.refute_called()
 927         stubs.api.container_requests().create.refute_called()
 928
 929         body = {
 930             "workflow": {
 931                 "owner_uuid": project_uuid,
 932                 "name": "testing 123",
 933                 "description": "",
 934                 "definition": self.expect_workflow,
 935             }
 936         }
 937         stubs.api.workflows().create.assert_called_with(
 938             body=JsonDiffMatcher(body))
 939
 940         self.assertEqual(capture_stdout.getvalue(),
 941                          stubs.expect_workflow_uuid + '\n')
 942
 943     @stubs
 944     def test_incompatible_api(self, stubs):
 945         capture_stderr = cStringIO.StringIO()
 946         logging.getLogger('arvados.cwl-runner').addHandler(
 947             logging.StreamHandler(capture_stderr))
 948
 949         exited = arvados_cwl.main(
 950             ["--update-workflow", self.existing_workflow_uuid,
 951              "--api=jobs",
 952              "--debug",
 953              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 954             sys.stderr, sys.stderr, api_client=stubs.api)
 955         self.assertEqual(exited, 1)
 956         self.assertRegexpMatches(
 957             capture_stderr.getvalue(),
 958             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
 959
 960     @stubs
 961     def test_update(self, stubs):
 962         capture_stdout = cStringIO.StringIO()
 963
 964         exited = arvados_cwl.main(
 965             ["--update-workflow", self.existing_workflow_uuid,
 966              "--debug",
 967              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 968             capture_stdout, sys.stderr, api_client=stubs.api)
 969         self.assertEqual(exited, 0)
 970
 971         body = {
 972             "workflow": {
 973                 "name": "submit_wf.cwl",
 974                 "description": "",
 975                 "definition": self.expect_workflow,
 976             }
 977         }
 978         stubs.api.workflows().update.assert_called_with(
 979             uuid=self.existing_workflow_uuid,
 980             body=JsonDiffMatcher(body))
 981         self.assertEqual(capture_stdout.getvalue(),
 982                          self.existing_workflow_uuid + '\n')
 983
 984
 985     @stubs
 986     def test_update_name(self, stubs):
 987         capture_stdout = cStringIO.StringIO()
 988
 989         exited = arvados_cwl.main(
 990             ["--update-workflow", self.existing_workflow_uuid,
 991              "--debug", "--name", "testing 123",
 992              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 993             capture_stdout, sys.stderr, api_client=stubs.api)
 994         self.assertEqual(exited, 0)
 995
 996         body = {
 997             "workflow": {
 998                 "name": "testing 123",
 999                 "description": "",
1000                 "definition": self.expect_workflow,
1001             }
1002         }
1003         stubs.api.workflows().update.assert_called_with(
1004             uuid=self.existing_workflow_uuid,
1005             body=JsonDiffMatcher(body))
1006         self.assertEqual(capture_stdout.getvalue(),
1007                          self.existing_workflow_uuid + '\n')
1008
1009
1010 class TestTemplateInputs(unittest.TestCase):
1011     expect_template = {
1012         "components": {
1013             "inputs_test.cwl": {
1014                 'runtime_constraints': {
1015                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
1016                     'min_ram_mb_per_node': 1024
1017                 },
1018                 'script_parameters': {
1019                     'cwl:tool':
1020                     '99999999999999999999999999999991+99/'
1021                     'wf/inputs_test.cwl',
1022                     'optionalFloatInput': None,
1023                     'fileInput': {
1024                         'type': 'File',
1025                         'dataclass': 'File',
1026                         'required': True,
1027                         'title': "It's a file; we expect to find some characters in it.",
1028                         'description': 'If there were anything further to say, it would be said here,\nor here.'
1029                     },
1030                     'floatInput': {
1031                         'type': 'float',
1032                         'dataclass': 'number',
1033                         'required': True,
1034                         'title': 'Floats like a duck',
1035                         'default': 0.1,
1036                         'value': 0.1,
1037                     },
1038                     'optionalFloatInput': {
1039                         'type': ['null', 'float'],
1040                         'dataclass': 'number',
1041                         'required': False,
1042                     },
1043                     'boolInput': {
1044                         'type': 'boolean',
1045                         'dataclass': 'boolean',
1046                         'required': True,
1047                         'title': 'True or false?',
1048                     },
1049                 },
1050                 'repository': 'arvados',
1051                 'script_version': 'master',
1052                 'minimum_script_version': '570509ab4d2ef93d870fd2b1f2eab178afb1bad9',
1053                 'script': 'cwl-runner',
1054             },
1055         },
1056         "name": "inputs_test.cwl",
1057     }
1058
1059     @stubs
1060     def test_inputs_empty(self, stubs):
1061         exited = arvados_cwl.main(
1062             ["--create-template",
1063              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
1064             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1065         self.assertEqual(exited, 0)
1066
1067         stubs.api.pipeline_templates().create.assert_called_with(
1068             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
1069
1070     @stubs
1071     def test_inputs(self, stubs):
1072         exited = arvados_cwl.main(
1073             ["--create-template",
1074              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
1075             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
1076         self.assertEqual(exited, 0)
1077
1078         expect_template = copy.deepcopy(self.expect_template)
1079         params = expect_template[
1080             "components"]["inputs_test.cwl"]["script_parameters"]
1081         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
1082         params["floatInput"]["value"] = 1.234
1083         params["boolInput"]["value"] = True
1084
1085         stubs.api.pipeline_templates().create.assert_called_with(
1086             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)