sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12 import json
  13 import logging
  14
  15 from .matcher import JsonDiffMatcher
  16
  17
  18 def stubs(func):
  19     @functools.wraps(func)
  20     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  21     @mock.patch("arvados.collection.KeepClient")
  22     @mock.patch("arvados.events.subscribe")
  23     def wrapped(self, events, keep_client, keepdocker, *args, **kwargs):
  24         class Stubs:
  25             pass
  26         stubs = Stubs()
  27         stubs.events = events
  28         stubs.keepdocker = keepdocker
  29         stubs.keep_client = keep_client
  30
  31         def putstub(p, **kwargs):
  32             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  33         stubs.keep_client().put.side_effect = putstub
  34         stubs.keep_client.put.side_effect = putstub
  35
  36         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  37         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  38
  39
  40         stubs.api = mock.MagicMock()
  41         stubs.api.users().current().execute.return_value = {
  42             "uuid": stubs.fake_user_uuid,
  43         }
  44         stubs.api.collections().list().execute.return_value = {"items": []}
  45         stubs.api.collections().create().execute.side_effect = ({
  46             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  47             "portable_data_hash": "99999999999999999999999999999991+99",
  48             "manifest_text": ""
  49         }, {
  50             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  51             "portable_data_hash": "99999999999999999999999999999992+99",
  52             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  53         },
  54         {
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  56             "portable_data_hash": "99999999999999999999999999999994+99",
  57             "manifest_text": ""
  58         },
  59         {
  60             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  61             "portable_data_hash": "99999999999999999999999999999995+99",
  62             "manifest_text": ""
  63         },
  64         {
  65             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  66             "portable_data_hash": "99999999999999999999999999999996+99",
  67             "manifest_text": ""
  68         }
  69         )
  70         stubs.api.collections().get().execute.return_value = {
  71             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  72
  73         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  74         stubs.api.jobs().create().execute.return_value = {
  75             "uuid": stubs.expect_job_uuid,
  76             "state": "Queued",
  77         }
  78
  79         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  80         stubs.api.container_requests().create().execute.return_value = {
  81             "uuid": stubs.expect_container_request_uuid,
  82             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  83             "state": "Queued"
  84         }
  85
  86         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  87         stubs.api.pipeline_templates().create().execute.return_value = {
  88             "uuid": stubs.expect_pipeline_template_uuid,
  89         }
  90         stubs.expect_job_spec = {
  91             'runtime_constraints': {
  92                 'docker_image': 'arvados/jobs'
  93             },
  94             'script_parameters': {
  95                 'x': {
  96                     'basename': 'blorp.txt',
  97                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
  98                     'class': 'File'
  99                 },
 100                 'y': {
 101                     'basename': '99999999999999999999999999999998+99',
 102                     'location': 'keep:99999999999999999999999999999998+99',
 103                     'class': 'Directory'
 104                 },
 105                 'z': {
 106                     'basename': 'anonymous',
 107                     "listing": [{
 108                         "basename": "renamed.txt",
 109                         "class": "File",
 110                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 111                     }],
 112                     'class': 'Directory'
 113                 },
 114                 'cwl:tool':
 115                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 116             },
 117             'repository': 'arvados',
 118             'script_version': 'master',
 119             'script': 'cwl-runner'
 120         }
 121         stubs.pipeline_component = stubs.expect_job_spec.copy()
 122         stubs.expect_pipeline_instance = {
 123             'name': 'submit_wf.cwl',
 124             'state': 'RunningOnServer',
 125             "components": {
 126                 "cwl-runner": {
 127                     'runtime_constraints': {'docker_image': 'arvados/jobs'},
 128                     'script_parameters': {
 129                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 130                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 131                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 132                               'listing': [
 133                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 134                               ]}},
 135                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 136                     },
 137                     'repository': 'arvados',
 138                     'script_version': 'master',
 139                     'script': 'cwl-runner'
 140                 }
 141             }
 142         }
 143         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 144         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 145         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 146         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 147         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 148             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 149             "state": "Queued"
 150         }
 151         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 152         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 153
 154         stubs.expect_container_spec = {
 155             'priority': 1,
 156             'mounts': {
 157                 '/var/spool/cwl': {
 158                     'writable': True,
 159                     'kind': 'collection'
 160                 },
 161                 '/var/lib/cwl/workflow': {
 162                     'portable_data_hash': '99999999999999999999999999999991+99',
 163                     'kind': 'collection'
 164                 },
 165                 'stdout': {
 166                     'path': '/var/spool/cwl/cwl.output.json',
 167                     'kind': 'file'
 168                 },
 169                 '/var/lib/cwl/job/cwl.input.json': {
 170                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 171                     'kind': 'collection'
 172                 }
 173             },
 174             'state': 'Committed',
 175             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 176             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 177             'name': 'submit_wf.cwl',
 178             'container_image': '99999999999999999999999999999993+99',
 179             'output_path': '/var/spool/cwl',
 180             'cwd': '/var/spool/cwl',
 181             'runtime_constraints': {
 182                 'API': True,
 183                 'vcpus': 1,
 184                 'ram': 268435456
 185             }
 186         }
 187
 188         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 189         stubs.api.workflows().create().execute.return_value = {
 190             "uuid": stubs.expect_workflow_uuid,
 191         }
 192
 193         return func(self, stubs, *args, **kwargs)
 194     return wrapped
 195
 196
 197 class TestSubmit(unittest.TestCase):
 198     @mock.patch("time.sleep")
 199     @stubs
 200     def test_submit(self, stubs, tm):
 201         capture_stdout = cStringIO.StringIO()
 202         exited = arvados_cwl.main(
 203             ["--submit", "--no-wait", "--debug",
 204              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 205             capture_stdout, sys.stderr, api_client=stubs.api)
 206         self.assertEqual(exited, 0)
 207
 208         stubs.api.collections().create.assert_has_calls([
 209             mock.call(),
 210             mock.call(body={
 211                 'manifest_text':
 212                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 213                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 214                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 215                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 216                 'name': 'submit_wf.cwl',
 217             }, ensure_unique_name=True),
 218             mock.call().execute(),
 219             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 220                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 221                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 222                             'replication_desired': None,
 223                             'name': 'New collection'
 224             }, ensure_unique_name=True),
 225             mock.call().execute(num_retries=4),
 226             mock.call(body={
 227                 'manifest_text':
 228                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 229                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 230                 'name': '#',
 231             }, ensure_unique_name=True),
 232             mock.call().execute()])
 233
 234         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 235         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 236         stubs.api.pipeline_instances().create.assert_called_with(
 237             body=expect_pipeline)
 238         self.assertEqual(capture_stdout.getvalue(),
 239                          stubs.expect_pipeline_uuid + '\n')
 240
 241     @mock.patch("time.sleep")
 242     @stubs
 243     def test_submit_with_project_uuid(self, stubs, tm):
 244         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 245
 246         exited = arvados_cwl.main(
 247             ["--submit", "--no-wait",
 248              "--project-uuid", project_uuid,
 249              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 250             sys.stdout, sys.stderr, api_client=stubs.api)
 251         self.assertEqual(exited, 0)
 252
 253         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 254         expect_pipeline["owner_uuid"] = project_uuid
 255         stubs.api.pipeline_instances().create.assert_called_with(
 256             body=expect_pipeline)
 257
 258     @stubs
 259     def test_submit_container(self, stubs):
 260         capture_stdout = cStringIO.StringIO()
 261         try:
 262             exited = arvados_cwl.main(
 263                 ["--submit", "--no-wait", "--api=containers", "--debug",
 264                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 265                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 266             self.assertEqual(exited, 0)
 267         except:
 268             logging.exception("")
 269
 270         stubs.api.collections().create.assert_has_calls([
 271             mock.call(),
 272             mock.call(body={
 273                 'manifest_text':
 274                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 275                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 276                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 277                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 278                 'name': 'submit_wf.cwl',
 279             }, ensure_unique_name=True),
 280             mock.call().execute(),
 281             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 282                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 283                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 284                             'name': 'New collection',
 285                             'replication_desired': None,
 286             }, ensure_unique_name=True),
 287             mock.call().execute(num_retries=4),
 288             mock.call(body={
 289                 'manifest_text':
 290                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 291                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 292                 'name': '#',
 293             }, ensure_unique_name=True),
 294             mock.call().execute()])
 295
 296         expect_container = copy.deepcopy(stubs.expect_container_spec)
 297         expect_container["owner_uuid"] = stubs.fake_user_uuid
 298         stubs.api.container_requests().create.assert_called_with(
 299             body=expect_container)
 300         self.assertEqual(capture_stdout.getvalue(),
 301                          stubs.expect_container_request_uuid + '\n')
 302
 303
 304 class TestCreateTemplate(unittest.TestCase):
 305     @stubs
 306     def test_create(self, stubs):
 307         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 308
 309         capture_stdout = cStringIO.StringIO()
 310
 311         exited = arvados_cwl.main(
 312             ["--create-template", "--debug",
 313              "--project-uuid", project_uuid,
 314              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 315             capture_stdout, sys.stderr, api_client=stubs.api)
 316         self.assertEqual(exited, 0)
 317
 318         stubs.api.pipeline_instances().create.refute_called()
 319         stubs.api.jobs().create.refute_called()
 320
 321         expect_component = copy.deepcopy(stubs.expect_job_spec)
 322         expect_component['script_parameters']['x'] = {
 323             'dataclass': 'File',
 324             'required': True,
 325             'type': 'File',
 326             'value': '99999999999999999999999999999994+99/blorp.txt',
 327         }
 328         expect_component['script_parameters']['y'] = {
 329             'dataclass': 'Collection',
 330             'required': True,
 331             'type': 'Directory',
 332             'value': '99999999999999999999999999999998+99',
 333         }
 334         expect_component['script_parameters']['z'] = {
 335             'dataclass': 'Collection',
 336             'required': True,
 337             'type': 'Directory',
 338         }
 339         expect_template = {
 340             "components": {
 341                 "submit_wf.cwl": expect_component,
 342             },
 343             "name": "submit_wf.cwl",
 344             "owner_uuid": project_uuid,
 345         }
 346         stubs.api.pipeline_templates().create.assert_called_with(
 347             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 348
 349         self.assertEqual(capture_stdout.getvalue(),
 350                          stubs.expect_pipeline_template_uuid + '\n')
 351
 352
 353 class TestCreateWorkflow(unittest.TestCase):
 354     @stubs
 355     def test_create(self, stubs):
 356         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 357
 358         capture_stdout = cStringIO.StringIO()
 359
 360         exited = arvados_cwl.main(
 361             ["--create-workflow", "--debug",
 362              "--project-uuid", project_uuid,
 363              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 364             capture_stdout, sys.stderr, api_client=stubs.api)
 365         self.assertEqual(exited, 0)
 366
 367         stubs.api.pipeline_templates().create.refute_called()
 368         stubs.api.container_requests().create.refute_called()
 369
 370         with open("tests/wf/expect_packed.cwl") as f:
 371             expect_workflow = f.read()
 372
 373         body = {
 374             "workflow": {
 375                 "owner_uuid": project_uuid,
 376                 "name": "submit_wf.cwl",
 377                 "description": "",
 378                 "definition": expect_workflow
 379                 }
 380         }
 381         stubs.api.workflows().create.assert_called_with(
 382             body=JsonDiffMatcher(body))
 383
 384         self.assertEqual(capture_stdout.getvalue(),
 385                          stubs.expect_workflow_uuid + '\n')
 386
 387
 388 class TestTemplateInputs(unittest.TestCase):
 389     expect_template = {
 390         "components": {
 391             "inputs_test.cwl": {
 392                 'runtime_constraints': {
 393                     'docker_image': 'arvados/jobs',
 394                 },
 395                 'script_parameters': {
 396                     'cwl:tool':
 397                     '99999999999999999999999999999991+99/'
 398                     'wf/inputs_test.cwl',
 399                     'optionalFloatInput': None,
 400                     'fileInput': {
 401                         'type': 'File',
 402                         'dataclass': 'File',
 403                         'required': True,
 404                         'title': "It's a file; we expect to find some characters in it.",
 405                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 406                     },
 407                     'floatInput': {
 408                         'type': 'float',
 409                         'dataclass': 'number',
 410                         'required': True,
 411                         'title': 'Floats like a duck',
 412                         'default': 0.1,
 413                         'value': 0.1,
 414                     },
 415                     'optionalFloatInput': {
 416                         'type': ['null', 'float'],
 417                         'dataclass': 'number',
 418                         'required': False,
 419                     },
 420                     'boolInput': {
 421                         'type': 'boolean',
 422                         'dataclass': 'boolean',
 423                         'required': True,
 424                         'title': 'True or false?',
 425                     },
 426                 },
 427                 'repository': 'arvados',
 428                 'script_version': 'master',
 429                 'script': 'cwl-runner',
 430             },
 431         },
 432         "name": "inputs_test.cwl",
 433     }
 434
 435     @stubs
 436     def test_inputs_empty(self, stubs):
 437         exited = arvados_cwl.main(
 438             ["--create-template", "--no-wait",
 439              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 440             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 441         self.assertEqual(exited, 0)
 442
 443         expect_template = copy.deepcopy(self.expect_template)
 444         expect_template["owner_uuid"] = stubs.fake_user_uuid
 445
 446         stubs.api.pipeline_templates().create.assert_called_with(
 447             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 448
 449     @stubs
 450     def test_inputs(self, stubs):
 451         exited = arvados_cwl.main(
 452             ["--create-template", "--no-wait",
 453              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 454             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 455         self.assertEqual(exited, 0)
 456
 457         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 458
 459         expect_template = copy.deepcopy(self.expect_template)
 460         expect_template["owner_uuid"] = stubs.fake_user_uuid
 461         params = expect_template[
 462             "components"]["inputs_test.cwl"]["script_parameters"]
 463         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 464         params["floatInput"]["value"] = 1.234
 465         params["boolInput"]["value"] = True
 466
 467         stubs.api.pipeline_templates().create.assert_called_with(
 468             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)