sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12 import json
  13 import logging
  14
  15 from .matcher import JsonDiffMatcher
  16
  17
  18 def stubs(func):
  19     @functools.wraps(func)
  20     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  21     @mock.patch("arvados.collection.KeepClient")
  22     @mock.patch("arvados.events.subscribe")
  23     def wrapped(self, events, keep_client, keepdocker, *args, **kwargs):
  24         class Stubs:
  25             pass
  26         stubs = Stubs()
  27         stubs.events = events
  28         stubs.keepdocker = keepdocker
  29         stubs.keep_client = keep_client
  30
  31         def putstub(p, **kwargs):
  32             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  33         stubs.keep_client().put.side_effect = putstub
  34         stubs.keep_client.put.side_effect = putstub
  35
  36         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  37         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  38
  39
  40         stubs.api = mock.MagicMock()
  41         stubs.api.users().current().execute.return_value = {
  42             "uuid": stubs.fake_user_uuid,
  43         }
  44         stubs.api.collections().list().execute.return_value = {"items": []}
  45         stubs.api.collections().create().execute.side_effect = ({
  46             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  47             "portable_data_hash": "99999999999999999999999999999991+99",
  48             "manifest_text": ""
  49         }, {
  50             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  51             "portable_data_hash": "99999999999999999999999999999992+99",
  52             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  53         },
  54         {
  55             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  56             "portable_data_hash": "99999999999999999999999999999994+99",
  57             "manifest_text": ""
  58         },
  59         {
  60             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  61             "portable_data_hash": "99999999999999999999999999999995+99",
  62             "manifest_text": ""
  63         }        )
  64         stubs.api.collections().get().execute.return_value = {
  65             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  66
  67         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  68         stubs.api.jobs().create().execute.return_value = {
  69             "uuid": stubs.expect_job_uuid,
  70             "state": "Queued",
  71         }
  72
  73         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  74         stubs.api.container_requests().create().execute.return_value = {
  75             "uuid": stubs.expect_container_request_uuid,
  76             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  77             "state": "Queued"
  78         }
  79
  80         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  81         stubs.api.pipeline_templates().create().execute.return_value = {
  82             "uuid": stubs.expect_pipeline_template_uuid,
  83         }
  84         stubs.expect_job_spec = {
  85             'runtime_constraints': {
  86                 'docker_image': 'arvados/jobs'
  87             },
  88             'script_parameters': {
  89                 'x': {
  90                     'basename': 'blorp.txt',
  91                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
  92                     'class': 'File'
  93                 },
  94                 'y': {
  95                     'basename': '99999999999999999999999999999998+99',
  96                     'location': 'keep:99999999999999999999999999999998+99',
  97                     'class': 'Directory'
  98                 },
  99                 'z': {
 100                     'basename': 'anonymous',
 101                     "listing": [{
 102                         "basename": "renamed.txt",
 103                         "class": "File",
 104                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 105                     }],
 106                     'class': 'Directory'
 107                 },
 108                 'cwl:tool':
 109                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 110             },
 111             'repository': 'arvados',
 112             'script_version': 'master',
 113             'script': 'cwl-runner'
 114         }
 115         stubs.pipeline_component = stubs.expect_job_spec.copy()
 116         stubs.expect_pipeline_instance = {
 117             'name': 'submit_wf.cwl',
 118             'state': 'RunningOnServer',
 119             "components": {
 120                 "cwl-runner": {
 121                     'runtime_constraints': {'docker_image': 'arvados/jobs'},
 122                     'script_parameters': {
 123                         'y': {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'},
 124                         'x': {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'},
 125                         'z': {'basename': 'anonymous', 'class': 'Directory',
 126                               'listing': [
 127                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 128                               ]},
 129                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 130                     },
 131                     'repository': 'arvados',
 132                     'script_version': 'master',
 133                     'script': 'cwl-runner'
 134                 }
 135             }
 136         }
 137         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 138         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 139         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 140         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 141         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 142             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 143             "state": "Queued"
 144         }
 145         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 146         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 147
 148         stubs.expect_container_spec = {
 149             'priority': 1,
 150             'mounts': {
 151                 '/var/spool/cwl': {
 152                     'writable': True,
 153                     'kind': 'collection'
 154                 },
 155                 '/var/lib/cwl/workflow': {
 156                     'portable_data_hash': '99999999999999999999999999999991+99',
 157                     'kind': 'collection'
 158                 },
 159                 'stdout': {
 160                     'path': '/var/spool/cwl/cwl.output.json',
 161                     'kind': 'file'
 162                 },
 163                 '/var/lib/cwl/job/cwl.input.json': {
 164                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 165                     'kind': 'collection'
 166                 }
 167             },
 168             'state': 'Committed',
 169             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 170             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 171             'name': 'submit_wf.cwl',
 172             'container_image': '99999999999999999999999999999993+99',
 173             'output_path': '/var/spool/cwl',
 174             'cwd': '/var/spool/cwl',
 175             'runtime_constraints': {
 176                 'API': True,
 177                 'vcpus': 1,
 178                 'ram': 268435456
 179             }
 180         }
 181
 182         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 183         stubs.api.workflows().create().execute.return_value = {
 184             "uuid": stubs.expect_workflow_uuid,
 185         }
 186
 187         return func(self, stubs, *args, **kwargs)
 188     return wrapped
 189
 190
 191 class TestSubmit(unittest.TestCase):
 192     @mock.patch("time.sleep")
 193     @stubs
 194     def test_submit(self, stubs, tm):
 195         capture_stdout = cStringIO.StringIO()
 196         exited = arvados_cwl.main(
 197             ["--submit", "--no-wait", "--debug",
 198              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 199             capture_stdout, sys.stderr, api_client=stubs.api)
 200         self.assertEqual(exited, 0)
 201
 202         stubs.api.collections().create.assert_has_calls([
 203             mock.call(),
 204             mock.call(body={
 205                 'manifest_text':
 206                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 207                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 208                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 209                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 210                 'name': 'submit_wf.cwl',
 211             }, ensure_unique_name=True),
 212             mock.call().execute(),
 213             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 214                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 215                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 216                             'replication_desired': None,
 217                             'name': 'New collection'
 218             }, ensure_unique_name=True),
 219             mock.call().execute(num_retries=4),
 220             mock.call(body={
 221                 'manifest_text':
 222                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 223                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 224                 'name': '#',
 225             }, ensure_unique_name=True),
 226             mock.call().execute()])
 227
 228         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 229         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 230         stubs.api.pipeline_instances().create.assert_called_with(
 231             body=expect_pipeline)
 232         self.assertEqual(capture_stdout.getvalue(),
 233                          stubs.expect_pipeline_uuid + '\n')
 234
 235     @mock.patch("time.sleep")
 236     @stubs
 237     def test_submit_with_project_uuid(self, stubs, tm):
 238         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 239
 240         exited = arvados_cwl.main(
 241             ["--submit", "--no-wait",
 242              "--project-uuid", project_uuid,
 243              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 244             sys.stdout, sys.stderr, api_client=stubs.api)
 245         self.assertEqual(exited, 0)
 246
 247         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 248         expect_pipeline["owner_uuid"] = project_uuid
 249         stubs.api.pipeline_instances().create.assert_called_with(
 250             body=expect_pipeline)
 251
 252     @stubs
 253     def test_submit_container(self, stubs):
 254         capture_stdout = cStringIO.StringIO()
 255         try:
 256             exited = arvados_cwl.main(
 257                 ["--submit", "--no-wait", "--api=containers", "--debug",
 258                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 259                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 260             self.assertEqual(exited, 0)
 261         except:
 262             logging.exception("")
 263
 264         stubs.api.collections().create.assert_has_calls([
 265             mock.call(),
 266             mock.call(body={
 267                 'manifest_text':
 268                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 269                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 270                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 271                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 272                 'name': 'submit_wf.cwl',
 273             }, ensure_unique_name=True),
 274             mock.call().execute(),
 275             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 276                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 277                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 278                             'name': 'New collection',
 279                             'replication_desired': None,
 280             }, ensure_unique_name=True),
 281             mock.call().execute(num_retries=4),
 282             mock.call(body={
 283                 'manifest_text':
 284                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 285                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 286                 'name': '#',
 287             }, ensure_unique_name=True),
 288             mock.call().execute()])
 289
 290         expect_container = copy.deepcopy(stubs.expect_container_spec)
 291         expect_container["owner_uuid"] = stubs.fake_user_uuid
 292         stubs.api.container_requests().create.assert_called_with(
 293             body=expect_container)
 294         self.assertEqual(capture_stdout.getvalue(),
 295                          stubs.expect_container_request_uuid + '\n')
 296
 297
 298 class TestCreateTemplate(unittest.TestCase):
 299     @stubs
 300     def test_create(self, stubs):
 301         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 302
 303         capture_stdout = cStringIO.StringIO()
 304
 305         exited = arvados_cwl.main(
 306             ["--create-template", "--debug",
 307              "--project-uuid", project_uuid,
 308              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 309             capture_stdout, sys.stderr, api_client=stubs.api)
 310         self.assertEqual(exited, 0)
 311
 312         stubs.api.pipeline_instances().create.refute_called()
 313         stubs.api.jobs().create.refute_called()
 314
 315         expect_component = copy.deepcopy(stubs.expect_job_spec)
 316         expect_component['script_parameters']['x'] = {
 317             'dataclass': 'File',
 318             'required': True,
 319             'type': 'File',
 320             'value': '99999999999999999999999999999994+99/blorp.txt',
 321         }
 322         expect_component['script_parameters']['y'] = {
 323             'dataclass': 'Collection',
 324             'required': True,
 325             'type': 'Directory',
 326             'value': '99999999999999999999999999999998+99',
 327         }
 328         expect_component['script_parameters']['z'] = {
 329             'dataclass': 'Collection',
 330             'required': True,
 331             'type': 'Directory',
 332         }
 333         expect_template = {
 334             "components": {
 335                 "submit_wf.cwl": expect_component,
 336             },
 337             "name": "submit_wf.cwl",
 338             "owner_uuid": project_uuid,
 339         }
 340         stubs.api.pipeline_templates().create.assert_called_with(
 341             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 342
 343         self.assertEqual(capture_stdout.getvalue(),
 344                          stubs.expect_pipeline_template_uuid + '\n')
 345
 346
 347 class TestCreateWorkflow(unittest.TestCase):
 348     @stubs
 349     def test_create(self, stubs):
 350         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 351
 352         capture_stdout = cStringIO.StringIO()
 353
 354         exited = arvados_cwl.main(
 355             ["--create-workflow", "--debug",
 356              "--project-uuid", project_uuid,
 357              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 358             capture_stdout, sys.stderr, api_client=stubs.api)
 359         self.assertEqual(exited, 0)
 360
 361         stubs.api.pipeline_templates().create.refute_called()
 362         stubs.api.container_requests().create.refute_called()
 363
 364         with open("tests/wf/expect_packed.cwl") as f:
 365             expect_workflow = f.read()
 366
 367         body = {
 368             "workflow": {
 369                 "owner_uuid": project_uuid,
 370                 "name": "submit_wf.cwl",
 371                 "description": "",
 372                 "definition": expect_workflow
 373                 }
 374         }
 375         stubs.api.workflows().create.assert_called_with(
 376             body=JsonDiffMatcher(body))
 377
 378         self.assertEqual(capture_stdout.getvalue(),
 379                          stubs.expect_workflow_uuid + '\n')
 380
 381
 382 class TestTemplateInputs(unittest.TestCase):
 383     expect_template = {
 384         "components": {
 385             "inputs_test.cwl": {
 386                 'runtime_constraints': {
 387                     'docker_image': 'arvados/jobs',
 388                 },
 389                 'script_parameters': {
 390                     'cwl:tool':
 391                     '99999999999999999999999999999991+99/'
 392                     'wf/inputs_test.cwl',
 393                     'optionalFloatInput': None,
 394                     'fileInput': {
 395                         'type': 'File',
 396                         'dataclass': 'File',
 397                         'required': True,
 398                         'title': "It's a file; we expect to find some characters in it.",
 399                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 400                     },
 401                     'floatInput': {
 402                         'type': 'float',
 403                         'dataclass': 'number',
 404                         'required': True,
 405                         'title': 'Floats like a duck',
 406                         'default': 0.1,
 407                         'value': 0.1,
 408                     },
 409                     'optionalFloatInput': {
 410                         'type': ['null', 'float'],
 411                         'dataclass': 'number',
 412                         'required': False,
 413                     },
 414                     'boolInput': {
 415                         'type': 'boolean',
 416                         'dataclass': 'boolean',
 417                         'required': True,
 418                         'title': 'True or false?',
 419                     },
 420                 },
 421                 'repository': 'arvados',
 422                 'script_version': 'master',
 423                 'script': 'cwl-runner',
 424             },
 425         },
 426         "name": "inputs_test.cwl",
 427     }
 428
 429     @stubs
 430     def test_inputs_empty(self, stubs):
 431         exited = arvados_cwl.main(
 432             ["--create-template", "--no-wait",
 433              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 434             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 435         self.assertEqual(exited, 0)
 436
 437         expect_template = copy.deepcopy(self.expect_template)
 438         expect_template["owner_uuid"] = stubs.fake_user_uuid
 439
 440         stubs.api.pipeline_templates().create.assert_called_with(
 441             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 442
 443     @stubs
 444     def test_inputs(self, stubs):
 445         exited = arvados_cwl.main(
 446             ["--create-template", "--no-wait",
 447              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 448             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 449         self.assertEqual(exited, 0)
 450
 451         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 452
 453         expect_template = copy.deepcopy(self.expect_template)
 454         expect_template["owner_uuid"] = stubs.fake_user_uuid
 455         params = expect_template[
 456             "components"]["inputs_test.cwl"]["script_parameters"]
 457         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 458         params["floatInput"]["value"] = 1.234
 459         params["boolInput"]["value"] = True
 460
 461         stubs.api.pipeline_templates().create.assert_called_with(
 462             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)