sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12 import json
  13
  14 from .matcher import JsonDiffMatcher
  15
  16
  17 def stubs(func):
  18     @functools.wraps(func)
  19     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  20     @mock.patch("arvados.collection.KeepClient")
  21     @mock.patch("arvados.events.subscribe")
  22     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
  23         class Stubs:
  24             pass
  25         stubs = Stubs()
  26         stubs.events = events
  27         stubs.KeepClient = KeepClient
  28         stubs.keepdocker = keepdocker
  29
  30         def putstub(p, **kwargs):
  31             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  32         stubs.KeepClient().put.side_effect = putstub
  33
  34         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  35         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  36
  37         stubs.api = mock.MagicMock()
  38         stubs.api.users().current().execute.return_value = {
  39             "uuid": stubs.fake_user_uuid,
  40         }
  41         stubs.api.collections().list().execute.return_value = {"items": []}
  42         stubs.api.collections().create().execute.side_effect = ({
  43             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  44             "portable_data_hash": "99999999999999999999999999999991+99",
  45             "manifest_text": ""
  46         }, {
  47             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  48             "portable_data_hash": "99999999999999999999999999999992+99",
  49             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  50         },
  51         {
  52             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  53             "portable_data_hash": "99999999999999999999999999999994+99",
  54             "manifest_text": ""
  55         },
  56         {
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  58             "portable_data_hash": "99999999999999999999999999999995+99",
  59             "manifest_text": ""
  60         }        )
  61         stubs.api.collections().get().execute.return_value = {
  62             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  63
  64         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  65         stubs.api.jobs().create().execute.return_value = {
  66             "uuid": stubs.expect_job_uuid,
  67             "state": "Queued",
  68         }
  69
  70         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  71         stubs.api.container_requests().create().execute.return_value = {
  72             "uuid": stubs.expect_container_request_uuid,
  73             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  74             "state": "Queued"
  75         }
  76
  77         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  78         stubs.api.pipeline_templates().create().execute.return_value = {
  79             "uuid": stubs.expect_pipeline_template_uuid,
  80         }
  81         stubs.expect_job_spec = {
  82             'runtime_constraints': {
  83                 'docker_image': 'arvados/jobs'
  84             },
  85             'script_parameters': {
  86                 'x': {
  87                     'basename': 'blorp.txt',
  88                     'location': '99999999999999999999999999999994+99/blorp.txt',
  89                     'class': 'File'
  90                 },
  91                 'cwl:tool':
  92                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
  93             },
  94             'repository': 'arvados',
  95             'script_version': 'master',
  96             'script': 'cwl-runner'
  97         }
  98
  99         stubs.expect_container_spec = {
 100             'priority': 1,
 101             'mounts': {
 102                 '/var/spool/cwl': {
 103                     'writable': True,
 104                     'kind': 'collection'
 105                 },
 106                 '/var/lib/cwl/workflow': {
 107                     'portable_data_hash': '99999999999999999999999999999991+99',
 108                     'kind': 'collection'
 109                 },
 110                 'stdout': {
 111                     'path': '/var/spool/cwl/cwl.output.json',
 112                     'kind': 'file'
 113                 },
 114                 '/var/lib/cwl/job/cwl.input.json': {
 115                     'portable_data_hash': '765fda0d9897729ff467a4609879c00a+60/cwl.input.json',
 116                     'kind': 'collection'
 117                 }
 118             },
 119             'state': 'Committed',
 120             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 121             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 122             'name': 'submit_wf.cwl',
 123             'container_image': '99999999999999999999999999999993+99',
 124             'output_path': '/var/spool/cwl',
 125             'cwd': '/var/spool/cwl',
 126             'runtime_constraints': {
 127                 'API': True,
 128                 'vcpus': 1,
 129                 'ram': 268435456
 130             }
 131         }
 132
 133         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 134         stubs.api.workflows().create().execute.return_value = {
 135             "uuid": stubs.expect_workflow_uuid,
 136         }
 137
 138         return func(self, stubs, *args, **kwargs)
 139     return wrapped
 140
 141
 142 class TestSubmit(unittest.TestCase):
 143     @stubs
 144     def test_submit(self, stubs):
 145         capture_stdout = cStringIO.StringIO()
 146         exited = arvados_cwl.main(
 147             ["--submit", "--no-wait",
 148              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 149             capture_stdout, sys.stderr, api_client=stubs.api)
 150         self.assertEqual(exited, 0)
 151
 152         stubs.api.collections().create.assert_has_calls([
 153             mock.call(),
 154             mock.call(body={
 155                 'manifest_text':
 156                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 157                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 158                 '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
 159                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 160                 'name': 'submit_wf.cwl',
 161             }, ensure_unique_name=True),
 162             mock.call().execute(),
 163             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 164                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 165                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 166                             'replication_desired': None,
 167                             'name': 'New collection'
 168             }, ensure_unique_name=True),
 169             mock.call().execute(num_retries=4),
 170             mock.call(body={
 171                 'manifest_text':
 172                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 173                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 174                 'name': '#',
 175             }, ensure_unique_name=True),
 176             mock.call().execute()])
 177
 178         expect_job = copy.deepcopy(stubs.expect_job_spec)
 179         expect_job["owner_uuid"] = stubs.fake_user_uuid
 180         stubs.api.jobs().create.assert_called_with(
 181             body=expect_job,
 182             find_or_create=True)
 183         self.assertEqual(capture_stdout.getvalue(),
 184                          stubs.expect_job_uuid + '\n')
 185
 186     @stubs
 187     def test_submit_with_project_uuid(self, stubs):
 188         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 189
 190         exited = arvados_cwl.main(
 191             ["--submit", "--no-wait",
 192              "--project-uuid", project_uuid,
 193              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 194             sys.stdout, sys.stderr, api_client=stubs.api)
 195         self.assertEqual(exited, 0)
 196
 197         expect_body = copy.deepcopy(stubs.expect_job_spec)
 198         expect_body["owner_uuid"] = project_uuid
 199         stubs.api.jobs().create.assert_called_with(
 200             body=expect_body,
 201             find_or_create=True)
 202
 203     @stubs
 204     def test_submit_container(self, stubs):
 205         capture_stdout = cStringIO.StringIO()
 206         exited = arvados_cwl.main(
 207             ["--submit", "--no-wait", "--api=containers", "--debug",
 208              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 209             capture_stdout, sys.stderr, api_client=stubs.api)
 210         self.assertEqual(exited, 0)
 211
 212         stubs.api.collections().create.assert_has_calls([
 213             mock.call(),
 214             mock.call(body={
 215                 'manifest_text':
 216                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 217                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 218                 '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
 219                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 220                 'name': 'submit_wf.cwl',
 221             }, ensure_unique_name=True),
 222             mock.call().execute(),
 223             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 224                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 225                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 226                             'name': 'New collection',
 227                             'replication_desired': None,
 228             }, ensure_unique_name=True),
 229             mock.call().execute(num_retries=4),
 230             mock.call(body={
 231                 'manifest_text':
 232                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 233                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 234                 'name': '#',
 235             }, ensure_unique_name=True),
 236             mock.call().execute()])
 237
 238         expect_container = copy.deepcopy(stubs.expect_container_spec)
 239         expect_container["owner_uuid"] = stubs.fake_user_uuid
 240         stubs.api.container_requests().create.assert_called_with(
 241             body=expect_container)
 242         self.assertEqual(capture_stdout.getvalue(),
 243                          stubs.expect_container_request_uuid + '\n')
 244
 245
 246 class TestCreateTemplate(unittest.TestCase):
 247     @stubs
 248     def test_create(self, stubs):
 249         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 250
 251         capture_stdout = cStringIO.StringIO()
 252
 253         exited = arvados_cwl.main(
 254             ["--create-template", "--debug",
 255              "--project-uuid", project_uuid,
 256              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 257             capture_stdout, sys.stderr, api_client=stubs.api)
 258         self.assertEqual(exited, 0)
 259
 260         stubs.api.pipeline_instances().create.refute_called()
 261         stubs.api.jobs().create.refute_called()
 262
 263         expect_component = copy.deepcopy(stubs.expect_job_spec)
 264         expect_component['script_parameters']['x'] = {
 265             'dataclass': 'File',
 266             'required': True,
 267             'type': 'File',
 268             'value': '99999999999999999999999999999994+99/blorp.txt',
 269         }
 270         expect_template = {
 271             "components": {
 272                 "submit_wf.cwl": expect_component,
 273             },
 274             "name": "submit_wf.cwl",
 275             "owner_uuid": project_uuid,
 276         }
 277         stubs.api.pipeline_templates().create.assert_called_with(
 278             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 279
 280         self.assertEqual(capture_stdout.getvalue(),
 281                          stubs.expect_pipeline_template_uuid + '\n')
 282
 283
 284 class TestCreateWorkflow(unittest.TestCase):
 285     @stubs
 286     def test_create(self, stubs):
 287         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 288
 289         capture_stdout = cStringIO.StringIO()
 290
 291         exited = arvados_cwl.main(
 292             ["--create-workflow", "--debug",
 293              "--project-uuid", project_uuid,
 294              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 295             capture_stdout, sys.stderr, api_client=stubs.api)
 296         self.assertEqual(exited, 0)
 297
 298         stubs.api.pipeline_templates().create.refute_called()
 299         stubs.api.container_requests().create.refute_called()
 300
 301         with open("tests/wf/expect_packed.cwl") as f:
 302             expect_workflow = f.read()
 303
 304         body = {
 305             "workflow": {
 306                 "owner_uuid": project_uuid,
 307                 "name": "submit_wf.cwl",
 308                 "description": "",
 309                 "workflow": expect_workflow
 310                 }
 311         }
 312         stubs.api.workflows().create.assert_called_with(
 313             body=JsonDiffMatcher(body))
 314
 315         self.assertEqual(capture_stdout.getvalue(),
 316                          stubs.expect_workflow_uuid + '\n')
 317
 318
 319 class TestTemplateInputs(unittest.TestCase):
 320     expect_template = {
 321         "components": {
 322             "inputs_test.cwl": {
 323                 'runtime_constraints': {
 324                     'docker_image': 'arvados/jobs',
 325                 },
 326                 'script_parameters': {
 327                     'cwl:tool':
 328                     '99999999999999999999999999999991+99/'
 329                     'wf/inputs_test.cwl',
 330                     'optionalFloatInput': None,
 331                     'fileInput': {
 332                         'type': 'File',
 333                         'dataclass': 'File',
 334                         'required': True,
 335                         'title': "It's a file; we expect to find some characters in it.",
 336                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 337                     },
 338                     'floatInput': {
 339                         'type': 'float',
 340                         'dataclass': 'number',
 341                         'required': True,
 342                         'title': 'Floats like a duck',
 343                         'default': 0.1,
 344                         'value': 0.1,
 345                     },
 346                     'optionalFloatInput': {
 347                         'type': ['null', 'float'],
 348                         'dataclass': 'number',
 349                         'required': False,
 350                     },
 351                     'boolInput': {
 352                         'type': 'boolean',
 353                         'dataclass': 'boolean',
 354                         'required': True,
 355                         'title': 'True or false?',
 356                     },
 357                 },
 358                 'repository': 'arvados',
 359                 'script_version': 'master',
 360                 'script': 'cwl-runner',
 361             },
 362         },
 363         "name": "inputs_test.cwl",
 364     }
 365
 366     @stubs
 367     def test_inputs_empty(self, stubs):
 368         exited = arvados_cwl.main(
 369             ["--create-template", "--no-wait",
 370              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 371             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 372         self.assertEqual(exited, 0)
 373
 374         expect_template = copy.deepcopy(self.expect_template)
 375         expect_template["owner_uuid"] = stubs.fake_user_uuid
 376
 377         stubs.api.pipeline_templates().create.assert_called_with(
 378             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 379
 380     @stubs
 381     def test_inputs(self, stubs):
 382         exited = arvados_cwl.main(
 383             ["--create-template", "--no-wait",
 384              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 385             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 386         self.assertEqual(exited, 0)
 387
 388         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 389
 390         expect_template = copy.deepcopy(self.expect_template)
 391         expect_template["owner_uuid"] = stubs.fake_user_uuid
 392         params = expect_template[
 393             "components"]["inputs_test.cwl"]["script_parameters"]
 394         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 395         params["floatInput"]["value"] = 1.234
 396         params["boolInput"]["value"] = True
 397
 398         stubs.api.pipeline_templates().create.assert_called_with(
 399             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)