sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12 import json
  13
  14 from .matcher import JsonDiffMatcher
  15
  16
  17 def stubs(func):
  18     @functools.wraps(func)
  19     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  20     @mock.patch("arvados.collection.KeepClient")
  21     @mock.patch("arvados.events.subscribe")
  22     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
  23         class Stubs:
  24             pass
  25         stubs = Stubs()
  26         stubs.events = events
  27         stubs.KeepClient = KeepClient
  28         stubs.keepdocker = keepdocker
  29
  30         def putstub(p, **kwargs):
  31             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  32         stubs.KeepClient().put.side_effect = putstub
  33
  34         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  35         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  36
  37         stubs.api = mock.MagicMock()
  38         stubs.api.users().current().execute.return_value = {
  39             "uuid": stubs.fake_user_uuid,
  40         }
  41         stubs.api.collections().list().execute.return_value = {"items": []}
  42         stubs.api.collections().create().execute.side_effect = ({
  43             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  44             "portable_data_hash": "99999999999999999999999999999991+99",
  45             "manifest_text": ""
  46         }, {
  47             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  48             "portable_data_hash": "99999999999999999999999999999992+99",
  49             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  50         },
  51         {
  52             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  53             "portable_data_hash": "99999999999999999999999999999994+99",
  54             "manifest_text": ""
  55         },
  56         {
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  58             "portable_data_hash": "99999999999999999999999999999995+99",
  59             "manifest_text": ""
  60         }        )
  61         stubs.api.collections().get().execute.return_value = {
  62             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  63
  64         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  65         stubs.api.jobs().create().execute.return_value = {
  66             "uuid": stubs.expect_job_uuid,
  67             "state": "Queued",
  68         }
  69
  70         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  71         stubs.api.container_requests().create().execute.return_value = {
  72             "uuid": stubs.expect_container_request_uuid,
  73             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  74             "state": "Queued"
  75         }
  76
  77         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  78         stubs.api.pipeline_templates().create().execute.return_value = {
  79             "uuid": stubs.expect_pipeline_template_uuid,
  80         }
  81         stubs.expect_job_spec = {
  82             'runtime_constraints': {
  83                 'docker_image': 'arvados/jobs'
  84             },
  85             'script_parameters': {
  86                 'x': {
  87                     'basename': 'blorp.txt',
  88                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
  89                     'class': 'File'
  90                 },
  91                 'y': {
  92                     'basename': '99999999999999999999999999999998+99',
  93                     'location': 'keep:99999999999999999999999999999998+99',
  94                     'class': 'Directory'
  95                 },
  96                 'cwl:tool':
  97                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
  98             },
  99             'repository': 'arvados',
 100             'script_version': 'master',
 101             'script': 'cwl-runner'
 102         }
 103
 104         stubs.expect_container_spec = {
 105             'priority': 1,
 106             'mounts': {
 107                 '/var/spool/cwl': {
 108                     'writable': True,
 109                     'kind': 'collection'
 110                 },
 111                 '/var/lib/cwl/workflow': {
 112                     'portable_data_hash': '99999999999999999999999999999991+99',
 113                     'kind': 'collection'
 114                 },
 115                 'stdout': {
 116                     'path': '/var/spool/cwl/cwl.output.json',
 117                     'kind': 'file'
 118                 },
 119                 '/var/lib/cwl/job/cwl.input.json': {
 120                     'portable_data_hash': '606be75b6e4f811a2f282d7fac867043+60/cwl.input.json',
 121                     'kind': 'collection'
 122                 }
 123             },
 124             'state': 'Committed',
 125             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 126             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 127             'name': 'submit_wf.cwl',
 128             'container_image': '99999999999999999999999999999993+99',
 129             'output_path': '/var/spool/cwl',
 130             'cwd': '/var/spool/cwl',
 131             'runtime_constraints': {
 132                 'API': True,
 133                 'vcpus': 1,
 134                 'ram': 268435456
 135             }
 136         }
 137
 138         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 139         stubs.api.workflows().create().execute.return_value = {
 140             "uuid": stubs.expect_workflow_uuid,
 141         }
 142
 143         return func(self, stubs, *args, **kwargs)
 144     return wrapped
 145
 146
 147 class TestSubmit(unittest.TestCase):
 148     @stubs
 149     def test_submit(self, stubs):
 150         capture_stdout = cStringIO.StringIO()
 151         exited = arvados_cwl.main(
 152             ["--submit", "--no-wait",
 153              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 154             capture_stdout, sys.stderr, api_client=stubs.api)
 155         self.assertEqual(exited, 0)
 156
 157         stubs.api.collections().create.assert_has_calls([
 158             mock.call(),
 159             mock.call(body={
 160                 'manifest_text':
 161                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 162                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 163                 '0f8864f292e901019c43fdabacd62c3e+383 0:383:submit_wf.cwl\n',
 164                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 165                 'name': 'submit_wf.cwl',
 166             }, ensure_unique_name=True),
 167             mock.call().execute(),
 168             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 169                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 170                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 171                             'replication_desired': None,
 172                             'name': 'New collection'
 173             }, ensure_unique_name=True),
 174             mock.call().execute(num_retries=4),
 175             mock.call(body={
 176                 'manifest_text':
 177                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 178                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 179                 'name': '#',
 180             }, ensure_unique_name=True),
 181             mock.call().execute()])
 182
 183         expect_job = copy.deepcopy(stubs.expect_job_spec)
 184         expect_job["owner_uuid"] = stubs.fake_user_uuid
 185         stubs.api.jobs().create.assert_called_with(
 186             body=expect_job,
 187             find_or_create=True)
 188         self.assertEqual(capture_stdout.getvalue(),
 189                          stubs.expect_job_uuid + '\n')
 190
 191     @stubs
 192     def test_submit_with_project_uuid(self, stubs):
 193         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 194
 195         exited = arvados_cwl.main(
 196             ["--submit", "--no-wait",
 197              "--project-uuid", project_uuid,
 198              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 199             sys.stdout, sys.stderr, api_client=stubs.api)
 200         self.assertEqual(exited, 0)
 201
 202         expect_body = copy.deepcopy(stubs.expect_job_spec)
 203         expect_body["owner_uuid"] = project_uuid
 204         stubs.api.jobs().create.assert_called_with(
 205             body=expect_body,
 206             find_or_create=True)
 207
 208     @stubs
 209     def test_submit_container(self, stubs):
 210         capture_stdout = cStringIO.StringIO()
 211         exited = arvados_cwl.main(
 212             ["--submit", "--no-wait", "--api=containers", "--debug",
 213              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 214             capture_stdout, sys.stderr, api_client=stubs.api)
 215         self.assertEqual(exited, 0)
 216
 217         stubs.api.collections().create.assert_has_calls([
 218             mock.call(),
 219             mock.call(body={
 220                 'manifest_text':
 221                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 222                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 223                 '0f8864f292e901019c43fdabacd62c3e+383 0:383:submit_wf.cwl\n',
 224                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 225                 'name': 'submit_wf.cwl',
 226             }, ensure_unique_name=True),
 227             mock.call().execute(),
 228             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 229                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 230                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 231                             'name': 'New collection',
 232                             'replication_desired': None,
 233             }, ensure_unique_name=True),
 234             mock.call().execute(num_retries=4),
 235             mock.call(body={
 236                 'manifest_text':
 237                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 238                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 239                 'name': '#',
 240             }, ensure_unique_name=True),
 241             mock.call().execute()])
 242
 243         expect_container = copy.deepcopy(stubs.expect_container_spec)
 244         expect_container["owner_uuid"] = stubs.fake_user_uuid
 245         stubs.api.container_requests().create.assert_called_with(
 246             body=expect_container)
 247         self.assertEqual(capture_stdout.getvalue(),
 248                          stubs.expect_container_request_uuid + '\n')
 249
 250
 251 class TestCreateTemplate(unittest.TestCase):
 252     @stubs
 253     def test_create(self, stubs):
 254         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 255
 256         capture_stdout = cStringIO.StringIO()
 257
 258         exited = arvados_cwl.main(
 259             ["--create-template", "--debug",
 260              "--project-uuid", project_uuid,
 261              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 262             capture_stdout, sys.stderr, api_client=stubs.api)
 263         self.assertEqual(exited, 0)
 264
 265         stubs.api.pipeline_instances().create.refute_called()
 266         stubs.api.jobs().create.refute_called()
 267
 268         expect_component = copy.deepcopy(stubs.expect_job_spec)
 269         expect_component['script_parameters']['x'] = {
 270             'dataclass': 'File',
 271             'required': True,
 272             'type': 'File',
 273             'value': '99999999999999999999999999999994+99/blorp.txt',
 274         }
 275         expect_component['script_parameters']['y'] = {
 276             'dataclass': 'Collection',
 277             'required': True,
 278             'type': 'Directory',
 279             'value': '99999999999999999999999999999998+99',
 280         }
 281         expect_template = {
 282             "components": {
 283                 "submit_wf.cwl": expect_component,
 284             },
 285             "name": "submit_wf.cwl",
 286             "owner_uuid": project_uuid,
 287         }
 288         stubs.api.pipeline_templates().create.assert_called_with(
 289             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 290
 291         self.assertEqual(capture_stdout.getvalue(),
 292                          stubs.expect_pipeline_template_uuid + '\n')
 293
 294
 295 class TestCreateWorkflow(unittest.TestCase):
 296     @stubs
 297     def test_create(self, stubs):
 298         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 299
 300         capture_stdout = cStringIO.StringIO()
 301
 302         exited = arvados_cwl.main(
 303             ["--create-workflow", "--debug",
 304              "--project-uuid", project_uuid,
 305              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 306             capture_stdout, sys.stderr, api_client=stubs.api)
 307         self.assertEqual(exited, 0)
 308
 309         stubs.api.pipeline_templates().create.refute_called()
 310         stubs.api.container_requests().create.refute_called()
 311
 312         with open("tests/wf/expect_packed.cwl") as f:
 313             expect_workflow = f.read()
 314
 315         body = {
 316             "workflow": {
 317                 "owner_uuid": project_uuid,
 318                 "name": "submit_wf.cwl",
 319                 "description": "",
 320                 "definition": expect_workflow
 321                 }
 322         }
 323         stubs.api.workflows().create.assert_called_with(
 324             body=JsonDiffMatcher(body))
 325
 326         self.assertEqual(capture_stdout.getvalue(),
 327                          stubs.expect_workflow_uuid + '\n')
 328
 329
 330 class TestTemplateInputs(unittest.TestCase):
 331     expect_template = {
 332         "components": {
 333             "inputs_test.cwl": {
 334                 'runtime_constraints': {
 335                     'docker_image': 'arvados/jobs',
 336                 },
 337                 'script_parameters': {
 338                     'cwl:tool':
 339                     '99999999999999999999999999999991+99/'
 340                     'wf/inputs_test.cwl',
 341                     'optionalFloatInput': None,
 342                     'fileInput': {
 343                         'type': 'File',
 344                         'dataclass': 'File',
 345                         'required': True,
 346                         'title': "It's a file; we expect to find some characters in it.",
 347                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 348                     },
 349                     'floatInput': {
 350                         'type': 'float',
 351                         'dataclass': 'number',
 352                         'required': True,
 353                         'title': 'Floats like a duck',
 354                         'default': 0.1,
 355                         'value': 0.1,
 356                     },
 357                     'optionalFloatInput': {
 358                         'type': ['null', 'float'],
 359                         'dataclass': 'number',
 360                         'required': False,
 361                     },
 362                     'boolInput': {
 363                         'type': 'boolean',
 364                         'dataclass': 'boolean',
 365                         'required': True,
 366                         'title': 'True or false?',
 367                     },
 368                 },
 369                 'repository': 'arvados',
 370                 'script_version': 'master',
 371                 'script': 'cwl-runner',
 372             },
 373         },
 374         "name": "inputs_test.cwl",
 375     }
 376
 377     @stubs
 378     def test_inputs_empty(self, stubs):
 379         exited = arvados_cwl.main(
 380             ["--create-template", "--no-wait",
 381              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 382             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 383         self.assertEqual(exited, 0)
 384
 385         expect_template = copy.deepcopy(self.expect_template)
 386         expect_template["owner_uuid"] = stubs.fake_user_uuid
 387
 388         stubs.api.pipeline_templates().create.assert_called_with(
 389             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 390
 391     @stubs
 392     def test_inputs(self, stubs):
 393         exited = arvados_cwl.main(
 394             ["--create-template", "--no-wait",
 395              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 396             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 397         self.assertEqual(exited, 0)
 398
 399         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 400
 401         expect_template = copy.deepcopy(self.expect_template)
 402         expect_template["owner_uuid"] = stubs.fake_user_uuid
 403         params = expect_template[
 404             "components"]["inputs_test.cwl"]["script_parameters"]
 405         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 406         params["floatInput"]["value"] = 1.234
 407         params["boolInput"]["value"] = True
 408
 409         stubs.api.pipeline_templates().create.assert_called_with(
 410             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)