sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12
  13 from .matcher import JsonDiffMatcher
  14
  15
  16 def stubs(func):
  17     @functools.wraps(func)
  18     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  19     @mock.patch("arvados.collection.KeepClient")
  20     @mock.patch("arvados.events.subscribe")
  21     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
  22         class Stubs:
  23             pass
  24         stubs = Stubs()
  25         stubs.events = events
  26         stubs.KeepClient = KeepClient
  27         stubs.keepdocker = keepdocker
  28
  29         def putstub(p, **kwargs):
  30             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  31         stubs.KeepClient().put.side_effect = putstub
  32
  33         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  34         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  35
  36         stubs.api = mock.MagicMock()
  37         stubs.api.users().current().execute.return_value = {
  38             "uuid": stubs.fake_user_uuid,
  39         }
  40         stubs.api.collections().list().execute.return_value = {"items": []}
  41         stubs.api.collections().create().execute.side_effect = ({
  42             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  43             "portable_data_hash": "99999999999999999999999999999991+99",
  44             "manifest_text": ""
  45         }, {
  46             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  47             "portable_data_hash": "99999999999999999999999999999992+99",
  48             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  49         },
  50         {
  51             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  52             "portable_data_hash": "99999999999999999999999999999994+99",
  53             "manifest_text": ""
  54         },
  55         {
  56             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  57             "portable_data_hash": "99999999999999999999999999999995+99",
  58             "manifest_text": ""
  59         }        )
  60         stubs.api.collections().get().execute.return_value = {
  61             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  62
  63         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  64         stubs.api.jobs().create().execute.return_value = {
  65             "uuid": stubs.expect_job_uuid,
  66             "state": "Queued",
  67         }
  68
  69         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  70         stubs.api.container_requests().create().execute.return_value = {
  71             "uuid": stubs.expect_container_request_uuid,
  72             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  73             "state": "Queued"
  74         }
  75
  76         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  77         stubs.api.pipeline_templates().create().execute.return_value = {
  78             "uuid": stubs.expect_pipeline_template_uuid,
  79         }
  80         stubs.expect_job_spec = {
  81             'runtime_constraints': {
  82                 'docker_image': 'arvados/jobs'
  83             },
  84             'script_parameters': {
  85                 'x': {
  86                     'basename': 'blorp.txt',
  87                     'location': '99999999999999999999999999999994+99/blorp.txt',
  88                     'class': 'File'
  89                 },
  90                 'cwl:tool':
  91                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
  92             },
  93             'repository': 'arvados',
  94             'script_version': 'master',
  95             'script': 'cwl-runner'
  96         }
  97
  98         stubs.expect_container_spec = {
  99             'priority': 1,
 100             'mounts': {
 101                 '/var/spool/cwl': {
 102                     'writable': True,
 103                     'kind': 'collection'
 104                 },
 105                 '/var/lib/cwl/workflow': {
 106                     'portable_data_hash': '99999999999999999999999999999991+99',
 107                     'kind': 'collection'
 108                 },
 109                 'stdout': {
 110                     'path': '/var/spool/cwl/cwl.output.json',
 111                     'kind': 'file'
 112                 },
 113                 '/var/lib/cwl/job/cwl.input.json': {
 114                     'portable_data_hash': '765fda0d9897729ff467a4609879c00a+60/cwl.input.json',
 115                     'kind': 'collection'
 116                 }
 117             },
 118             'state': 'Committed',
 119             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 120             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 121             'name': 'submit_wf.cwl',
 122             'container_image': '99999999999999999999999999999993+99',
 123             'output_path': '/var/spool/cwl',
 124             'cwd': '/var/spool/cwl',
 125             'runtime_constraints': {
 126                 'API': True,
 127                 'vcpus': 1,
 128                 'ram': 268435456
 129             }
 130         }
 131         return func(self, stubs, *args, **kwargs)
 132     return wrapped
 133
 134
 135 class TestSubmit(unittest.TestCase):
 136     @stubs
 137     def test_submit(self, stubs):
 138         capture_stdout = cStringIO.StringIO()
 139         exited = arvados_cwl.main(
 140             ["--submit", "--no-wait",
 141              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 142             capture_stdout, sys.stderr, api_client=stubs.api)
 143         self.assertEqual(exited, 0)
 144
 145         stubs.api.collections().create.assert_has_calls([
 146             mock.call(),
 147             mock.call(body={
 148                 'manifest_text':
 149                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 150                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 151                 '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
 152                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 153                 'name': 'submit_wf.cwl',
 154             }, ensure_unique_name=True),
 155             mock.call().execute(),
 156             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 157                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 158                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 159                             'name': 'New collection',
 160                             'replication_desired': None,
 161             }, ensure_unique_name=True),
 162             mock.call().execute(num_retries=4),
 163             mock.call(body={
 164                 'manifest_text':
 165                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 166                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 167                 'name': '#',
 168             }, ensure_unique_name=True),
 169             mock.call().execute()])
 170
 171         expect_job = copy.deepcopy(stubs.expect_job_spec)
 172         expect_job["owner_uuid"] = stubs.fake_user_uuid
 173         stubs.api.jobs().create.assert_called_with(
 174             body=expect_job,
 175             find_or_create=True)
 176         self.assertEqual(capture_stdout.getvalue(),
 177                          stubs.expect_job_uuid + '\n')
 178
 179     @stubs
 180     def test_submit_with_project_uuid(self, stubs):
 181         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 182
 183         exited = arvados_cwl.main(
 184             ["--submit", "--no-wait",
 185              "--project-uuid", project_uuid,
 186              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 187             sys.stdout, sys.stderr, api_client=stubs.api)
 188         self.assertEqual(exited, 0)
 189
 190         expect_body = copy.deepcopy(stubs.expect_job_spec)
 191         expect_body["owner_uuid"] = project_uuid
 192         stubs.api.jobs().create.assert_called_with(
 193             body=expect_body,
 194             find_or_create=True)
 195
 196     @stubs
 197     def test_submit_container(self, stubs):
 198         capture_stdout = cStringIO.StringIO()
 199         exited = arvados_cwl.main(
 200             ["--submit", "--no-wait", "--api=containers", "--debug",
 201              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 202             capture_stdout, sys.stderr, api_client=stubs.api)
 203         self.assertEqual(exited, 0)
 204
 205         stubs.api.collections().create.assert_has_calls([
 206             mock.call(),
 207             mock.call(body={
 208                 'manifest_text':
 209                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 210                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 211                 '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
 212                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 213                 'name': 'submit_wf.cwl',
 214             }, ensure_unique_name=True),
 215             mock.call().execute(),
 216             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 217                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 218                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 219                             'name': 'New collection',
 220                             'replication_desired': None,
 221             }, ensure_unique_name=True),
 222             mock.call().execute(num_retries=4),
 223             mock.call(body={
 224                 'manifest_text':
 225                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 226                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 227                 'name': '#',
 228             }, ensure_unique_name=True),
 229             mock.call().execute()])
 230
 231         expect_container = copy.deepcopy(stubs.expect_container_spec)
 232         expect_container["owner_uuid"] = stubs.fake_user_uuid
 233         stubs.api.container_requests().create.assert_called_with(
 234             body=expect_container)
 235         self.assertEqual(capture_stdout.getvalue(),
 236                          stubs.expect_container_request_uuid + '\n')
 237
 238
 239 class TestCreateTemplate(unittest.TestCase):
 240     @stubs
 241     def test_create(self, stubs):
 242         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 243
 244         capture_stdout = cStringIO.StringIO()
 245
 246         exited = arvados_cwl.main(
 247             ["--create-template", "--no-wait", "--debug",
 248              "--project-uuid", project_uuid,
 249              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 250             capture_stdout, sys.stderr, api_client=stubs.api)
 251         self.assertEqual(exited, 0)
 252
 253         stubs.api.pipeline_instances().create.refute_called()
 254         stubs.api.jobs().create.refute_called()
 255
 256         expect_component = copy.deepcopy(stubs.expect_job_spec)
 257         expect_component['script_parameters']['x'] = {
 258             'dataclass': 'File',
 259             'required': True,
 260             'type': 'File',
 261             'value': '99999999999999999999999999999994+99/blorp.txt',
 262         }
 263         expect_template = {
 264             "components": {
 265                 "submit_wf.cwl": expect_component,
 266             },
 267             "name": "submit_wf.cwl",
 268             "owner_uuid": project_uuid,
 269         }
 270         stubs.api.pipeline_templates().create.assert_called_with(
 271             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 272
 273         self.assertEqual(capture_stdout.getvalue(),
 274                          stubs.expect_pipeline_template_uuid + '\n')
 275
 276
 277 class TestTemplateInputs(unittest.TestCase):
 278     expect_template = {
 279         "components": {
 280             "inputs_test.cwl": {
 281                 'runtime_constraints': {
 282                     'docker_image': 'arvados/jobs',
 283                 },
 284                 'script_parameters': {
 285                     'cwl:tool':
 286                     '99999999999999999999999999999991+99/'
 287                     'wf/inputs_test.cwl',
 288                     'optionalFloatInput': None,
 289                     'fileInput': {
 290                         'type': 'File',
 291                         'dataclass': 'File',
 292                         'required': True,
 293                         'title': "It's a file; we expect to find some characters in it.",
 294                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 295                     },
 296                     'floatInput': {
 297                         'type': 'float',
 298                         'dataclass': 'number',
 299                         'required': True,
 300                         'title': 'Floats like a duck',
 301                         'default': 0.1,
 302                         'value': 0.1,
 303                     },
 304                     'optionalFloatInput': {
 305                         'type': ['null', 'float'],
 306                         'dataclass': 'number',
 307                         'required': False,
 308                     },
 309                     'boolInput': {
 310                         'type': 'boolean',
 311                         'dataclass': 'boolean',
 312                         'required': True,
 313                         'title': 'True or false?',
 314                     },
 315                 },
 316                 'repository': 'arvados',
 317                 'script_version': 'master',
 318                 'script': 'cwl-runner',
 319             },
 320         },
 321         "name": "inputs_test.cwl",
 322     }
 323
 324     @stubs
 325     def test_inputs_empty(self, stubs):
 326         exited = arvados_cwl.main(
 327             ["--create-template", "--no-wait",
 328              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 329             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 330         self.assertEqual(exited, 0)
 331
 332         expect_template = copy.deepcopy(self.expect_template)
 333         expect_template["owner_uuid"] = stubs.fake_user_uuid
 334
 335         stubs.api.pipeline_templates().create.assert_called_with(
 336             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 337
 338     @stubs
 339     def test_inputs(self, stubs):
 340         exited = arvados_cwl.main(
 341             ["--create-template", "--no-wait",
 342              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 343             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 344         self.assertEqual(exited, 0)
 345
 346         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 347
 348         expect_template = copy.deepcopy(self.expect_template)
 349         expect_template["owner_uuid"] = stubs.fake_user_uuid
 350         params = expect_template[
 351             "components"]["inputs_test.cwl"]["script_parameters"]
 352         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 353         params["floatInput"]["value"] = 1.234
 354         params["boolInput"]["value"] = True
 355
 356         stubs.api.pipeline_templates().create.assert_called_with(
 357             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)