sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12
  13 from .matcher import JsonDiffMatcher
  14
  15
  16 def stubs(func):
  17     @functools.wraps(func)
  18     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  19     @mock.patch("arvados.collection.KeepClient")
  20     @mock.patch("arvados.events.subscribe")
  21     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
  22         class Stubs:
  23             pass
  24         stubs = Stubs()
  25         stubs.events = events
  26         stubs.KeepClient = KeepClient
  27         stubs.keepdocker = keepdocker
  28
  29         def putstub(p, **kwargs):
  30             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  31         stubs.KeepClient().put.side_effect = putstub
  32
  33         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  34         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  35
  36         stubs.api = mock.MagicMock()
  37         stubs.api.users().current().execute.return_value = {
  38             "uuid": stubs.fake_user_uuid,
  39         }
  40         stubs.api.collections().list().execute.return_value = {"items": []}
  41         stubs.api.collections().create().execute.side_effect = ({
  42             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  43             "portable_data_hash": "99999999999999999999999999999991+99",
  44         }, {
  45             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  46             "portable_data_hash": "99999999999999999999999999999992+99",
  47         },
  48         {
  49             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  50             "portable_data_hash": "99999999999999999999999999999994+99",
  51             "manifest_text": ""
  52         })
  53         stubs.api.collections().get().execute.return_value = {
  54             "portable_data_hash": "99999999999999999999999999999993+99"}
  55
  56         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  57         stubs.api.jobs().create().execute.return_value = {
  58             "uuid": stubs.expect_job_uuid,
  59             "state": "Queued",
  60         }
  61
  62         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  63         stubs.api.container_requests().create().execute.return_value = {
  64             "uuid": stubs.expect_container_request_uuid,
  65             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  66             "state": "Queued"
  67         }
  68
  69         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  70         stubs.api.pipeline_templates().create().execute.return_value = {
  71             "uuid": stubs.expect_pipeline_template_uuid,
  72         }
  73         stubs.expect_job_spec = {
  74             'runtime_constraints': {
  75                 'docker_image': 'arvados/jobs'
  76             },
  77             'script_parameters': {
  78                 'x': {
  79                     'path': '99999999999999999999999999999992+99/blorp.txt',
  80                     'class': 'File'
  81                 },
  82                 'cwl:tool':
  83                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
  84             },
  85             'repository': 'arvados',
  86             'script_version': 'master',
  87             'script': 'cwl-runner'
  88         }
  89
  90         stubs.expect_container_spec = {
  91             'priority': 1,
  92             'mounts': {
  93                 '/var/spool/cwl': {
  94                     'writable': True,
  95                     'kind': 'collection'
  96                 },
  97                 '/var/lib/cwl/workflow': {
  98                     'portable_data_hash': '99999999999999999999999999999991+99',
  99                     'kind': 'collection'
 100                 },
 101                 'stdout': {
 102                     'path': '/var/spool/cwl/cwl.output.json',
 103                     'kind': 'file'
 104                 },
 105                 '/var/lib/cwl/job/cwl.input.json': {
 106                     'portable_data_hash': '33be5c865fe12e1e4788d2f1bc627f7a+60/cwl.input.json',
 107                     'kind': 'collection'
 108                 }
 109             },
 110             'state': 'Committed',
 111             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 112             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 113             'name': 'submit_wf.cwl',
 114             'container_image': '99999999999999999999999999999993+99',
 115             'output_path': '/var/spool/cwl',
 116             'cwd': '/var/spool/cwl',
 117             'runtime_constraints': {
 118                 'API': True,
 119                 'vcpus': 1,
 120                 'ram': 268435456
 121             }
 122         }
 123         return func(self, stubs, *args, **kwargs)
 124     return wrapped
 125
 126
 127 class TestSubmit(unittest.TestCase):
 128     @stubs
 129     def test_submit(self, stubs):
 130         capture_stdout = cStringIO.StringIO()
 131         exited = arvados_cwl.main(
 132             ["--submit", "--no-wait",
 133              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 134             capture_stdout, sys.stderr, api_client=stubs.api)
 135         self.assertEqual(exited, 0)
 136
 137         stubs.api.collections().create.assert_has_calls([
 138             mock.call(),
 139             mock.call(body={
 140                 'manifest_text':
 141                 './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
 142                 '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
 143                 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
 144                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 145                 'name': 'submit_wf.cwl',
 146             }, ensure_unique_name=True),
 147             mock.call().execute(),
 148             mock.call(body={
 149                 'manifest_text':
 150                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 151                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 152                 'name': '#',
 153             }, ensure_unique_name=True),
 154             mock.call().execute()])
 155
 156         expect_job = copy.deepcopy(stubs.expect_job_spec)
 157         expect_job["owner_uuid"] = stubs.fake_user_uuid
 158         stubs.api.jobs().create.assert_called_with(
 159             body=expect_job,
 160             find_or_create=True)
 161         self.assertEqual(capture_stdout.getvalue(),
 162                          stubs.expect_job_uuid + '\n')
 163
 164     @stubs
 165     def test_submit_with_project_uuid(self, stubs):
 166         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 167
 168         exited = arvados_cwl.main(
 169             ["--submit", "--no-wait",
 170              "--project-uuid", project_uuid,
 171              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 172             sys.stdout, sys.stderr, api_client=stubs.api)
 173         self.assertEqual(exited, 0)
 174
 175         expect_body = copy.deepcopy(stubs.expect_job_spec)
 176         expect_body["owner_uuid"] = project_uuid
 177         stubs.api.jobs().create.assert_called_with(
 178             body=expect_body,
 179             find_or_create=True)
 180
 181     @stubs
 182     def test_submit_container(self, stubs):
 183         capture_stdout = cStringIO.StringIO()
 184         exited = arvados_cwl.main(
 185             ["--submit", "--no-wait", "--api=containers", "--debug",
 186              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 187             capture_stdout, sys.stderr, api_client=stubs.api)
 188         self.assertEqual(exited, 0)
 189
 190         stubs.api.collections().create.assert_has_calls([
 191             mock.call(),
 192             mock.call(body={
 193                 'manifest_text':
 194                 './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
 195                 '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
 196                 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
 197                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 198                 'name': 'submit_wf.cwl',
 199             }, ensure_unique_name=True),
 200             mock.call().execute(),
 201             mock.call(body={
 202                 'manifest_text':
 203                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 204                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 205                 'name': '#',
 206             }, ensure_unique_name=True),
 207             mock.call().execute()])
 208
 209         expect_container = copy.deepcopy(stubs.expect_container_spec)
 210         expect_container["owner_uuid"] = stubs.fake_user_uuid
 211         stubs.api.container_requests().create.assert_called_with(
 212             body=expect_container)
 213         self.assertEqual(capture_stdout.getvalue(),
 214                          stubs.expect_container_request_uuid + '\n')
 215
 216
 217 class TestCreateTemplate(unittest.TestCase):
 218     @stubs
 219     def test_create(self, stubs):
 220         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 221
 222         capture_stdout = cStringIO.StringIO()
 223
 224         exited = arvados_cwl.main(
 225             ["--create-template", "--no-wait",
 226              "--project-uuid", project_uuid,
 227              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 228             capture_stdout, sys.stderr, api_client=stubs.api)
 229         self.assertEqual(exited, 0)
 230
 231         stubs.api.pipeline_instances().create.refute_called()
 232         stubs.api.jobs().create.refute_called()
 233
 234         expect_component = copy.deepcopy(stubs.expect_job_spec)
 235         expect_component['script_parameters']['x'] = {
 236             'dataclass': 'File',
 237             'required': True,
 238             'type': 'File',
 239             'value': '99999999999999999999999999999992+99/blorp.txt',
 240         }
 241         expect_template = {
 242             "components": {
 243                 "submit_wf.cwl": expect_component,
 244             },
 245             "name": "submit_wf.cwl",
 246             "owner_uuid": project_uuid,
 247         }
 248         stubs.api.pipeline_templates().create.assert_called_with(
 249             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 250
 251         self.assertEqual(capture_stdout.getvalue(),
 252                          stubs.expect_pipeline_template_uuid + '\n')
 253
 254
 255 class TestTemplateInputs(unittest.TestCase):
 256     expect_template = {
 257         "components": {
 258             "inputs_test.cwl": {
 259                 'runtime_constraints': {
 260                     'docker_image': 'arvados/jobs',
 261                 },
 262                 'script_parameters': {
 263                     'cwl:tool':
 264                     '99999999999999999999999999999991+99/'
 265                     'wf/inputs_test.cwl',
 266                     'optionalFloatInput': None,
 267                     'fileInput': {
 268                         'type': 'File',
 269                         'dataclass': 'File',
 270                         'required': True,
 271                         'title': "It's a file; we expect to find some characters in it.",
 272                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 273                     },
 274                     'floatInput': {
 275                         'type': 'float',
 276                         'dataclass': 'number',
 277                         'required': True,
 278                         'title': 'Floats like a duck',
 279                         'default': 0.1,
 280                         'value': 0.1,
 281                     },
 282                     'optionalFloatInput': {
 283                         'type': ['null', 'float'],
 284                         'dataclass': 'number',
 285                         'required': False,
 286                     },
 287                     'boolInput': {
 288                         'type': 'boolean',
 289                         'dataclass': 'boolean',
 290                         'required': True,
 291                         'title': 'True or false?',
 292                     },
 293                 },
 294                 'repository': 'arvados',
 295                 'script_version': 'master',
 296                 'script': 'cwl-runner',
 297             },
 298         },
 299         "name": "inputs_test.cwl",
 300     }
 301
 302     @stubs
 303     def test_inputs_empty(self, stubs):
 304         exited = arvados_cwl.main(
 305             ["--create-template", "--no-wait",
 306              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 307             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 308         self.assertEqual(exited, 0)
 309
 310         expect_template = copy.deepcopy(self.expect_template)
 311         expect_template["owner_uuid"] = stubs.fake_user_uuid
 312
 313         stubs.api.pipeline_templates().create.assert_called_with(
 314             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 315
 316     @stubs
 317     def test_inputs(self, stubs):
 318         exited = arvados_cwl.main(
 319             ["--create-template", "--no-wait",
 320              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 321             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 322         self.assertEqual(exited, 0)
 323
 324         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 325
 326         expect_template = copy.deepcopy(self.expect_template)
 327         expect_template["owner_uuid"] = stubs.fake_user_uuid
 328         params = expect_template[
 329             "components"]["inputs_test.cwl"]["script_parameters"]
 330         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
 331         params["floatInput"]["value"] = 1.234
 332         params["boolInput"]["value"] = True
 333
 334         stubs.api.pipeline_templates().create.assert_called_with(
 335             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)