sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12
  13 from .matcher import JsonDiffMatcher
  14
  15
  16 def stubs(func):
  17     @functools.wraps(func)
  18     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  19     @mock.patch("arvados.collection.KeepClient")
  20     @mock.patch("arvados.events.subscribe")
  21     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
  22         class Stubs:
  23             pass
  24         stubs = Stubs()
  25         stubs.events = events
  26         stubs.KeepClient = KeepClient
  27         stubs.keepdocker = keepdocker
  28
  29         def putstub(p, **kwargs):
  30             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  31         stubs.KeepClient().put.side_effect = putstub
  32
  33         stubs.keepdocker.return_value = True
  34         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  35
  36         stubs.api = mock.MagicMock()
  37         stubs.api.users().current().execute.return_value = {
  38             "uuid": stubs.fake_user_uuid,
  39         }
  40         stubs.api.collections().list().execute.return_value = {"items": []}
  41         stubs.api.collections().create().execute.side_effect = ({
  42             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  43             "portable_data_hash": "99999999999999999999999999999991+99",
  44         }, {
  45             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  46             "portable_data_hash": "99999999999999999999999999999992+99",
  47         })
  48         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  49         stubs.api.jobs().create().execute.return_value = {
  50             "uuid": stubs.expect_job_uuid,
  51             "state": "Queued",
  52         }
  53         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  54         stubs.api.pipeline_templates().create().execute.return_value = {
  55             "uuid": stubs.expect_pipeline_template_uuid,
  56         }
  57         stubs.expect_job_spec = {
  58             'runtime_constraints': {
  59                 'docker_image': 'arvados/jobs'
  60             },
  61             'script_parameters': {
  62                 'x': {
  63                     'path': '99999999999999999999999999999992+99/blorp.txt',
  64                     'class': 'File'
  65                 },
  66                 'cwl:tool':
  67                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
  68             },
  69             'repository': 'arvados',
  70             'script_version': 'master',
  71             'script': 'cwl-runner'
  72         }
  73         return func(self, stubs, *args, **kwargs)
  74     return wrapped
  75
  76
  77 class TestSubmit(unittest.TestCase):
  78     @stubs
  79     def test_submit(self, stubs):
  80         capture_stdout = cStringIO.StringIO()
  81         exited = arvados_cwl.main(
  82             ["--submit", "--no-wait",
  83              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
  84             capture_stdout, sys.stderr, api_client=stubs.api)
  85         self.assertEqual(exited, 0)
  86
  87         stubs.api.collections().create.assert_has_calls([
  88             mock.call(),
  89             mock.call(body={
  90                 'manifest_text':
  91                 './tool 84ec4df683711de31b782505389a8843+429 '
  92                 '0:16:blub.txt 16:413:submit_tool.cwl\n./wf '
  93                 '81d977a245a41b8e79859fbe00623fd0+344 0:344:submit_wf.cwl\n',
  94                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
  95                 'name': 'submit_wf.cwl',
  96             }, ensure_unique_name=True),
  97             mock.call().execute(),
  98             mock.call(body={
  99                 'manifest_text':
 100                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 101                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 102                 'name': '#',
 103             }, ensure_unique_name=True),
 104             mock.call().execute()])
 105
 106         expect_job = copy.deepcopy(stubs.expect_job_spec)
 107         expect_job["owner_uuid"] = stubs.fake_user_uuid
 108         stubs.api.jobs().create.assert_called_with(
 109             body=expect_job,
 110             find_or_create=True)
 111         self.assertEqual(capture_stdout.getvalue(),
 112                          stubs.expect_job_uuid + '\n')
 113
 114     @stubs
 115     def test_submit_with_project_uuid(self, stubs):
 116         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 117
 118         exited = arvados_cwl.main(
 119             ["--submit", "--no-wait",
 120              "--project-uuid", project_uuid,
 121              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 122             sys.stdout, sys.stderr, api_client=stubs.api)
 123         self.assertEqual(exited, 0)
 124
 125         expect_body = copy.deepcopy(stubs.expect_job_spec)
 126         expect_body["owner_uuid"] = project_uuid
 127         stubs.api.jobs().create.assert_called_with(
 128             body=expect_body,
 129             find_or_create=True)
 130
 131
 132 class TestCreateTemplate(unittest.TestCase):
 133     @stubs
 134     def test_create(self, stubs):
 135         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 136
 137         capture_stdout = cStringIO.StringIO()
 138
 139         exited = arvados_cwl.main(
 140             ["--create-template", "--no-wait",
 141              "--project-uuid", project_uuid,
 142              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 143             capture_stdout, sys.stderr, api_client=stubs.api)
 144         self.assertEqual(exited, 0)
 145
 146         stubs.api.pipeline_instances().create.refute_called()
 147         stubs.api.jobs().create.refute_called()
 148
 149         expect_component = copy.deepcopy(stubs.expect_job_spec)
 150         expect_component['script_parameters']['x'] = {
 151             'dataclass': 'File',
 152             'required': True,
 153             'type': 'File',
 154             'value': '99999999999999999999999999999992+99/blorp.txt',
 155         }
 156         expect_template = {
 157             "components": {
 158                 "submit_wf.cwl": expect_component,
 159             },
 160             "name": "submit_wf.cwl",
 161             "owner_uuid": project_uuid,
 162         }
 163         stubs.api.pipeline_templates().create.assert_called_with(
 164             body=JsonDiffMatcher(expect_template))
 165
 166         self.assertEqual(capture_stdout.getvalue(),
 167                          stubs.expect_pipeline_template_uuid + '\n')
 168
 169
 170 class TestTemplateInputs(unittest.TestCase):
 171     expect_template = {
 172         "components": {
 173             "inputs_test.cwl": {
 174                 'runtime_constraints': {
 175                     'docker_image': 'arvados/jobs',
 176                 },
 177                 'script_parameters': {
 178                     'cwl:tool':
 179                     '99999999999999999999999999999991+99/'
 180                     'wf/inputs_test.cwl',
 181                     'optionalFloatInput': None,
 182                     'fileInput': {
 183                         'type': 'File',
 184                         'dataclass': 'File',
 185                         'required': True,
 186                         'title': "It's a file; we expect to find some characters in it.",
 187                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 188                     },
 189                     'floatInput': {
 190                         'type': 'float',
 191                         'dataclass': 'number',
 192                         'required': True,
 193                         'title': 'Floats like a duck',
 194                         'default': 0.1,
 195                         'value': 0.1,
 196                     },
 197                     'optionalFloatInput': {
 198                         'type': ['null', 'float'],
 199                         'dataclass': 'number',
 200                         'required': False,
 201                     },
 202                     'boolInput': {
 203                         'type': 'boolean',
 204                         'dataclass': 'boolean',
 205                         'required': True,
 206                         'title': 'True or false?',
 207                     },
 208                 },
 209                 'repository': 'arvados',
 210                 'script_version': 'master',
 211                 'script': 'cwl-runner',
 212             },
 213         },
 214         "name": "inputs_test.cwl",
 215     }
 216
 217     @stubs
 218     def test_inputs_empty(self, stubs):
 219         exited = arvados_cwl.main(
 220             ["--create-template", "--no-wait",
 221              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 222             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 223         self.assertEqual(exited, 0)
 224
 225         expect_template = copy.deepcopy(self.expect_template)
 226         expect_template["owner_uuid"] = stubs.fake_user_uuid
 227
 228         stubs.api.pipeline_templates().create.assert_called_with(
 229             body=JsonDiffMatcher(expect_template))
 230
 231     @stubs
 232     def test_inputs(self, stubs):
 233         exited = arvados_cwl.main(
 234             ["--create-template", "--no-wait",
 235              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 236             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 237         self.assertEqual(exited, 0)
 238
 239         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 240
 241         expect_template = copy.deepcopy(self.expect_template)
 242         expect_template["owner_uuid"] = stubs.fake_user_uuid
 243         params = expect_template[
 244             "components"]["inputs_test.cwl"]["script_parameters"]
 245         params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
 246         params["floatInput"]["value"] = 1.234
 247         params["boolInput"]["value"] = True
 248
 249         stubs.api.pipeline_templates().create.assert_called_with(
 250             body=JsonDiffMatcher(expect_template))