sdk/cwl/tests/test_submit.py

   1 import arvados
   2 import arvados.keep
   3 import arvados.collection
   4 import arvados_cwl
   5 import copy
   6 import cStringIO
   7 import functools
   8 import hashlib
   9 import mock
  10 import sys
  11 import unittest
  12 import json
  13
  14 from .matcher import JsonDiffMatcher
  15
  16
  17 def stubs(func):
  18     @functools.wraps(func)
  19     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  20     @mock.patch("arvados.collection.KeepClient")
  21     @mock.patch("arvados.events.subscribe")
  22     def wrapped(self, events, KeepClient, keepdocker, *args, **kwargs):
  23         class Stubs:
  24             pass
  25         stubs = Stubs()
  26         stubs.events = events
  27         stubs.KeepClient = KeepClient
  28         stubs.keepdocker = keepdocker
  29
  30         def putstub(p, **kwargs):
  31             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  32         stubs.KeepClient().put.side_effect = putstub
  33
  34         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  35         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  36
  37         stubs.api = mock.MagicMock()
  38         stubs.api.users().current().execute.return_value = {
  39             "uuid": stubs.fake_user_uuid,
  40         }
  41         stubs.api.collections().list().execute.return_value = {"items": []}
  42         stubs.api.collections().create().execute.side_effect = ({
  43             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  44             "portable_data_hash": "99999999999999999999999999999991+99",
  45             "manifest_text": ""
  46         }, {
  47             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  48             "portable_data_hash": "99999999999999999999999999999992+99",
  49             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  50         },
  51         {
  52             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  53             "portable_data_hash": "99999999999999999999999999999994+99",
  54             "manifest_text": ""
  55         },
  56         {
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  58             "portable_data_hash": "99999999999999999999999999999995+99",
  59             "manifest_text": ""
  60         }        )
  61         stubs.api.collections().get().execute.return_value = {
  62             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  63
  64         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  65         stubs.api.jobs().create().execute.return_value = {
  66             "uuid": stubs.expect_job_uuid,
  67             "state": "Queued",
  68         }
  69
  70         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  71         stubs.api.container_requests().create().execute.return_value = {
  72             "uuid": stubs.expect_container_request_uuid,
  73             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  74             "state": "Queued"
  75         }
  76
  77         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  78         stubs.api.pipeline_templates().create().execute.return_value = {
  79             "uuid": stubs.expect_pipeline_template_uuid,
  80         }
  81         stubs.expect_job_spec = {
  82             'runtime_constraints': {
  83                 'docker_image': 'arvados/jobs'
  84             },
  85             'script_parameters': {
  86                 'x': {
  87                     'basename': 'blorp.txt',
  88                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
  89                     'class': 'File'
  90                 },
  91                 'y': {
  92                     'basename': '99999999999999999999999999999998+99',
  93                     'location': 'keep:99999999999999999999999999999998+99',
  94                     'class': 'Directory'
  95                 },
  96                 'z': {
  97                     'basename': 'anonymous',
  98                     "listing": [{
  99                         "basename": "renamed.txt",
 100                         "class": "File",
 101                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 102                     }],
 103                     'class': 'Directory'
 104                 },
 105                 'cwl:tool':
 106                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 107             },
 108             'repository': 'arvados',
 109             'script_version': 'master',
 110             'script': 'cwl-runner'
 111         }
 112
 113         stubs.expect_container_spec = {
 114             'priority': 1,
 115             'mounts': {
 116                 '/var/spool/cwl': {
 117                     'writable': True,
 118                     'kind': 'collection'
 119                 },
 120                 '/var/lib/cwl/workflow': {
 121                     'portable_data_hash': '99999999999999999999999999999991+99',
 122                     'kind': 'collection'
 123                 },
 124                 'stdout': {
 125                     'path': '/var/spool/cwl/cwl.output.json',
 126                     'kind': 'file'
 127                 },
 128                 '/var/lib/cwl/job/cwl.input.json': {
 129                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 130                     'kind': 'collection'
 131                 }
 132             },
 133             'state': 'Committed',
 134             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 135             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 136             'name': 'submit_wf.cwl',
 137             'container_image': '99999999999999999999999999999993+99',
 138             'output_path': '/var/spool/cwl',
 139             'cwd': '/var/spool/cwl',
 140             'runtime_constraints': {
 141                 'API': True,
 142                 'vcpus': 1,
 143                 'ram': 268435456
 144             }
 145         }
 146
 147         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 148         stubs.api.workflows().create().execute.return_value = {
 149             "uuid": stubs.expect_workflow_uuid,
 150         }
 151
 152         return func(self, stubs, *args, **kwargs)
 153     return wrapped
 154
 155
 156 class TestSubmit(unittest.TestCase):
 157     @stubs
 158     def test_submit(self, stubs):
 159         capture_stdout = cStringIO.StringIO()
 160         exited = arvados_cwl.main(
 161             ["--submit", "--no-wait",
 162              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 163             capture_stdout, sys.stderr, api_client=stubs.api)
 164         self.assertEqual(exited, 0)
 165
 166         stubs.api.collections().create.assert_has_calls([
 167             mock.call(),
 168             mock.call(body={
 169                 'manifest_text':
 170                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 171                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 172                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 173                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 174                 'name': 'submit_wf.cwl',
 175             }, ensure_unique_name=True),
 176             mock.call().execute(),
 177             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 178                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 179                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 180                             'replication_desired': None,
 181                             'name': 'New collection'
 182             }, ensure_unique_name=True),
 183             mock.call().execute(num_retries=4),
 184             mock.call(body={
 185                 'manifest_text':
 186                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 187                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 188                 'name': '#',
 189             }, ensure_unique_name=True),
 190             mock.call().execute()])
 191
 192         expect_job = copy.deepcopy(stubs.expect_job_spec)
 193         expect_job["owner_uuid"] = stubs.fake_user_uuid
 194         stubs.api.jobs().create.assert_called_with(
 195             body=expect_job,
 196             find_or_create=True)
 197         self.assertEqual(capture_stdout.getvalue(),
 198                          stubs.expect_job_uuid + '\n')
 199
 200     @stubs
 201     def test_submit_with_project_uuid(self, stubs):
 202         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 203
 204         exited = arvados_cwl.main(
 205             ["--submit", "--no-wait",
 206              "--project-uuid", project_uuid,
 207              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 208             sys.stdout, sys.stderr, api_client=stubs.api)
 209         self.assertEqual(exited, 0)
 210
 211         expect_body = copy.deepcopy(stubs.expect_job_spec)
 212         expect_body["owner_uuid"] = project_uuid
 213         stubs.api.jobs().create.assert_called_with(
 214             body=expect_body,
 215             find_or_create=True)
 216
 217     @stubs
 218     def test_submit_container(self, stubs):
 219         capture_stdout = cStringIO.StringIO()
 220         exited = arvados_cwl.main(
 221             ["--submit", "--no-wait", "--api=containers", "--debug",
 222              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 223             capture_stdout, sys.stderr, api_client=stubs.api)
 224         self.assertEqual(exited, 0)
 225
 226         stubs.api.collections().create.assert_has_calls([
 227             mock.call(),
 228             mock.call(body={
 229                 'manifest_text':
 230                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 231                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 232                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 233                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 234                 'name': 'submit_wf.cwl',
 235             }, ensure_unique_name=True),
 236             mock.call().execute(),
 237             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 238                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 239                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 240                             'name': 'New collection',
 241                             'replication_desired': None,
 242             }, ensure_unique_name=True),
 243             mock.call().execute(num_retries=4),
 244             mock.call(body={
 245                 'manifest_text':
 246                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 247                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 248                 'name': '#',
 249             }, ensure_unique_name=True),
 250             mock.call().execute()])
 251
 252         expect_container = copy.deepcopy(stubs.expect_container_spec)
 253         expect_container["owner_uuid"] = stubs.fake_user_uuid
 254         stubs.api.container_requests().create.assert_called_with(
 255             body=expect_container)
 256         self.assertEqual(capture_stdout.getvalue(),
 257                          stubs.expect_container_request_uuid + '\n')
 258
 259
 260 class TestCreateTemplate(unittest.TestCase):
 261     @stubs
 262     def test_create(self, stubs):
 263         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 264
 265         capture_stdout = cStringIO.StringIO()
 266
 267         exited = arvados_cwl.main(
 268             ["--create-template", "--debug",
 269              "--project-uuid", project_uuid,
 270              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 271             capture_stdout, sys.stderr, api_client=stubs.api)
 272         self.assertEqual(exited, 0)
 273
 274         stubs.api.pipeline_instances().create.refute_called()
 275         stubs.api.jobs().create.refute_called()
 276
 277         expect_component = copy.deepcopy(stubs.expect_job_spec)
 278         expect_component['script_parameters']['x'] = {
 279             'dataclass': 'File',
 280             'required': True,
 281             'type': 'File',
 282             'value': '99999999999999999999999999999994+99/blorp.txt',
 283         }
 284         expect_component['script_parameters']['y'] = {
 285             'dataclass': 'Collection',
 286             'required': True,
 287             'type': 'Directory',
 288             'value': '99999999999999999999999999999998+99',
 289         }
 290         expect_component['script_parameters']['z'] = {
 291             'dataclass': 'Collection',
 292             'required': True,
 293             'type': 'Directory',
 294         }
 295         expect_template = {
 296             "components": {
 297                 "submit_wf.cwl": expect_component,
 298             },
 299             "name": "submit_wf.cwl",
 300             "owner_uuid": project_uuid,
 301         }
 302         stubs.api.pipeline_templates().create.assert_called_with(
 303             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 304
 305         self.assertEqual(capture_stdout.getvalue(),
 306                          stubs.expect_pipeline_template_uuid + '\n')
 307
 308
 309 class TestCreateWorkflow(unittest.TestCase):
 310     @stubs
 311     def test_create(self, stubs):
 312         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 313
 314         capture_stdout = cStringIO.StringIO()
 315
 316         exited = arvados_cwl.main(
 317             ["--create-workflow", "--debug",
 318              "--project-uuid", project_uuid,
 319              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 320             capture_stdout, sys.stderr, api_client=stubs.api)
 321         self.assertEqual(exited, 0)
 322
 323         stubs.api.pipeline_templates().create.refute_called()
 324         stubs.api.container_requests().create.refute_called()
 325
 326         with open("tests/wf/expect_packed.cwl") as f:
 327             expect_workflow = f.read()
 328
 329         body = {
 330             "workflow": {
 331                 "owner_uuid": project_uuid,
 332                 "name": "submit_wf.cwl",
 333                 "description": "",
 334                 "definition": expect_workflow
 335                 }
 336         }
 337         stubs.api.workflows().create.assert_called_with(
 338             body=JsonDiffMatcher(body))
 339
 340         self.assertEqual(capture_stdout.getvalue(),
 341                          stubs.expect_workflow_uuid + '\n')
 342
 343
 344 class TestTemplateInputs(unittest.TestCase):
 345     expect_template = {
 346         "components": {
 347             "inputs_test.cwl": {
 348                 'runtime_constraints': {
 349                     'docker_image': 'arvados/jobs',
 350                 },
 351                 'script_parameters': {
 352                     'cwl:tool':
 353                     '99999999999999999999999999999991+99/'
 354                     'wf/inputs_test.cwl',
 355                     'optionalFloatInput': None,
 356                     'fileInput': {
 357                         'type': 'File',
 358                         'dataclass': 'File',
 359                         'required': True,
 360                         'title': "It's a file; we expect to find some characters in it.",
 361                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 362                     },
 363                     'floatInput': {
 364                         'type': 'float',
 365                         'dataclass': 'number',
 366                         'required': True,
 367                         'title': 'Floats like a duck',
 368                         'default': 0.1,
 369                         'value': 0.1,
 370                     },
 371                     'optionalFloatInput': {
 372                         'type': ['null', 'float'],
 373                         'dataclass': 'number',
 374                         'required': False,
 375                     },
 376                     'boolInput': {
 377                         'type': 'boolean',
 378                         'dataclass': 'boolean',
 379                         'required': True,
 380                         'title': 'True or false?',
 381                     },
 382                 },
 383                 'repository': 'arvados',
 384                 'script_version': 'master',
 385                 'script': 'cwl-runner',
 386             },
 387         },
 388         "name": "inputs_test.cwl",
 389     }
 390
 391     @stubs
 392     def test_inputs_empty(self, stubs):
 393         exited = arvados_cwl.main(
 394             ["--create-template", "--no-wait",
 395              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 396             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 397         self.assertEqual(exited, 0)
 398
 399         expect_template = copy.deepcopy(self.expect_template)
 400         expect_template["owner_uuid"] = stubs.fake_user_uuid
 401
 402         stubs.api.pipeline_templates().create.assert_called_with(
 403             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 404
 405     @stubs
 406     def test_inputs(self, stubs):
 407         exited = arvados_cwl.main(
 408             ["--create-template", "--no-wait",
 409              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 410             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 411         self.assertEqual(exited, 0)
 412
 413         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 414
 415         expect_template = copy.deepcopy(self.expect_template)
 416         expect_template["owner_uuid"] = stubs.fake_user_uuid
 417         params = expect_template[
 418             "components"]["inputs_test.cwl"]["script_parameters"]
 419         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 420         params["floatInput"]["value"] = 1.234
 421         params["boolInput"]["value"] = True
 422
 423         stubs.api.pipeline_templates().create.assert_called_with(
 424             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)