sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados.keep
  15
  16 from .matcher import JsonDiffMatcher
  17
  18
  19 def stubs(func):
  20     @functools.wraps(func)
  21     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  22     @mock.patch("arvados.collection.KeepClient")
  23     @mock.patch("arvados.events.subscribe")
  24     def wrapped(self, events, keep_client, keepdocker, *args, **kwargs):
  25         class Stubs:
  26             pass
  27         stubs = Stubs()
  28         stubs.events = events
  29         stubs.keepdocker = keepdocker
  30         stubs.keep_client = keep_client
  31
  32         def putstub(p, **kwargs):
  33             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  34         stubs.keep_client().put.side_effect = putstub
  35         stubs.keep_client.put.side_effect = putstub
  36
  37         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  38         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  39
  40
  41         stubs.api = mock.MagicMock()
  42         stubs.api._rootDesc = arvados.api('v1')._rootDesc
  43         stubs.api.users().current().execute.return_value = {
  44             "uuid": stubs.fake_user_uuid,
  45         }
  46         stubs.api.collections().list().execute.return_value = {"items": []}
  47         stubs.api.collections().create().execute.side_effect = ({
  48             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  49             "portable_data_hash": "99999999999999999999999999999991+99",
  50             "manifest_text": ""
  51         }, {
  52             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  53             "portable_data_hash": "99999999999999999999999999999992+99",
  54             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  55         },
  56         {
  57             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  58             "portable_data_hash": "99999999999999999999999999999994+99",
  59             "manifest_text": ""
  60         },
  61         {
  62             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  63             "portable_data_hash": "99999999999999999999999999999995+99",
  64             "manifest_text": ""
  65         },
  66         {
  67             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  68             "portable_data_hash": "99999999999999999999999999999996+99",
  69             "manifest_text": ""
  70         }
  71         )
  72         stubs.api.collections().get().execute.return_value = {
  73             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  74
  75         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  76         stubs.api.jobs().create().execute.return_value = {
  77             "uuid": stubs.expect_job_uuid,
  78             "state": "Queued",
  79         }
  80
  81         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  82         stubs.api.container_requests().create().execute.return_value = {
  83             "uuid": stubs.expect_container_request_uuid,
  84             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  85             "state": "Queued"
  86         }
  87
  88         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  89         stubs.api.pipeline_templates().create().execute.return_value = {
  90             "uuid": stubs.expect_pipeline_template_uuid,
  91         }
  92         stubs.expect_job_spec = {
  93             'runtime_constraints': {
  94                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
  95             },
  96             'script_parameters': {
  97                 'x': {
  98                     'basename': 'blorp.txt',
  99                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 100                     'class': 'File'
 101                 },
 102                 'y': {
 103                     'basename': '99999999999999999999999999999998+99',
 104                     'location': 'keep:99999999999999999999999999999998+99',
 105                     'class': 'Directory'
 106                 },
 107                 'z': {
 108                     'basename': 'anonymous',
 109                     "listing": [{
 110                         "basename": "renamed.txt",
 111                         "class": "File",
 112                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 113                     }],
 114                     'class': 'Directory'
 115                 },
 116                 'cwl:tool':
 117                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 118             },
 119             'repository': 'arvados',
 120             'script_version': arvados_cwl.__version__,
 121             'script': 'cwl-runner'
 122         }
 123         stubs.pipeline_component = stubs.expect_job_spec.copy()
 124         stubs.expect_pipeline_instance = {
 125             'name': 'submit_wf.cwl',
 126             'state': 'RunningOnServer',
 127             "components": {
 128                 "cwl-runner": {
 129                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
 130                     'script_parameters': {
 131                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 132                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 133                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 134                               'listing': [
 135                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 136                               ]}},
 137                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 138                     },
 139                     'repository': 'arvados',
 140                     'script_version': arvados_cwl.__version__,
 141                     'script': 'cwl-runner',
 142                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 143                 }
 144             }
 145         }
 146         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 147         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 148         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 149         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 150         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 151             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 152             "state": "Queued"
 153         }
 154         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 155         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 156
 157         stubs.expect_container_spec = {
 158             'priority': 1,
 159             'mounts': {
 160                 '/var/spool/cwl': {
 161                     'writable': True,
 162                     'kind': 'collection'
 163                 },
 164                 '/var/lib/cwl/workflow': {
 165                     'portable_data_hash': '99999999999999999999999999999991+99',
 166                     'kind': 'collection'
 167                 },
 168                 'stdout': {
 169                     'path': '/var/spool/cwl/cwl.output.json',
 170                     'kind': 'file'
 171                 },
 172                 '/var/lib/cwl/job/cwl.input.json': {
 173                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 174                     'kind': 'collection'
 175                 }
 176             },
 177             'state': 'Committed',
 178             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 179             'command': ['arvados-cwl-runner', '--local', '--api=containers', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 180             'name': 'submit_wf.cwl',
 181             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 182             'output_path': '/var/spool/cwl',
 183             'cwd': '/var/spool/cwl',
 184             'runtime_constraints': {
 185                 'API': True,
 186                 'vcpus': 1,
 187                 'ram': 268435456
 188             }
 189         }
 190
 191         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 192         stubs.api.workflows().create().execute.return_value = {
 193             "uuid": stubs.expect_workflow_uuid,
 194         }
 195
 196         return func(self, stubs, *args, **kwargs)
 197     return wrapped
 198
 199
 200 class TestSubmit(unittest.TestCase):
 201     @mock.patch("time.sleep")
 202     @stubs
 203     def test_submit(self, stubs, tm):
 204         capture_stdout = cStringIO.StringIO()
 205         exited = arvados_cwl.main(
 206             ["--submit", "--no-wait", "--debug",
 207              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 208             capture_stdout, sys.stderr, api_client=stubs.api)
 209         self.assertEqual(exited, 0)
 210
 211         stubs.api.collections().create.assert_has_calls([
 212             mock.call(),
 213             mock.call(body={
 214                 'manifest_text':
 215                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 216                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 217                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 218                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 219                 'name': 'submit_wf.cwl',
 220             }, ensure_unique_name=True),
 221             mock.call().execute(),
 222             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 223                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 224                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 225                             'replication_desired': None,
 226                             'name': 'New collection'
 227             }, ensure_unique_name=True),
 228             mock.call().execute(num_retries=4),
 229             mock.call(body={
 230                 'manifest_text':
 231                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 232                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 233                 'name': '#',
 234             }, ensure_unique_name=True),
 235             mock.call().execute()])
 236
 237         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 238         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 239         stubs.api.pipeline_instances().create.assert_called_with(
 240             body=expect_pipeline)
 241         self.assertEqual(capture_stdout.getvalue(),
 242                          stubs.expect_pipeline_uuid + '\n')
 243
 244     @mock.patch("time.sleep")
 245     @stubs
 246     def test_submit_with_project_uuid(self, stubs, tm):
 247         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 248
 249         exited = arvados_cwl.main(
 250             ["--submit", "--no-wait",
 251              "--project-uuid", project_uuid,
 252              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 253             sys.stdout, sys.stderr, api_client=stubs.api)
 254         self.assertEqual(exited, 0)
 255
 256         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 257         expect_pipeline["owner_uuid"] = project_uuid
 258         stubs.api.pipeline_instances().create.assert_called_with(
 259             body=expect_pipeline)
 260
 261     @stubs
 262     def test_submit_container(self, stubs):
 263         capture_stdout = cStringIO.StringIO()
 264         try:
 265             exited = arvados_cwl.main(
 266                 ["--submit", "--no-wait", "--api=containers", "--debug",
 267                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 268                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 269             self.assertEqual(exited, 0)
 270         except:
 271             logging.exception("")
 272
 273         stubs.api.collections().create.assert_has_calls([
 274             mock.call(),
 275             mock.call(body={
 276                 'manifest_text':
 277                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 278                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 279                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 280                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 281                 'name': 'submit_wf.cwl',
 282             }, ensure_unique_name=True),
 283             mock.call().execute(),
 284             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 285                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 286                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 287                             'name': 'New collection',
 288                             'replication_desired': None,
 289             }, ensure_unique_name=True),
 290             mock.call().execute(num_retries=4),
 291             mock.call(body={
 292                 'manifest_text':
 293                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 294                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 295                 'name': '#',
 296             }, ensure_unique_name=True),
 297             mock.call().execute()])
 298
 299         expect_container = copy.deepcopy(stubs.expect_container_spec)
 300         expect_container["owner_uuid"] = stubs.fake_user_uuid
 301         stubs.api.container_requests().create.assert_called_with(
 302             body=expect_container)
 303         self.assertEqual(capture_stdout.getvalue(),
 304                          stubs.expect_container_request_uuid + '\n')
 305
 306
 307 class TestCreateTemplate(unittest.TestCase):
 308     @stubs
 309     def test_create(self, stubs):
 310         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 311
 312         capture_stdout = cStringIO.StringIO()
 313
 314         exited = arvados_cwl.main(
 315             ["--create-template", "--debug",
 316              "--project-uuid", project_uuid,
 317              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 318             capture_stdout, sys.stderr, api_client=stubs.api)
 319         self.assertEqual(exited, 0)
 320
 321         stubs.api.pipeline_instances().create.refute_called()
 322         stubs.api.jobs().create.refute_called()
 323
 324         expect_component = copy.deepcopy(stubs.expect_job_spec)
 325         expect_component['script_parameters']['x'] = {
 326             'dataclass': 'File',
 327             'required': True,
 328             'type': 'File',
 329             'value': '99999999999999999999999999999994+99/blorp.txt',
 330         }
 331         expect_component['script_parameters']['y'] = {
 332             'dataclass': 'Collection',
 333             'required': True,
 334             'type': 'Directory',
 335             'value': '99999999999999999999999999999998+99',
 336         }
 337         expect_component['script_parameters']['z'] = {
 338             'dataclass': 'Collection',
 339             'required': True,
 340             'type': 'Directory',
 341         }
 342         expect_template = {
 343             "components": {
 344                 "submit_wf.cwl": expect_component,
 345             },
 346             "name": "submit_wf.cwl",
 347             "owner_uuid": project_uuid,
 348         }
 349         stubs.api.pipeline_templates().create.assert_called_with(
 350             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 351
 352         self.assertEqual(capture_stdout.getvalue(),
 353                          stubs.expect_pipeline_template_uuid + '\n')
 354
 355
 356 class TestCreateWorkflow(unittest.TestCase):
 357     @stubs
 358     def test_create(self, stubs):
 359         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 360
 361         capture_stdout = cStringIO.StringIO()
 362
 363         exited = arvados_cwl.main(
 364             ["--create-workflow", "--debug",
 365              "--project-uuid", project_uuid,
 366              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 367             capture_stdout, sys.stderr, api_client=stubs.api)
 368         self.assertEqual(exited, 0)
 369
 370         stubs.api.pipeline_templates().create.refute_called()
 371         stubs.api.container_requests().create.refute_called()
 372
 373         with open("tests/wf/expect_packed.cwl") as f:
 374             expect_workflow = f.read()
 375
 376         body = {
 377             "workflow": {
 378                 "owner_uuid": project_uuid,
 379                 "name": "submit_wf.cwl",
 380                 "description": "",
 381                 "definition": expect_workflow
 382                 }
 383         }
 384         stubs.api.workflows().create.assert_called_with(
 385             body=JsonDiffMatcher(body))
 386
 387         self.assertEqual(capture_stdout.getvalue(),
 388                          stubs.expect_workflow_uuid + '\n')
 389
 390
 391 class TestTemplateInputs(unittest.TestCase):
 392     expect_template = {
 393         "components": {
 394             "inputs_test.cwl": {
 395                 'runtime_constraints': {
 396                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 397                 },
 398                 'script_parameters': {
 399                     'cwl:tool':
 400                     '99999999999999999999999999999991+99/'
 401                     'wf/inputs_test.cwl',
 402                     'optionalFloatInput': None,
 403                     'fileInput': {
 404                         'type': 'File',
 405                         'dataclass': 'File',
 406                         'required': True,
 407                         'title': "It's a file; we expect to find some characters in it.",
 408                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 409                     },
 410                     'floatInput': {
 411                         'type': 'float',
 412                         'dataclass': 'number',
 413                         'required': True,
 414                         'title': 'Floats like a duck',
 415                         'default': 0.1,
 416                         'value': 0.1,
 417                     },
 418                     'optionalFloatInput': {
 419                         'type': ['null', 'float'],
 420                         'dataclass': 'number',
 421                         'required': False,
 422                     },
 423                     'boolInput': {
 424                         'type': 'boolean',
 425                         'dataclass': 'boolean',
 426                         'required': True,
 427                         'title': 'True or false?',
 428                     },
 429                 },
 430                 'repository': 'arvados',
 431                 'script_version': arvados_cwl.__version__,
 432                 'script': 'cwl-runner',
 433             },
 434         },
 435         "name": "inputs_test.cwl",
 436     }
 437
 438     @stubs
 439     def test_inputs_empty(self, stubs):
 440         exited = arvados_cwl.main(
 441             ["--create-template", "--no-wait",
 442              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 443             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 444         self.assertEqual(exited, 0)
 445
 446         expect_template = copy.deepcopy(self.expect_template)
 447         expect_template["owner_uuid"] = stubs.fake_user_uuid
 448
 449         stubs.api.pipeline_templates().create.assert_called_with(
 450             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 451
 452     @stubs
 453     def test_inputs(self, stubs):
 454         exited = arvados_cwl.main(
 455             ["--create-template", "--no-wait",
 456              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 457             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 458         self.assertEqual(exited, 0)
 459
 460         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 461
 462         expect_template = copy.deepcopy(self.expect_template)
 463         expect_template["owner_uuid"] = stubs.fake_user_uuid
 464         params = expect_template[
 465             "components"]["inputs_test.cwl"]["script_parameters"]
 466         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 467         params["floatInput"]["value"] = 1.234
 468         params["boolInput"]["value"] = True
 469
 470         stubs.api.pipeline_templates().create.assert_called_with(
 471             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)