sdk/cwl/tests/test_submit.py

   1 import copy
   2 import cStringIO
   3 import functools
   4 import hashlib
   5 import json
   6 import logging
   7 import mock
   8 import sys
   9 import unittest
  10
  11 import arvados
  12 import arvados.collection
  13 import arvados_cwl
  14 import arvados.keep
  15
  16 from .matcher import JsonDiffMatcher
  17 from .mock_discovery import get_rootDesc
  18
  19 _rootDesc = None
  20
  21 def stubs(func):
  22     @functools.wraps(func)
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     @mock.patch("arvados.collection.KeepClient")
  25     @mock.patch("arvados.keep.KeepClient")
  26     @mock.patch("arvados.events.subscribe")
  27     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
  28         class Stubs:
  29             pass
  30         stubs = Stubs()
  31         stubs.events = events
  32         stubs.keepdocker = keepdocker
  33
  34
  35         def putstub(p, **kwargs):
  36             return "%s+%i" % (hashlib.md5(p).hexdigest(), len(p))
  37         keep_client1().put.side_effect = putstub
  38         keep_client1.put.side_effect = putstub
  39         keep_client2().put.side_effect = putstub
  40         keep_client2.put.side_effect = putstub
  41
  42         stubs.keep_client = keep_client2
  43         stubs.keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  44         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
  45
  46         stubs.api = mock.MagicMock()
  47         stubs.api._rootDesc = get_rootDesc()
  48
  49         stubs.api.users().current().execute.return_value = {
  50             "uuid": stubs.fake_user_uuid,
  51         }
  52         stubs.api.collections().list().execute.return_value = {"items": []}
  53         stubs.api.collections().create().execute.side_effect = ({
  54             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
  55             "portable_data_hash": "99999999999999999999999999999991+99",
  56             "manifest_text": ""
  57         }, {
  58             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
  59             "portable_data_hash": "99999999999999999999999999999992+99",
  60             "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
  61         },
  62         {
  63             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
  64             "portable_data_hash": "99999999999999999999999999999994+99",
  65             "manifest_text": ""
  66         },
  67         {
  68             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
  69             "portable_data_hash": "99999999999999999999999999999995+99",
  70             "manifest_text": ""
  71         },
  72         {
  73             "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz6",
  74             "portable_data_hash": "99999999999999999999999999999996+99",
  75             "manifest_text": ""
  76         }
  77         )
  78         stubs.api.collections().get().execute.return_value = {
  79             "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
  80
  81         stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  82         stubs.api.jobs().create().execute.return_value = {
  83             "uuid": stubs.expect_job_uuid,
  84             "state": "Queued",
  85         }
  86
  87         stubs.expect_container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
  88         stubs.api.container_requests().create().execute.return_value = {
  89             "uuid": stubs.expect_container_request_uuid,
  90             "container_uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz",
  91             "state": "Queued"
  92         }
  93
  94         stubs.expect_pipeline_template_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
  95         stubs.api.pipeline_templates().create().execute.return_value = {
  96             "uuid": stubs.expect_pipeline_template_uuid,
  97         }
  98         stubs.expect_job_spec = {
  99             'runtime_constraints': {
 100                 'docker_image': 'arvados/jobs:'+arvados_cwl.__version__
 101             },
 102             'script_parameters': {
 103                 'x': {
 104                     'basename': 'blorp.txt',
 105                     'location': 'keep:99999999999999999999999999999994+99/blorp.txt',
 106                     'class': 'File'
 107                 },
 108                 'y': {
 109                     'basename': '99999999999999999999999999999998+99',
 110                     'location': 'keep:99999999999999999999999999999998+99',
 111                     'class': 'Directory'
 112                 },
 113                 'z': {
 114                     'basename': 'anonymous',
 115                     "listing": [{
 116                         "basename": "renamed.txt",
 117                         "class": "File",
 118                         "location": "keep:99999999999999999999999999999998+99/file1.txt"
 119                     }],
 120                     'class': 'Directory'
 121                 },
 122                 'cwl:tool':
 123                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
 124             },
 125             'repository': 'arvados',
 126             'script_version': arvados_cwl.__version__,
 127             'script': 'cwl-runner'
 128         }
 129         stubs.pipeline_component = stubs.expect_job_spec.copy()
 130         stubs.expect_pipeline_instance = {
 131             'name': 'submit_wf.cwl',
 132             'state': 'RunningOnServer',
 133             "components": {
 134                 "cwl-runner": {
 135                     'runtime_constraints': {'docker_image': 'arvados/jobs:'+arvados_cwl.__version__},
 136                     'script_parameters': {
 137                         'y': {"value": {'basename': '99999999999999999999999999999998+99', 'location': 'keep:99999999999999999999999999999998+99', 'class': 'Directory'}},
 138                         'x': {"value": {'basename': 'blorp.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999994+99/blorp.txt'}},
 139                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
 140                               'listing': [
 141                                   {'basename': 'renamed.txt', 'class': 'File', 'location': 'keep:99999999999999999999999999999998+99/file1.txt'}
 142                               ]}},
 143                         'cwl:tool': '99999999999999999999999999999991+99/wf/submit_wf.cwl',
 144                         'arv:enable_reuse': True
 145                     },
 146                     'repository': 'arvados',
 147                     'script_version': arvados_cwl.__version__,
 148                     'script': 'cwl-runner',
 149                     'job': {'state': 'Queued', 'uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}
 150                 }
 151             }
 152         }
 153         stubs.pipeline_create = copy.deepcopy(stubs.expect_pipeline_instance)
 154         stubs.expect_pipeline_uuid = "zzzzz-d1hrv-zzzzzzzzzzzzzzz"
 155         stubs.pipeline_create["uuid"] = stubs.expect_pipeline_uuid
 156         stubs.pipeline_with_job = copy.deepcopy(stubs.pipeline_create)
 157         stubs.pipeline_with_job["components"]["cwl-runner"]["job"] = {
 158             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 159             "state": "Queued"
 160         }
 161         stubs.api.pipeline_instances().create().execute.return_value = stubs.pipeline_create
 162         stubs.api.pipeline_instances().get().execute.return_value = stubs.pipeline_with_job
 163
 164         stubs.expect_container_spec = {
 165             'priority': 1,
 166             'mounts': {
 167                 '/var/spool/cwl': {
 168                     'writable': True,
 169                     'kind': 'collection'
 170                 },
 171                 '/var/lib/cwl/workflow': {
 172                     'portable_data_hash': '99999999999999999999999999999991+99',
 173                     'kind': 'collection'
 174                 },
 175                 'stdout': {
 176                     'path': '/var/spool/cwl/cwl.output.json',
 177                     'kind': 'file'
 178                 },
 179                 '/var/lib/cwl/job/cwl.input.json': {
 180                     'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
 181                     'kind': 'collection'
 182                 }
 183             },
 184             'state': 'Committed',
 185             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 186             'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json'],
 187             'name': 'submit_wf.cwl',
 188             'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
 189             'output_path': '/var/spool/cwl',
 190             'cwd': '/var/spool/cwl',
 191             'runtime_constraints': {
 192                 'API': True,
 193                 'vcpus': 1,
 194                 'ram': 268435456
 195             }
 196         }
 197
 198         stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz"
 199         stubs.api.workflows().create().execute.return_value = {
 200             "uuid": stubs.expect_workflow_uuid,
 201         }
 202
 203         return func(self, stubs, *args, **kwargs)
 204     return wrapped
 205
 206
 207 class TestSubmit(unittest.TestCase):
 208     @mock.patch("time.sleep")
 209     @stubs
 210     def test_submit(self, stubs, tm):
 211         capture_stdout = cStringIO.StringIO()
 212         exited = arvados_cwl.main(
 213             ["--submit", "--no-wait", "--debug",
 214              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 215             capture_stdout, sys.stderr, api_client=stubs.api)
 216         self.assertEqual(exited, 0)
 217
 218         stubs.api.collections().create.assert_has_calls([
 219             mock.call(),
 220             mock.call(body={
 221                 'manifest_text':
 222                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 223                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 224                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 225                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 226                 'name': 'submit_wf.cwl',
 227             }, ensure_unique_name=True),
 228             mock.call().execute(),
 229             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 230                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 231                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 232                             'replication_desired': None,
 233                             'name': 'New collection'
 234             }, ensure_unique_name=True),
 235             mock.call().execute(num_retries=4),
 236             mock.call(body={
 237                 'manifest_text':
 238                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 239                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 240                 'name': '#',
 241             }, ensure_unique_name=True),
 242             mock.call().execute()])
 243
 244         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 245         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 246         stubs.api.pipeline_instances().create.assert_called_with(
 247             body=expect_pipeline)
 248         self.assertEqual(capture_stdout.getvalue(),
 249                          stubs.expect_pipeline_uuid + '\n')
 250
 251
 252     @mock.patch("time.sleep")
 253     @stubs
 254     def test_submit_no_reuse(self, stubs, tm):
 255         capture_stdout = cStringIO.StringIO()
 256         exited = arvados_cwl.main(
 257             ["--submit", "--no-wait", "--debug", "--disable-reuse",
 258              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 259             capture_stdout, sys.stderr, api_client=stubs.api)
 260         self.assertEqual(exited, 0)
 261
 262         stubs.expect_pipeline_instance["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
 263
 264         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 265         expect_pipeline["owner_uuid"] = stubs.fake_user_uuid
 266         stubs.api.pipeline_instances().create.assert_called_with(
 267             body=expect_pipeline)
 268         self.assertEqual(capture_stdout.getvalue(),
 269                          stubs.expect_pipeline_uuid + '\n')
 270
 271     @mock.patch("time.sleep")
 272     @stubs
 273     def test_submit_with_project_uuid(self, stubs, tm):
 274         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 275
 276         exited = arvados_cwl.main(
 277             ["--submit", "--no-wait",
 278              "--project-uuid", project_uuid,
 279              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 280             sys.stdout, sys.stderr, api_client=stubs.api)
 281         self.assertEqual(exited, 0)
 282
 283         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
 284         expect_pipeline["owner_uuid"] = project_uuid
 285         stubs.api.pipeline_instances().create.assert_called_with(
 286             body=expect_pipeline)
 287
 288     @stubs
 289     def test_submit_container(self, stubs):
 290         capture_stdout = cStringIO.StringIO()
 291         try:
 292             exited = arvados_cwl.main(
 293                 ["--submit", "--no-wait", "--api=containers", "--debug",
 294                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 295                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 296             self.assertEqual(exited, 0)
 297         except:
 298             logging.exception("")
 299
 300         stubs.api.collections().create.assert_has_calls([
 301             mock.call(),
 302             mock.call(body={
 303                 'manifest_text':
 304                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
 305                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
 306                 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
 307                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 308                 'name': 'submit_wf.cwl',
 309             }, ensure_unique_name=True),
 310             mock.call().execute(),
 311             mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
 312                             '0:0:blub.txt 0:0:submit_tool.cwl\n',
 313                             'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 314                             'name': 'New collection',
 315                             'replication_desired': None,
 316             }, ensure_unique_name=True),
 317             mock.call().execute(num_retries=4),
 318             mock.call(body={
 319                 'manifest_text':
 320                 '. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
 321                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
 322                 'name': '#',
 323             }, ensure_unique_name=True),
 324             mock.call().execute()])
 325
 326         expect_container = copy.deepcopy(stubs.expect_container_spec)
 327         expect_container["owner_uuid"] = stubs.fake_user_uuid
 328         stubs.api.container_requests().create.assert_called_with(
 329             body=expect_container)
 330         self.assertEqual(capture_stdout.getvalue(),
 331                          stubs.expect_container_request_uuid + '\n')
 332
 333     @stubs
 334     def test_submit_container_no_reuse(self, stubs):
 335         capture_stdout = cStringIO.StringIO()
 336         try:
 337             exited = arvados_cwl.main(
 338                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
 339                  "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 340                 capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 341             self.assertEqual(exited, 0)
 342         except:
 343             logging.exception("")
 344
 345         stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--disable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/job/cwl.input.json']
 346
 347         expect_container = copy.deepcopy(stubs.expect_container_spec)
 348         expect_container["owner_uuid"] = stubs.fake_user_uuid
 349         stubs.api.container_requests().create.assert_called_with(
 350             body=expect_container)
 351         self.assertEqual(capture_stdout.getvalue(),
 352                          stubs.expect_container_request_uuid + '\n')
 353
 354
 355 class TestCreateTemplate(unittest.TestCase):
 356     @stubs
 357     def test_create(self, stubs):
 358         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 359
 360         capture_stdout = cStringIO.StringIO()
 361
 362         exited = arvados_cwl.main(
 363             ["--create-template", "--debug",
 364              "--project-uuid", project_uuid,
 365              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 366             capture_stdout, sys.stderr, api_client=stubs.api)
 367         self.assertEqual(exited, 0)
 368
 369         stubs.api.pipeline_instances().create.refute_called()
 370         stubs.api.jobs().create.refute_called()
 371
 372         expect_component = copy.deepcopy(stubs.expect_job_spec)
 373         expect_component['script_parameters']['x'] = {
 374             'dataclass': 'File',
 375             'required': True,
 376             'type': 'File',
 377             'value': '99999999999999999999999999999994+99/blorp.txt',
 378         }
 379         expect_component['script_parameters']['y'] = {
 380             'dataclass': 'Collection',
 381             'required': True,
 382             'type': 'Directory',
 383             'value': '99999999999999999999999999999998+99',
 384         }
 385         expect_component['script_parameters']['z'] = {
 386             'dataclass': 'Collection',
 387             'required': True,
 388             'type': 'Directory',
 389         }
 390         expect_template = {
 391             "components": {
 392                 "submit_wf.cwl": expect_component,
 393             },
 394             "name": "submit_wf.cwl",
 395             "owner_uuid": project_uuid,
 396         }
 397         stubs.api.pipeline_templates().create.assert_called_with(
 398             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 399
 400         self.assertEqual(capture_stdout.getvalue(),
 401                          stubs.expect_pipeline_template_uuid + '\n')
 402
 403
 404 class TestCreateWorkflow(unittest.TestCase):
 405     @stubs
 406     def test_create(self, stubs):
 407         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 408
 409         capture_stdout = cStringIO.StringIO()
 410
 411         exited = arvados_cwl.main(
 412             ["--create-workflow", "--debug",
 413              "--project-uuid", project_uuid,
 414              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
 415             capture_stdout, sys.stderr, api_client=stubs.api)
 416         self.assertEqual(exited, 0)
 417
 418         stubs.api.pipeline_templates().create.refute_called()
 419         stubs.api.container_requests().create.refute_called()
 420
 421         with open("tests/wf/expect_packed.cwl") as f:
 422             expect_workflow = f.read()
 423
 424         body = {
 425             "workflow": {
 426                 "owner_uuid": project_uuid,
 427                 "name": "submit_wf.cwl",
 428                 "description": "",
 429                 "definition": expect_workflow
 430                 }
 431         }
 432         stubs.api.workflows().create.assert_called_with(
 433             body=JsonDiffMatcher(body))
 434
 435         self.assertEqual(capture_stdout.getvalue(),
 436                          stubs.expect_workflow_uuid + '\n')
 437
 438
 439 class TestTemplateInputs(unittest.TestCase):
 440     expect_template = {
 441         "components": {
 442             "inputs_test.cwl": {
 443                 'runtime_constraints': {
 444                     'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
 445                 },
 446                 'script_parameters': {
 447                     'cwl:tool':
 448                     '99999999999999999999999999999991+99/'
 449                     'wf/inputs_test.cwl',
 450                     'optionalFloatInput': None,
 451                     'fileInput': {
 452                         'type': 'File',
 453                         'dataclass': 'File',
 454                         'required': True,
 455                         'title': "It's a file; we expect to find some characters in it.",
 456                         'description': 'If there were anything further to say, it would be said here,\nor here.'
 457                     },
 458                     'floatInput': {
 459                         'type': 'float',
 460                         'dataclass': 'number',
 461                         'required': True,
 462                         'title': 'Floats like a duck',
 463                         'default': 0.1,
 464                         'value': 0.1,
 465                     },
 466                     'optionalFloatInput': {
 467                         'type': ['null', 'float'],
 468                         'dataclass': 'number',
 469                         'required': False,
 470                     },
 471                     'boolInput': {
 472                         'type': 'boolean',
 473                         'dataclass': 'boolean',
 474                         'required': True,
 475                         'title': 'True or false?',
 476                     },
 477                 },
 478                 'repository': 'arvados',
 479                 'script_version': arvados_cwl.__version__,
 480                 'script': 'cwl-runner',
 481             },
 482         },
 483         "name": "inputs_test.cwl",
 484     }
 485
 486     @stubs
 487     def test_inputs_empty(self, stubs):
 488         exited = arvados_cwl.main(
 489             ["--create-template", "--no-wait",
 490              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
 491             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 492         self.assertEqual(exited, 0)
 493
 494         expect_template = copy.deepcopy(self.expect_template)
 495         expect_template["owner_uuid"] = stubs.fake_user_uuid
 496
 497         stubs.api.pipeline_templates().create.assert_called_with(
 498             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 499
 500     @stubs
 501     def test_inputs(self, stubs):
 502         exited = arvados_cwl.main(
 503             ["--create-template", "--no-wait",
 504              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
 505             cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
 506         self.assertEqual(exited, 0)
 507
 508         self.expect_template["owner_uuid"] = stubs.fake_user_uuid
 509
 510         expect_template = copy.deepcopy(self.expect_template)
 511         expect_template["owner_uuid"] = stubs.fake_user_uuid
 512         params = expect_template[
 513             "components"]["inputs_test.cwl"]["script_parameters"]
 514         params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
 515         params["floatInput"]["value"] = 1.234
 516         params["boolInput"]["value"] = True
 517
 518         stubs.api.pipeline_templates().create.assert_called_with(
 519             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)