sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 from arvados_cwl.arvdocker import arv_docker_clear_cache
   7 import logging
   8 import mock
   9 import unittest
  10 import os
  11 import functools
  12 import cwltool.process
  13 from schema_salad.ref_resolver import Loader
  14 from schema_salad.sourceline import cmap
  15
  16 from .matcher import JsonDiffMatcher
  17
  18 if not os.getenv('ARVADOS_DEBUG'):
  19     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  20     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  21
  22
  23 class TestContainer(unittest.TestCase):
  24
  25     # The test passes no builder.resources
  26     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  27     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  28     def test_run(self, keepdocker):
  29         for enable_reuse in (True, False):
  30             arv_docker_clear_cache()
  31
  32             runner = mock.MagicMock()
  33             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  34             runner.ignore_docker_for_reuse = False
  35             runner.intermediate_output_ttl = 0
  36
  37             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  38             runner.api.collections().get().execute.return_value = {
  39                 "portable_data_hash": "99999999999999999999999999999993+99"}
  40
  41             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  42
  43             tool = cmap({
  44                 "inputs": [],
  45                 "outputs": [],
  46                 "baseCommand": "ls",
  47                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  48                 "id": "#",
  49                 "class": "CommandLineTool"
  50             })
  51             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  52                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  53             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  54                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  55             arvtool.formatgraph = None
  56             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  57                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  58                 j.run(enable_reuse=enable_reuse, priority=500)
  59                 runner.api.container_requests().create.assert_called_with(
  60                     body=JsonDiffMatcher({
  61                         'environment': {
  62                             'HOME': '/var/spool/cwl',
  63                             'TMPDIR': '/tmp'
  64                         },
  65                         'name': 'test_run_'+str(enable_reuse),
  66                         'runtime_constraints': {
  67                             'vcpus': 1,
  68                             'ram': 1073741824
  69                         },
  70                         'use_existing': enable_reuse,
  71                         'priority': 500,
  72                         'mounts': {
  73                             '/tmp': {'kind': 'tmp',
  74                                      "capacity": 1073741824
  75                                  },
  76                             '/var/spool/cwl': {'kind': 'tmp',
  77                                                "capacity": 1073741824 }
  78                         },
  79                         'state': 'Committed',
  80                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  81                         'output_path': '/var/spool/cwl',
  82                         'output_ttl': 0,
  83                         'container_image': 'arvados/jobs',
  84                         'command': ['ls', '/var/spool/cwl'],
  85                         'cwd': '/var/spool/cwl',
  86                         'scheduling_parameters': {},
  87                         'properties': {},
  88                     }))
  89
  90     # The test passes some fields in builder.resources
  91     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  92     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  93     def test_resource_requirements(self, keepdocker):
  94         arv_docker_clear_cache()
  95         runner = mock.MagicMock()
  96         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  97         runner.ignore_docker_for_reuse = False
  98         runner.intermediate_output_ttl = 3600
  99         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 100
 101         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 102         runner.api.collections().get().execute.return_value = {
 103             "portable_data_hash": "99999999999999999999999999999993+99"}
 104
 105         tool = cmap({
 106             "inputs": [],
 107             "outputs": [],
 108             "hints": [{
 109                 "class": "ResourceRequirement",
 110                 "coresMin": 3,
 111                 "ramMin": 3000,
 112                 "tmpdirMin": 4000,
 113                 "outdirMin": 5000
 114             }, {
 115                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 116                 "keep_cache": 512
 117             }, {
 118                 "class": "http://arvados.org/cwl#APIRequirement",
 119             }, {
 120                 "class": "http://arvados.org/cwl#PartitionRequirement",
 121                 "partition": "blurb"
 122             }, {
 123                 "class": "http://arvados.org/cwl#IntermediateOutput",
 124                 "outputTTL": 7200
 125             }, {
 126                 "class": "http://arvados.org/cwl#ReuseRequirement",
 127                 "enableReuse": False
 128             }],
 129             "baseCommand": "ls",
 130             "id": "#",
 131             "class": "CommandLineTool"
 132         })
 133         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 134                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 135         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 136                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 137                                                  loader=Loader({}))
 138         arvtool.formatgraph = None
 139         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 140                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 141             j.run(enable_reuse=True, priority=500)
 142
 143         call_args, call_kwargs = runner.api.container_requests().create.call_args
 144
 145         call_body_expected = {
 146             'environment': {
 147                 'HOME': '/var/spool/cwl',
 148                 'TMPDIR': '/tmp'
 149             },
 150             'name': 'test_resource_requirements',
 151             'runtime_constraints': {
 152                 'vcpus': 3,
 153                 'ram': 3145728000,
 154                 'keep_cache_ram': 536870912,
 155                 'API': True
 156             },
 157             'use_existing': False,
 158             'priority': 500,
 159             'mounts': {
 160                 '/tmp': {'kind': 'tmp',
 161                          "capacity": 4194304000 },
 162                 '/var/spool/cwl': {'kind': 'tmp',
 163                                    "capacity": 5242880000 }
 164             },
 165             'state': 'Committed',
 166             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 167             'output_path': '/var/spool/cwl',
 168             'output_ttl': 7200,
 169             'container_image': 'arvados/jobs',
 170             'command': ['ls'],
 171             'cwd': '/var/spool/cwl',
 172             'scheduling_parameters': {
 173                 'partitions': ['blurb']
 174             },
 175             'properties': {}
 176         }
 177
 178         call_body = call_kwargs.get('body', None)
 179         self.assertNotEqual(None, call_body)
 180         for key in call_body:
 181             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 182
 183
 184     # The test passes some fields in builder.resources
 185     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 186     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 187     @mock.patch("arvados.collection.Collection")
 188     def test_initial_work_dir(self, collection_mock, keepdocker):
 189         arv_docker_clear_cache()
 190         runner = mock.MagicMock()
 191         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 192         runner.ignore_docker_for_reuse = False
 193         runner.intermediate_output_ttl = 0
 194         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 195
 196         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 197         runner.api.collections().get().execute.return_value = {
 198             "portable_data_hash": "99999999999999999999999999999993+99"}
 199
 200         sourcemock = mock.MagicMock()
 201         def get_collection_mock(p):
 202             if "/" in p:
 203                 return (sourcemock, p.split("/", 1)[1])
 204             else:
 205                 return (sourcemock, "")
 206         runner.fs_access.get_collection.side_effect = get_collection_mock
 207
 208         vwdmock = mock.MagicMock()
 209         collection_mock.return_value = vwdmock
 210         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 211
 212         tool = cmap({
 213             "inputs": [],
 214             "outputs": [],
 215             "hints": [{
 216                 "class": "InitialWorkDirRequirement",
 217                 "listing": [{
 218                     "class": "File",
 219                     "basename": "foo",
 220                     "location": "keep:99999999999999999999999999999995+99/bar"
 221                 },
 222                 {
 223                     "class": "Directory",
 224                     "basename": "foo2",
 225                     "location": "keep:99999999999999999999999999999995+99"
 226                 },
 227                 {
 228                     "class": "File",
 229                     "basename": "filename",
 230                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 231                 },
 232                 {
 233                     "class": "Directory",
 234                     "basename": "subdir",
 235                     "location": "keep:99999999999999999999999999999995+99/subdir"
 236                 }                        ]
 237             }],
 238             "baseCommand": "ls",
 239             "id": "#",
 240             "class": "CommandLineTool"
 241         })
 242         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 243                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 244         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 245                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 246                                                  loader=Loader({}))
 247         arvtool.formatgraph = None
 248         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 249                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 250             j.run(priority=500)
 251
 252         call_args, call_kwargs = runner.api.container_requests().create.call_args
 253
 254         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 255         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 256         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 257         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 258
 259         call_body_expected = {
 260             'environment': {
 261                 'HOME': '/var/spool/cwl',
 262                 'TMPDIR': '/tmp'
 263             },
 264             'name': 'test_initial_work_dir',
 265             'runtime_constraints': {
 266                 'vcpus': 1,
 267                 'ram': 1073741824
 268             },
 269             'use_existing': True,
 270             'priority': 500,
 271             'mounts': {
 272                 '/tmp': {'kind': 'tmp',
 273                          "capacity": 1073741824 },
 274                 '/var/spool/cwl': {'kind': 'tmp',
 275                                    "capacity": 1073741824 },
 276                 '/var/spool/cwl/foo': {
 277                     'kind': 'collection',
 278                     'path': 'foo',
 279                     'portable_data_hash': '99999999999999999999999999999996+99'
 280                 },
 281                 '/var/spool/cwl/foo2': {
 282                     'kind': 'collection',
 283                     'path': 'foo2',
 284                     'portable_data_hash': '99999999999999999999999999999996+99'
 285                 },
 286                 '/var/spool/cwl/filename': {
 287                     'kind': 'collection',
 288                     'path': 'filename',
 289                     'portable_data_hash': '99999999999999999999999999999996+99'
 290                 },
 291                 '/var/spool/cwl/subdir': {
 292                     'kind': 'collection',
 293                     'path': 'subdir',
 294                     'portable_data_hash': '99999999999999999999999999999996+99'
 295                 }
 296             },
 297             'state': 'Committed',
 298             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 299             'output_path': '/var/spool/cwl',
 300             'output_ttl': 0,
 301             'container_image': 'arvados/jobs',
 302             'command': ['ls'],
 303             'cwd': '/var/spool/cwl',
 304             'scheduling_parameters': {
 305             },
 306             'properties': {}
 307         }
 308
 309         call_body = call_kwargs.get('body', None)
 310         self.assertNotEqual(None, call_body)
 311         for key in call_body:
 312             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 313
 314
 315     # Test redirecting stdin/stdout/stderr
 316     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 317     def test_redirects(self, keepdocker):
 318         arv_docker_clear_cache()
 319
 320         runner = mock.MagicMock()
 321         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 322         runner.ignore_docker_for_reuse = False
 323         runner.intermediate_output_ttl = 0
 324
 325         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 326         runner.api.collections().get().execute.return_value = {
 327             "portable_data_hash": "99999999999999999999999999999993+99"}
 328
 329         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 330
 331         tool = cmap({
 332             "inputs": [],
 333             "outputs": [],
 334             "baseCommand": "ls",
 335             "stdout": "stdout.txt",
 336             "stderr": "stderr.txt",
 337             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 338             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 339             "id": "#",
 340             "class": "CommandLineTool"
 341         })
 342         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 343                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 344         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 345                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 346         arvtool.formatgraph = None
 347         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 348                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 349             j.run(priority=500)
 350             runner.api.container_requests().create.assert_called_with(
 351                 body=JsonDiffMatcher({
 352                     'environment': {
 353                         'HOME': '/var/spool/cwl',
 354                         'TMPDIR': '/tmp'
 355                     },
 356                     'name': 'test_run_redirect',
 357                     'runtime_constraints': {
 358                         'vcpus': 1,
 359                         'ram': 1073741824
 360                     },
 361                     'use_existing': True,
 362                     'priority': 500,
 363                     'mounts': {
 364                         '/tmp': {'kind': 'tmp',
 365                                  "capacity": 1073741824 },
 366                         '/var/spool/cwl': {'kind': 'tmp',
 367                                            "capacity": 1073741824 },
 368                         "stderr": {
 369                             "kind": "file",
 370                             "path": "/var/spool/cwl/stderr.txt"
 371                         },
 372                         "stdin": {
 373                             "kind": "collection",
 374                             "path": "file.txt",
 375                             "portable_data_hash": "99999999999999999999999999999996+99"
 376                         },
 377                         "stdout": {
 378                             "kind": "file",
 379                             "path": "/var/spool/cwl/stdout.txt"
 380                         },
 381                     },
 382                     'state': 'Committed',
 383                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 384                     'output_path': '/var/spool/cwl',
 385                     'output_ttl': 0,
 386                     'container_image': 'arvados/jobs',
 387                     'command': ['ls', '/var/spool/cwl'],
 388                     'cwd': '/var/spool/cwl',
 389                     'scheduling_parameters': {},
 390                     'properties': {},
 391                 }))
 392
 393     @mock.patch("arvados.collection.Collection")
 394     def test_done(self, col):
 395         api = mock.MagicMock()
 396
 397         runner = mock.MagicMock()
 398         runner.api = api
 399         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 400         runner.num_retries = 0
 401         runner.ignore_docker_for_reuse = False
 402         runner.intermediate_output_ttl = 0
 403
 404         runner.api.containers().get().execute.return_value = {"state":"Complete",
 405                                                               "output": "abc+123",
 406                                                               "exit_code": 0}
 407
 408         col().open.return_value = []
 409
 410         arvjob = arvados_cwl.ArvadosContainer(runner)
 411         arvjob.name = "testjob"
 412         arvjob.builder = mock.MagicMock()
 413         arvjob.output_callback = mock.MagicMock()
 414         arvjob.collect_outputs = mock.MagicMock()
 415         arvjob.successCodes = [0]
 416         arvjob.outdir = "/var/spool/cwl"
 417         arvjob.output_ttl = 3600
 418
 419         arvjob.collect_outputs.return_value = {"out": "stuff"}
 420
 421         arvjob.done({
 422             "state": "Final",
 423             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 424             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 425             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 426             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 427             "modified_at": "2017-05-26T12:01:22Z"
 428         })
 429
 430         self.assertFalse(api.collections().create.called)
 431
 432         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 433         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 434         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 435
 436     # The test passes no builder.resources
 437     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 438     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 439     def test_mounts(self, keepdocker):
 440         arv_docker_clear_cache()
 441
 442         runner = mock.MagicMock()
 443         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 444         runner.ignore_docker_for_reuse = False
 445         runner.intermediate_output_ttl = 0
 446
 447         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 448         runner.api.collections().get().execute.return_value = {
 449             "portable_data_hash": "99999999999999999999999999999993+99"}
 450
 451         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 452
 453         tool = cmap({
 454             "inputs": [
 455                 {"id": "p1",
 456                  "type": "Directory"}
 457             ],
 458             "outputs": [],
 459             "baseCommand": "ls",
 460             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 461             "id": "#",
 462             "class": "CommandLineTool"
 463         })
 464         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 465                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 466         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 467                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 468         arvtool.formatgraph = None
 469         job_order = {
 470             "p1": {
 471                 "class": "Directory",
 472                 "location": "keep:99999999999999999999999999999994+44",
 473                 "listing": [
 474                     {
 475                         "class": "File",
 476                         "location": "keep:99999999999999999999999999999994+44/file1",
 477                     },
 478                     {
 479                         "class": "File",
 480                         "location": "keep:99999999999999999999999999999994+44/file2",
 481                     }
 482                 ]
 483             }
 484         }
 485         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 486                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 487             j.run(priority=500)
 488             runner.api.container_requests().create.assert_called_with(
 489                 body=JsonDiffMatcher({
 490                     'environment': {
 491                         'HOME': '/var/spool/cwl',
 492                         'TMPDIR': '/tmp'
 493                     },
 494                     'name': 'test_run_mounts',
 495                     'runtime_constraints': {
 496                         'vcpus': 1,
 497                         'ram': 1073741824
 498                     },
 499                     'use_existing': True,
 500                     'priority': 500,
 501                     'mounts': {
 502                         "/keep/99999999999999999999999999999994+44": {
 503                             "kind": "collection",
 504                             "portable_data_hash": "99999999999999999999999999999994+44"
 505                         },
 506                         '/tmp': {'kind': 'tmp',
 507                                  "capacity": 1073741824 },
 508                         '/var/spool/cwl': {'kind': 'tmp',
 509                                            "capacity": 1073741824 }
 510                     },
 511                     'state': 'Committed',
 512                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 513                     'output_path': '/var/spool/cwl',
 514                     'output_ttl': 0,
 515                     'container_image': 'arvados/jobs',
 516                     'command': ['ls', '/var/spool/cwl'],
 517                     'cwd': '/var/spool/cwl',
 518                     'scheduling_parameters': {},
 519                     'properties': {},
 520                 }))