sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 from arvados_cwl.arvdocker import arv_docker_clear_cache
   7 import logging
   8 import mock
   9 import unittest
  10 import os
  11 import functools
  12 import cwltool.process
  13 from schema_salad.ref_resolver import Loader
  14 from schema_salad.sourceline import cmap
  15
  16 from .matcher import JsonDiffMatcher
  17
  18 if not os.getenv('ARVADOS_DEBUG'):
  19     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  20     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  21
  22
  23 class TestContainer(unittest.TestCase):
  24
  25     # The test passes no builder.resources
  26     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  27     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  28     def test_run(self, keepdocker):
  29         for enable_reuse in (True, False):
  30             arv_docker_clear_cache()
  31
  32             runner = mock.MagicMock()
  33             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  34             runner.ignore_docker_for_reuse = False
  35             runner.intermediate_output_ttl = 0
  36
  37             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  38             runner.api.collections().get().execute.return_value = {
  39                 "portable_data_hash": "99999999999999999999999999999993+99"}
  40
  41             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  42
  43             tool = cmap({
  44                 "inputs": [],
  45                 "outputs": [],
  46                 "baseCommand": "ls",
  47                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  48             })
  49             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  50                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  51             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  52                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  53             arvtool.formatgraph = None
  54             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  55                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  56                 j.run(enable_reuse=enable_reuse)
  57                 runner.api.container_requests().create.assert_called_with(
  58                     body=JsonDiffMatcher({
  59                         'environment': {
  60                             'HOME': '/var/spool/cwl',
  61                             'TMPDIR': '/tmp'
  62                         },
  63                         'name': 'test_run_'+str(enable_reuse),
  64                         'runtime_constraints': {
  65                             'vcpus': 1,
  66                             'ram': 1073741824
  67                         },
  68                         'use_existing': enable_reuse,
  69                         'priority': 1,
  70                         'mounts': {
  71                             '/tmp': {'kind': 'tmp',
  72                                      "capacity": 1073741824
  73                                  },
  74                             '/var/spool/cwl': {'kind': 'tmp',
  75                                                "capacity": 1073741824 }
  76                         },
  77                         'state': 'Committed',
  78                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  79                         'output_path': '/var/spool/cwl',
  80                         'output_ttl': 0,
  81                         'container_image': 'arvados/jobs',
  82                         'command': ['ls', '/var/spool/cwl'],
  83                         'cwd': '/var/spool/cwl',
  84                         'scheduling_parameters': {},
  85                         'properties': {},
  86                     }))
  87
  88     # The test passes some fields in builder.resources
  89     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  90     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  91     def test_resource_requirements(self, keepdocker):
  92         arv_docker_clear_cache()
  93         runner = mock.MagicMock()
  94         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  95         runner.ignore_docker_for_reuse = False
  96         runner.intermediate_output_ttl = 3600
  97         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  98
  99         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 100         runner.api.collections().get().execute.return_value = {
 101             "portable_data_hash": "99999999999999999999999999999993+99"}
 102
 103         tool = cmap({
 104             "inputs": [],
 105             "outputs": [],
 106             "hints": [{
 107                 "class": "ResourceRequirement",
 108                 "coresMin": 3,
 109                 "ramMin": 3000,
 110                 "tmpdirMin": 4000,
 111                 "outdirMin": 5000
 112             }, {
 113                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 114                 "keep_cache": 512
 115             }, {
 116                 "class": "http://arvados.org/cwl#APIRequirement",
 117             }, {
 118                 "class": "http://arvados.org/cwl#PartitionRequirement",
 119                 "partition": "blurb"
 120             }, {
 121                 "class": "http://arvados.org/cwl#IntermediateOutput",
 122                 "outputTTL": 7200
 123             }],
 124             "baseCommand": "ls"
 125         })
 126         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 127                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 128         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 129                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 130                                                  loader=Loader({}))
 131         arvtool.formatgraph = None
 132         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 133                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 134             j.run()
 135
 136         call_args, call_kwargs = runner.api.container_requests().create.call_args
 137
 138         call_body_expected = {
 139             'environment': {
 140                 'HOME': '/var/spool/cwl',
 141                 'TMPDIR': '/tmp'
 142             },
 143             'name': 'test_resource_requirements',
 144             'runtime_constraints': {
 145                 'vcpus': 3,
 146                 'ram': 3145728000,
 147                 'keep_cache_ram': 536870912,
 148                 'API': True
 149             },
 150             'use_existing': True,
 151             'priority': 1,
 152             'mounts': {
 153                 '/tmp': {'kind': 'tmp',
 154                          "capacity": 4194304000 },
 155                 '/var/spool/cwl': {'kind': 'tmp',
 156                                    "capacity": 5242880000 }
 157             },
 158             'state': 'Committed',
 159             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 160             'output_path': '/var/spool/cwl',
 161             'output_ttl': 7200,
 162             'container_image': 'arvados/jobs',
 163             'command': ['ls'],
 164             'cwd': '/var/spool/cwl',
 165             'scheduling_parameters': {
 166                 'partitions': ['blurb']
 167             },
 168             'properties': {}
 169         }
 170
 171         call_body = call_kwargs.get('body', None)
 172         self.assertNotEqual(None, call_body)
 173         for key in call_body:
 174             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 175
 176
 177     # The test passes some fields in builder.resources
 178     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 179     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 180     @mock.patch("arvados.collection.Collection")
 181     def test_initial_work_dir(self, collection_mock, keepdocker):
 182         arv_docker_clear_cache()
 183         runner = mock.MagicMock()
 184         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 185         runner.ignore_docker_for_reuse = False
 186         runner.intermediate_output_ttl = 0
 187         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 188
 189         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 190         runner.api.collections().get().execute.return_value = {
 191             "portable_data_hash": "99999999999999999999999999999993+99"}
 192
 193         sourcemock = mock.MagicMock()
 194         def get_collection_mock(p):
 195             if "/" in p:
 196                 return (sourcemock, p.split("/", 1)[1])
 197             else:
 198                 return (sourcemock, "")
 199         runner.fs_access.get_collection.side_effect = get_collection_mock
 200
 201         vwdmock = mock.MagicMock()
 202         collection_mock.return_value = vwdmock
 203         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 204
 205         tool = cmap({
 206             "inputs": [],
 207             "outputs": [],
 208             "hints": [{
 209                 "class": "InitialWorkDirRequirement",
 210                 "listing": [{
 211                     "class": "File",
 212                     "basename": "foo",
 213                     "location": "keep:99999999999999999999999999999995+99/bar"
 214                 },
 215                 {
 216                     "class": "Directory",
 217                     "basename": "foo2",
 218                     "location": "keep:99999999999999999999999999999995+99"
 219                 },
 220                 {
 221                     "class": "File",
 222                     "basename": "filename",
 223                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 224                 },
 225                 {
 226                     "class": "Directory",
 227                     "basename": "subdir",
 228                     "location": "keep:99999999999999999999999999999995+99/subdir"
 229                 }                        ]
 230             }],
 231             "baseCommand": "ls"
 232         })
 233         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 234                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 235         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 236                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 237                                                  loader=Loader({}))
 238         arvtool.formatgraph = None
 239         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 240                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 241             j.run()
 242
 243         call_args, call_kwargs = runner.api.container_requests().create.call_args
 244
 245         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 246         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 247         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 248         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 249
 250         call_body_expected = {
 251             'environment': {
 252                 'HOME': '/var/spool/cwl',
 253                 'TMPDIR': '/tmp'
 254             },
 255             'name': 'test_initial_work_dir',
 256             'runtime_constraints': {
 257                 'vcpus': 1,
 258                 'ram': 1073741824
 259             },
 260             'use_existing': True,
 261             'priority': 1,
 262             'mounts': {
 263                 '/tmp': {'kind': 'tmp',
 264                          "capacity": 1073741824 },
 265                 '/var/spool/cwl': {'kind': 'tmp',
 266                                    "capacity": 1073741824 },
 267                 '/var/spool/cwl/foo': {
 268                     'kind': 'collection',
 269                     'path': 'foo',
 270                     'portable_data_hash': '99999999999999999999999999999996+99'
 271                 },
 272                 '/var/spool/cwl/foo2': {
 273                     'kind': 'collection',
 274                     'path': 'foo2',
 275                     'portable_data_hash': '99999999999999999999999999999996+99'
 276                 },
 277                 '/var/spool/cwl/filename': {
 278                     'kind': 'collection',
 279                     'path': 'filename',
 280                     'portable_data_hash': '99999999999999999999999999999996+99'
 281                 },
 282                 '/var/spool/cwl/subdir': {
 283                     'kind': 'collection',
 284                     'path': 'subdir',
 285                     'portable_data_hash': '99999999999999999999999999999996+99'
 286                 }
 287             },
 288             'state': 'Committed',
 289             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 290             'output_path': '/var/spool/cwl',
 291             'output_ttl': 0,
 292             'container_image': 'arvados/jobs',
 293             'command': ['ls'],
 294             'cwd': '/var/spool/cwl',
 295             'scheduling_parameters': {
 296             },
 297             'properties': {}
 298         }
 299
 300         call_body = call_kwargs.get('body', None)
 301         self.assertNotEqual(None, call_body)
 302         for key in call_body:
 303             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 304
 305
 306     # Test redirecting stdin/stdout/stderr
 307     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 308     def test_redirects(self, keepdocker):
 309         arv_docker_clear_cache()
 310
 311         runner = mock.MagicMock()
 312         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 313         runner.ignore_docker_for_reuse = False
 314         runner.intermediate_output_ttl = 0
 315
 316         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 317         runner.api.collections().get().execute.return_value = {
 318             "portable_data_hash": "99999999999999999999999999999993+99"}
 319
 320         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 321
 322         tool = cmap({
 323             "inputs": [],
 324             "outputs": [],
 325             "baseCommand": "ls",
 326             "stdout": "stdout.txt",
 327             "stderr": "stderr.txt",
 328             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 329             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 330         })
 331         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 332                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 333         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 334                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 335         arvtool.formatgraph = None
 336         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 337                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 338             j.run()
 339             runner.api.container_requests().create.assert_called_with(
 340                 body=JsonDiffMatcher({
 341                     'environment': {
 342                         'HOME': '/var/spool/cwl',
 343                         'TMPDIR': '/tmp'
 344                     },
 345                     'name': 'test_run_redirect',
 346                     'runtime_constraints': {
 347                         'vcpus': 1,
 348                         'ram': 1073741824
 349                     },
 350                     'use_existing': True,
 351                     'priority': 1,
 352                     'mounts': {
 353                         '/tmp': {'kind': 'tmp',
 354                                  "capacity": 1073741824 },
 355                         '/var/spool/cwl': {'kind': 'tmp',
 356                                            "capacity": 1073741824 },
 357                         "stderr": {
 358                             "kind": "file",
 359                             "path": "/var/spool/cwl/stderr.txt"
 360                         },
 361                         "stdin": {
 362                             "kind": "collection",
 363                             "path": "file.txt",
 364                             "portable_data_hash": "99999999999999999999999999999996+99"
 365                         },
 366                         "stdout": {
 367                             "kind": "file",
 368                             "path": "/var/spool/cwl/stdout.txt"
 369                         },
 370                     },
 371                     'state': 'Committed',
 372                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 373                     'output_path': '/var/spool/cwl',
 374                     'output_ttl': 0,
 375                     'container_image': 'arvados/jobs',
 376                     'command': ['ls', '/var/spool/cwl'],
 377                     'cwd': '/var/spool/cwl',
 378                     'scheduling_parameters': {},
 379                     'properties': {},
 380                 }))
 381
 382     @mock.patch("arvados.collection.Collection")
 383     def test_done(self, col):
 384         api = mock.MagicMock()
 385
 386         runner = mock.MagicMock()
 387         runner.api = api
 388         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 389         runner.num_retries = 0
 390         runner.ignore_docker_for_reuse = False
 391         runner.intermediate_output_ttl = 0
 392
 393         runner.api.containers().get().execute.return_value = {"state":"Complete",
 394                                                               "output": "abc+123",
 395                                                               "exit_code": 0}
 396
 397         col().open.return_value = []
 398
 399         arvjob = arvados_cwl.ArvadosContainer(runner)
 400         arvjob.name = "testjob"
 401         arvjob.builder = mock.MagicMock()
 402         arvjob.output_callback = mock.MagicMock()
 403         arvjob.collect_outputs = mock.MagicMock()
 404         arvjob.successCodes = [0]
 405         arvjob.outdir = "/var/spool/cwl"
 406         arvjob.output_ttl = 3600
 407
 408         arvjob.collect_outputs.return_value = {"out": "stuff"}
 409
 410         arvjob.done({
 411             "state": "Final",
 412             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 413             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 414             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 415             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 416             "modified_at": "2017-05-26T12:01:22Z"
 417         })
 418
 419         self.assertFalse(api.collections().create.called)
 420
 421         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 422         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 423         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 424
 425     # The test passes no builder.resources
 426     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 427     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 428     def test_mounts(self, keepdocker):
 429         arv_docker_clear_cache()
 430
 431         runner = mock.MagicMock()
 432         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 433         runner.ignore_docker_for_reuse = False
 434         runner.intermediate_output_ttl = 0
 435
 436         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 437         runner.api.collections().get().execute.return_value = {
 438             "portable_data_hash": "99999999999999999999999999999993+99"}
 439
 440         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 441
 442         tool = cmap({
 443             "inputs": [
 444                 {"id": "p1",
 445                  "type": "Directory"}
 446             ],
 447             "outputs": [],
 448             "baseCommand": "ls",
 449             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 450         })
 451         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 452                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 453         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 454                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 455         arvtool.formatgraph = None
 456         job_order = {
 457             "p1": {
 458                 "class": "Directory",
 459                 "location": "keep:99999999999999999999999999999994+44",
 460                 "listing": [
 461                     {
 462                         "class": "File",
 463                         "location": "keep:99999999999999999999999999999994+44/file1",
 464                     },
 465                     {
 466                         "class": "File",
 467                         "location": "keep:99999999999999999999999999999994+44/file2",
 468                     }
 469                 ]
 470             }
 471         }
 472         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 473                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 474             j.run()
 475             runner.api.container_requests().create.assert_called_with(
 476                 body=JsonDiffMatcher({
 477                     'environment': {
 478                         'HOME': '/var/spool/cwl',
 479                         'TMPDIR': '/tmp'
 480                     },
 481                     'name': 'test_run_mounts',
 482                     'runtime_constraints': {
 483                         'vcpus': 1,
 484                         'ram': 1073741824
 485                     },
 486                     'use_existing': True,
 487                     'priority': 1,
 488                     'mounts': {
 489                         "/keep/99999999999999999999999999999994+44": {
 490                             "kind": "collection",
 491                             "portable_data_hash": "99999999999999999999999999999994+44"
 492                         },
 493                         '/tmp': {'kind': 'tmp',
 494                                  "capacity": 1073741824 },
 495                         '/var/spool/cwl': {'kind': 'tmp',
 496                                            "capacity": 1073741824 }
 497                     },
 498                     'state': 'Committed',
 499                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 500                     'output_path': '/var/spool/cwl',
 501                     'output_ttl': 0,
 502                     'container_image': 'arvados/jobs',
 503                     'command': ['ls', '/var/spool/cwl'],
 504                     'cwd': '/var/spool/cwl',
 505                     'scheduling_parameters': {},
 506                     'properties': {},
 507                 }))