sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 from arvados_cwl.arvdocker import arv_docker_clear_cache
   7 import logging
   8 import mock
   9 import unittest
  10 import os
  11 import functools
  12 import cwltool.process
  13 from schema_salad.ref_resolver import Loader
  14 from schema_salad.sourceline import cmap
  15
  16 from .matcher import JsonDiffMatcher
  17
  18 if not os.getenv('ARVADOS_DEBUG'):
  19     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  20     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  21
  22
  23 class TestContainer(unittest.TestCase):
  24
  25     # The test passes no builder.resources
  26     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  27     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  28     def test_run(self, keepdocker):
  29         for enable_reuse in (True, False):
  30             arv_docker_clear_cache()
  31
  32             runner = mock.MagicMock()
  33             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  34             runner.ignore_docker_for_reuse = False
  35             runner.intermediate_output_ttl = 0
  36
  37             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  38             runner.api.collections().get().execute.return_value = {
  39                 "portable_data_hash": "99999999999999999999999999999993+99"}
  40
  41             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  42
  43             tool = cmap({
  44                 "inputs": [],
  45                 "outputs": [],
  46                 "baseCommand": "ls",
  47                 "arguments": [{"valueFrom": "$(runtime.outdir)"}],
  48                 "id": "#"
  49             })
  50             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  51                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  52             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  53                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  54             arvtool.formatgraph = None
  55             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  56                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  57                 j.run(enable_reuse=enable_reuse)
  58                 runner.api.container_requests().create.assert_called_with(
  59                     body=JsonDiffMatcher({
  60                         'environment': {
  61                             'HOME': '/var/spool/cwl',
  62                             'TMPDIR': '/tmp'
  63                         },
  64                         'name': 'test_run_'+str(enable_reuse),
  65                         'runtime_constraints': {
  66                             'vcpus': 1,
  67                             'ram': 1073741824
  68                         },
  69                         'use_existing': enable_reuse,
  70                         'priority': 1,
  71                         'mounts': {
  72                             '/tmp': {'kind': 'tmp',
  73                                      "capacity": 1073741824
  74                                  },
  75                             '/var/spool/cwl': {'kind': 'tmp',
  76                                                "capacity": 1073741824 }
  77                         },
  78                         'state': 'Committed',
  79                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  80                         'output_path': '/var/spool/cwl',
  81                         'output_ttl': 0,
  82                         'container_image': 'arvados/jobs',
  83                         'command': ['ls', '/var/spool/cwl'],
  84                         'cwd': '/var/spool/cwl',
  85                         'scheduling_parameters': {},
  86                         'properties': {},
  87                     }))
  88
  89     # The test passes some fields in builder.resources
  90     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  91     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  92     def test_resource_requirements(self, keepdocker):
  93         arv_docker_clear_cache()
  94         runner = mock.MagicMock()
  95         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  96         runner.ignore_docker_for_reuse = False
  97         runner.intermediate_output_ttl = 3600
  98         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  99
 100         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 101         runner.api.collections().get().execute.return_value = {
 102             "portable_data_hash": "99999999999999999999999999999993+99"}
 103
 104         tool = cmap({
 105             "inputs": [],
 106             "outputs": [],
 107             "hints": [{
 108                 "class": "ResourceRequirement",
 109                 "coresMin": 3,
 110                 "ramMin": 3000,
 111                 "tmpdirMin": 4000,
 112                 "outdirMin": 5000
 113             }, {
 114                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 115                 "keep_cache": 512
 116             }, {
 117                 "class": "http://arvados.org/cwl#APIRequirement",
 118             }, {
 119                 "class": "http://arvados.org/cwl#PartitionRequirement",
 120                 "partition": "blurb"
 121             }, {
 122                 "class": "http://arvados.org/cwl#IntermediateOutput",
 123                 "outputTTL": 7200
 124             }, {
 125                 "class": "http://arvados.org/cwl#ReuseRequirement",
 126                 "enableReuse": False
 127             }],
 128             "baseCommand": "ls",
 129             "id": "#"
 130         })
 131         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 132                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 133         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 134                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 135                                                  loader=Loader({}))
 136         arvtool.formatgraph = None
 137         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 138                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 139             j.run(enable_reuse=True)
 140
 141         call_args, call_kwargs = runner.api.container_requests().create.call_args
 142
 143         call_body_expected = {
 144             'environment': {
 145                 'HOME': '/var/spool/cwl',
 146                 'TMPDIR': '/tmp'
 147             },
 148             'name': 'test_resource_requirements',
 149             'runtime_constraints': {
 150                 'vcpus': 3,
 151                 'ram': 3145728000,
 152                 'keep_cache_ram': 536870912,
 153                 'API': True
 154             },
 155             'use_existing': False,
 156             'priority': 1,
 157             'mounts': {
 158                 '/tmp': {'kind': 'tmp',
 159                          "capacity": 4194304000 },
 160                 '/var/spool/cwl': {'kind': 'tmp',
 161                                    "capacity": 5242880000 }
 162             },
 163             'state': 'Committed',
 164             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 165             'output_path': '/var/spool/cwl',
 166             'output_ttl': 7200,
 167             'container_image': 'arvados/jobs',
 168             'command': ['ls'],
 169             'cwd': '/var/spool/cwl',
 170             'scheduling_parameters': {
 171                 'partitions': ['blurb']
 172             },
 173             'properties': {}
 174         }
 175
 176         call_body = call_kwargs.get('body', None)
 177         self.assertNotEqual(None, call_body)
 178         for key in call_body:
 179             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 180
 181
 182     # The test passes some fields in builder.resources
 183     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 184     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 185     @mock.patch("arvados.collection.Collection")
 186     def test_initial_work_dir(self, collection_mock, keepdocker):
 187         arv_docker_clear_cache()
 188         runner = mock.MagicMock()
 189         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 190         runner.ignore_docker_for_reuse = False
 191         runner.intermediate_output_ttl = 0
 192         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 193
 194         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 195         runner.api.collections().get().execute.return_value = {
 196             "portable_data_hash": "99999999999999999999999999999993+99"}
 197
 198         sourcemock = mock.MagicMock()
 199         def get_collection_mock(p):
 200             if "/" in p:
 201                 return (sourcemock, p.split("/", 1)[1])
 202             else:
 203                 return (sourcemock, "")
 204         runner.fs_access.get_collection.side_effect = get_collection_mock
 205
 206         vwdmock = mock.MagicMock()
 207         collection_mock.return_value = vwdmock
 208         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 209
 210         tool = cmap({
 211             "inputs": [],
 212             "outputs": [],
 213             "hints": [{
 214                 "class": "InitialWorkDirRequirement",
 215                 "listing": [{
 216                     "class": "File",
 217                     "basename": "foo",
 218                     "location": "keep:99999999999999999999999999999995+99/bar"
 219                 },
 220                 {
 221                     "class": "Directory",
 222                     "basename": "foo2",
 223                     "location": "keep:99999999999999999999999999999995+99"
 224                 },
 225                 {
 226                     "class": "File",
 227                     "basename": "filename",
 228                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 229                 },
 230                 {
 231                     "class": "Directory",
 232                     "basename": "subdir",
 233                     "location": "keep:99999999999999999999999999999995+99/subdir"
 234                 }                        ]
 235             }],
 236             "baseCommand": "ls",
 237             "id": "#"
 238         })
 239         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 240                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 241         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 242                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 243                                                  loader=Loader({}))
 244         arvtool.formatgraph = None
 245         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 246                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 247             j.run()
 248
 249         call_args, call_kwargs = runner.api.container_requests().create.call_args
 250
 251         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 252         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 253         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 254         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 255
 256         call_body_expected = {
 257             'environment': {
 258                 'HOME': '/var/spool/cwl',
 259                 'TMPDIR': '/tmp'
 260             },
 261             'name': 'test_initial_work_dir',
 262             'runtime_constraints': {
 263                 'vcpus': 1,
 264                 'ram': 1073741824
 265             },
 266             'use_existing': True,
 267             'priority': 1,
 268             'mounts': {
 269                 '/tmp': {'kind': 'tmp',
 270                          "capacity": 1073741824 },
 271                 '/var/spool/cwl': {'kind': 'tmp',
 272                                    "capacity": 1073741824 },
 273                 '/var/spool/cwl/foo': {
 274                     'kind': 'collection',
 275                     'path': 'foo',
 276                     'portable_data_hash': '99999999999999999999999999999996+99'
 277                 },
 278                 '/var/spool/cwl/foo2': {
 279                     'kind': 'collection',
 280                     'path': 'foo2',
 281                     'portable_data_hash': '99999999999999999999999999999996+99'
 282                 },
 283                 '/var/spool/cwl/filename': {
 284                     'kind': 'collection',
 285                     'path': 'filename',
 286                     'portable_data_hash': '99999999999999999999999999999996+99'
 287                 },
 288                 '/var/spool/cwl/subdir': {
 289                     'kind': 'collection',
 290                     'path': 'subdir',
 291                     'portable_data_hash': '99999999999999999999999999999996+99'
 292                 }
 293             },
 294             'state': 'Committed',
 295             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 296             'output_path': '/var/spool/cwl',
 297             'output_ttl': 0,
 298             'container_image': 'arvados/jobs',
 299             'command': ['ls'],
 300             'cwd': '/var/spool/cwl',
 301             'scheduling_parameters': {
 302             },
 303             'properties': {}
 304         }
 305
 306         call_body = call_kwargs.get('body', None)
 307         self.assertNotEqual(None, call_body)
 308         for key in call_body:
 309             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 310
 311
 312     # Test redirecting stdin/stdout/stderr
 313     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 314     def test_redirects(self, keepdocker):
 315         arv_docker_clear_cache()
 316
 317         runner = mock.MagicMock()
 318         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 319         runner.ignore_docker_for_reuse = False
 320         runner.intermediate_output_ttl = 0
 321
 322         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 323         runner.api.collections().get().execute.return_value = {
 324             "portable_data_hash": "99999999999999999999999999999993+99"}
 325
 326         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 327
 328         tool = cmap({
 329             "inputs": [],
 330             "outputs": [],
 331             "baseCommand": "ls",
 332             "stdout": "stdout.txt",
 333             "stderr": "stderr.txt",
 334             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 335             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 336             "id": "#"
 337         })
 338         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 339                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 340         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 341                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 342         arvtool.formatgraph = None
 343         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 344                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 345             j.run()
 346             runner.api.container_requests().create.assert_called_with(
 347                 body=JsonDiffMatcher({
 348                     'environment': {
 349                         'HOME': '/var/spool/cwl',
 350                         'TMPDIR': '/tmp'
 351                     },
 352                     'name': 'test_run_redirect',
 353                     'runtime_constraints': {
 354                         'vcpus': 1,
 355                         'ram': 1073741824
 356                     },
 357                     'use_existing': True,
 358                     'priority': 1,
 359                     'mounts': {
 360                         '/tmp': {'kind': 'tmp',
 361                                  "capacity": 1073741824 },
 362                         '/var/spool/cwl': {'kind': 'tmp',
 363                                            "capacity": 1073741824 },
 364                         "stderr": {
 365                             "kind": "file",
 366                             "path": "/var/spool/cwl/stderr.txt"
 367                         },
 368                         "stdin": {
 369                             "kind": "collection",
 370                             "path": "file.txt",
 371                             "portable_data_hash": "99999999999999999999999999999996+99"
 372                         },
 373                         "stdout": {
 374                             "kind": "file",
 375                             "path": "/var/spool/cwl/stdout.txt"
 376                         },
 377                     },
 378                     'state': 'Committed',
 379                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 380                     'output_path': '/var/spool/cwl',
 381                     'output_ttl': 0,
 382                     'container_image': 'arvados/jobs',
 383                     'command': ['ls', '/var/spool/cwl'],
 384                     'cwd': '/var/spool/cwl',
 385                     'scheduling_parameters': {},
 386                     'properties': {},
 387                 }))
 388
 389     @mock.patch("arvados.collection.Collection")
 390     def test_done(self, col):
 391         api = mock.MagicMock()
 392
 393         runner = mock.MagicMock()
 394         runner.api = api
 395         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 396         runner.num_retries = 0
 397         runner.ignore_docker_for_reuse = False
 398         runner.intermediate_output_ttl = 0
 399
 400         runner.api.containers().get().execute.return_value = {"state":"Complete",
 401                                                               "output": "abc+123",
 402                                                               "exit_code": 0}
 403
 404         col().open.return_value = []
 405
 406         arvjob = arvados_cwl.ArvadosContainer(runner)
 407         arvjob.name = "testjob"
 408         arvjob.builder = mock.MagicMock()
 409         arvjob.output_callback = mock.MagicMock()
 410         arvjob.collect_outputs = mock.MagicMock()
 411         arvjob.successCodes = [0]
 412         arvjob.outdir = "/var/spool/cwl"
 413         arvjob.output_ttl = 3600
 414
 415         arvjob.collect_outputs.return_value = {"out": "stuff"}
 416
 417         arvjob.done({
 418             "state": "Final",
 419             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 420             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 421             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 422             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 423             "modified_at": "2017-05-26T12:01:22Z"
 424         })
 425
 426         self.assertFalse(api.collections().create.called)
 427
 428         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 429         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 430         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 431
 432     # The test passes no builder.resources
 433     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 434     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 435     def test_mounts(self, keepdocker):
 436         arv_docker_clear_cache()
 437
 438         runner = mock.MagicMock()
 439         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 440         runner.ignore_docker_for_reuse = False
 441         runner.intermediate_output_ttl = 0
 442
 443         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 444         runner.api.collections().get().execute.return_value = {
 445             "portable_data_hash": "99999999999999999999999999999993+99"}
 446
 447         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 448
 449         tool = cmap({
 450             "inputs": [
 451                 {"id": "p1",
 452                  "type": "Directory"}
 453             ],
 454             "outputs": [],
 455             "baseCommand": "ls",
 456             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
 457             "id": "#"
 458         })
 459         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 460                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 461         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 462                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 463         arvtool.formatgraph = None
 464         job_order = {
 465             "p1": {
 466                 "class": "Directory",
 467                 "location": "keep:99999999999999999999999999999994+44",
 468                 "listing": [
 469                     {
 470                         "class": "File",
 471                         "location": "keep:99999999999999999999999999999994+44/file1",
 472                     },
 473                     {
 474                         "class": "File",
 475                         "location": "keep:99999999999999999999999999999994+44/file2",
 476                     }
 477                 ]
 478             }
 479         }
 480         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 481                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 482             j.run()
 483             runner.api.container_requests().create.assert_called_with(
 484                 body=JsonDiffMatcher({
 485                     'environment': {
 486                         'HOME': '/var/spool/cwl',
 487                         'TMPDIR': '/tmp'
 488                     },
 489                     'name': 'test_run_mounts',
 490                     'runtime_constraints': {
 491                         'vcpus': 1,
 492                         'ram': 1073741824
 493                     },
 494                     'use_existing': True,
 495                     'priority': 1,
 496                     'mounts': {
 497                         "/keep/99999999999999999999999999999994+44": {
 498                             "kind": "collection",
 499                             "portable_data_hash": "99999999999999999999999999999994+44"
 500                         },
 501                         '/tmp': {'kind': 'tmp',
 502                                  "capacity": 1073741824 },
 503                         '/var/spool/cwl': {'kind': 'tmp',
 504                                            "capacity": 1073741824 }
 505                     },
 506                     'state': 'Committed',
 507                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 508                     'output_path': '/var/spool/cwl',
 509                     'output_ttl': 0,
 510                     'container_image': 'arvados/jobs',
 511                     'command': ['ls', '/var/spool/cwl'],
 512                     'cwd': '/var/spool/cwl',
 513                     'scheduling_parameters': {},
 514                     'properties': {},
 515                 }))