sdk/cwl/tests/test_container.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados_cwl
   6 from arvados_cwl.arvdocker import arv_docker_clear_cache
   7 import logging
   8 import mock
   9 import unittest
  10 import os
  11 import functools
  12 import cwltool.process
  13 from schema_salad.ref_resolver import Loader
  14 from schema_salad.sourceline import cmap
  15
  16 from .matcher import JsonDiffMatcher
  17
  18 if not os.getenv('ARVADOS_DEBUG'):
  19     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  20     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  21
  22
  23 class TestContainer(unittest.TestCase):
  24
  25     # The test passes no builder.resources
  26     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  27     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  28     def test_run(self, keepdocker):
  29         for enable_reuse in (True, False):
  30             arv_docker_clear_cache()
  31
  32             runner = mock.MagicMock()
  33             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  34             runner.ignore_docker_for_reuse = False
  35             runner.intermediate_output_ttl = 0
  36
  37             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  38             runner.api.collections().get().execute.return_value = {
  39                 "portable_data_hash": "99999999999999999999999999999993+99"}
  40
  41             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  42
  43             tool = cmap({
  44                 "inputs": [],
  45                 "outputs": [],
  46                 "baseCommand": "ls",
  47                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  48             })
  49             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  50                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  51             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  52                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  53             arvtool.formatgraph = None
  54             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  55                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  56                 j.run(enable_reuse=enable_reuse)
  57                 runner.api.container_requests().create.assert_called_with(
  58                     body=JsonDiffMatcher({
  59                         'environment': {
  60                             'HOME': '/var/spool/cwl',
  61                             'TMPDIR': '/tmp'
  62                         },
  63                         'name': 'test_run_'+str(enable_reuse),
  64                         'runtime_constraints': {
  65                             'vcpus': 1,
  66                             'ram': 1073741824
  67                         },
  68                         'use_existing': enable_reuse,
  69                         'priority': 1,
  70                         'mounts': {
  71                             '/tmp': {'kind': 'tmp',
  72                                      "capacity": 1073741824
  73                                  },
  74                             '/var/spool/cwl': {'kind': 'tmp',
  75                                                "capacity": 1073741824 }
  76                         },
  77                         'state': 'Committed',
  78                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  79                         'output_path': '/var/spool/cwl',
  80                         'output_ttl': 0,
  81                         'container_image': 'arvados/jobs',
  82                         'command': ['ls', '/var/spool/cwl'],
  83                         'cwd': '/var/spool/cwl',
  84                         'scheduling_parameters': {},
  85                         'properties': {},
  86                     }))
  87
  88     # The test passes some fields in builder.resources
  89     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  90     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  91     def test_resource_requirements(self, keepdocker):
  92         arv_docker_clear_cache()
  93         runner = mock.MagicMock()
  94         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  95         runner.ignore_docker_for_reuse = False
  96         runner.intermediate_output_ttl = 3600
  97         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  98
  99         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 100         runner.api.collections().get().execute.return_value = {
 101             "portable_data_hash": "99999999999999999999999999999993+99"}
 102
 103         tool = cmap({
 104             "inputs": [],
 105             "outputs": [],
 106             "hints": [{
 107                 "class": "ResourceRequirement",
 108                 "coresMin": 3,
 109                 "ramMin": 3000,
 110                 "tmpdirMin": 4000,
 111                 "outdirMin": 5000
 112             }, {
 113                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 114                 "keep_cache": 512
 115             }, {
 116                 "class": "http://arvados.org/cwl#APIRequirement",
 117             }, {
 118                 "class": "http://arvados.org/cwl#PartitionRequirement",
 119                 "partition": "blurb"
 120             }, {
 121                 "class": "http://arvados.org/cwl#IntermediateOutput",
 122                 "outputTTL": 7200
 123             }, {
 124                 "class": "http://arvados.org/cwl#ReuseRequirement",
 125                 "enableReuse": False
 126             }],
 127             "baseCommand": "ls"
 128         })
 129         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 130                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 131         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 132                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 133                                                  loader=Loader({}))
 134         arvtool.formatgraph = None
 135         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 136                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 137             j.run(enable_reuse=True)
 138
 139         call_args, call_kwargs = runner.api.container_requests().create.call_args
 140
 141         call_body_expected = {
 142             'environment': {
 143                 'HOME': '/var/spool/cwl',
 144                 'TMPDIR': '/tmp'
 145             },
 146             'name': 'test_resource_requirements',
 147             'runtime_constraints': {
 148                 'vcpus': 3,
 149                 'ram': 3145728000,
 150                 'keep_cache_ram': 536870912,
 151                 'API': True
 152             },
 153             'use_existing': False,
 154             'priority': 1,
 155             'mounts': {
 156                 '/tmp': {'kind': 'tmp',
 157                          "capacity": 4194304000 },
 158                 '/var/spool/cwl': {'kind': 'tmp',
 159                                    "capacity": 5242880000 }
 160             },
 161             'state': 'Committed',
 162             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 163             'output_path': '/var/spool/cwl',
 164             'output_ttl': 7200,
 165             'container_image': 'arvados/jobs',
 166             'command': ['ls'],
 167             'cwd': '/var/spool/cwl',
 168             'scheduling_parameters': {
 169                 'partitions': ['blurb']
 170             },
 171             'properties': {}
 172         }
 173
 174         call_body = call_kwargs.get('body', None)
 175         self.assertNotEqual(None, call_body)
 176         for key in call_body:
 177             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 178
 179
 180     # The test passes some fields in builder.resources
 181     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 182     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 183     @mock.patch("arvados.collection.Collection")
 184     def test_initial_work_dir(self, collection_mock, keepdocker):
 185         arv_docker_clear_cache()
 186         runner = mock.MagicMock()
 187         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 188         runner.ignore_docker_for_reuse = False
 189         runner.intermediate_output_ttl = 0
 190         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 191
 192         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 193         runner.api.collections().get().execute.return_value = {
 194             "portable_data_hash": "99999999999999999999999999999993+99"}
 195
 196         sourcemock = mock.MagicMock()
 197         def get_collection_mock(p):
 198             if "/" in p:
 199                 return (sourcemock, p.split("/", 1)[1])
 200             else:
 201                 return (sourcemock, "")
 202         runner.fs_access.get_collection.side_effect = get_collection_mock
 203
 204         vwdmock = mock.MagicMock()
 205         collection_mock.return_value = vwdmock
 206         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 207
 208         tool = cmap({
 209             "inputs": [],
 210             "outputs": [],
 211             "hints": [{
 212                 "class": "InitialWorkDirRequirement",
 213                 "listing": [{
 214                     "class": "File",
 215                     "basename": "foo",
 216                     "location": "keep:99999999999999999999999999999995+99/bar"
 217                 },
 218                 {
 219                     "class": "Directory",
 220                     "basename": "foo2",
 221                     "location": "keep:99999999999999999999999999999995+99"
 222                 },
 223                 {
 224                     "class": "File",
 225                     "basename": "filename",
 226                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 227                 },
 228                 {
 229                     "class": "Directory",
 230                     "basename": "subdir",
 231                     "location": "keep:99999999999999999999999999999995+99/subdir"
 232                 }                        ]
 233             }],
 234             "baseCommand": "ls"
 235         })
 236         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 237                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 238         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 239                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 240                                                  loader=Loader({}))
 241         arvtool.formatgraph = None
 242         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 243                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 244             j.run()
 245
 246         call_args, call_kwargs = runner.api.container_requests().create.call_args
 247
 248         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 249         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 250         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 251         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 252
 253         call_body_expected = {
 254             'environment': {
 255                 'HOME': '/var/spool/cwl',
 256                 'TMPDIR': '/tmp'
 257             },
 258             'name': 'test_initial_work_dir',
 259             'runtime_constraints': {
 260                 'vcpus': 1,
 261                 'ram': 1073741824
 262             },
 263             'use_existing': True,
 264             'priority': 1,
 265             'mounts': {
 266                 '/tmp': {'kind': 'tmp',
 267                          "capacity": 1073741824 },
 268                 '/var/spool/cwl': {'kind': 'tmp',
 269                                    "capacity": 1073741824 },
 270                 '/var/spool/cwl/foo': {
 271                     'kind': 'collection',
 272                     'path': 'foo',
 273                     'portable_data_hash': '99999999999999999999999999999996+99'
 274                 },
 275                 '/var/spool/cwl/foo2': {
 276                     'kind': 'collection',
 277                     'path': 'foo2',
 278                     'portable_data_hash': '99999999999999999999999999999996+99'
 279                 },
 280                 '/var/spool/cwl/filename': {
 281                     'kind': 'collection',
 282                     'path': 'filename',
 283                     'portable_data_hash': '99999999999999999999999999999996+99'
 284                 },
 285                 '/var/spool/cwl/subdir': {
 286                     'kind': 'collection',
 287                     'path': 'subdir',
 288                     'portable_data_hash': '99999999999999999999999999999996+99'
 289                 }
 290             },
 291             'state': 'Committed',
 292             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 293             'output_path': '/var/spool/cwl',
 294             'output_ttl': 0,
 295             'container_image': 'arvados/jobs',
 296             'command': ['ls'],
 297             'cwd': '/var/spool/cwl',
 298             'scheduling_parameters': {
 299             },
 300             'properties': {}
 301         }
 302
 303         call_body = call_kwargs.get('body', None)
 304         self.assertNotEqual(None, call_body)
 305         for key in call_body:
 306             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 307
 308
 309     # Test redirecting stdin/stdout/stderr
 310     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 311     def test_redirects(self, keepdocker):
 312         arv_docker_clear_cache()
 313
 314         runner = mock.MagicMock()
 315         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 316         runner.ignore_docker_for_reuse = False
 317         runner.intermediate_output_ttl = 0
 318
 319         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 320         runner.api.collections().get().execute.return_value = {
 321             "portable_data_hash": "99999999999999999999999999999993+99"}
 322
 323         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 324
 325         tool = cmap({
 326             "inputs": [],
 327             "outputs": [],
 328             "baseCommand": "ls",
 329             "stdout": "stdout.txt",
 330             "stderr": "stderr.txt",
 331             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 332             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 333         })
 334         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 335                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 336         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 337                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 338         arvtool.formatgraph = None
 339         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 340                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 341             j.run()
 342             runner.api.container_requests().create.assert_called_with(
 343                 body=JsonDiffMatcher({
 344                     'environment': {
 345                         'HOME': '/var/spool/cwl',
 346                         'TMPDIR': '/tmp'
 347                     },
 348                     'name': 'test_run_redirect',
 349                     'runtime_constraints': {
 350                         'vcpus': 1,
 351                         'ram': 1073741824
 352                     },
 353                     'use_existing': True,
 354                     'priority': 1,
 355                     'mounts': {
 356                         '/tmp': {'kind': 'tmp',
 357                                  "capacity": 1073741824 },
 358                         '/var/spool/cwl': {'kind': 'tmp',
 359                                            "capacity": 1073741824 },
 360                         "stderr": {
 361                             "kind": "file",
 362                             "path": "/var/spool/cwl/stderr.txt"
 363                         },
 364                         "stdin": {
 365                             "kind": "collection",
 366                             "path": "file.txt",
 367                             "portable_data_hash": "99999999999999999999999999999996+99"
 368                         },
 369                         "stdout": {
 370                             "kind": "file",
 371                             "path": "/var/spool/cwl/stdout.txt"
 372                         },
 373                     },
 374                     'state': 'Committed',
 375                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 376                     'output_path': '/var/spool/cwl',
 377                     'output_ttl': 0,
 378                     'container_image': 'arvados/jobs',
 379                     'command': ['ls', '/var/spool/cwl'],
 380                     'cwd': '/var/spool/cwl',
 381                     'scheduling_parameters': {},
 382                     'properties': {},
 383                 }))
 384
 385     @mock.patch("arvados.collection.Collection")
 386     def test_done(self, col):
 387         api = mock.MagicMock()
 388
 389         runner = mock.MagicMock()
 390         runner.api = api
 391         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 392         runner.num_retries = 0
 393         runner.ignore_docker_for_reuse = False
 394         runner.intermediate_output_ttl = 0
 395
 396         runner.api.containers().get().execute.return_value = {"state":"Complete",
 397                                                               "output": "abc+123",
 398                                                               "exit_code": 0}
 399
 400         col().open.return_value = []
 401
 402         arvjob = arvados_cwl.ArvadosContainer(runner)
 403         arvjob.name = "testjob"
 404         arvjob.builder = mock.MagicMock()
 405         arvjob.output_callback = mock.MagicMock()
 406         arvjob.collect_outputs = mock.MagicMock()
 407         arvjob.successCodes = [0]
 408         arvjob.outdir = "/var/spool/cwl"
 409         arvjob.output_ttl = 3600
 410
 411         arvjob.collect_outputs.return_value = {"out": "stuff"}
 412
 413         arvjob.done({
 414             "state": "Final",
 415             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 416             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 417             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 418             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 419             "modified_at": "2017-05-26T12:01:22Z"
 420         })
 421
 422         self.assertFalse(api.collections().create.called)
 423
 424         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 425         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 426         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 427
 428     # The test passes no builder.resources
 429     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 430     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 431     def test_mounts(self, keepdocker):
 432         arv_docker_clear_cache()
 433
 434         runner = mock.MagicMock()
 435         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 436         runner.ignore_docker_for_reuse = False
 437         runner.intermediate_output_ttl = 0
 438
 439         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 440         runner.api.collections().get().execute.return_value = {
 441             "portable_data_hash": "99999999999999999999999999999993+99"}
 442
 443         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 444
 445         tool = cmap({
 446             "inputs": [
 447                 {"id": "p1",
 448                  "type": "Directory"}
 449             ],
 450             "outputs": [],
 451             "baseCommand": "ls",
 452             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 453         })
 454         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 455                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 456         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 457                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 458         arvtool.formatgraph = None
 459         job_order = {
 460             "p1": {
 461                 "class": "Directory",
 462                 "location": "keep:99999999999999999999999999999994+44",
 463                 "listing": [
 464                     {
 465                         "class": "File",
 466                         "location": "keep:99999999999999999999999999999994+44/file1",
 467                     },
 468                     {
 469                         "class": "File",
 470                         "location": "keep:99999999999999999999999999999994+44/file2",
 471                     }
 472                 ]
 473             }
 474         }
 475         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 476                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 477             j.run()
 478             runner.api.container_requests().create.assert_called_with(
 479                 body=JsonDiffMatcher({
 480                     'environment': {
 481                         'HOME': '/var/spool/cwl',
 482                         'TMPDIR': '/tmp'
 483                     },
 484                     'name': 'test_run_mounts',
 485                     'runtime_constraints': {
 486                         'vcpus': 1,
 487                         'ram': 1073741824
 488                     },
 489                     'use_existing': True,
 490                     'priority': 1,
 491                     'mounts': {
 492                         "/keep/99999999999999999999999999999994+44": {
 493                             "kind": "collection",
 494                             "portable_data_hash": "99999999999999999999999999999994+44"
 495                         },
 496                         '/tmp': {'kind': 'tmp',
 497                                  "capacity": 1073741824 },
 498                         '/var/spool/cwl': {'kind': 'tmp',
 499                                            "capacity": 1073741824 }
 500                     },
 501                     'state': 'Committed',
 502                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 503                     'output_path': '/var/spool/cwl',
 504                     'output_ttl': 0,
 505                     'container_image': 'arvados/jobs',
 506                     'command': ['ls', '/var/spool/cwl'],
 507                     'cwd': '/var/spool/cwl',
 508                     'scheduling_parameters': {},
 509                     'properties': {},
 510                 }))