sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10 from schema_salad.sourceline import cmap
  11
  12 from .matcher import JsonDiffMatcher
  13
  14 if not os.getenv('ARVADOS_DEBUG'):
  15     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  16     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  17
  18
  19 class TestContainer(unittest.TestCase):
  20
  21     # The test passes no builder.resources
  22     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     def test_run(self, keepdocker):
  25         for enable_reuse in (True, False):
  26             arv_docker_clear_cache()
  27
  28             runner = mock.MagicMock()
  29             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  30             runner.ignore_docker_for_reuse = False
  31
  32             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  33             runner.api.collections().get().execute.return_value = {
  34                 "portable_data_hash": "99999999999999999999999999999993+99"}
  35
  36             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  37
  38             tool = cmap({
  39                 "inputs": [],
  40                 "outputs": [],
  41                 "baseCommand": "ls",
  42                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  43             })
  44             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  45                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  46             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  47                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  48             arvtool.formatgraph = None
  49             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  50                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  51                 j.run(enable_reuse=enable_reuse)
  52                 runner.api.container_requests().create.assert_called_with(
  53                     body=JsonDiffMatcher({
  54                         'environment': {
  55                             'HOME': '/var/spool/cwl',
  56                             'TMPDIR': '/tmp'
  57                         },
  58                         'name': 'test_run_'+str(enable_reuse),
  59                         'runtime_constraints': {
  60                             'vcpus': 1,
  61                             'ram': 1073741824
  62                         },
  63                         'use_existing': enable_reuse,
  64                         'priority': 1,
  65                         'mounts': {
  66                             '/tmp': {'kind': 'tmp'},
  67                             '/var/spool/cwl': {'kind': 'tmp'}
  68                         },
  69                         'state': 'Committed',
  70                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  71                         'output_path': '/var/spool/cwl',
  72                         'container_image': 'arvados/jobs',
  73                         'command': ['ls', '/var/spool/cwl'],
  74                         'cwd': '/var/spool/cwl',
  75                         'scheduling_parameters': {},
  76                         'properties': {},
  77                     }))
  78
  79     # The test passes some fields in builder.resources
  80     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  81     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  82     def test_resource_requirements(self, keepdocker):
  83         arv_docker_clear_cache()
  84         runner = mock.MagicMock()
  85         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  86         runner.ignore_docker_for_reuse = False
  87         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  88
  89         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  90         runner.api.collections().get().execute.return_value = {
  91             "portable_data_hash": "99999999999999999999999999999993+99"}
  92
  93         tool = cmap({
  94             "inputs": [],
  95             "outputs": [],
  96             "hints": [{
  97                 "class": "ResourceRequirement",
  98                 "coresMin": 3,
  99                 "ramMin": 3000,
 100                 "tmpdirMin": 4000
 101             }, {
 102                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 103                 "keep_cache": 512
 104             }, {
 105                 "class": "http://arvados.org/cwl#APIRequirement",
 106             }, {
 107                 "class": "http://arvados.org/cwl#PartitionRequirement",
 108                 "partition": "blurb"
 109             }],
 110             "baseCommand": "ls"
 111         })
 112         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 113                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 114         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 115                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 116                                                  loader=Loader({}))
 117         arvtool.formatgraph = None
 118         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 119                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 120             j.run()
 121
 122         call_args, call_kwargs = runner.api.container_requests().create.call_args
 123
 124         call_body_expected = {
 125                 'environment': {
 126                     'HOME': '/var/spool/cwl',
 127                     'TMPDIR': '/tmp'
 128                 },
 129                 'name': 'test_resource_requirements',
 130                 'runtime_constraints': {
 131                     'vcpus': 3,
 132                     'ram': 3145728000,
 133                     'keep_cache_ram': 536870912,
 134                     'API': True
 135                 },
 136                 'use_existing': True,
 137                 'priority': 1,
 138                 'mounts': {
 139                     '/tmp': {'kind': 'tmp'},
 140                     '/var/spool/cwl': {'kind': 'tmp'}
 141                 },
 142                 'state': 'Committed',
 143                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 144                 'output_path': '/var/spool/cwl',
 145                 'container_image': 'arvados/jobs',
 146                 'command': ['ls'],
 147                 'cwd': '/var/spool/cwl',
 148                 'scheduling_parameters': {
 149                     'partitions': ['blurb']
 150                 },
 151                 'properties': {}
 152         }
 153
 154         call_body = call_kwargs.get('body', None)
 155         self.assertNotEqual(None, call_body)
 156         for key in call_body:
 157             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 158
 159
 160     # The test passes some fields in builder.resources
 161     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 162     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 163     @mock.patch("arvados.collection.Collection")
 164     def test_initial_work_dir(self, collection_mock, keepdocker):
 165         arv_docker_clear_cache()
 166         runner = mock.MagicMock()
 167         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 168         runner.ignore_docker_for_reuse = False
 169         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 170
 171         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 172         runner.api.collections().get().execute.return_value = {
 173             "portable_data_hash": "99999999999999999999999999999993+99"}
 174
 175         sourcemock = mock.MagicMock()
 176         def get_collection_mock(p):
 177             if "/" in p:
 178                 return (sourcemock, p.split("/", 1)[1])
 179             else:
 180                 return (sourcemock, "")
 181         runner.fs_access.get_collection.side_effect = get_collection_mock
 182
 183         vwdmock = mock.MagicMock()
 184         collection_mock.return_value = vwdmock
 185         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 186
 187         tool = cmap({
 188             "inputs": [],
 189             "outputs": [],
 190             "hints": [{
 191                 "class": "InitialWorkDirRequirement",
 192                 "listing": [{
 193                     "class": "File",
 194                     "basename": "foo",
 195                     "location": "keep:99999999999999999999999999999995+99/bar"
 196                 },
 197                 {
 198                     "class": "Directory",
 199                     "basename": "foo2",
 200                     "location": "keep:99999999999999999999999999999995+99"
 201                 },
 202                 {
 203                     "class": "File",
 204                     "basename": "filename",
 205                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 206                 },
 207                 {
 208                     "class": "Directory",
 209                     "basename": "subdir",
 210                     "location": "keep:99999999999999999999999999999995+99/subdir"
 211                 }                        ]
 212             }],
 213             "baseCommand": "ls"
 214         })
 215         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 216                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 217         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 218                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 219                                                  loader=Loader({}))
 220         arvtool.formatgraph = None
 221         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 222                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 223             j.run()
 224
 225         call_args, call_kwargs = runner.api.container_requests().create.call_args
 226
 227         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 228         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 229         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 230         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 231
 232         call_body_expected = {
 233                 'environment': {
 234                     'HOME': '/var/spool/cwl',
 235                     'TMPDIR': '/tmp'
 236                 },
 237                 'name': 'test_initial_work_dir',
 238                 'runtime_constraints': {
 239                     'vcpus': 1,
 240                     'ram': 1073741824
 241                 },
 242                 'use_existing': True,
 243                 'priority': 1,
 244                 'mounts': {
 245                     '/tmp': {'kind': 'tmp'},
 246                     '/var/spool/cwl': {'kind': 'tmp'},
 247                     '/var/spool/cwl/foo': {
 248                         'kind': 'collection',
 249                         'path': 'foo',
 250                         'portable_data_hash': '99999999999999999999999999999996+99'
 251                     },
 252                     '/var/spool/cwl/foo2': {
 253                         'kind': 'collection',
 254                         'path': 'foo2',
 255                         'portable_data_hash': '99999999999999999999999999999996+99'
 256                     },
 257                     '/var/spool/cwl/filename': {
 258                         'kind': 'collection',
 259                         'path': 'filename',
 260                         'portable_data_hash': '99999999999999999999999999999996+99'
 261                     },
 262                     '/var/spool/cwl/subdir': {
 263                         'kind': 'collection',
 264                         'path': 'subdir',
 265                         'portable_data_hash': '99999999999999999999999999999996+99'
 266                     }
 267                 },
 268                 'state': 'Committed',
 269                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 270                 'output_path': '/var/spool/cwl',
 271                 'container_image': 'arvados/jobs',
 272                 'command': ['ls'],
 273                 'cwd': '/var/spool/cwl',
 274                 'scheduling_parameters': {
 275                 },
 276                 'properties': {}
 277         }
 278
 279         call_body = call_kwargs.get('body', None)
 280         self.assertNotEqual(None, call_body)
 281         for key in call_body:
 282             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 283
 284
 285     # Test redirecting stdin/stdout/stderr
 286     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 287     def test_redirects(self, keepdocker):
 288         arv_docker_clear_cache()
 289
 290         runner = mock.MagicMock()
 291         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 292         runner.ignore_docker_for_reuse = False
 293
 294         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 295         runner.api.collections().get().execute.return_value = {
 296             "portable_data_hash": "99999999999999999999999999999993+99"}
 297
 298         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 299
 300         tool = cmap({
 301             "inputs": [],
 302             "outputs": [],
 303             "baseCommand": "ls",
 304             "stdout": "stdout.txt",
 305             "stderr": "stderr.txt",
 306             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 307             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 308         })
 309         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 310                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 311         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 312                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 313         arvtool.formatgraph = None
 314         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 315                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 316             j.run()
 317             runner.api.container_requests().create.assert_called_with(
 318                 body=JsonDiffMatcher({
 319                     'environment': {
 320                         'HOME': '/var/spool/cwl',
 321                         'TMPDIR': '/tmp'
 322                     },
 323                     'name': 'test_run_redirect',
 324                     'runtime_constraints': {
 325                         'vcpus': 1,
 326                         'ram': 1073741824
 327                     },
 328                     'use_existing': True,
 329                     'priority': 1,
 330                     'mounts': {
 331                         '/tmp': {'kind': 'tmp'},
 332                         '/var/spool/cwl': {'kind': 'tmp'},
 333                         "stderr": {
 334                             "kind": "file",
 335                             "path": "/var/spool/cwl/stderr.txt"
 336                         },
 337                         "stdin": {
 338                             "kind": "collection",
 339                             "path": "file.txt",
 340                             "portable_data_hash": "99999999999999999999999999999996+99"
 341                         },
 342                         "stdout": {
 343                             "kind": "file",
 344                             "path": "/var/spool/cwl/stdout.txt"
 345                         },
 346                     },
 347                     'state': 'Committed',
 348                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 349                     'output_path': '/var/spool/cwl',
 350                     'container_image': 'arvados/jobs',
 351                     'command': ['ls', '/var/spool/cwl'],
 352                     'cwd': '/var/spool/cwl',
 353                     'scheduling_parameters': {},
 354                     'properties': {},
 355                 }))
 356
 357     @mock.patch("arvados.collection.Collection")
 358     def test_done(self, col):
 359         api = mock.MagicMock()
 360
 361         runner = mock.MagicMock()
 362         runner.api = api
 363         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 364         runner.num_retries = 0
 365         runner.ignore_docker_for_reuse = False
 366
 367         runner.api.containers().get().execute.return_value = {"state":"Complete",
 368                                                               "output": "abc+123",
 369                                                               "exit_code": 0}
 370
 371         col().open.return_value = []
 372
 373         arvjob = arvados_cwl.ArvadosContainer(runner)
 374         arvjob.name = "testjob"
 375         arvjob.builder = mock.MagicMock()
 376         arvjob.output_callback = mock.MagicMock()
 377         arvjob.collect_outputs = mock.MagicMock()
 378         arvjob.successCodes = [0]
 379         arvjob.outdir = "/var/spool/cwl"
 380
 381         arvjob.collect_outputs.return_value = {"out": "stuff"}
 382
 383         arvjob.done({
 384             "state": "Final",
 385             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 386             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 387             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 388             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 389         })
 390
 391         self.assertFalse(api.collections().create.called)
 392
 393         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 394         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 395
 396     # The test passes no builder.resources
 397     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 398     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 399     def test_mounts(self, keepdocker):
 400         arv_docker_clear_cache()
 401
 402         runner = mock.MagicMock()
 403         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 404         runner.ignore_docker_for_reuse = False
 405
 406         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 407         runner.api.collections().get().execute.return_value = {
 408             "portable_data_hash": "99999999999999999999999999999993+99"}
 409
 410         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 411
 412         tool = cmap({
 413             "inputs": [
 414                 {"id": "p1",
 415                  "type": "Directory"}
 416             ],
 417             "outputs": [],
 418             "baseCommand": "ls",
 419             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 420         })
 421         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 422                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 423         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 424                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 425         arvtool.formatgraph = None
 426         job_order = {
 427             "p1": {
 428                 "class": "Directory",
 429                 "location": "keep:99999999999999999999999999999994+44",
 430                 "listing": [
 431                     {
 432                         "class": "File",
 433                         "location": "keep:99999999999999999999999999999994+44/file1",
 434                     },
 435                     {
 436                         "class": "File",
 437                         "location": "keep:99999999999999999999999999999994+44/file2",
 438                     }
 439                 ]
 440             }
 441         }
 442         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 443                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 444             j.run()
 445             runner.api.container_requests().create.assert_called_with(
 446                 body=JsonDiffMatcher({
 447                     'environment': {
 448                         'HOME': '/var/spool/cwl',
 449                         'TMPDIR': '/tmp'
 450                     },
 451                     'name': 'test_run_mounts',
 452                     'runtime_constraints': {
 453                         'vcpus': 1,
 454                         'ram': 1073741824
 455                     },
 456                     'use_existing': True,
 457                     'priority': 1,
 458                     'mounts': {
 459                         "/keep/99999999999999999999999999999994+44": {
 460                             "kind": "collection",
 461                             "portable_data_hash": "99999999999999999999999999999994+44"
 462                         },
 463                         '/tmp': {'kind': 'tmp'},
 464                         '/var/spool/cwl': {'kind': 'tmp'}
 465                     },
 466                     'state': 'Committed',
 467                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 468                     'output_path': '/var/spool/cwl',
 469                     'container_image': 'arvados/jobs',
 470                     'command': ['ls', '/var/spool/cwl'],
 471                     'cwd': '/var/spool/cwl',
 472                     'scheduling_parameters': {},
 473                     'properties': {},
 474                 }))