sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10 from schema_salad.sourceline import cmap
  11
  12 from .matcher import JsonDiffMatcher
  13
  14 if not os.getenv('ARVADOS_DEBUG'):
  15     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  16     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  17
  18
  19 class TestContainer(unittest.TestCase):
  20
  21     # The test passes no builder.resources
  22     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     def test_run(self, keepdocker):
  25         for enable_reuse in (True, False):
  26             arv_docker_clear_cache()
  27
  28             runner = mock.MagicMock()
  29             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  30             runner.ignore_docker_for_reuse = False
  31
  32             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  33             runner.api.collections().get().execute.return_value = {
  34                 "portable_data_hash": "99999999999999999999999999999993+99"}
  35
  36             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  37
  38             tool = cmap({
  39                 "inputs": [],
  40                 "outputs": [],
  41                 "baseCommand": "ls",
  42                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  43             })
  44             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  45                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  46             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  47                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  48             arvtool.formatgraph = None
  49             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  50                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  51                 j.run(enable_reuse=enable_reuse)
  52                 runner.api.container_requests().create.assert_called_with(
  53                     body=JsonDiffMatcher({
  54                         'environment': {
  55                             'HOME': '/var/spool/cwl',
  56                             'TMPDIR': '/tmp'
  57                         },
  58                         'name': 'test_run_'+str(enable_reuse),
  59                         'runtime_constraints': {
  60                             'vcpus': 1,
  61                             'ram': 1073741824
  62                         },
  63                         'use_existing': enable_reuse,
  64                         'priority': 1,
  65                         'mounts': {
  66                             '/tmp': {'kind': 'tmp',
  67                                      "capacity": 1073741824
  68                                  },
  69                             '/var/spool/cwl': {'kind': 'tmp',
  70                                                "capacity": 1073741824 }
  71                         },
  72                         'state': 'Committed',
  73                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  74                         'output_path': '/var/spool/cwl',
  75                         'container_image': 'arvados/jobs',
  76                         'command': ['ls', '/var/spool/cwl'],
  77                         'cwd': '/var/spool/cwl',
  78                         'scheduling_parameters': {},
  79                         'properties': {},
  80                     }))
  81
  82     # The test passes some fields in builder.resources
  83     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  84     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  85     def test_resource_requirements(self, keepdocker):
  86         arv_docker_clear_cache()
  87         runner = mock.MagicMock()
  88         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  89         runner.ignore_docker_for_reuse = False
  90         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  91
  92         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  93         runner.api.collections().get().execute.return_value = {
  94             "portable_data_hash": "99999999999999999999999999999993+99"}
  95
  96         tool = cmap({
  97             "inputs": [],
  98             "outputs": [],
  99             "hints": [{
 100                 "class": "ResourceRequirement",
 101                 "coresMin": 3,
 102                 "ramMin": 3000,
 103                 "tmpdirMin": 4000,
 104                 "outdirMin": 5000
 105             }, {
 106                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 107                 "keep_cache": 512
 108             }, {
 109                 "class": "http://arvados.org/cwl#APIRequirement",
 110             }, {
 111                 "class": "http://arvados.org/cwl#PartitionRequirement",
 112                 "partition": "blurb"
 113             }],
 114             "baseCommand": "ls"
 115         })
 116         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 117                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 118         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 119                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 120                                                  loader=Loader({}))
 121         arvtool.formatgraph = None
 122         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 123                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 124             j.run()
 125
 126         call_args, call_kwargs = runner.api.container_requests().create.call_args
 127
 128         call_body_expected = {
 129                 'environment': {
 130                     'HOME': '/var/spool/cwl',
 131                     'TMPDIR': '/tmp'
 132                 },
 133                 'name': 'test_resource_requirements',
 134                 'runtime_constraints': {
 135                     'vcpus': 3,
 136                     'ram': 3145728000,
 137                     'keep_cache_ram': 536870912,
 138                     'API': True
 139                 },
 140                 'use_existing': True,
 141                 'priority': 1,
 142                 'mounts': {
 143                     '/tmp': {'kind': 'tmp',
 144                              "capacity": 4194304000 },
 145                     '/var/spool/cwl': {'kind': 'tmp',
 146                                        "capacity": 5242880000 }
 147                 },
 148                 'state': 'Committed',
 149                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 150                 'output_path': '/var/spool/cwl',
 151                 'container_image': 'arvados/jobs',
 152                 'command': ['ls'],
 153                 'cwd': '/var/spool/cwl',
 154                 'scheduling_parameters': {
 155                     'partitions': ['blurb']
 156                 },
 157                 'properties': {}
 158         }
 159
 160         call_body = call_kwargs.get('body', None)
 161         self.assertNotEqual(None, call_body)
 162         for key in call_body:
 163             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 164
 165
 166     # The test passes some fields in builder.resources
 167     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 168     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 169     @mock.patch("arvados.collection.Collection")
 170     def test_initial_work_dir(self, collection_mock, keepdocker):
 171         arv_docker_clear_cache()
 172         runner = mock.MagicMock()
 173         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 174         runner.ignore_docker_for_reuse = False
 175         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 176
 177         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 178         runner.api.collections().get().execute.return_value = {
 179             "portable_data_hash": "99999999999999999999999999999993+99"}
 180
 181         sourcemock = mock.MagicMock()
 182         def get_collection_mock(p):
 183             if "/" in p:
 184                 return (sourcemock, p.split("/", 1)[1])
 185             else:
 186                 return (sourcemock, "")
 187         runner.fs_access.get_collection.side_effect = get_collection_mock
 188
 189         vwdmock = mock.MagicMock()
 190         collection_mock.return_value = vwdmock
 191         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 192
 193         tool = cmap({
 194             "inputs": [],
 195             "outputs": [],
 196             "hints": [{
 197                 "class": "InitialWorkDirRequirement",
 198                 "listing": [{
 199                     "class": "File",
 200                     "basename": "foo",
 201                     "location": "keep:99999999999999999999999999999995+99/bar"
 202                 },
 203                 {
 204                     "class": "Directory",
 205                     "basename": "foo2",
 206                     "location": "keep:99999999999999999999999999999995+99"
 207                 },
 208                 {
 209                     "class": "File",
 210                     "basename": "filename",
 211                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 212                 },
 213                 {
 214                     "class": "Directory",
 215                     "basename": "subdir",
 216                     "location": "keep:99999999999999999999999999999995+99/subdir"
 217                 }                        ]
 218             }],
 219             "baseCommand": "ls"
 220         })
 221         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 222                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 223         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 224                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 225                                                  loader=Loader({}))
 226         arvtool.formatgraph = None
 227         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 228                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 229             j.run()
 230
 231         call_args, call_kwargs = runner.api.container_requests().create.call_args
 232
 233         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 234         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 235         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 236         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 237
 238         call_body_expected = {
 239                 'environment': {
 240                     'HOME': '/var/spool/cwl',
 241                     'TMPDIR': '/tmp'
 242                 },
 243                 'name': 'test_initial_work_dir',
 244                 'runtime_constraints': {
 245                     'vcpus': 1,
 246                     'ram': 1073741824
 247                 },
 248                 'use_existing': True,
 249                 'priority': 1,
 250                 'mounts': {
 251                     '/tmp': {'kind': 'tmp',
 252                              "capacity": 1073741824 },
 253                     '/var/spool/cwl': {'kind': 'tmp',
 254                                        "capacity": 1073741824 },
 255                     '/var/spool/cwl/foo': {
 256                         'kind': 'collection',
 257                         'path': 'foo',
 258                         'portable_data_hash': '99999999999999999999999999999996+99'
 259                     },
 260                     '/var/spool/cwl/foo2': {
 261                         'kind': 'collection',
 262                         'path': 'foo2',
 263                         'portable_data_hash': '99999999999999999999999999999996+99'
 264                     },
 265                     '/var/spool/cwl/filename': {
 266                         'kind': 'collection',
 267                         'path': 'filename',
 268                         'portable_data_hash': '99999999999999999999999999999996+99'
 269                     },
 270                     '/var/spool/cwl/subdir': {
 271                         'kind': 'collection',
 272                         'path': 'subdir',
 273                         'portable_data_hash': '99999999999999999999999999999996+99'
 274                     }
 275                 },
 276                 'state': 'Committed',
 277                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 278                 'output_path': '/var/spool/cwl',
 279                 'container_image': 'arvados/jobs',
 280                 'command': ['ls'],
 281                 'cwd': '/var/spool/cwl',
 282                 'scheduling_parameters': {
 283                 },
 284                 'properties': {}
 285         }
 286
 287         call_body = call_kwargs.get('body', None)
 288         self.assertNotEqual(None, call_body)
 289         for key in call_body:
 290             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 291
 292
 293     # Test redirecting stdin/stdout/stderr
 294     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 295     def test_redirects(self, keepdocker):
 296         arv_docker_clear_cache()
 297
 298         runner = mock.MagicMock()
 299         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 300         runner.ignore_docker_for_reuse = False
 301
 302         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 303         runner.api.collections().get().execute.return_value = {
 304             "portable_data_hash": "99999999999999999999999999999993+99"}
 305
 306         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 307
 308         tool = cmap({
 309             "inputs": [],
 310             "outputs": [],
 311             "baseCommand": "ls",
 312             "stdout": "stdout.txt",
 313             "stderr": "stderr.txt",
 314             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 315             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 316         })
 317         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 318                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 319         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 320                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 321         arvtool.formatgraph = None
 322         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 323                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 324             j.run()
 325             runner.api.container_requests().create.assert_called_with(
 326                 body=JsonDiffMatcher({
 327                     'environment': {
 328                         'HOME': '/var/spool/cwl',
 329                         'TMPDIR': '/tmp'
 330                     },
 331                     'name': 'test_run_redirect',
 332                     'runtime_constraints': {
 333                         'vcpus': 1,
 334                         'ram': 1073741824
 335                     },
 336                     'use_existing': True,
 337                     'priority': 1,
 338                     'mounts': {
 339                         '/tmp': {'kind': 'tmp',
 340                                  "capacity": 1073741824 },
 341                         '/var/spool/cwl': {'kind': 'tmp',
 342                                            "capacity": 1073741824 },
 343                         "stderr": {
 344                             "kind": "file",
 345                             "path": "/var/spool/cwl/stderr.txt"
 346                         },
 347                         "stdin": {
 348                             "kind": "collection",
 349                             "path": "file.txt",
 350                             "portable_data_hash": "99999999999999999999999999999996+99"
 351                         },
 352                         "stdout": {
 353                             "kind": "file",
 354                             "path": "/var/spool/cwl/stdout.txt"
 355                         },
 356                     },
 357                     'state': 'Committed',
 358                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 359                     'output_path': '/var/spool/cwl',
 360                     'container_image': 'arvados/jobs',
 361                     'command': ['ls', '/var/spool/cwl'],
 362                     'cwd': '/var/spool/cwl',
 363                     'scheduling_parameters': {},
 364                     'properties': {},
 365                 }))
 366
 367     @mock.patch("arvados.collection.Collection")
 368     def test_done(self, col):
 369         api = mock.MagicMock()
 370
 371         runner = mock.MagicMock()
 372         runner.api = api
 373         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 374         runner.num_retries = 0
 375         runner.ignore_docker_for_reuse = False
 376
 377         runner.api.containers().get().execute.return_value = {"state":"Complete",
 378                                                               "output": "abc+123",
 379                                                               "exit_code": 0}
 380
 381         col().open.return_value = []
 382
 383         arvjob = arvados_cwl.ArvadosContainer(runner)
 384         arvjob.name = "testjob"
 385         arvjob.builder = mock.MagicMock()
 386         arvjob.output_callback = mock.MagicMock()
 387         arvjob.collect_outputs = mock.MagicMock()
 388         arvjob.successCodes = [0]
 389         arvjob.outdir = "/var/spool/cwl"
 390
 391         arvjob.collect_outputs.return_value = {"out": "stuff"}
 392
 393         arvjob.done({
 394             "state": "Final",
 395             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 396             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 397             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 398             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 399         })
 400
 401         self.assertFalse(api.collections().create.called)
 402
 403         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 404         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 405
 406     # The test passes no builder.resources
 407     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 408     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 409     def test_mounts(self, keepdocker):
 410         arv_docker_clear_cache()
 411
 412         runner = mock.MagicMock()
 413         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 414         runner.ignore_docker_for_reuse = False
 415
 416         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 417         runner.api.collections().get().execute.return_value = {
 418             "portable_data_hash": "99999999999999999999999999999993+99"}
 419
 420         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 421
 422         tool = cmap({
 423             "inputs": [
 424                 {"id": "p1",
 425                  "type": "Directory"}
 426             ],
 427             "outputs": [],
 428             "baseCommand": "ls",
 429             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 430         })
 431         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 432                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 433         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 434                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 435         arvtool.formatgraph = None
 436         job_order = {
 437             "p1": {
 438                 "class": "Directory",
 439                 "location": "keep:99999999999999999999999999999994+44",
 440                 "listing": [
 441                     {
 442                         "class": "File",
 443                         "location": "keep:99999999999999999999999999999994+44/file1",
 444                     },
 445                     {
 446                         "class": "File",
 447                         "location": "keep:99999999999999999999999999999994+44/file2",
 448                     }
 449                 ]
 450             }
 451         }
 452         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 453                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 454             j.run()
 455             runner.api.container_requests().create.assert_called_with(
 456                 body=JsonDiffMatcher({
 457                     'environment': {
 458                         'HOME': '/var/spool/cwl',
 459                         'TMPDIR': '/tmp'
 460                     },
 461                     'name': 'test_run_mounts',
 462                     'runtime_constraints': {
 463                         'vcpus': 1,
 464                         'ram': 1073741824
 465                     },
 466                     'use_existing': True,
 467                     'priority': 1,
 468                     'mounts': {
 469                         "/keep/99999999999999999999999999999994+44": {
 470                             "kind": "collection",
 471                             "portable_data_hash": "99999999999999999999999999999994+44"
 472                         },
 473                         '/tmp': {'kind': 'tmp',
 474                                  "capacity": 1073741824 },
 475                         '/var/spool/cwl': {'kind': 'tmp',
 476                                            "capacity": 1073741824 }
 477                     },
 478                     'state': 'Committed',
 479                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 480                     'output_path': '/var/spool/cwl',
 481                     'container_image': 'arvados/jobs',
 482                     'command': ['ls', '/var/spool/cwl'],
 483                     'cwd': '/var/spool/cwl',
 484                     'scheduling_parameters': {},
 485                     'properties': {},
 486                 }))