sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10 from schema_salad.sourceline import cmap
  11
  12 from .matcher import JsonDiffMatcher
  13
  14 if not os.getenv('ARVADOS_DEBUG'):
  15     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  16     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  17
  18
  19 class TestContainer(unittest.TestCase):
  20
  21     # The test passes no builder.resources
  22     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     def test_run(self, keepdocker):
  25         for enable_reuse in (True, False):
  26             arv_docker_clear_cache()
  27
  28             runner = mock.MagicMock()
  29             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  30             runner.ignore_docker_for_reuse = False
  31             runner.intermediate_output_ttl = 0
  32
  33             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  34             runner.api.collections().get().execute.return_value = {
  35                 "portable_data_hash": "99999999999999999999999999999993+99"}
  36
  37             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  38
  39             tool = cmap({
  40                 "inputs": [],
  41                 "outputs": [],
  42                 "baseCommand": "ls",
  43                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  44             })
  45             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  46                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  47             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  48                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  49             arvtool.formatgraph = None
  50             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  51                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  52                 j.run(enable_reuse=enable_reuse)
  53                 runner.api.container_requests().create.assert_called_with(
  54                     body=JsonDiffMatcher({
  55                         'environment': {
  56                             'HOME': '/var/spool/cwl',
  57                             'TMPDIR': '/tmp'
  58                         },
  59                         'name': 'test_run_'+str(enable_reuse),
  60                         'runtime_constraints': {
  61                             'vcpus': 1,
  62                             'ram': 1073741824
  63                         },
  64                         'use_existing': enable_reuse,
  65                         'priority': 1,
  66                         'mounts': {
  67                             '/tmp': {'kind': 'tmp',
  68                                      "capacity": 1073741824
  69                                  },
  70                             '/var/spool/cwl': {'kind': 'tmp',
  71                                                "capacity": 1073741824 }
  72                         },
  73                         'state': 'Committed',
  74                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  75                         'output_path': '/var/spool/cwl',
  76                         'output_ttl': 0,
  77                         'container_image': 'arvados/jobs',
  78                         'command': ['ls', '/var/spool/cwl'],
  79                         'cwd': '/var/spool/cwl',
  80                         'scheduling_parameters': {},
  81                         'properties': {},
  82                     }))
  83
  84     # The test passes some fields in builder.resources
  85     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  86     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  87     def test_resource_requirements(self, keepdocker):
  88         arv_docker_clear_cache()
  89         runner = mock.MagicMock()
  90         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  91         runner.ignore_docker_for_reuse = False
  92         runner.intermediate_output_ttl = 3600
  93         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  94
  95         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  96         runner.api.collections().get().execute.return_value = {
  97             "portable_data_hash": "99999999999999999999999999999993+99"}
  98
  99         tool = cmap({
 100             "inputs": [],
 101             "outputs": [],
 102             "hints": [{
 103                 "class": "ResourceRequirement",
 104                 "coresMin": 3,
 105                 "ramMin": 3000,
 106                 "tmpdirMin": 4000,
 107                 "outdirMin": 5000
 108             }, {
 109                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 110                 "keep_cache": 512
 111             }, {
 112                 "class": "http://arvados.org/cwl#APIRequirement",
 113             }, {
 114                 "class": "http://arvados.org/cwl#PartitionRequirement",
 115                 "partition": "blurb"
 116             }, {
 117                 "class": "http://arvados.org/cwl#IntermediateOutput",
 118                 "outputTTL": 7200
 119             }],
 120             "baseCommand": "ls"
 121         })
 122         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 123                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 124         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 125                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 126                                                  loader=Loader({}))
 127         arvtool.formatgraph = None
 128         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 129                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 130             j.run()
 131
 132         call_args, call_kwargs = runner.api.container_requests().create.call_args
 133
 134         call_body_expected = {
 135             'environment': {
 136                 'HOME': '/var/spool/cwl',
 137                 'TMPDIR': '/tmp'
 138             },
 139             'name': 'test_resource_requirements',
 140             'runtime_constraints': {
 141                 'vcpus': 3,
 142                 'ram': 3145728000,
 143                 'keep_cache_ram': 536870912,
 144                 'API': True
 145             },
 146             'use_existing': True,
 147             'priority': 1,
 148             'mounts': {
 149                 '/tmp': {'kind': 'tmp',
 150                          "capacity": 4194304000 },
 151                 '/var/spool/cwl': {'kind': 'tmp',
 152                                    "capacity": 5242880000 }
 153             },
 154             'state': 'Committed',
 155             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 156             'output_path': '/var/spool/cwl',
 157             'output_ttl': 7200,
 158             'container_image': 'arvados/jobs',
 159             'command': ['ls'],
 160             'cwd': '/var/spool/cwl',
 161             'scheduling_parameters': {
 162                 'partitions': ['blurb']
 163             },
 164             'properties': {}
 165         }
 166
 167         call_body = call_kwargs.get('body', None)
 168         self.assertNotEqual(None, call_body)
 169         for key in call_body:
 170             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 171
 172
 173     # The test passes some fields in builder.resources
 174     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 175     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 176     @mock.patch("arvados.collection.Collection")
 177     def test_initial_work_dir(self, collection_mock, keepdocker):
 178         arv_docker_clear_cache()
 179         runner = mock.MagicMock()
 180         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 181         runner.ignore_docker_for_reuse = False
 182         runner.intermediate_output_ttl = 0
 183         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 184
 185         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 186         runner.api.collections().get().execute.return_value = {
 187             "portable_data_hash": "99999999999999999999999999999993+99"}
 188
 189         sourcemock = mock.MagicMock()
 190         def get_collection_mock(p):
 191             if "/" in p:
 192                 return (sourcemock, p.split("/", 1)[1])
 193             else:
 194                 return (sourcemock, "")
 195         runner.fs_access.get_collection.side_effect = get_collection_mock
 196
 197         vwdmock = mock.MagicMock()
 198         collection_mock.return_value = vwdmock
 199         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 200
 201         tool = cmap({
 202             "inputs": [],
 203             "outputs": [],
 204             "hints": [{
 205                 "class": "InitialWorkDirRequirement",
 206                 "listing": [{
 207                     "class": "File",
 208                     "basename": "foo",
 209                     "location": "keep:99999999999999999999999999999995+99/bar"
 210                 },
 211                 {
 212                     "class": "Directory",
 213                     "basename": "foo2",
 214                     "location": "keep:99999999999999999999999999999995+99"
 215                 },
 216                 {
 217                     "class": "File",
 218                     "basename": "filename",
 219                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 220                 },
 221                 {
 222                     "class": "Directory",
 223                     "basename": "subdir",
 224                     "location": "keep:99999999999999999999999999999995+99/subdir"
 225                 }                        ]
 226             }],
 227             "baseCommand": "ls"
 228         })
 229         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 230                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 231         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 232                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 233                                                  loader=Loader({}))
 234         arvtool.formatgraph = None
 235         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 236                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 237             j.run()
 238
 239         call_args, call_kwargs = runner.api.container_requests().create.call_args
 240
 241         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 242         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 243         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 244         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 245
 246         call_body_expected = {
 247             'environment': {
 248                 'HOME': '/var/spool/cwl',
 249                 'TMPDIR': '/tmp'
 250             },
 251             'name': 'test_initial_work_dir',
 252             'runtime_constraints': {
 253                 'vcpus': 1,
 254                 'ram': 1073741824
 255             },
 256             'use_existing': True,
 257             'priority': 1,
 258             'mounts': {
 259                 '/tmp': {'kind': 'tmp',
 260                          "capacity": 1073741824 },
 261                 '/var/spool/cwl': {'kind': 'tmp',
 262                                    "capacity": 1073741824 },
 263                 '/var/spool/cwl/foo': {
 264                     'kind': 'collection',
 265                     'path': 'foo',
 266                     'portable_data_hash': '99999999999999999999999999999996+99'
 267                 },
 268                 '/var/spool/cwl/foo2': {
 269                     'kind': 'collection',
 270                     'path': 'foo2',
 271                     'portable_data_hash': '99999999999999999999999999999996+99'
 272                 },
 273                 '/var/spool/cwl/filename': {
 274                     'kind': 'collection',
 275                     'path': 'filename',
 276                     'portable_data_hash': '99999999999999999999999999999996+99'
 277                 },
 278                 '/var/spool/cwl/subdir': {
 279                     'kind': 'collection',
 280                     'path': 'subdir',
 281                     'portable_data_hash': '99999999999999999999999999999996+99'
 282                 }
 283             },
 284             'state': 'Committed',
 285             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 286             'output_path': '/var/spool/cwl',
 287             'output_ttl': 0,
 288             'container_image': 'arvados/jobs',
 289             'command': ['ls'],
 290             'cwd': '/var/spool/cwl',
 291             'scheduling_parameters': {
 292             },
 293             'properties': {}
 294         }
 295
 296         call_body = call_kwargs.get('body', None)
 297         self.assertNotEqual(None, call_body)
 298         for key in call_body:
 299             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 300
 301
 302     # Test redirecting stdin/stdout/stderr
 303     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 304     def test_redirects(self, keepdocker):
 305         arv_docker_clear_cache()
 306
 307         runner = mock.MagicMock()
 308         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 309         runner.ignore_docker_for_reuse = False
 310         runner.intermediate_output_ttl = 0
 311
 312         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 313         runner.api.collections().get().execute.return_value = {
 314             "portable_data_hash": "99999999999999999999999999999993+99"}
 315
 316         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 317
 318         tool = cmap({
 319             "inputs": [],
 320             "outputs": [],
 321             "baseCommand": "ls",
 322             "stdout": "stdout.txt",
 323             "stderr": "stderr.txt",
 324             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 325             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 326         })
 327         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 328                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 329         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 330                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 331         arvtool.formatgraph = None
 332         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 333                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 334             j.run()
 335             runner.api.container_requests().create.assert_called_with(
 336                 body=JsonDiffMatcher({
 337                     'environment': {
 338                         'HOME': '/var/spool/cwl',
 339                         'TMPDIR': '/tmp'
 340                     },
 341                     'name': 'test_run_redirect',
 342                     'runtime_constraints': {
 343                         'vcpus': 1,
 344                         'ram': 1073741824
 345                     },
 346                     'use_existing': True,
 347                     'priority': 1,
 348                     'mounts': {
 349                         '/tmp': {'kind': 'tmp',
 350                                  "capacity": 1073741824 },
 351                         '/var/spool/cwl': {'kind': 'tmp',
 352                                            "capacity": 1073741824 },
 353                         "stderr": {
 354                             "kind": "file",
 355                             "path": "/var/spool/cwl/stderr.txt"
 356                         },
 357                         "stdin": {
 358                             "kind": "collection",
 359                             "path": "file.txt",
 360                             "portable_data_hash": "99999999999999999999999999999996+99"
 361                         },
 362                         "stdout": {
 363                             "kind": "file",
 364                             "path": "/var/spool/cwl/stdout.txt"
 365                         },
 366                     },
 367                     'state': 'Committed',
 368                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 369                     'output_path': '/var/spool/cwl',
 370                     'output_ttl': 0,
 371                     'container_image': 'arvados/jobs',
 372                     'command': ['ls', '/var/spool/cwl'],
 373                     'cwd': '/var/spool/cwl',
 374                     'scheduling_parameters': {},
 375                     'properties': {},
 376                 }))
 377
 378     @mock.patch("arvados.collection.Collection")
 379     def test_done(self, col):
 380         api = mock.MagicMock()
 381
 382         runner = mock.MagicMock()
 383         runner.api = api
 384         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 385         runner.num_retries = 0
 386         runner.ignore_docker_for_reuse = False
 387         runner.intermediate_output_ttl = 0
 388
 389         runner.api.containers().get().execute.return_value = {"state":"Complete",
 390                                                               "output": "abc+123",
 391                                                               "exit_code": 0}
 392
 393         col().open.return_value = []
 394
 395         arvjob = arvados_cwl.ArvadosContainer(runner)
 396         arvjob.name = "testjob"
 397         arvjob.builder = mock.MagicMock()
 398         arvjob.output_callback = mock.MagicMock()
 399         arvjob.collect_outputs = mock.MagicMock()
 400         arvjob.successCodes = [0]
 401         arvjob.outdir = "/var/spool/cwl"
 402         arvjob.output_ttl = 3600
 403
 404         arvjob.collect_outputs.return_value = {"out": "stuff"}
 405
 406         arvjob.done({
 407             "state": "Final",
 408             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 409             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 410             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 411             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 412             "modified_at": "2017-05-26T12:01:22Z"
 413         })
 414
 415         self.assertFalse(api.collections().create.called)
 416
 417         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 418         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 419         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 420
 421     # The test passes no builder.resources
 422     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 423     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 424     def test_mounts(self, keepdocker):
 425         arv_docker_clear_cache()
 426
 427         runner = mock.MagicMock()
 428         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 429         runner.ignore_docker_for_reuse = False
 430         runner.intermediate_output_ttl = 0
 431
 432         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 433         runner.api.collections().get().execute.return_value = {
 434             "portable_data_hash": "99999999999999999999999999999993+99"}
 435
 436         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 437
 438         tool = cmap({
 439             "inputs": [
 440                 {"id": "p1",
 441                  "type": "Directory"}
 442             ],
 443             "outputs": [],
 444             "baseCommand": "ls",
 445             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 446         })
 447         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 448                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 449         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 450                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 451         arvtool.formatgraph = None
 452         job_order = {
 453             "p1": {
 454                 "class": "Directory",
 455                 "location": "keep:99999999999999999999999999999994+44",
 456                 "listing": [
 457                     {
 458                         "class": "File",
 459                         "location": "keep:99999999999999999999999999999994+44/file1",
 460                     },
 461                     {
 462                         "class": "File",
 463                         "location": "keep:99999999999999999999999999999994+44/file2",
 464                     }
 465                 ]
 466             }
 467         }
 468         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 469                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 470             j.run()
 471             runner.api.container_requests().create.assert_called_with(
 472                 body=JsonDiffMatcher({
 473                     'environment': {
 474                         'HOME': '/var/spool/cwl',
 475                         'TMPDIR': '/tmp'
 476                     },
 477                     'name': 'test_run_mounts',
 478                     'runtime_constraints': {
 479                         'vcpus': 1,
 480                         'ram': 1073741824
 481                     },
 482                     'use_existing': True,
 483                     'priority': 1,
 484                     'mounts': {
 485                         "/keep/99999999999999999999999999999994+44": {
 486                             "kind": "collection",
 487                             "portable_data_hash": "99999999999999999999999999999994+44"
 488                         },
 489                         '/tmp': {'kind': 'tmp',
 490                                  "capacity": 1073741824 },
 491                         '/var/spool/cwl': {'kind': 'tmp',
 492                                            "capacity": 1073741824 }
 493                     },
 494                     'state': 'Committed',
 495                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 496                     'output_path': '/var/spool/cwl',
 497                     'output_ttl': 0,
 498                     'container_image': 'arvados/jobs',
 499                     'command': ['ls', '/var/spool/cwl'],
 500                     'cwd': '/var/spool/cwl',
 501                     'scheduling_parameters': {},
 502                     'properties': {},
 503                 }))