sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10 from schema_salad.sourceline import cmap
  11
  12 from .matcher import JsonDiffMatcher
  13
  14 if not os.getenv('ARVADOS_DEBUG'):
  15     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  16     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  17
  18
  19 class TestContainer(unittest.TestCase):
  20
  21     # The test passes no builder.resources
  22     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     def test_run(self, keepdocker):
  25         for enable_reuse in (True, False):
  26             arv_docker_clear_cache()
  27
  28             runner = mock.MagicMock()
  29             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  30             runner.ignore_docker_for_reuse = False
  31             runner.intermediate_output_ttl = 0
  32
  33             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  34             runner.api.collections().get().execute.return_value = {
  35                 "portable_data_hash": "99999999999999999999999999999993+99"}
  36
  37             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  38
  39             tool = cmap({
  40                 "inputs": [],
  41                 "outputs": [],
  42                 "baseCommand": "ls",
  43                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  44             })
  45             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
  46                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
  47             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  48                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  49             arvtool.formatgraph = None
  50             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  51                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  52                 j.run(enable_reuse=enable_reuse)
  53                 runner.api.container_requests().create.assert_called_with(
  54                     body=JsonDiffMatcher({
  55                         'environment': {
  56                             'HOME': '/var/spool/cwl',
  57                             'TMPDIR': '/tmp'
  58                         },
  59                         'name': 'test_run_'+str(enable_reuse),
  60                         'runtime_constraints': {
  61                             'vcpus': 1,
  62                             'ram': 1073741824
  63                         },
  64                         'use_existing': enable_reuse,
  65                         'priority': 1,
  66                         'mounts': {
  67                             '/tmp': {'kind': 'tmp',
  68                                      "capacity": 1073741824
  69                                  },
  70                             '/var/spool/cwl': {'kind': 'tmp',
  71                                                "capacity": 1073741824 }
  72                         },
  73                         'state': 'Committed',
  74                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  75                         'output_path': '/var/spool/cwl',
  76                         'output_ttl': 0,
  77                         'container_image': 'arvados/jobs',
  78                         'command': ['ls', '/var/spool/cwl'],
  79                         'cwd': '/var/spool/cwl',
  80                         'scheduling_parameters': {},
  81                         'properties': {},
  82                     }))
  83
  84     # The test passes some fields in builder.resources
  85     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  86     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  87     def test_resource_requirements(self, keepdocker):
  88         arv_docker_clear_cache()
  89         runner = mock.MagicMock()
  90         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  91         runner.ignore_docker_for_reuse = False
  92         runner.intermediate_output_ttl = 3600
  93         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  94
  95         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  96         runner.api.collections().get().execute.return_value = {
  97             "portable_data_hash": "99999999999999999999999999999993+99"}
  98
  99         tool = cmap({
 100             "inputs": [],
 101             "outputs": [],
 102             "hints": [{
 103                 "class": "ResourceRequirement",
 104                 "coresMin": 3,
 105                 "ramMin": 3000,
 106                 "tmpdirMin": 4000,
 107                 "outdirMin": 5000
 108             }, {
 109                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 110                 "keep_cache": 512
 111             }, {
 112                 "class": "http://arvados.org/cwl#APIRequirement",
 113             }, {
 114                 "class": "http://arvados.org/cwl#PartitionRequirement",
 115                 "partition": "blurb"
 116             }, {
 117                 "class": "http://arvados.org/cwl#IntermediateOutput",
 118                 "outputTTL": 7200
 119             }, {
 120                 "class": "http://arvados.org/cwl#ReuseRequirement",
 121                 "enableReuse": False
 122             }],
 123             "baseCommand": "ls"
 124         })
 125         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 126                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 127         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 128                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 129                                                  loader=Loader({}))
 130         arvtool.formatgraph = None
 131         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 132                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 133             j.run(enable_reuse=True)
 134
 135         call_args, call_kwargs = runner.api.container_requests().create.call_args
 136
 137         call_body_expected = {
 138             'environment': {
 139                 'HOME': '/var/spool/cwl',
 140                 'TMPDIR': '/tmp'
 141             },
 142             'name': 'test_resource_requirements',
 143             'runtime_constraints': {
 144                 'vcpus': 3,
 145                 'ram': 3145728000,
 146                 'keep_cache_ram': 536870912,
 147                 'API': True
 148             },
 149             'use_existing': False,
 150             'priority': 1,
 151             'mounts': {
 152                 '/tmp': {'kind': 'tmp',
 153                          "capacity": 4194304000 },
 154                 '/var/spool/cwl': {'kind': 'tmp',
 155                                    "capacity": 5242880000 }
 156             },
 157             'state': 'Committed',
 158             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 159             'output_path': '/var/spool/cwl',
 160             'output_ttl': 7200,
 161             'container_image': 'arvados/jobs',
 162             'command': ['ls'],
 163             'cwd': '/var/spool/cwl',
 164             'scheduling_parameters': {
 165                 'partitions': ['blurb']
 166             },
 167             'properties': {}
 168         }
 169
 170         call_body = call_kwargs.get('body', None)
 171         self.assertNotEqual(None, call_body)
 172         for key in call_body:
 173             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 174
 175
 176     # The test passes some fields in builder.resources
 177     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 178     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 179     @mock.patch("arvados.collection.Collection")
 180     def test_initial_work_dir(self, collection_mock, keepdocker):
 181         arv_docker_clear_cache()
 182         runner = mock.MagicMock()
 183         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 184         runner.ignore_docker_for_reuse = False
 185         runner.intermediate_output_ttl = 0
 186         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 187
 188         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 189         runner.api.collections().get().execute.return_value = {
 190             "portable_data_hash": "99999999999999999999999999999993+99"}
 191
 192         sourcemock = mock.MagicMock()
 193         def get_collection_mock(p):
 194             if "/" in p:
 195                 return (sourcemock, p.split("/", 1)[1])
 196             else:
 197                 return (sourcemock, "")
 198         runner.fs_access.get_collection.side_effect = get_collection_mock
 199
 200         vwdmock = mock.MagicMock()
 201         collection_mock.return_value = vwdmock
 202         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 203
 204         tool = cmap({
 205             "inputs": [],
 206             "outputs": [],
 207             "hints": [{
 208                 "class": "InitialWorkDirRequirement",
 209                 "listing": [{
 210                     "class": "File",
 211                     "basename": "foo",
 212                     "location": "keep:99999999999999999999999999999995+99/bar"
 213                 },
 214                 {
 215                     "class": "Directory",
 216                     "basename": "foo2",
 217                     "location": "keep:99999999999999999999999999999995+99"
 218                 },
 219                 {
 220                     "class": "File",
 221                     "basename": "filename",
 222                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 223                 },
 224                 {
 225                     "class": "Directory",
 226                     "basename": "subdir",
 227                     "location": "keep:99999999999999999999999999999995+99/subdir"
 228                 }                        ]
 229             }],
 230             "baseCommand": "ls"
 231         })
 232         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 233                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 234         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 235                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 236                                                  loader=Loader({}))
 237         arvtool.formatgraph = None
 238         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 239                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 240             j.run()
 241
 242         call_args, call_kwargs = runner.api.container_requests().create.call_args
 243
 244         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 245         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 246         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 247         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 248
 249         call_body_expected = {
 250             'environment': {
 251                 'HOME': '/var/spool/cwl',
 252                 'TMPDIR': '/tmp'
 253             },
 254             'name': 'test_initial_work_dir',
 255             'runtime_constraints': {
 256                 'vcpus': 1,
 257                 'ram': 1073741824
 258             },
 259             'use_existing': True,
 260             'priority': 1,
 261             'mounts': {
 262                 '/tmp': {'kind': 'tmp',
 263                          "capacity": 1073741824 },
 264                 '/var/spool/cwl': {'kind': 'tmp',
 265                                    "capacity": 1073741824 },
 266                 '/var/spool/cwl/foo': {
 267                     'kind': 'collection',
 268                     'path': 'foo',
 269                     'portable_data_hash': '99999999999999999999999999999996+99'
 270                 },
 271                 '/var/spool/cwl/foo2': {
 272                     'kind': 'collection',
 273                     'path': 'foo2',
 274                     'portable_data_hash': '99999999999999999999999999999996+99'
 275                 },
 276                 '/var/spool/cwl/filename': {
 277                     'kind': 'collection',
 278                     'path': 'filename',
 279                     'portable_data_hash': '99999999999999999999999999999996+99'
 280                 },
 281                 '/var/spool/cwl/subdir': {
 282                     'kind': 'collection',
 283                     'path': 'subdir',
 284                     'portable_data_hash': '99999999999999999999999999999996+99'
 285                 }
 286             },
 287             'state': 'Committed',
 288             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 289             'output_path': '/var/spool/cwl',
 290             'output_ttl': 0,
 291             'container_image': 'arvados/jobs',
 292             'command': ['ls'],
 293             'cwd': '/var/spool/cwl',
 294             'scheduling_parameters': {
 295             },
 296             'properties': {}
 297         }
 298
 299         call_body = call_kwargs.get('body', None)
 300         self.assertNotEqual(None, call_body)
 301         for key in call_body:
 302             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 303
 304
 305     # Test redirecting stdin/stdout/stderr
 306     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 307     def test_redirects(self, keepdocker):
 308         arv_docker_clear_cache()
 309
 310         runner = mock.MagicMock()
 311         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 312         runner.ignore_docker_for_reuse = False
 313         runner.intermediate_output_ttl = 0
 314
 315         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 316         runner.api.collections().get().execute.return_value = {
 317             "portable_data_hash": "99999999999999999999999999999993+99"}
 318
 319         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 320
 321         tool = cmap({
 322             "inputs": [],
 323             "outputs": [],
 324             "baseCommand": "ls",
 325             "stdout": "stdout.txt",
 326             "stderr": "stderr.txt",
 327             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 328             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 329         })
 330         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 331                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 332         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 333                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 334         arvtool.formatgraph = None
 335         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 336                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 337             j.run()
 338             runner.api.container_requests().create.assert_called_with(
 339                 body=JsonDiffMatcher({
 340                     'environment': {
 341                         'HOME': '/var/spool/cwl',
 342                         'TMPDIR': '/tmp'
 343                     },
 344                     'name': 'test_run_redirect',
 345                     'runtime_constraints': {
 346                         'vcpus': 1,
 347                         'ram': 1073741824
 348                     },
 349                     'use_existing': True,
 350                     'priority': 1,
 351                     'mounts': {
 352                         '/tmp': {'kind': 'tmp',
 353                                  "capacity": 1073741824 },
 354                         '/var/spool/cwl': {'kind': 'tmp',
 355                                            "capacity": 1073741824 },
 356                         "stderr": {
 357                             "kind": "file",
 358                             "path": "/var/spool/cwl/stderr.txt"
 359                         },
 360                         "stdin": {
 361                             "kind": "collection",
 362                             "path": "file.txt",
 363                             "portable_data_hash": "99999999999999999999999999999996+99"
 364                         },
 365                         "stdout": {
 366                             "kind": "file",
 367                             "path": "/var/spool/cwl/stdout.txt"
 368                         },
 369                     },
 370                     'state': 'Committed',
 371                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 372                     'output_path': '/var/spool/cwl',
 373                     'output_ttl': 0,
 374                     'container_image': 'arvados/jobs',
 375                     'command': ['ls', '/var/spool/cwl'],
 376                     'cwd': '/var/spool/cwl',
 377                     'scheduling_parameters': {},
 378                     'properties': {},
 379                 }))
 380
 381     @mock.patch("arvados.collection.Collection")
 382     def test_done(self, col):
 383         api = mock.MagicMock()
 384
 385         runner = mock.MagicMock()
 386         runner.api = api
 387         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 388         runner.num_retries = 0
 389         runner.ignore_docker_for_reuse = False
 390         runner.intermediate_output_ttl = 0
 391
 392         runner.api.containers().get().execute.return_value = {"state":"Complete",
 393                                                               "output": "abc+123",
 394                                                               "exit_code": 0}
 395
 396         col().open.return_value = []
 397
 398         arvjob = arvados_cwl.ArvadosContainer(runner)
 399         arvjob.name = "testjob"
 400         arvjob.builder = mock.MagicMock()
 401         arvjob.output_callback = mock.MagicMock()
 402         arvjob.collect_outputs = mock.MagicMock()
 403         arvjob.successCodes = [0]
 404         arvjob.outdir = "/var/spool/cwl"
 405         arvjob.output_ttl = 3600
 406
 407         arvjob.collect_outputs.return_value = {"out": "stuff"}
 408
 409         arvjob.done({
 410             "state": "Final",
 411             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 412             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 413             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 414             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 415             "modified_at": "2017-05-26T12:01:22Z"
 416         })
 417
 418         self.assertFalse(api.collections().create.called)
 419
 420         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 421         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
 422         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 423
 424     # The test passes no builder.resources
 425     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 426     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 427     def test_mounts(self, keepdocker):
 428         arv_docker_clear_cache()
 429
 430         runner = mock.MagicMock()
 431         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 432         runner.ignore_docker_for_reuse = False
 433         runner.intermediate_output_ttl = 0
 434
 435         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 436         runner.api.collections().get().execute.return_value = {
 437             "portable_data_hash": "99999999999999999999999999999993+99"}
 438
 439         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 440
 441         tool = cmap({
 442             "inputs": [
 443                 {"id": "p1",
 444                  "type": "Directory"}
 445             ],
 446             "outputs": [],
 447             "baseCommand": "ls",
 448             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 449         })
 450         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
 451                                      collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
 452         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 453                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 454         arvtool.formatgraph = None
 455         job_order = {
 456             "p1": {
 457                 "class": "Directory",
 458                 "location": "keep:99999999999999999999999999999994+44",
 459                 "listing": [
 460                     {
 461                         "class": "File",
 462                         "location": "keep:99999999999999999999999999999994+44/file1",
 463                     },
 464                     {
 465                         "class": "File",
 466                         "location": "keep:99999999999999999999999999999994+44/file2",
 467                     }
 468                 ]
 469             }
 470         }
 471         for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
 472                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 473             j.run()
 474             runner.api.container_requests().create.assert_called_with(
 475                 body=JsonDiffMatcher({
 476                     'environment': {
 477                         'HOME': '/var/spool/cwl',
 478                         'TMPDIR': '/tmp'
 479                     },
 480                     'name': 'test_run_mounts',
 481                     'runtime_constraints': {
 482                         'vcpus': 1,
 483                         'ram': 1073741824
 484                     },
 485                     'use_existing': True,
 486                     'priority': 1,
 487                     'mounts': {
 488                         "/keep/99999999999999999999999999999994+44": {
 489                             "kind": "collection",
 490                             "portable_data_hash": "99999999999999999999999999999994+44"
 491                         },
 492                         '/tmp': {'kind': 'tmp',
 493                                  "capacity": 1073741824 },
 494                         '/var/spool/cwl': {'kind': 'tmp',
 495                                            "capacity": 1073741824 }
 496                     },
 497                     'state': 'Committed',
 498                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 499                     'output_path': '/var/spool/cwl',
 500                     'output_ttl': 0,
 501                     'container_image': 'arvados/jobs',
 502                     'command': ['ls', '/var/spool/cwl'],
 503                     'cwd': '/var/spool/cwl',
 504                     'scheduling_parameters': {},
 505                     'properties': {},
 506                 }))