sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10 from schema_salad.sourceline import cmap
  11
  12 from .matcher import JsonDiffMatcher
  13
  14 if not os.getenv('ARVADOS_DEBUG'):
  15     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  16     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  17
  18
  19 class TestContainer(unittest.TestCase):
  20
  21     # The test passes no builder.resources
  22     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     def test_run(self, keepdocker):
  25         for enable_reuse in (True, False):
  26             arv_docker_clear_cache()
  27
  28             runner = mock.MagicMock()
  29             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  30             runner.ignore_docker_for_reuse = False
  31
  32             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  33             runner.api.collections().get().execute.return_value = {
  34                 "portable_data_hash": "99999999999999999999999999999993+99"}
  35
  36             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  37
  38             tool = cmap({
  39                 "inputs": [],
  40                 "outputs": [],
  41                 "baseCommand": "ls",
  42                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  43             })
  44             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  45             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  46                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  47             arvtool.formatgraph = None
  48             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  49                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  50                 j.run(enable_reuse=enable_reuse)
  51                 runner.api.container_requests().create.assert_called_with(
  52                     body=JsonDiffMatcher({
  53                         'environment': {
  54                             'HOME': '/var/spool/cwl',
  55                             'TMPDIR': '/tmp'
  56                         },
  57                         'name': 'test_run_'+str(enable_reuse),
  58                         'runtime_constraints': {
  59                             'vcpus': 1,
  60                             'ram': 1073741824
  61                         },
  62                         'use_existing': enable_reuse,
  63                         'priority': 1,
  64                         'mounts': {
  65                             '/var/spool/cwl': {'kind': 'tmp'}
  66                         },
  67                         'state': 'Committed',
  68                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  69                         'output_path': '/var/spool/cwl',
  70                         'container_image': 'arvados/jobs',
  71                         'command': ['ls', '/var/spool/cwl'],
  72                         'cwd': '/var/spool/cwl',
  73                         'scheduling_parameters': {},
  74                         'properties': {},
  75                     }))
  76
  77     # The test passes some fields in builder.resources
  78     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  79     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  80     def test_resource_requirements(self, keepdocker):
  81         arv_docker_clear_cache()
  82         runner = mock.MagicMock()
  83         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  84         runner.ignore_docker_for_reuse = False
  85         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  86
  87         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  88         runner.api.collections().get().execute.return_value = {
  89             "portable_data_hash": "99999999999999999999999999999993+99"}
  90
  91         tool = cmap({
  92             "inputs": [],
  93             "outputs": [],
  94             "hints": [{
  95                 "class": "ResourceRequirement",
  96                 "coresMin": 3,
  97                 "ramMin": 3000,
  98                 "tmpdirMin": 4000
  99             }, {
 100                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 101                 "keep_cache": 512
 102             }, {
 103                 "class": "http://arvados.org/cwl#APIRequirement",
 104             }, {
 105                 "class": "http://arvados.org/cwl#PartitionRequirement",
 106                 "partition": "blurb"
 107             }],
 108             "baseCommand": "ls"
 109         })
 110         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 111         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 112                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 113                                                  loader=Loader({}))
 114         arvtool.formatgraph = None
 115         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 116                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 117             j.run()
 118
 119         call_args, call_kwargs = runner.api.container_requests().create.call_args
 120
 121         call_body_expected = {
 122                 'environment': {
 123                     'HOME': '/var/spool/cwl',
 124                     'TMPDIR': '/tmp'
 125                 },
 126                 'name': 'test_resource_requirements',
 127                 'runtime_constraints': {
 128                     'vcpus': 3,
 129                     'ram': 3145728000,
 130                     'keep_cache_ram': 536870912,
 131                     'API': True
 132                 },
 133                 'use_existing': True,
 134                 'priority': 1,
 135                 'mounts': {
 136                     '/var/spool/cwl': {'kind': 'tmp'}
 137                 },
 138                 'state': 'Committed',
 139                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 140                 'output_path': '/var/spool/cwl',
 141                 'container_image': 'arvados/jobs',
 142                 'command': ['ls'],
 143                 'cwd': '/var/spool/cwl',
 144                 'scheduling_parameters': {
 145                     'partitions': ['blurb']
 146                 },
 147                 'properties': {}
 148         }
 149
 150         call_body = call_kwargs.get('body', None)
 151         self.assertNotEqual(None, call_body)
 152         for key in call_body:
 153             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 154
 155
 156     # The test passes some fields in builder.resources
 157     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
 158     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 159     @mock.patch("arvados.collection.Collection")
 160     def test_initial_work_dir(self, collection_mock, keepdocker):
 161         arv_docker_clear_cache()
 162         runner = mock.MagicMock()
 163         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 164         runner.ignore_docker_for_reuse = False
 165         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 166
 167         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 168         runner.api.collections().get().execute.return_value = {
 169             "portable_data_hash": "99999999999999999999999999999993+99"}
 170
 171         sourcemock = mock.MagicMock()
 172         def get_collection_mock(p):
 173             if "/" in p:
 174                 return (sourcemock, p.split("/", 1)[1])
 175             else:
 176                 return (sourcemock, "")
 177         runner.fs_access.get_collection.side_effect = get_collection_mock
 178
 179         vwdmock = mock.MagicMock()
 180         collection_mock.return_value = vwdmock
 181         vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99"
 182
 183         tool = cmap({
 184             "inputs": [],
 185             "outputs": [],
 186             "hints": [{
 187                 "class": "InitialWorkDirRequirement",
 188                 "listing": [{
 189                     "class": "File",
 190                     "basename": "foo",
 191                     "location": "keep:99999999999999999999999999999995+99/bar"
 192                 },
 193                 {
 194                     "class": "Directory",
 195                     "basename": "foo2",
 196                     "location": "keep:99999999999999999999999999999995+99"
 197                 },
 198                 {
 199                     "class": "File",
 200                     "basename": "filename",
 201                     "location": "keep:99999999999999999999999999999995+99/baz/filename"
 202                 },
 203                 {
 204                     "class": "Directory",
 205                     "basename": "subdir",
 206                     "location": "keep:99999999999999999999999999999995+99/subdir"
 207                 }                        ]
 208             }],
 209             "baseCommand": "ls"
 210         })
 211         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 212         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 213                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 214                                                  loader=Loader({}))
 215         arvtool.formatgraph = None
 216         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir",
 217                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 218             j.run()
 219
 220         call_args, call_kwargs = runner.api.container_requests().create.call_args
 221
 222         vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
 223         vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
 224         vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
 225         vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 226
 227         call_body_expected = {
 228                 'environment': {
 229                     'HOME': '/var/spool/cwl',
 230                     'TMPDIR': '/tmp'
 231                 },
 232                 'name': 'test_initial_work_dir',
 233                 'runtime_constraints': {
 234                     'vcpus': 1,
 235                     'ram': 1073741824
 236                 },
 237                 'use_existing': True,
 238                 'priority': 1,
 239                 'mounts': {
 240                     '/var/spool/cwl': {'kind': 'tmp'},
 241                     '/var/spool/cwl/foo': {
 242                         'kind': 'collection',
 243                         'path': 'foo',
 244                         'portable_data_hash': '99999999999999999999999999999996+99'
 245                     },
 246                     '/var/spool/cwl/foo2': {
 247                         'kind': 'collection',
 248                         'path': 'foo2',
 249                         'portable_data_hash': '99999999999999999999999999999996+99'
 250                     },
 251                     '/var/spool/cwl/filename': {
 252                         'kind': 'collection',
 253                         'path': 'filename',
 254                         'portable_data_hash': '99999999999999999999999999999996+99'
 255                     },
 256                     '/var/spool/cwl/subdir': {
 257                         'kind': 'collection',
 258                         'path': 'subdir',
 259                         'portable_data_hash': '99999999999999999999999999999996+99'
 260                     }
 261                 },
 262                 'state': 'Committed',
 263                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 264                 'output_path': '/var/spool/cwl',
 265                 'container_image': 'arvados/jobs',
 266                 'command': ['ls'],
 267                 'cwd': '/var/spool/cwl',
 268                 'scheduling_parameters': {
 269                 },
 270                 'properties': {}
 271         }
 272
 273         call_body = call_kwargs.get('body', None)
 274         self.assertNotEqual(None, call_body)
 275         for key in call_body:
 276             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 277
 278
 279     # Test redirecting stdin/stdout/stderr
 280     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
 281     def test_redirects(self, keepdocker):
 282         arv_docker_clear_cache()
 283
 284         runner = mock.MagicMock()
 285         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 286         runner.ignore_docker_for_reuse = False
 287
 288         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
 289         runner.api.collections().get().execute.return_value = {
 290             "portable_data_hash": "99999999999999999999999999999993+99"}
 291
 292         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 293
 294         tool = cmap({
 295             "inputs": [],
 296             "outputs": [],
 297             "baseCommand": "ls",
 298             "stdout": "stdout.txt",
 299             "stderr": "stderr.txt",
 300             "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
 301             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
 302         })
 303         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 304         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
 305                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
 306         arvtool.formatgraph = None
 307         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
 308                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 309             j.run()
 310             runner.api.container_requests().create.assert_called_with(
 311                 body=JsonDiffMatcher({
 312                     'environment': {
 313                         'HOME': '/var/spool/cwl',
 314                         'TMPDIR': '/tmp'
 315                     },
 316                     'name': 'test_run_redirect',
 317                     'runtime_constraints': {
 318                         'vcpus': 1,
 319                         'ram': 1073741824
 320                     },
 321                     'use_existing': True,
 322                     'priority': 1,
 323                     'mounts': {
 324                         '/var/spool/cwl': {'kind': 'tmp'},
 325                         "stderr": {
 326                             "kind": "file",
 327                             "path": "/var/spool/cwl/stderr.txt"
 328                         },
 329                         "stdin": {
 330                             "kind": "collection",
 331                             "path": "file.txt",
 332                             "portable_data_hash": "99999999999999999999999999999996+99"
 333                         },
 334                         "stdout": {
 335                             "kind": "file",
 336                             "path": "/var/spool/cwl/stdout.txt"
 337                         },
 338                     },
 339                     'state': 'Committed',
 340                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 341                     'output_path': '/var/spool/cwl',
 342                     'container_image': 'arvados/jobs',
 343                     'command': ['ls', '/var/spool/cwl'],
 344                     'cwd': '/var/spool/cwl',
 345                     'scheduling_parameters': {},
 346                     'properties': {},
 347                 }))
 348
 349     @mock.patch("arvados.collection.Collection")
 350     def test_done(self, col):
 351         api = mock.MagicMock()
 352
 353         runner = mock.MagicMock()
 354         runner.api = api
 355         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 356         runner.num_retries = 0
 357         runner.ignore_docker_for_reuse = False
 358
 359         runner.api.containers().get().execute.return_value = {"state":"Complete",
 360                                                               "output": "abc+123",
 361                                                               "exit_code": 0}
 362
 363         col().open.return_value = []
 364
 365         arvjob = arvados_cwl.ArvadosContainer(runner)
 366         arvjob.name = "testjob"
 367         arvjob.builder = mock.MagicMock()
 368         arvjob.output_callback = mock.MagicMock()
 369         arvjob.collect_outputs = mock.MagicMock()
 370         arvjob.successCodes = [0]
 371         arvjob.outdir = "/var/spool/cwl"
 372
 373         arvjob.collect_outputs.return_value = {"out": "stuff"}
 374
 375         arvjob.done({
 376             "state": "Final",
 377             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 378             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 379             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 380             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 381         })
 382
 383         self.assertFalse(api.collections().create.called)
 384
 385         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 386         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")