sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10
  11 from schema_salad.ref_resolver import Loader
  12
  13 if not os.getenv('ARVADOS_DEBUG'):
  14     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  15     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  16
  17
  18 class TestContainer(unittest.TestCase):
  19
  20     # The test passes no builder.resources
  21     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  22     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  23     def test_run(self, keepdocker):
  24         for enable_reuse in (True, False):
  25             arv_docker_clear_cache()
  26
  27             runner = mock.MagicMock()
  28             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  29             runner.ignore_docker_for_reuse = False
  30
  31             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  32             runner.api.collections().get().execute.return_value = {
  33                 "portable_data_hash": "99999999999999999999999999999993+99"}
  34
  35             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  36
  37             tool = {
  38                 "inputs": [],
  39                 "outputs": [],
  40                 "baseCommand": "ls",
  41                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  42             }
  43             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  44             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  45                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  46             arvtool.formatgraph = None
  47             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  48                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  49                 j.run(enable_reuse=enable_reuse)
  50                 runner.api.container_requests().create.assert_called_with(
  51                     body={
  52                         'environment': {
  53                             'HOME': '/var/spool/cwl',
  54                             'TMPDIR': '/tmp'
  55                         },
  56                         'name': 'test_run_'+str(enable_reuse),
  57                         'runtime_constraints': {
  58                             'vcpus': 1,
  59                             'ram': 1073741824
  60                         },
  61                         'use_existing': enable_reuse,
  62                         'priority': 1,
  63                         'mounts': {
  64                             '/var/spool/cwl': {'kind': 'tmp'}
  65                         },
  66                         'state': 'Committed',
  67                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  68                         'output_path': '/var/spool/cwl',
  69                         'container_image': '99999999999999999999999999999993+99',
  70                         'command': ['ls', '/var/spool/cwl'],
  71                         'cwd': '/var/spool/cwl',
  72                         'scheduling_parameters': {}
  73                     })
  74
  75     # The test passes some fields in builder.resources
  76     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  77     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  78     def test_resource_requirements(self, keepdocker):
  79         arv_docker_clear_cache()
  80         runner = mock.MagicMock()
  81         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  82         runner.ignore_docker_for_reuse = False
  83         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  84
  85         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  86         runner.api.collections().get().execute.return_value = {
  87             "portable_data_hash": "99999999999999999999999999999993+99"}
  88
  89         tool = {
  90             "inputs": [],
  91             "outputs": [],
  92             "hints": [{
  93                 "class": "ResourceRequirement",
  94                 "coresMin": 3,
  95                 "ramMin": 3000,
  96                 "tmpdirMin": 4000
  97             }, {
  98                 "class": "http://arvados.org/cwl#RuntimeConstraints",
  99                 "keep_cache": 512
 100             }, {
 101                 "class": "http://arvados.org/cwl#APIRequirement",
 102             }, {
 103                 "class": "http://arvados.org/cwl#PartitionRequirement",
 104                 "partition": "blurb"
 105             }],
 106             "baseCommand": "ls"
 107         }
 108         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 109         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 110                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 111                                                  loader=Loader({}))
 112         arvtool.formatgraph = None
 113         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 114                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 115             j.run()
 116
 117         call_args, call_kwargs = runner.api.container_requests().create.call_args
 118
 119         call_body_expected = {
 120                 'environment': {
 121                     'HOME': '/var/spool/cwl',
 122                     'TMPDIR': '/tmp'
 123                 },
 124                 'name': 'test_resource_requirements',
 125                 'runtime_constraints': {
 126                     'vcpus': 3,
 127                     'ram': 3145728000,
 128                     'keep_cache_ram': 512,
 129                     'API': True
 130                 },
 131                 'use_existing': True,
 132                 'priority': 1,
 133                 'mounts': {
 134                     '/var/spool/cwl': {'kind': 'tmp'}
 135                 },
 136                 'state': 'Committed',
 137                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 138                 'output_path': '/var/spool/cwl',
 139                 'container_image': '99999999999999999999999999999993+99',
 140                 'command': ['ls'],
 141                 'cwd': '/var/spool/cwl',
 142                 'scheduling_parameters': {
 143                     'partitions': ['blurb']
 144                 }
 145         }
 146
 147         call_body = call_kwargs.get('body', None)
 148         self.assertNotEqual(None, call_body)
 149         for key in call_body:
 150             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 151
 152     @mock.patch("arvados.collection.Collection")
 153     def test_done(self, col):
 154         api = mock.MagicMock()
 155
 156         runner = mock.MagicMock()
 157         runner.api = api
 158         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 159         runner.num_retries = 0
 160         runner.ignore_docker_for_reuse = False
 161
 162         runner.api.containers().get().execute.return_value = {"state":"Complete",
 163                                                               "output": "abc+123",
 164                                                               "exit_code": 0}
 165
 166         col().open.return_value = []
 167
 168         arvjob = arvados_cwl.ArvadosContainer(runner)
 169         arvjob.name = "testjob"
 170         arvjob.builder = mock.MagicMock()
 171         arvjob.output_callback = mock.MagicMock()
 172         arvjob.collect_outputs = mock.MagicMock()
 173         arvjob.successCodes = [0]
 174         arvjob.outdir = "/var/spool/cwl"
 175
 176         arvjob.collect_outputs.return_value = {"out": "stuff"}
 177
 178         arvjob.done({
 179             "state": "Final",
 180             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 181             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 182             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 183             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 184         })
 185
 186         self.assertFalse(api.collections().create.called)
 187
 188         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 189         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")