sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10 from schema_salad.sourceline import cmap
  11
  12 from .matcher import JsonDiffMatcher
  13
  14 if not os.getenv('ARVADOS_DEBUG'):
  15     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  16     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  17
  18
  19 class TestContainer(unittest.TestCase):
  20
  21     # The test passes no builder.resources
  22     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  23     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  24     def test_run(self, keepdocker):
  25         for enable_reuse in (True, False):
  26             arv_docker_clear_cache()
  27
  28             runner = mock.MagicMock()
  29             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  30             runner.ignore_docker_for_reuse = False
  31
  32             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  33             runner.api.collections().get().execute.return_value = {
  34                 "portable_data_hash": "99999999999999999999999999999993+99"}
  35
  36             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  37
  38             tool = cmap({
  39                 "inputs": [],
  40                 "outputs": [],
  41                 "baseCommand": "ls",
  42                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  43             })
  44             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  45             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  46                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  47             arvtool.formatgraph = None
  48             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  49                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  50                 j.run(enable_reuse=enable_reuse)
  51                 runner.api.container_requests().create.assert_called_with(
  52                     body=JsonDiffMatcher({
  53                         'environment': {
  54                             'HOME': '/var/spool/cwl',
  55                             'TMPDIR': '/tmp'
  56                         },
  57                         'name': 'test_run_'+str(enable_reuse),
  58                         'runtime_constraints': {
  59                             'vcpus': 1,
  60                             'ram': 1073741824
  61                         },
  62                         'use_existing': enable_reuse,
  63                         'priority': 1,
  64                         'mounts': {
  65                             '/var/spool/cwl': {'kind': 'tmp'}
  66                         },
  67                         'state': 'Committed',
  68                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  69                         'output_path': '/var/spool/cwl',
  70                         'container_image': '99999999999999999999999999999993+99',
  71                         'command': ['ls', '/var/spool/cwl'],
  72                         'cwd': '/var/spool/cwl',
  73                         'scheduling_parameters': {},
  74                         'properties': {},
  75                     }))
  76
  77     # The test passes some fields in builder.resources
  78     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  79     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  80     def test_resource_requirements(self, keepdocker):
  81         arv_docker_clear_cache()
  82         runner = mock.MagicMock()
  83         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  84         runner.ignore_docker_for_reuse = False
  85         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  86
  87         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  88         runner.api.collections().get().execute.return_value = {
  89             "portable_data_hash": "99999999999999999999999999999993+99"}
  90
  91         tool = cmap({
  92             "inputs": [],
  93             "outputs": [],
  94             "hints": [{
  95                 "class": "ResourceRequirement",
  96                 "coresMin": 3,
  97                 "ramMin": 3000,
  98                 "tmpdirMin": 4000
  99             }, {
 100                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 101                 "keep_cache": 512
 102             }, {
 103                 "class": "http://arvados.org/cwl#APIRequirement",
 104             }, {
 105                 "class": "http://arvados.org/cwl#PartitionRequirement",
 106                 "partition": "blurb"
 107             }],
 108             "baseCommand": "ls"
 109         })
 110         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 111         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 112                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 113                                                  loader=Loader({}))
 114         arvtool.formatgraph = None
 115         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 116                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 117             j.run()
 118
 119         call_args, call_kwargs = runner.api.container_requests().create.call_args
 120
 121         call_body_expected = {
 122                 'environment': {
 123                     'HOME': '/var/spool/cwl',
 124                     'TMPDIR': '/tmp'
 125                 },
 126                 'name': 'test_resource_requirements',
 127                 'runtime_constraints': {
 128                     'vcpus': 3,
 129                     'ram': 3145728000,
 130                     'keep_cache_ram': 512,
 131                     'API': True
 132                 },
 133                 'use_existing': True,
 134                 'priority': 1,
 135                 'mounts': {
 136                     '/var/spool/cwl': {'kind': 'tmp'}
 137                 },
 138                 'state': 'Committed',
 139                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 140                 'output_path': '/var/spool/cwl',
 141                 'container_image': '99999999999999999999999999999993+99',
 142                 'command': ['ls'],
 143                 'cwd': '/var/spool/cwl',
 144                 'scheduling_parameters': {
 145                     'partitions': ['blurb']
 146                 },
 147                 'properties': {}
 148         }
 149
 150         call_body = call_kwargs.get('body', None)
 151         self.assertNotEqual(None, call_body)
 152         for key in call_body:
 153             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 154
 155     @mock.patch("arvados.collection.Collection")
 156     def test_done(self, col):
 157         api = mock.MagicMock()
 158
 159         runner = mock.MagicMock()
 160         runner.api = api
 161         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 162         runner.num_retries = 0
 163         runner.ignore_docker_for_reuse = False
 164
 165         runner.api.containers().get().execute.return_value = {"state":"Complete",
 166                                                               "output": "abc+123",
 167                                                               "exit_code": 0}
 168
 169         col().open.return_value = []
 170
 171         arvjob = arvados_cwl.ArvadosContainer(runner)
 172         arvjob.name = "testjob"
 173         arvjob.builder = mock.MagicMock()
 174         arvjob.output_callback = mock.MagicMock()
 175         arvjob.collect_outputs = mock.MagicMock()
 176         arvjob.successCodes = [0]
 177         arvjob.outdir = "/var/spool/cwl"
 178
 179         arvjob.collect_outputs.return_value = {"out": "stuff"}
 180
 181         arvjob.done({
 182             "state": "Final",
 183             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 184             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 185             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 186             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 187         })
 188
 189         self.assertFalse(api.collections().create.called)
 190
 191         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 192         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")