sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10
  11 from .matcher import JsonDiffMatcher
  12
  13 if not os.getenv('ARVADOS_DEBUG'):
  14     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  15     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  16
  17
  18 class TestContainer(unittest.TestCase):
  19
  20     # The test passes no builder.resources
  21     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  22     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  23     def test_run(self, keepdocker):
  24         for enable_reuse in (True, False):
  25             arv_docker_clear_cache()
  26
  27             runner = mock.MagicMock()
  28             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  29             runner.ignore_docker_for_reuse = False
  30
  31             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  32             runner.api.collections().get().execute.return_value = {
  33                 "portable_data_hash": "99999999999999999999999999999993+99"}
  34
  35             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  36
  37             tool = {
  38                 "inputs": [],
  39                 "outputs": [],
  40                 "baseCommand": "ls",
  41                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  42             }
  43             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  44             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  45                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  46             arvtool.formatgraph = None
  47             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  48                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  49                 j.run(enable_reuse=enable_reuse)
  50                 runner.api.container_requests().create.assert_called_with(
  51                     body=JsonDiffMatcher({
  52                         'environment': {
  53                             'HOME': '/var/spool/cwl',
  54                             'TMPDIR': '/tmp'
  55                         },
  56                         'name': 'test_run_'+str(enable_reuse),
  57                         'runtime_constraints': {
  58                             'vcpus': 1,
  59                             'ram': 1073741824
  60                         },
  61                         'use_existing': enable_reuse,
  62                         'priority': 1,
  63                         'mounts': {
  64                             '/var/spool/cwl': {'kind': 'tmp'}
  65                         },
  66                         'state': 'Committed',
  67                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  68                         'output_path': '/var/spool/cwl',
  69                         'container_image': '99999999999999999999999999999993+99',
  70                         'command': ['ls', '/var/spool/cwl'],
  71                         'cwd': '/var/spool/cwl',
  72                         'scheduling_parameters': {},
  73                         'properties': {},
  74                     }))
  75
  76     # The test passes some fields in builder.resources
  77     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  78     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  79     def test_resource_requirements(self, keepdocker):
  80         arv_docker_clear_cache()
  81         runner = mock.MagicMock()
  82         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  83         runner.ignore_docker_for_reuse = False
  84         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  85
  86         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  87         runner.api.collections().get().execute.return_value = {
  88             "portable_data_hash": "99999999999999999999999999999993+99"}
  89
  90         tool = {
  91             "inputs": [],
  92             "outputs": [],
  93             "hints": [{
  94                 "class": "ResourceRequirement",
  95                 "coresMin": 3,
  96                 "ramMin": 3000,
  97                 "tmpdirMin": 4000
  98             }, {
  99                 "class": "http://arvados.org/cwl#RuntimeConstraints",
 100                 "keep_cache": 512
 101             }, {
 102                 "class": "http://arvados.org/cwl#APIRequirement",
 103             }, {
 104                 "class": "http://arvados.org/cwl#PartitionRequirement",
 105                 "partition": "blurb"
 106             }],
 107             "baseCommand": "ls"
 108         }
 109         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 110         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 111                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 112                                                  loader=Loader({}))
 113         arvtool.formatgraph = None
 114         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 115                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 116             j.run()
 117
 118         call_args, call_kwargs = runner.api.container_requests().create.call_args
 119
 120         call_body_expected = {
 121                 'environment': {
 122                     'HOME': '/var/spool/cwl',
 123                     'TMPDIR': '/tmp'
 124                 },
 125                 'name': 'test_resource_requirements',
 126                 'runtime_constraints': {
 127                     'vcpus': 3,
 128                     'ram': 3145728000,
 129                     'keep_cache_ram': 512,
 130                     'API': True
 131                 },
 132                 'use_existing': True,
 133                 'priority': 1,
 134                 'mounts': {
 135                     '/var/spool/cwl': {'kind': 'tmp'}
 136                 },
 137                 'state': 'Committed',
 138                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 139                 'output_path': '/var/spool/cwl',
 140                 'container_image': '99999999999999999999999999999993+99',
 141                 'command': ['ls'],
 142                 'cwd': '/var/spool/cwl',
 143                 'scheduling_parameters': {
 144                     'partitions': ['blurb']
 145                 },
 146                 'properties': {}
 147         }
 148
 149         call_body = call_kwargs.get('body', None)
 150         self.assertNotEqual(None, call_body)
 151         for key in call_body:
 152             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 153
 154     @mock.patch("arvados.collection.Collection")
 155     def test_done(self, col):
 156         api = mock.MagicMock()
 157
 158         runner = mock.MagicMock()
 159         runner.api = api
 160         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 161         runner.num_retries = 0
 162         runner.ignore_docker_for_reuse = False
 163
 164         runner.api.containers().get().execute.return_value = {"state":"Complete",
 165                                                               "output": "abc+123",
 166                                                               "exit_code": 0}
 167
 168         col().open.return_value = []
 169
 170         arvjob = arvados_cwl.ArvadosContainer(runner)
 171         arvjob.name = "testjob"
 172         arvjob.builder = mock.MagicMock()
 173         arvjob.output_callback = mock.MagicMock()
 174         arvjob.collect_outputs = mock.MagicMock()
 175         arvjob.successCodes = [0]
 176         arvjob.outdir = "/var/spool/cwl"
 177
 178         arvjob.collect_outputs.return_value = {"out": "stuff"}
 179
 180         arvjob.done({
 181             "state": "Final",
 182             "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
 183             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
 184             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
 185             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 186         })
 187
 188         self.assertFalse(api.collections().create.called)
 189
 190         arvjob.collect_outputs.assert_called_with("keep:abc+123")
 191         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")