sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 from arvados_cwl.arvdocker import arv_docker_clear_cache
   3 import logging
   4 import mock
   5 import unittest
   6 import os
   7 import functools
   8 import cwltool.process
   9 from schema_salad.ref_resolver import Loader
  10
  11 from schema_salad.ref_resolver import Loader
  12
  13 if not os.getenv('ARVADOS_DEBUG'):
  14     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  15     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  16
  17
  18 class TestContainer(unittest.TestCase):
  19
  20     # The test passes no builder.resources
  21     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  22     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  23     def test_run(self, keepdocker):
  24         for enable_reuse in (True, False):
  25             arv_docker_clear_cache()
  26
  27             runner = mock.MagicMock()
  28             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  29             runner.ignore_docker_for_reuse = False
  30
  31             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  32             runner.api.collections().get().execute.return_value = {
  33                 "portable_data_hash": "99999999999999999999999999999993+99"}
  34
  35             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  36
  37             tool = {
  38                 "inputs": [],
  39                 "outputs": [],
  40                 "baseCommand": "ls",
  41                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  42             }
  43             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  44             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  45                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  46             arvtool.formatgraph = None
  47             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  48                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  49                 j.run(enable_reuse=enable_reuse)
  50                 runner.api.container_requests().create.assert_called_with(
  51                     body={
  52                         'environment': {
  53                             'HOME': '/var/spool/cwl',
  54                             'TMPDIR': '/tmp'
  55                         },
  56                         'name': 'test_run_'+str(enable_reuse),
  57                         'runtime_constraints': {
  58                             'vcpus': 1,
  59                             'ram': 1073741824
  60                         },
  61                         'use_existing': enable_reuse,
  62                         'priority': 1,
  63                         'mounts': {
  64                             '/var/spool/cwl': {'kind': 'tmp'}
  65                         },
  66                         'state': 'Committed',
  67                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  68                         'output_path': '/var/spool/cwl',
  69                         'container_image': '99999999999999999999999999999993+99',
  70                         'command': ['ls', '/var/spool/cwl'],
  71                         'cwd': '/var/spool/cwl'
  72                     })
  73
  74     # The test passes some fields in builder.resources
  75     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  76     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  77     def test_resource_requirements(self, keepdocker):
  78         arv_docker_clear_cache()
  79         runner = mock.MagicMock()
  80         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  81         runner.ignore_docker_for_reuse = False
  82         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  83
  84         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  85         runner.api.collections().get().execute.return_value = {
  86             "portable_data_hash": "99999999999999999999999999999993+99"}
  87
  88         tool = {
  89             "inputs": [],
  90             "outputs": [],
  91             "hints": [{
  92                 "class": "ResourceRequirement",
  93                 "coresMin": 3,
  94                 "ramMin": 3000,
  95                 "tmpdirMin": 4000
  96             }, {
  97                 "class": "http://arvados.org/cwl#RuntimeConstraints",
  98                 "keep_cache": 512
  99             }, {
 100                 "class": "http://arvados.org/cwl#APIRequirement",
 101             }, {
 102                 "class": "http://arvados.org/cwl#PartitionRequirement",
 103                 "partition": "blurb"
 104             }],
 105             "baseCommand": "ls"
 106         }
 107         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 108         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 109                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 110                                                  loader=Loader({}))
 111         arvtool.formatgraph = None
 112         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 113                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 114             j.run()
 115
 116         runner.api.container_requests().create.assert_called_with(
 117             body={
 118                 'environment': {
 119                     'HOME': '/var/spool/cwl',
 120                     'TMPDIR': '/tmp'
 121                 },
 122                 'name': 'test_resource_requirements',
 123                 'runtime_constraints': {
 124                     'vcpus': 3,
 125                     'ram': 3145728000,
 126                     'keep_cache_ram': 512,
 127                     'API': True,
 128                     'partition': ['blurb']
 129                 },
 130                 'use_existing': True,
 131                 'priority': 1,
 132                 'mounts': {
 133                     '/var/spool/cwl': {'kind': 'tmp'}
 134                 },
 135                 'state': 'Committed',
 136                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 137                 'output_path': '/var/spool/cwl',
 138                 'container_image': '99999999999999999999999999999993+99',
 139                 'command': ['ls'],
 140                 'cwd': '/var/spool/cwl'
 141             })
 142
 143     @mock.patch("arvados.collection.Collection")
 144     def test_done(self, col):
 145         api = mock.MagicMock()
 146
 147         runner = mock.MagicMock()
 148         runner.api = api
 149         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 150         runner.num_retries = 0
 151         runner.ignore_docker_for_reuse = False
 152
 153         col().open.return_value = []
 154         api.collections().list().execute.side_effect = ({"items": []},
 155                                                         {"items": [{"manifest_text": "XYZ"}]})
 156
 157         arvjob = arvados_cwl.ArvadosContainer(runner)
 158         arvjob.name = "testjob"
 159         arvjob.builder = mock.MagicMock()
 160         arvjob.output_callback = mock.MagicMock()
 161         arvjob.collect_outputs = mock.MagicMock()
 162         arvjob.successCodes = [0]
 163         arvjob.outdir = "/var/spool/cwl"
 164
 165         arvjob.done({
 166             "state": "Complete",
 167             "output": "99999999999999999999999999999993+99",
 168             "log": "99999999999999999999999999999994+99",
 169             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 170             "exit_code": 0
 171         })
 172
 173         api.collections().list.assert_has_calls([
 174             mock.call(),
 175             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 176                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 177                           ['name', '=', 'Output 9999999 of testjob']]),
 178             mock.call().execute(num_retries=0),
 179             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 180                  select=['manifest_text']),
 181             mock.call().execute(num_retries=0)])
 182
 183         api.collections().create.assert_called_with(
 184             ensure_unique_name=True,
 185             body={'portable_data_hash': '99999999999999999999999999999993+99',
 186                   'manifest_text': 'XYZ',
 187                   'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 188                   'name': 'Output 9999999 of testjob'})
 189
 190     @mock.patch("arvados.collection.Collection")
 191     def test_done_use_existing_collection(self, col):
 192         api = mock.MagicMock()
 193
 194         runner = mock.MagicMock()
 195         runner.api = api
 196         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 197         runner.num_retries = 0
 198
 199         col().open.return_value = []
 200         api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
 201
 202         arvjob = arvados_cwl.ArvadosContainer(runner)
 203         arvjob.name = "testjob"
 204         arvjob.builder = mock.MagicMock()
 205         arvjob.output_callback = mock.MagicMock()
 206         arvjob.collect_outputs = mock.MagicMock()
 207         arvjob.successCodes = [0]
 208         arvjob.outdir = "/var/spool/cwl"
 209
 210         arvjob.done({
 211             "state": "Complete",
 212             "output": "99999999999999999999999999999993+99",
 213             "log": "99999999999999999999999999999994+99",
 214             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 215             "exit_code": 0
 216         })
 217
 218         api.collections().list.assert_has_calls([
 219             mock.call(),
 220             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 221                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 222                                ['name', '=', 'Output 9999999 of testjob']]),
 223             mock.call().execute(num_retries=0)])
 224
 225         self.assertFalse(api.collections().create.called)