sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 import logging
   3 import mock
   4 import unittest
   5 import os
   6 import functools
   7 import cwltool.process
   8 from schema_salad.ref_resolver import Loader
   9
  10 from schema_salad.ref_resolver import Loader
  11
  12 if not os.getenv('ARVADOS_DEBUG'):
  13     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  14     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  15
  16
  17 class TestContainer(unittest.TestCase):
  18
  19     # The test passes no builder.resources
  20     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  21     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  22     def test_run(self, keepdocker):
  23         for enable_reuse in (True, False):
  24             runner = mock.MagicMock()
  25             runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  26             runner.ignore_docker_for_reuse = False
  27
  28             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  29             runner.api.collections().get().execute.return_value = {
  30                 "portable_data_hash": "99999999999999999999999999999993+99"}
  31
  32             document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  33
  34             tool = {
  35                 "inputs": [],
  36                 "outputs": [],
  37                 "baseCommand": "ls",
  38                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  39             }
  40             make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  41             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  42                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  43             arvtool.formatgraph = None
  44             for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
  45                                  make_fs_access=make_fs_access, tmpdir="/tmp"):
  46                 j.run(enable_reuse=enable_reuse)
  47                 runner.api.container_requests().create.assert_called_with(
  48                     body={
  49                         'environment': {
  50                             'HOME': '/var/spool/cwl',
  51                             'TMPDIR': '/tmp'
  52                         },
  53                         'name': 'test_run_'+str(enable_reuse),
  54                         'runtime_constraints': {
  55                             'vcpus': 1,
  56                             'ram': 1073741824
  57                         },
  58                         'use_existing': enable_reuse,
  59                         'priority': 1,
  60                         'mounts': {
  61                             '/var/spool/cwl': {'kind': 'tmp'}
  62                         },
  63                         'state': 'Committed',
  64                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  65                         'output_path': '/var/spool/cwl',
  66                         'container_image': '99999999999999999999999999999993+99',
  67                         'command': ['ls', '/var/spool/cwl'],
  68                         'cwd': '/var/spool/cwl'
  69                     })
  70
  71     # The test passes some fields in builder.resources
  72     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  73     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  74     def test_resource_requirements(self, keepdocker):
  75         runner = mock.MagicMock()
  76         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  77         runner.ignore_docker_for_reuse = False
  78         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
  79
  80         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  81         runner.api.collections().get().execute.return_value = {
  82             "portable_data_hash": "99999999999999999999999999999993+99"}
  83
  84         tool = {
  85             "inputs": [],
  86             "outputs": [],
  87             "hints": [{
  88                 "class": "ResourceRequirement",
  89                 "coresMin": 3,
  90                 "ramMin": 3000,
  91                 "tmpdirMin": 4000
  92             }, {
  93                 "class": "http://arvados.org/cwl#RuntimeConstraints",
  94                 "keep_cache": 512
  95             }, {
  96                 "class": "http://arvados.org/cwl#APIRequirement",
  97             }, {
  98                 "class": "http://arvados.org/cwl#PartitionRequirement",
  99                 "partition": "blurb"
 100             }],
 101             "baseCommand": "ls"
 102         }
 103         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
 104         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
 105                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
 106                                                  loader=Loader({}))
 107         arvtool.formatgraph = None
 108         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
 109                              make_fs_access=make_fs_access, tmpdir="/tmp"):
 110             j.run()
 111
 112         runner.api.container_requests().create.assert_called_with(
 113             body={
 114                 'environment': {
 115                     'HOME': '/var/spool/cwl',
 116                     'TMPDIR': '/tmp'
 117                 },
 118                 'name': 'test_resource_requirements',
 119                 'runtime_constraints': {
 120                     'vcpus': 3,
 121                     'ram': 3145728000,
 122                     'keep_cache_ram': 512,
 123                     'API': True,
 124                     'partition': ['blurb']
 125                 },
 126                 'use_existing': True,
 127                 'priority': 1,
 128                 'mounts': {
 129                     '/var/spool/cwl': {'kind': 'tmp'}
 130                 },
 131                 'state': 'Committed',
 132                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 133                 'output_path': '/var/spool/cwl',
 134                 'container_image': '99999999999999999999999999999993+99',
 135                 'command': ['ls'],
 136                 'cwd': '/var/spool/cwl'
 137             })
 138
 139     @mock.patch("arvados.collection.Collection")
 140     def test_done(self, col):
 141         api = mock.MagicMock()
 142
 143         runner = mock.MagicMock()
 144         runner.api = api
 145         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 146         runner.num_retries = 0
 147         runner.ignore_docker_for_reuse = False
 148
 149         col().open.return_value = []
 150         api.collections().list().execute.side_effect = ({"items": []},
 151                                                         {"items": [{"manifest_text": "XYZ"}]})
 152
 153         arvjob = arvados_cwl.ArvadosContainer(runner)
 154         arvjob.name = "testjob"
 155         arvjob.builder = mock.MagicMock()
 156         arvjob.output_callback = mock.MagicMock()
 157         arvjob.collect_outputs = mock.MagicMock()
 158         arvjob.successCodes = [0]
 159         arvjob.outdir = "/var/spool/cwl"
 160
 161         arvjob.done({
 162             "state": "Complete",
 163             "output": "99999999999999999999999999999993+99",
 164             "log": "99999999999999999999999999999994+99",
 165             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 166             "exit_code": 0
 167         })
 168
 169         api.collections().list.assert_has_calls([
 170             mock.call(),
 171             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 172                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 173                           ['name', '=', 'Output 9999999 of testjob']]),
 174             mock.call().execute(num_retries=0),
 175             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 176                  select=['manifest_text']),
 177             mock.call().execute(num_retries=0)])
 178
 179         api.collections().create.assert_called_with(
 180             ensure_unique_name=True,
 181             body={'portable_data_hash': '99999999999999999999999999999993+99',
 182                   'manifest_text': 'XYZ',
 183                   'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 184                   'name': 'Output 9999999 of testjob'})
 185
 186     @mock.patch("arvados.collection.Collection")
 187     def test_done_use_existing_collection(self, col):
 188         api = mock.MagicMock()
 189
 190         runner = mock.MagicMock()
 191         runner.api = api
 192         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 193         runner.num_retries = 0
 194
 195         col().open.return_value = []
 196         api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
 197
 198         arvjob = arvados_cwl.ArvadosContainer(runner)
 199         arvjob.name = "testjob"
 200         arvjob.builder = mock.MagicMock()
 201         arvjob.output_callback = mock.MagicMock()
 202         arvjob.collect_outputs = mock.MagicMock()
 203         arvjob.successCodes = [0]
 204         arvjob.outdir = "/var/spool/cwl"
 205
 206         arvjob.done({
 207             "state": "Complete",
 208             "output": "99999999999999999999999999999993+99",
 209             "log": "99999999999999999999999999999994+99",
 210             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 211             "exit_code": 0
 212         })
 213
 214         api.collections().list.assert_has_calls([
 215             mock.call(),
 216             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 217                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 218                                ['name', '=', 'Output 9999999 of testjob']]),
 219             mock.call().execute(num_retries=0)])
 220
 221         self.assertFalse(api.collections().create.called)