sdk/cwl/tests/test_container.py

   1 import arvados_cwl
   2 import logging
   3 import mock
   4 import unittest
   5 import os
   6 import functools
   7 import cwltool.process
   8 from schema_salad.ref_resolver import Loader
   9
  10 if not os.getenv('ARVADOS_DEBUG'):
  11     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  12     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  13
  14
  15 class TestContainer(unittest.TestCase):
  16
  17     # The test passes no builder.resources
  18     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  19     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  20     def test_run(self, keepdocker):
  21         runner = mock.MagicMock()
  22         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  23         runner.ignore_docker_for_reuse = False
  24
  25         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  26         runner.api.collections().get().execute.return_value = {
  27             "portable_data_hash": "99999999999999999999999999999993+99"}
  28
  29         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
  30
  31         tool = {
  32             "inputs": [],
  33             "outputs": [],
  34             "baseCommand": "ls",
  35             "arguments": [{"valueFrom": "$(runtime.outdir)"}]
  36         }
  37         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  38         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
  39                                                  basedir="", make_fs_access=make_fs_access, loader=Loader({}))
  40         arvtool.formatgraph = None
  41         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run",
  42                              make_fs_access=make_fs_access, tmpdir="/tmp"):
  43             j.run()
  44             runner.api.container_requests().create.assert_called_with(
  45                 body={
  46                     'environment': {
  47                         'HOME': '/var/spool/cwl',
  48                         'TMPDIR': '/tmp'
  49                     },
  50                     'name': 'test_run',
  51                     'runtime_constraints': {
  52                         'vcpus': 1,
  53                         'ram': 1073741824
  54                     }, 'priority': 1,
  55                     'mounts': {
  56                         '/var/spool/cwl': {'kind': 'tmp'}
  57                     },
  58                     'state': 'Committed',
  59                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  60                     'output_path': '/var/spool/cwl',
  61                     'container_image': '99999999999999999999999999999993+99',
  62                     'command': ['ls', '/var/spool/cwl'],
  63                     'cwd': '/var/spool/cwl'
  64                 })
  65
  66     # The test passes some fields in builder.resources
  67     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  68     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
  69     def test_resource_requirements(self, keepdocker):
  70         runner = mock.MagicMock()
  71         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  72         runner.ignore_docker_for_reuse = False
  73         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
  74
  75         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
  76         runner.api.collections().get().execute.return_value = {
  77             "portable_data_hash": "99999999999999999999999999999993+99"}
  78
  79         tool = {
  80             "inputs": [],
  81             "outputs": [],
  82             "hints": [{
  83                 "class": "ResourceRequirement",
  84                 "coresMin": 3,
  85                 "ramMin": 3000,
  86                 "tmpdirMin": 4000
  87             }],
  88             "baseCommand": "ls"
  89         }
  90         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
  91         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
  92                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
  93                                                  loader=Loader({}))
  94         arvtool.formatgraph = None
  95         for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
  96                              make_fs_access=make_fs_access, tmpdir="/tmp"):
  97             j.run()
  98
  99         runner.api.container_requests().create.assert_called_with(
 100             body={
 101                 'environment': {
 102                     'HOME': '/var/spool/cwl',
 103                     'TMPDIR': '/tmp'
 104                 },
 105                 'name': 'test_resource_requirements',
 106                 'runtime_constraints': {
 107                     'vcpus': 3,
 108                     'ram': 3145728000
 109                 }, 'priority': 1,
 110                 'mounts': {
 111                     '/var/spool/cwl': {'kind': 'tmp'}
 112                 },
 113                 'state': 'Committed',
 114                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 115                 'output_path': '/var/spool/cwl',
 116                 'container_image': '99999999999999999999999999999993+99',
 117                 'command': ['ls'],
 118                 'cwd': '/var/spool/cwl'
 119             })
 120
 121     @mock.patch("arvados.collection.Collection")
 122     def test_done(self, col):
 123         api = mock.MagicMock()
 124
 125         runner = mock.MagicMock()
 126         runner.api = api
 127         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 128         runner.num_retries = 0
 129         runner.ignore_docker_for_reuse = False
 130
 131         col().open.return_value = []
 132         api.collections().list().execute.side_effect = ({"items": []},
 133                                                         {"items": [{"manifest_text": "XYZ"}]})
 134
 135         arvjob = arvados_cwl.ArvadosContainer(runner)
 136         arvjob.name = "testjob"
 137         arvjob.builder = mock.MagicMock()
 138         arvjob.output_callback = mock.MagicMock()
 139         arvjob.collect_outputs = mock.MagicMock()
 140         arvjob.successCodes = [0]
 141         arvjob.outdir = "/var/spool/cwl"
 142
 143         arvjob.done({
 144             "state": "Complete",
 145             "output": "99999999999999999999999999999993+99",
 146             "log": "99999999999999999999999999999994+99",
 147             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 148             "exit_code": 0
 149         })
 150
 151         api.collections().list.assert_has_calls([
 152             mock.call(),
 153             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 154                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 155                           ['name', '=', 'Output 9999999 of testjob']]),
 156             mock.call().execute(num_retries=0),
 157             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 158                  select=['manifest_text']),
 159             mock.call().execute(num_retries=0)])
 160
 161         api.collections().create.assert_called_with(
 162             ensure_unique_name=True,
 163             body={'portable_data_hash': '99999999999999999999999999999993+99',
 164                   'manifest_text': 'XYZ',
 165                   'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 166                   'name': 'Output 9999999 of testjob'})
 167
 168     @mock.patch("arvados.collection.Collection")
 169     def test_done_use_existing_collection(self, col):
 170         api = mock.MagicMock()
 171
 172         runner = mock.MagicMock()
 173         runner.api = api
 174         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 175         runner.num_retries = 0
 176
 177         col().open.return_value = []
 178         api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
 179
 180         arvjob = arvados_cwl.ArvadosContainer(runner)
 181         arvjob.name = "testjob"
 182         arvjob.builder = mock.MagicMock()
 183         arvjob.output_callback = mock.MagicMock()
 184         arvjob.collect_outputs = mock.MagicMock()
 185         arvjob.successCodes = [0]
 186         arvjob.outdir = "/var/spool/cwl"
 187
 188         arvjob.done({
 189             "state": "Complete",
 190             "output": "99999999999999999999999999999993+99",
 191             "log": "99999999999999999999999999999994+99",
 192             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
 193             "exit_code": 0
 194         })
 195
 196         api.collections().list.assert_has_calls([
 197             mock.call(),
 198             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 199                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 200                                ['name', '=', 'Output 9999999 of testjob']]),
 201             mock.call().execute(num_retries=0)])
 202
 203         self.assertFalse(api.collections().create.called)