sdk/cwl/tests/test_job.py

   1 import arvados_cwl
   2 import logging
   3 import mock
   4 import unittest
   5 import os
   6 import cwltool.process
   7
   8 if not os.getenv('ARVADOS_DEBUG'):
   9     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
  10     logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  11
  12
  13 class TestJob(unittest.TestCase):
  14
  15     # The test passes no builder.resources
  16     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  17     def test_run(self):
  18         runner = mock.MagicMock()
  19         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  20         runner.ignore_docker_for_reuse = False
  21         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
  22
  23         tool = {
  24             "inputs": [],
  25             "outputs": [],
  26             "baseCommand": "ls"
  27         }
  28         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, avsc_names=avsc_names, basedir="")
  29         arvtool.formatgraph = None
  30         for j in arvtool.job({}, mock.MagicMock(), basedir=""):
  31             j.run()
  32             runner.api.jobs().create.assert_called_with(
  33                 body={
  34                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  35                     'runtime_constraints': {},
  36                     'script_parameters': {
  37                         'tasks': [{
  38                             'task.env': {'TMPDIR': '$(task.tmpdir)'},
  39                             'command': ['ls']
  40                         }],
  41                     },
  42                     'script_version': 'master',
  43                     'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
  44                     'repository': 'arvados',
  45                     'script': 'crunchrunner',
  46                     'runtime_constraints': {
  47                         'docker_image': 'arvados/jobs',
  48                         'min_cores_per_node': 1,
  49                         'min_ram_mb_per_node': 1024,
  50                         'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
  51                     }
  52                 },
  53                 find_or_create=True,
  54                 filters=[['repository', '=', 'arvados'],
  55                          ['script', '=', 'crunchrunner'],
  56                          ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
  57                          ['docker_image_locator', 'in docker', 'arvados/jobs']]
  58             )
  59
  60     # The test passes some fields in builder.resources
  61     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  62     def test_resource_requirements(self):
  63         runner = mock.MagicMock()
  64         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  65         runner.ignore_docker_for_reuse = False
  66         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
  67
  68         tool = {
  69             "inputs": [],
  70             "outputs": [],
  71             "hints": [{
  72                 "class": "ResourceRequirement",
  73                 "coresMin": 3,
  74                 "ramMin": 3000,
  75                 "tmpdirMin": 4000
  76             }],
  77             "baseCommand": "ls"
  78         }
  79         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, avsc_names=avsc_names)
  80         arvtool.formatgraph = None
  81         for j in arvtool.job({}, mock.MagicMock(), basedir=""):
  82             j.run()
  83         runner.api.jobs().create.assert_called_with(
  84             body={
  85                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  86                 'runtime_constraints': {},
  87                 'script_parameters': {
  88                     'tasks': [{
  89                         'task.env': {'TMPDIR': '$(task.tmpdir)'},
  90                         'command': ['ls']
  91                     }]
  92             },
  93             'script_version': 'master',
  94                 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
  95                 'repository': 'arvados',
  96                 'script': 'crunchrunner',
  97                 'runtime_constraints': {
  98                     'docker_image': 'arvados/jobs',
  99                     'min_cores_per_node': 3,
 100                     'min_ram_mb_per_node': 3000,
 101                     'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
 102                 }
 103             },
 104             find_or_create=True,
 105             filters=[['repository', '=', 'arvados'],
 106                      ['script', '=', 'crunchrunner'],
 107                      ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
 108                      ['docker_image_locator', 'in docker', 'arvados/jobs']])
 109
 110     @mock.patch("arvados.collection.Collection")
 111     def test_done(self, col):
 112         api = mock.MagicMock()
 113
 114         runner = mock.MagicMock()
 115         runner.api = api
 116         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 117         runner.num_retries = 0
 118         runner.ignore_docker_for_reuse = False
 119
 120         col().open.return_value = []
 121         api.collections().list().execute.side_effect = ({"items": []},
 122                                                         {"items": [{"manifest_text": "XYZ"}]})
 123
 124         arvjob = arvados_cwl.ArvadosJob(runner)
 125         arvjob.name = "testjob"
 126         arvjob.builder = mock.MagicMock()
 127         arvjob.output_callback = mock.MagicMock()
 128         arvjob.collect_outputs = mock.MagicMock()
 129
 130         arvjob.done({
 131             "state": "Complete",
 132             "output": "99999999999999999999999999999993+99",
 133             "log": "99999999999999999999999999999994+99",
 134             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 135         })
 136
 137         api.collections().list.assert_has_calls([
 138             mock.call(),
 139             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 140                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 141                           ['name', '=', 'Output 9999999 of testjob']]),
 142             mock.call().execute(num_retries=0),
 143             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 144                  select=['manifest_text']),
 145             mock.call().execute(num_retries=0)])
 146
 147         api.collections().create.assert_called_with(
 148             ensure_unique_name=True,
 149             body={'portable_data_hash': '99999999999999999999999999999993+99',
 150                   'manifest_text': 'XYZ',
 151                   'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 152                   'name': 'Output 9999999 of testjob'})
 153
 154     @mock.patch("arvados.collection.Collection")
 155     def test_done_use_existing_collection(self, col):
 156         api = mock.MagicMock()
 157
 158         runner = mock.MagicMock()
 159         runner.api = api
 160         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 161         runner.num_retries = 0
 162
 163         col().open.return_value = []
 164         api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
 165
 166         arvjob = arvados_cwl.ArvadosJob(runner)
 167         arvjob.name = "testjob"
 168         arvjob.builder = mock.MagicMock()
 169         arvjob.output_callback = mock.MagicMock()
 170         arvjob.collect_outputs = mock.MagicMock()
 171
 172         arvjob.done({
 173             "state": "Complete",
 174             "output": "99999999999999999999999999999993+99",
 175             "log": "99999999999999999999999999999994+99",
 176             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 177         })
 178
 179         api.collections().list.assert_has_calls([
 180             mock.call(),
 181             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 182                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 183                                ['name', '=', 'Output 9999999 of testjob']]),
 184             mock.call().execute(num_retries=0)])
 185
 186         self.assertFalse(api.collections().create.called)