sdk/cwl/tests/test_job.py

   1 import unittest
   2 import mock
   3 import arvados_cwl
   4
   5 class TestJob(unittest.TestCase):
   6
   7     # The test passes no builder.resources
   8     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
   9     def test_run(self):
  10         runner = mock.MagicMock()
  11         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  12         runner.ignore_docker_for_reuse = False
  13
  14         tool = {
  15             "inputs": [],
  16             "outputs": [],
  17             "baseCommand": "ls"
  18         }
  19         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool)
  20         arvtool.formatgraph = None
  21         for j in arvtool.job({}, "", mock.MagicMock()):
  22             j.run()
  23             runner.api.jobs().create.assert_called_with(
  24                 body={
  25                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  26                     'runtime_constraints': {},
  27                     'script_parameters': {
  28                         'tasks': [{
  29                             'task.env': {'TMPDIR': '$(task.tmpdir)'},
  30                             'command': ['ls']
  31                         }]
  32             },
  33             'script_version': 'master',
  34                     'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
  35                     'repository': 'arvados',
  36                     'script': 'crunchrunner',
  37                     'runtime_constraints': {
  38                         'docker_image': 'arvados/jobs',
  39                         'min_cores_per_node': 1,
  40                         'min_ram_mb_per_node': 1024,
  41                         'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
  42                     }
  43         },
  44                                                     find_or_create=True,
  45                 filters=[['repository', '=', 'arvados'],
  46                          ['script', '=', 'crunchrunner'],
  47                          ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
  48                          ['docker_image_locator', 'in docker', 'arvados/jobs']]
  49             )
  50
  51     # The test passes some fields in builder.resources
  52     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
  53     def test_resource_requirements(self):
  54         runner = mock.MagicMock()
  55         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
  56         runner.ignore_docker_for_reuse = False
  57
  58         tool = {
  59             "inputs": [],
  60             "outputs": [],
  61             "hints": [{
  62                 "class": "ResourceRequirement",
  63                 "coresMin": 3,
  64                 "ramMin": 3000,
  65                 "tmpdirMin": 4000
  66             }],
  67             "baseCommand": "ls"
  68         }
  69         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool)
  70         arvtool.formatgraph = None
  71         for j in arvtool.job({}, "", mock.MagicMock()):
  72             j.run()
  73         runner.api.jobs().create.assert_called_with(
  74             body={
  75                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
  76                 'runtime_constraints': {},
  77                 'script_parameters': {
  78                     'tasks': [{
  79                         'task.env': {'TMPDIR': '$(task.tmpdir)'},
  80                         'command': ['ls']
  81                     }]
  82             },
  83             'script_version': 'master',
  84                 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
  85                 'repository': 'arvados',
  86                 'script': 'crunchrunner',
  87                 'runtime_constraints': {
  88                     'docker_image': 'arvados/jobs',
  89                     'min_cores_per_node': 3,
  90                     'min_ram_mb_per_node': 3000,
  91                     'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
  92                 }
  93         },
  94             find_or_create=True,
  95             filters=[['repository', '=', 'arvados'],
  96                      ['script', '=', 'crunchrunner'],
  97                      ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
  98                      ['docker_image_locator', 'in docker', 'arvados/jobs']])
  99
 100     @mock.patch("arvados.collection.Collection")
 101     def test_done(self, col):
 102         api = mock.MagicMock()
 103
 104         runner = mock.MagicMock()
 105         runner.api = api
 106         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 107         runner.num_retries = 0
 108         runner.ignore_docker_for_reuse = False
 109
 110         col().open.return_value = []
 111         api.collections().list().execute.side_effect = ({"items": []},
 112                                                         {"items": [{"manifest_text": "XYZ"}]})
 113
 114         arvjob = arvados_cwl.ArvadosJob(runner)
 115         arvjob.name = "testjob"
 116         arvjob.builder = mock.MagicMock()
 117         arvjob.output_callback = mock.MagicMock()
 118         arvjob.collect_outputs = mock.MagicMock()
 119
 120         arvjob.done({
 121             "state": "Complete",
 122             "output": "99999999999999999999999999999993+99",
 123             "log": "99999999999999999999999999999994+99",
 124             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 125         })
 126
 127         api.collections().list.assert_has_calls([
 128             mock.call(),
 129             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 130                           ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 131                           ['name', '=', 'Output 9999999 of testjob']]),
 132             mock.call().execute(num_retries=0),
 133             mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
 134                  select=['manifest_text']),
 135             mock.call().execute(num_retries=0)])
 136
 137         api.collections().create.assert_called_with(
 138             ensure_unique_name=True,
 139             body={'portable_data_hash': '99999999999999999999999999999993+99',
 140                   'manifest_text': 'XYZ',
 141                   'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
 142                   'name': 'Output 9999999 of testjob'})
 143
 144     @mock.patch("arvados.collection.Collection")
 145     def test_done_use_existing_collection(self, col):
 146         api = mock.MagicMock()
 147
 148         runner = mock.MagicMock()
 149         runner.api = api
 150         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 151         runner.num_retries = 0
 152
 153         col().open.return_value = []
 154         api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
 155
 156         arvjob = arvados_cwl.ArvadosJob(runner)
 157         arvjob.name = "testjob"
 158         arvjob.builder = mock.MagicMock()
 159         arvjob.output_callback = mock.MagicMock()
 160         arvjob.collect_outputs = mock.MagicMock()
 161
 162         arvjob.done({
 163             "state": "Complete",
 164             "output": "99999999999999999999999999999993+99",
 165             "log": "99999999999999999999999999999994+99",
 166             "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
 167         })
 168
 169         api.collections().list.assert_has_calls([
 170             mock.call(),
 171             mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
 172                                ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
 173                                ['name', '=', 'Output 9999999 of testjob']]),
 174             mock.call().execute(num_retries=0)])
 175
 176         self.assertFalse(api.collections().create.called)