9 if not os.getenv('ARVADOS_DEBUG'):
10 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
11 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
14 class TestJob(unittest.TestCase):
16 # The test passes no builder.resources
17 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
19 runner = mock.MagicMock()
20 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
21 runner.ignore_docker_for_reuse = False
22 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
28 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
30 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="")
31 arvtool.formatgraph = None
32 for j in arvtool.job({}, mock.MagicMock(), basedir="",
33 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)):
35 runner.api.jobs().create.assert_called_with(
37 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
38 'runtime_constraints': {},
39 'script_parameters': {
41 'task.env': {'TMPDIR': '$(task.tmpdir)'},
42 'command': ['ls', '$(task.outdir)']
45 'script_version': 'master',
46 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
47 'repository': 'arvados',
48 'script': 'crunchrunner',
49 'runtime_constraints': {
50 'docker_image': 'arvados/jobs',
51 'min_cores_per_node': 1,
52 'min_ram_mb_per_node': 1024,
53 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
57 filters=[['repository', '=', 'arvados'],
58 ['script', '=', 'crunchrunner'],
59 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
60 ['docker_image_locator', 'in docker', 'arvados/jobs']]
63 # The test passes some fields in builder.resources
64 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
65 def test_resource_requirements(self):
66 runner = mock.MagicMock()
67 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
68 runner.ignore_docker_for_reuse = False
69 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
75 "class": "ResourceRequirement",
82 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names)
83 arvtool.formatgraph = None
84 for j in arvtool.job({}, mock.MagicMock(), basedir=""):
86 runner.api.jobs().create.assert_called_with(
88 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
89 'runtime_constraints': {},
90 'script_parameters': {
92 'task.env': {'TMPDIR': '$(task.tmpdir)'},
96 'script_version': 'master',
97 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
98 'repository': 'arvados',
99 'script': 'crunchrunner',
100 'runtime_constraints': {
101 'docker_image': 'arvados/jobs',
102 'min_cores_per_node': 3,
103 'min_ram_mb_per_node': 3000,
104 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
108 filters=[['repository', '=', 'arvados'],
109 ['script', '=', 'crunchrunner'],
110 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
111 ['docker_image_locator', 'in docker', 'arvados/jobs']])
113 @mock.patch("arvados.collection.Collection")
114 def test_done(self, col):
115 api = mock.MagicMock()
117 runner = mock.MagicMock()
119 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
120 runner.num_retries = 0
121 runner.ignore_docker_for_reuse = False
123 col().open.return_value = []
124 api.collections().list().execute.side_effect = ({"items": []},
125 {"items": [{"manifest_text": "XYZ"}]})
127 arvjob = arvados_cwl.ArvadosJob(runner)
128 arvjob.name = "testjob"
129 arvjob.builder = mock.MagicMock()
130 arvjob.output_callback = mock.MagicMock()
131 arvjob.collect_outputs = mock.MagicMock()
135 "output": "99999999999999999999999999999993+99",
136 "log": "99999999999999999999999999999994+99",
137 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
140 api.collections().list.assert_has_calls([
142 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
143 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
144 ['name', '=', 'Output 9999999 of testjob']]),
145 mock.call().execute(num_retries=0),
146 mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
147 select=['manifest_text']),
148 mock.call().execute(num_retries=0)])
150 api.collections().create.assert_called_with(
151 ensure_unique_name=True,
152 body={'portable_data_hash': '99999999999999999999999999999993+99',
153 'manifest_text': 'XYZ',
154 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
155 'name': 'Output 9999999 of testjob'})
157 @mock.patch("arvados.collection.Collection")
158 def test_done_use_existing_collection(self, col):
159 api = mock.MagicMock()
161 runner = mock.MagicMock()
163 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
164 runner.num_retries = 0
166 col().open.return_value = []
167 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
169 arvjob = arvados_cwl.ArvadosJob(runner)
170 arvjob.name = "testjob"
171 arvjob.builder = mock.MagicMock()
172 arvjob.output_callback = mock.MagicMock()
173 arvjob.collect_outputs = mock.MagicMock()
177 "output": "99999999999999999999999999999993+99",
178 "log": "99999999999999999999999999999994+99",
179 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
182 api.collections().list.assert_has_calls([
184 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
185 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
186 ['name', '=', 'Output 9999999 of testjob']]),
187 mock.call().execute(num_retries=0)])
189 self.assertFalse(api.collections().create.called)