9 if not os.getenv('ARVADOS_DEBUG'):
10 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
11 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
14 class TestJob(unittest.TestCase):
16 # The test passes no builder.resources
17 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
19 runner = mock.MagicMock()
20 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
21 runner.ignore_docker_for_reuse = False
22 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
28 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
30 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
31 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access)
32 arvtool.formatgraph = None
33 for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
35 runner.api.jobs().create.assert_called_with(
37 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
38 'runtime_constraints': {},
39 'script_parameters': {
41 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
42 'command': ['ls', '$(task.outdir)']
45 'script_version': 'master',
46 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
47 'repository': 'arvados',
48 'script': 'crunchrunner',
49 'runtime_constraints': {
50 'docker_image': 'arvados/jobs',
51 'min_cores_per_node': 1,
52 'min_ram_mb_per_node': 1024,
53 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
57 filters=[['repository', '=', 'arvados'],
58 ['script', '=', 'crunchrunner'],
59 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
60 ['docker_image_locator', 'in docker', 'arvados/jobs']]
63 # The test passes some fields in builder.resources
64 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
65 def test_resource_requirements(self):
66 runner = mock.MagicMock()
67 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
68 runner.ignore_docker_for_reuse = False
69 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
75 "class": "ResourceRequirement",
82 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
83 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, make_fs_access=make_fs_access)
84 arvtool.formatgraph = None
85 for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access):
87 runner.api.jobs().create.assert_called_with(
89 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
90 'runtime_constraints': {},
91 'script_parameters': {
93 'task.env': {'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)'},
97 'script_version': 'master',
98 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6',
99 'repository': 'arvados',
100 'script': 'crunchrunner',
101 'runtime_constraints': {
102 'docker_image': 'arvados/jobs',
103 'min_cores_per_node': 3,
104 'min_ram_mb_per_node': 3000,
105 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
109 filters=[['repository', '=', 'arvados'],
110 ['script', '=', 'crunchrunner'],
111 ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'],
112 ['docker_image_locator', 'in docker', 'arvados/jobs']])
114 @mock.patch("arvados.collection.Collection")
115 def test_done(self, col):
116 api = mock.MagicMock()
118 runner = mock.MagicMock()
120 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
121 runner.num_retries = 0
122 runner.ignore_docker_for_reuse = False
124 col().open.return_value = []
125 api.collections().list().execute.side_effect = ({"items": []},
126 {"items": [{"manifest_text": "XYZ"}]})
128 arvjob = arvados_cwl.ArvadosJob(runner)
129 arvjob.name = "testjob"
130 arvjob.builder = mock.MagicMock()
131 arvjob.output_callback = mock.MagicMock()
132 arvjob.collect_outputs = mock.MagicMock()
136 "output": "99999999999999999999999999999993+99",
137 "log": "99999999999999999999999999999994+99",
138 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
141 api.collections().list.assert_has_calls([
143 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
144 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
145 ['name', '=', 'Output 9999999 of testjob']]),
146 mock.call().execute(num_retries=0),
147 mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
148 select=['manifest_text']),
149 mock.call().execute(num_retries=0)])
151 api.collections().create.assert_called_with(
152 ensure_unique_name=True,
153 body={'portable_data_hash': '99999999999999999999999999999993+99',
154 'manifest_text': 'XYZ',
155 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
156 'name': 'Output 9999999 of testjob'})
158 @mock.patch("arvados.collection.Collection")
159 def test_done_use_existing_collection(self, col):
160 api = mock.MagicMock()
162 runner = mock.MagicMock()
164 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
165 runner.num_retries = 0
167 col().open.return_value = []
168 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
170 arvjob = arvados_cwl.ArvadosJob(runner)
171 arvjob.name = "testjob"
172 arvjob.builder = mock.MagicMock()
173 arvjob.output_callback = mock.MagicMock()
174 arvjob.collect_outputs = mock.MagicMock()
178 "output": "99999999999999999999999999999993+99",
179 "log": "99999999999999999999999999999994+99",
180 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
183 api.collections().list.assert_has_calls([
185 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
186 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
187 ['name', '=', 'Output 9999999 of testjob']]),
188 mock.call().execute(num_retries=0)])
190 self.assertFalse(api.collections().create.called)