8 from schema_salad.ref_resolver import Loader
10 from schema_salad.ref_resolver import Loader
12 if not os.getenv('ARVADOS_DEBUG'):
13 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
14 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
17 class TestContainer(unittest.TestCase):
19 # The test passes no builder.resources
20 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
21 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
22 def test_run(self, keepdocker):
23 runner = mock.MagicMock()
24 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
25 runner.ignore_docker_for_reuse = False
27 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
28 runner.api.collections().get().execute.return_value = {
29 "portable_data_hash": "99999999999999999999999999999993+99"}
31 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
37 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
39 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
40 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
41 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
42 arvtool.formatgraph = None
43 for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run",
44 make_fs_access=make_fs_access, tmpdir="/tmp"):
46 runner.api.container_requests().create.assert_called_with(
49 'HOME': '/var/spool/cwl',
53 'runtime_constraints': {
58 '/var/spool/cwl': {'kind': 'tmp'}
61 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
62 'output_path': '/var/spool/cwl',
63 'container_image': '99999999999999999999999999999993+99',
64 'command': ['ls', '/var/spool/cwl'],
65 'cwd': '/var/spool/cwl'
68 # The test passes some fields in builder.resources
69 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
70 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
71 def test_resource_requirements(self, keepdocker):
72 runner = mock.MagicMock()
73 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
74 runner.ignore_docker_for_reuse = False
75 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
77 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
78 runner.api.collections().get().execute.return_value = {
79 "portable_data_hash": "99999999999999999999999999999993+99"}
85 "class": "ResourceRequirement",
90 "class": "http://arvados.org/cwl#RuntimeConstraints",
93 "class": "http://arvados.org/cwl#APIRequirement",
95 "class": "http://arvados.org/cwl#PartitionRequirement",
100 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
101 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
102 avsc_names=avsc_names, make_fs_access=make_fs_access,
104 arvtool.formatgraph = None
105 for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
106 make_fs_access=make_fs_access, tmpdir="/tmp"):
109 runner.api.container_requests().create.assert_called_with(
112 'HOME': '/var/spool/cwl',
115 'name': 'test_resource_requirements',
116 'runtime_constraints': {
120 'partition': ['blurb']
123 '/var/spool/cwl': {'kind': 'tmp'}
125 'state': 'Committed',
126 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
127 'output_path': '/var/spool/cwl',
128 'container_image': '99999999999999999999999999999993+99',
130 'cwd': '/var/spool/cwl'
133 @mock.patch("arvados.collection.Collection")
134 def test_done(self, col):
135 api = mock.MagicMock()
137 runner = mock.MagicMock()
139 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
140 runner.num_retries = 0
141 runner.ignore_docker_for_reuse = False
143 col().open.return_value = []
144 api.collections().list().execute.side_effect = ({"items": []},
145 {"items": [{"manifest_text": "XYZ"}]})
147 arvjob = arvados_cwl.ArvadosContainer(runner)
148 arvjob.name = "testjob"
149 arvjob.builder = mock.MagicMock()
150 arvjob.output_callback = mock.MagicMock()
151 arvjob.collect_outputs = mock.MagicMock()
152 arvjob.successCodes = [0]
153 arvjob.outdir = "/var/spool/cwl"
157 "output": "99999999999999999999999999999993+99",
158 "log": "99999999999999999999999999999994+99",
159 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
163 api.collections().list.assert_has_calls([
165 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
166 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
167 ['name', '=', 'Output 9999999 of testjob']]),
168 mock.call().execute(num_retries=0),
169 mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
170 select=['manifest_text']),
171 mock.call().execute(num_retries=0)])
173 api.collections().create.assert_called_with(
174 ensure_unique_name=True,
175 body={'portable_data_hash': '99999999999999999999999999999993+99',
176 'manifest_text': 'XYZ',
177 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
178 'name': 'Output 9999999 of testjob'})
180 @mock.patch("arvados.collection.Collection")
181 def test_done_use_existing_collection(self, col):
182 api = mock.MagicMock()
184 runner = mock.MagicMock()
186 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
187 runner.num_retries = 0
189 col().open.return_value = []
190 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
192 arvjob = arvados_cwl.ArvadosContainer(runner)
193 arvjob.name = "testjob"
194 arvjob.builder = mock.MagicMock()
195 arvjob.output_callback = mock.MagicMock()
196 arvjob.collect_outputs = mock.MagicMock()
197 arvjob.successCodes = [0]
198 arvjob.outdir = "/var/spool/cwl"
202 "output": "99999999999999999999999999999993+99",
203 "log": "99999999999999999999999999999994+99",
204 "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
208 api.collections().list.assert_has_calls([
210 mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
211 ['portable_data_hash', '=', '99999999999999999999999999999993+99'],
212 ['name', '=', 'Output 9999999 of testjob']]),
213 mock.call().execute(num_retries=0)])
215 self.assertFalse(api.collections().create.called)