2 from arvados_cwl.arvdocker import arv_docker_clear_cache
9 from schema_salad.ref_resolver import Loader
11 from schema_salad.ref_resolver import Loader
13 if not os.getenv('ARVADOS_DEBUG'):
14 logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
15 logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
18 class TestContainer(unittest.TestCase):
20 # The test passes no builder.resources
21 # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
22 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
23 def test_run(self, keepdocker):
24 for enable_reuse in (True, False):
25 arv_docker_clear_cache()
27 runner = mock.MagicMock()
28 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
29 runner.ignore_docker_for_reuse = False
31 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
32 runner.api.collections().get().execute.return_value = {
33 "portable_data_hash": "99999999999999999999999999999993+99"}
35 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
41 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
43 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
44 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
45 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
46 arvtool.formatgraph = None
47 for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
48 make_fs_access=make_fs_access, tmpdir="/tmp"):
49 j.run(enable_reuse=enable_reuse)
50 runner.api.container_requests().create.assert_called_with(
53 'HOME': '/var/spool/cwl',
56 'name': 'test_run_'+str(enable_reuse),
57 'runtime_constraints': {
61 'use_existing': enable_reuse,
64 '/var/spool/cwl': {'kind': 'tmp'}
67 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
68 'output_path': '/var/spool/cwl',
69 'container_image': '99999999999999999999999999999993+99',
70 'command': ['ls', '/var/spool/cwl'],
71 'cwd': '/var/spool/cwl',
72 'scheduling_parameters': {}
75 # The test passes some fields in builder.resources
76 # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
77 @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
78 def test_resource_requirements(self, keepdocker):
79 arv_docker_clear_cache()
80 runner = mock.MagicMock()
81 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
82 runner.ignore_docker_for_reuse = False
83 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
85 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
86 runner.api.collections().get().execute.return_value = {
87 "portable_data_hash": "99999999999999999999999999999993+99"}
93 "class": "ResourceRequirement",
98 "class": "http://arvados.org/cwl#RuntimeConstraints",
101 "class": "http://arvados.org/cwl#APIRequirement",
103 "class": "http://arvados.org/cwl#PartitionRequirement",
108 make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
109 arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
110 avsc_names=avsc_names, make_fs_access=make_fs_access,
112 arvtool.formatgraph = None
113 for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements",
114 make_fs_access=make_fs_access, tmpdir="/tmp"):
117 call_args, call_kwargs = runner.api.container_requests().create.call_args
119 call_body_expected = {
121 'HOME': '/var/spool/cwl',
124 'name': 'test_resource_requirements',
125 'runtime_constraints': {
128 'keep_cache_ram': 512,
131 'use_existing': True,
134 '/var/spool/cwl': {'kind': 'tmp'}
136 'state': 'Committed',
137 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
138 'output_path': '/var/spool/cwl',
139 'container_image': '99999999999999999999999999999993+99',
141 'cwd': '/var/spool/cwl',
142 'scheduling_parameters': {
143 'partitions': ['blurb']
147 call_body = call_kwargs.get('body', None)
148 self.assertNotEqual(None, call_body)
149 for key in call_body:
150 self.assertEqual(call_body_expected.get(key), call_body.get(key))
152 @mock.patch("arvados.collection.Collection")
153 def test_done(self, col):
154 api = mock.MagicMock()
156 runner = mock.MagicMock()
158 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
159 runner.num_retries = 0
160 runner.ignore_docker_for_reuse = False
162 col().open.return_value = []
163 api.collections().list().execute.side_effect = ({"items": []},
164 {"items": [{"manifest_text": "XYZ"}]})
166 arvjob = arvados_cwl.ArvadosContainer(runner)
167 arvjob.name = "testjob"
168 arvjob.builder = mock.MagicMock()
169 arvjob.output_callback = mock.MagicMock()
170 arvjob.collect_outputs = mock.MagicMock()
171 arvjob.successCodes = [0]
172 arvjob.outdir = "/var/spool/cwl"
176 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
177 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
178 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
179 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
182 self.assertFalse(api.collections().create.called)
184 @mock.patch("arvados.collection.Collection")
185 def test_done_use_existing_collection(self, col):
186 api = mock.MagicMock()
188 runner = mock.MagicMock()
190 runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
191 runner.num_retries = 0
193 col().open.return_value = []
194 api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
196 arvjob = arvados_cwl.ArvadosContainer(runner)
197 arvjob.name = "testjob"
198 arvjob.builder = mock.MagicMock()
199 arvjob.output_callback = mock.MagicMock()
200 arvjob.collect_outputs = mock.MagicMock()
201 arvjob.successCodes = [0]
202 arvjob.outdir = "/var/spool/cwl"
206 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
207 "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
208 "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
209 "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
210 "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
213 self.assertFalse(api.collections().create.called)