From: Fuad Muhic Date: Thu, 5 Jul 2018 15:36:24 +0000 (+0200) Subject: Merge branch 'master' of git.curoverse.com:arvados into 13330-cwl-intermediate-collec... X-Git-Tag: 1.2.0~85^2~3 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/5c4d9d38dcee73a7ffb6221c80f707c3924da64f Merge branch 'master' of git.curoverse.com:arvados into 13330-cwl-intermediate-collections-cleanup Arvados-DCO-1.1-Signed-off-by: Fuad Muhic --- 5c4d9d38dcee73a7ffb6221c80f707c3924da64f diff --cc sdk/cwl/arvados_cwl/arvjob.py index 332c6a7b9f,70c2173db9..2d112c87a7 --- a/sdk/cwl/arvados_cwl/arvjob.py +++ b/sdk/cwl/arvados_cwl/arvjob.py @@@ -6,10 -6,9 +6,10 @@@ import loggin import re import copy import json +import datetime import time - from cwltool.process import get_feature, shortname, UnsupportedRequirement + from cwltool.process import shortname, UnsupportedRequirement from cwltool.errors import WorkflowException from cwltool.command_line_tool import revmap_file, CommandLineTool from cwltool.load_tool import fetch_document diff --cc sdk/cwl/tests/test_container.py index a8a91efd3a,dd484690c1..bd5b1a1b61 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@@ -641,15 -649,45 +656,59 @@@ class TestContainer(unittest.TestCase) } })) + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_timelimit(self, keepdocker): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "#", + "class": "CommandLineTool", + "hints": [ + { + "class": "http://commonwl.org/cwltool#TimeLimit", + "timelimit": 42 + } + ] + }) + + loadingContext, runtimeContext = self.helper(runner) + runtimeContext.name = "test_timelimit" + + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + + _, kwargs = runner.api.container_requests().create.call_args + self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time')) ++ ++ + def test_get_intermediate_collection_info(self): + arvrunner = mock.MagicMock() + arvrunner.intermediate_output_ttl = 60 + arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"} + + container = arvados_cwl.ArvadosContainer(arvrunner) + + info = container._get_intermediate_collection_info() + + self.assertEqual(info["name"], "Intermediate collection") + self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1)) + self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}) diff --cc sdk/cwl/tests/test_job.py index 1841fd3f89,c110bc5d53..b9f1c39617 --- a/sdk/cwl/tests/test_job.py +++ b/sdk/cwl/tests/test_job.py @@@ -25,15 -24,30 +25,37 @@@ if not os.getenv('ARVADOS_DEBUG') logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN) logging.getLogger('arvados.arv-run').setLevel(logging.WARN) +class MockDateTime(datetime.datetime): + @classmethod + def now(cls): + return datetime.datetime(2018, 1, 1, 0, 0, 0, 0) + +datetime.datetime = MockDateTime + class TestJob(unittest.TestCase): + def helper(self, runner, enable_reuse=True): + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + loadingContext = arvados_cwl.context.ArvLoadingContext( + {"avsc_names": avsc_names, + "basedir": "", + "make_fs_access": make_fs_access, + "loader": Loader({}), + "metadata": {"cwlVersion": "v1.0"}, + "makeTool": runner.arv_make_tool}) + runtimeContext = arvados_cwl.context.ArvRuntimeContext( + {"work_api": "jobs", + "basedir": "", + "name": "test_run_job_"+str(enable_reuse), + "make_fs_access": make_fs_access, + "enable_reuse": enable_reuse, + "priority": 500}) + + return loadingContext, runtimeContext + # The test passes no builder.resources # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch('arvados.commands.keepdocker.list_images_in_arv') @@@ -315,22 -331,36 +339,50 @@@ arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") + def test_get_intermediate_collection_info(self): + arvrunner = mock.MagicMock() + arvrunner.intermediate_output_ttl = 60 + arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"} + + job = arvados_cwl.ArvadosJob(arvrunner) + + info = job._get_intermediate_collection_info() + + self.assertEqual(info["name"], "Intermediate collection") + self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1)) + self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}) + + class TestWorkflow(unittest.TestCase): + def helper(self, runner, enable_reuse=True): + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + + document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access("")) + document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session) + document_loader.fetch_text = document_loader.fetcher.fetch_text + document_loader.check_exists = document_loader.fetcher.check_exists + + loadingContext = arvados_cwl.context.ArvLoadingContext( + {"avsc_names": avsc_names, + "basedir": "", + "make_fs_access": make_fs_access, + "loader": document_loader, + "metadata": {"cwlVersion": "v1.0"}, + "construct_tool_object": runner.arv_make_tool}) + runtimeContext = arvados_cwl.context.ArvRuntimeContext( + {"work_api": "jobs", + "basedir": "", + "name": "test_run_wf_"+str(enable_reuse), + "make_fs_access": make_fs_access, + "enable_reuse": enable_reuse, + "priority": 500}) + + return loadingContext, runtimeContext + # The test passes no builder.resources # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.collection.CollectionReader")