From ac0dc6c57a2b4f736b6faf62421de56a3355db04 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 5 Apr 2019 13:12:45 -0400 Subject: [PATCH] 15028: Update cwltool/schema-salad deps, fix tests Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvworkflow.py | 4 ++++ sdk/cwl/arvados_cwl/executor.py | 1 + sdk/cwl/arvados_cwl/runner.py | 5 +++++ sdk/cwl/setup.py | 4 ++-- sdk/cwl/tests/test_container.py | 12 ++++++------ sdk/cwl/tests/test_job.py | 14 +++++++------- sdk/cwl/tests/test_submit.py | 1 + sdk/cwl/tests/wf/scatter2_subwf.cwl | 10 +++++----- 8 files changed, 31 insertions(+), 20 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 8e6bff4f04..325fccb2c9 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -288,6 +288,10 @@ class ArvadosWorkflow(Workflow): adjustDirObjs(packed, keepmount) self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed) + self.loadingContext = self.loadingContext.copy() + self.loadingContext.metadata = self.loadingContext.metadata.copy() + self.loadingContext.metadata["http://commonwl.org/cwltool#original_cwlVersion"] = "v1.0" + wf_runner = cmap({ "class": "CommandLineTool", "baseCommand": "cwltool", diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index eeb44dbd7f..2875dbc222 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -187,6 +187,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods self.loadingContext.fetcher_constructor = self.fetcher_constructor self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries) self.loadingContext.construct_tool_object = self.arv_make_tool + self.loadingContext.do_update = False # Add a custom logging handler to the root logger for runtime status reporting # if running inside a container diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index 9385bde63c..3235f4763c 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -31,6 +31,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class from cwltool.utils import aslist from cwltool.builder import substitute from cwltool.pack import pack +from cwltool.update import INTERNAL_VERSION import schema_salad.validate as validate import arvados.collection @@ -451,6 +452,10 @@ class Runner(Process): collection_cache_size=256, collection_cache_is_default=True): + loadingContext = loadingContext.copy() + loadingContext.metadata = loadingContext.metadata.copy() + loadingContext.metadata["cwlVersion"] = INTERNAL_VERSION + super(Runner, self).__init__(tool.tool, loadingContext) self.arvrunner = runner diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index 1052fb0d76..3ccf49f9e8 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -33,8 +33,8 @@ setup(name='arvados-cwl-runner', # Note that arvados/build/run-build-packages.sh looks at this # file to determine what version of cwltool and schema-salad to build. install_requires=[ - 'cwltool==1.0.20181217162649', - 'schema-salad==3.0.20181129082112', + 'cwltool==1.0.20190422193858', + 'schema-salad==4.1.20190305210046', 'typing >= 3.6.4', 'ruamel.yaml >=0.15.54, <=0.15.77', 'arvados-python-client>=1.3.0.20190205182514', diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 07d962bf9b..2ddd06535e 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -57,7 +57,7 @@ class CollectionMock(object): class TestContainer(unittest.TestCase): def helper(self, runner, enable_reuse=True): - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1") make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) @@ -66,7 +66,7 @@ class TestContainer(unittest.TestCase): "basedir": "", "make_fs_access": make_fs_access, "loader": Loader({}), - "metadata": {"cwlVersion": "v1.0"}}) + "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}}) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", "basedir": "", @@ -400,7 +400,7 @@ class TestContainer(unittest.TestCase): runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99"} - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1") tool = cmap({ "inputs": [], @@ -607,7 +607,7 @@ class TestContainer(unittest.TestCase): "portable_data_hash": "99999999999999999999999999999994+99", "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"} - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1") tool = cmap({ "inputs": [ @@ -697,7 +697,7 @@ class TestContainer(unittest.TestCase): runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99"} - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1") tool = cmap({"arguments": ["md5sum", "example.conf"], "class": "CommandLineTool", @@ -803,7 +803,7 @@ class TestContainer(unittest.TestCase): "class": "CommandLineTool", "hints": [ { - "class": "http://commonwl.org/cwltool#TimeLimit", + "class": "ToolTimeLimit", "timelimit": 42 } ] diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py index 022d75be11..9cac68aa6c 100644 --- a/sdk/cwl/tests/test_job.py +++ b/sdk/cwl/tests/test_job.py @@ -34,7 +34,7 @@ if not os.getenv('ARVADOS_DEBUG'): class TestJob(unittest.TestCase): def helper(self, runner, enable_reuse=True): - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1") make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) @@ -43,7 +43,7 @@ class TestJob(unittest.TestCase): "basedir": "", "make_fs_access": make_fs_access, "loader": Loader({}), - "metadata": {"cwlVersion": "v1.0"}, + "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}, "makeTool": runner.arv_make_tool}) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "jobs", @@ -343,7 +343,7 @@ class TestJob(unittest.TestCase): class TestWorkflow(unittest.TestCase): def helper(self, runner, enable_reuse=True): - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1") make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) @@ -358,7 +358,7 @@ class TestWorkflow(unittest.TestCase): "basedir": "", "make_fs_access": make_fs_access, "loader": document_loader, - "metadata": {"cwlVersion": "v1.0"}, + "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}, "construct_tool_object": runner.arv_make_tool}) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "jobs", @@ -458,6 +458,7 @@ bytes(b'''{ @mock.patch("arvados.collection.Collection") @mock.patch('arvados.commands.keepdocker.list_images_in_arv') def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader): + # TODO copy this over to test_container arvados_cwl.add_arv_hints() api = mock.MagicMock() @@ -475,16 +476,15 @@ bytes(b'''{ runner.num_retries = 0 loadingContext, runtimeContext = self.helper(runner) - + loadingContext.do_update = True tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl") - metadata["cwlVersion"] = tool["cwlVersion"] mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs) arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext) arvtool.formatgraph = None it = arvtool.job({}, mock.MagicMock(), runtimeContext) - + next(it).run(runtimeContext) next(it).run(runtimeContext) diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 9535f6ba20..5f92cee942 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -84,6 +84,7 @@ def stubs(func): stubs.api = mock.MagicMock() stubs.api._rootDesc = get_rootDesc() + stubs.api._rootDesc["uuidPrefix"] = "zzzzz" stubs.api.users().current().execute.return_value = { "uuid": stubs.fake_user_uuid, diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl index 2af1155437..ecd36026f2 100644 --- a/sdk/cwl/tests/wf/scatter2_subwf.cwl +++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl @@ -6,7 +6,7 @@ "$graph": [ { "class": "Workflow", - "cwlVersion": "v1.0", + "cwlVersion": "v1.1.0-dev1", "hints": [], "id": "#main", "inputs": [ @@ -59,10 +59,10 @@ "run": { "baseCommand": "sleep", "class": "CommandLineTool", - "id": "#main/sleep1/subtool", + "id": "#main/sleep1/run/subtool", "inputs": [ { - "id": "#main/sleep1/subtool/sleeptime", + "id": "#main/sleep1/run/subtool/sleeptime", "inputBinding": { "position": 1 }, @@ -71,7 +71,7 @@ ], "outputs": [ { - "id": "#main/sleep1/subtool/out", + "id": "#main/sleep1/run/subtool/out", "outputBinding": { "outputEval": "out" }, @@ -83,5 +83,5 @@ ] } ], - "cwlVersion": "v1.0" + "cwlVersion": "v1.1.0-dev1" } \ No newline at end of file -- 2.30.2