From c042cd8120be2c54f54ee2a6041adbc30e141190 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 3 May 2019 14:32:21 -0400 Subject: [PATCH] 15028: More conformance fixes Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/__init__.py | 9 +++++++++ sdk/cwl/arvados_cwl/arvcontainer.py | 4 ++++ sdk/cwl/arvados_cwl/executor.py | 27 ++++++++++++++++++++++----- sdk/cwl/setup.py | 2 +- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index def7203814..f8069a3cb3 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -318,6 +318,15 @@ def main(args, stdout, stderr, api_client=None, keep_client=None, else: arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) + if stdout is sys.stdout: + # cwltool.main has code to work around encoding issues with + # sys.stdout and unix pipes (they default to ASCII encoding, + # we want utf-8), so when stdout is sys.stdout set it to None + # to take advantage of that. Don't override it for all cases + # since we still want to be able to capture stdout for the + # unit tests. + stdout = None + return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index b0dff491f5..860bd954c6 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -225,6 +225,10 @@ class ArvadosContainer(JobBase): runtimeContext.pull_image, runtimeContext.project_uuid) + network_req, _ = self.get_requirement("NetworkAccess") + if network_req: + runtime_constraints["API"] = network_req["networkAccess"] + api_req, _ = self.get_requirement("http://arvados.org/cwl#APIRequirement") if api_req: runtime_constraints["API"] = True diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 9b9c2f1610..9ff06fa4aa 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -6,7 +6,7 @@ from __future__ import division from builtins import next from builtins import object from builtins import str -from future.utils import viewvalues +from future.utils import viewvalues, viewitems import argparse import logging @@ -406,7 +406,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods except (KeyboardInterrupt, SystemExit): break - def check_features(self, obj): + def check_features(self, obj, parentfield=""): if isinstance(obj, dict): if obj.get("writable") and self.work_api != "containers": raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs") @@ -420,12 +420,15 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods "Option 'dockerOutputDirectory' must be an absolute path.") if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers": raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs") - for v in viewvalues(obj): - self.check_features(v) + if obj.get("class") == "InplaceUpdateRequirement": + if obj["inplaceUpdate"] and parentfield == "requirements": + raise SourceLine(obj, "class", UnsupportedRequirement).makeError("InplaceUpdateRequirement not supported for keep collections.") + for k,v in viewitems(obj): + self.check_features(v, parentfield=k) elif isinstance(obj, list): for i,v in enumerate(obj): with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)): - self.check_features(v) + self.check_features(v, parentfield=parentfield) def make_output_collection(self, name, storage_classes, tagsString, outputObj): outputObj = copy.deepcopy(outputObj) @@ -525,6 +528,18 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods 'progress':1.0 }).execute(num_retries=self.num_retries) + def apply_reqs(self, job_order_object, tool): + if "https://w3id.org/cwl/cwl#requirements" in job_order_object: + if tool.metadata.get("http://commonwl.org/cwltool#original_cwlVersion") == 'v1.0': + raise WorkflowException( + "`cwl:requirements` in the input object is not part of CWL " + "v1.0. You can adjust to use `cwltool:overrides` instead; or you " + "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with " + "--enable-dev.") + job_reqs = job_order_object["https://w3id.org/cwl/cwl#requirements"] + for req in job_reqs: + tool.requirements.append(req) + def arv_executor(self, tool, job_order, runtimeContext, logger=None): self.debug = runtimeContext.debug @@ -607,6 +622,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods merged_map=merged_map), "success") + self.apply_reqs(job_order, tool) + self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse self.eval_timeout = runtimeContext.eval_timeout diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index 3af3dfb4da..def79ece40 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner', # Note that arvados/build/run-build-packages.sh looks at this # file to determine what version of cwltool and schema-salad to build. install_requires=[ - 'cwltool==1.0.20190502174223', + 'cwltool==1.0.20190503175737', 'schema-salad==4.1.20190305210046', 'typing >= 3.6.4', 'ruamel.yaml >=0.15.54, <=0.15.77', -- 2.30.2