From 580c55d60829f17118f0c8e57d1d72c84a71b62b Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 26 Nov 2018 10:13:39 -0500 Subject: [PATCH] 14198: Restore behavior of creating an empty Builder object Used for do_eval() on set_cluster_target(). Calling _init_job() turns out to have some unintended side effects as well as doing redundant work. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvtool.py | 28 +++++++++++++++++++++++++++- sdk/cwl/arvados_cwl/arvworkflow.py | 5 ++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py index 5257645304..cd319e55b1 100644 --- a/sdk/cwl/arvados_cwl/arvtool.py +++ b/sdk/cwl/arvados_cwl/arvtool.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from cwltool.command_line_tool import CommandLineTool +from cwltool.builder import Builder from .arvjob import ArvadosJob from .arvcontainer import ArvadosContainer from .pathmapper import ArvPathMapper @@ -36,6 +37,31 @@ def set_cluster_target(tool, arvrunner, builder, runtimeContext): return runtimeContext +def make_builder(joborder, hints, requirements, runtimeContext): + return Builder( + job=joborder, + files=[], # type: List[Dict[Text, Text]] + bindings=[], # type: List[Dict[Text, Any]] + schemaDefs={}, # type: Dict[Text, Dict[Text, Any]] + names=None, # type: Names + requirements=requirements, # type: List[Dict[Text, Any]] + hints=hints, # type: List[Dict[Text, Any]] + resources={}, # type: Dict[str, int] + mutation_manager=None, # type: Optional[MutationManager] + formatgraph=None, # type: Optional[Graph] + make_fs_access=None, # type: Type[StdFsAccess] + fs_access=None, # type: StdFsAccess + job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any] + timeout=runtimeContext.eval_timeout, # type: float + debug=runtimeContext.debug, # type: bool + js_console=runtimeContext.js_console, # type: bool + force_docker_pull=runtimeContext.force_docker_pull, # type: bool + loadListing="", # type: Text + outdir="", # type: Text + tmpdir="", # type: Text + stagedir="", # type: Text + ) + class ArvadosCommandTool(CommandLineTool): """Wrap cwltool CommandLineTool to override selected methods.""" @@ -62,7 +88,7 @@ class ArvadosCommandTool(CommandLineTool): "$(task.keep)/%s/%s") def job(self, joborder, output_callback, runtimeContext): - builder = self._init_job(joborder, runtimeContext) + builder = make_builder(joborder, self.hints, self.requirements, runtimeContext) runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext) if runtimeContext.work_api == "containers": diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 4cc01a91ce..f514476b90 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -14,7 +14,6 @@ from cwltool.load_tool import fetch_document from cwltool.process import shortname from cwltool.workflow import Workflow, WorkflowException, WorkflowStep from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class -from cwltool.builder import Builder from cwltool.context import LoadingContext import ruamel.yaml as yaml @@ -22,7 +21,7 @@ import ruamel.yaml as yaml from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields, discover_secondary_files) from .pathmapper import ArvPathMapper, trim_listing -from .arvtool import ArvadosCommandTool, set_cluster_target +from .arvtool import ArvadosCommandTool, set_cluster_target, make_builder from .perf import Perf logger = logging.getLogger('arvados.cwl-runner') @@ -136,7 +135,7 @@ class ArvadosWorkflowStep(WorkflowStep): runtimeContext = runtimeContext.copy() runtimeContext.toplevel = True # Preserve behavior for #13365 - builder = self._init_job({shortname(k): v for k,v in joborder.items()}, runtimeContext) + builder = make_builder({shortname(k): v for k,v in joborder.items()}, self.hints, self.requirements, runtimeContext) runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext) return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext) -- 2.30.2