17778: Merge branch 'master' into 17778-doc-update

[arvados.git] / sdk / cwl / arvados_cwl / arvtool.py

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 31e6be12b533cb9d96c65f74189935fad8c3fcbe..89176923519ee4c737d9376032463dee08a80102 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -3,10 +3,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from cwltool.command_line_tool import CommandLineTool, ExpressionTool
-from cwltool.builder import Builder
-from .arvjob import ArvadosJob
 from .arvcontainer import ArvadosContainer
 from .pathmapper import ArvPathMapper
+from .runner import make_builder
 from functools import partial
 from schema_salad.sourceline import SourceLine
 from cwltool.errors import WorkflowException
@@ -16,7 +15,16 @@ def validate_cluster_target(arvrunner, runtimeContext):
         runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and
         runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]):
         raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster,
-                                                                                                  ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
+                                                                                                     ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
+    if runtimeContext.project_uuid:
+        cluster_target = runtimeContext.submit_runner_cluster or arvrunner.api._rootDesc["uuidPrefix"]
+        if not runtimeContext.project_uuid.startswith(cluster_target):
+            raise WorkflowException("Project uuid '%s' must be for target cluster '%s'" % (runtimeContext.project_uuid, cluster_target))
+        try:
+            arvrunner.api.groups().get(uuid=runtimeContext.project_uuid).execute()
+        except Exception as e:
+            raise WorkflowException("Invalid project uuid '%s': %s" % (runtimeContext.project_uuid, e))
+
 def set_cluster_target(tool, arvrunner, builder, runtimeContext):
     cluster_target_req = None
     for field in ("hints", "requirements"):
@@ -37,30 +45,6 @@ def set_cluster_target(tool, arvrunner, builder, runtimeContext):
 
     return runtimeContext
 
-def make_builder(joborder, hints, requirements, runtimeContext):
-    return Builder(
-                 job=joborder,
-                 files=[],               # type: List[Dict[Text, Text]]
-                 bindings=[],            # type: List[Dict[Text, Any]]
-                 schemaDefs={},          # type: Dict[Text, Dict[Text, Any]]
-                 names=None,               # type: Names
-                 requirements=requirements,        # type: List[Dict[Text, Any]]
-                 hints=hints,               # type: List[Dict[Text, Any]]
-                 resources={},           # type: Dict[str, int]
-                 mutation_manager=None,    # type: Optional[MutationManager]
-                 formatgraph=None,         # type: Optional[Graph]
-                 make_fs_access=None,      # type: Type[StdFsAccess]
-                 fs_access=None,           # type: StdFsAccess
-                 job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any]
-                 timeout=runtimeContext.eval_timeout,             # type: float
-                 debug=runtimeContext.debug,               # type: bool
-                 js_console=runtimeContext.js_console,          # type: bool
-                 force_docker_pull=runtimeContext.force_docker_pull,   # type: bool
-                 loadListing="",         # type: Text
-                 outdir="",              # type: Text
-                 tmpdir="",              # type: Text
-                 stagedir="",            # type: Text
-                )
 
 class ArvadosCommandTool(CommandLineTool):
     """Wrap cwltool CommandLineTool to override selected methods."""
@@ -72,8 +56,6 @@ class ArvadosCommandTool(CommandLineTool):
     def make_job_runner(self, runtimeContext):
         if runtimeContext.work_api == "containers":
             return partial(ArvadosContainer, self.arvrunner, runtimeContext)
-        elif runtimeContext.work_api == "jobs":
-            return partial(ArvadosJob, self.arvrunner)
         else:
             raise Exception("Unsupported work_api %s", runtimeContext.work_api)
 
@@ -82,13 +64,9 @@ class ArvadosCommandTool(CommandLineTool):
             return ArvPathMapper(self.arvrunner, reffiles+runtimeContext.extra_reffiles, runtimeContext.basedir,
                                  "/keep/%s",
                                  "/keep/%s/%s")
-        elif runtimeContext.work_api == "jobs":
-            return ArvPathMapper(self.arvrunner, reffiles, runtimeContext.basedir,
-                                 "$(task.keep)/%s",
-                                 "$(task.keep)/%s/%s")
 
     def job(self, joborder, output_callback, runtimeContext):
-        builder = make_builder(joborder, self.hints, self.requirements, runtimeContext)
+        builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
         runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
 
         if runtimeContext.work_api == "containers":
@@ -99,11 +77,6 @@ class ArvadosCommandTool(CommandLineTool):
             else:
                 runtimeContext.outdir = "/var/spool/cwl"
                 runtimeContext.docker_outdir = "/var/spool/cwl"
-        elif runtimeContext.work_api == "jobs":
-            runtimeContext.outdir = "$(task.outdir)"
-            runtimeContext.docker_outdir = "$(task.outdir)"
-            runtimeContext.tmpdir = "$(task.tmpdir)"
-            runtimeContext.docker_tmpdir = "$(task.tmpdir)"
         return super(ArvadosCommandTool, self).job(joborder, output_callback, runtimeContext)
 
 class ArvadosExpressionTool(ExpressionTool):