9442: Override kwargs["outdir"] so that $(runtime.outdir) is set correctly.

[arvados.git] / sdk / cwl / arvados_cwl / arvcontainer.py

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index be1140469ade22369f5ae51fac6da24a4826ddcd..94a7579f202a62d88f7a69481e4f2fe65dfa0f7b 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -14,11 +14,12 @@ from .runner import Runner
 logger = logging.getLogger('arvados.cwl-runner')
 
 class ArvadosContainer(object):
-    """Submit and manage a Crunch job for executing a CWL CommandLineTool."""
+    """Submit and manage a Crunch container request for executing a CWL CommandLineTool."""
 
     def __init__(self, runner):
         self.arvrunner = runner
         self.running = False
+        self.uuid = None
 
     def update_pipeline_component(self, r):
         pass
@@ -50,20 +51,6 @@ class ArvadosContainer(object):
         if self.generatefiles:
             raise UnsupportedRequirement("Generate files not supported")
 
-            vwd = arvados.collection.Collection(api_client=self.arvrunner.api_client)
-            container_request["task.vwd"] = {}
-            for t in self.generatefiles:
-                if isinstance(self.generatefiles[t], dict):
-                    src, rest = self.arvrunner.fs_access.get_collection(self.generatefiles[t]["path"].replace("$(task.keep)/", "keep:"))
-                    vwd.copy(rest, t, source_collection=src)
-                else:
-                    with vwd.open(t, "w") as f:
-                        f.write(self.generatefiles[t])
-            vwd.save_new()
-            # TODO
-            # for t in self.generatefiles:
-            #     container_request["task.vwd"][t] = "$(task.keep)/%s/%s" % (vwd.portable_data_hash(), t)
-
         container_request["environment"] = {"TMPDIR": "/tmp"}
         if self.environment:
             container_request["environment"].update(self.environment)
@@ -88,7 +75,6 @@ class ArvadosContainer(object):
         if resources is not None:
             runtime_constraints["vcpus"] = resources.get("cores", 1)
             runtime_constraints["ram"] = resources.get("ram") * 2**20
-            #runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
 
         container_request["mounts"] = mounts
         container_request["runtime_constraints"] = runtime_constraints
@@ -98,7 +84,7 @@ class ArvadosContainer(object):
                 body=container_request
             ).execute(num_retries=self.arvrunner.num_retries)
 
-            self.arvrunner.jobs[response["container_uuid"]] = self
+            self.arvrunner.processes[response["container_uuid"]] = self
 
             logger.info("Container %s (%s) request state is %s", self.name, response["container_uuid"], response["state"])
 
@@ -111,7 +97,17 @@ class ArvadosContainer(object):
     def done(self, record):
         try:
             if record["state"] == "Complete":
-                processStatus = "success"
+                rcode = record["exit_code"]
+                if self.successCodes and rcode in self.successCodes:
+                    processStatus = "success"
+                elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
+                    processStatus = "temporaryFail"
+                elif self.permanentFailCodes and rcode in self.permanentFailCodes:
+                    processStatus = "permanentFail"
+                elif rcode == 0:
+                    processStatus = "success"
+                else:
+                    processStatus = "permanentFail"
             else:
                 processStatus = "permanentFail"
 
@@ -128,18 +124,17 @@ class ArvadosContainer(object):
 
             self.output_callback(outputs, processStatus)
         finally:
-            del self.arvrunner.jobs[record["uuid"]]
+            del self.arvrunner.processes[record["uuid"]]
 
 
 class RunnerContainer(Runner):
     """Submit and manage a container that runs arvados-cwl-runner."""
 
     def arvados_job_spec(self, dry_run=False, pull_image=True, **kwargs):
-        """Create an Arvados job specification for this workflow.
+        """Create an Arvados container request for this workflow.
 
-        The returned dict can be used to create a job (i.e., passed as
-        the +body+ argument to jobs().create()), or as a component in
-        a pipeline template or pipeline instance.
+        The returned dict can be used to create a container passed as
+        the +body+ argument to container_requests().create().
         """
 
         workflowmapper = super(RunnerContainer, self).arvados_job_spec(dry_run=dry_run, pull_image=pull_image, **kwargs)
@@ -161,7 +156,7 @@ class RunnerContainer(Runner):
                                                self.arvrunner.project_uuid)
 
         return {
-            "command": ["arvados-cwl-runner", "--local", "--crunch2", workflowpath, jobpath],
+            "command": ["arvados-cwl-runner", "--local", "--api=containers", workflowpath, jobpath],
             "owner_uuid": self.arvrunner.project_uuid,
             "name": self.name,
             "output_path": "/var/spool/cwl",
@@ -170,7 +165,7 @@ class RunnerContainer(Runner):
             "state": "Committed",
             "container_image": container_image,
             "mounts": {
-                workflowpath: {
+                "/var/lib/cwl/workflow": {
                     "kind": "collection",
                     "portable_data_hash": "%s" % workflowcollection
                 },
@@ -181,15 +176,21 @@ class RunnerContainer(Runner):
                 "stdout": {
                     "kind": "file",
                     "path": "/var/spool/cwl/cwl.output.json"
+                },
+                "/var/spool/cwl": {
+                    "kind": "collection",
+                    "writable": True
                 }
             },
             "runtime_constraints": {
                 "vcpus": 1,
-                "ram": 1024*1024*256
+                "ram": 1024*1024*256,
+                "API": True
             }
         }
 
     def run(self, *args, **kwargs):
+        kwargs["keepprefix"] = "keep:"
         job_spec = self.arvados_job_spec(*args, **kwargs)
         job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
 
@@ -198,7 +199,7 @@ class RunnerContainer(Runner):
         ).execute(num_retries=self.arvrunner.num_retries)
 
         self.uuid = response["uuid"]
-        self.arvrunner.jobs[response["container_uuid"]] = self
+        self.arvrunner.processes[response["container_uuid"]] = self
 
         logger.info("Submitted container %s", response["uuid"])