Merge branch '10346-rearrange-api-docs' closes #10346

[arvados.git] / sdk / cwl / arvados_cwl / arvjob.py

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index a97453151b0568ab57f79ca5e30e932fa704d1fa..8a62204f8fb9ec22298abec15529411ace70ed9e 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -13,17 +13,16 @@ from cwltool.builder import Builder
 import arvados.collection
 
 from .arvdocker import arv_docker_get_image
-from .runner import Runner
+from .runner import Runner, arvados_jobs_image
 from .pathmapper import InitialWorkDirPathMapper
 from .perf import Perf
 from . import done
+from ._version import __version__
 
 logger = logging.getLogger('arvados.cwl-runner')
 metrics = logging.getLogger('arvados.cwl-runner.metrics')
 
-tmpdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.tmpdir\)=(.*)")
-outdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.outdir\)=(.*)")
-keepre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.keep\)=(.*)")
+crunchrunner_re = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.(tmpdir|outdir|keep)\)=(.*)")
 
 class ArvadosJob(object):
     """Submit and manage a Crunch job for executing a CWL CommandLineTool."""
@@ -86,9 +85,11 @@ class ArvadosJob(object):
         with Perf(metrics, "arv_docker_get_image %s" % self.name):
             (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
             if docker_req and kwargs.get("use_container") is not False:
+                if docker_req.get("dockerOutputDirectory"):
+                    raise UnsupportedRequirement("Option 'dockerOutputDirectory' of DockerRequirement not supported.")
                 runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
             else:
-                runtime_constraints["docker_image"] = "arvados/jobs"
+                runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
 
         resources = self.builder.resources
         if resources is not None:
@@ -183,6 +184,7 @@ class ArvadosJob(object):
                                                                    keep_client=self.arvrunner.keep_client,
                                                                    num_retries=self.arvrunner.num_retries)
                         log = logc.open(logc.keys()[0])
+                        dirs = {}
                         tmpdir = None
                         outdir = None
                         keepdir = None
@@ -194,19 +196,13 @@ class ArvadosJob(object):
                             # the job restarts on a different node these values
                             # will different runs, and we need to know about the
                             # final run that actually produced output.
-
-                            g = tmpdirre.match(l)
-                            if g:
-                                tmpdir = g.group(1)
-                            g = outdirre.match(l)
-                            if g:
-                                outdir = g.group(1)
-                            g = keepre.match(l)
+                            g = crunchrunner_re.match(l)
                             if g:
-                                keepdir = g.group(1)
+                                dirs[g.group(1)] = g.group(2)
 
                     with Perf(metrics, "output collection %s" % self.name):
-                        outputs = done.done(self, record, tmpdir, outdir, keepdir)
+                        outputs = done.done(self, record, dirs["tmpdir"],
+                                            dirs["outdir"], dirs["keep"])
             except WorkflowException as e:
                 logger.error("Error while collecting job outputs:\n%s", e, exc_info=(e if self.arvrunner.debug else False))
                 processStatus = "permanentFail"
@@ -241,25 +237,38 @@ class RunnerJob(Runner):
             del self.job_order["job_order"]
 
         self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"]).target[5:]
+
         if self.output_name:
             self.job_order["arv:output_name"] = self.output_name
+
+        self.job_order["arv:enable_reuse"] = self.enable_reuse
+
         return {
             "script": "cwl-runner",
-            "script_version": "master",
+            "script_version": __version__,
             "repository": "arvados",
             "script_parameters": self.job_order,
             "runtime_constraints": {
-                "docker_image": "arvados/jobs"
+                "docker_image": arvados_jobs_image(self.arvrunner)
             }
         }
 
     def run(self, *args, **kwargs):
         job_spec = self.arvados_job_spec(*args, **kwargs)
 
+        job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
+
+        job = self.arvrunner.api.jobs().create(
+            body=job_spec,
+            find_or_create=self.enable_reuse
+        ).execute(num_retries=self.arvrunner.num_retries)
+
         for k,v in job_spec["script_parameters"].items():
             if v is False or v is None or isinstance(v, dict):
                 job_spec["script_parameters"][k] = {"value": v}
 
+        del job_spec["owner_uuid"]
+        job_spec["job"] = job
         self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
             body={
                 "owner_uuid": self.arvrunner.project_uuid,
@@ -272,16 +281,6 @@ class RunnerJob(Runner):
             self.uuid = self.arvrunner.pipeline["uuid"]
             return
 
-        job = None
-        while not job:
-            time.sleep(2)
-            self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().get(
-                uuid=self.arvrunner.pipeline["uuid"]).execute(
-                    num_retries=self.arvrunner.num_retries)
-            job = self.arvrunner.pipeline["components"]["cwl-runner"].get("job")
-            if not job and self.arvrunner.pipeline["state"] != "RunningOnServer":
-                raise WorkflowException("Submitted pipeline is %s" % (self.arvrunner.pipeline["state"]))
-
         self.uuid = job["uuid"]
         self.arvrunner.processes[self.uuid] = self
 
@@ -317,6 +316,7 @@ class RunnerTemplate(object):
         Specifically, translate CWL input specs to Arvados pipeline
         format, like {"dataclass":"File","value":"xyz"}.
         """
+
         spec = self.job.arvados_job_spec()
 
         # Most of the component spec is exactly the same as the job