import arvados.collection
from .arvdocker import arv_docker_get_image
-from .runner import Runner
+from .runner import Runner, arvados_jobs_image
from .pathmapper import InitialWorkDirPathMapper
from .perf import Perf
from . import done
+from ._version import __version__
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
-tmpdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.tmpdir\)=(.*)")
-outdirre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.outdir\)=(.*)")
-keepre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.keep\)=(.*)")
+crunchrunner_re = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.(tmpdir|outdir|keep)\)=(.*)")
class ArvadosJob(object):
"""Submit and manage a Crunch job for executing a CWL CommandLineTool."""
with Perf(metrics, "arv_docker_get_image %s" % self.name):
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if docker_req and kwargs.get("use_container") is not False:
+ if docker_req.get("dockerOutputDirectory"):
+ raise UnsupportedRequirement("Option 'dockerOutputDirectory' of DockerRequirement not supported.")
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
else:
- runtime_constraints["docker_image"] = "arvados/jobs"
+ runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
resources = self.builder.resources
if resources is not None:
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
log = logc.open(logc.keys()[0])
+ dirs = {}
tmpdir = None
outdir = None
keepdir = None
# the job restarts on a different node these values
# will different runs, and we need to know about the
# final run that actually produced output.
-
- g = tmpdirre.match(l)
- if g:
- tmpdir = g.group(1)
- g = outdirre.match(l)
- if g:
- outdir = g.group(1)
- g = keepre.match(l)
+ g = crunchrunner_re.match(l)
if g:
- keepdir = g.group(1)
+ dirs[g.group(1)] = g.group(2)
with Perf(metrics, "output collection %s" % self.name):
- outputs = done.done(self, record, tmpdir, outdir, keepdir)
+ outputs = done.done(self, record, dirs["tmpdir"],
+ dirs["outdir"], dirs["keep"])
except WorkflowException as e:
logger.error("Error while collecting job outputs:\n%s", e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
del self.job_order["job_order"]
self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"]).target[5:]
+
if self.output_name:
self.job_order["arv:output_name"] = self.output_name
+
+ self.job_order["arv:enable_reuse"] = self.enable_reuse
+
return {
"script": "cwl-runner",
- "script_version": "master",
+ "script_version": __version__,
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
- "docker_image": "arvados/jobs"
+ "docker_image": arvados_jobs_image(self.arvrunner)
}
}
def run(self, *args, **kwargs):
job_spec = self.arvados_job_spec(*args, **kwargs)
+ job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
+
+ job = self.arvrunner.api.jobs().create(
+ body=job_spec,
+ find_or_create=self.enable_reuse
+ ).execute(num_retries=self.arvrunner.num_retries)
+
for k,v in job_spec["script_parameters"].items():
- if isinstance(v, dict):
+ if v is False or v is None or isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
+ del job_spec["owner_uuid"]
+ job_spec["job"] = job
self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
body={
"owner_uuid": self.arvrunner.project_uuid,
self.uuid = self.arvrunner.pipeline["uuid"]
return
- job = None
- while not job:
- time.sleep(2)
- self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().get(
- uuid=self.arvrunner.pipeline["uuid"]).execute(
- num_retries=self.arvrunner.num_retries)
- job = self.arvrunner.pipeline["components"]["cwl-runner"].get("job")
- if not job and self.arvrunner.pipeline["state"] != "RunningOnServer":
- raise WorkflowException("Submitted pipeline is %s" % (self.arvrunner.pipeline["state"]))
-
self.uuid = job["uuid"]
self.arvrunner.processes[self.uuid] = self
Specifically, translate CWL input specs to Arvados pipeline
format, like {"dataclass":"File","value":"xyz"}.
"""
+
spec = self.job.arvados_job_spec()
# Most of the component spec is exactly the same as the job