X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8d4ec10fc26d93d282845c789cd61da79e4b2836..80459d52161120ae8e33da140984d596271d5195:/sdk/cwl/arvados_cwl/arvjob.py diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py index d6055d3710..877f4238a3 100644 --- a/sdk/cwl/arvados_cwl/arvjob.py +++ b/sdk/cwl/arvados_cwl/arvjob.py @@ -4,17 +4,23 @@ import copy import json import time -from cwltool.process import get_feature, shortname +from cwltool.process import get_feature, shortname, UnsupportedRequirement from cwltool.errors import WorkflowException from cwltool.draft2tool import revmap_file, CommandLineTool from cwltool.load_tool import fetch_document from cwltool.builder import Builder +from cwltool.pathmapper import adjustFileObjs, adjustDirObjs + +from schema_salad.sourceline import SourceLine + +import ruamel.yaml as yaml import arvados.collection +from arvados.errors import ApiError from .arvdocker import arv_docker_get_image -from .runner import Runner, arvados_jobs_image -from .pathmapper import InitialWorkDirPathMapper +from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location +from .pathmapper import VwdPathMapper, trim_listing from .perf import Perf from . import done from ._version import __version__ @@ -46,8 +52,8 @@ class ArvadosJob(object): keep_client=self.arvrunner.keep_client, num_retries=self.arvrunner.num_retries) script_parameters["task.vwd"] = {} - generatemapper = InitialWorkDirPathMapper([self.generatefiles], "", "", - separateDirs=False) + generatemapper = VwdPathMapper([self.generatefiles], "", "", + separateDirs=False) with Perf(metrics, "createfiles %s" % self.name): for f, p in generatemapper.items(): @@ -55,8 +61,9 @@ class ArvadosJob(object): with vwd.open(p.target, "w") as n: n.write(p.resolved.encode("utf-8")) - with Perf(metrics, "generatefiles.save_new %s" % self.name): - vwd.save_new() + if vwd: + with Perf(metrics, "generatefiles.save_new %s" % self.name): + vwd.save_new() for f, p in generatemapper.items(): if p.type == "File": @@ -92,7 +99,7 @@ class ArvadosJob(object): "Option 'dockerOutputDirectory' of DockerRequirement not supported.") runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid) else: - runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner) + runtime_constraints["docker_image"] = "arvados/jobs" resources = self.builder.resources if resources is not None: @@ -104,6 +111,7 @@ class ArvadosJob(object): if runtime_req: if "keep_cache" in runtime_req: runtime_constraints["keep_cache_mb_per_task"] = runtime_req["keep_cache"] + runtime_constraints["min_ram_mb_per_node"] += runtime_req["keep_cache"] if "outputDirType" in runtime_req: if runtime_req["outputDirType"] == "local_output_dir": script_parameters["task.keepTmpOutput"] = False @@ -116,6 +124,12 @@ class ArvadosJob(object): if not self.arvrunner.ignore_docker_for_reuse: filters.append(["docker_image_locator", "in docker", runtime_constraints["docker_image"]]) + enable_reuse = kwargs.get("enable_reuse", True) + if enable_reuse: + reuse_req, _ = get_feature(self, "http://arvados.org/cwl#ReuseRequirement") + if reuse_req: + enable_reuse = reuse_req["enableReuse"] + try: with Perf(metrics, "create %s" % self.name): response = self.arvrunner.api.jobs().create( @@ -129,20 +143,37 @@ class ArvadosJob(object): "runtime_constraints": runtime_constraints }, filters=filters, - find_or_create=kwargs.get("enable_reuse", True) + find_or_create=enable_reuse ).execute(num_retries=self.arvrunner.num_retries) self.arvrunner.processes[response["uuid"]] = self self.update_pipeline_component(response) - logger.info("Job %s (%s) is %s", self.name, response["uuid"], response["state"]) + if response["state"] == "Complete": + logger.info("%s reused job %s", self.arvrunner.label(self), response["uuid"]) + # Give read permission to the desired project on reused jobs + if response["owner_uuid"] != self.arvrunner.project_uuid: + try: + self.arvrunner.api.links().create(body={ + 'link_class': 'permission', + 'name': 'can_read', + 'tail_uuid': self.arvrunner.project_uuid, + 'head_uuid': response["uuid"], + }).execute(num_retries=self.arvrunner.num_retries) + except ApiError as e: + # The user might not have "manage" access on the job: log + # a message and continue. + logger.info("Creating read permission on job %s: %s", + response["uuid"], + e) - if response["state"] in ("Complete", "Failed", "Cancelled"): with Perf(metrics, "done %s" % self.name): self.done(response) + else: + logger.info("%s %s is %s", self.arvrunner.label(self), response["uuid"], response["state"]) except Exception as e: - logger.exception("Job %s error" % (self.name)) + logger.exception("%s error" % (self.arvrunner.label(self))) self.output_callback({}, "permanentFail") def update_pipeline_component(self, record): @@ -203,11 +234,15 @@ class ArvadosJob(object): if g: dirs[g.group(1)] = g.group(2) + if processStatus == "permanentFail": + done.logtail(logc, logger, "%s error log:" % self.arvrunner.label(self)) + with Perf(metrics, "output collection %s" % self.name): outputs = done.done(self, record, dirs["tmpdir"], dirs["outdir"], dirs["keep"]) except WorkflowException as e: - logger.error("Error while collecting output for job %s:\n%s", self.name, e, exc_info=(e if self.arvrunner.debug else False)) + logger.error("%s unable to collect output from %s:\n%s", + self.arvrunner.label(self), record["output"], e, exc_info=(e if self.arvrunner.debug else False)) processStatus = "permanentFail" except Exception as e: logger.exception("Got unknown exception while collecting output for job %s:", self.name) @@ -219,11 +254,10 @@ class ArvadosJob(object): logger.error("Unexpected output type %s '%s'", type(outputs), outputs) outputs = {} processStatus = "permanentFail" - - self.output_callback(outputs, processStatus) finally: - del self.arvrunner.processes[record["uuid"]] - + self.output_callback(outputs, processStatus) + if record["uuid"] in self.arvrunner.processes: + del self.arvrunner.processes[record["uuid"]] class RunnerJob(Runner): """Submit and manage a Crunch job that runs crunch_scripts/cwl-runner.""" @@ -236,9 +270,16 @@ class RunnerJob(Runner): a pipeline template or pipeline instance. """ - workflowmapper = super(RunnerJob, self).arvados_job_spec(dry_run=dry_run, pull_image=pull_image, **kwargs) + if self.tool.tool["id"].startswith("keep:"): + self.job_order["cwl:tool"] = self.tool.tool["id"][5:] + else: + packed = packed_workflow(self.arvrunner, self.tool) + wf_pdh = upload_workflow_collection(self.arvrunner, self.name, packed) + self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh - self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"]).target[5:] + adjustDirObjs(self.job_order, trim_listing) + adjustFileObjs(self.job_order, trim_anonymous_location) + adjustDirObjs(self.job_order, trim_anonymous_location) if self.output_name: self.job_order["arv:output_name"] = self.output_name @@ -248,6 +289,9 @@ class RunnerJob(Runner): self.job_order["arv:enable_reuse"] = self.enable_reuse + if self.on_error: + self.job_order["arv:on_error"] = self.on_error + return { "script": "cwl-runner", "script_version": "master", @@ -255,7 +299,7 @@ class RunnerJob(Runner): "repository": "arvados", "script_parameters": self.job_order, "runtime_constraints": { - "docker_image": arvados_jobs_image(self.arvrunner), + "docker_image": arvados_jobs_image(self.arvrunner, self.jobs_image), "min_ram_mb_per_node": self.submit_runner_ram } } @@ -349,10 +393,12 @@ class RunnerTemplate(object): if not isinstance(types, list): types = [types] param['required'] = 'null' not in types - non_null_types = set(types) - set(['null']) + non_null_types = [t for t in types if t != "null"] if len(non_null_types) == 1: the_type = [c for c in non_null_types][0] - dataclass = self.type_to_dataclass.get(the_type) + dataclass = None + if isinstance(the_type, basestring): + dataclass = self.type_to_dataclass.get(the_type) if dataclass: param['dataclass'] = dataclass # Note: If we didn't figure out a single appropriate