X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/09a2e88c51e5432e607f2a38466e55b4ba15e887..eb58fd945645f5a670c761f7046b10885941167e:/sdk/cwl/arvados_cwl/arvjob.py diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py index 1afb9afc0e..9a03372d32 100644 --- a/sdk/cwl/arvados_cwl/arvjob.py +++ b/sdk/cwl/arvados_cwl/arvjob.py @@ -1,38 +1,61 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + import logging import re import copy import json import time -from cwltool.process import get_feature, shortname +from cwltool.process import shortname, UnsupportedRequirement from cwltool.errors import WorkflowException -from cwltool.draft2tool import revmap_file, CommandLineTool +from cwltool.command_line_tool import revmap_file, CommandLineTool from cwltool.load_tool import fetch_document from cwltool.builder import Builder +from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class +from cwltool.job import JobBase + +from schema_salad.sourceline import SourceLine + +import arvados_cwl.util +import ruamel.yaml as yaml import arvados.collection +from arvados.errors import ApiError from .arvdocker import arv_docker_get_image -from .runner import Runner, arvados_jobs_image -from .pathmapper import InitialWorkDirPathMapper +from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields +from .pathmapper import VwdPathMapper, trim_listing from .perf import Perf from . import done from ._version import __version__ +from .util import get_intermediate_collection_info logger = logging.getLogger('arvados.cwl-runner') metrics = logging.getLogger('arvados.cwl-runner.metrics') -crunchrunner_re = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.(tmpdir|outdir|keep)\)=(.*)") +crunchrunner_re = re.compile(r"^.*crunchrunner: \$\(task\.(tmpdir|outdir|keep)\)=(.*)$") -class ArvadosJob(object): +crunchrunner_git_commit = 'a3f2cb186e437bfce0031b024b2157b73ed2717d' + +class ArvadosJob(JobBase): """Submit and manage a Crunch job for executing a CWL CommandLineTool.""" - def __init__(self, runner): + def __init__(self, runner, + builder, # type: Builder + joborder, # type: Dict[Text, Union[Dict[Text, Any], List, Text]] + make_path_mapper, # type: Callable[..., PathMapper] + requirements, # type: List[Dict[Text, Text]] + hints, # type: List[Dict[Text, Text]] + name # type: Text + ): + super(ArvadosJob, self).__init__(builder, joborder, make_path_mapper, requirements, hints, name) self.arvrunner = runner self.running = False self.uuid = None - def run(self, dry_run=False, pull_image=True, **kwargs): + def run(self, runtimeContext): script_parameters = { "command": self.command_line } @@ -44,8 +67,8 @@ class ArvadosJob(object): keep_client=self.arvrunner.keep_client, num_retries=self.arvrunner.num_retries) script_parameters["task.vwd"] = {} - generatemapper = InitialWorkDirPathMapper([self.generatefiles], "", "", - separateDirs=False) + generatemapper = VwdPathMapper([self.generatefiles], "", "", + separateDirs=False) with Perf(metrics, "createfiles %s" % self.name): for f, p in generatemapper.items(): @@ -53,8 +76,14 @@ class ArvadosJob(object): with vwd.open(p.target, "w") as n: n.write(p.resolved.encode("utf-8")) - with Perf(metrics, "generatefiles.save_new %s" % self.name): - vwd.save_new() + if vwd: + with Perf(metrics, "generatefiles.save_new %s" % self.name): + info = get_intermediate_collection_info(self.name, None, runtimeContext.intermediate_output_ttl) + vwd.save_new(name=info["name"], + owner_uuid=self.arvrunner.project_uuid, + ensure_unique_name=True, + trash_at=info["trash_at"], + properties=info["properties"]) for f, p in generatemapper.items(): if p.type == "File": @@ -83,13 +112,17 @@ class ArvadosJob(object): script_parameters["task.permanentFailCodes"] = self.permanentFailCodes with Perf(metrics, "arv_docker_get_image %s" % self.name): - (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") - if docker_req and kwargs.get("use_container") is not False: + (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") + if docker_req and runtimeContext.use_container is not False: if docker_req.get("dockerOutputDirectory"): - raise UnsupportedRequirement("Option 'dockerOutputDirectory' of DockerRequirement not supported.") - runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid) + raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError( + "Option 'dockerOutputDirectory' of DockerRequirement not supported.") + runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, + docker_req, + runtimeContext.pull_image, + self.arvrunner.project_uuid) else: - runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner) + runtime_constraints["docker_image"] = "arvados/jobs" resources = self.builder.resources if resources is not None: @@ -97,10 +130,11 @@ class ArvadosJob(object): runtime_constraints["min_ram_mb_per_node"] = resources.get("ram") runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0) - runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints") + runtime_req, _ = self.get_requirement("http://arvados.org/cwl#RuntimeConstraints") if runtime_req: if "keep_cache" in runtime_req: runtime_constraints["keep_cache_mb_per_task"] = runtime_req["keep_cache"] + runtime_constraints["min_ram_mb_per_node"] += runtime_req["keep_cache"] if "outputDirType" in runtime_req: if runtime_req["outputDirType"] == "local_output_dir": script_parameters["task.keepTmpOutput"] = False @@ -109,10 +143,18 @@ class ArvadosJob(object): filters = [["repository", "=", "arvados"], ["script", "=", "crunchrunner"], - ["script_version", "in git", "9e5b98e8f5f4727856b53447191f9c06e3da2ba6"]] + ["script_version", "in git", crunchrunner_git_commit]] if not self.arvrunner.ignore_docker_for_reuse: filters.append(["docker_image_locator", "in docker", runtime_constraints["docker_image"]]) + enable_reuse = runtimeContext.enable_reuse + if enable_reuse: + reuse_req, _ = self.get_requirement("http://arvados.org/cwl#ReuseRequirement") + if reuse_req: + enable_reuse = reuse_req["enableReuse"] + + self.output_callback = self.arvrunner.get_wrapped_callback(self.output_callback) + try: with Perf(metrics, "create %s" % self.name): response = self.arvrunner.api.jobs().create( @@ -121,47 +163,65 @@ class ArvadosJob(object): "script": "crunchrunner", "repository": "arvados", "script_version": "master", - "minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6", + "minimum_script_version": crunchrunner_git_commit, "script_parameters": {"tasks": [script_parameters]}, "runtime_constraints": runtime_constraints }, filters=filters, - find_or_create=kwargs.get("enable_reuse", True) + find_or_create=enable_reuse ).execute(num_retries=self.arvrunner.num_retries) - self.arvrunner.processes[response["uuid"]] = self + self.uuid = response["uuid"] + self.arvrunner.process_submitted(self) self.update_pipeline_component(response) - logger.info("Job %s (%s) is %s", self.name, response["uuid"], response["state"]) - - if response["state"] in ("Complete", "Failed", "Cancelled"): - with Perf(metrics, "done %s" % self.name): - self.done(response) + if response["state"] == "Complete": + logger.info("%s reused job %s", self.arvrunner.label(self), response["uuid"]) + # Give read permission to the desired project on reused jobs + if response["owner_uuid"] != self.arvrunner.project_uuid: + try: + self.arvrunner.api.links().create(body={ + 'link_class': 'permission', + 'name': 'can_read', + 'tail_uuid': self.arvrunner.project_uuid, + 'head_uuid': response["uuid"], + }).execute(num_retries=self.arvrunner.num_retries) + except ApiError as e: + # The user might not have "manage" access on the job: log + # a message and continue. + logger.info("Creating read permission on job %s: %s", + response["uuid"], + e) + else: + logger.info("%s %s is %s", self.arvrunner.label(self), response["uuid"], response["state"]) except Exception as e: - logger.exception("Job %s error" % (self.name)) + logger.exception("%s error" % (self.arvrunner.label(self))) self.output_callback({}, "permanentFail") def update_pipeline_component(self, record): - if self.arvrunner.pipeline: - self.arvrunner.pipeline["components"][self.name] = {"job": record} - with Perf(metrics, "update_pipeline_component %s" % self.name): - self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().update(uuid=self.arvrunner.pipeline["uuid"], - body={ - "components": self.arvrunner.pipeline["components"] - }).execute(num_retries=self.arvrunner.num_retries) - if self.arvrunner.uuid: - try: - job = self.arvrunner.api.jobs().get(uuid=self.arvrunner.uuid).execute() - if job: - components = job["components"] - components[self.name] = record["uuid"] - self.arvrunner.api.jobs().update(uuid=self.arvrunner.uuid, + with self.arvrunner.workflow_eval_lock: + if self.arvrunner.pipeline: + self.arvrunner.pipeline["components"][self.name] = {"job": record} + with Perf(metrics, "update_pipeline_component %s" % self.name): + self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().update( + uuid=self.arvrunner.pipeline["uuid"], body={ - "components": components + "components": self.arvrunner.pipeline["components"] }).execute(num_retries=self.arvrunner.num_retries) - except Exception as e: - logger.info("Error adding to components: %s", e) + if self.arvrunner.uuid: + try: + job = self.arvrunner.api.jobs().get(uuid=self.arvrunner.uuid).execute() + if job: + components = job["components"] + components[self.name] = record["uuid"] + self.arvrunner.api.jobs().update( + uuid=self.arvrunner.uuid, + body={ + "components": components + }).execute(num_retries=self.arvrunner.num_retries) + except Exception as e: + logger.info("Error adding to components: %s", e) def done(self, record): try: @@ -184,12 +244,13 @@ class ArvadosJob(object): keep_client=self.arvrunner.keep_client, num_retries=self.arvrunner.num_retries) log = logc.open(logc.keys()[0]) - dirs = {} - tmpdir = None - outdir = None - keepdir = None + dirs = { + "tmpdir": "/tmpdir", + "outdir": "/outdir", + "keep": "/keep" + } for l in log: - # Determine the tmpdir, outdir and keepdir paths from + # Determine the tmpdir, outdir and keep paths from # the job run. Unfortunately, we can't take the first # values we find (which are expected to be near the # top) and stop scanning because if the node fails and @@ -200,11 +261,15 @@ class ArvadosJob(object): if g: dirs[g.group(1)] = g.group(2) + if processStatus == "permanentFail": + done.logtail(logc, logger.error, "%s (%s) error log:" % (self.arvrunner.label(self), record["uuid"]), maxlen=40) + with Perf(metrics, "output collection %s" % self.name): outputs = done.done(self, record, dirs["tmpdir"], dirs["outdir"], dirs["keep"]) except WorkflowException as e: - logger.error("Error while collecting output for job %s:\n%s", self.name, e, exc_info=(e if self.arvrunner.debug else False)) + logger.error("%s unable to collect output from %s:\n%s", + self.arvrunner.label(self), record["output"], e, exc_info=(e if self.arvrunner.debug else False)) processStatus = "permanentFail" except Exception as e: logger.exception("Got unknown exception while collecting output for job %s:", self.name) @@ -216,16 +281,14 @@ class ArvadosJob(object): logger.error("Unexpected output type %s '%s'", type(outputs), outputs) outputs = {} processStatus = "permanentFail" - - self.output_callback(outputs, processStatus) finally: - del self.arvrunner.processes[record["uuid"]] + self.output_callback(outputs, processStatus) class RunnerJob(Runner): """Submit and manage a Crunch job that runs crunch_scripts/cwl-runner.""" - def arvados_job_spec(self, dry_run=False, pull_image=True, **kwargs): + def arvados_job_spec(self, debug=False): """Create an Arvados job specification for this workflow. The returned dict can be used to create a job (i.e., passed as @@ -233,15 +296,16 @@ class RunnerJob(Runner): a pipeline template or pipeline instance. """ - workflowmapper = super(RunnerJob, self).arvados_job_spec(dry_run=dry_run, pull_image=pull_image, **kwargs) - - # Need to filter this out, gets added by cwltool when providing - # parameters on the command line, and arv-run-pipeline-instance doesn't - # like it. - if "job_order" in self.job_order: - del self.job_order["job_order"] + if self.tool.tool["id"].startswith("keep:"): + self.job_order["cwl:tool"] = self.tool.tool["id"][5:] + else: + packed = packed_workflow(self.arvrunner, self.tool, self.merged_map) + wf_pdh = upload_workflow_collection(self.arvrunner, self.name, packed) + self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh - self.job_order["cwl:tool"] = workflowmapper.mapper(self.tool.tool["id"]).target[5:] + adjustDirObjs(self.job_order, trim_listing) + visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location) + visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields) if self.output_name: self.job_order["arv:output_name"] = self.output_name @@ -251,18 +315,26 @@ class RunnerJob(Runner): self.job_order["arv:enable_reuse"] = self.enable_reuse + if self.on_error: + self.job_order["arv:on_error"] = self.on_error + + if debug: + self.job_order["arv:debug"] = True + return { "script": "cwl-runner", - "script_version": __version__, + "script_version": "master", + "minimum_script_version": "570509ab4d2ef93d870fd2b1f2eab178afb1bad9", "repository": "arvados", "script_parameters": self.job_order, "runtime_constraints": { - "docker_image": arvados_jobs_image(self.arvrunner) + "docker_image": arvados_jobs_image(self.arvrunner, self.jobs_image), + "min_ram_mb_per_node": self.submit_runner_ram } } - def run(self, *args, **kwargs): - job_spec = self.arvados_job_spec(*args, **kwargs) + def run(self, runtimeContext): + job_spec = self.arvados_job_spec(runtimeContext.debug) job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid) @@ -277,23 +349,28 @@ class RunnerJob(Runner): del job_spec["owner_uuid"] job_spec["job"] = job + + instance_spec = { + "owner_uuid": self.arvrunner.project_uuid, + "name": self.name, + "components": { + "cwl-runner": job_spec, + }, + "state": "RunningOnServer", + } + if not self.enable_reuse: + instance_spec["properties"] = {"run_options": {"enable_job_reuse": False}} + self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create( - body={ - "owner_uuid": self.arvrunner.project_uuid, - "name": shortname(self.tool.tool["id"]), - "components": {"cwl-runner": job_spec }, - "state": "RunningOnServer"}).execute(num_retries=self.arvrunner.num_retries) + body=instance_spec).execute(num_retries=self.arvrunner.num_retries) logger.info("Created pipeline %s", self.arvrunner.pipeline["uuid"]) - if kwargs.get("wait") is False: + if runtimeContext.wait is False: self.uuid = self.arvrunner.pipeline["uuid"] return self.uuid = job["uuid"] - self.arvrunner.processes[self.uuid] = self - - if job["state"] in ("Complete", "Failed", "Cancelled"): - self.done(job) + self.arvrunner.process_submitted(self) class RunnerTemplate(object): @@ -308,7 +385,8 @@ class RunnerTemplate(object): 'string': 'text', } - def __init__(self, runner, tool, job_order, enable_reuse, uuid): + def __init__(self, runner, tool, job_order, enable_reuse, uuid, + submit_runner_ram=0, name=None, merged_map=None): self.runner = runner self.tool = tool self.job = RunnerJob( @@ -317,7 +395,10 @@ class RunnerTemplate(object): job_order=job_order, enable_reuse=enable_reuse, output_name=None, - output_tags=None) + output_tags=None, + submit_runner_ram=submit_runner_ram, + name=name, + merged_map=merged_map) self.uuid = uuid def pipeline_component_spec(self): @@ -347,10 +428,12 @@ class RunnerTemplate(object): if not isinstance(types, list): types = [types] param['required'] = 'null' not in types - non_null_types = set(types) - set(['null']) + non_null_types = [t for t in types if t != "null"] if len(non_null_types) == 1: the_type = [c for c in non_null_types][0] - dataclass = self.type_to_dataclass.get(the_type) + dataclass = None + if isinstance(the_type, basestring): + dataclass = self.type_to_dataclass.get(the_type) if dataclass: param['dataclass'] = dataclass # Note: If we didn't figure out a single appropriate