X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/07d92519438a592d531f2c7558cd51788da262ca..332b0d1b4a9095f4e43893ec741f901b74b36ceb:/sdk/cwl/arvados_cwl/executor.py diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index eed2fe19df..680ca0b7b2 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -18,6 +18,7 @@ import json import re from functools import partial import time +import urllib from cwltool.errors import WorkflowException import cwltool.workflow @@ -37,12 +38,12 @@ from .arvworkflow import ArvadosWorkflow, upload_workflow from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size from .perf import Perf from .pathmapper import NoFollowPathMapper -from .task_queue import TaskQueue +from cwltool.task_queue import TaskQueue from .context import ArvLoadingContext, ArvRuntimeContext from ._version import __version__ from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class +from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class, aslist from cwltool.command_line_tool import compute_checksums from cwltool.load_tool import load_tool @@ -99,7 +100,8 @@ class ArvCwlExecutor(object): arvargs=None, keep_client=None, num_retries=4, - thread_count=4): + thread_count=4, + stdout=sys.stdout): if arvargs is None: arvargs = argparse.Namespace() @@ -132,6 +134,7 @@ class ArvCwlExecutor(object): self.should_estimate_cache_size = True self.fs_access = None self.secret_store = None + self.stdout = stdout if keep_client is not None: self.keep_client = keep_client @@ -250,50 +253,37 @@ The 'jobs' API is no longer supported. activity statuses, for example in the RuntimeStatusLoggingHandler. """ with self.workflow_eval_lock: - current = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) + current = None + try: + current = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) + except Exception as e: + logger.info("Couldn't get current container: %s", e) if current is None: return runtime_status = current.get('runtime_status', {}) - # In case of status being an error, only report the first one. - if kind == 'error': - if not runtime_status.get('error'): - runtime_status.update({ - 'error': message - }) - if detail is not None: - runtime_status.update({ - 'errorDetail': detail - }) - # Further errors are only mentioned as a count. - else: - # Get anything before an optional 'and N more' string. - try: - error_msg = re.match( - r'^(.*?)(?=\s*\(and \d+ more\)|$)', runtime_status.get('error')).groups()[0] - more_failures = re.match( - r'.*\(and (\d+) more\)', runtime_status.get('error')) - except TypeError: - # Ignore tests stubbing errors - return - if more_failures: - failure_qty = int(more_failures.groups()[0]) - runtime_status.update({ - 'error': "%s (and %d more)" % (error_msg, failure_qty+1) - }) - else: - runtime_status.update({ - 'error': "%s (and 1 more)" % error_msg - }) - elif kind in ['warning', 'activity']: - # Record the last warning/activity status without regard of - # previous occurences. + if kind in ('error', 'warning'): + updatemessage = runtime_status.get(kind, "") + if not updatemessage: + updatemessage = message + + # Subsequent messages tacked on in detail + updatedetail = runtime_status.get(kind+'Detail', "") + maxlines = 40 + if updatedetail.count("\n") < maxlines: + if updatedetail: + updatedetail += "\n" + updatedetail += message + "\n" + + if detail: + updatedetail += detail + "\n" + + if updatedetail.count("\n") >= maxlines: + updatedetail += "\nSome messages may have been omitted. Check the full log." + runtime_status.update({ - kind: message + kind: updatemessage, + kind+'Detail': updatedetail, }) - if detail is not None: - runtime_status.update({ - kind+"Detail": detail - }) else: # Ignore any other status kind return @@ -444,7 +434,7 @@ The 'jobs' API is no longer supported. srccollection = sp[0][5:] try: reader = self.collection_cache.get(srccollection) - srcpath = "/".join(sp[1:]) if len(sp) > 1 else "." + srcpath = urllib.parse.unquote("/".join(sp[1:]) if len(sp) > 1 else ".") final.copy(srcpath, v.target, source_collection=reader, overwrite=False) except arvados.errors.ArgumentError as e: logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e) @@ -503,7 +493,7 @@ The 'jobs' API is no longer supported. }).execute(num_retries=self.num_retries) except Exception: logger.exception("Setting container output") - return + raise def apply_reqs(self, job_order_object, tool): if "https://w3id.org/cwl/cwl#requirements" in job_order_object: @@ -517,10 +507,15 @@ The 'jobs' API is no longer supported. for req in job_reqs: tool.requirements.append(req) - def arv_executor(self, tool, job_order, runtimeContext, logger=None): + def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None): self.debug = runtimeContext.debug - tool.visit(self.check_features) + workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"] + workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"] + controller = self.api.config()["Services"]["Controller"]["ExternalURL"] + logger.info("Using cluster %s (%s)", self.api.config()["ClusterID"], workbench2 or workbench1 or controller) + + updated_tool.visit(self.check_features) self.project_uuid = runtimeContext.project_uuid self.pipeline = None @@ -540,28 +535,41 @@ The 'jobs' API is no longer supported. if runtimeContext.submit_request_uuid and self.work_api != "containers": raise Exception("--submit-request-uuid requires containers API, but using '{}' api".format(self.work_api)) + default_storage_classes = ",".join([k for k,v in self.api.config().get("StorageClasses", {"default": {"Default": True}}).items() if v.get("Default") is True]) + if runtimeContext.storage_classes == "default": + runtimeContext.storage_classes = default_storage_classes + if runtimeContext.intermediate_storage_classes == "default": + runtimeContext.intermediate_storage_classes = default_storage_classes + if not runtimeContext.name: - runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"]) + runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"]) # Upload local file references in the job order. job_order = upload_job_order(self, "%s input" % runtimeContext.name, - tool, job_order) + updated_tool, job_order) + + # the last clause means: if it is a command line tool, and we + # are going to wait for the result, and always_submit_runner + # is false, then we don't submit a runner process. submitting = (runtimeContext.update_workflow or runtimeContext.create_workflow or (runtimeContext.submit and not - (tool.tool["class"] == "CommandLineTool" and + (updated_tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and not runtimeContext.always_submit_runner))) loadingContext = self.loadingContext.copy() loadingContext.do_validate = False - loadingContext.do_update = False if submitting: + loadingContext.do_update = False # Document may have been auto-updated. Reload the original # document with updating disabled because we want to - # submit the original document, not the auto-updated one. - tool = load_tool(tool.tool["id"], loadingContext) + # submit the document with its original CWL version, not + # the auto-updated one. + tool = load_tool(updated_tool.tool["id"], loadingContext) + else: + tool = updated_tool # Upload direct dependencies of workflow steps, get back mapping of files to keep references. # Also uploads docker images. @@ -580,13 +588,15 @@ The 'jobs' API is no longer supported. if existing_uuid or runtimeContext.create_workflow: # Create a pipeline template or workflow record and exit. if self.work_api == "containers": - return (upload_workflow(self, tool, job_order, + uuid = upload_workflow(self, tool, job_order, self.project_uuid, uuid=existing_uuid, submit_runner_ram=runtimeContext.submit_runner_ram, name=runtimeContext.name, - merged_map=merged_map), - "success") + merged_map=merged_map, + submit_runner_image=runtimeContext.submit_runner_image) + self.stdout.write(uuid + "\n") + return (None, "success") self.apply_reqs(job_order, tool) @@ -628,22 +638,23 @@ The 'jobs' API is no longer supported. if runtimeContext.submit: # Submit a runner job to run the workflow for us. if self.work_api == "containers": - if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner): - runtimeContext.runnerjob = tool.tool["id"] + if submitting: + tool = RunnerContainer(self, updated_tool, + tool, loadingContext, runtimeContext.enable_reuse, + self.output_name, + self.output_tags, + submit_runner_ram=runtimeContext.submit_runner_ram, + name=runtimeContext.name, + on_error=runtimeContext.on_error, + submit_runner_image=runtimeContext.submit_runner_image, + intermediate_output_ttl=runtimeContext.intermediate_output_ttl, + merged_map=merged_map, + priority=runtimeContext.priority, + secret_store=self.secret_store, + collection_cache_size=runtimeContext.collection_cache_size, + collection_cache_is_default=self.should_estimate_cache_size) else: - tool = RunnerContainer(self, tool, loadingContext, runtimeContext.enable_reuse, - self.output_name, - self.output_tags, - submit_runner_ram=runtimeContext.submit_runner_ram, - name=runtimeContext.name, - on_error=runtimeContext.on_error, - submit_runner_image=runtimeContext.submit_runner_image, - intermediate_output_ttl=runtimeContext.intermediate_output_ttl, - merged_map=merged_map, - priority=runtimeContext.priority, - secret_store=self.secret_store, - collection_cache_size=runtimeContext.collection_cache_size, - collection_cache_is_default=self.should_estimate_cache_size) + runtimeContext.runnerjob = tool.tool["id"] if runtimeContext.cwl_runner_job is not None: self.uuid = runtimeContext.cwl_runner_job.get('uuid') @@ -655,7 +666,8 @@ The 'jobs' API is no longer supported. if runtimeContext.submit and not runtimeContext.wait: runnerjob = next(jobiter) runnerjob.run(runtimeContext) - return (runnerjob.uuid, "success") + self.stdout.write(runnerjob.uuid+"\n") + return (None, "success") current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) if current_container: @@ -745,13 +757,21 @@ The 'jobs' API is no longer supported. if runtimeContext.submit and isinstance(tool, Runner): logger.info("Final output collection %s", tool.final_output) + if workbench2 or workbench1: + logger.info("Output at %scollections/%s", workbench2 or workbench1, tool.final_output) else: if self.output_name is None: self.output_name = "Output of %s" % (shortname(tool.tool["id"])) if self.output_tags is None: self.output_tags = "" - storage_classes = runtimeContext.storage_classes.strip().split(",") + storage_classes = "" + storage_class_req, _ = tool.get_requirement("http://arvados.org/cwl#OutputStorageClass") + if storage_class_req and storage_class_req.get("finalStorageClass"): + storage_classes = aslist(storage_class_req["finalStorageClass"]) + else: + storage_classes = runtimeContext.storage_classes.strip().split(",") + self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output) self.set_crunch_output()