X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2de2c96925cc3439305f16dced7f89bd9124853d..873fcf181c037cc1e42419bfeaf5bb70c9d9e239:/sdk/cwl/arvados_cwl/executor.py diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index ef84dd4983..240e014e5a 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -2,12 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -from __future__ import division -from builtins import next -from builtins import object -from builtins import str -from future.utils import viewvalues, viewitems - import argparse import logging import os @@ -34,7 +28,7 @@ from arvados.errors import ApiError import arvados_cwl.util from .arvcontainer import RunnerContainer, cleanup_name_for_collection -from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder, update_from_merged_map +from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder, update_from_merged_map, print_keep_deps from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool from .arvworkflow import ArvadosWorkflow, upload_workflow, make_workflow_record from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size @@ -70,7 +64,7 @@ class RuntimeStatusLoggingHandler(logging.Handler): kind = 'error' elif record.levelno >= logging.WARNING: kind = 'warning' - if kind == 'warning' and record.name == "salad": + if kind == 'warning' and record.name in ("salad", "crunchstat_summary"): # Don't send validation warnings to runtime status, # they're noisy and unhelpful. return @@ -146,6 +140,7 @@ class ArvCwlExecutor(object): self.stdout = stdout self.fast_submit = False self.git_info = arvargs.git_info + self.debug = False if keep_client is not None: self.keep_client = keep_client @@ -266,7 +261,7 @@ The 'jobs' API is no longer supported. activity statuses, for example in the RuntimeStatusLoggingHandler. """ - if kind not in ('error', 'warning'): + if kind not in ('error', 'warning', 'activity'): # Ignore any other status kind return @@ -281,7 +276,7 @@ The 'jobs' API is no longer supported. runtime_status = current.get('runtime_status', {}) original_updatemessage = updatemessage = runtime_status.get(kind, "") - if not updatemessage: + if kind == "activity" or not updatemessage: updatemessage = message # Subsequent messages tacked on in detail @@ -368,9 +363,11 @@ The 'jobs' API is no longer supported. while keys: page = keys[:pageSize] try: - proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries) + proc_states = table.list(filters=[["uuid", "in", page]], select=["uuid", "container_uuid", "state", "log_uuid", + "output_uuid", "modified_at", "properties", + "runtime_constraints"]).execute(num_retries=self.num_retries) except Exception as e: - logger.exception("Error checking states on API server: %s", e) + logger.warning("Temporary error checking states on API server: %s", e) remain_wait = self.poll_interval continue @@ -418,7 +415,7 @@ The 'jobs' API is no longer supported. if obj.get("class") == "InplaceUpdateRequirement": if obj["inplaceUpdate"] and parentfield == "requirements": raise SourceLine(obj, "class", UnsupportedRequirement).makeError("InplaceUpdateRequirement not supported for keep collections.") - for k,v in viewitems(obj): + for k,v in obj.items(): self.check_features(v, parentfield=k) elif isinstance(obj, list): for i,v in enumerate(obj): @@ -593,6 +590,8 @@ The 'jobs' API is no longer supported. def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None): self.debug = runtimeContext.debug + self.runtime_status_update("activity", "initialization") + git_info = self.get_git_info(updated_tool) if self.git_info else {} if git_info: logger.info("Git provenance") @@ -600,6 +599,8 @@ The 'jobs' API is no longer supported. if git_info[g]: logger.info(" %s: %s", g.split("#", 1)[1], git_info[g]) + runtimeContext.git_info = git_info + workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"] workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"] controller = self.api.config()["Services"]["Controller"]["ExternalURL"] @@ -646,6 +647,10 @@ The 'jobs' API is no longer supported. runtimeContext.copy_deps = True runtimeContext.match_local_docker = True + if runtimeContext.print_keep_deps: + runtimeContext.copy_deps = False + runtimeContext.match_local_docker = False + if runtimeContext.update_workflow and self.project_uuid is None: # If we are updating a workflow, make sure anything that # gets uploaded goes into the same parent project, unless @@ -655,6 +660,8 @@ The 'jobs' API is no longer supported. self.project_uuid = runtimeContext.project_uuid + self.runtime_status_update("activity", "data transfer") + # Upload local file references in the job order. with Perf(metrics, "upload_job_order"): job_order, jobmapper = upload_job_order(self, "%s input" % runtimeContext.name, @@ -666,12 +673,10 @@ The 'jobs' API is no longer supported. # are going to wait for the result, and always_submit_runner # is false, then we don't submit a runner process. - submitting = (runtimeContext.update_workflow or - runtimeContext.create_workflow or - (runtimeContext.submit and not + submitting = (runtimeContext.submit and not (updated_tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and - not runtimeContext.always_submit_runner))) + not runtimeContext.always_submit_runner)) loadingContext = self.loadingContext.copy() loadingContext.do_validate = False @@ -697,7 +702,7 @@ The 'jobs' API is no longer supported. loadingContext.skip_resolve_all = True workflow_wrapper = None - if submitting and not self.fast_submit: + if (submitting and not self.fast_submit) or runtimeContext.update_workflow or runtimeContext.create_workflow or runtimeContext.print_keep_deps: # upload workflow and get back the workflow wrapper workflow_wrapper = upload_workflow(self, tool, job_order, @@ -720,6 +725,11 @@ The 'jobs' API is no longer supported. self.stdout.write(uuid + "\n") return (None, "success") + if runtimeContext.print_keep_deps: + # Just find and print out all the collection dependencies and exit + print_keep_deps(self, runtimeContext, merged_map, tool) + return (None, "success") + # Did not register a workflow, we're going to submit # it instead. loadingContext.loader.idx.clear() @@ -823,6 +833,8 @@ The 'jobs' API is no longer supported. # We either running the workflow directly, or submitting it # and will wait for a final result. + self.runtime_status_update("activity", "workflow execution") + current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) if current_container: logger.info("Running inside container %s", current_container.get("uuid")) @@ -860,7 +872,8 @@ The 'jobs' API is no longer supported. if (self.task_queue.in_flight + len(self.processes)) > 0: self.workflow_eval_lock.wait(3) else: - logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.") + if self.final_status is None: + logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.") break if self.stop_polling.is_set(): @@ -910,6 +923,11 @@ The 'jobs' API is no longer supported. if self.final_output is None: raise WorkflowException("Workflow did not return a result.") + if runtimeContext.usage_report_notes: + logger.info("Steps with low resource utilization (possible optimization opportunities):") + for x in runtimeContext.usage_report_notes: + logger.info(" %s", x) + if runtimeContext.submit and isinstance(tool, Runner): logger.info("Final output collection %s", tool.final_output) if workbench2 or workbench1: