X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/60e38e35f49de11b5752c2bfb7f9f10443d82c49..07baa0ed049746514495d1648c1aef0c40545141:/sdk/cwl/arvados_cwl/executor.py diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 381b885431..694f77baf2 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -17,6 +17,7 @@ import copy import json import re from functools import partial +import subprocess import time import urllib @@ -24,6 +25,7 @@ from cwltool.errors import WorkflowException import cwltool.workflow from schema_salad.sourceline import SourceLine import schema_salad.validate as validate +from schema_salad.ref_resolver import file_uri, uri_file_path import arvados import arvados.config @@ -31,8 +33,8 @@ from arvados.keep import KeepClient from arvados.errors import ApiError import arvados_cwl.util -from .arvcontainer import RunnerContainer -from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps +from .arvcontainer import RunnerContainer, cleanup_name_for_collection +from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool from .arvworkflow import ArvadosWorkflow, upload_workflow from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size @@ -252,6 +254,11 @@ The 'jobs' API is no longer supported. Called when there's a need to report errors, warnings or just activity statuses, for example in the RuntimeStatusLoggingHandler. """ + + if kind not in ('error', 'warning'): + # Ignore any other status kind + return + with self.workflow_eval_lock: current = None try: @@ -261,32 +268,35 @@ The 'jobs' API is no longer supported. if current is None: return runtime_status = current.get('runtime_status', {}) - if kind in ('error', 'warning'): - updatemessage = runtime_status.get(kind, "") - if not updatemessage: - updatemessage = message - - # Subsequent messages tacked on in detail - updatedetail = runtime_status.get(kind+'Detail', "") - maxlines = 40 - if updatedetail.count("\n") < maxlines: - if updatedetail: - updatedetail += "\n" - updatedetail += message + "\n" - - if detail: - updatedetail += detail + "\n" - - if updatedetail.count("\n") >= maxlines: - updatedetail += "\nSome messages may have been omitted. Check the full log." - - runtime_status.update({ - kind: updatemessage, - kind+'Detail': updatedetail, - }) - else: - # Ignore any other status kind + + original_updatemessage = updatemessage = runtime_status.get(kind, "") + if not updatemessage: + updatemessage = message + + # Subsequent messages tacked on in detail + original_updatedetail = updatedetail = runtime_status.get(kind+'Detail', "") + maxlines = 40 + if updatedetail.count("\n") < maxlines: + if updatedetail: + updatedetail += "\n" + updatedetail += message + "\n" + + if detail: + updatedetail += detail + "\n" + + if updatedetail.count("\n") >= maxlines: + updatedetail += "\nSome messages may have been omitted. Check the full log." + + if updatemessage == original_updatemessage and updatedetail == original_updatedetail: + # don't waste time doing an update if nothing changed + # (usually because we exceeded the max lines) return + + runtime_status.update({ + kind: updatemessage, + kind+'Detail': updatedetail, + }) + try: self.api.containers().update(uuid=current['uuid'], body={ @@ -510,9 +520,75 @@ The 'jobs' API is no longer supported. for req in job_reqs: tool.requirements.append(req) + @staticmethod + def get_git_info(tool): + in_a_git_repo = False + cwd = None + filepath = None + + if tool.tool["id"].startswith("file://"): + # check if git is installed + try: + filepath = uri_file_path(tool.tool["id"]) + cwd = os.path.dirname(filepath) + subprocess.run(["git", "log", "--format=%H", "-n1", "HEAD"], cwd=cwd, check=True, capture_output=True, text=True) + in_a_git_repo = True + except Exception as e: + pass + + gitproperties = {} + + if in_a_git_repo: + git_commit = subprocess.run(["git", "log", "--format=%H", "-n1", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_date = subprocess.run(["git", "log", "--format=%cD", "-n1", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_committer = subprocess.run(["git", "log", "--format=%cn <%ce>", "-n1", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_branch = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_origin = subprocess.run(["git", "remote", "get-url", "origin"], cwd=cwd, capture_output=True, text=True).stdout + git_status = subprocess.run(["git", "status", "--untracked-files=no", "--porcelain"], cwd=cwd, capture_output=True, text=True).stdout + git_describe = subprocess.run(["git", "describe", "--always", "--tags"], cwd=cwd, capture_output=True, text=True).stdout + git_toplevel = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=cwd, capture_output=True, text=True).stdout + git_path = filepath[len(git_toplevel):] + + gitproperties = { + "http://arvados.org/cwl#gitCommit": git_commit.strip(), + "http://arvados.org/cwl#gitDate": git_date.strip(), + "http://arvados.org/cwl#gitCommitter": git_committer.strip(), + "http://arvados.org/cwl#gitBranch": git_branch.strip(), + "http://arvados.org/cwl#gitOrigin": git_origin.strip(), + "http://arvados.org/cwl#gitStatus": git_status.strip(), + "http://arvados.org/cwl#gitDescribe": git_describe.strip(), + "http://arvados.org/cwl#gitPath": git_path.strip(), + } + else: + for g in ("http://arvados.org/cwl#gitCommit", + "http://arvados.org/cwl#gitDate", + "http://arvados.org/cwl#gitCommitter", + "http://arvados.org/cwl#gitBranch", + "http://arvados.org/cwl#gitOrigin", + "http://arvados.org/cwl#gitStatus", + "http://arvados.org/cwl#gitDescribe", + "http://arvados.org/cwl#gitPath"): + if g in tool.metadata: + gitproperties[g] = tool.metadata[g] + + return gitproperties + + def set_container_request_properties(self, container, properties): + resp = self.api.container_requests().list(filters=[["container_uuid", "=", container["uuid"]]], select=["uuid", "properties"]).execute(num_retries=self.num_retries) + for cr in resp["items"]: + cr["properties"].update({k.replace("http://arvados.org/cwl#", "arv:"): v for k, v in properties.items()}) + self.api.container_requests().update(uuid=cr["uuid"], body={"container_request": {"properties": cr["properties"]}}).execute(num_retries=self.num_retries) + def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None): self.debug = runtimeContext.debug + git_info = self.get_git_info(updated_tool) + if git_info: + logger.info("Git provenance") + for g in git_info: + if git_info[g]: + logger.info(" %s: %s", g.split("#", 1)[1], git_info[g]) + workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"] workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"] controller = self.api.config()["Services"]["Controller"]["ExternalURL"] @@ -546,7 +622,10 @@ The 'jobs' API is no longer supported. runtimeContext.intermediate_storage_classes = default_storage_classes if not runtimeContext.name: - runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"]) + self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"]) + if git_info.get("http://arvados.org/cwl#gitDescribe"): + self.name = "%s (%s)" % (self.name, git_info.get("http://arvados.org/cwl#gitDescribe")) + runtimeContext.name = self.name if runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow): # When creating or updating workflow record, by default @@ -565,8 +644,9 @@ The 'jobs' API is no longer supported. self.project_uuid = runtimeContext.project_uuid # Upload local file references in the job order. - job_order = upload_job_order(self, "%s input" % runtimeContext.name, - updated_tool, job_order, runtimeContext) + with Perf(metrics, "upload_job_order"): + job_order = upload_job_order(self, "%s input" % runtimeContext.name, + updated_tool, job_order, runtimeContext) # the last clause means: if it is a command line tool, and we # are going to wait for the result, and always_submit_runner @@ -581,19 +661,23 @@ The 'jobs' API is no longer supported. loadingContext = self.loadingContext.copy() loadingContext.do_validate = False + loadingContext.disable_js_validation = True if submitting: loadingContext.do_update = False # Document may have been auto-updated. Reload the original # document with updating disabled because we want to # submit the document with its original CWL version, not # the auto-updated one. - tool = load_tool(updated_tool.tool["id"], loadingContext) + with Perf(metrics, "load_tool original"): + tool = load_tool(updated_tool.tool["id"], loadingContext) else: tool = updated_tool # Upload direct dependencies of workflow steps, get back mapping of files to keep references. # Also uploads docker images. - merged_map = upload_workflow_deps(self, tool, runtimeContext) + logger.info("Uploading workflow dependencies") + with Perf(metrics, "upload_workflow_deps"): + merged_map = upload_workflow_deps(self, tool, runtimeContext) # Recreate process object (ArvadosWorkflow or # ArvadosCommandTool) because tool document may have been @@ -602,7 +686,8 @@ The 'jobs' API is no longer supported. loadingContext.loader = tool.doc_loader loadingContext.avsc_names = tool.doc_schema loadingContext.metadata = tool.metadata - tool = load_tool(tool.tool, loadingContext) + with Perf(metrics, "load_tool"): + tool = load_tool(tool.tool, loadingContext) if runtimeContext.update_workflow or runtimeContext.create_workflow: # Create a pipeline template or workflow record and exit. @@ -614,7 +699,8 @@ The 'jobs' API is no longer supported. submit_runner_ram=runtimeContext.submit_runner_ram, name=runtimeContext.name, merged_map=merged_map, - submit_runner_image=runtimeContext.submit_runner_image) + submit_runner_image=runtimeContext.submit_runner_image, + git_info=git_info) self.stdout.write(uuid + "\n") return (None, "success") @@ -627,6 +713,11 @@ The 'jobs' API is no longer supported. runtimeContext.tmpdir_prefix = "tmp" runtimeContext.work_api = self.work_api + if not self.output_name: + self.output_name = "Output from workflow %s" % runtimeContext.name + + self.output_name = cleanup_name_for_collection(self.output_name) + if self.work_api == "containers": if self.ignore_docker_for_reuse: raise Exception("--ignore-docker-for-reuse not supported with containers API.") @@ -671,7 +762,8 @@ The 'jobs' API is no longer supported. priority=runtimeContext.priority, secret_store=self.secret_store, collection_cache_size=runtimeContext.collection_cache_size, - collection_cache_is_default=self.should_estimate_cache_size) + collection_cache_is_default=self.should_estimate_cache_size, + git_info=git_info) else: runtimeContext.runnerjob = tool.tool["id"] @@ -691,6 +783,7 @@ The 'jobs' API is no longer supported. current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) if current_container: logger.info("Running inside container %s", current_container.get("uuid")) + self.set_container_request_properties(current_container, git_info) self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout) self.polling_thread = threading.Thread(target=self.poll_states) @@ -779,8 +872,6 @@ The 'jobs' API is no longer supported. if workbench2 or workbench1: logger.info("Output at %scollections/%s", workbench2 or workbench1, tool.final_output) else: - if self.output_name is None: - self.output_name = "Output of %s" % (shortname(tool.tool["id"])) if self.output_tags is None: self.output_tags = "" @@ -792,10 +883,11 @@ The 'jobs' API is no longer supported. storage_classes = runtimeContext.storage_classes.strip().split(",") output_properties = {} - output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties") + output_properties_req, _ = tool.get_requirement("http://arvados.org/cwl#OutputCollectionProperties") if output_properties_req: - for pr in output_properties_req["processProperties"]: - output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"]) + builder = make_builder(job_order, tool.hints, tool.requirements, runtimeContext, tool.metadata) + for pr in output_properties_req["outputProperties"]: + output_properties[pr["propertyName"]] = builder.do_eval(pr["propertyValue"]) self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, output_properties,