X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a699e7edea2e2a0384ff331c3f78a2e9ba8da294..60da027b51ed0d29ae2aad263b9ebdf8cfb6f9b3:/sdk/cwl/arvados_cwl/executor.py diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 8635d5fcfe..a447f46b93 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -17,13 +17,15 @@ import copy import json import re from functools import partial +import subprocess import time import urllib from cwltool.errors import WorkflowException import cwltool.workflow -from schema_salad.sourceline import SourceLine +from schema_salad.sourceline import SourceLine, cmap import schema_salad.validate as validate +from schema_salad.ref_resolver import file_uri, uri_file_path import arvados import arvados.config @@ -34,7 +36,7 @@ import arvados_cwl.util from .arvcontainer import RunnerContainer, cleanup_name_for_collection from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool -from .arvworkflow import ArvadosWorkflow, upload_workflow +from .arvworkflow import ArvadosWorkflow, upload_workflow, new_upload_workflow, make_workflow_record from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size from .perf import Perf from .pathmapper import NoFollowPathMapper @@ -68,6 +70,10 @@ class RuntimeStatusLoggingHandler(logging.Handler): kind = 'error' elif record.levelno >= logging.WARNING: kind = 'warning' + if kind == 'warning' and record.name == "salad": + # Don't send validation warnings to runtime status, + # they're noisy and unhelpful. + return if kind is not None and self.updatingRuntimeStatus is not True: self.updatingRuntimeStatus = True try: @@ -110,6 +116,9 @@ class ArvCwlExecutor(object): arvargs.output_tags = None arvargs.thread_count = 1 arvargs.collection_cache_size = None + arvargs.git_info = True + arvargs.submit = False + arvargs.defer_downloads = False self.api = api_client self.processes = {} @@ -135,6 +144,8 @@ class ArvCwlExecutor(object): self.fs_access = None self.secret_store = None self.stdout = stdout + self.fast_submit = False + self.git_info = arvargs.git_info if keep_client is not None: self.keep_client = keep_client @@ -201,6 +212,8 @@ The 'jobs' API is no longer supported. self.toplevel_runtimeContext.make_fs_access = partial(CollectionFsAccess, collection_cache=self.collection_cache) + self.defer_downloads = arvargs.submit and arvargs.defer_downloads + validate_cluster_target(self, self.toplevel_runtimeContext) @@ -356,8 +369,8 @@ The 'jobs' API is no longer supported. page = keys[:pageSize] try: proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries) - except Exception: - logger.exception("Error checking states on API server: %s") + except Exception as e: + logger.exception("Error checking states on API server: %s", e) remain_wait = self.poll_interval continue @@ -518,15 +531,82 @@ The 'jobs' API is no longer supported. for req in job_reqs: tool.requirements.append(req) + @staticmethod + def get_git_info(tool): + in_a_git_repo = False + cwd = None + filepath = None + + if tool.tool["id"].startswith("file://"): + # check if git is installed + try: + filepath = uri_file_path(tool.tool["id"]) + cwd = os.path.dirname(filepath) + subprocess.run(["git", "log", "--format=%H", "-n1", "HEAD"], cwd=cwd, check=True, capture_output=True, text=True) + in_a_git_repo = True + except Exception as e: + pass + + gitproperties = {} + + if in_a_git_repo: + git_commit = subprocess.run(["git", "log", "--format=%H", "-n1", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_date = subprocess.run(["git", "log", "--format=%cD", "-n1", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_committer = subprocess.run(["git", "log", "--format=%cn <%ce>", "-n1", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_branch = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd, capture_output=True, text=True).stdout + git_origin = subprocess.run(["git", "remote", "get-url", "origin"], cwd=cwd, capture_output=True, text=True).stdout + git_status = subprocess.run(["git", "status", "--untracked-files=no", "--porcelain"], cwd=cwd, capture_output=True, text=True).stdout + git_describe = subprocess.run(["git", "describe", "--always", "--tags"], cwd=cwd, capture_output=True, text=True).stdout + git_toplevel = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=cwd, capture_output=True, text=True).stdout + git_path = filepath[len(git_toplevel):] + + gitproperties = { + "http://arvados.org/cwl#gitCommit": git_commit.strip(), + "http://arvados.org/cwl#gitDate": git_date.strip(), + "http://arvados.org/cwl#gitCommitter": git_committer.strip(), + "http://arvados.org/cwl#gitBranch": git_branch.strip(), + "http://arvados.org/cwl#gitOrigin": git_origin.strip(), + "http://arvados.org/cwl#gitStatus": git_status.strip(), + "http://arvados.org/cwl#gitDescribe": git_describe.strip(), + "http://arvados.org/cwl#gitPath": git_path.strip(), + } + else: + for g in ("http://arvados.org/cwl#gitCommit", + "http://arvados.org/cwl#gitDate", + "http://arvados.org/cwl#gitCommitter", + "http://arvados.org/cwl#gitBranch", + "http://arvados.org/cwl#gitOrigin", + "http://arvados.org/cwl#gitStatus", + "http://arvados.org/cwl#gitDescribe", + "http://arvados.org/cwl#gitPath"): + if g in tool.metadata: + gitproperties[g] = tool.metadata[g] + + return gitproperties + + def set_container_request_properties(self, container, properties): + resp = self.api.container_requests().list(filters=[["container_uuid", "=", container["uuid"]]], select=["uuid", "properties"]).execute(num_retries=self.num_retries) + for cr in resp["items"]: + cr["properties"].update({k.replace("http://arvados.org/cwl#", "arv:"): v for k, v in properties.items()}) + self.api.container_requests().update(uuid=cr["uuid"], body={"container_request": {"properties": cr["properties"]}}).execute(num_retries=self.num_retries) + def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None): self.debug = runtimeContext.debug + git_info = self.get_git_info(updated_tool) if self.git_info else {} + if git_info: + logger.info("Git provenance") + for g in git_info: + if git_info[g]: + logger.info(" %s: %s", g.split("#", 1)[1], git_info[g]) + workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"] workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"] controller = self.api.config()["Services"]["Controller"]["ExternalURL"] logger.info("Using cluster %s (%s)", self.api.config()["ClusterID"], workbench2 or workbench1 or controller) - updated_tool.visit(self.check_features) + if not self.fast_submit: + updated_tool.visit(self.check_features) self.pipeline = None self.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir) @@ -554,7 +634,10 @@ The 'jobs' API is no longer supported. runtimeContext.intermediate_storage_classes = default_storage_classes if not runtimeContext.name: - runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"]) + self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"]) + if git_info.get("http://arvados.org/cwl#gitDescribe"): + self.name = "%s (%s)" % (self.name, git_info.get("http://arvados.org/cwl#gitDescribe")) + runtimeContext.name = self.name if runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow): # When creating or updating workflow record, by default @@ -591,47 +674,72 @@ The 'jobs' API is no longer supported. loadingContext = self.loadingContext.copy() loadingContext.do_validate = False loadingContext.disable_js_validation = True - if submitting: - loadingContext.do_update = False - # Document may have been auto-updated. Reload the original - # document with updating disabled because we want to - # submit the document with its original CWL version, not - # the auto-updated one. - with Perf(metrics, "load_tool original"): - tool = load_tool(updated_tool.tool["id"], loadingContext) - else: - tool = updated_tool + # if submitting and not self.fast_submit: + # loadingContext.do_update = False + # # Document may have been auto-updated. Reload the original + # # document with updating disabled because we want to + # # submit the document with its original CWL version, not + # # the auto-updated one. + # with Perf(metrics, "load_tool original"): + # tool = load_tool(updated_tool.tool["id"], loadingContext) + # else: + tool = updated_tool # Upload direct dependencies of workflow steps, get back mapping of files to keep references. # Also uploads docker images. - logger.info("Uploading workflow dependencies") - with Perf(metrics, "upload_workflow_deps"): - merged_map = upload_workflow_deps(self, tool, runtimeContext) + if not self.fast_submit: + logger.info("Uploading workflow dependencies") + with Perf(metrics, "upload_workflow_deps"): + merged_map = upload_workflow_deps(self, tool, runtimeContext) + else: + merged_map = {} # Recreate process object (ArvadosWorkflow or # ArvadosCommandTool) because tool document may have been # updated by upload_workflow_deps in ways that modify - # inheritance of hints or requirements. + # hints or requirements. loadingContext.loader = tool.doc_loader loadingContext.avsc_names = tool.doc_schema loadingContext.metadata = tool.metadata - with Perf(metrics, "load_tool"): - tool = load_tool(tool.tool, loadingContext) - - if runtimeContext.update_workflow or runtimeContext.create_workflow: - # Create a pipeline template or workflow record and exit. - if self.work_api == "containers": - uuid = upload_workflow(self, tool, job_order, - runtimeContext.project_uuid, - runtimeContext, - uuid=runtimeContext.update_workflow, - submit_runner_ram=runtimeContext.submit_runner_ram, - name=runtimeContext.name, - merged_map=merged_map, - submit_runner_image=runtimeContext.submit_runner_image) + loadingContext.skip_resolve_all = True + #with Perf(metrics, "load_tool"): + # tool = load_tool(tool.tool, loadingContext) + + if submitting and not self.fast_submit: + # upload workflow and get back the workflow wrapper + + workflow_wrapper = new_upload_workflow(self, tool, job_order, + runtimeContext.project_uuid, + runtimeContext, + uuid=runtimeContext.update_workflow, + submit_runner_ram=runtimeContext.submit_runner_ram, + name=runtimeContext.name, + merged_map=merged_map, + submit_runner_image=runtimeContext.submit_runner_image, + git_info=git_info, + set_defaults=(runtimeContext.update_workflow or runtimeContext.create_workflow)) + + if runtimeContext.update_workflow or runtimeContext.create_workflow: + # Now create a workflow record and exit. + uuid = make_workflow_record(self, workflow_wrapper, runtimeContext.name, tool, + runtimeContext.project_uuid, runtimeContext.update_workflow) self.stdout.write(uuid + "\n") return (None, "success") + loadingContext.loader.idx.clear() + loadingContext.loader.idx["_:main"] = workflow_wrapper + workflow_wrapper["id"] = "_:main" + + # Reload just the wrapper workflow. + self.fast_submit = True + #print("bah bah", loadingContext.requirements) + #workflow_wrapper, _ = loadingContext.loader.resolve_all(cmap(workflow_wrapper), "_:main", checklinks=True) + + #tool = load_tool(workflow_wrapper[0], loadingContext) + #print(json.dumps(workflow_wrapper, indent=2)) + tool = load_tool(workflow_wrapper, loadingContext) + + self.apply_reqs(job_order, tool) self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse @@ -690,7 +798,8 @@ The 'jobs' API is no longer supported. priority=runtimeContext.priority, secret_store=self.secret_store, collection_cache_size=runtimeContext.collection_cache_size, - collection_cache_is_default=self.should_estimate_cache_size) + collection_cache_is_default=self.should_estimate_cache_size, + git_info=git_info) else: runtimeContext.runnerjob = tool.tool["id"] @@ -710,6 +819,7 @@ The 'jobs' API is no longer supported. current_container = arvados_cwl.util.get_current_container(self.api, self.num_retries, logger) if current_container: logger.info("Running inside container %s", current_container.get("uuid")) + self.set_container_request_properties(current_container, git_info) self.poll_api = arvados.api('v1', timeout=runtimeContext.http_timeout) self.polling_thread = threading.Thread(target=self.poll_states)