19280: Add metrics checkpoints

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 9620079ed1eef1e841c5d9b45d8a39b915b571b6..3e8115ddb13d21716639746aa39cf5a096f9e83b 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -594,6 +594,7 @@ The 'jobs' API is no longer supported.
 
         # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
         # Also uploads docker images.
 
         # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
         # Also uploads docker images.

+        logger.info("Uploading workflow dependencies")

         merged_map = upload_workflow_deps(self, tool, runtimeContext)
 
         # Recreate process object (ArvadosWorkflow or
         merged_map = upload_workflow_deps(self, tool, runtimeContext)
 
         # Recreate process object (ArvadosWorkflow or

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ed4abbe2ee15c87a677ca5987ca15f22c09716a7..934aeb4018ab89f427676a741e7f113896900d12 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -49,8 +49,10 @@ from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, col
 from ._version import __version__
 from . import done
 from . context import ArvRuntimeContext
 from ._version import __version__
 from . import done
 from . context import ArvRuntimeContext

+from .perf import Perf

 
 logger = logging.getLogger('arvados.cwl-runner')
 
 logger = logging.getLogger('arvados.cwl-runner')

+metrics = logging.getLogger('arvados.cwl-runner.metrics')

 
 def trim_anonymous_location(obj):
     """Remove 'location' field from File and Directory literals.
 
 def trim_anonymous_location(obj):
     """Remove 'location' field from File and Directory literals.


@@ -653,7 +655,8 @@ FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
 def upload_workflow_deps(arvrunner, tool, runtimeContext):
     # Ensure that Docker images needed by this workflow are available
 
 def upload_workflow_deps(arvrunner, tool, runtimeContext):
     # Ensure that Docker images needed by this workflow are available
 

-    upload_docker(arvrunner, tool, runtimeContext)
+    with Perf(metrics, "upload_docker"):
+        upload_docker(arvrunner, tool, runtimeContext)

 
     document_loader = tool.doc_loader
 
 
     document_loader = tool.doc_loader
 


@@ -662,16 +665,17 @@ def upload_workflow_deps(arvrunner, tool, runtimeContext):
     def upload_tool_deps(deptool):
         if "id" in deptool:
             discovered_secondaryfiles = {}
     def upload_tool_deps(deptool):
         if "id" in deptool:
             discovered_secondaryfiles = {}

-            pm = upload_dependencies(arvrunner,
-                                     "%s dependencies" % (shortname(deptool["id"])),
-                                     document_loader,
-                                     deptool,
-                                     deptool["id"],
-                                     False,
-                                     runtimeContext,
-                                     include_primary=False,
-                                     discovered_secondaryfiles=discovered_secondaryfiles,
-                                     cache=tool_dep_cache)
+            with Perf(metrics, "upload_dependencies %s" % shortname(deptool["id"]):
+                pm = upload_dependencies(arvrunner,
+                                         "%s dependencies" % (shortname(deptool["id"])),
+                                         document_loader,
+                                         deptool,
+                                         deptool["id"],
+                                         False,
+                                         runtimeContext,
+                                         include_primary=False,
+                                         discovered_secondaryfiles=discovered_secondaryfiles,
+                                         cache=tool_dep_cache)

             document_loader.idx[deptool["id"]] = deptool
             toolmap = {}
             for k,v in pm.items():
             document_loader.idx[deptool["id"]] = deptool
             toolmap = {}
             for k,v in pm.items():