8654: Add comments

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 80713037f4193e5b8e77df00fc45a574dfa86006..d628f1c12d2adc80707523e84a8be6b835cf426d 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -1,5 +1,13 @@
 #!/usr/bin/env python
 
+# Crunch script integration for running arvados-cwl-runner (importing
+# arvados_cwl module) inside a crunch job.
+#
+# This gets the job record, transforms the script parameters into a valid CWL
+# input object, then executes the CWL runner to run the underlying workflow or
+# tool.  When the workflow completes, record the output object in an output
+# collection for this runner job.
+
 import arvados
 import arvados_cwl
 import arvados.collection
@@ -13,7 +21,8 @@ import argparse
 from arvados.api import OrderedJsonModel
 from cwltool.process import adjustFiles
 
-print cwltool.main.versionstring()
+# Print package versions
+logging.info(cwltool.main.versionstring())
 
 api = arvados.api("v1")
 
@@ -32,13 +41,13 @@ try:
 
     t = cwltool.main.load_tool(job_order_object, False, True, runner.arvMakeTool, True)
 
-    np = argparse.Namespace()
-    np.project_uuid = arvados.current_job()["owner_uuid"]
-    np.enable_reuse = True
-    np.submit = False
-    np.debug = True
-    np.quiet = False
-    outputObj = runner.arvExecutor(t, job_order_object, "", np, cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]})
+    args = argparse.Namespace()
+    args.project_uuid = arvados.current_job()["owner_uuid"]
+    args.enable_reuse = True
+    args.submit = False
+    args.debug = True
+    args.quiet = False
+    outputObj = runner.arvExecutor(t, job_order_object, "", args, cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]})
 
     files = {}
     def capture(path):

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3762536d5913095507afae7b3cc9ed3673be3501..5958100ffa2060449abb0fa37299a5a0b09acb66 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+# Implement cwl-runner interface for submitting and running jobs on Arvados.
+
 import argparse
 import arvados
 import arvados.events
@@ -35,6 +37,8 @@ keepre = re.compile(r"^\S+ \S+ \d+ \d+ stderr \S+ \S+ crunchrunner: \$\(task\.ke
 
 
 def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
+    """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
+
     if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
         dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
 
@@ -58,6 +62,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
 
 
 class CollectionFsAccess(cwltool.process.StdFsAccess):
+    """Implement the cwltool FsAccess interface for Arvados Collections."""
+
     def __init__(self, basedir):
         self.collections = {}
         self.basedir = basedir
@@ -114,6 +120,8 @@ class CollectionFsAccess(cwltool.process.StdFsAccess):
             return os.path.exists(self._abs(fn))
 
 class ArvadosJob(object):
+    """Submit and manage a Crunch job for executing a CWL CommandLineTool."""
+
     def __init__(self, runner):
         self.arvrunner = runner
         self.running = False
@@ -283,6 +291,8 @@ class ArvadosJob(object):
 
 
 class RunnerJob(object):
+    """Submit and manage a Crunch job that runs crunch_scripts/cwl-runner."""
+
     def __init__(self, runner, tool, job_order, enable_reuse):
         self.arvrunner = runner
         self.tool = tool
@@ -381,6 +391,8 @@ class RunnerJob(object):
             del self.arvrunner.jobs[record["uuid"]]
 
 class ArvPathMapper(cwltool.pathmapper.PathMapper):
+    """Convert container-local paths to and from Keep collection ids."""
+
     def __init__(self, arvrunner, referenced_files, basedir,
                  collection_pattern, file_pattern, name=None, **kwargs):
         self._pathmap = arvrunner.get_uploaded()
@@ -430,6 +442,8 @@ class ArvPathMapper(cwltool.pathmapper.PathMapper):
 
 
 class ArvadosCommandTool(cwltool.draft2tool.CommandLineTool):
+    """Wrap cwltool CommandLineTool to override selected methods."""
+
     def __init__(self, arvrunner, toolpath_object, **kwargs):
         super(ArvadosCommandTool, self).__init__(toolpath_object, **kwargs)
         self.arvrunner = arvrunner
@@ -445,6 +459,9 @@ class ArvadosCommandTool(cwltool.draft2tool.CommandLineTool):
 
 
 class ArvCwlRunner(object):
+    """Execute a CWL tool or workflow, submit crunch jobs, wait for them to
+    complete, and report output."""
+
     def __init__(self, api_client):
         self.api = api_client
         self.jobs = {}
@@ -593,6 +610,8 @@ class ArvCwlRunner(object):
             return self.final_output
 
 def versionstring():
+    """Print version string of key packages for provenance and debugging."""
+
     cwlpkg = pkg_resources.require("cwltool")
     arvpkg = pkg_resources.require("arvados-python-client")
     arvcwlpkg = pkg_resources.require("arvados-cwl-runner")

diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 7a298b4c1723ae2f5c3c1836d897a22c08547249..5d29c45117acd71e924838bb9b758af77d8e9b91 100644 (file)
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -110,7 +110,7 @@ def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
 def uploadfiles(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
     # Find the smallest path prefix that includes all the files that need to be uploaded.
     # This starts at the root and iteratively removes common parent directory prefixes
-    # until all file pathes no longer have a common parent.
+    # until all file paths no longer have a common parent.
     n = True
     pathprefix = "/"
     while n: