16169: Monkey patch load_tool.resolve_and_validate_document to fix bug

There is a bug in upstream cwltool where the version updater needs to
replace the document fragments in the loader index with the updated
ones, but actually it only does it for the root document.  Normally we
just fix the bug in upstream but that's challenging because current
cwltool dropped support for Python 2.7 and we're still supporting py2
in Arvados 2.0 (although py2 support will most likely be dropped in
Arvados 2.1).  Making a bugfix fork comes with its own
complications (it would need to be added to PyPi) so monkey patching
is the least disruptive fix (and is relatively safe because our
cwltool dependency is pinned to a specific version).  This
should be removed as soon as a bugfix goes into upstream cwltool and
we upgrade to it.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3dd04040ab5d728b5eac21bab601225135ce810e..2b2acd5688ff3475c24af8242ae0a7c5ef3ffcd9 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -16,6 +16,43 @@ import sys
 import re
 import pkg_resources  # part of setuptools
 
+### begin monkey patch ###
+# Monkey patch solution for bug #16169
+#
+# There is a bug in upstream cwltool where the version updater needs
+# to replace the document fragments in the loader index with the
+# updated ones, but actually it only does it for the root document.
+# Normally we just fix the bug in upstream but that's challenging
+# because current cwltool dropped support for Python 2.7 and we're
+# still supporting py2 in Arvados 2.0 (although py2 support will most
+# likely be dropped in Arvados 2.1).  Making a bugfix fork comes with
+# its own complications (it would need to be added to PyPi) so monkey
+# patching is the least disruptive fix (and is relatively safe because
+# our cwltool dependency is pinned to a specific version).  This
+# should be removed as soon as a bugfix goes into upstream cwltool and
+# we upgrade to it.
+#
+import cwltool.load_tool
+from cwltool.utils import visit_class
+from six.moves import urllib
+original_resolve_and_validate_document = cwltool.load_tool.resolve_and_validate_document
+def wrapped_resolve_and_validate_document(
+        loadingContext,            # type: LoadingContext
+        workflowobj,               # type: Union[CommentedMap, CommentedSeq]
+        uri,                       # type: Text
+        preprocess_only=False,     # type: bool
+        skip_schemas=None,         # type: Optional[bool]
+        ):
+    loadingContext, uri = original_resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only, skip_schemas)
+    if loadingContext.do_update in (True, None):
+        fileuri = urllib.parse.urldefrag(uri)[0]
+        def update_index(pr):
+            loadingContext.loader.idx[pr["id"]] = pr
+        visit_class(loadingContext.loader.idx[fileuri], ("CommandLineTool", "Workflow", "ExpressionTool"), update_index)
+    return loadingContext, uri
+cwltool.load_tool.resolve_and_validate_document = wrapped_resolve_and_validate_document
+### end monkey patch ###
+
 from schema_salad.sourceline import SourceLine
 import schema_salad.validate as validate
 import cwltool.main

diff --git a/sdk/cwl/tests/wf/16169-step.cwl b/sdk/cwl/tests/wf/16169-step.cwl
index 8386df517b986137b37d73a367b5550a26094f3b..ce6f2c0c936fd91bb5583432cd8dfd1d2be34623 100644 (file)
--- a/sdk/cwl/tests/wf/16169-step.cwl
+++ b/sdk/cwl/tests/wf/16169-step.cwl
@@ -6,10 +6,12 @@ class: CommandLineTool
 cwlVersion: v1.0
 requirements:
   InlineJavascriptRequirement: {}
+  DockerRequirement:
+    dockerPull: debian:stretch-slim
 inputs:
   d: Directory
 outputs:
   out: stdout
 stdout: output.txt
 arguments:
-  [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
\ No newline at end of file
+  [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index c89fa644ccf295e7d62bb16ed594c0951fb90e0d..6673888ab5e3f98a39c46df8baf673c5a610df76 100644 (file)
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -390,7 +390,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
     try:
         image_hash = find_one_image_hash(args.image, args.tag)
     except DockerError as error:
-        logger.error(error.message)
+        logger.error(str(error))
         sys.exit(1)
 
     if not docker_image_compatible(api, image_hash):