19385: Reference rewriting fixups

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index af838d68e8c7e33ac5f7d1d0f10e52fa7b95b47f..bf1ab3418937a0d2a9bbe16937f9aae257543e60 100755 (executable)
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -16,6 +16,7 @@ Syntax:
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 CWLTOOL=path           (optional) Path to cwltool git repository.
 SALAD=path             (optional) Path to schema_salad git repository.
+CWL_UTILS=path         (optional) Path to cwl-utils git repository.
 PYCMD=pythonexec       (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
 
 EOF
@@ -67,6 +68,14 @@ if [[ -n "$CWLTOOL" ]] ; then
     cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
 fi
 
+rm -rf sdk/cwl/cwlutils_dist
+mkdir -p sdk/cwl/cwlutils_dist
+if [[ -n "$CWL_UTILS" ]] ; then
+    (cd "$CWL_UTILS" && python3 setup.py sdist)
+    cwlutils=$(cd "$CWL_UTILS/dist" && ls -t cwl-utils-*.tar.gz | head -n1)
+    cp "$CWL_UTILS/dist/$cwlutils" $WORKSPACE/sdk/cwl/cwlutils_dist
+fi
+
 . build/run-library.sh
 
 # This defines python_sdk_version and cwl_runner_version with python-style
@@ -74,6 +83,17 @@ fi
 calculate_python_sdk_cwl_package_versions
 
 set -x
-docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+docker build --no-cache \
+       --build-arg sdk=$sdk \
+       --build-arg runner=$runner \
+       --build-arg salad=$salad \
+       --build-arg cwltool=$cwltool \
+       --build-arg pythoncmd=$py \
+       --build-arg pipcmd=$pipcmd \
+       --build-arg cwlutils=$cwlutils \
+       -f "$WORKSPACE/sdk/dev-jobs.dockerfile" \
+       -t arvados/jobs:$cwl_runner_version \
+       "$WORKSPACE/sdk"
+
 echo arv-keepdocker arvados/jobs $cwl_runner_version
 arv-keepdocker arvados/jobs $cwl_runner_version

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 3b15fcede5d5a8528d8d5f11c8a3a385b7090efa..35cbf94353bdd6f0235d0bc3d24df65a746d25ca 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -128,22 +128,34 @@ def make_wrapper_workflow(arvRunner, main, packed, project_uuid, name, git_info,
     return json.dumps(doc, sort_keys=True, indent=4, separators=(',',': '))
 
 def rel_ref(s, baseuri, urlexpander, merged_map):
+    if s.startswith("keep:"):
+        return s
+
+    #print("BBB", s, baseuri)
     uri = urlexpander(s, baseuri)
+    #print("CCC", uri)
+
     fileuri = urllib.parse.urldefrag(baseuri)[0]
-    if fileuri in merged_map:
-        replacements = merged_map[fileuri].resolved
-        if uri in replacements:
-            return replacements[uri]
 
-    if s.startswith("keep:"):
-        return s
+    for u in (baseuri, fileuri):
+        if u in merged_map:
+            replacements = merged_map[u].resolved
+            #print(uri, replacements)
+            if uri in replacements:
+                return replacements[uri]
 
-    p1 = os.path.dirname(uri_file_path(baseuri))
+    p1 = os.path.dirname(uri_file_path(fileuri))
     p2 = os.path.dirname(uri_file_path(uri))
     p3 = os.path.basename(uri_file_path(uri))
+
+    #print("PPP", p1, p2, p3)
+
     r = os.path.relpath(p2, p1)
     if r == ".":
         r = ""
+
+    #print("RRR", r)
+
     return os.path.join(r, p3)
 
 
@@ -167,10 +179,26 @@ def update_refs(d, baseuri, urlexpander, merged_map, set_block_style, runtimeCon
             d["http://arvados.org/cwl#dockerCollectionPDH"] = runtimeContext.cached_docker_lookups.get(dockerImageId)
 
         for s in d:
-            for field in ("$include", "$import", "location", "run"):
+            for field in ("location", "run", "name"):
                 if field in d and isinstance(d[field], str):
                     d[field] = rel_ref(d[field], baseuri, urlexpander, merged_map)
 
+            for field in ("$include", "$import"):
+                if field in d and isinstance(d[field], str):
+                    d[field] = rel_ref(d[field], baseuri, urlexpander, {})
+
+            basetypes = ("null", "boolean", "int", "long", "float", "double", "string", "File", "Directory")
+
+            if ("type" in d and
+                isinstance(d["type"], str) and
+                d["type"] not in basetypes):
+                d["type"] = rel_ref(d["type"], baseuri, urlexpander, merged_map)
+
+            if "inputs" in d and isinstance(d["inputs"], MutableMapping):
+                for inp in d["inputs"]:
+                    if isinstance(d["inputs"][inp], str) and d["inputs"][inp] not in basetypes:
+                        d["inputs"][inp] = rel_ref(d["inputs"][inp], baseuri, urlexpander, merged_map)
+
             if "$schemas" in d:
                 for n, s in enumerate(d["$schemas"]):
                     d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map)
@@ -221,6 +249,8 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
 
     col = arvados.collection.Collection(api_client=arvRunner.api)
 
+    #print(merged_map)
+
     for w in workflow_files | import_files:
         # 1. load YAML
 
@@ -363,8 +393,13 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
         for g in git_info:
             doc[g] = git_info[g]
 
+    #print("MMM", main["id"])
+    #print(yamlloader.dump(wrapper, stream=sys.stdout))
+
     update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, False, runtimeContext, main["id"]+"#", "#main/")
 
+    #print(yamlloader.dump(wrapper, stream=sys.stdout))
+
     return doc
 
 

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 053983b0110abf328bc611ccfa250d8b0df54df0..ee8917b881f2f24d7171bd523d818b1024e59fb1 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -23,7 +23,7 @@ import urllib
 
 from cwltool.errors import WorkflowException
 import cwltool.workflow
-from schema_salad.sourceline import SourceLine
+from schema_salad.sourceline import SourceLine, cmap
 import schema_salad.validate as validate
 from schema_salad.ref_resolver import file_uri, uri_file_path
 
@@ -705,7 +705,7 @@ The 'jobs' API is no longer supported.
         #with Perf(metrics, "load_tool"):
         #    tool = load_tool(tool.tool, loadingContext)
 
-        if runtimeContext.update_workflow or runtimeContext.create_workflow or (runtimeContext.submit and not self.fast_submit):
+        if submitting and not self.fast_submit:
             # upload workflow and get back the workflow wrapper
 
             workflow_wrapper = new_upload_workflow(self, tool, job_order,
@@ -730,7 +730,8 @@ The 'jobs' API is no longer supported.
 
             # Reload just the wrapper workflow.
             self.fast_submit = True
-            tool = load_tool(workflow_wrapper, loadingContext)
+            workflow_wrapper, _ = loadingContext.loader.resolve_all(cmap(workflow_wrapper), tool.tool["id"])
+            tool = load_tool(workflow_wrapper[0], loadingContext)
 
 
         self.apply_reqs(job_order, tool)

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index ae9f6d0d41c0ec3bf0accb4b9fe79c991a9b9162..bf5a9a9f1f70d6aa8643596a38d9f553ed07d9d2 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -36,7 +36,7 @@ setup(name='arvados-cwl-runner',
       # file to determine what version of cwltool and schema-salad to
       # build.
       install_requires=[
-          'cwltool==3.1.20221224142944',
+          'cwltool==3.1.20230119183816',
           'schema-salad>8.3.20220913105718',
           'arvados-python-client{}'.format(pysdk_dep),
           'ciso8601 >= 2.0.0',

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 60db4a889815c1f66e9c1070b35b1077fc5b1d8b..95b039eba9231588b0fce92af7d6b3a369af1f24 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -28,16 +28,19 @@ RUN apt-get update -q && apt-get install -qy --no-install-recommends \
 ARG sdk
 ARG runner
 ARG salad
+ARG cwlutils
 ARG cwltool
 
 ADD python/dist/$sdk /tmp/
 ADD cwl/salad_dist/$salad /tmp/
 ADD cwl/cwltool_dist/$cwltool /tmp/
+ADD cwl/cwlutils_dist/$cwlutils /tmp/
 ADD cwl/dist/$runner /tmp/
 
 RUN $pipcmd install wheel
 RUN cd /tmp/arvados-python-client-* && $pipcmd install .
 RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
+RUN if test -d /tmp/cwl-utils-* ; then cd /tmp/cwl-utils-* && $pipcmd install . ; fi
 RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
 RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .