10129: Ensure that directory literals get preserved when trimming keep directory...

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 59760810f3038673ac00cc04323b63612a6eefea..3bb0a34197594774f81049ff5ccdf606dc2dfcde 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -85,12 +85,13 @@ class ArvadosWorkflow(Workflow):
             def keepmount(obj):
                 if obj["location"].startswith("keep:"):
                     obj["location"] = "/keep/" + obj["location"][5:]
+                    if "listing" in obj:
+                        del obj["listing"]
                 elif obj["location"].startswith("_:"):
-                    pass
+                    del obj["location"]
                 else:
                     raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
-                if "listing" in obj:
-                    del obj["listing"]
+
             adjustFileObjs(joborder_keepmount, keepmount)
             adjustDirObjs(joborder_keepmount, keepmount)
             adjustFileObjs(packed, keepmount)

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4e91d6628a337d7e549220ec76887c19ec60ccb5..aafe7c3dd4560d7c3e48c4bbeba99012159b36b4 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -26,6 +26,8 @@ cwltool.draft2tool.ACCEPTLIST_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
 def del_listing(obj):
     if obj.get("location", "").startswith("keep:") and "listing" in obj:
         del obj["listing"]
+    if obj.get("location", "").startswith("_:"):
+        del obj["location"]
 
 def upload_dependencies(arvrunner, name, document_loader,
                         workflowobj, uri, loadref_run):
@@ -84,7 +86,7 @@ def upload_dependencies(arvrunner, name, document_loader,
                            name=name)
 
     def setloc(p):
-        if not p["location"].startswith("_:") and not p["location"].startswith("keep:"):
+        if "location" in p and (not p["location"].startswith("_:")) and (not p["location"].startswith("keep:")):
             p["location"] = mapper.mapper(p["location"]).resolved
     adjustFileObjs(workflowobj, setloc)
     adjustDirObjs(workflowobj, setloc)

diff --git a/sdk/cwl/tests/submit_test_job.json b/sdk/cwl/tests/submit_test_job.json
index 02d61fa558b334ef74f88524b0907334ce161dd7..49d5944c06d81f6f3ece48c8f37ce5421859a942 100644 (file)
--- a/sdk/cwl/tests/submit_test_job.json
+++ b/sdk/cwl/tests/submit_test_job.json
@@ -10,5 +10,14 @@
             "class": "File",
             "location": "keep:99999999999999999999999999999998+99/file1.txt"
         }]
+    },
+    "z": {
+        "class": "Directory",
+        "basename": "anonymous",
+        "listing": [{
+            "basename": "renamed.txt",
+            "class": "File",
+            "location": "keep:99999999999999999999999999999998+99/file1.txt"
+        }]
     }
 }

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index f57ff320b2a9492e875a79c10c9e651919e2c1f1..eb7199295144368d30da1122e435209e2c6323fa 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -93,6 +93,15 @@ def stubs(func):
                     'location': 'keep:99999999999999999999999999999998+99',
                     'class': 'Directory'
                 },
+                'z': {
+                    'basename': 'anonymous',
+                    "listing": [{
+                        "basename": "renamed.txt",
+                        "class": "File",
+                        "location": "keep:99999999999999999999999999999998+99/file1.txt"
+                    }],
+                    'class': 'Directory'
+                },
                 'cwl:tool':
                 '99999999999999999999999999999991+99/wf/submit_wf.cwl'
             },
@@ -117,7 +126,7 @@ def stubs(func):
                     'kind': 'file'
                 },
                 '/var/lib/cwl/job/cwl.input.json': {
-                    'portable_data_hash': '606be75b6e4f811a2f282d7fac867043+60/cwl.input.json',
+                    'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
                     'kind': 'collection'
                 }
             },
@@ -160,7 +169,7 @@ class TestSubmit(unittest.TestCase):
                 'manifest_text':
                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
-                '0f8864f292e901019c43fdabacd62c3e+383 0:383:submit_wf.cwl\n',
+                'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
                 'name': 'submit_wf.cwl',
             }, ensure_unique_name=True),
@@ -220,7 +229,7 @@ class TestSubmit(unittest.TestCase):
                 'manifest_text':
                 './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
                 '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
-                '0f8864f292e901019c43fdabacd62c3e+383 0:383:submit_wf.cwl\n',
+                'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
                 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
                 'name': 'submit_wf.cwl',
             }, ensure_unique_name=True),
@@ -278,6 +287,11 @@ class TestCreateTemplate(unittest.TestCase):
             'type': 'Directory',
             'value': '99999999999999999999999999999998+99',
         }
+        expect_component['script_parameters']['z'] = {
+            'dataclass': 'Collection',
+            'required': True,
+            'type': 'Directory',
+        }
         expect_template = {
             "components": {
                 "submit_wf.cwl": expect_component,

diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index f21053732d6d70510475d2b7f5479a288e083112..25d02b2b809dcb7ad0272485eadb357b9e6f2eb6 100644 (file)
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -19,6 +19,13 @@ $graph:
   - default: {basename: 99999999999999999999999999999998+99, class: Directory, location: 'keep:99999999999999999999999999999998+99'}
     id: '#main/y'
     type: Directory
+  - default:
+      basename: anonymous
+      class: Directory
+      listing:
+      - {basename: renamed.txt, class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt'}
+    id: '#main/z'
+    type: Directory
   outputs: []
   steps:
   - id: '#main/step1'

diff --git a/sdk/cwl/tests/wf/submit_wf.cwl b/sdk/cwl/tests/wf/submit_wf.cwl
index b4446559cc0bf246b3f6d0137d6cb24632a61d2a..9aab5cf4b008c20765598a50790a410cd6446820 100644 (file)
--- a/sdk/cwl/tests/wf/submit_wf.cwl
+++ b/sdk/cwl/tests/wf/submit_wf.cwl
@@ -10,6 +10,8 @@ inputs:
     type: File
   - id: y
     type: Directory
+  - id: z
+    type: Directory
 outputs: []
 steps:
   - id: step1