9751: Fix direct to keep directory references.

Add arvados-specific integration test for arvados-cwl-runner.

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 2a1873a84e6925f9c1edf6057dc2b01a392d25ff..fe4e8009aca1af0b04fb483983a09fd59de25390 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -18,8 +18,9 @@ import logging
 import os
 import json
 import argparse
+import re
 from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFileObjs
+from cwltool.process import adjustFileObjs, adjustDirObjs
 from cwltool.load_tool import load_tool
 
 # Print package versions
@@ -30,8 +31,10 @@ api = arvados.api("v1")
 try:
     job_order_object = arvados.current_job()['script_parameters']
 
+    pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
+
     def keeppath(v):
-        if arvados.util.keep_locator_pattern.match(v):
+        if pdh_path.match(v):
             return "keep:%s" % v
         else:
             return v
@@ -49,6 +52,7 @@ try:
             }
 
     adjustFileObjs(job_order_object, keeppathObj)
+    adjustDirObjs(job_order_object, keeppathObj)
 
     runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
 

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7bd9df344cb927713fcafb6d1ff98ad68adb1cc2..fd9e74f8575167f7aa003ef32d9c449596810408 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -124,7 +124,9 @@ class ArvCwlRunner(object):
         self.debug = kwargs.get("debug")
         self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
 
-        self.fs_access = kwargs["make_fs_access"](kwargs["basedir"])
+        make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess, api_client=self.api)
+        self.fs_access = make_fs_access(kwargs["basedir"])
+        kwargs["make_fs_access"] = make_fs_access
         kwargs["enable_reuse"] = kwargs.get("enable_reuse")
         kwargs["use_container"] = True
         kwargs["tmpdir_prefix"] = "tmp"

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 24c319cce5620dd9110f76932d45cb20967040bc..7e9a159747d1a9cce0ae68a4d56fe86e5e8c01aa 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -52,7 +52,6 @@ class ArvPathMapper(PathMapper):
         elif srcobj["class"] == "Directory":
             if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
                 self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
-            else:
                 for l in srcobj["listing"]:
                     self.visit(l, uploadfiles)
 

diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
new file mode 100755 (executable)
index 0000000..8646704
--- /dev/null
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+if ! arv-get d7514270f356df848477718d58308cc4+94 > /dev/null ; then
+    arv-put --portable-data-hash testdir
+fi
+exec cwltest --test arvados-tests.yml --tool $PWD/runner.sh

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
new file mode 100644 (file)
index 0000000..2526258
--- /dev/null
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -0,0 +1,10 @@
+- job: dir-job.yml
+  output:
+    "outlist": {
+        "size": 20,
+        "location": "output.txt",
+        "class": "File",
+        "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592"
+    }
+  tool: dir6.cwl
+  doc: Test directory in keep

diff --git a/sdk/cwl/tests/dir-job.yml b/sdk/cwl/tests/dir-job.yml
new file mode 100644 (file)
index 0000000..91204d7
--- /dev/null
+++ b/sdk/cwl/tests/dir-job.yml
@@ -0,0 +1,3 @@
+indir:
+  class: Directory
+  location: keep:d7514270f356df848477718d58308cc4+94
\ No newline at end of file

diff --git a/sdk/cwl/tests/dir6.cwl b/sdk/cwl/tests/dir6.cwl
new file mode 100644 (file)
index 0000000..93362b5
--- /dev/null
+++ b/sdk/cwl/tests/dir6.cwl
@@ -0,0 +1,21 @@
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+  - class: ShellCommandRequirement
+inputs:
+  indir:
+    type: Directory
+    inputBinding:
+      prefix: cd
+      position: -1
+outputs:
+  outlist:
+    type: File
+    outputBinding:
+      glob: output.txt
+arguments: [
+  {shellQuote: false, valueFrom: "&&"},
+  "find", ".",
+  {shellQuote: false, valueFrom: "|"},
+  "sort"]
+stdout: output.txt
\ No newline at end of file

diff --git a/sdk/cwl/tests/runner.sh b/sdk/cwl/tests/runner.sh
new file mode 100755 (executable)
index 0000000..22ede5c
--- /dev/null
+++ b/sdk/cwl/tests/runner.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec arvados-cwl-runner --disable-reuse --compute-checksum "$@"

diff --git a/sdk/cwl/tests/testdir/a b/sdk/cwl/tests/testdir/a
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sdk/cwl/tests/testdir/b b/sdk/cwl/tests/testdir/b
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sdk/cwl/tests/testdir/c/d b/sdk/cwl/tests/testdir/c/d
new file mode 100644 (file)
index 0000000..e69de29