Merge branch '13931-cwl-size' refs #13931

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 15689a9010934cf2b8847ec08825cf30bd3e13eb..9a893df781f477dadac19264fb49cfa77b459bb7 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -139,6 +139,17 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
         else:
             return super(CollectionFsAccess, self).exists(fn)
 
+    def size(self, fn):  # type: (unicode) -> bool
+        collection, rest = self.get_collection(fn)
+        if collection is not None:
+            if rest:
+                arvfile = collection.find(rest)
+                if isinstance(arvfile, arvados.arvfile.ArvadosFile):
+                    return arvfile.size()
+            raise IOError(errno.EINVAL, "Not a path to a file %s" % (fn))
+        else:
+            return super(CollectionFsAccess, self).size(fn)
+
     def isfile(self, fn):  # type: (unicode) -> bool
         collection, rest = self.get_collection(fn)
         if collection is not None:

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 0cab074d9a8a9755f941c6a59e226d4bd9d1e5f3..3836cf5a23cd9f5320aa9a66eb5d508e7ddb6165 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,8 +33,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20180615183820',
-          'schema-salad==2.7.20180501211602',
+          'cwltool==1.0.20180806194258',
+          'schema-salad==2.7.20180719125426',
           'typing >= 3.5.3',
           'ruamel.yaml >=0.13.11, <0.15',
           'arvados-python-client>=1.1.4.20180607143841',

diff --git a/sdk/cwl/tests/13931-size-job.yml b/sdk/cwl/tests/13931-size-job.yml
new file mode 100644 (file)
index 0000000..97b46dd
--- /dev/null
+++ b/sdk/cwl/tests/13931-size-job.yml
@@ -0,0 +1,3 @@
+fastq1:
+  class: File
+  location: keep:20850f01122e860fb878758ac1320877+71/sample1_S01_R1_001.fastq.gz
\ No newline at end of file

diff --git a/sdk/cwl/tests/13931-size.cwl b/sdk/cwl/tests/13931-size.cwl
new file mode 100644 (file)
index 0000000..aed1bd6
--- /dev/null
+++ b/sdk/cwl/tests/13931-size.cwl
@@ -0,0 +1,10 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  fastq1: File
+outputs:
+  out: stdout
+baseCommand: echo
+arguments:
+  - $(inputs.fastq1.size)
+stdout: size.txt
\ No newline at end of file

diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
index 4869e3e524153af30feb6a654e65e2cac6c57f3f..8635aae65507fadb6be76d27156167855440ac68 100755 (executable)
--- a/sdk/cwl/tests/arvados-tests.sh
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -12,4 +12,8 @@ fi
 if ! arv-get 4d8a70b1e63b2aad6984e40e338e2373+69 > /dev/null ; then
     arv-put --portable-data-hash secondaryFiles/hello.txt*
 fi
+if ! arv-get 20850f01122e860fb878758ac1320877+71 > /dev/null ; then
+    arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz
+fi
+
 exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 8eac71886cbf643ca97db1e033b9ba2808b40137..21191a5b3e1553952e1031f9c1ef019db56ab1c4 100644 (file)
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -226,3 +226,14 @@
     out: null
   tool: wf-defaults/wf7.cwl
   doc: workflow level default in RunInSingleContainer
+
+- job: 13931-size-job.yml
+  output:
+    "out": {
+        "checksum": "sha1$5bf6e5357bd42a6b1d2a3a040e16a91490064d26",
+        "location": "size.txt",
+        "class": "File",
+        "size": 3
+    }
+  tool: 13931-size.cwl
+  doc: Test that size is set for files in Keep

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index ae234414a3df90888cfbe9028c06aa5efbba9f55..3f8a32816ddccdad01c78eedfdce1ed0b2be5e64 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -481,7 +481,8 @@ class TestContainer(unittest.TestCase):
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
-            "portable_data_hash": "99999999999999999999999999999993+99"}
+            "portable_data_hash": "99999999999999999999999999999994+99",
+            "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
 
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index c110bc5d53cd4634656d93fab2937954be973d07..4473b88ca0d785dbb2eaff961bf64fd21c25c280 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -389,6 +389,7 @@ class TestWorkflow(unittest.TestCase):
         metadata["cwlVersion"] = tool["cwlVersion"]
 
         mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
+        mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "token.txt")
 
         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
         arvtool.formatgraph = None
@@ -435,7 +436,8 @@ class TestWorkflow(unittest.TestCase):
   "fileblub": {
     "basename": "token.txt",
     "class": "File",
-    "location": "/keep/99999999999999999999999999999999+118/token.txt"
+    "location": "/keep/99999999999999999999999999999999+118/token.txt",
+    "size": 0
   },
   "sleeptime": 5
 }''')])

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index cd46251300dfb95862cb7957f510e108dd78b281..d980db575dd8d6e3db1ac3dbb0f7709cb14a894e 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -132,7 +132,8 @@ def stubs(func):
                     "listing": [{
                         "basename": "renamed.txt",
                         "class": "File",
-                        "location": "keep:99999999999999999999999999999998+99/file1.txt"
+                        "location": "keep:99999999999999999999999999999998+99/file1.txt",
+                        "size": 0
                     }],
                     'class': 'Directory'
                 },
@@ -164,7 +165,8 @@ def stubs(func):
                                   {
                                       'basename': 'renamed.txt',
                                       'class': 'File', 'location':
-                                      'keep:99999999999999999999999999999998+99/file1.txt'
+                                      'keep:99999999999999999999999999999998+99/file1.txt',
+                                      'size': 0
                                   }
                               ]}},
                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
@@ -225,7 +227,8 @@ def stubs(func):
                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
                             {'basename': 'renamed.txt',
                              'class': 'File',
-                             'location': 'keep:99999999999999999999999999999998+99/file1.txt'
+                             'location': 'keep:99999999999999999999999999999998+99/file1.txt',
+                             'size': 0
                             }
                         ]}
                     },
@@ -781,6 +784,7 @@ class TestSubmit(unittest.TestCase):
     @stubs
     def test_submit_file_keepref(self, stubs, tm, collectionReader):
         capture_stdout = cStringIO.StringIO()
+        collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_keepref_wf.cwl"],

diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index 7b3b4503efc239661f5b03b2afb0cfac3ca8cc4d..c84252c7b8c135b0eb6105881dab64f70424006b 100644 (file)
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -64,7 +64,8 @@
                                 "class": "File",
                                 "location": "keep:99999999999999999999999999999998+99/file1.txt",
                                 "nameext": ".txt",
-                                "nameroot": "renamed"
+                                "nameroot": "renamed",
+                                "size": 0
                             }
                         ]
                     },

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index f9f1e967b94f7e589a60888261eae4a7916a88c1..aa1f18052f8afcbe289da18d597b6e66d62d3db6 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -20,7 +20,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
 
-RUN pip install -U setuptools
+RUN pip install -U setuptools six
 
 ARG sdk
 ARG runner