Merge branch '17531-remove-forcelegacyapi14-config'

closes #17531

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/.licenseignore b/.licenseignore
index 7ebc82667ce80565575029ff2df12fa44703297e..e3289aa7c79e5cb6dd825a3cecaadca977481987 100644 (file)
--- a/.licenseignore
+++ b/.licenseignore
@@ -86,3 +86,5 @@ sdk/python/tests/fed-migrate/CWLFile
 sdk/python/tests/fed-migrate/*.cwl
 sdk/python/tests/fed-migrate/*.cwlex
 doc/install/*.xlsx
+sdk/cwl/tests/wf/hello.txt
+sdk/cwl/tests/wf/indir1/hello2.txt
\ No newline at end of file

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 6067ae9f442b70c6d42db62df1581ab32a7cea37..4fe82a6fe1d6fc32f709dd909577da7010970e07 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,7 @@ from cwltool.pack import pack
 from cwltool.load_tool import fetch_document, resolve_and_validate_document
 from cwltool.process import shortname
 from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
-from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class, normalizeFilesDirs
 from cwltool.context import LoadingContext
 
 import ruamel.yaml as yaml
@@ -186,6 +186,7 @@ class ArvadosWorkflow(Workflow):
 
         discover_secondary_files(self.arvrunner.fs_access, builder,
                                  self.tool["inputs"], joborder)
+        normalizeFilesDirs(joborder)
 
         with Perf(metrics, "subworkflow upload_deps"):
             upload_dependencies(self.arvrunner,

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 668604f88470e18ea566d807301b712bfafa935e..48255f85756a67ed7fa6026940695c8652bdb41c 100644 (file)
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -355,3 +355,8 @@
       "size": 4
   tool: 17267-broken-schemas.cwl
   doc: "Test issue 17267 - inaccessible $schemas URL is not a fatal error"
+
+- job: null
+  output: {}
+  tool: wf/trick_defaults2.cwl
+  doc: "Test issue 17462 - secondary file objects on file defaults are not resolved"

diff --git a/sdk/cwl/tests/wf/hello.txt b/sdk/cwl/tests/wf/hello.txt
new file mode 100644 (file)
index 0000000..ce01362
--- /dev/null
+++ b/sdk/cwl/tests/wf/hello.txt
@@ -0,0 +1 @@
+hello

diff --git a/sdk/cwl/tests/wf/indir1/hello2.txt b/sdk/cwl/tests/wf/indir1/hello2.txt
new file mode 100644 (file)
index 0000000..14be0d4
--- /dev/null
+++ b/sdk/cwl/tests/wf/indir1/hello2.txt
@@ -0,0 +1 @@
+hello2

diff --git a/sdk/cwl/tests/wf/trick_defaults2.cwl b/sdk/cwl/tests/wf/trick_defaults2.cwl
new file mode 100644 (file)
index 0000000..9f42922
--- /dev/null
+++ b/sdk/cwl/tests/wf/trick_defaults2.cwl
@@ -0,0 +1,18 @@
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+  inp1:
+    type: File
+    default:
+      class: File
+      location: hello.txt
+      secondaryFiles:
+        - class: Directory
+          location: indir1
+outputs: []
+baseCommand: 'true'

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 616a94e903eba20794ac589e696b5eb7c14f50aa..9596a2dc2d26efcddebbd7921e32ff38ebdb7352 100644 (file)
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -77,8 +77,7 @@ Synonym for --stream.
 _group.add_argument('--stream', action='store_true',
                     help="""
 Store the file content and display the resulting manifest on
-stdout. Do not write the manifest to Keep or save a Collection object
-in Arvados.
+stdout. Do not save a Collection object in Arvados.
 """)
 
 _group.add_argument('--as-manifest', action='store_true', dest='manifest',
@@ -743,7 +742,7 @@ class ArvPutUploadJob(object):
 
     def _write_stdin(self, filename):
         output = self._local_collection.open(filename, 'wb')
-        self._write(sys.stdin, output)
+        self._write(sys.stdin.buffer, output)
         output.close()
 
     def _check_file(self, source, filename):

diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index a8c4a853cf9f35f94ab04a191bc0188de723f2ad..e75d39d8793ffb6032a8591a37d9d5da2168ba72 100644 (file)
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -912,7 +912,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
             [sys.executable, arv_put.__file__, '--stream'],
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT, env=self.ENVIRON)
-        pipe.stdin.write(b'stdin test\n')
+        pipe.stdin.write(b'stdin test\xa6\n')
         pipe.stdin.close()
         deadline = time.time() + 5
         while (pipe.poll() is None) and (time.time() < deadline):
@@ -924,7 +924,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
         elif returncode != 0:
             sys.stdout.write(pipe.stdout.read())
             self.fail("arv-put returned exit code {}".format(returncode))
-        self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11',
+        self.assertIn('1cb671b355a0c23d5d1c61d59cdb1b2b+12',
                       pipe.stdout.read().decode())
 
     def test_sigint_logs_request_id(self):