From: Peter Amstutz Date: Fri, 13 May 2022 15:08:31 +0000 (-0400) Subject: Merge branch '19109-upload-secondary' refs #19109 X-Git-Tag: 2.5.0~178 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a89fbc8b4f2d8db8654175428bd1f041eed6f109?hp=93380e6aec7e11607019cdce88419b6f708327d7 Merge branch '19109-upload-secondary' refs #19109 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/.licenseignore b/.licenseignore index 387aeda944..203c378bdc 100644 --- a/.licenseignore +++ b/.licenseignore @@ -50,6 +50,7 @@ docker/jobs/1078ECD7.key */script/rails sdk/cwl/tests/input/blorp.txt sdk/cwl/tests/tool/blub.txt +sdk/cwl/tests/19109-upload-secondary/* sdk/cwl/tests/federation/data/* sdk/cwl/tests/fake-keep-mount/fake_collection_dir/.arvados#collection sdk/go/manifest/testdata/*_manifest diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index 50b3bb94d8..f232178c5d 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -129,6 +129,9 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov set_secondary(fsaccess, builder, i, secondaryspec, primary, discovered) return + if inputschema == "File": + inputschema = {"type": "File"} + if isinstance(inputschema, basestring): sd = search_schemadef(inputschema, reversed(builder.hints+builder.requirements)) if sd: @@ -164,10 +167,13 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov set_secondary(fsaccess, builder, {"type": inputschema["items"]}, secondaryspec, p, discovered) elif (inputschema["type"] == "File" and - secondaryspec and isinstance(primary, Mapping) and - primary.get("class") == "File" and - "secondaryFiles" not in primary): + primary.get("class") == "File"): + + if "secondaryFiles" in primary or not secondaryspec: + # Nothing to do. + return + # # Found a file, check for secondaryFiles # @@ -175,9 +181,9 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov primary["secondaryFiles"] = secondaryspec for i, sf in enumerate(aslist(secondaryspec)): if builder.cwlVersion == "v1.0": - pattern = builder.do_eval(sf, context=primary) + pattern = sf else: - pattern = builder.do_eval(sf["pattern"], context=primary) + pattern = sf["pattern"] if pattern is None: continue if isinstance(pattern, list): @@ -214,7 +220,20 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov "Expression must return list, object, string or null") if pattern is not None: - sfpath = substitute(primary["location"], pattern) + if "${" in pattern or "$(" in pattern: + sfname = builder.do_eval(pattern, context=primary) + else: + sfname = substitute(primary["basename"], pattern) + + if sfname is None: + continue + + p_location = primary["location"] + if "/" in p_location: + sfpath = ( + p_location[0 : p_location.rindex("/") + 1] + + sfname + ) required = builder.do_eval(required, context=primary) diff --git a/sdk/cwl/tests/19109-upload-secondary.cwl b/sdk/cwl/tests/19109-upload-secondary.cwl new file mode 100644 index 0000000000..933496b6a5 --- /dev/null +++ b/sdk/cwl/tests/19109-upload-secondary.cwl @@ -0,0 +1,42 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.2 +class: Workflow + +requirements: + InlineJavascriptRequirement: {} + +inputs: + file1: + type: File? + secondaryFiles: + - pattern: .tbi + required: true + file2: + type: File + secondaryFiles: + - pattern: | + ${ + return self.basename + '.tbi'; + } + required: true +outputs: + out: + type: File + outputSource: cat/out + out2: + type: File + outputSource: cat2/out +steps: + cat: + in: + inp: file1 + run: cat2.cwl + out: [out] + cat2: + in: + inp: file2 + run: cat2.cwl + out: [out] diff --git a/sdk/cwl/tests/19109-upload-secondary.yml b/sdk/cwl/tests/19109-upload-secondary.yml new file mode 100644 index 0000000000..af4147ae9a --- /dev/null +++ b/sdk/cwl/tests/19109-upload-secondary.yml @@ -0,0 +1,10 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +file1: + class: File + location: 19109-upload-secondary/file1.txt +file2: + class: File + location: 19109-upload-secondary/file2.txt diff --git a/sdk/cwl/tests/19109-upload-secondary/file1.txt b/sdk/cwl/tests/19109-upload-secondary/file1.txt new file mode 100644 index 0000000000..41549dbf8c --- /dev/null +++ b/sdk/cwl/tests/19109-upload-secondary/file1.txt @@ -0,0 +1 @@ +strawberry diff --git a/sdk/cwl/tests/19109-upload-secondary/file1.txt.tbi b/sdk/cwl/tests/19109-upload-secondary/file1.txt.tbi new file mode 100644 index 0000000000..c9438bd232 --- /dev/null +++ b/sdk/cwl/tests/19109-upload-secondary/file1.txt.tbi @@ -0,0 +1 @@ +blueberry \ No newline at end of file diff --git a/sdk/cwl/tests/19109-upload-secondary/file2.txt b/sdk/cwl/tests/19109-upload-secondary/file2.txt new file mode 100644 index 0000000000..637a09b86a --- /dev/null +++ b/sdk/cwl/tests/19109-upload-secondary/file2.txt @@ -0,0 +1 @@ +banana diff --git a/sdk/cwl/tests/19109-upload-secondary/file2.txt.tbi b/sdk/cwl/tests/19109-upload-secondary/file2.txt.tbi new file mode 100644 index 0000000000..4d5e62525a --- /dev/null +++ b/sdk/cwl/tests/19109-upload-secondary/file2.txt.tbi @@ -0,0 +1 @@ +mango \ No newline at end of file diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml index 9e691bdba5..2f309cfe81 100644 --- a/sdk/cwl/tests/arvados-tests.yml +++ b/sdk/cwl/tests/arvados-tests.yml @@ -449,3 +449,23 @@ output: {} tool: 18994-basename/wf_ren.cwl doc: "Test issue 18994 - correctly stage file with modified basename" + +- job: 19109-upload-secondary.yml + output: { + "out": { + "basename": "file1.catted", + "class": "File", + "location": "file1.catted", + "size": 20, + "checksum": "sha1$c4cead17cebdd829f38c48e18a28f1da72339ef7" + }, + "out2": { + "basename": "file2.catted", + "checksum": "sha1$6f71c5d1512519ede45bedfdd624e05fd8037b0d", + "class": "File", + "location": "file2.catted", + "size": 12 + } + } + tool: 19109-upload-secondary.cwl + doc: "Test issue 19109 - correctly discover & upload secondary files" diff --git a/sdk/cwl/tests/cat2.cwl b/sdk/cwl/tests/cat2.cwl new file mode 100644 index 0000000000..82d93efbf0 --- /dev/null +++ b/sdk/cwl/tests/cat2.cwl @@ -0,0 +1,17 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.2 +class: CommandLineTool +inputs: + - id: inp + type: File + secondaryFiles: + - pattern: .tbi + required: true +stdout: $(inputs.inp.nameroot).catted +outputs: + out: + type: stdout +arguments: [cat, '$(inputs.inp.path)', '$(inputs.inp.secondaryFiles[0].path)']