Fix setting HOME on singularity. Fix a-c-r secondaryFiles.

To fix CWL conformance tests.

refs #18238

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 99c5cef95c169155d9673b5e59c90f7f6d81e5a6..a768dbfa2cf09bad4c2142ac7d2d14844fb4f214 100644 (file)
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -252,11 +252,12 @@ func (e *singularityExecutor) Start() error {
        env := make([]string, 0, len(e.spec.Env))
        for k, v := range e.spec.Env {
                if k == "HOME" {
-                       // $HOME is a special case
+                       // $HOME is a special case on Singularity 3.5,
+                       // but is just a normal variable on Singularity 3.6+
+                       // I think this will work with both
                        args = append(args, "--home="+v)
-               } else {
-                       env = append(env, "SINGULARITYENV_"+k+"="+v)
                }
+               env = append(env, "SINGULARITYENV_"+k+"="+v)
        }
 
        args = append(args, e.imageFilename)

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 126a2b511e4bf42d928624a03d4f57c54ad8bb36..ada64ae69aa62f0c1e4487bd28160c071fb3d0a1 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -184,7 +184,7 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
             elif isinstance(pattern, dict):
                 specs.append(pattern)
             elif isinstance(pattern, str):
-                specs.append({"pattern": pattern})
+                specs.append({"pattern": pattern, "required": sf.get("required")})
             else:
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Expression must return list, object, string or null")
@@ -193,7 +193,9 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         for i, sf in enumerate(specs):
             if isinstance(sf, dict):
                 if sf.get("class") == "File":
-                    pattern = sf["basename"]
+                    pattern = None
+                    sfpath = sf["location"]
+                    required = True
                 else:
                     pattern = sf["pattern"]
                     required = sf.get("required")
@@ -204,11 +206,16 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Expression must return list, object, string or null")
 
-            sfpath = substitute(primary["location"], pattern)
+            if pattern is not None:
+                sfpath = substitute(primary["location"], pattern)
+
             required = builder.do_eval(required, context=primary)
 
             if fsaccess.exists(sfpath):
-                found.append({"location": sfpath, "class": "File"})
+                if pattern is not None:
+                    found.append({"location": sfpath, "class": "File"})
+                else:
+                    found.append(sf)
             elif required:
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Required secondary file '%s' does not exist" % sfpath)