12630: request both 'compute' and 'utility' nvidia capabilities

[arvados.git] / lib / crunchrun / singularity.go

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index cda10aa611b7cf8d3d1d03822495f91a9ee0f8d6..942de4300e087a95306fc2de1d20288535581a67 100644 (file)
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -10,6 +10,7 @@ import (
        "os"
        "os/exec"
        "sort"
+       "strings"
        "syscall"
        "time"
 
@@ -284,6 +285,15 @@ func (e *singularityExecutor) execCmd(path string) *exec.Cmd {
                env = append(env, "SINGULARITYENV_"+k+"="+v)
        }
 
+       // Singularity always makes all nvidia devices visible to the
+       // container.  If a resource manager such as slurm or LSF told
+       // us to select specific devices we need to propagate that.
+       for _, s := range os.Environ() {
+               if strings.HasPrefix(s, "CUDA_VISIBLE_DEVICES=") {
+                       env = append(env, "SINGULARITYENV_"+s)
+               }
+       }
+
        args = append(args, e.imageFilename)
        args = append(args, e.spec.Command...)