From: Tom Clegg Date: Tue, 17 May 2022 18:13:34 +0000 (-0400) Subject: Merge branch '19081-singularity-no-eval' X-Git-Tag: 2.5.0~170 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/844ff7cc1dc1c93a29b7ad8eca2987b987cf89e6?hp=f7954ab0a45cbc302aa07fa60697363895395dde Merge branch '19081-singularity-no-eval' refs #19081 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go index 474fbf4ade..0253ac3fa8 100644 --- a/lib/crunchrun/crunchrun.go +++ b/lib/crunchrun/crunchrun.go @@ -422,7 +422,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) { "--storage-classes", strings.Join(runner.Container.OutputStorageClasses, ","), fmt.Sprintf("--crunchstat-interval=%v", runner.statInterval.Seconds())} - if runner.executor.Runtime() == "docker" { + if _, isdocker := runner.executor.(*dockerExecutor); isdocker { arvMountCmd = append(arvMountCmd, "--allow-other") } @@ -1479,7 +1479,10 @@ func (runner *ContainerRunner) NewArvLogWriter(name string) (io.WriteCloser, err func (runner *ContainerRunner) Run() (err error) { runner.CrunchLog.Printf("crunch-run %s started", cmd.Version.String()) runner.CrunchLog.Printf("%s", currentUserAndGroups()) - runner.CrunchLog.Printf("Executing container '%s' using %s runtime", runner.Container.UUID, runner.executor.Runtime()) + v, _ := exec.Command("arv-mount", "--version").CombinedOutput() + runner.CrunchLog.Printf("Using FUSE mount: %s", v) + runner.CrunchLog.Printf("Using container runtime: %s", runner.executor.Runtime()) + runner.CrunchLog.Printf("Executing container: %s", runner.Container.UUID) hostname, hosterr := os.Hostname() if hosterr != nil { diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go index 1d2c7b09fd..347703a95b 100644 --- a/lib/crunchrun/crunchrun_test.go +++ b/lib/crunchrun/crunchrun_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" @@ -128,6 +129,7 @@ func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvad return e.loadErr } func (e *stubExecutor) Runtime() string { return "stub" } +func (e *stubExecutor) Version() string { return "stub " + cmd.Version.String() } func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr } func (e *stubExecutor) Start() error { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr } func (e *stubExecutor) CgroupID() string { return "cgroupid" } @@ -885,7 +887,8 @@ func (s *TestSuite) TestLogVersionAndRuntime(c *C) { c.Assert(s.api.Logs["crunch-run"], NotNil) c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`) c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`) - c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`) + c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container: zzzzz-zzzzz-zzzzzzzzzzzzzzz.*`) + c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Using container runtime: stub.*`) } func (s *TestSuite) TestContainerRecordLog(c *C) { diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go index e62f2a39ba..f3808cb357 100644 --- a/lib/crunchrun/docker.go +++ b/lib/crunchrun/docker.go @@ -46,7 +46,20 @@ func newDockerExecutor(containerUUID string, logf func(string, ...interface{}), }, err } -func (e *dockerExecutor) Runtime() string { return "docker" } +func (e *dockerExecutor) Runtime() string { + v, _ := e.dockerclient.ServerVersion(context.Background()) + info := "" + for _, cv := range v.Components { + if info != "" { + info += ", " + } + info += cv.Name + " " + cv.Version + } + if info == "" { + info = "(unknown version)" + } + return "docker " + info +} func (e *dockerExecutor) LoadImage(imageID string, imageTarballPath string, container arvados.Container, arvMountPoint string, containerClient *arvados.Client) error { diff --git a/lib/crunchrun/executor.go b/lib/crunchrun/executor.go index dc1bc20b7c..0a65f4d634 100644 --- a/lib/crunchrun/executor.go +++ b/lib/crunchrun/executor.go @@ -60,6 +60,6 @@ type containerExecutor interface { // Release resources (temp dirs, stopped containers) Close() - // Name of runtime engine ("docker", "singularity") + // Name and version of runtime engine ("docker 20.10.16", "singularity-ce version 3.9.9") Runtime() string } diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go index 99af0530ff..3301a6e63b 100644 --- a/lib/crunchrun/executor_test.go +++ b/lib/crunchrun/executor_test.go @@ -80,6 +80,7 @@ func (s *executorSuite) TearDownTest(c *C) { } func (s *executorSuite) TestExecTrivialContainer(c *C) { + c.Logf("Using container runtime: %s", s.executor.Runtime()) s.spec.Command = []string{"echo", "ok"} s.checkRun(c, 0) c.Check(s.stdout.String(), Equals, "ok\n") diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go index 0b139dd97d..ec08937283 100644 --- a/lib/crunchrun/integration_test.go +++ b/lib/crunchrun/integration_test.go @@ -162,11 +162,13 @@ func (s *integrationSuite) setup(c *C) { func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) { s.engine = "docker" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) { s.engine = "singularity" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) { diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go index 61992b312d..1af0d420e4 100644 --- a/lib/crunchrun/singularity.go +++ b/lib/crunchrun/singularity.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "sort" + "strings" "syscall" "time" @@ -36,7 +37,13 @@ func newSingularityExecutor(logf func(string, ...interface{})) (*singularityExec }, nil } -func (e *singularityExecutor) Runtime() string { return "singularity" } +func (e *singularityExecutor) Runtime() string { + buf, err := exec.Command("singularity", "--version").CombinedOutput() + if err != nil { + return "singularity (unknown version)" + } + return strings.TrimSuffix(string(buf), "\n") +} func (e *singularityExecutor) getOrCreateProject(ownerUuid string, name string, containerClient *arvados.Client) (*arvados.Group, error) { var gp arvados.GroupList