19099: Enable container shell when using singularity runtime.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go
index 1c963f92110f20cbd7ef1723466a04e2c3c7f4e0..ea8eedaa1be2a3ccdda93759aacdeed55e3f23b7 100644 (file)
--- a/lib/crunchrun/executor_test.go
+++ b/lib/crunchrun/executor_test.go
@@ -6,6 +6,7 @@ package crunchrun
 
 import (
        "bytes"
+       "fmt"
        "io"
        "io/ioutil"
        "net"
@@ -208,7 +209,11 @@ func (s *executorSuite) TestIPAddress(c *C) {
 }
 
 func (s *executorSuite) TestInject(c *C) {
+       hostdir := c.MkDir()
+       c.Assert(os.WriteFile(hostdir+"/testfile", []byte("first tube"), 0777), IsNil)
+       mountdir := fmt.Sprintf("/injecttest-%d", os.Getpid())
        s.spec.Command = []string{"sleep", "10"}
+       s.spec.BindMounts = map[string]bindmount{mountdir: {HostPath: hostdir, ReadOnly: true}}
        c.Assert(s.executor.Create(s.spec), IsNil)
        c.Assert(s.executor.Start(), IsNil)
        starttime := time.Now()
@@ -216,13 +221,23 @@ func (s *executorSuite) TestInject(c *C) {
        ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second))
        defer cancel()
 
-       injectcmd := []string{"cat", "/proc/1/cmdline"}
+       // Allow InjectCommand to fail a few times while the container
+       // is starting
+       for ctx.Err() == nil {
+               _, err := s.executor.InjectCommand(ctx, "", "root", false, []string{"true"})
+               if err == nil {
+                       break
+               }
+               time.Sleep(time.Second / 10)
+       }
+
+       injectcmd := []string{"cat", mountdir + "/testfile"}
        cmd, err := s.executor.InjectCommand(ctx, "", "root", false, injectcmd)
        c.Assert(err, IsNil)
        out, err := cmd.CombinedOutput()
        c.Logf("inject %s => %q", injectcmd, out)
        c.Check(err, IsNil)
-       c.Check(string(out), Equals, "sleep\00010\000")
+       c.Check(string(out), Equals, "first tube")
 
        s.executor.Stop()
        code, _ := s.executor.Wait(ctx)

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 6ba65200d2741b8ac66bb3dd109986a152e30d3f..921f58ff0a3d7fd8a4ed1992aef0fc3bfcc1bbb7 100644 (file)
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -5,12 +5,16 @@
 package crunchrun
 
 import (
+       "bytes"
        "errors"
        "fmt"
        "io/ioutil"
+       "net"
        "os"
        "os/exec"
+       "regexp"
        "sort"
+       "strconv"
        "syscall"
        "time"
 
@@ -243,11 +247,10 @@ func (e *singularityExecutor) Create(spec containerSpec) error {
 }
 
 func (e *singularityExecutor) execCmd(path string) *exec.Cmd {
-       args := []string{path, "exec", "--containall", "--cleanenv", "--pwd", e.spec.WorkingDir}
+       args := []string{path, "exec", "--containall", "--cleanenv", "--pwd", e.spec.WorkingDir, "--net"}
        if !e.spec.EnableNetwork {
-               args = append(args, "--net", "--network=none")
+               args = append(args, "--network=none")
        }
-
        if e.spec.CUDADeviceCount != 0 {
                args = append(args, "--nv")
        }
@@ -352,9 +355,116 @@ func (e *singularityExecutor) Close() {
 }
 
 func (e *singularityExecutor) InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, injectcmd []string) (*exec.Cmd, error) {
-       return nil, errors.New("unimplemented")
+       target, err := e.containedProcess()
+       if err != nil {
+               return nil, err
+       }
+       return exec.CommandContext(ctx, "nsenter", append([]string{fmt.Sprintf("--target=%d", target), "--all"}, injectcmd...)...), nil
 }
 
+var (
+       errContainerHasNoIPAddress = errors.New("container has no IP address distinct from host")
+)
+
 func (e *singularityExecutor) IPAddress() (string, error) {
-       return "", errors.New("unimplemented")
+       target, err := e.containedProcess()
+       if err != nil {
+               return "", err
+       }
+       targetIPs, err := processIPs(target)
+       if err != nil {
+               return "", err
+       }
+       selfIPs, err := processIPs(os.Getpid())
+       if err != nil {
+               return "", err
+       }
+       for ip := range targetIPs {
+               if !selfIPs[ip] {
+                       return ip, nil
+               }
+       }
+       return "", errContainerHasNoIPAddress
+}
+
+func processIPs(pid int) (map[string]bool, error) {
+       fibtrie, err := os.ReadFile(fmt.Sprintf("/proc/%d/net/fib_trie", pid))
+       if err != nil {
+               return nil, err
+       }
+
+       addrs := map[string]bool{}
+       // When we see a pair of lines like this:
+       //
+       //              |-- 10.1.2.3
+       //                 /32 host LOCAL
+       //
+       // ...we set addrs["10.1.2.3"] = true
+       lines := bytes.Split(fibtrie, []byte{'\n'})
+       for linenumber, line := range lines {
+               if !bytes.HasSuffix(line, []byte("/32 host LOCAL")) {
+                       continue
+               }
+               if linenumber < 1 {
+                       continue
+               }
+               i := bytes.LastIndexByte(lines[linenumber-1], ' ')
+               if i < 0 || i >= len(line)-7 {
+                       continue
+               }
+               addr := string(lines[linenumber-1][i+1:])
+               if net.ParseIP(addr).To4() != nil {
+                       addrs[addr] = true
+               }
+       }
+       return addrs, nil
+}
+
+var (
+       errContainerNotStarted = errors.New("container has not started yet")
+       errCannotFindChild     = errors.New("failed to find any process inside the container")
+       reProcStatusPPid       = regexp.MustCompile(`\nPPid:\t(\d+)\n`)
+)
+
+// Return the PID of a process that is inside the container (not
+// necessarily the topmost/pid=1 process in the container).
+func (e *singularityExecutor) containedProcess() (int, error) {
+       if e.child == nil || e.child.Process == nil {
+               return 0, errContainerNotStarted
+       }
+       lsns, err := exec.Command("lsns").CombinedOutput()
+       if err != nil {
+               return 0, fmt.Errorf("lsns: %w", err)
+       }
+       for _, line := range bytes.Split(lsns, []byte{'\n'}) {
+               fields := bytes.Fields(line)
+               if len(fields) < 4 {
+                       continue
+               }
+               if !bytes.Equal(fields[1], []byte("pid")) {
+                       continue
+               }
+               pid, err := strconv.ParseInt(string(fields[3]), 10, 64)
+               if err != nil {
+                       return 0, fmt.Errorf("error parsing PID field in lsns output: %q", fields[3])
+               }
+               for parent := pid; ; {
+                       procstatus, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", parent))
+                       if err != nil {
+                               break
+                       }
+                       m := reProcStatusPPid.FindSubmatch(procstatus)
+                       if m == nil {
+                               break
+                       }
+                       parent, err = strconv.ParseInt(string(m[1]), 10, 64)
+                       if err != nil {
+                               break
+                       }
+                       if int(parent) == e.child.Process.Pid {
+                               return int(pid), nil
+                       }
+               }
+       }
+       return 0, errCannotFindChild
 }

diff --git a/lib/crunchrun/singularity_test.go b/lib/crunchrun/singularity_test.go
index cdeafee88242b3330adcf2c5ae7550fdcb104f46..8a2e62d7e77c232618a9525873983ac3f321d594 100644 (file)
--- a/lib/crunchrun/singularity_test.go
+++ b/lib/crunchrun/singularity_test.go
@@ -28,6 +28,14 @@ func (s *singularitySuite) SetUpSuite(c *C) {
        }
 }
 
+func (s *singularitySuite) TestInject(c *C) {
+       path, err := exec.LookPath("nsenter")
+       if err != nil || path != "/var/lib/arvados/bin/nsenter" {
+               c.Skip("looks like /var/lib/arvados/bin/nsenter is not installed -- re-run `arvados-server install`?")
+       }
+       s.executorSuite.TestInject(c)
+}
+
 var _ = Suite(&singularityStubSuite{})
 
 // singularityStubSuite tests don't really invoke singularity, so we

diff --git a/lib/install/deps.go b/lib/install/deps.go
index cdf28e09c69c23bd9259710380fdbaa8476101bf..2d9da72b9785d5419469d1a27c277e300795b738 100644 (file)
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -338,6 +338,16 @@ make -C ./builddir install
                        }
                }
 
+               err = inst.runBash(`
+install /usr/bin/nsenter /var/lib/arvados/bin/nsenter
+setcap "cap_sys_admin+pei cap_sys_chroot+pei" /var/lib/arvados/bin/nsenter
+singularity config global --set 'allow net networks' bridge
+singularity config global --set 'allow net groups' sudo
+`, stdout, stderr)
+               if err != nil {
+                       return 1
+               }
+
                // The entry in /etc/locale.gen is "en_US.UTF-8"; once
                // it's installed, locale -a reports it as
                // "en_US.utf8".