From 43fdb06a1620d926bdaec00582c82a4190805d86 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Fri, 13 May 2022 02:43:14 -0400 Subject: [PATCH] 19099: Enable container shell when using singularity runtime. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- lib/crunchrun/executor_test.go | 19 ++++- lib/crunchrun/singularity.go | 120 ++++++++++++++++++++++++++++-- lib/crunchrun/singularity_test.go | 8 ++ lib/install/deps.go | 10 +++ 4 files changed, 150 insertions(+), 7 deletions(-) diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go index 1c963f9211..ea8eedaa1b 100644 --- a/lib/crunchrun/executor_test.go +++ b/lib/crunchrun/executor_test.go @@ -6,6 +6,7 @@ package crunchrun import ( "bytes" + "fmt" "io" "io/ioutil" "net" @@ -208,7 +209,11 @@ func (s *executorSuite) TestIPAddress(c *C) { } func (s *executorSuite) TestInject(c *C) { + hostdir := c.MkDir() + c.Assert(os.WriteFile(hostdir+"/testfile", []byte("first tube"), 0777), IsNil) + mountdir := fmt.Sprintf("/injecttest-%d", os.Getpid()) s.spec.Command = []string{"sleep", "10"} + s.spec.BindMounts = map[string]bindmount{mountdir: {HostPath: hostdir, ReadOnly: true}} c.Assert(s.executor.Create(s.spec), IsNil) c.Assert(s.executor.Start(), IsNil) starttime := time.Now() @@ -216,13 +221,23 @@ func (s *executorSuite) TestInject(c *C) { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) defer cancel() - injectcmd := []string{"cat", "/proc/1/cmdline"} + // Allow InjectCommand to fail a few times while the container + // is starting + for ctx.Err() == nil { + _, err := s.executor.InjectCommand(ctx, "", "root", false, []string{"true"}) + if err == nil { + break + } + time.Sleep(time.Second / 10) + } + + injectcmd := []string{"cat", mountdir + "/testfile"} cmd, err := s.executor.InjectCommand(ctx, "", "root", false, injectcmd) c.Assert(err, IsNil) out, err := cmd.CombinedOutput() c.Logf("inject %s => %q", injectcmd, out) c.Check(err, IsNil) - c.Check(string(out), Equals, "sleep\00010\000") + c.Check(string(out), Equals, "first tube") s.executor.Stop() code, _ := s.executor.Wait(ctx) diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go index 6ba65200d2..921f58ff0a 100644 --- a/lib/crunchrun/singularity.go +++ b/lib/crunchrun/singularity.go @@ -5,12 +5,16 @@ package crunchrun import ( + "bytes" "errors" "fmt" "io/ioutil" + "net" "os" "os/exec" + "regexp" "sort" + "strconv" "syscall" "time" @@ -243,11 +247,10 @@ func (e *singularityExecutor) Create(spec containerSpec) error { } func (e *singularityExecutor) execCmd(path string) *exec.Cmd { - args := []string{path, "exec", "--containall", "--cleanenv", "--pwd", e.spec.WorkingDir} + args := []string{path, "exec", "--containall", "--cleanenv", "--pwd", e.spec.WorkingDir, "--net"} if !e.spec.EnableNetwork { - args = append(args, "--net", "--network=none") + args = append(args, "--network=none") } - if e.spec.CUDADeviceCount != 0 { args = append(args, "--nv") } @@ -352,9 +355,116 @@ func (e *singularityExecutor) Close() { } func (e *singularityExecutor) InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, injectcmd []string) (*exec.Cmd, error) { - return nil, errors.New("unimplemented") + target, err := e.containedProcess() + if err != nil { + return nil, err + } + return exec.CommandContext(ctx, "nsenter", append([]string{fmt.Sprintf("--target=%d", target), "--all"}, injectcmd...)...), nil } +var ( + errContainerHasNoIPAddress = errors.New("container has no IP address distinct from host") +) + func (e *singularityExecutor) IPAddress() (string, error) { - return "", errors.New("unimplemented") + target, err := e.containedProcess() + if err != nil { + return "", err + } + targetIPs, err := processIPs(target) + if err != nil { + return "", err + } + selfIPs, err := processIPs(os.Getpid()) + if err != nil { + return "", err + } + for ip := range targetIPs { + if !selfIPs[ip] { + return ip, nil + } + } + return "", errContainerHasNoIPAddress +} + +func processIPs(pid int) (map[string]bool, error) { + fibtrie, err := os.ReadFile(fmt.Sprintf("/proc/%d/net/fib_trie", pid)) + if err != nil { + return nil, err + } + + addrs := map[string]bool{} + // When we see a pair of lines like this: + // + // |-- 10.1.2.3 + // /32 host LOCAL + // + // ...we set addrs["10.1.2.3"] = true + lines := bytes.Split(fibtrie, []byte{'\n'}) + for linenumber, line := range lines { + if !bytes.HasSuffix(line, []byte("/32 host LOCAL")) { + continue + } + if linenumber < 1 { + continue + } + i := bytes.LastIndexByte(lines[linenumber-1], ' ') + if i < 0 || i >= len(line)-7 { + continue + } + addr := string(lines[linenumber-1][i+1:]) + if net.ParseIP(addr).To4() != nil { + addrs[addr] = true + } + } + return addrs, nil +} + +var ( + errContainerNotStarted = errors.New("container has not started yet") + errCannotFindChild = errors.New("failed to find any process inside the container") + reProcStatusPPid = regexp.MustCompile(`\nPPid:\t(\d+)\n`) +) + +// Return the PID of a process that is inside the container (not +// necessarily the topmost/pid=1 process in the container). +func (e *singularityExecutor) containedProcess() (int, error) { + if e.child == nil || e.child.Process == nil { + return 0, errContainerNotStarted + } + lsns, err := exec.Command("lsns").CombinedOutput() + if err != nil { + return 0, fmt.Errorf("lsns: %w", err) + } + for _, line := range bytes.Split(lsns, []byte{'\n'}) { + fields := bytes.Fields(line) + if len(fields) < 4 { + continue + } + if !bytes.Equal(fields[1], []byte("pid")) { + continue + } + pid, err := strconv.ParseInt(string(fields[3]), 10, 64) + if err != nil { + return 0, fmt.Errorf("error parsing PID field in lsns output: %q", fields[3]) + } + for parent := pid; ; { + procstatus, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", parent)) + if err != nil { + break + } + m := reProcStatusPPid.FindSubmatch(procstatus) + if m == nil { + break + } + parent, err = strconv.ParseInt(string(m[1]), 10, 64) + if err != nil { + break + } + if int(parent) == e.child.Process.Pid { + return int(pid), nil + } + } + } + return 0, errCannotFindChild } diff --git a/lib/crunchrun/singularity_test.go b/lib/crunchrun/singularity_test.go index cdeafee882..8a2e62d7e7 100644 --- a/lib/crunchrun/singularity_test.go +++ b/lib/crunchrun/singularity_test.go @@ -28,6 +28,14 @@ func (s *singularitySuite) SetUpSuite(c *C) { } } +func (s *singularitySuite) TestInject(c *C) { + path, err := exec.LookPath("nsenter") + if err != nil || path != "/var/lib/arvados/bin/nsenter" { + c.Skip("looks like /var/lib/arvados/bin/nsenter is not installed -- re-run `arvados-server install`?") + } + s.executorSuite.TestInject(c) +} + var _ = Suite(&singularityStubSuite{}) // singularityStubSuite tests don't really invoke singularity, so we diff --git a/lib/install/deps.go b/lib/install/deps.go index cdf28e09c6..2d9da72b97 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -338,6 +338,16 @@ make -C ./builddir install } } + err = inst.runBash(` +install /usr/bin/nsenter /var/lib/arvados/bin/nsenter +setcap "cap_sys_admin+pei cap_sys_chroot+pei" /var/lib/arvados/bin/nsenter +singularity config global --set 'allow net networks' bridge +singularity config global --set 'allow net groups' sudo +`, stdout, stderr) + if err != nil { + return 1 + } + // The entry in /etc/locale.gen is "en_US.UTF-8"; once // it's installed, locale -a reports it as // "en_US.utf8". -- 2.30.2