lib/crunchrun/executor_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net"
  14         "net/http"
  15         "net/netip"
  16         "os"
  17         "regexp"
  18         "strings"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/diagnostics"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         . "gopkg.in/check.v1"
  25 )
  26
  27 type nopWriteCloser struct{ io.Writer }
  28
  29 func (nopWriteCloser) Close() error { return nil }
  30
  31 // embedded by dockerSuite and singularitySuite so they can share
  32 // tests.
  33 type executorSuite struct {
  34         newExecutor func(*C) // embedding struct's SetUpSuite method must set this
  35         executor    containerExecutor
  36         spec        containerSpec
  37         stdout      bytes.Buffer
  38         stderr      bytes.Buffer
  39 }
  40
  41 func (s *executorSuite) SetUpTest(c *C) {
  42         s.newExecutor(c)
  43         s.stdout = bytes.Buffer{}
  44         s.stderr = bytes.Buffer{}
  45         s.spec = containerSpec{
  46                 Image:       "busybox:uclibc",
  47                 VCPUs:       1,
  48                 WorkingDir:  "",
  49                 Env:         map[string]string{"PATH": "/bin:/usr/bin"},
  50                 NetworkMode: "default",
  51                 Stdout:      nopWriteCloser{&s.stdout},
  52                 Stderr:      nopWriteCloser{&s.stderr},
  53         }
  54         err := s.executor.LoadImage("", arvadostest.BusyboxDockerImage(c), arvados.Container{}, "", nil)
  55         c.Assert(err, IsNil)
  56 }
  57
  58 func (s *executorSuite) TearDownTest(c *C) {
  59         s.executor.Close()
  60 }
  61
  62 func (s *executorSuite) TestExecTrivialContainer(c *C) {
  63         c.Logf("Using container runtime: %s", s.executor.Runtime())
  64         s.spec.Command = []string{"echo", "ok"}
  65         s.checkRun(c, 0)
  66         c.Check(s.stdout.String(), Equals, "ok\n")
  67         c.Check(s.stderr.String(), Equals, "")
  68 }
  69
  70 func (s *executorSuite) TestDiagnosticsImage(c *C) {
  71         s.newExecutor(c)
  72         imagefile := c.MkDir() + "/hello-world.tar"
  73         err := ioutil.WriteFile(imagefile, diagnostics.HelloWorldDockerImage, 0777)
  74         c.Assert(err, IsNil)
  75         err = s.executor.LoadImage("", imagefile, arvados.Container{}, "", nil)
  76         c.Assert(err, IsNil)
  77
  78         c.Logf("Using container runtime: %s", s.executor.Runtime())
  79         s.spec.Image = "hello-world"
  80         s.spec.Command = []string{"/hello"}
  81         s.checkRun(c, 0)
  82         c.Check(s.stdout.String(), Matches, `(?ms)\nHello from Docker!\n.*`)
  83 }
  84
  85 func (s *executorSuite) TestExitStatus(c *C) {
  86         s.spec.Command = []string{"false"}
  87         s.checkRun(c, 1)
  88 }
  89
  90 func (s *executorSuite) TestSignalExitStatus(c *C) {
  91         if _, isdocker := s.executor.(*dockerExecutor); isdocker {
  92                 // It's not quite this easy to make busybox kill
  93                 // itself in docker where it's pid 1.
  94                 c.Skip("kill -9 $$ doesn't work on busybox with pid=1 in docker")
  95                 return
  96         }
  97         s.spec.Command = []string{"sh", "-c", "kill -9 $$"}
  98         s.checkRun(c, 0x80+9)
  99 }
 100
 101 func (s *executorSuite) TestExecStop(c *C) {
 102         s.spec.Command = []string{"sh", "-c", "sleep 10; echo ok"}
 103         err := s.executor.Create(s.spec)
 104         c.Assert(err, IsNil)
 105         err = s.executor.Start()
 106         c.Assert(err, IsNil)
 107         go func() {
 108                 time.Sleep(time.Second / 10)
 109                 s.executor.Stop()
 110         }()
 111         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second))
 112         defer cancel()
 113         code, err := s.executor.Wait(ctx)
 114         c.Check(code, Not(Equals), 0)
 115         c.Check(err, IsNil)
 116         c.Check(s.stdout.String(), Equals, "")
 117         c.Check(s.stderr.String(), Equals, "")
 118 }
 119
 120 func (s *executorSuite) TestExecCleanEnv(c *C) {
 121         s.spec.Command = []string{"env"}
 122         s.checkRun(c, 0)
 123         c.Check(s.stderr.String(), Equals, "")
 124         got := map[string]string{}
 125         for _, kv := range strings.Split(s.stdout.String(), "\n") {
 126                 if kv == "" {
 127                         continue
 128                 }
 129                 kv := strings.SplitN(kv, "=", 2)
 130                 switch kv[0] {
 131                 case "HOSTNAME", "HOME":
 132                         // docker sets these by itself
 133                 case "LD_LIBRARY_PATH", "SINGULARITY_NAME", "PWD", "LANG", "SHLVL", "SINGULARITY_INIT", "SINGULARITY_CONTAINER":
 134                         // singularity sets these by itself (cf. https://sylabs.io/guides/3.5/user-guide/environment_and_metadata.html)
 135                 case "SINGULARITY_APPNAME":
 136                         // singularity also sets this by itself (v3.5.2, but not v3.7.4)
 137                 case "PROMPT_COMMAND", "PS1", "SINGULARITY_BIND", "SINGULARITY_COMMAND", "SINGULARITY_ENVIRONMENT":
 138                         // singularity also sets these by itself (v3.7.4)
 139                 case "SINGULARITY_NO_EVAL":
 140                         // our singularity driver sets this to control
 141                         // singularity behavior, and it gets passed
 142                         // through to the container
 143                 default:
 144                         got[kv[0]] = kv[1]
 145                 }
 146         }
 147         c.Check(got, DeepEquals, s.spec.Env)
 148 }
 149 func (s *executorSuite) TestExecEnableNetwork(c *C) {
 150         for _, enable := range []bool{false, true} {
 151                 s.SetUpTest(c)
 152                 s.spec.Command = []string{"ip", "route"}
 153                 s.spec.EnableNetwork = enable
 154                 s.checkRun(c, 0)
 155                 if enable {
 156                         c.Check(s.stdout.String(), Matches, "(?ms).*default via.*")
 157                 } else {
 158                         c.Check(s.stdout.String(), Equals, "")
 159                 }
 160         }
 161 }
 162
 163 func (s *executorSuite) TestExecWorkingDir(c *C) {
 164         s.spec.WorkingDir = "/tmp"
 165         s.spec.Command = []string{"sh", "-c", "pwd"}
 166         s.checkRun(c, 0)
 167         c.Check(s.stdout.String(), Equals, "/tmp\n")
 168 }
 169
 170 func (s *executorSuite) TestExecStdoutStderr(c *C) {
 171         s.spec.Command = []string{"sh", "-c", "echo foo; echo -n bar >&2; echo baz; echo waz >&2"}
 172         s.checkRun(c, 0)
 173         c.Check(s.stdout.String(), Equals, "foo\nbaz\n")
 174         c.Check(s.stderr.String(), Equals, "barwaz\n")
 175 }
 176
 177 func (s *executorSuite) TestEnableNetwork_Listen(c *C) {
 178         // Listen on an available port on the host.
 179         ln, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", "0"))
 180         c.Assert(err, IsNil)
 181         defer ln.Close()
 182         _, port, err := net.SplitHostPort(ln.Addr().String())
 183         c.Assert(err, IsNil)
 184
 185         // Start a container that listens on the same port number that
 186         // is already in use on the host.
 187         s.spec.Command = []string{"nc", "-l", "-p", port, "-e", "printf", `HTTP/1.1 418 I'm a teapot\r\n\r\n`}
 188         s.spec.EnableNetwork = true
 189         c.Assert(s.executor.Create(s.spec), IsNil)
 190         c.Assert(s.executor.Start(), IsNil)
 191         starttime := time.Now()
 192
 193         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second))
 194         defer cancel()
 195
 196         for {
 197                 time.Sleep(time.Second / 10)
 198                 if ctx.Err() != nil {
 199                         c.Error("timed out")
 200                         break
 201                 }
 202
 203                 ip, err := s.executor.IPAddress()
 204                 if err != nil {
 205                         c.Logf("s.executor.IPAddress: %s", err)
 206                         continue
 207                 }
 208                 c.Assert(ip, Not(Equals), "")
 209
 210                 // When we connect to the port using
 211                 // s.executor.IPAddress(), we should reach the nc
 212                 // process running inside the container, not the
 213                 // net.Listen() running outside the container, even
 214                 // though both listen on the same port.
 215                 ctx, cancel := context.WithDeadline(ctx, time.Now().Add(time.Second))
 216                 defer cancel()
 217                 req, err := http.NewRequestWithContext(ctx, "BREW", "http://"+net.JoinHostPort(ip, port), nil)
 218                 c.Assert(err, IsNil)
 219                 resp, err := http.DefaultClient.Do(req)
 220                 if err != nil {
 221                         c.Logf("%s (retrying...)", err)
 222                         continue
 223                 }
 224                 c.Check(resp.StatusCode, Equals, http.StatusTeapot)
 225                 c.Logf("%s %q: %s", req.Method, req.URL, resp.Status)
 226                 break
 227         }
 228
 229         s.executor.Stop()
 230         code, _ := s.executor.Wait(ctx)
 231         c.Logf("container ran for %v", time.Now().Sub(starttime))
 232         c.Check(code, Equals, -1)
 233
 234         c.Logf("stdout:\n%s\n\n", s.stdout.String())
 235         c.Logf("stderr:\n%s\n\n", s.stderr.String())
 236 }
 237
 238 func (s *executorSuite) TestEnableNetwork_IPAddress(c *C) {
 239         s.spec.Command = []string{"ip", "ad"}
 240         s.spec.EnableNetwork = true
 241         c.Assert(s.executor.Create(s.spec), IsNil)
 242         c.Assert(s.executor.Start(), IsNil)
 243         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second))
 244         defer cancel()
 245         code, _ := s.executor.Wait(ctx)
 246         c.Check(code, Equals, 0)
 247         c.Logf("stdout:\n%s\n\n", s.stdout.String())
 248         c.Logf("stderr:\n%s\n\n", s.stderr.String())
 249
 250         found := false
 251         for _, m := range regexp.MustCompile(` inet (.+?)/`).FindAllStringSubmatch(s.stdout.String(), -1) {
 252                 if addr, err := netip.ParseAddr(m[1]); err == nil && !addr.IsLoopback() {
 253                         found = true
 254                         c.Logf("found non-loopback IP address %q", m[1])
 255                         break
 256                 }
 257         }
 258         c.Check(found, Equals, true, Commentf("container does not appear to have a non-loopback IP address"))
 259 }
 260
 261 func (s *executorSuite) TestInject(c *C) {
 262         hostdir := c.MkDir()
 263         c.Assert(os.WriteFile(hostdir+"/testfile", []byte("first tube"), 0777), IsNil)
 264         mountdir := fmt.Sprintf("/injecttest-%d", os.Getpid())
 265         s.spec.Command = []string{"sleep", "10"}
 266         s.spec.BindMounts = map[string]bindmount{mountdir: {HostPath: hostdir, ReadOnly: true}}
 267         c.Assert(s.executor.Create(s.spec), IsNil)
 268         c.Assert(s.executor.Start(), IsNil)
 269         starttime := time.Now()
 270
 271         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second))
 272         defer cancel()
 273
 274         // Allow InjectCommand to fail a few times while the container
 275         // is starting
 276         for ctx.Err() == nil {
 277                 _, err := s.executor.InjectCommand(ctx, "", "root", false, []string{"true"})
 278                 if err == nil {
 279                         break
 280                 }
 281                 time.Sleep(time.Second / 10)
 282         }
 283
 284         injectcmd := []string{"cat", mountdir + "/testfile"}
 285         cmd, err := s.executor.InjectCommand(ctx, "", "root", false, injectcmd)
 286         c.Assert(err, IsNil)
 287         out, err := cmd.CombinedOutput()
 288         c.Logf("inject %s => %q", injectcmd, out)
 289         c.Check(err, IsNil)
 290         c.Check(string(out), Equals, "first tube")
 291
 292         s.executor.Stop()
 293         code, _ := s.executor.Wait(ctx)
 294         c.Logf("container ran for %v", time.Now().Sub(starttime))
 295         c.Check(code, Equals, -1)
 296 }
 297
 298 func (s *executorSuite) checkRun(c *C, expectCode int) {
 299         c.Assert(s.executor.Create(s.spec), IsNil)
 300         c.Assert(s.executor.Start(), IsNil)
 301         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second))
 302         defer cancel()
 303         code, err := s.executor.Wait(ctx)
 304         c.Assert(err, IsNil)
 305         c.Check(code, Equals, expectCode)
 306 }