lib/crunchrun/singularity_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "fmt"
   9         "os"
  10         "os/exec"
  11         "strings"
  12         "sync"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/arvados"
  16         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  17         . "gopkg.in/check.v1"
  18 )
  19
  20 var _ = Suite(&singularitySuite{})
  21
  22 type singularitySuite struct {
  23         executorSuite
  24 }
  25
  26 func (s *singularitySuite) SetUpSuite(c *C) {
  27         _, err := exec.LookPath("singularity")
  28         if err != nil {
  29                 c.Skip("looks like singularity is not installed")
  30         }
  31         s.newExecutor = func(c *C) {
  32                 var err error
  33                 s.executor, err = newSingularityExecutor(c.Logf)
  34                 c.Assert(err, IsNil)
  35         }
  36         arvadostest.StartKeep(2, true)
  37 }
  38
  39 func (s *singularitySuite) TearDownSuite(c *C) {
  40         if s.executor != nil {
  41                 s.executor.Close()
  42         }
  43 }
  44
  45 func (s *singularitySuite) TestEnableNetwork_Listen(c *C) {
  46         // With modern iptables, singularity (as of 4.2.1) cannot
  47         // enable networking when invoked by a regular user. Under
  48         // arvados-dispatch-cloud, crunch-run runs as root, so it's
  49         // OK. For testing, assuming tests are not running as root, we
  50         // use sudo -- but only if requested via environment variable.
  51         if os.Getuid() == 0 {
  52                 // already root
  53         } else if os.Getenv("ARVADOS_TEST_PRIVESC") == "sudo" {
  54                 c.Logf("ARVADOS_TEST_PRIVESC is 'sudo', invoking 'sudo singularity ...'")
  55                 s.executor.(*singularityExecutor).sudo = true
  56         } else {
  57                 c.Skip("test case needs to run singularity as root -- set ARVADOS_TEST_PRIVESC=sudo to enable this test")
  58         }
  59         s.executorSuite.TestEnableNetwork_Listen(c)
  60 }
  61
  62 func (s *singularitySuite) TestInject(c *C) {
  63         path, err := exec.LookPath("nsenter")
  64         if err != nil || path != "/var/lib/arvados/bin/nsenter" {
  65                 c.Skip("looks like /var/lib/arvados/bin/nsenter is not installed -- re-run `arvados-server install`?")
  66         }
  67         s.executorSuite.TestInject(c)
  68 }
  69
  70 var _ = Suite(&singularityStubSuite{})
  71
  72 // singularityStubSuite tests don't really invoke singularity, so we
  73 // can run them even if singularity is not installed.
  74 type singularityStubSuite struct{}
  75
  76 func (s *singularityStubSuite) TestSingularityExecArgs(c *C) {
  77         e, err := newSingularityExecutor(c.Logf)
  78         c.Assert(err, IsNil)
  79         err = e.Create(containerSpec{
  80                 WorkingDir:     "/WorkingDir",
  81                 Env:            map[string]string{"FOO": "bar"},
  82                 BindMounts:     map[string]bindmount{"/mnt": {HostPath: "/hostpath", ReadOnly: true}},
  83                 EnableNetwork:  false,
  84                 GPUStack:       "cuda",
  85                 GPUDeviceCount: 3,
  86                 VCPUs:          2,
  87                 RAM:            12345678,
  88         })
  89         c.Check(err, IsNil)
  90         e.imageFilename = "/fake/image.sif"
  91         cmd := e.execCmd("./singularity")
  92         expectArgs := []string{"./singularity", "exec", "--containall", "--cleanenv", "--pwd=/WorkingDir", "--net", "--network=none", "--nv"}
  93         if cgroupSupport["cpu"] {
  94                 expectArgs = append(expectArgs, "--cpus", "2")
  95         }
  96         if cgroupSupport["memory"] {
  97                 expectArgs = append(expectArgs, "--memory", "12345678")
  98         }
  99         expectArgs = append(expectArgs, "--bind", "/hostpath:/mnt:ro", "/fake/image.sif")
 100         c.Check(cmd.Args, DeepEquals, expectArgs)
 101         c.Check(cmd.Env, DeepEquals, []string{
 102                 "SINGULARITYENV_FOO=bar",
 103                 "SINGULARITY_NO_EVAL=1",
 104                 "XDG_RUNTIME_DIR=" + os.Getenv("XDG_RUNTIME_DIR"),
 105                 "DBUS_SESSION_BUS_ADDRESS=" + os.Getenv("DBUS_SESSION_BUS_ADDRESS"),
 106         })
 107 }
 108
 109 func (s *singularitySuite) setupMount(c *C) (mountdir string) {
 110         mountdir = c.MkDir()
 111         cmd := exec.Command("arv-mount",
 112                 "--foreground", "--read-write",
 113                 "--storage-classes", "default",
 114                 "--mount-by-pdh", "by_id", "--mount-by-id", "by_uuid",
 115                 "--disable-event-listening",
 116                 mountdir)
 117         cmd.Stdout = os.Stderr
 118         cmd.Stderr = os.Stderr
 119         err := cmd.Start()
 120         c.Assert(err, IsNil)
 121         return
 122 }
 123
 124 func (s *singularitySuite) teardownMount(c *C, mountdir string) {
 125         exec.Command("arv-mount", "--unmount", mountdir).Run()
 126 }
 127
 128 type singularitySuiteLoadTestSetup struct {
 129         containerClient   *arvados.Client
 130         imageCacheProject *arvados.Group
 131         dockerImageID     string
 132         collectionName    string
 133 }
 134
 135 func (s *singularitySuite) setupLoadTest(c *C, e *singularityExecutor) (setup singularitySuiteLoadTestSetup) {
 136         // remove symlink and converted image already written by
 137         // (executorSuite)SetupTest
 138         os.Remove(e.tmpdir + "/image.tar")
 139         os.Remove(e.tmpdir + "/image.sif")
 140
 141         setup.containerClient = arvados.NewClientFromEnv()
 142         setup.containerClient.AuthToken = arvadostest.ActiveTokenV2
 143
 144         var err error
 145         setup.imageCacheProject, err = e.getImageCacheProject(arvadostest.ActiveUserUUID, setup.containerClient)
 146         c.Assert(err, IsNil)
 147
 148         setup.dockerImageID = "sha256:388056c9a6838deea3792e8f00705b35b439cf57b3c9c2634fb4e95cfc896de6"
 149         setup.collectionName = fmt.Sprintf("singularity image for %s", setup.dockerImageID)
 150
 151         // Remove existing cache entry, if any.
 152         var cl arvados.CollectionList
 153         err = setup.containerClient.RequestAndDecode(&cl,
 154                 arvados.EndpointCollectionList.Method,
 155                 arvados.EndpointCollectionList.Path,
 156                 nil, arvados.ListOptions{Filters: []arvados.Filter{
 157                         arvados.Filter{"owner_uuid", "=", setup.imageCacheProject.UUID},
 158                         arvados.Filter{"name", "=", setup.collectionName},
 159                 },
 160                         Limit: 1})
 161         c.Assert(err, IsNil)
 162         if len(cl.Items) == 1 {
 163                 setup.containerClient.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+cl.Items[0].UUID, nil, nil)
 164         }
 165
 166         return
 167 }
 168
 169 func (s *singularitySuite) checkCacheCollectionExists(c *C, setup singularitySuiteLoadTestSetup) {
 170         var cl arvados.CollectionList
 171         err := setup.containerClient.RequestAndDecode(&cl,
 172                 arvados.EndpointCollectionList.Method,
 173                 arvados.EndpointCollectionList.Path,
 174                 nil, arvados.ListOptions{Filters: []arvados.Filter{
 175                         arvados.Filter{"owner_uuid", "=", setup.imageCacheProject.UUID},
 176                         arvados.Filter{"name", "=", setup.collectionName},
 177                 },
 178                         Limit: 1})
 179         c.Assert(err, IsNil)
 180         if !c.Check(cl.Items, HasLen, 1) {
 181                 return
 182         }
 183         c.Check(cl.Items[0].PortableDataHash, Not(Equals), "d41d8cd98f00b204e9800998ecf8427e+0")
 184 }
 185
 186 func (s *singularitySuite) TestImageCache_New(c *C) {
 187         mountdir := s.setupMount(c)
 188         defer s.teardownMount(c, mountdir)
 189         e, err := newSingularityExecutor(c.Logf)
 190         c.Assert(err, IsNil)
 191         setup := s.setupLoadTest(c, e)
 192         err = e.LoadImage(setup.dockerImageID, arvadostest.BusyboxDockerImage(c), arvados.Container{RuntimeUserUUID: arvadostest.ActiveUserUUID}, mountdir, setup.containerClient)
 193         c.Check(err, IsNil)
 194         _, err = os.Stat(e.tmpdir + "/image.sif")
 195         c.Check(err, NotNil)
 196         c.Check(os.IsNotExist(err), Equals, true)
 197         s.checkCacheCollectionExists(c, setup)
 198 }
 199
 200 func (s *singularitySuite) TestImageCache_SkipEmpty(c *C) {
 201         mountdir := s.setupMount(c)
 202         defer s.teardownMount(c, mountdir)
 203         e, err := newSingularityExecutor(c.Logf)
 204         c.Assert(err, IsNil)
 205         setup := s.setupLoadTest(c, e)
 206
 207         var emptyCollection arvados.Collection
 208         exp := time.Now().Add(24 * 7 * 2 * time.Hour)
 209         err = setup.containerClient.RequestAndDecode(&emptyCollection,
 210                 arvados.EndpointCollectionCreate.Method,
 211                 arvados.EndpointCollectionCreate.Path,
 212                 nil, map[string]interface{}{
 213                         "collection": map[string]string{
 214                                 "owner_uuid": setup.imageCacheProject.UUID,
 215                                 "name":       setup.collectionName,
 216                                 "trash_at":   exp.UTC().Format(time.RFC3339),
 217                         },
 218                 })
 219         c.Assert(err, IsNil)
 220
 221         err = e.LoadImage(setup.dockerImageID, arvadostest.BusyboxDockerImage(c), arvados.Container{RuntimeUserUUID: arvadostest.ActiveUserUUID}, mountdir, setup.containerClient)
 222         c.Check(err, IsNil)
 223         c.Check(e.imageFilename, Equals, e.tmpdir+"/image.sif")
 224
 225         // tmpdir should contain symlink to docker image archive.
 226         tarListing, err := exec.Command("tar", "tvf", e.tmpdir+"/image.tar").CombinedOutput()
 227         c.Check(err, IsNil)
 228         c.Check(string(tarListing), Matches, `(?ms).*/layer.tar.*`)
 229
 230         // converted singularity image should be non-empty.
 231         fi, err := os.Stat(e.imageFilename)
 232         if c.Check(err, IsNil) {
 233                 c.Check(int(fi.Size()), Not(Equals), 0)
 234         }
 235 }
 236
 237 func (s *singularitySuite) TestImageCache_Concurrency_1(c *C) {
 238         s.testImageCache(c, 1)
 239 }
 240
 241 func (s *singularitySuite) TestImageCache_Concurrency_2(c *C) {
 242         s.testImageCache(c, 2)
 243 }
 244
 245 func (s *singularitySuite) TestImageCache_Concurrency_10(c *C) {
 246         s.testImageCache(c, 10)
 247 }
 248
 249 func (s *singularitySuite) testImageCache(c *C, concurrency int) {
 250         mountdirs := make([]string, concurrency)
 251         execs := make([]*singularityExecutor, concurrency)
 252         setups := make([]singularitySuiteLoadTestSetup, concurrency)
 253         for i := range execs {
 254                 mountdirs[i] = s.setupMount(c)
 255                 defer s.teardownMount(c, mountdirs[i])
 256                 e, err := newSingularityExecutor(c.Logf)
 257                 c.Assert(err, IsNil)
 258                 defer e.Close()
 259                 execs[i] = e
 260                 setups[i] = s.setupLoadTest(c, e)
 261         }
 262
 263         var wg sync.WaitGroup
 264         for i, e := range execs {
 265                 i, e := i, e
 266                 wg.Add(1)
 267                 go func() {
 268                         defer wg.Done()
 269                         err := e.LoadImage(setups[i].dockerImageID, arvadostest.BusyboxDockerImage(c), arvados.Container{RuntimeUserUUID: arvadostest.ActiveUserUUID}, mountdirs[i], setups[i].containerClient)
 270                         c.Check(err, IsNil)
 271                 }()
 272         }
 273         wg.Wait()
 274
 275         for i, e := range execs {
 276                 fusepath := strings.TrimPrefix(e.imageFilename, mountdirs[i])
 277                 // imageFilename should be in the fuse mount, not
 278                 // e.tmpdir.
 279                 c.Check(fusepath, Not(Equals), execs[0].imageFilename)
 280                 // Below fuse mountpoint, paths should all be equal.
 281                 fusepath0 := strings.TrimPrefix(execs[0].imageFilename, mountdirs[0])
 282                 c.Check(fusepath, Equals, fusepath0)
 283         }
 284 }