lib/crunchrun/copier_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "io"
  10         "io/ioutil"
  11         "os"
  12         "syscall"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  16         "github.com/sirupsen/logrus"
  17         check "gopkg.in/check.v1"
  18 )
  19
  20 var _ = check.Suite(&copierSuite{})
  21
  22 type copierSuite struct {
  23         cp  copier
  24         log bytes.Buffer
  25 }
  26
  27 func (s *copierSuite) SetUpTest(c *check.C) {
  28         tmpdir := c.MkDir()
  29         s.log = bytes.Buffer{}
  30         s.cp = copier{
  31                 client:        arvados.NewClientFromEnv(),
  32                 hostOutputDir: tmpdir,
  33                 ctrOutputDir:  "/ctr/outdir",
  34                 mounts: map[string]arvados.Mount{
  35                         "/ctr/outdir": {Kind: "tmp"},
  36                 },
  37                 secretMounts: map[string]arvados.Mount{
  38                         "/secret_text": {Kind: "text", Content: "xyzzy"},
  39                 },
  40                 logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel},
  41         }
  42 }
  43
  44 func (s *copierSuite) TestEmptyOutput(c *check.C) {
  45         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  46         c.Check(err, check.IsNil)
  47         c.Check(s.cp.dirs, check.DeepEquals, []string(nil))
  48         c.Check(len(s.cp.files), check.Equals, 0)
  49 }
  50
  51 func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
  52         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
  53         c.Assert(err, check.IsNil)
  54         f, err := os.OpenFile(s.cp.hostOutputDir+"/dir1/foo", os.O_CREATE|os.O_WRONLY, 0644)
  55         c.Assert(err, check.IsNil)
  56         _, err = io.WriteString(f, "foo")
  57         c.Assert(err, check.IsNil)
  58         c.Assert(f.Close(), check.IsNil)
  59         err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644)
  60         c.Assert(err, check.IsNil)
  61
  62         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  63         c.Check(err, check.IsNil)
  64         c.Check(s.cp.dirs, check.DeepEquals, []string{"/dir1", "/dir1/dir2", "/dir1/dir2/dir3"})
  65         c.Check(s.cp.files, check.DeepEquals, []filetodo{
  66                 {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
  67                 {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
  68         })
  69         c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`)
  70 }
  71
  72 func (s *copierSuite) TestSymlinkCycle(c *check.C) {
  73         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir1", 0755), check.IsNil)
  74         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir2", 0755), check.IsNil)
  75         c.Assert(os.Symlink("../dir2", s.cp.hostOutputDir+"/dir1/l_dir2"), check.IsNil)
  76         c.Assert(os.Symlink("../dir1", s.cp.hostOutputDir+"/dir2/l_dir1"), check.IsNil)
  77         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  78         c.Check(err, check.ErrorMatches, `.*cycle.*`)
  79 }
  80
  81 func (s *copierSuite) TestSymlinkTargetMissing(c *check.C) {
  82         c.Assert(os.Symlink("./missing", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  83         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  84         c.Check(err, check.ErrorMatches, `.*/ctr/outdir/missing.*`)
  85 }
  86
  87 func (s *copierSuite) TestSymlinkTargetNotMounted(c *check.C) {
  88         c.Assert(os.Symlink("../boop", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  89         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  90         c.Check(err, check.ErrorMatches, `.*/ctr/boop.*`)
  91 }
  92
  93 func (s *copierSuite) TestSymlinkToSecret(c *check.C) {
  94         c.Assert(os.Symlink("/secret_text", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  95         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  96         c.Check(err, check.IsNil)
  97         c.Check(len(s.cp.dirs), check.Equals, 0)
  98         c.Check(len(s.cp.files), check.Equals, 0)
  99 }
 100
 101 func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
 102         s.cp.secretMounts["/ctr/outdir/secret_text"] = s.cp.secretMounts["/secret_text"]
 103         s.writeFileInOutputDir(c, "secret_text", "xyzzy")
 104         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 105         c.Check(err, check.IsNil)
 106         c.Check(len(s.cp.dirs), check.Equals, 0)
 107         c.Check(len(s.cp.files), check.Equals, 0)
 108 }
 109
 110 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
 111         // simulate mounted read-only collection
 112         s.cp.mounts["/mnt"] = arvados.Mount{
 113                 Kind:             "collection",
 114                 PortableDataHash: arvadostest.FooCollectionPDH,
 115         }
 116
 117         // simulate mounted writable collection
 118         bindtmp, err := ioutil.TempDir("", "crunch-run.test.")
 119         c.Assert(err, check.IsNil)
 120         defer os.RemoveAll(bindtmp)
 121         f, err := os.OpenFile(bindtmp+"/.arvados#collection", os.O_CREATE|os.O_WRONLY, 0644)
 122         c.Assert(err, check.IsNil)
 123         _, err = io.WriteString(f, `{"manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"}`)
 124         c.Assert(err, check.IsNil)
 125         c.Assert(f.Close(), check.IsNil)
 126         s.cp.mounts["/mnt-w"] = arvados.Mount{
 127                 Kind:             "collection",
 128                 PortableDataHash: arvadostest.FooCollectionPDH,
 129                 Writable:         true,
 130         }
 131         s.cp.bindmounts = map[string]bindmount{
 132                 "/mnt-w": bindmount{HostPath: bindtmp, ReadOnly: false},
 133         }
 134
 135         c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
 136         c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
 137         c.Assert(os.Symlink("/mnt-w/bar", s.cp.hostOutputDir+"/l_file_w"), check.IsNil)
 138
 139         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 140         c.Check(err, check.IsNil)
 141         c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n\. 37b5\S+ 0:3:l_file_w\n`)
 142 }
 143
 144 func (s *copierSuite) TestSymlink(c *check.C) {
 145         hostfile := s.cp.hostOutputDir + "/dir1/file"
 146
 147         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
 148         c.Assert(err, check.IsNil)
 149         s.writeFileInOutputDir(c, "dir1/file", "file")
 150         for _, err := range []error{
 151                 os.Symlink(s.cp.ctrOutputDir+"/dir1/file", s.cp.hostOutputDir+"/l_abs_file"),
 152                 os.Symlink(s.cp.ctrOutputDir+"/dir1/dir2", s.cp.hostOutputDir+"/l_abs_dir2"),
 153                 os.Symlink("../../dir1/file", s.cp.hostOutputDir+"/dir1/dir2/l_rel_file"),
 154                 os.Symlink("dir1/file", s.cp.hostOutputDir+"/l_rel_file"),
 155                 os.MkdirAll(s.cp.hostOutputDir+"/morelinks", 0755),
 156                 os.Symlink("../dir1/dir2", s.cp.hostOutputDir+"/morelinks/l_rel_dir2"),
 157                 os.Symlink("dir1/dir2/dir3", s.cp.hostOutputDir+"/l_rel_dir3"),
 158                 // rel. symlink -> rel. symlink -> regular file
 159                 os.Symlink("../dir1/dir2/l_rel_file", s.cp.hostOutputDir+"/morelinks/l_rel_l_rel_file"),
 160         } {
 161                 c.Assert(err, check.IsNil)
 162         }
 163
 164         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 165         c.Check(err, check.IsNil)
 166         c.Check(s.cp.dirs, check.DeepEquals, []string{
 167                 "/dir1", "/dir1/dir2", "/dir1/dir2/dir3",
 168                 "/l_abs_dir2", "/l_abs_dir2/dir3",
 169                 "/l_rel_dir3",
 170                 "/morelinks", "/morelinks/l_rel_dir2", "/morelinks/l_rel_dir2/dir3",
 171         })
 172         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 173                 {dst: "/dir1/dir2/dir3/.keep", src: os.DevNull},
 174                 {dst: "/dir1/dir2/l_rel_file", src: hostfile, size: 4},
 175                 {dst: "/dir1/file", src: hostfile, size: 4},
 176                 {dst: "/l_abs_dir2/dir3/.keep", src: os.DevNull},
 177                 {dst: "/l_abs_dir2/l_rel_file", src: hostfile, size: 4},
 178                 {dst: "/l_abs_file", src: hostfile, size: 4},
 179                 {dst: "/l_rel_dir3/.keep", src: os.DevNull},
 180                 {dst: "/l_rel_file", src: hostfile, size: 4},
 181                 {dst: "/morelinks/l_rel_dir2/dir3/.keep", src: os.DevNull},
 182                 {dst: "/morelinks/l_rel_dir2/l_rel_file", src: hostfile, size: 4},
 183                 {dst: "/morelinks/l_rel_l_rel_file", src: hostfile, size: 4},
 184         })
 185 }
 186
 187 func (s *copierSuite) TestUnsupportedOutputMount(c *check.C) {
 188         s.cp.mounts["/ctr/outdir"] = arvados.Mount{Kind: "waz"}
 189         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 190         c.Check(err, check.NotNil)
 191 }
 192
 193 func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
 194         s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{Kind: "waz"}
 195         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 196         c.Check(err, check.NotNil)
 197 }
 198
 199 func (s *copierSuite) TestWritableMountBelow(c *check.C) {
 200         s.cp.mounts["/ctr/outdir/mount"] = arvados.Mount{
 201                 Kind:             "collection",
 202                 PortableDataHash: arvadostest.FooCollectionPDH,
 203                 Writable:         true,
 204         }
 205         c.Assert(os.MkdirAll(s.cp.hostOutputDir+"/mount", 0755), check.IsNil)
 206         s.writeFileInOutputDir(c, "file", "file")
 207         s.writeFileInOutputDir(c, "mount/foo", "foo")
 208
 209         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 210         c.Check(err, check.IsNil)
 211         c.Check(s.cp.dirs, check.DeepEquals, []string{"/mount"})
 212         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 213                 {src: s.cp.hostOutputDir + "/file", dst: "/file", size: 4},
 214                 {src: s.cp.hostOutputDir + "/mount/foo", dst: "/mount/foo", size: 3},
 215         })
 216 }
 217
 218 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {
 219         f, err := os.OpenFile(s.cp.hostOutputDir+"/"+path, os.O_CREATE|os.O_WRONLY, 0644)
 220         c.Assert(err, check.IsNil)
 221         _, err = io.WriteString(f, data)
 222         c.Assert(err, check.IsNil)
 223         c.Assert(f.Close(), check.IsNil)
 224 }