lib/crunchrun/copier_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "bytes"
   9         "io"
  10         "io/ioutil"
  11         "os"
  12         "syscall"
  13
  14         "git.arvados.org/arvados.git/sdk/go/arvados"
  15         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  16         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  17         "github.com/sirupsen/logrus"
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 var _ = check.Suite(&copierSuite{})
  22
  23 type copierSuite struct {
  24         cp  copier
  25         log bytes.Buffer
  26 }
  27
  28 func (s *copierSuite) SetUpTest(c *check.C) {
  29         tmpdir := c.MkDir()
  30         api, err := arvadosclient.MakeArvadosClient()
  31         c.Assert(err, check.IsNil)
  32         s.log = bytes.Buffer{}
  33         s.cp = copier{
  34                 client:        arvados.NewClientFromEnv(),
  35                 arvClient:     api,
  36                 hostOutputDir: tmpdir,
  37                 ctrOutputDir:  "/ctr/outdir",
  38                 mounts: map[string]arvados.Mount{
  39                         "/ctr/outdir": {Kind: "tmp"},
  40                 },
  41                 secretMounts: map[string]arvados.Mount{
  42                         "/secret_text": {Kind: "text", Content: "xyzzy"},
  43                 },
  44                 logger: &logrus.Logger{Out: &s.log, Formatter: &logrus.TextFormatter{}, Level: logrus.InfoLevel},
  45         }
  46 }
  47
  48 func (s *copierSuite) TestEmptyOutput(c *check.C) {
  49         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  50         c.Check(err, check.IsNil)
  51         c.Check(s.cp.dirs, check.DeepEquals, []string(nil))
  52         c.Check(len(s.cp.files), check.Equals, 0)
  53 }
  54
  55 func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
  56         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
  57         c.Assert(err, check.IsNil)
  58         f, err := os.OpenFile(s.cp.hostOutputDir+"/dir1/foo", os.O_CREATE|os.O_WRONLY, 0644)
  59         c.Assert(err, check.IsNil)
  60         _, err = io.WriteString(f, "foo")
  61         c.Assert(err, check.IsNil)
  62         c.Assert(f.Close(), check.IsNil)
  63         err = syscall.Mkfifo(s.cp.hostOutputDir+"/dir1/fifo", 0644)
  64         c.Assert(err, check.IsNil)
  65
  66         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  67         c.Check(err, check.IsNil)
  68         c.Check(s.cp.dirs, check.DeepEquals, []string{"/dir1", "/dir1/dir2", "/dir1/dir2/dir3"})
  69         c.Check(s.cp.files, check.DeepEquals, []filetodo{
  70                 {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
  71                 {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
  72         })
  73         c.Check(s.log.String(), check.Matches, `.* msg="Skipping unsupported file type \(mode 200000644\) in output dir: \\"/ctr/outdir/dir1/fifo\\""\n`)
  74 }
  75
  76 func (s *copierSuite) TestSymlinkCycle(c *check.C) {
  77         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir1", 0755), check.IsNil)
  78         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir2", 0755), check.IsNil)
  79         c.Assert(os.Symlink("../dir2", s.cp.hostOutputDir+"/dir1/l_dir2"), check.IsNil)
  80         c.Assert(os.Symlink("../dir1", s.cp.hostOutputDir+"/dir2/l_dir1"), check.IsNil)
  81         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  82         c.Check(err, check.ErrorMatches, `.*cycle.*`)
  83 }
  84
  85 func (s *copierSuite) TestSymlinkTargetMissing(c *check.C) {
  86         c.Assert(os.Symlink("./missing", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  87         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  88         c.Check(err, check.ErrorMatches, `.*/ctr/outdir/missing.*`)
  89 }
  90
  91 func (s *copierSuite) TestSymlinkTargetNotMounted(c *check.C) {
  92         c.Assert(os.Symlink("../boop", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  93         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  94         c.Check(err, check.ErrorMatches, `.*/ctr/boop.*`)
  95 }
  96
  97 func (s *copierSuite) TestSymlinkToSecret(c *check.C) {
  98         c.Assert(os.Symlink("/secret_text", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  99         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 100         c.Check(err, check.IsNil)
 101         c.Check(len(s.cp.dirs), check.Equals, 0)
 102         c.Check(len(s.cp.files), check.Equals, 0)
 103 }
 104
 105 func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
 106         s.cp.secretMounts["/ctr/outdir/secret_text"] = s.cp.secretMounts["/secret_text"]
 107         s.writeFileInOutputDir(c, "secret_text", "xyzzy")
 108         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 109         c.Check(err, check.IsNil)
 110         c.Check(len(s.cp.dirs), check.Equals, 0)
 111         c.Check(len(s.cp.files), check.Equals, 0)
 112 }
 113
 114 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
 115         // simulate mounted read-only collection
 116         s.cp.mounts["/mnt"] = arvados.Mount{
 117                 Kind:             "collection",
 118                 PortableDataHash: arvadostest.FooCollectionPDH,
 119         }
 120
 121         // simulate mounted writable collection
 122         bindtmp, err := ioutil.TempDir("", "crunch-run.test.")
 123         c.Assert(err, check.IsNil)
 124         defer os.RemoveAll(bindtmp)
 125         f, err := os.OpenFile(bindtmp+"/.arvados#collection", os.O_CREATE|os.O_WRONLY, 0644)
 126         c.Assert(err, check.IsNil)
 127         _, err = io.WriteString(f, `{"manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"}`)
 128         c.Assert(err, check.IsNil)
 129         c.Assert(f.Close(), check.IsNil)
 130         s.cp.mounts["/mnt-w"] = arvados.Mount{
 131                 Kind:             "collection",
 132                 PortableDataHash: arvadostest.FooCollectionPDH,
 133                 Writable:         true,
 134         }
 135         s.cp.binds = append(s.cp.binds, bindtmp+":/mnt-w")
 136
 137         c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
 138         c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
 139         c.Assert(os.Symlink("/mnt-w/bar", s.cp.hostOutputDir+"/l_file_w"), check.IsNil)
 140
 141         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 142         c.Check(err, check.IsNil)
 143         c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n\. 37b5\S+ 0:3:l_file_w\n`)
 144 }
 145
 146 func (s *copierSuite) TestSymlink(c *check.C) {
 147         hostfile := s.cp.hostOutputDir + "/dir1/file"
 148
 149         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
 150         c.Assert(err, check.IsNil)
 151         s.writeFileInOutputDir(c, "dir1/file", "file")
 152         for _, err := range []error{
 153                 os.Symlink(s.cp.ctrOutputDir+"/dir1/file", s.cp.hostOutputDir+"/l_abs_file"),
 154                 os.Symlink(s.cp.ctrOutputDir+"/dir1/dir2", s.cp.hostOutputDir+"/l_abs_dir2"),
 155                 os.Symlink("../../dir1/file", s.cp.hostOutputDir+"/dir1/dir2/l_rel_file"),
 156                 os.Symlink("dir1/file", s.cp.hostOutputDir+"/l_rel_file"),
 157                 os.MkdirAll(s.cp.hostOutputDir+"/morelinks", 0755),
 158                 os.Symlink("../dir1/dir2", s.cp.hostOutputDir+"/morelinks/l_rel_dir2"),
 159                 os.Symlink("dir1/dir2/dir3", s.cp.hostOutputDir+"/l_rel_dir3"),
 160                 // rel. symlink -> rel. symlink -> regular file
 161                 os.Symlink("../dir1/dir2/l_rel_file", s.cp.hostOutputDir+"/morelinks/l_rel_l_rel_file"),
 162         } {
 163                 c.Assert(err, check.IsNil)
 164         }
 165
 166         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 167         c.Check(err, check.IsNil)
 168         c.Check(s.cp.dirs, check.DeepEquals, []string{
 169                 "/dir1", "/dir1/dir2", "/dir1/dir2/dir3",
 170                 "/l_abs_dir2", "/l_abs_dir2/dir3",
 171                 "/l_rel_dir3",
 172                 "/morelinks", "/morelinks/l_rel_dir2", "/morelinks/l_rel_dir2/dir3",
 173         })
 174         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 175                 {dst: "/dir1/dir2/dir3/.keep", src: os.DevNull},
 176                 {dst: "/dir1/dir2/l_rel_file", src: hostfile, size: 4},
 177                 {dst: "/dir1/file", src: hostfile, size: 4},
 178                 {dst: "/l_abs_dir2/dir3/.keep", src: os.DevNull},
 179                 {dst: "/l_abs_dir2/l_rel_file", src: hostfile, size: 4},
 180                 {dst: "/l_abs_file", src: hostfile, size: 4},
 181                 {dst: "/l_rel_dir3/.keep", src: os.DevNull},
 182                 {dst: "/l_rel_file", src: hostfile, size: 4},
 183                 {dst: "/morelinks/l_rel_dir2/dir3/.keep", src: os.DevNull},
 184                 {dst: "/morelinks/l_rel_dir2/l_rel_file", src: hostfile, size: 4},
 185                 {dst: "/morelinks/l_rel_l_rel_file", src: hostfile, size: 4},
 186         })
 187 }
 188
 189 func (s *copierSuite) TestUnsupportedOutputMount(c *check.C) {
 190         s.cp.mounts["/ctr/outdir"] = arvados.Mount{Kind: "waz"}
 191         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 192         c.Check(err, check.NotNil)
 193 }
 194
 195 func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
 196         s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{Kind: "waz"}
 197         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 198         c.Check(err, check.NotNil)
 199 }
 200
 201 func (s *copierSuite) TestWritableMountBelow(c *check.C) {
 202         s.cp.mounts["/ctr/outdir/mount"] = arvados.Mount{
 203                 Kind:             "collection",
 204                 PortableDataHash: arvadostest.FooCollectionPDH,
 205                 Writable:         true,
 206         }
 207         c.Assert(os.MkdirAll(s.cp.hostOutputDir+"/mount", 0755), check.IsNil)
 208         s.writeFileInOutputDir(c, "file", "file")
 209         s.writeFileInOutputDir(c, "mount/foo", "foo")
 210
 211         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 212         c.Check(err, check.IsNil)
 213         c.Check(s.cp.dirs, check.DeepEquals, []string{"/mount"})
 214         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 215                 {src: s.cp.hostOutputDir + "/file", dst: "/file", size: 4},
 216                 {src: s.cp.hostOutputDir + "/mount/foo", dst: "/mount/foo", size: 3},
 217         })
 218 }
 219
 220 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {
 221         f, err := os.OpenFile(s.cp.hostOutputDir+"/"+path, os.O_CREATE|os.O_WRONLY, 0644)
 222         c.Assert(err, check.IsNil)
 223         _, err = io.WriteString(f, data)
 224         c.Assert(err, check.IsNil)
 225         c.Assert(f.Close(), check.IsNil)
 226 }