services/crunch-run/copier_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "io"
   9         "io/ioutil"
  10         "os"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvados"
  13         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 var _ = check.Suite(&copierSuite{})
  19
  20 type copierSuite struct {
  21         cp copier
  22 }
  23
  24 func (s *copierSuite) SetUpTest(c *check.C) {
  25         tmpdir, err := ioutil.TempDir("", "crunch-run.test.")
  26         c.Assert(err, check.IsNil)
  27         api, err := arvadosclient.MakeArvadosClient()
  28         c.Assert(err, check.IsNil)
  29         s.cp = copier{
  30                 client:        arvados.NewClientFromEnv(),
  31                 arvClient:     api,
  32                 hostOutputDir: tmpdir,
  33                 ctrOutputDir:  "/ctr/outdir",
  34                 mounts: map[string]arvados.Mount{
  35                         "/ctr/outdir": {Kind: "tmp"},
  36                 },
  37                 secretMounts: map[string]arvados.Mount{
  38                         "/secret_text": {Kind: "text", Content: "xyzzy"},
  39                 },
  40         }
  41 }
  42
  43 func (s *copierSuite) TearDownTest(c *check.C) {
  44         os.RemoveAll(s.cp.hostOutputDir)
  45 }
  46
  47 func (s *copierSuite) TestEmptyOutput(c *check.C) {
  48         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  49         c.Check(err, check.IsNil)
  50         c.Check(s.cp.dirs, check.DeepEquals, []string(nil))
  51         c.Check(len(s.cp.files), check.Equals, 0)
  52 }
  53
  54 func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
  55         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
  56         c.Assert(err, check.IsNil)
  57         f, err := os.OpenFile(s.cp.hostOutputDir+"/dir1/foo", os.O_CREATE|os.O_WRONLY, 0644)
  58         c.Assert(err, check.IsNil)
  59         _, err = io.WriteString(f, "foo")
  60         c.Assert(err, check.IsNil)
  61         c.Assert(f.Close(), check.IsNil)
  62
  63         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  64         c.Check(err, check.IsNil)
  65         c.Check(s.cp.dirs, check.DeepEquals, []string{"/dir1", "/dir1/dir2", "/dir1/dir2/dir3"})
  66         c.Check(s.cp.files, check.DeepEquals, []filetodo{
  67                 {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
  68                 {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
  69         })
  70 }
  71
  72 func (s *copierSuite) TestSymlinkCycle(c *check.C) {
  73         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir1", 0755), check.IsNil)
  74         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir2", 0755), check.IsNil)
  75         c.Assert(os.Symlink("../dir2", s.cp.hostOutputDir+"/dir1/l_dir2"), check.IsNil)
  76         c.Assert(os.Symlink("../dir1", s.cp.hostOutputDir+"/dir2/l_dir1"), check.IsNil)
  77         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  78         c.Check(err, check.ErrorMatches, `.*cycle.*`)
  79 }
  80
  81 func (s *copierSuite) TestSymlinkTargetMissing(c *check.C) {
  82         c.Assert(os.Symlink("./missing", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  83         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  84         c.Check(err, check.ErrorMatches, `.*/ctr/outdir/missing.*`)
  85 }
  86
  87 func (s *copierSuite) TestSymlinkTargetNotMounted(c *check.C) {
  88         c.Assert(os.Symlink("../boop", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  89         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  90         c.Check(err, check.ErrorMatches, `.*/ctr/boop.*`)
  91 }
  92
  93 func (s *copierSuite) TestSymlinkToSecret(c *check.C) {
  94         c.Assert(os.Symlink("/secret_text", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  95         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  96         c.Check(err, check.IsNil)
  97         c.Check(len(s.cp.dirs), check.Equals, 0)
  98         c.Check(len(s.cp.files), check.Equals, 0)
  99 }
 100
 101 func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
 102         s.cp.secretMounts["/ctr/outdir/secret_text"] = s.cp.secretMounts["/secret_text"]
 103         s.writeFileInOutputDir(c, "secret_text", "xyzzy")
 104         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 105         c.Check(err, check.IsNil)
 106         c.Check(len(s.cp.dirs), check.Equals, 0)
 107         c.Check(len(s.cp.files), check.Equals, 0)
 108 }
 109
 110 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
 111         // simulate mounted read-only collection
 112         s.cp.mounts["/mnt"] = arvados.Mount{
 113                 Kind:             "collection",
 114                 PortableDataHash: arvadostest.FooPdh,
 115         }
 116
 117         // simulate mounted writable collection
 118         bindtmp, err := ioutil.TempDir("", "crunch-run.test.")
 119         c.Assert(err, check.IsNil)
 120         defer os.RemoveAll(bindtmp)
 121         f, err := os.OpenFile(bindtmp+"/.arvados#collection", os.O_CREATE|os.O_WRONLY, 0644)
 122         c.Assert(err, check.IsNil)
 123         _, err = io.WriteString(f, `{"manifest_text":". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"}`)
 124         c.Assert(err, check.IsNil)
 125         c.Assert(f.Close(), check.IsNil)
 126         s.cp.mounts["/mnt-w"] = arvados.Mount{
 127                 Kind:             "collection",
 128                 PortableDataHash: arvadostest.FooPdh,
 129                 Writable:         true,
 130         }
 131         s.cp.binds = append(s.cp.binds, bindtmp+":/mnt-w")
 132
 133         c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
 134         c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
 135         c.Assert(os.Symlink("/mnt-w/bar", s.cp.hostOutputDir+"/l_file_w"), check.IsNil)
 136
 137         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 138         c.Check(err, check.IsNil)
 139         c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n\. 37b5\S+ 0:3:l_file_w\n`)
 140 }
 141
 142 func (s *copierSuite) TestSymlink(c *check.C) {
 143         hostfile := s.cp.hostOutputDir + "/dir1/file"
 144
 145         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
 146         c.Assert(err, check.IsNil)
 147         s.writeFileInOutputDir(c, "dir1/file", "file")
 148         for _, err := range []error{
 149                 os.Symlink(s.cp.ctrOutputDir+"/dir1/file", s.cp.hostOutputDir+"/l_abs_file"),
 150                 os.Symlink(s.cp.ctrOutputDir+"/dir1/dir2", s.cp.hostOutputDir+"/l_abs_dir2"),
 151                 os.Symlink("../../dir1/file", s.cp.hostOutputDir+"/dir1/dir2/l_rel_file"),
 152                 os.Symlink("dir1/file", s.cp.hostOutputDir+"/l_rel_file"),
 153                 os.MkdirAll(s.cp.hostOutputDir+"/morelinks", 0755),
 154                 os.Symlink("../dir1/dir2", s.cp.hostOutputDir+"/morelinks/l_rel_dir2"),
 155                 os.Symlink("dir1/dir2/dir3", s.cp.hostOutputDir+"/l_rel_dir3"),
 156                 // rel. symlink -> rel. symlink -> regular file
 157                 os.Symlink("../dir1/dir2/l_rel_file", s.cp.hostOutputDir+"/morelinks/l_rel_l_rel_file"),
 158         } {
 159                 c.Assert(err, check.IsNil)
 160         }
 161
 162         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 163         c.Check(err, check.IsNil)
 164         c.Check(s.cp.dirs, check.DeepEquals, []string{
 165                 "/dir1", "/dir1/dir2", "/dir1/dir2/dir3",
 166                 "/l_abs_dir2", "/l_abs_dir2/dir3",
 167                 "/l_rel_dir3",
 168                 "/morelinks", "/morelinks/l_rel_dir2", "/morelinks/l_rel_dir2/dir3",
 169         })
 170         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 171                 {dst: "/dir1/dir2/dir3/.keep", src: os.DevNull},
 172                 {dst: "/dir1/dir2/l_rel_file", src: hostfile, size: 4},
 173                 {dst: "/dir1/file", src: hostfile, size: 4},
 174                 {dst: "/l_abs_dir2/dir3/.keep", src: os.DevNull},
 175                 {dst: "/l_abs_dir2/l_rel_file", src: hostfile, size: 4},
 176                 {dst: "/l_abs_file", src: hostfile, size: 4},
 177                 {dst: "/l_rel_dir3/.keep", src: os.DevNull},
 178                 {dst: "/l_rel_file", src: hostfile, size: 4},
 179                 {dst: "/morelinks/l_rel_dir2/dir3/.keep", src: os.DevNull},
 180                 {dst: "/morelinks/l_rel_dir2/l_rel_file", src: hostfile, size: 4},
 181                 {dst: "/morelinks/l_rel_l_rel_file", src: hostfile, size: 4},
 182         })
 183 }
 184
 185 func (s *copierSuite) TestUnsupportedOutputMount(c *check.C) {
 186         s.cp.mounts["/ctr/outdir"] = arvados.Mount{Kind: "waz"}
 187         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 188         c.Check(err, check.NotNil)
 189 }
 190
 191 func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
 192         s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{Kind: "waz"}
 193         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 194         c.Check(err, check.NotNil)
 195 }
 196
 197 func (s *copierSuite) TestWritableMountBelow(c *check.C) {
 198         s.cp.mounts["/ctr/outdir/mount"] = arvados.Mount{
 199                 Kind:             "collection",
 200                 PortableDataHash: arvadostest.FooPdh,
 201                 Writable:         true,
 202         }
 203         c.Assert(os.MkdirAll(s.cp.hostOutputDir+"/mount", 0755), check.IsNil)
 204         s.writeFileInOutputDir(c, "file", "file")
 205         s.writeFileInOutputDir(c, "mount/foo", "foo")
 206
 207         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 208         c.Check(err, check.IsNil)
 209         c.Check(s.cp.dirs, check.DeepEquals, []string{"/mount"})
 210         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 211                 {src: s.cp.hostOutputDir + "/file", dst: "/file", size: 4},
 212                 {src: s.cp.hostOutputDir + "/mount/foo", dst: "/mount/foo", size: 3},
 213         })
 214 }
 215
 216 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {
 217         f, err := os.OpenFile(s.cp.hostOutputDir+"/"+path, os.O_CREATE|os.O_WRONLY, 0644)
 218         c.Assert(err, check.IsNil)
 219         _, err = io.WriteString(f, data)
 220         c.Assert(err, check.IsNil)
 221         c.Assert(f.Close(), check.IsNil)
 222 }