services/crunch-run/copier_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "io"
   9         "io/ioutil"
  10         "os"
  11
  12         "git.curoverse.com/arvados.git/sdk/go/arvados"
  13         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  15         check "gopkg.in/check.v1"
  16 )
  17
  18 var _ = check.Suite(&copierSuite{})
  19
  20 type copierSuite struct {
  21         cp copier
  22 }
  23
  24 func (s *copierSuite) SetUpTest(c *check.C) {
  25         tmpdir, err := ioutil.TempDir("", "crunch-run.test.")
  26         c.Assert(err, check.IsNil)
  27         api, err := arvadosclient.MakeArvadosClient()
  28         c.Assert(err, check.IsNil)
  29         s.cp = copier{
  30                 client:        arvados.NewClientFromEnv(),
  31                 arvClient:     api,
  32                 hostOutputDir: tmpdir,
  33                 ctrOutputDir:  "/ctr/outdir",
  34                 mounts: map[string]arvados.Mount{
  35                         "/ctr/outdir": {Kind: "tmp"},
  36                 },
  37                 secretMounts: map[string]arvados.Mount{
  38                         "/secret_text": {Kind: "text", Content: "xyzzy"},
  39                 },
  40         }
  41 }
  42
  43 func (s *copierSuite) TearDownTest(c *check.C) {
  44         os.RemoveAll(s.cp.hostOutputDir)
  45 }
  46
  47 func (s *copierSuite) TestEmptyOutput(c *check.C) {
  48         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  49         c.Check(err, check.IsNil)
  50         c.Check(s.cp.dirs, check.DeepEquals, []string(nil))
  51         c.Check(len(s.cp.files), check.Equals, 0)
  52 }
  53
  54 func (s *copierSuite) TestRegularFilesAndDirs(c *check.C) {
  55         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
  56         c.Assert(err, check.IsNil)
  57         f, err := os.OpenFile(s.cp.hostOutputDir+"/dir1/foo", os.O_CREATE|os.O_WRONLY, 0644)
  58         c.Assert(err, check.IsNil)
  59         _, err = io.WriteString(f, "foo")
  60         c.Assert(err, check.IsNil)
  61         c.Assert(f.Close(), check.IsNil)
  62
  63         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  64         c.Check(err, check.IsNil)
  65         c.Check(s.cp.dirs, check.DeepEquals, []string{"/dir1", "/dir1/dir2", "/dir1/dir2/dir3"})
  66         c.Check(s.cp.files, check.DeepEquals, []filetodo{
  67                 {src: os.DevNull, dst: "/dir1/dir2/dir3/.keep"},
  68                 {src: s.cp.hostOutputDir + "/dir1/foo", dst: "/dir1/foo", size: 3},
  69         })
  70 }
  71
  72 func (s *copierSuite) TestSymlinkCycle(c *check.C) {
  73         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir1", 0755), check.IsNil)
  74         c.Assert(os.Mkdir(s.cp.hostOutputDir+"/dir2", 0755), check.IsNil)
  75         c.Assert(os.Symlink("../dir2", s.cp.hostOutputDir+"/dir1/l_dir2"), check.IsNil)
  76         c.Assert(os.Symlink("../dir1", s.cp.hostOutputDir+"/dir2/l_dir1"), check.IsNil)
  77         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  78         c.Check(err, check.ErrorMatches, `.*cycle.*`)
  79 }
  80
  81 func (s *copierSuite) TestSymlinkTargetMissing(c *check.C) {
  82         c.Assert(os.Symlink("./missing", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  83         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  84         c.Check(err, check.ErrorMatches, `.*/ctr/outdir/missing.*`)
  85 }
  86
  87 func (s *copierSuite) TestSymlinkTargetNotMounted(c *check.C) {
  88         c.Assert(os.Symlink("../boop", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  89         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  90         c.Check(err, check.ErrorMatches, `.*/ctr/boop.*`)
  91 }
  92
  93 func (s *copierSuite) TestSymlinkToSecret(c *check.C) {
  94         c.Assert(os.Symlink("/secret_text", s.cp.hostOutputDir+"/symlink"), check.IsNil)
  95         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
  96         c.Check(err, check.IsNil)
  97         c.Check(len(s.cp.dirs), check.Equals, 0)
  98         c.Check(len(s.cp.files), check.Equals, 0)
  99 }
 100
 101 func (s *copierSuite) TestSecretInOutputDir(c *check.C) {
 102         s.cp.secretMounts["/ctr/outdir/secret_text"] = s.cp.secretMounts["/secret_text"]
 103         s.writeFileInOutputDir(c, "secret_text", "xyzzy")
 104         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 105         c.Check(err, check.IsNil)
 106         c.Check(len(s.cp.dirs), check.Equals, 0)
 107         c.Check(len(s.cp.files), check.Equals, 0)
 108 }
 109
 110 func (s *copierSuite) TestSymlinkToMountedCollection(c *check.C) {
 111         s.cp.mounts["/mnt"] = arvados.Mount{
 112                 Kind:             "collection",
 113                 PortableDataHash: arvadostest.FooPdh,
 114         }
 115         c.Assert(os.Symlink("../../mnt", s.cp.hostOutputDir+"/l_dir"), check.IsNil)
 116         c.Assert(os.Symlink("/mnt/foo", s.cp.hostOutputDir+"/l_file"), check.IsNil)
 117
 118         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 119         c.Check(err, check.IsNil)
 120         c.Check(s.cp.manifest, check.Matches, `(?ms)\./l_dir acbd\S+ 0:3:foo\n\. acbd\S+ 0:3:l_file\n`)
 121 }
 122
 123 func (s *copierSuite) TestSymlink(c *check.C) {
 124         hostfile := s.cp.hostOutputDir + "/dir1/file"
 125
 126         err := os.MkdirAll(s.cp.hostOutputDir+"/dir1/dir2/dir3", 0755)
 127         c.Assert(err, check.IsNil)
 128         s.writeFileInOutputDir(c, "dir1/file", "file")
 129         for _, err := range []error{
 130                 os.Symlink(s.cp.ctrOutputDir+"/dir1/file", s.cp.hostOutputDir+"/l_abs_file"),
 131                 os.Symlink(s.cp.ctrOutputDir+"/dir1/dir2", s.cp.hostOutputDir+"/l_abs_dir2"),
 132                 os.Symlink("../../dir1/file", s.cp.hostOutputDir+"/dir1/dir2/l_rel_file"),
 133                 os.Symlink("dir1/file", s.cp.hostOutputDir+"/l_rel_file"),
 134                 os.MkdirAll(s.cp.hostOutputDir+"/morelinks", 0755),
 135                 os.Symlink("../dir1/dir2", s.cp.hostOutputDir+"/morelinks/l_rel_dir2"),
 136                 os.Symlink("dir1/dir2/dir3", s.cp.hostOutputDir+"/l_rel_dir3"),
 137         } {
 138                 c.Assert(err, check.IsNil)
 139         }
 140
 141         err = s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 142         c.Check(err, check.IsNil)
 143         c.Check(s.cp.dirs, check.DeepEquals, []string{
 144                 "/dir1", "/dir1/dir2", "/dir1/dir2/dir3",
 145                 "/l_abs_dir2", "/l_abs_dir2/dir3",
 146                 "/l_rel_dir3",
 147                 "/morelinks", "/morelinks/l_rel_dir2", "/morelinks/l_rel_dir2/dir3",
 148         })
 149         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 150                 {dst: "/dir1/dir2/dir3/.keep", src: os.DevNull},
 151                 {dst: "/dir1/dir2/l_rel_file", src: hostfile, size: 4},
 152                 {dst: "/dir1/file", src: hostfile, size: 4},
 153                 {dst: "/l_abs_dir2/dir3/.keep", src: os.DevNull},
 154                 {dst: "/l_abs_dir2/l_rel_file", src: hostfile, size: 4},
 155                 {dst: "/l_abs_file", src: hostfile, size: 4},
 156                 {dst: "/l_rel_dir3/.keep", src: os.DevNull},
 157                 {dst: "/l_rel_file", src: hostfile, size: 4},
 158                 {dst: "/morelinks/l_rel_dir2/dir3/.keep", src: os.DevNull},
 159                 {dst: "/morelinks/l_rel_dir2/l_rel_file", src: hostfile, size: 4},
 160         })
 161 }
 162
 163 func (s *copierSuite) TestUnsupportedOutputMount(c *check.C) {
 164         s.cp.mounts["/ctr/outdir"] = arvados.Mount{Kind: "waz"}
 165         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 166         c.Check(err, check.NotNil)
 167 }
 168
 169 func (s *copierSuite) TestUnsupportedMountKindBelow(c *check.C) {
 170         s.cp.mounts["/ctr/outdir/dirk"] = arvados.Mount{Kind: "waz"}
 171         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 172         c.Check(err, check.NotNil)
 173 }
 174
 175 func (s *copierSuite) TestWritableMountBelow(c *check.C) {
 176         s.cp.mounts["/ctr/outdir/mount"] = arvados.Mount{
 177                 Kind:             "collection",
 178                 PortableDataHash: arvadostest.FooPdh,
 179                 Writable:         true,
 180         }
 181         c.Assert(os.MkdirAll(s.cp.hostOutputDir+"/mount", 0755), check.IsNil)
 182         s.writeFileInOutputDir(c, "file", "file")
 183         s.writeFileInOutputDir(c, "mount/foo", "foo")
 184
 185         err := s.cp.walkMount("", s.cp.ctrOutputDir, 10, true)
 186         c.Check(err, check.IsNil)
 187         c.Check(s.cp.dirs, check.DeepEquals, []string{"/mount"})
 188         c.Check(s.cp.files, check.DeepEquals, []filetodo{
 189                 {src: s.cp.hostOutputDir + "/file", dst: "/file", size: 4},
 190                 {src: s.cp.hostOutputDir + "/mount/foo", dst: "/mount/foo", size: 3},
 191         })
 192 }
 193
 194 func (s *copierSuite) writeFileInOutputDir(c *check.C, path, data string) {
 195         f, err := os.OpenFile(s.cp.hostOutputDir+"/"+path, os.O_CREATE|os.O_WRONLY, 0644)
 196         c.Assert(err, check.IsNil)
 197         _, err = io.WriteString(f, data)
 198         c.Assert(err, check.IsNil)
 199         c.Assert(f.Close(), check.IsNil)
 200 }