sdk/go/arvados/fs_site_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "net/http"
  12         "os"
  13         "syscall"
  14         "time"
  15
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 const (
  20         // Importing arvadostest would be an import cycle, so these
  21         // fixtures are duplicated here [until fs moves to a separate
  22         // package].
  23         fixtureActiveToken                  = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
  24         fixtureAProjectUUID                 = "zzzzz-j7d0g-v955i6s2oi1cbso"
  25         fixtureThisFilterGroupUUID          = "zzzzz-j7d0g-thisfiltergroup"
  26         fixtureAFilterGroupTwoUUID          = "zzzzz-j7d0g-afiltergrouptwo"
  27         fixtureAFilterGroupThreeUUID        = "zzzzz-j7d0g-filtergroupthre"
  28         fixtureAFilterGroupFourUUID         = "zzzzz-j7d0g-filtergroupfour"
  29         fixtureAFilterGroupFiveUUID         = "zzzzz-j7d0g-filtergroupfive"
  30         fixtureFooAndBarFilesInDirUUID      = "zzzzz-4zz18-foonbarfilesdir"
  31         fixtureFooCollectionName            = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
  32         fixtureFooCollectionPDH             = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
  33         fixtureFooCollection                = "zzzzz-4zz18-fy296fx3hot09f7"
  34         fixtureNonexistentCollection        = "zzzzz-4zz18-totallynotexist"
  35         fixtureStorageClassesDesiredArchive = "zzzzz-4zz18-3t236wr12769qqa"
  36         fixtureBlobSigningKey               = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
  37         fixtureBlobSigningTTL               = 336 * time.Hour
  38 )
  39
  40 var _ = check.Suite(&SiteFSSuite{})
  41
  42 func init() {
  43         // Enable DebugLocksPanicMode sometimes. Don't enable it all
  44         // the time, though -- it adds many calls to time.Sleep(),
  45         // which could hide different bugs.
  46         if time.Now().Second()&1 == 0 {
  47                 DebugLocksPanicMode = true
  48         }
  49 }
  50
  51 type SiteFSSuite struct {
  52         client *Client
  53         fs     CustomFileSystem
  54         kc     keepClient
  55 }
  56
  57 func (s *SiteFSSuite) SetUpTest(c *check.C) {
  58         s.client = &Client{
  59                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  60                 AuthToken: fixtureActiveToken,
  61                 Insecure:  true,
  62         }
  63         s.kc = &keepClientStub{
  64                 blocks: map[string][]byte{
  65                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  66                 },
  67                 sigkey:    fixtureBlobSigningKey,
  68                 sigttl:    fixtureBlobSigningTTL,
  69                 authToken: fixtureActiveToken,
  70         }
  71         s.fs = s.client.SiteFileSystem(s.kc)
  72 }
  73
  74 func (s *SiteFSSuite) TestHttpFileSystemInterface(c *check.C) {
  75         _, ok := s.fs.(http.FileSystem)
  76         c.Check(ok, check.Equals, true)
  77 }
  78
  79 func (s *SiteFSSuite) TestByIDEmpty(c *check.C) {
  80         f, err := s.fs.Open("/by_id")
  81         c.Assert(err, check.IsNil)
  82         fis, err := f.Readdir(-1)
  83         c.Check(err, check.IsNil)
  84         c.Check(len(fis), check.Equals, 0)
  85 }
  86
  87 func (s *SiteFSSuite) TestUpdateStorageClasses(c *check.C) {
  88         f, err := s.fs.OpenFile("/by_id/"+fixtureStorageClassesDesiredArchive+"/newfile", os.O_CREATE|os.O_RDWR, 0777)
  89         c.Assert(err, check.IsNil)
  90         _, err = f.Write([]byte("nope"))
  91         c.Assert(err, check.IsNil)
  92         err = f.Close()
  93         c.Assert(err, check.IsNil)
  94         err = s.fs.Sync()
  95         c.Assert(err, check.ErrorMatches, `.*stub does not write storage class "archive"`)
  96 }
  97
  98 func (s *SiteFSSuite) TestByUUIDAndPDH(c *check.C) {
  99         f, err := s.fs.Open("/by_id")
 100         c.Assert(err, check.IsNil)
 101         fis, err := f.Readdir(-1)
 102         c.Check(err, check.IsNil)
 103         c.Check(len(fis), check.Equals, 0)
 104
 105         err = s.fs.Mkdir("/by_id/"+fixtureFooCollection, 0755)
 106         c.Check(err, check.Equals, os.ErrExist)
 107
 108         f, err = s.fs.Open("/by_id/" + fixtureNonexistentCollection)
 109         c.Assert(err, check.Equals, os.ErrNotExist)
 110
 111         for _, path := range []string{
 112                 fixtureFooCollection,
 113                 fixtureFooCollectionPDH,
 114                 fixtureAProjectUUID + "/" + fixtureFooCollectionName,
 115         } {
 116                 f, err = s.fs.Open("/by_id/" + path)
 117                 c.Assert(err, check.IsNil)
 118                 fis, err = f.Readdir(-1)
 119                 c.Assert(err, check.IsNil)
 120                 var names []string
 121                 for _, fi := range fis {
 122                         names = append(names, fi.Name())
 123                 }
 124                 c.Check(names, check.DeepEquals, []string{"foo"})
 125         }
 126
 127         f, err = s.fs.Open("/by_id/" + fixtureAProjectUUID + "/A Subproject/baz_file")
 128         c.Assert(err, check.IsNil)
 129         fis, err = f.Readdir(-1)
 130         c.Assert(err, check.IsNil)
 131         var names []string
 132         for _, fi := range fis {
 133                 names = append(names, fi.Name())
 134         }
 135         c.Check(names, check.DeepEquals, []string{"baz"})
 136
 137         _, err = s.fs.OpenFile("/by_id/"+fixtureNonexistentCollection, os.O_RDWR|os.O_CREATE, 0755)
 138         c.Check(err, check.Equals, ErrInvalidArgument)
 139         err = s.fs.Rename("/by_id/"+fixtureFooCollection, "/by_id/beep")
 140         c.Check(err, check.Equals, ErrInvalidArgument)
 141         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/beep")
 142         c.Check(err, check.Equals, ErrInvalidArgument)
 143         _, err = s.fs.Stat("/by_id/beep")
 144         c.Check(err, check.Equals, os.ErrNotExist)
 145         err = s.fs.Rename("/by_id/"+fixtureFooCollection+"/foo", "/by_id/"+fixtureFooCollection+"/bar")
 146         c.Check(err, check.IsNil)
 147
 148         err = s.fs.Rename("/by_id", "/beep")
 149         c.Check(err, check.Equals, ErrInvalidArgument)
 150 }
 151
 152 // Copy subtree from OS src to dst path inside fs. If src is a
 153 // directory, dst must exist and be a directory.
 154 func copyFromOS(fs FileSystem, dst, src string) error {
 155         inf, err := os.Open(src)
 156         if err != nil {
 157                 return err
 158         }
 159         defer inf.Close()
 160         dirents, err := inf.Readdir(-1)
 161         if e, ok := err.(*os.PathError); ok {
 162                 if e, ok := e.Err.(syscall.Errno); ok {
 163                         if e == syscall.ENOTDIR {
 164                                 err = syscall.ENOTDIR
 165                         }
 166                 }
 167         }
 168         if err == syscall.ENOTDIR {
 169                 outf, err := fs.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_TRUNC|os.O_WRONLY, 0700)
 170                 if err != nil {
 171                         return fmt.Errorf("open %s: %s", dst, err)
 172                 }
 173                 defer outf.Close()
 174                 _, err = io.Copy(outf, inf)
 175                 if err != nil {
 176                         return fmt.Errorf("%s: copying data from %s: %s", dst, src, err)
 177                 }
 178                 err = outf.Close()
 179                 if err != nil {
 180                         return err
 181                 }
 182         } else if err != nil {
 183                 return fmt.Errorf("%s: readdir: %T %s", src, err, err)
 184         } else {
 185                 {
 186                         d, err := fs.Open(dst)
 187                         if err != nil {
 188                                 return fmt.Errorf("opendir(%s): %s", dst, err)
 189                         }
 190                         d.Close()
 191                 }
 192                 for _, ent := range dirents {
 193                         if ent.Name() == "." || ent.Name() == ".." {
 194                                 continue
 195                         }
 196                         dstname := dst + "/" + ent.Name()
 197                         if ent.IsDir() {
 198                                 err = fs.Mkdir(dstname, 0700)
 199                                 if err != nil {
 200                                         return fmt.Errorf("mkdir %s: %s", dstname, err)
 201                                 }
 202                         }
 203                         err = copyFromOS(fs, dstname, src+"/"+ent.Name())
 204                         if err != nil {
 205                                 return err
 206                         }
 207                 }
 208         }
 209         return nil
 210 }
 211
 212 func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 213         s.fs.MountProject("home", "")
 214
 215         var src1 Collection
 216         err := s.client.RequestAndDecode(&src1, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 217                 "collection": map[string]string{
 218                         "name":       "TestSnapshotSplice src1",
 219                         "owner_uuid": fixtureAProjectUUID,
 220                 },
 221         })
 222         c.Assert(err, check.IsNil)
 223         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src1.UUID, nil, nil)
 224         err = s.fs.Sync()
 225         c.Assert(err, check.IsNil)
 226         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src1", "..") // arvados.git/sdk/go
 227         c.Assert(err, check.IsNil)
 228
 229         var src2 Collection
 230         err = s.client.RequestAndDecode(&src2, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 231                 "collection": map[string]string{
 232                         "name":       "TestSnapshotSplice src2",
 233                         "owner_uuid": fixtureAProjectUUID,
 234                 },
 235         })
 236         c.Assert(err, check.IsNil)
 237         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+src2.UUID, nil, nil)
 238         err = s.fs.Sync()
 239         c.Assert(err, check.IsNil)
 240         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice src2", "..") // arvados.git/sdk/go
 241         c.Assert(err, check.IsNil)
 242
 243         var dst Collection
 244         err = s.client.RequestAndDecode(&dst, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 245                 "collection": map[string]string{
 246                         "name":       "TestSnapshotSplice dst",
 247                         "owner_uuid": fixtureAProjectUUID,
 248                 },
 249         })
 250         c.Assert(err, check.IsNil)
 251         defer s.client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+dst.UUID, nil, nil)
 252         err = s.fs.Sync()
 253         c.Assert(err, check.IsNil)
 254         err = copyFromOS(s.fs, "/home/A Project/TestSnapshotSplice dst", "..") // arvados.git/sdk/go
 255         c.Assert(err, check.IsNil)
 256
 257         snap1, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice src1/ctxlog")
 258         c.Assert(err, check.IsNil)
 259         err = Splice(s.fs, "/home/A Project/TestSnapshotSplice dst/ctxlog-copy", snap1)
 260         c.Assert(err, check.IsNil)
 261         err = Splice(s.fs, "/home/A Project/TestSnapshotSplice dst/ctxlog-copy2", snap1)
 262         c.Assert(err, check.IsNil)
 263
 264         snap2, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice dst/ctxlog-copy")
 265         c.Assert(err, check.IsNil)
 266         err = Splice(s.fs, "/home/A Project/TestSnapshotSplice dst/ctxlog-copy-copy", snap2)
 267         c.Assert(err, check.IsNil)
 268
 269         snapDst, err := Snapshot(s.fs, "/home/A Project/TestSnapshotSplice dst")
 270         c.Assert(err, check.IsNil)
 271         err = Splice(s.fs, "/home/A Project/TestSnapshotSplice dst", snapDst)
 272         c.Assert(err, check.IsNil)
 273         err = Splice(s.fs, "/home/A Project/TestSnapshotSplice dst/copy1", snapDst)
 274         c.Assert(err, check.IsNil)
 275         err = Splice(s.fs, "/home/A Project/TestSnapshotSplice dst/copy2", snapDst)
 276         c.Assert(err, check.IsNil)
 277         err = s.fs.RemoveAll("/home/A Project/TestSnapshotSplice dst/arvados")
 278         c.Assert(err, check.IsNil)
 279         _, err = s.fs.Open("/home/A Project/TestSnapshotSplice dst/arvados/fs_site_test.go")
 280         c.Assert(err, check.Equals, os.ErrNotExist)
 281         f, err := s.fs.Open("/home/A Project/TestSnapshotSplice dst/copy2/arvados/fs_site_test.go")
 282         c.Assert(err, check.IsNil)
 283         defer f.Close()
 284         buf, err := ioutil.ReadAll(f)
 285         c.Check(err, check.IsNil)
 286         c.Check(string(buf), check.Not(check.Equals), "")
 287         err = f.Close()
 288         c.Assert(err, check.IsNil)
 289 }