sdk/go/arvados/collection_fs_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "math/rand"
  12         "net/http"
  13         "os"
  14         "sync"
  15         "testing"
  16
  17         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 var _ = check.Suite(&CollectionFSSuite{})
  22
  23 type keepClientStub struct {
  24         blocks map[string][]byte
  25 }
  26
  27 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
  28         buf := kcs.blocks[locator[:32]]
  29         if buf == nil {
  30                 return 0, os.ErrNotExist
  31         }
  32         return copy(p, buf[off:]), nil
  33 }
  34
  35 type CollectionFSSuite struct {
  36         client *Client
  37         coll   Collection
  38         fs     CollectionFileSystem
  39         kc     keepClient
  40 }
  41
  42 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
  43         s.client = NewClientFromEnv()
  44         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
  45         c.Assert(err, check.IsNil)
  46         s.kc = &keepClientStub{
  47                 blocks: map[string][]byte{
  48                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  49                 }}
  50         s.fs = s.coll.FileSystem(s.client, s.kc)
  51 }
  52
  53 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
  54         _, ok := s.fs.(http.FileSystem)
  55         c.Check(ok, check.Equals, true)
  56 }
  57
  58 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
  59         f, err := s.fs.Open("/dir1")
  60         c.Assert(err, check.IsNil)
  61
  62         st, err := f.Stat()
  63         c.Assert(err, check.IsNil)
  64         c.Check(st.Size(), check.Equals, int64(2))
  65         c.Check(st.IsDir(), check.Equals, true)
  66
  67         fis, err := f.Readdir(0)
  68         c.Check(err, check.IsNil)
  69         c.Check(len(fis), check.Equals, 2)
  70         if len(fis) > 0 {
  71                 c.Check(fis[0].Size(), check.Equals, int64(3))
  72         }
  73 }
  74
  75 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
  76         f, err := s.fs.Open("./dir1")
  77         c.Assert(err, check.IsNil)
  78
  79         fis, err := f.Readdir(1)
  80         c.Check(err, check.IsNil)
  81         c.Check(len(fis), check.Equals, 1)
  82         if len(fis) > 0 {
  83                 c.Check(fis[0].Size(), check.Equals, int64(3))
  84         }
  85
  86         fis, err = f.Readdir(1)
  87         c.Check(err, check.IsNil)
  88         c.Check(len(fis), check.Equals, 1)
  89         if len(fis) > 0 {
  90                 c.Check(fis[0].Size(), check.Equals, int64(3))
  91         }
  92
  93         fis, err = f.Readdir(1)
  94         c.Check(len(fis), check.Equals, 0)
  95         c.Check(err, check.NotNil)
  96         c.Check(err, check.Equals, io.EOF)
  97
  98         f, err = s.fs.Open("dir1")
  99         c.Assert(err, check.IsNil)
 100         fis, err = f.Readdir(1)
 101         c.Check(len(fis), check.Equals, 1)
 102         c.Assert(err, check.IsNil)
 103         fis, err = f.Readdir(2)
 104         c.Check(len(fis), check.Equals, 1)
 105         c.Assert(err, check.IsNil)
 106         fis, err = f.Readdir(2)
 107         c.Check(len(fis), check.Equals, 0)
 108         c.Assert(err, check.Equals, io.EOF)
 109 }
 110
 111 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
 112         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
 113                 f, err := s.fs.Open(path)
 114                 c.Assert(err, check.IsNil)
 115
 116                 st, err := f.Stat()
 117                 c.Assert(err, check.IsNil)
 118                 c.Check(st.Size(), check.Equals, int64(1))
 119                 c.Check(st.IsDir(), check.Equals, true)
 120         }
 121         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
 122                 c.Logf("%q", path)
 123                 f, err := s.fs.Open(path)
 124                 c.Assert(err, check.IsNil)
 125
 126                 st, err := f.Stat()
 127                 c.Assert(err, check.IsNil)
 128                 c.Check(st.Size(), check.Equals, int64(2))
 129                 c.Check(st.IsDir(), check.Equals, true)
 130         }
 131 }
 132
 133 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
 134         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
 135                 f, err := s.fs.Open(path)
 136                 c.Assert(f, check.IsNil)
 137                 c.Assert(err, check.NotNil)
 138                 c.Assert(os.IsNotExist(err), check.Equals, true)
 139         }
 140 }
 141
 142 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
 143         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
 144         c.Assert(err, check.IsNil)
 145         st, err := f.Stat()
 146         c.Assert(err, check.IsNil)
 147         c.Check(st.Size(), check.Equals, int64(3))
 148         n, err := f.Write([]byte("bar"))
 149         c.Check(n, check.Equals, 0)
 150         c.Check(err, check.Equals, ErrReadOnlyFile)
 151 }
 152
 153 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
 154         f, err := s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE, 0)
 155         c.Assert(err, check.IsNil)
 156         st, err := f.Stat()
 157         c.Assert(err, check.IsNil)
 158         c.Check(st.Size(), check.Equals, int64(0))
 159
 160         n, err := f.Write([]byte("bar"))
 161         c.Check(n, check.Equals, 3)
 162         c.Check(err, check.IsNil)
 163
 164         c.Check(f.Close(), check.IsNil)
 165
 166         f, err = s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 167         c.Check(f, check.IsNil)
 168         c.Assert(err, check.NotNil)
 169
 170         f, err = s.fs.OpenFile("/newfile", os.O_RDWR, 0)
 171         c.Assert(err, check.IsNil)
 172         st, err = f.Stat()
 173         c.Assert(err, check.IsNil)
 174         c.Check(st.Size(), check.Equals, int64(3))
 175
 176         c.Check(f.Close(), check.IsNil)
 177
 178         // TODO: serialize to Collection, confirm manifest contents,
 179         // make new FileSystem, confirm file contents.
 180 }
 181
 182 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
 183         maxBlockSize = 8
 184         defer func() { maxBlockSize = 2 << 26 }()
 185
 186         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 187         c.Assert(err, check.IsNil)
 188         defer f.Close()
 189         st, err := f.Stat()
 190         c.Assert(err, check.IsNil)
 191         c.Check(st.Size(), check.Equals, int64(3))
 192
 193         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 194         c.Assert(err, check.IsNil)
 195         defer f2.Close()
 196
 197         buf := make([]byte, 64)
 198         n, err := f.Read(buf)
 199         c.Check(n, check.Equals, 3)
 200         c.Check(err, check.Equals, io.EOF)
 201         c.Check(string(buf[:3]), check.DeepEquals, "foo")
 202
 203         pos, err := f.Seek(-2, os.SEEK_CUR)
 204         c.Check(pos, check.Equals, int64(1))
 205         c.Check(err, check.IsNil)
 206
 207         // Split a storedExtent in two, and insert a memExtent
 208         n, err = f.Write([]byte("*"))
 209         c.Check(n, check.Equals, 1)
 210         c.Check(err, check.IsNil)
 211
 212         pos, err = f.Seek(0, os.SEEK_CUR)
 213         c.Check(pos, check.Equals, int64(2))
 214         c.Check(err, check.IsNil)
 215
 216         pos, err = f.Seek(0, os.SEEK_SET)
 217         c.Check(pos, check.Equals, int64(0))
 218         c.Check(err, check.IsNil)
 219
 220         rbuf, err := ioutil.ReadAll(f)
 221         c.Check(len(rbuf), check.Equals, 3)
 222         c.Check(err, check.IsNil)
 223         c.Check(string(rbuf), check.Equals, "f*o")
 224
 225         // Write multiple blocks in one call
 226         f.Seek(1, os.SEEK_SET)
 227         n, err = f.Write([]byte("0123456789abcdefg"))
 228         c.Check(n, check.Equals, 17)
 229         c.Check(err, check.IsNil)
 230         pos, err = f.Seek(0, os.SEEK_CUR)
 231         c.Check(pos, check.Equals, int64(18))
 232         pos, err = f.Seek(-18, os.SEEK_CUR)
 233         c.Check(err, check.IsNil)
 234         n, err = io.ReadFull(f, buf)
 235         c.Check(n, check.Equals, 18)
 236         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
 237         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
 238
 239         buf2, err := ioutil.ReadAll(f2)
 240         c.Check(err, check.IsNil)
 241         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 242
 243         // truncate to current size
 244         err = f.Truncate(18)
 245         f2.Seek(0, os.SEEK_SET)
 246         buf2, err = ioutil.ReadAll(f2)
 247         c.Check(err, check.IsNil)
 248         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 249
 250         // shrink to zero some data
 251         f.Truncate(15)
 252         f2.Seek(0, os.SEEK_SET)
 253         buf2, err = ioutil.ReadAll(f2)
 254         c.Check(err, check.IsNil)
 255         c.Check(string(buf2), check.Equals, "f0123456789abcd")
 256
 257         // grow to partial block/extent
 258         f.Truncate(20)
 259         f2.Seek(0, os.SEEK_SET)
 260         buf2, err = ioutil.ReadAll(f2)
 261         c.Check(err, check.IsNil)
 262         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
 263
 264         f.Truncate(0)
 265         f2.Write([]byte("12345678abcdefghijkl"))
 266
 267         // grow to block/extent boundary
 268         f.Truncate(64)
 269         f2.Seek(0, os.SEEK_SET)
 270         buf2, err = ioutil.ReadAll(f2)
 271         c.Check(err, check.IsNil)
 272         c.Check(len(buf2), check.Equals, 64)
 273         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 8)
 274
 275         // shrink to block/extent boundary
 276         err = f.Truncate(32)
 277         f2.Seek(0, os.SEEK_SET)
 278         buf2, err = ioutil.ReadAll(f2)
 279         c.Check(err, check.IsNil)
 280         c.Check(len(buf2), check.Equals, 32)
 281         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 4)
 282
 283         // shrink to partial block/extent
 284         err = f.Truncate(15)
 285         f2.Seek(0, os.SEEK_SET)
 286         buf2, err = ioutil.ReadAll(f2)
 287         c.Check(err, check.IsNil)
 288         c.Check(string(buf2), check.Equals, "12345678abcdefg")
 289         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 2)
 290
 291         // Truncate to size=3 while f2's ptr is at 15
 292         err = f.Truncate(3)
 293         c.Check(err, check.IsNil)
 294         buf2, err = ioutil.ReadAll(f2)
 295         c.Check(err, check.IsNil)
 296         c.Check(string(buf2), check.Equals, "")
 297         f2.Seek(0, os.SEEK_SET)
 298         buf2, err = ioutil.ReadAll(f2)
 299         c.Check(err, check.IsNil)
 300         c.Check(string(buf2), check.Equals, "123")
 301         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 1)
 302 }
 303
 304 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
 305         maxBlockSize = 8
 306         defer func() { maxBlockSize = 2 << 26 }()
 307
 308         var wg sync.WaitGroup
 309         for n := 0; n < 128; n++ {
 310                 wg.Add(1)
 311                 go func() {
 312                         defer wg.Done()
 313                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 314                         c.Assert(err, check.IsNil)
 315                         defer f.Close()
 316                         for i := 0; i < 6502; i++ {
 317                                 switch rand.Int() & 3 {
 318                                 case 0:
 319                                         f.Truncate(int64(rand.Intn(64)))
 320                                 case 1:
 321                                         f.Seek(int64(rand.Intn(64)), os.SEEK_SET)
 322                                 case 2:
 323                                         _, err := f.Write([]byte("beep boop"))
 324                                         c.Check(err, check.IsNil)
 325                                 case 3:
 326                                         _, err := ioutil.ReadAll(f)
 327                                         c.Check(err, check.IsNil)
 328                                 }
 329                         }
 330                 }()
 331         }
 332         wg.Wait()
 333
 334         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 335         c.Assert(err, check.IsNil)
 336         defer f.Close()
 337         buf, err := ioutil.ReadAll(f)
 338         c.Check(err, check.IsNil)
 339         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
 340 }
 341
 342 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
 343         maxBlockSize = 8
 344         defer func() { maxBlockSize = 2 << 26 }()
 345
 346         var wg sync.WaitGroup
 347         for n := 0; n < 128; n++ {
 348                 wg.Add(1)
 349                 go func(n int) {
 350                         defer wg.Done()
 351                         expect := make([]byte, 0, 64)
 352                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
 353                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 354                         c.Assert(err, check.IsNil)
 355                         defer f.Close()
 356                         for i := 0; i < 6502; i++ {
 357                                 trunc := rand.Intn(65)
 358                                 woff := rand.Intn(trunc + 1)
 359                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
 360                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
 361                                         buf[i] = 0
 362                                 }
 363                                 expect = expect[:trunc]
 364                                 if trunc < woff+len(wbytes) {
 365                                         expect = expect[:woff+len(wbytes)]
 366                                 }
 367                                 copy(expect[woff:], wbytes)
 368                                 f.Truncate(int64(trunc))
 369                                 pos, err := f.Seek(int64(woff), os.SEEK_SET)
 370                                 c.Check(pos, check.Equals, int64(woff))
 371                                 c.Check(err, check.IsNil)
 372                                 n, err := f.Write(wbytes)
 373                                 c.Check(n, check.Equals, len(wbytes))
 374                                 c.Check(err, check.IsNil)
 375                                 pos, err = f.Seek(0, os.SEEK_SET)
 376                                 c.Check(pos, check.Equals, int64(0))
 377                                 c.Check(err, check.IsNil)
 378                                 buf, err := ioutil.ReadAll(f)
 379                                 c.Check(string(buf), check.Equals, string(expect))
 380                                 c.Check(err, check.IsNil)
 381                         }
 382                 }(n)
 383         }
 384         wg.Wait()
 385
 386         root, err := s.fs.Open("/")
 387         c.Assert(err, check.IsNil)
 388         defer root.Close()
 389         fi, err := root.Readdir(-1)
 390         c.Check(err, check.IsNil)
 391         c.Logf("Readdir(): %#v", fi)
 392 }
 393
 394 // Gocheck boilerplate
 395 func Test(t *testing.T) {
 396         check.TestingT(t)
 397 }