sdk/go/arvados/collection_fs_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "fmt"
   9         "io"
  10         "io/ioutil"
  11         "math/rand"
  12         "net/http"
  13         "os"
  14         "regexp"
  15         "sync"
  16         "testing"
  17
  18         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  19         check "gopkg.in/check.v1"
  20 )
  21
  22 var _ = check.Suite(&CollectionFSSuite{})
  23
  24 type keepClientStub struct {
  25         blocks map[string][]byte
  26 }
  27
  28 func (kcs *keepClientStub) ReadAt(locator string, p []byte, off int) (int, error) {
  29         buf := kcs.blocks[locator[:32]]
  30         if buf == nil {
  31                 return 0, os.ErrNotExist
  32         }
  33         return copy(p, buf[off:]), nil
  34 }
  35
  36 type CollectionFSSuite struct {
  37         client *Client
  38         coll   Collection
  39         fs     CollectionFileSystem
  40         kc     keepClient
  41 }
  42
  43 func (s *CollectionFSSuite) SetUpTest(c *check.C) {
  44         s.client = NewClientFromEnv()
  45         err := s.client.RequestAndDecode(&s.coll, "GET", "arvados/v1/collections/"+arvadostest.FooAndBarFilesInDirUUID, nil, nil)
  46         c.Assert(err, check.IsNil)
  47         s.kc = &keepClientStub{
  48                 blocks: map[string][]byte{
  49                         "3858f62230ac3c915f300c664312c63f": []byte("foobar"),
  50                 }}
  51         s.fs = s.coll.FileSystem(s.client, s.kc)
  52 }
  53
  54 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
  55         _, ok := s.fs.(http.FileSystem)
  56         c.Check(ok, check.Equals, true)
  57 }
  58
  59 func (s *CollectionFSSuite) TestReaddirFull(c *check.C) {
  60         f, err := s.fs.Open("/dir1")
  61         c.Assert(err, check.IsNil)
  62
  63         st, err := f.Stat()
  64         c.Assert(err, check.IsNil)
  65         c.Check(st.Size(), check.Equals, int64(2))
  66         c.Check(st.IsDir(), check.Equals, true)
  67
  68         fis, err := f.Readdir(0)
  69         c.Check(err, check.IsNil)
  70         c.Check(len(fis), check.Equals, 2)
  71         if len(fis) > 0 {
  72                 c.Check(fis[0].Size(), check.Equals, int64(3))
  73         }
  74 }
  75
  76 func (s *CollectionFSSuite) TestReaddirLimited(c *check.C) {
  77         f, err := s.fs.Open("./dir1")
  78         c.Assert(err, check.IsNil)
  79
  80         fis, err := f.Readdir(1)
  81         c.Check(err, check.IsNil)
  82         c.Check(len(fis), check.Equals, 1)
  83         if len(fis) > 0 {
  84                 c.Check(fis[0].Size(), check.Equals, int64(3))
  85         }
  86
  87         fis, err = f.Readdir(1)
  88         c.Check(err, check.IsNil)
  89         c.Check(len(fis), check.Equals, 1)
  90         if len(fis) > 0 {
  91                 c.Check(fis[0].Size(), check.Equals, int64(3))
  92         }
  93
  94         fis, err = f.Readdir(1)
  95         c.Check(len(fis), check.Equals, 0)
  96         c.Check(err, check.NotNil)
  97         c.Check(err, check.Equals, io.EOF)
  98
  99         f, err = s.fs.Open("dir1")
 100         c.Assert(err, check.IsNil)
 101         fis, err = f.Readdir(1)
 102         c.Check(len(fis), check.Equals, 1)
 103         c.Assert(err, check.IsNil)
 104         fis, err = f.Readdir(2)
 105         c.Check(len(fis), check.Equals, 1)
 106         c.Assert(err, check.IsNil)
 107         fis, err = f.Readdir(2)
 108         c.Check(len(fis), check.Equals, 0)
 109         c.Assert(err, check.Equals, io.EOF)
 110 }
 111
 112 func (s *CollectionFSSuite) TestPathMunge(c *check.C) {
 113         for _, path := range []string{".", "/", "./", "///", "/../", "/./.."} {
 114                 f, err := s.fs.Open(path)
 115                 c.Assert(err, check.IsNil)
 116
 117                 st, err := f.Stat()
 118                 c.Assert(err, check.IsNil)
 119                 c.Check(st.Size(), check.Equals, int64(1))
 120                 c.Check(st.IsDir(), check.Equals, true)
 121         }
 122         for _, path := range []string{"/dir1", "dir1", "./dir1", "///dir1//.//", "../dir1/../dir1/"} {
 123                 c.Logf("%q", path)
 124                 f, err := s.fs.Open(path)
 125                 c.Assert(err, check.IsNil)
 126
 127                 st, err := f.Stat()
 128                 c.Assert(err, check.IsNil)
 129                 c.Check(st.Size(), check.Equals, int64(2))
 130                 c.Check(st.IsDir(), check.Equals, true)
 131         }
 132 }
 133
 134 func (s *CollectionFSSuite) TestNotExist(c *check.C) {
 135         for _, path := range []string{"/no", "no", "./no", "n/o", "/n/o"} {
 136                 f, err := s.fs.Open(path)
 137                 c.Assert(f, check.IsNil)
 138                 c.Assert(err, check.NotNil)
 139                 c.Assert(os.IsNotExist(err), check.Equals, true)
 140         }
 141 }
 142
 143 func (s *CollectionFSSuite) TestReadOnlyFile(c *check.C) {
 144         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDONLY, 0)
 145         c.Assert(err, check.IsNil)
 146         st, err := f.Stat()
 147         c.Assert(err, check.IsNil)
 148         c.Check(st.Size(), check.Equals, int64(3))
 149         n, err := f.Write([]byte("bar"))
 150         c.Check(n, check.Equals, 0)
 151         c.Check(err, check.Equals, ErrReadOnlyFile)
 152 }
 153
 154 func (s *CollectionFSSuite) TestCreateFile(c *check.C) {
 155         f, err := s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE, 0)
 156         c.Assert(err, check.IsNil)
 157         st, err := f.Stat()
 158         c.Assert(err, check.IsNil)
 159         c.Check(st.Size(), check.Equals, int64(0))
 160
 161         n, err := f.Write([]byte("bar"))
 162         c.Check(n, check.Equals, 3)
 163         c.Check(err, check.IsNil)
 164
 165         c.Check(f.Close(), check.IsNil)
 166
 167         f, err = s.fs.OpenFile("/newfile", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 168         c.Check(f, check.IsNil)
 169         c.Assert(err, check.NotNil)
 170
 171         f, err = s.fs.OpenFile("/newfile", os.O_RDWR, 0)
 172         c.Assert(err, check.IsNil)
 173         st, err = f.Stat()
 174         c.Assert(err, check.IsNil)
 175         c.Check(st.Size(), check.Equals, int64(3))
 176
 177         c.Check(f.Close(), check.IsNil)
 178
 179         // TODO: serialize to Collection, confirm manifest contents,
 180         // make new FileSystem, confirm file contents.
 181 }
 182
 183 func (s *CollectionFSSuite) TestReadWriteFile(c *check.C) {
 184         maxBlockSize = 8
 185         defer func() { maxBlockSize = 2 << 26 }()
 186
 187         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 188         c.Assert(err, check.IsNil)
 189         defer f.Close()
 190         st, err := f.Stat()
 191         c.Assert(err, check.IsNil)
 192         c.Check(st.Size(), check.Equals, int64(3))
 193
 194         f2, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 195         c.Assert(err, check.IsNil)
 196         defer f2.Close()
 197
 198         buf := make([]byte, 64)
 199         n, err := f.Read(buf)
 200         c.Check(n, check.Equals, 3)
 201         c.Check(err, check.Equals, io.EOF)
 202         c.Check(string(buf[:3]), check.DeepEquals, "foo")
 203
 204         pos, err := f.Seek(-2, os.SEEK_CUR)
 205         c.Check(pos, check.Equals, int64(1))
 206         c.Check(err, check.IsNil)
 207
 208         // Split a storedExtent in two, and insert a memExtent
 209         n, err = f.Write([]byte("*"))
 210         c.Check(n, check.Equals, 1)
 211         c.Check(err, check.IsNil)
 212
 213         pos, err = f.Seek(0, os.SEEK_CUR)
 214         c.Check(pos, check.Equals, int64(2))
 215         c.Check(err, check.IsNil)
 216
 217         pos, err = f.Seek(0, os.SEEK_SET)
 218         c.Check(pos, check.Equals, int64(0))
 219         c.Check(err, check.IsNil)
 220
 221         rbuf, err := ioutil.ReadAll(f)
 222         c.Check(len(rbuf), check.Equals, 3)
 223         c.Check(err, check.IsNil)
 224         c.Check(string(rbuf), check.Equals, "f*o")
 225
 226         // Write multiple blocks in one call
 227         f.Seek(1, os.SEEK_SET)
 228         n, err = f.Write([]byte("0123456789abcdefg"))
 229         c.Check(n, check.Equals, 17)
 230         c.Check(err, check.IsNil)
 231         pos, err = f.Seek(0, os.SEEK_CUR)
 232         c.Check(pos, check.Equals, int64(18))
 233         pos, err = f.Seek(-18, os.SEEK_CUR)
 234         c.Check(err, check.IsNil)
 235         n, err = io.ReadFull(f, buf)
 236         c.Check(n, check.Equals, 18)
 237         c.Check(err, check.Equals, io.ErrUnexpectedEOF)
 238         c.Check(string(buf[:n]), check.Equals, "f0123456789abcdefg")
 239
 240         buf2, err := ioutil.ReadAll(f2)
 241         c.Check(err, check.IsNil)
 242         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 243
 244         // truncate to current size
 245         err = f.Truncate(18)
 246         f2.Seek(0, os.SEEK_SET)
 247         buf2, err = ioutil.ReadAll(f2)
 248         c.Check(err, check.IsNil)
 249         c.Check(string(buf2), check.Equals, "f0123456789abcdefg")
 250
 251         // shrink to zero some data
 252         f.Truncate(15)
 253         f2.Seek(0, os.SEEK_SET)
 254         buf2, err = ioutil.ReadAll(f2)
 255         c.Check(err, check.IsNil)
 256         c.Check(string(buf2), check.Equals, "f0123456789abcd")
 257
 258         // grow to partial block/extent
 259         f.Truncate(20)
 260         f2.Seek(0, os.SEEK_SET)
 261         buf2, err = ioutil.ReadAll(f2)
 262         c.Check(err, check.IsNil)
 263         c.Check(string(buf2), check.Equals, "f0123456789abcd\x00\x00\x00\x00\x00")
 264
 265         f.Truncate(0)
 266         f2.Write([]byte("12345678abcdefghijkl"))
 267
 268         // grow to block/extent boundary
 269         f.Truncate(64)
 270         f2.Seek(0, os.SEEK_SET)
 271         buf2, err = ioutil.ReadAll(f2)
 272         c.Check(err, check.IsNil)
 273         c.Check(len(buf2), check.Equals, 64)
 274         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 8)
 275
 276         // shrink to block/extent boundary
 277         err = f.Truncate(32)
 278         f2.Seek(0, os.SEEK_SET)
 279         buf2, err = ioutil.ReadAll(f2)
 280         c.Check(err, check.IsNil)
 281         c.Check(len(buf2), check.Equals, 32)
 282         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 4)
 283
 284         // shrink to partial block/extent
 285         err = f.Truncate(15)
 286         f2.Seek(0, os.SEEK_SET)
 287         buf2, err = ioutil.ReadAll(f2)
 288         c.Check(err, check.IsNil)
 289         c.Check(string(buf2), check.Equals, "12345678abcdefg")
 290         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 2)
 291
 292         // Truncate to size=3 while f2's ptr is at 15
 293         err = f.Truncate(3)
 294         c.Check(err, check.IsNil)
 295         buf2, err = ioutil.ReadAll(f2)
 296         c.Check(err, check.IsNil)
 297         c.Check(string(buf2), check.Equals, "")
 298         f2.Seek(0, os.SEEK_SET)
 299         buf2, err = ioutil.ReadAll(f2)
 300         c.Check(err, check.IsNil)
 301         c.Check(string(buf2), check.Equals, "123")
 302         c.Check(len(f.(*file).inode.(*filenode).extents), check.Equals, 1)
 303
 304         m, err := s.fs.MarshalManifest(".")
 305         c.Check(err, check.IsNil)
 306         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 307         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 202cb962ac59075b964b07152d234b70+3 3:3:bar 6:3:foo\n")
 308 }
 309
 310 func (s *CollectionFSSuite) TestMarshalSmallBlocks(c *check.C) {
 311         maxBlockSize = 8
 312         defer func() { maxBlockSize = 2 << 26 }()
 313
 314         s.fs = (&Collection{}).FileSystem(s.client, s.kc)
 315         for _, name := range []string{"foo", "bar", "baz"} {
 316                 f, err := s.fs.OpenFile(name, os.O_WRONLY|os.O_CREATE, 0)
 317                 c.Assert(err, check.IsNil)
 318                 f.Write([]byte(name))
 319                 f.Close()
 320         }
 321
 322         m, err := s.fs.MarshalManifest(".")
 323         c.Check(err, check.IsNil)
 324         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 325         c.Check(m, check.Equals, ". c3c23db5285662ef7172373df0003206+6 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:bar 3:3:baz 6:3:foo\n")
 326 }
 327
 328 func (s *CollectionFSSuite) TestMkdir(c *check.C) {
 329         err := s.fs.Mkdir("foo/bar", 0755)
 330         c.Check(err, check.Equals, os.ErrNotExist)
 331
 332         f, err := s.fs.OpenFile("foo/bar", os.O_CREATE, 0)
 333         c.Check(err, check.Equals, os.ErrNotExist)
 334
 335         err = s.fs.Mkdir("foo", 0755)
 336         c.Check(err, check.IsNil)
 337
 338         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_WRONLY, 0)
 339         c.Check(err, check.IsNil)
 340         if err == nil {
 341                 defer f.Close()
 342                 f.Write([]byte("foo"))
 343         }
 344
 345         // mkdir fails if a file already exists with that name
 346         err = s.fs.Mkdir("foo/bar", 0755)
 347         c.Check(err, check.NotNil)
 348
 349         err = s.fs.Remove("foo/bar")
 350         c.Check(err, check.IsNil)
 351
 352         // mkdir succeds after the file is deleted
 353         err = s.fs.Mkdir("foo/bar", 0755)
 354         c.Check(err, check.IsNil)
 355
 356         // creating a file in a nonexistent subdir should still fail
 357         f, err = s.fs.OpenFile("foo/bar/baz/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 358         c.Check(err, check.Equals, os.ErrNotExist)
 359
 360         f, err = s.fs.OpenFile("foo/bar/foo.txt", os.O_CREATE|os.O_WRONLY, 0)
 361         c.Check(err, check.IsNil)
 362         if err == nil {
 363                 defer f.Close()
 364                 f.Write([]byte("foo"))
 365         }
 366
 367         // creating foo/bar as a regular file should fail
 368         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, 0)
 369         c.Check(err, check.NotNil)
 370
 371         // creating foo/bar as a directory should fail
 372         f, err = s.fs.OpenFile("foo/bar", os.O_CREATE|os.O_EXCL, os.ModeDir)
 373         c.Check(err, check.NotNil)
 374         err = s.fs.Mkdir("foo/bar")
 375         c.Check(err, check.NotNil)
 376
 377         m, err := s.fs.MarshalManifest(".")
 378         c.Check(err, check.IsNil)
 379         m = regexp.MustCompile(`\+A[^\+ ]+`).ReplaceAllLiteralString(m, "")
 380         c.Check(m, check.Equals, "./dir1 3858f62230ac3c915f300c664312c63f+6 3:3:bar 0:3:foo\n./foo/bar acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
 381 }
 382
 383 func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
 384         maxBlockSize = 8
 385         defer func() { maxBlockSize = 2 << 26 }()
 386
 387         var wg sync.WaitGroup
 388         for n := 0; n < 128; n++ {
 389                 wg.Add(1)
 390                 go func() {
 391                         defer wg.Done()
 392                         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 393                         c.Assert(err, check.IsNil)
 394                         defer f.Close()
 395                         for i := 0; i < 6502; i++ {
 396                                 switch rand.Int() & 3 {
 397                                 case 0:
 398                                         f.Truncate(int64(rand.Intn(64)))
 399                                 case 1:
 400                                         f.Seek(int64(rand.Intn(64)), os.SEEK_SET)
 401                                 case 2:
 402                                         _, err := f.Write([]byte("beep boop"))
 403                                         c.Check(err, check.IsNil)
 404                                 case 3:
 405                                         _, err := ioutil.ReadAll(f)
 406                                         c.Check(err, check.IsNil)
 407                                 }
 408                         }
 409                 }()
 410         }
 411         wg.Wait()
 412
 413         f, err := s.fs.OpenFile("/dir1/foo", os.O_RDWR, 0)
 414         c.Assert(err, check.IsNil)
 415         defer f.Close()
 416         buf, err := ioutil.ReadAll(f)
 417         c.Check(err, check.IsNil)
 418         c.Logf("after lots of random r/w/seek/trunc, buf is %q", buf)
 419 }
 420
 421 func (s *CollectionFSSuite) TestRandomWrites(c *check.C) {
 422         maxBlockSize = 8
 423         defer func() { maxBlockSize = 2 << 26 }()
 424
 425         var wg sync.WaitGroup
 426         for n := 0; n < 128; n++ {
 427                 wg.Add(1)
 428                 go func(n int) {
 429                         defer wg.Done()
 430                         expect := make([]byte, 0, 64)
 431                         wbytes := []byte("there's no simple explanation for anything important that any of us do")
 432                         f, err := s.fs.OpenFile(fmt.Sprintf("random-%d", n), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0)
 433                         c.Assert(err, check.IsNil)
 434                         defer f.Close()
 435                         for i := 0; i < 6502; i++ {
 436                                 trunc := rand.Intn(65)
 437                                 woff := rand.Intn(trunc + 1)
 438                                 wbytes = wbytes[:rand.Intn(64-woff+1)]
 439                                 for buf, i := expect[:cap(expect)], len(expect); i < trunc; i++ {
 440                                         buf[i] = 0
 441                                 }
 442                                 expect = expect[:trunc]
 443                                 if trunc < woff+len(wbytes) {
 444                                         expect = expect[:woff+len(wbytes)]
 445                                 }
 446                                 copy(expect[woff:], wbytes)
 447                                 f.Truncate(int64(trunc))
 448                                 pos, err := f.Seek(int64(woff), os.SEEK_SET)
 449                                 c.Check(pos, check.Equals, int64(woff))
 450                                 c.Check(err, check.IsNil)
 451                                 n, err := f.Write(wbytes)
 452                                 c.Check(n, check.Equals, len(wbytes))
 453                                 c.Check(err, check.IsNil)
 454                                 pos, err = f.Seek(0, os.SEEK_SET)
 455                                 c.Check(pos, check.Equals, int64(0))
 456                                 c.Check(err, check.IsNil)
 457                                 buf, err := ioutil.ReadAll(f)
 458                                 c.Check(string(buf), check.Equals, string(expect))
 459                                 c.Check(err, check.IsNil)
 460                         }
 461                 }(n)
 462         }
 463         wg.Wait()
 464
 465         root, err := s.fs.Open("/")
 466         c.Assert(err, check.IsNil)
 467         defer root.Close()
 468         fi, err := root.Readdir(-1)
 469         c.Check(err, check.IsNil)
 470         c.Logf("Readdir(): %#v", fi)
 471
 472         m, err := s.fs.MarshalManifest(".")
 473         c.Check(err, check.IsNil)
 474         c.Logf("%s", m)
 475 }
 476
 477 // Gocheck boilerplate
 478 func Test(t *testing.T) {
 479         check.TestingT(t)
 480 }