X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/28f892a37c81486a3fc353fdb6c875e708e0f7c9..45ebdd1005f12c3c18355ab511e7a2e7f623358a:/sdk/go/arvados/fs_collection_test.go

diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index b221aaa083..a29371b76c 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -124,6 +124,38 @@ func (s *CollectionFSSuite) SetUpTest(c *check.C) {
 	c.Assert(err, check.IsNil)
 }
 
+func (s *CollectionFSSuite) TestSyncNonCanonicalManifest(c *check.C) {
+	var coll Collection
+	err := s.client.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+fixtureFooAndBarFilesInDirUUID, nil, nil)
+	c.Assert(err, check.IsNil)
+	mtxt := strings.Replace(coll.ManifestText, "3:3:bar 0:3:foo", "0:3:foo 3:3:bar", -1)
+	c.Assert(mtxt, check.Not(check.Equals), coll.ManifestText)
+	err = s.client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+		"collection": map[string]interface{}{
+			"manifest_text": mtxt}})
+	c.Assert(err, check.IsNil)
+	// In order for the rest of the test to work as intended, the API server
+	// needs to retain the file ordering we set manually. We check that here.
+	// We can't check `mtxt == coll.ManifestText` because the API server
+	// might've returned new block signatures if the GET and POST happened in
+	// different seconds.
+	expectPattern := `\./dir1 \S+ 0:3:foo 3:3:bar\n`
+	c.Assert(coll.ManifestText, check.Matches, expectPattern)
+
+	fs, err := coll.FileSystem(s.client, s.kc)
+	c.Assert(err, check.IsNil)
+	err = fs.Sync()
+	c.Check(err, check.IsNil)
+
+	// fs had no local changes, so Sync should not have saved
+	// anything back to the API/database. (If it did, we would see
+	// the manifest rewritten in canonical order.)
+	var saved Collection
+	err = s.client.RequestAndDecode(&saved, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+	c.Assert(err, check.IsNil)
+	c.Check(saved.ManifestText, check.Matches, expectPattern)
+}
+
 func (s *CollectionFSSuite) TestHttpFileSystemInterface(c *check.C) {
 	_, ok := s.fs.(http.FileSystem)
 	c.Check(ok, check.Equals, true)
@@ -1209,11 +1241,12 @@ func (s *CollectionFSSuite) TestFlushFullBlocksOnly(c *check.C) {
 	}
 
 	nDirs := int64(8)
+	nFiles := int64(67)
 	megabyte := make([]byte, 1<<20)
 	for i := int64(0); i < nDirs; i++ {
 		dir := fmt.Sprintf("dir%d", i)
 		fs.Mkdir(dir, 0755)
-		for j := 0; j < 67; j++ {
+		for j := int64(0); j < nFiles; j++ {
 			f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
 			c.Assert(err, check.IsNil)
 			defer f.Close()
@@ -1221,7 +1254,8 @@ func (s *CollectionFSSuite) TestFlushFullBlocksOnly(c *check.C) {
 			c.Assert(err, check.IsNil)
 		}
 	}
-	c.Check(fs.MemorySize(), check.Equals, int64(nDirs*67<<20))
+	inodebytes := int64((nDirs*(nFiles+1) + 1) * 64)
+	c.Check(fs.MemorySize(), check.Equals, nDirs*nFiles*(1<<20+64)+inodebytes)
 	c.Check(flushed, check.Equals, int64(0))
 
 	waitForFlush := func(expectUnflushed, expectFlushed int64) {
@@ -1232,27 +1266,29 @@ func (s *CollectionFSSuite) TestFlushFullBlocksOnly(c *check.C) {
 	}
 
 	// Nothing flushed yet
-	waitForFlush((nDirs*67)<<20, 0)
+	waitForFlush(nDirs*nFiles*(1<<20+64)+inodebytes, 0)
 
 	// Flushing a non-empty dir "/" is non-recursive and there are
 	// no top-level files, so this has no effect
 	fs.Flush("/", false)
-	waitForFlush((nDirs*67)<<20, 0)
+	waitForFlush(nDirs*nFiles*(1<<20+64)+inodebytes, 0)
 
 	// Flush the full block in dir0
 	fs.Flush("dir0", false)
-	waitForFlush((nDirs*67-64)<<20, 64<<20)
+	bigloclen := int64(32 + 9 + 51 + 64) // md5 + "+" + "67xxxxxx" + "+Axxxxxx..." + 64 (see (storedSegment)memorySize)
+	waitForFlush((nDirs*nFiles-64)*(1<<20+64)+inodebytes+bigloclen*64, 64<<20)
 
 	err = fs.Flush("dir-does-not-exist", false)
 	c.Check(err, check.NotNil)
 
 	// Flush full blocks in all dirs
 	fs.Flush("", false)
-	waitForFlush(nDirs*3<<20, nDirs*64<<20)
+	waitForFlush(nDirs*3*(1<<20+64)+inodebytes+bigloclen*64*nDirs, nDirs*64<<20)
 
 	// Flush non-full blocks, too
 	fs.Flush("", true)
-	waitForFlush(0, nDirs*67<<20)
+	smallloclen := int64(32 + 8 + 51 + 64) // md5 + "+" + "3xxxxxx" + "+Axxxxxx..." + 64 (see (storedSegment)memorySize)
+	waitForFlush(inodebytes+bigloclen*64*nDirs+smallloclen*3*nDirs, nDirs*67<<20)
 }
 
 // Even when writing lots of files/dirs from different goroutines, as
@@ -1631,7 +1667,7 @@ func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
 	runtime.ReadMemStats(&memstats)
 	c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
 
-	f, err := coll.FileSystem(nil, nil)
+	f, err := coll.FileSystem(NewClientFromEnv(), &keepClientStub{})
 	c.Check(err, check.IsNil)
 	c.Logf("%s loaded", time.Now())
 	c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))