From: mishaz Date: Tue, 9 Jun 2015 17:51:31 +0000 (+0000) Subject: Renamed BlockToReplication BlockToDesiredReplication. X-Git-Tag: 1.1.0~1505^2~6 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/982f7db5b67194ddb3b3dd1fae594784f58b35b8 Renamed BlockToReplication BlockToDesiredReplication. Added protocol field to servers in pull list. --- diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go index ed6df9dc5c..93fad0e427 100644 --- a/services/datamanager/collection/collection.go +++ b/services/datamanager/collection/collection.go @@ -39,13 +39,13 @@ type Collection struct { } type ReadCollections struct { - ReadAllCollections bool - UuidToCollection map[string]Collection - OwnerToCollectionSize map[string]int - BlockToReplication map[blockdigest.DigestWithSize]int - CollectionUuidToIndex map[string]int - CollectionIndexToUuid []string - BlockToCollectionIndices map[blockdigest.DigestWithSize][]int + ReadAllCollections bool + UuidToCollection map[string]Collection + OwnerToCollectionSize map[string]int + BlockToDesiredReplication map[blockdigest.DigestWithSize]int + CollectionUuidToIndex map[string]int + CollectionIndexToUuid []string + BlockToCollectionIndices map[blockdigest.DigestWithSize][]int } type GetCollectionsParams struct { @@ -283,7 +283,7 @@ func ProcessCollections(arvLogger *logger.Logger, func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) { readCollections.OwnerToCollectionSize = make(map[string]int) - readCollections.BlockToReplication = make(map[blockdigest.DigestWithSize]int) + readCollections.BlockToDesiredReplication = make(map[blockdigest.DigestWithSize]int) numCollections := len(readCollections.UuidToCollection) readCollections.CollectionUuidToIndex = make(map[string]int, numCollections) readCollections.CollectionIndexToUuid = make([]string, 0, numCollections) @@ -303,9 +303,10 @@ func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) { readCollections.BlockToCollectionIndices[locator] = append(readCollections.BlockToCollectionIndices[locator], collectionIndex) - storedReplication := readCollections.BlockToReplication[locator] + storedReplication := readCollections.BlockToDesiredReplication[locator] if coll.ReplicationLevel > storedReplication { - readCollections.BlockToReplication[locator] = coll.ReplicationLevel + readCollections.BlockToDesiredReplication[locator] = + coll.ReplicationLevel } } } @@ -319,7 +320,7 @@ func (readCollections *ReadCollections) Summarize(arvLogger *logger.Logger) { collectionInfo["owner_to_collection_size"] = readCollections.OwnerToCollectionSize collectionInfo["distinct_blocks_named"] = - len(readCollections.BlockToReplication) + len(readCollections.BlockToDesiredReplication) }) } diff --git a/services/datamanager/collection/collection_test.go b/services/datamanager/collection/collection_test.go index 4af5d4c8c4..1669bb7784 100644 --- a/services/datamanager/collection/collection_test.go +++ b/services/datamanager/collection/collection_test.go @@ -20,9 +20,9 @@ var _ = Suite(&MySuite{}) // indeterminate, we replace BlockToCollectionIndices with // BlockToCollectionUuids. type ExpectedSummary struct { - OwnerToCollectionSize map[string]int - BlockToReplication map[blockdigest.DigestWithSize]int - BlockToCollectionUuids map[blockdigest.DigestWithSize][]string + OwnerToCollectionSize map[string]int + BlockToDesiredReplication map[blockdigest.DigestWithSize]int + BlockToCollectionUuids map[blockdigest.DigestWithSize][]string } func CompareSummarizedReadCollections(c *C, @@ -32,8 +32,8 @@ func CompareSummarizedReadCollections(c *C, c.Assert(summarized.OwnerToCollectionSize, DeepEquals, expected.OwnerToCollectionSize) - c.Assert(summarized.BlockToReplication, DeepEquals, - expected.BlockToReplication) + c.Assert(summarized.BlockToDesiredReplication, DeepEquals, + expected.BlockToDesiredReplication) summarizedBlockToCollectionUuids := make(map[blockdigest.DigestWithSize]map[string]struct{}) @@ -73,9 +73,9 @@ func (s *MySuite) TestSummarizeSimple(checker *C) { blockDigest2 := blockdigest.MakeTestDigestWithSize(2) expected := ExpectedSummary{ - OwnerToCollectionSize: map[string]int{c.OwnerUuid: c.TotalSize}, - BlockToReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5}, - BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.Uuid}, blockDigest2: []string{c.Uuid}}, + OwnerToCollectionSize: map[string]int{c.OwnerUuid: c.TotalSize}, + BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5}, + BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.Uuid}, blockDigest2: []string{c.Uuid}}, } CompareSummarizedReadCollections(checker, rc, expected) @@ -107,7 +107,7 @@ func (s *MySuite) TestSummarizeOverlapping(checker *C) { c0.OwnerUuid: c0.TotalSize, c1.OwnerUuid: c1.TotalSize, }, - BlockToReplication: map[blockdigest.DigestWithSize]int{ + BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{ blockDigest1: 5, blockDigest2: 8, blockDigest3: 8, diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go index 3e84308580..91612668b5 100644 --- a/services/datamanager/datamanager.go +++ b/services/datamanager/datamanager.go @@ -102,7 +102,7 @@ func singlerun() { log.Printf("Blocks In Collections: %d, "+ "\nBlocks In Keep: %d.", - len(readCollections.BlockToReplication), + len(readCollections.BlockToDesiredReplication), len(keepServerInfo.BlockToServers)) log.Println(replicationCounts.PrettyPrint()) @@ -121,7 +121,7 @@ func singlerun() { pullServers := summary.ComputePullServers(kc, &keepServerInfo, - readCollections.BlockToReplication, + readCollections.BlockToDesiredReplication, replicationSummary.UnderReplicatedBlocks) pullLists := summary.BuildPullLists(pullServers) diff --git a/services/datamanager/summary/pull_list.go b/services/datamanager/summary/pull_list.go index e9bd5d1877..542a6693d2 100644 --- a/services/datamanager/summary/pull_list.go +++ b/services/datamanager/summary/pull_list.go @@ -17,7 +17,6 @@ import ( type Locator blockdigest.DigestWithSize func (l Locator) MarshalJSON() ([]byte, error) { - //return []byte("\"" + l.Digest.String() + "\""), nil return []byte("\"" + blockdigest.DigestWithSize(l).String() + "\""), nil } @@ -127,7 +126,8 @@ func CreatePullServers(cs CanonicalString, server := cs.Get(RemoveProtocolPrefix(host)) _, hasBlock := serverHasBlock[server] if hasBlock { - ps.From = append(ps.From, server) + // The from field should include the protocol. + ps.From = append(ps.From, cs.Get(host)) } else if len(ps.To) < maxToFields { _, writable := writableServers[server] if writable { diff --git a/services/datamanager/summary/summary.go b/services/datamanager/summary/summary.go index efb6061c9a..edd760b035 100644 --- a/services/datamanager/summary/summary.go +++ b/services/datamanager/summary/summary.go @@ -196,7 +196,7 @@ func BucketReplication(readCollections collection.ReadCollections, keepServerInfo keep.ReadServers) (rlbsm ReplicationLevelBlockSetMap) { rlbsm = make(ReplicationLevelBlockSetMap) - for block, requestedReplication := range readCollections.BlockToReplication { + for block, requestedReplication := range readCollections.BlockToDesiredReplication { rlbsm.Insert( ReplicationLevels{ Requested: requestedReplication, @@ -205,7 +205,7 @@ func BucketReplication(readCollections collection.ReadCollections, } for block, servers := range keepServerInfo.BlockToServers { - if 0 == readCollections.BlockToReplication[block] { + if 0 == readCollections.BlockToDesiredReplication[block] { rlbsm.Insert( ReplicationLevels{Requested: 0, Actual: len(servers)}, block)