projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
refs #7167
[arvados.git]
/
services
/
datamanager
/
summary
/
pull_list.go
diff --git
a/services/datamanager/summary/pull_list.go
b/services/datamanager/summary/pull_list.go
index d2eef9316ba8ab91794bb0f84822a9cac1ac7b7b..cc01249a624a7f4947cdcfc8dafd73dd7e347377 100644
(file)
--- a/
services/datamanager/summary/pull_list.go
+++ b/
services/datamanager/summary/pull_list.go
@@
-1,4
+1,5
@@
// Code for generating pull lists as described in https://arvados.org/projects/arvados/wiki/Keep_Design_Doc#Pull-List
// Code for generating pull lists as described in https://arvados.org/projects/arvados/wiki/Keep_Design_Doc#Pull-List
+
package summary
import (
package summary
import (
@@
-14,19
+15,21
@@
import (
"strings"
)
"strings"
)
+// Locator is a block digest
type Locator blockdigest.DigestWithSize
type Locator blockdigest.DigestWithSize
+// MarshalJSON encoding
func (l Locator) MarshalJSON() ([]byte, error) {
return []byte("\"" + blockdigest.DigestWithSize(l).String() + "\""), nil
}
func (l Locator) MarshalJSON() ([]byte, error) {
return []byte("\"" + blockdigest.DigestWithSize(l).String() + "\""), nil
}
-//
O
ne entry in the Pull List
+//
PullRequest represents o
ne entry in the Pull List
type PullRequest struct {
Locator Locator `json:"locator"`
Servers []string `json:"servers"`
}
type PullRequest struct {
Locator Locator `json:"locator"`
Servers []string `json:"servers"`
}
-//
The Pull
List for a particular server
+//
Pull
List for a particular server
type PullList []PullRequest
// PullListByLocator implements sort.Interface for PullList based on
type PullList []PullRequest
// PullListByLocator implements sort.Interface for PullList based on
@@
-49,6
+52,7
@@
func (a PullListByLocator) Less(i, j int) bool {
return false
}
return false
}
+// PullServers struct
// For a given under-replicated block, this structure represents which
// servers should pull the specified block and which servers they can
// pull it from.
// For a given under-replicated block, this structure represents which
// servers should pull the specified block and which servers they can
// pull it from.
@@
-57,8
+61,8
@@
type PullServers struct {
From []string // Servers that already contain the specified block
}
From []string // Servers that already contain the specified block
}
-// C
reates a map from block locator to PullServers with one entry for
-// each under-replicated block.
+// C
omputePullServers creates a map from block locator to PullServers
+//
with one entry for
each under-replicated block.
//
// This method ignores zero-replica blocks since there are no servers
// to pull them from, so callers should feel free to omit them, but
//
// This method ignores zero-replica blocks since there are no servers
// to pull them from, so callers should feel free to omit them, but
@@
-78,7
+82,7
@@
func ComputePullServers(kc *keepclient.KeepClient,
writableServers[cs.Get(url)] = struct{}{}
}
writableServers[cs.Get(url)] = struct{}{}
}
- for block
, _
:= range underReplicated {
+ for block := range underReplicated {
serversStoringBlock := keepServerInfo.BlockToServers[block]
numCopies := len(serversStoringBlock)
numCopiesMissing := blockToDesiredReplication[block] - numCopies
serversStoringBlock := keepServerInfo.BlockToServers[block]
numCopies := len(serversStoringBlock)
numCopiesMissing := blockToDesiredReplication[block] - numCopies
@@
-94,7
+98,7
@@
func ComputePullServers(kc *keepclient.KeepClient,
serverHasBlock := map[string]struct{}{}
for _, info := range serversStoringBlock {
sa := keepServerInfo.KeepServerIndexToAddress[info.ServerIndex]
serverHasBlock := map[string]struct{}{}
for _, info := range serversStoringBlock {
sa := keepServerInfo.KeepServerIndexToAddress[info.ServerIndex]
- serverHasBlock[cs.Get(sa.
HostPort
())] = struct{}{}
+ serverHasBlock[cs.Get(sa.
URL
())] = struct{}{}
}
roots := keepclient.NewRootSorter(kc.LocalRoots(),
}
roots := keepclient.NewRootSorter(kc.LocalRoots(),
@@
-109,9
+113,9
@@
func ComputePullServers(kc *keepclient.KeepClient,
return m
}
return m
}
-// Create
s a pull list in which the To and From fields preserve the
-//
ordering of sorted servers and the contents are all canonical
-// strings.
+// Create
PullServers creates a pull list in which the To and From
+//
fields preserve the ordering of sorted servers and the contents
+//
are all canonical
strings.
func CreatePullServers(cs CanonicalString,
serverHasBlock map[string]struct{},
writableServers map[string]struct{},
func CreatePullServers(cs CanonicalString,
serverHasBlock map[string]struct{},
writableServers map[string]struct{},
@@
-126,7
+130,7
@@
func CreatePullServers(cs CanonicalString,
for _, host := range sortedServers {
// Strip the protocol portion of the url.
// Use the canonical copy of the string to avoid memory waste.
for _, host := range sortedServers {
// Strip the protocol portion of the url.
// Use the canonical copy of the string to avoid memory waste.
- server := cs.Get(
RemoveProtocolPrefix(host)
)
+ server := cs.Get(
host
)
_, hasBlock := serverHasBlock[server]
if hasBlock {
// The from field should include the protocol.
_, hasBlock := serverHasBlock[server]
if hasBlock {
// The from field should include the protocol.
@@
-142,12
+146,12
@@
func CreatePullServers(cs CanonicalString,
return
}
return
}
-//
S
trips the protocol prefix from a url.
+//
RemoveProtocolPrefix s
trips the protocol prefix from a url.
func RemoveProtocolPrefix(url string) string {
return url[(strings.LastIndex(url, "/") + 1):]
}
func RemoveProtocolPrefix(url string) string {
return url[(strings.LastIndex(url, "/") + 1):]
}
-//
P
roduces a PullList for each keep server.
+//
BuildPullLists p
roduces a PullList for each keep server.
func BuildPullLists(lps map[Locator]PullServers) (spl map[string]PullList) {
spl = map[string]PullList{}
// We don't worry about canonicalizing our strings here, because we
func BuildPullLists(lps map[Locator]PullServers) (spl map[string]PullList) {
spl = map[string]PullList{}
// We don't worry about canonicalizing our strings here, because we
@@
-166,7
+170,7
@@
func BuildPullLists(lps map[Locator]PullServers) (spl map[string]PullList) {
return
}
return
}
-// Writes each pull list to a file.
+// Write
PullLists write
s each pull list to a file.
// The filename is based on the hostname.
//
// This is just a hack for prototyping, it is not expected to be used
// The filename is based on the hostname.
//
// This is just a hack for prototyping, it is not expected to be used
@@
-175,7
+179,7
@@
func WritePullLists(arvLogger *logger.Logger,
pullLists map[string]PullList) {
r := strings.NewReplacer(":", ".")
for host, list := range pullLists {
pullLists map[string]PullList) {
r := strings.NewReplacer(":", ".")
for host, list := range pullLists {
- filename := fmt.Sprintf("pull_list.%s", r.Replace(
host
))
+ filename := fmt.Sprintf("pull_list.%s", r.Replace(
RemoveProtocolPrefix(host)
))
pullListFile, err := os.Create(filename)
if err != nil {
loggerutil.FatalWithMessage(arvLogger,
pullListFile, err := os.Create(filename)
if err != nil {
loggerutil.FatalWithMessage(arvLogger,