7 "git.curoverse.com/arvados.git/sdk/go/keepclient"
15 var keepClient keepclient.KeepClient
18 Keepstore initiates pull worker channel goroutine.
19 The channel will process pull list.
20 For each (next) pull request:
21 For each locator listed, execute Pull on the server(s) listed
22 Skip the rest of the servers if no errors
25 func RunPullWorker(nextItem <-chan interface{}, kc keepclient.KeepClient) {
27 for item := range nextItem {
28 Pull(item.(PullRequest))
33 For each Pull request:
34 Generate a random API token.
35 Generate a permission signature using this token, timestamp ~60 seconds in the future, and desired block hash.
36 Using this token & signature, retrieve the given block.
39 func Pull(pullRequest PullRequest) (err error) {
42 log.Printf("Pull %s success", pullRequest)
44 log.Printf("Pull %s error: %s", pullRequest, err)
48 service_roots := make(map[string]string)
49 for _, addr := range pullRequest.Servers {
50 service_roots[addr] = addr
52 keepClient.SetServiceRoots(service_roots)
54 // Generate signature with a random token
55 PermissionSecret = []byte(os.Getenv("ARVADOS_API_TOKEN"))
56 expires_at := time.Now().Add(60 * time.Second)
57 signedLocator := SignLocator(pullRequest.Locator, GenerateRandomApiToken(), expires_at)
59 reader, contentLen, _, err := GetContent(pullRequest.Locator, signedLocator)
64 return errors.New(fmt.Sprintf("No reader found for : %s", signedLocator))
68 read_content, err := ioutil.ReadAll(reader)
73 if (read_content == nil) || (int64(len(read_content)) != contentLen) {
74 return errors.New(fmt.Sprintf("Content not found for: %s", signedLocator))
77 err = PutContent(read_content, pullRequest.Locator)
81 // Fetch the content for the given locator using keepclient.
82 var GetContent = func(locator string, signedLocator string) (reader io.ReadCloser, contentLength int64, url string, err error) {
83 reader, blocklen, url, err := keepClient.Get(signedLocator)
84 return reader, blocklen, url, err
87 const ALPHA_NUMERIC = "0123456789abcdefghijklmnopqrstuvwxyz"
89 func GenerateRandomApiToken() string {
90 var bytes = make([]byte, 36)
92 for i, b := range bytes {
93 bytes[i] = ALPHA_NUMERIC[b%byte(len(ALPHA_NUMERIC))]
95 return (string(bytes))
99 var PutContent = func(content []byte, locator string) (err error) {
100 err = PutBlock(content, locator)