X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/19ae770973482257117fe8ded5619c3018c4b60f..10440ac12d6771ab80469adf551d2cac8d3461e6:/tools/keep-rsync/keep-rsync.go diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go index 820772eb5b..7e519f775b 100644 --- a/tools/keep-rsync/keep-rsync.go +++ b/tools/keep-rsync/keep-rsync.go @@ -1,3 +1,7 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( @@ -6,17 +10,20 @@ import ( "errors" "flag" "fmt" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/keepclient" "io/ioutil" "log" "net/http" "os" - "regexp" "strings" "time" + + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/keepclient" ) +var version = "dev" + func main() { err := doMain() if err != nil { @@ -60,8 +67,22 @@ func doMain() error { "", "Index prefix") - // Parse args; omit the first arg which is the command name - flags.Parse(os.Args[1:]) + srcBlobSignatureTTLFlag := flags.Duration( + "src-blob-signature-ttl", + 0, + "Lifetime of blob permission signatures on source keepservers. If not provided, this will be retrieved from the API server's discovery document.") + + getVersion := flags.Bool( + "version", + false, + "Print version information and exit.") + + if ok, code := cmd.ParseFlags(flags, os.Args[0], os.Args[1:], "", os.Stderr); !ok { + os.Exit(code) + } else if *getVersion { + fmt.Printf("%s %s\n", os.Args[0], version) + os.Exit(0) + } srcConfig, srcBlobSigningKey, err := loadConfig(*srcConfigFile) if err != nil { @@ -74,18 +95,18 @@ func doMain() error { } // setup src and dst keepclients - kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0) + kcSrc, srcBlobSignatureTTL, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTLFlag) if err != nil { return fmt.Errorf("Error configuring src keepclient: %s", err.Error()) } - kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications) + kcDst, _, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0) if err != nil { return fmt.Errorf("Error configuring dst keepclient: %s", err.Error()) } // Copy blocks not found in dst from src - err = performKeepRsync(kcSrc, kcDst, srcBlobSigningKey, *prefix) + err = performKeepRsync(kcSrc, kcDst, srcBlobSignatureTTL, srcBlobSigningKey, *prefix) if err != nil { return fmt.Errorf("Error while syncing data: %s", err.Error()) } @@ -97,7 +118,6 @@ type apiConfig struct { APIToken string APIHost string APIHostInsecure bool - ExternalClient bool } // Load src and dst config from given files @@ -114,8 +134,6 @@ func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err return } -var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$") - // Read config from file func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) { if !strings.Contains(filename, "/") { @@ -144,9 +162,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin case "ARVADOS_API_HOST": config.APIHost = value case "ARVADOS_API_HOST_INSECURE": - config.APIHostInsecure = matchTrue.MatchString(value) - case "ARVADOS_EXTERNAL_CLIENT": - config.ExternalClient = matchTrue.MatchString(value) + config.APIHostInsecure = arvadosclient.StringBool(value) case "ARVADOS_BLOB_SIGNING_KEY": blobSigningKey = value } @@ -155,27 +171,26 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } // setup keepclient using the config provided -func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int) (kc *keepclient.KeepClient, err error) { +func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, blobSignatureTTL time.Duration, err error) { arv := arvadosclient.ArvadosClient{ ApiToken: config.APIToken, ApiServer: config.APIHost, ApiInsecure: config.APIHostInsecure, Client: &http.Client{Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}}, - External: config.ExternalClient, } - // if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers + // If keepServicesJSON is provided, use it instead of service discovery if keepServicesJSON == "" { kc, err = keepclient.MakeKeepClient(&arv) if err != nil { - return nil, err + return nil, 0, err } } else { kc = keepclient.New(&arv) err = kc.LoadKeepServicesFromJSON(keepServicesJSON) if err != nil { - return kc, err + return kc, 0, err } } @@ -186,19 +201,30 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl if err == nil { replications = int(value.(float64)) } else { - return nil, err + return nil, 0, err } } kc.Want_replicas = replications } - return kc, nil + // If srcBlobSignatureTTL is not provided, get it from API server discovery doc + blobSignatureTTL = srcBlobSignatureTTL + if !isDst && srcBlobSignatureTTL == 0 { + value, err := arv.Discovery("blobSignatureTtl") + if err == nil { + blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second + } else { + return nil, 0, err + } + } + + return kc, blobSignatureTTL, nil } // Get unique block locators from src and dst // Copy any blocks missing in dst -func performKeepRsync(kcSrc, kcDst *keepclient.KeepClient, blobSigningKey, prefix string) error { +func performKeepRsync(kcSrc, kcDst *keepclient.KeepClient, srcBlobSignatureTTL time.Duration, blobSigningKey, prefix string) error { // Get unique locators from src srcIndex, err := getUniqueLocators(kcSrc, prefix) if err != nil { @@ -218,7 +244,7 @@ func performKeepRsync(kcSrc, kcDst *keepclient.KeepClient, blobSigningKey, prefi log.Printf("Before keep-rsync, there are %d blocks in src and %d blocks in dst. Start copying %d blocks from src not found in dst.", len(srcIndex), len(dstIndex), len(toBeCopied)) - err = copyBlocksToDst(toBeCopied, kcSrc, kcDst, blobSigningKey) + err = copyBlocksToDst(toBeCopied, kcSrc, kcDst, srcBlobSignatureTTL, blobSigningKey) return err } @@ -254,7 +280,7 @@ func getMissingLocators(srcLocators, dstLocators map[string]bool) []string { } // Copy blocks from src to dst; only those that are missing in dst are copied -func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, blobSigningKey string) error { +func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, srcBlobSignatureTTL time.Duration, blobSigningKey string) error { total := len(toBeCopied) startedAt := time.Now() @@ -271,7 +297,7 @@ func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, b getLocator := locator expiresAt := time.Now().AddDate(0, 0, 1) if blobSigningKey != "" { - getLocator = keepclient.SignLocator(getLocator, kcSrc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey)) + getLocator = keepclient.SignLocator(getLocator, kcSrc.Arvados.ApiToken, expiresAt, srcBlobSignatureTTL, []byte(blobSigningKey)) } reader, len, _, err := kcSrc.Get(getLocator)