X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/db868a3c02cb84f068de77b438b4f2cb498826ae..4d9ece42c8482948e4eca9f587cdb36f62a00e2b:/tools/keep-rsync/keep-rsync.go diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go index 705025b079..912238c3fb 100644 --- a/tools/keep-rsync/keep-rsync.go +++ b/tools/keep-rsync/keep-rsync.go @@ -18,77 +18,84 @@ import ( ) func main() { - var srcConfigFile, dstConfigFile, srcKeepServicesJSON, dstKeepServicesJSON, prefix string - var replications int - var srcBlobSigningKey string + err := doMain() + if err != nil { + log.Fatalf("%v", err) + } +} + +func doMain() error { + flags := flag.NewFlagSet("keep-rsync", flag.ExitOnError) - flag.StringVar( - &srcConfigFile, + srcConfigFile := flags.String( "src", "", - "Source configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf") + "Source configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY for the source.") - flag.StringVar( - &dstConfigFile, + dstConfigFile := flags.String( "dst", "", - "Destination configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf") + "Destination configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, and ARVADOS_API_HOST_INSECURE for the destination.") - flag.StringVar( - &srcKeepServicesJSON, + srcKeepServicesJSON := flags.String( "src-keep-services-json", "", "An optional list of available source keepservices. "+ "If not provided, this list is obtained from api server configured in src-config-file.") - flag.StringVar( - &dstKeepServicesJSON, + dstKeepServicesJSON := flags.String( "dst-keep-services-json", "", "An optional list of available destination keepservices. "+ "If not provided, this list is obtained from api server configured in dst-config-file.") - flag.IntVar( - &replications, + replications := flags.Int( "replications", 0, "Number of replications to write to the destination. If replications not specified, "+ "default replication level configured on destination server will be used.") - flag.StringVar( - &prefix, + prefix := flags.String( "prefix", "", "Index prefix") - flag.Parse() + srcBlobSignatureTTL := flags.Duration( + "blob-signing-ttl", + 0, + "Lifetime of blob permission signatures on source keepservers. If not provided, this will be retrieved from the API server's discovery document.") + + // Parse args; omit the first arg which is the command name + flags.Parse(os.Args[1:]) - srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile) + srcConfig, srcBlobSigningKey, err := loadConfig(*srcConfigFile) if err != nil { - log.Fatalf("Error loading src configuration from file: %s", err.Error()) + return fmt.Errorf("Error loading src configuration from file: %s", err.Error()) } - dstConfig, _, err := loadConfig(dstConfigFile) + dstConfig, _, err := loadConfig(*dstConfigFile) if err != nil { - log.Fatalf("Error loading dst configuration from file: %s", err.Error()) + return fmt.Errorf("Error loading dst configuration from file: %s", err.Error()) } // setup src and dst keepclients - kcSrc, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0) + kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTL) if err != nil { - log.Fatalf("Error configuring src keepclient: %s", err.Error()) + return fmt.Errorf("Error configuring src keepclient: %s", err.Error()) } - kcDst, err := setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications) + kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0) if err != nil { - log.Fatalf("Error configuring dst keepclient: %s", err.Error()) + return fmt.Errorf("Error configuring dst keepclient: %s", err.Error()) } // Copy blocks not found in dst from src - err = performKeepRsync(kcSrc, kcDst, srcBlobSigningKey, prefix) + err = performKeepRsync(kcSrc, kcDst, *srcBlobSignatureTTL, srcBlobSigningKey, *prefix) if err != nil { - log.Fatalf("Error while syncing data: %s", err.Error()) + return fmt.Errorf("Error while syncing data: %s", err.Error()) } + + return nil } type apiConfig struct { @@ -153,7 +160,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } // setup keepclient using the config provided -func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int) (kc *keepclient.KeepClient, err error) { +func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) { arv := arvadosclient.ArvadosClient{ ApiToken: config.APIToken, ApiServer: config.APIHost, @@ -191,12 +198,22 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl kc.Want_replicas = replications } + // If srcBlobSignatureTTL is not provided, get it from API server discovery doc + if !isDst && srcBlobSignatureTTL == 0 { + value, err := arv.Discovery("blobSignatureTtl") + if err == nil { + srcBlobSignatureTTL = time.Duration(int(value.(float64))) * time.Second + } else { + return nil, err + } + } + return kc, nil } // Get unique block locators from src and dst // Copy any blocks missing in dst -func performKeepRsync(kcSrc, kcDst *keepclient.KeepClient, blobSigningKey, prefix string) error { +func performKeepRsync(kcSrc, kcDst *keepclient.KeepClient, srcBlobSignatureTTL time.Duration, blobSigningKey, prefix string) error { // Get unique locators from src srcIndex, err := getUniqueLocators(kcSrc, prefix) if err != nil { @@ -216,7 +233,7 @@ func performKeepRsync(kcSrc, kcDst *keepclient.KeepClient, blobSigningKey, prefi log.Printf("Before keep-rsync, there are %d blocks in src and %d blocks in dst. Start copying %d blocks from src not found in dst.", len(srcIndex), len(dstIndex), len(toBeCopied)) - err = copyBlocksToDst(toBeCopied, kcSrc, kcDst, blobSigningKey) + err = copyBlocksToDst(toBeCopied, kcSrc, kcDst, srcBlobSignatureTTL, blobSigningKey) return err } @@ -252,7 +269,7 @@ func getMissingLocators(srcLocators, dstLocators map[string]bool) []string { } // Copy blocks from src to dst; only those that are missing in dst are copied -func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, blobSigningKey string) error { +func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, srcBlobSignatureTTL time.Duration, blobSigningKey string) error { total := len(toBeCopied) startedAt := time.Now() @@ -269,7 +286,7 @@ func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, b getLocator := locator expiresAt := time.Now().AddDate(0, 0, 1) if blobSigningKey != "" { - getLocator = keepclient.SignLocator(getLocator, kcSrc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey)) + getLocator = keepclient.SignLocator(getLocator, kcSrc.Arvados.ApiToken, expiresAt, srcBlobSignatureTTL, []byte(blobSigningKey)) } reader, len, _, err := kcSrc.Get(getLocator)