X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/845f7b9048d4b674a05e04fe919f2eb37479b1a0..89c2982376a7275e873d203c949d13b40fbd3bc4:/tools/keep-rsync/keep-rsync.go diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go index a5362879a9..820772eb5b 100644 --- a/tools/keep-rsync/keep-rsync.go +++ b/tools/keep-rsync/keep-rsync.go @@ -18,77 +18,79 @@ import ( ) func main() { - var srcConfigFile, dstConfigFile, srcKeepServicesJSON, dstKeepServicesJSON, prefix string - var replications int - var srcBlobSigningKey string + err := doMain() + if err != nil { + log.Fatalf("%v", err) + } +} - flag.StringVar( - &srcConfigFile, - "src-config-file", +func doMain() error { + flags := flag.NewFlagSet("keep-rsync", flag.ExitOnError) + + srcConfigFile := flags.String( + "src", "", - "Source configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf") + "Source configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY for the source.") - flag.StringVar( - &dstConfigFile, - "dst-config-file", + dstConfigFile := flags.String( + "dst", "", - "Destination configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf") + "Destination configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, and ARVADOS_API_HOST_INSECURE for the destination.") - flag.StringVar( - &srcKeepServicesJSON, + srcKeepServicesJSON := flags.String( "src-keep-services-json", "", "An optional list of available source keepservices. "+ "If not provided, this list is obtained from api server configured in src-config-file.") - flag.StringVar( - &dstKeepServicesJSON, + dstKeepServicesJSON := flags.String( "dst-keep-services-json", "", "An optional list of available destination keepservices. "+ "If not provided, this list is obtained from api server configured in dst-config-file.") - flag.IntVar( - &replications, + replications := flags.Int( "replications", 0, "Number of replications to write to the destination. If replications not specified, "+ "default replication level configured on destination server will be used.") - flag.StringVar( - &prefix, + prefix := flags.String( "prefix", "", "Index prefix") - flag.Parse() + // Parse args; omit the first arg which is the command name + flags.Parse(os.Args[1:]) - srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile) + srcConfig, srcBlobSigningKey, err := loadConfig(*srcConfigFile) if err != nil { - log.Fatalf("Error loading src configuration from file: %s", err.Error()) + return fmt.Errorf("Error loading src configuration from file: %s", err.Error()) } - dstConfig, _, err := loadConfig(dstConfigFile) + dstConfig, _, err := loadConfig(*dstConfigFile) if err != nil { - log.Fatalf("Error loading dst configuration from file: %s", err.Error()) + return fmt.Errorf("Error loading dst configuration from file: %s", err.Error()) } // setup src and dst keepclients - kcSrc, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0) + kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0) if err != nil { - log.Fatalf("Error configuring src keepclient: %s", err.Error()) + return fmt.Errorf("Error configuring src keepclient: %s", err.Error()) } - kcDst, err := setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications) + kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications) if err != nil { - log.Fatalf("Error configuring dst keepclient: %s", err.Error()) + return fmt.Errorf("Error configuring dst keepclient: %s", err.Error()) } // Copy blocks not found in dst from src - err = performKeepRsync(kcSrc, kcDst, srcBlobSigningKey, prefix) + err = performKeepRsync(kcSrc, kcDst, srcBlobSigningKey, *prefix) if err != nil { - log.Fatalf("Error while syncing data: %s", err.Error()) + return fmt.Errorf("Error while syncing data: %s", err.Error()) } + + return nil } type apiConfig struct { @@ -117,10 +119,7 @@ var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$") // Read config from file func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) { if !strings.Contains(filename, "/") { - filename = os.Getenv("HOME") + "/.config/arvados/" + filename - if !strings.HasSuffix(filename, ".conf") { - filename = filename + ".conf" - } + filename = os.Getenv("HOME") + "/.config/arvados/" + filename + ".conf" } content, err := ioutil.ReadFile(filename) @@ -256,13 +255,18 @@ func getMissingLocators(srcLocators, dstLocators map[string]bool) []string { // Copy blocks from src to dst; only those that are missing in dst are copied func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, blobSigningKey string) error { - done := 0 total := len(toBeCopied) startedAt := time.Now() - var blockTime int64 - for _, locator := range toBeCopied { - log.Printf("Getting block %d of %d: %v", done+1, total, locator) + for done, locator := range toBeCopied { + if done == 0 { + log.Printf("Copying data block %d of %d (%.2f%% done): %v", done+1, total, + float64(done)/float64(total)*100, locator) + } else { + timePerBlock := time.Since(startedAt) / time.Duration(done) + log.Printf("Copying data block %d of %d (%.2f%% done, %v est. time remaining): %v", done+1, total, + float64(done)/float64(total)*100, timePerBlock*time.Duration(total-done), locator) + } getLocator := locator expiresAt := time.Now().AddDate(0, 0, 1) @@ -275,23 +279,10 @@ func copyBlocksToDst(toBeCopied []string, kcSrc, kcDst *keepclient.KeepClient, b return fmt.Errorf("Error getting block: %v %v", locator, err) } - if done == 0 { - log.Printf("Copying data block %d of %d (%.2f%% done): %v", done+1, total, - float64(done)/float64(total)*100, locator) - } else { - log.Printf("Copying data block %d of %d (%.2f%% done, ETA %v): %v", done+1, total, - float64(done)/float64(total)*100, time.Duration(blockTime*int64(total-done)), locator) - } _, _, err = kcDst.PutHR(getLocator[:32], reader, len) if err != nil { return fmt.Errorf("Error copying data block: %v %v", locator, err) } - - if done == 0 { - blockTime = int64(time.Now().Sub(startedAt)) - } - - done++ } log.Printf("Successfully copied to destination %d blocks.", total)