tools/keep-rsync/keep-rsync.go

   1 package main
   2
   3 import (
   4         "flag"
   5         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   6         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   7         "io/ioutil"
   8         "log"
   9         "strings"
  10 )
  11
  12 // keep-rsync arguments
  13 var (
  14         srcConfig           map[string]string
  15         dstConfig           map[string]string
  16         srcKeepServicesJSON string
  17         dstKeepServicesJSON string
  18         replications        int
  19         prefix              string
  20 )
  21
  22 func main() {
  23         var srcConfigFile string
  24         var dstConfigFile string
  25
  26         flag.StringVar(
  27                 &srcConfigFile,
  28                 "src-config-file",
  29                 "",
  30                 "Source configuration filename with full path that contains "+
  31                         "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
  32                         "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
  33
  34         flag.StringVar(
  35                 &dstConfigFile,
  36                 "dst-config-file",
  37                 "",
  38                 "Destination configuration filename with full path that contains "+
  39                         "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
  40                         "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
  41
  42         flag.StringVar(
  43                 &srcKeepServicesJSON,
  44                 "src-keep-services-json",
  45                 "",
  46                 "An optional list of available source keepservices. "+
  47                         "If not provided, this list is obtained from api server configured in src-config-file.")
  48
  49         flag.StringVar(
  50                 &dstKeepServicesJSON,
  51                 "dst-keep-services-json",
  52                 "",
  53                 "An optional list of available destination keepservices. "+
  54                         "If not provided, this list is obtained from api server configured in dst-config-file.")
  55
  56         flag.IntVar(
  57                 &replications,
  58                 "replications",
  59                 3,
  60                 "Number of replications to write to the destination.")
  61
  62         flag.StringVar(
  63                 &prefix,
  64                 "prefix",
  65                 "",
  66                 "Index prefix")
  67
  68         flag.Parse()
  69
  70         var err error
  71
  72         if srcConfigFile == "" {
  73                 log.Fatal("-src-config-file must be specified.")
  74         }
  75         srcConfig, err = readConfigFromFile(srcConfigFile)
  76         if err != nil {
  77                 log.Fatal("Error reading source configuration: %s", err.Error())
  78         }
  79
  80         if dstConfigFile == "" {
  81                 log.Fatal("-dst-config-file must be specified.")
  82         }
  83         dstConfig, err = readConfigFromFile(dstConfigFile)
  84         if err != nil {
  85                 log.Fatal("Error reading destination configuration: %s", err.Error())
  86         }
  87
  88         err = initializeKeepRsync()
  89         if err != nil {
  90                 log.Fatal("Error configurating keep-rsync: %s", err.Error())
  91         }
  92 }
  93
  94 // Reads config from file
  95 func readConfigFromFile(filename string) (map[string]string, error) {
  96         content, err := ioutil.ReadFile(filename)
  97         if err != nil {
  98                 return nil, err
  99         }
 100
 101         config := make(map[string]string)
 102         lines := strings.Split(string(content), "\n")
 103         for _, line := range lines {
 104                 if line == "" {
 105                         continue
 106                 }
 107                 kv := strings.Split(line, "=")
 108                 config[kv[0]] = kv[1]
 109         }
 110         return config, nil
 111 }
 112
 113 // keep-rsync source and destination clients
 114 var (
 115         arvSrc arvadosclient.ArvadosClient
 116         arvDst arvadosclient.ArvadosClient
 117         kcSrc  *keepclient.KeepClient
 118         kcDst  *keepclient.KeepClient
 119 )
 120
 121 // Initializes keep-rsync using the config provided
 122 func initializeKeepRsync() (err error) {
 123         arvSrc, err = arvadosclient.MakeArvadosClientWithConfig(srcConfig)
 124         if err != nil {
 125                 return
 126         }
 127
 128         arvDst, err = arvadosclient.MakeArvadosClientWithConfig(dstConfig)
 129         if err != nil {
 130                 return
 131         }
 132
 133         kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
 134         if err != nil {
 135                 return
 136         }
 137
 138         kcDst, err = keepclient.MakeKeepClient(&arvDst)
 139
 140         return
 141 }