tools/keep-rsync/keep-rsync.go

   1 package main
   2
   3 import (
   4         "flag"
   5         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   6         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   7         "io/ioutil"
   8         "log"
   9         "regexp"
  10         "strings"
  11 )
  12
  13 // keep-rsync arguments
  14 var (
  15         srcConfig           arvadosclient.APIConfig
  16         dstConfig           arvadosclient.APIConfig
  17         srcKeepServicesJSON string
  18         dstKeepServicesJSON string
  19         replications        int
  20         prefix              string
  21 )
  22
  23 func main() {
  24         var srcConfigFile string
  25         var dstConfigFile string
  26
  27         flag.StringVar(
  28                 &srcConfigFile,
  29                 "src-config-file",
  30                 "",
  31                 "Source configuration filename with full path that contains "+
  32                         "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
  33                         "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
  34
  35         flag.StringVar(
  36                 &dstConfigFile,
  37                 "dst-config-file",
  38                 "",
  39                 "Destination configuration filename with full path that contains "+
  40                         "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
  41                         "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
  42
  43         flag.StringVar(
  44                 &srcKeepServicesJSON,
  45                 "src-keep-services-json",
  46                 "",
  47                 "An optional list of available source keepservices. "+
  48                         "If not provided, this list is obtained from api server configured in src-config-file.")
  49
  50         flag.StringVar(
  51                 &dstKeepServicesJSON,
  52                 "dst-keep-services-json",
  53                 "",
  54                 "An optional list of available destination keepservices. "+
  55                         "If not provided, this list is obtained from api server configured in dst-config-file.")
  56
  57         flag.IntVar(
  58                 &replications,
  59                 "replications",
  60                 3,
  61                 "Number of replications to write to the destination.")
  62
  63         flag.StringVar(
  64                 &prefix,
  65                 "prefix",
  66                 "",
  67                 "Index prefix")
  68
  69         flag.Parse()
  70
  71         var err error
  72
  73         // Load config
  74         if srcConfigFile == "" {
  75                 log.Fatal("-src-config-file must be specified.")
  76         }
  77         srcConfig, err = readConfigFromFile(srcConfigFile)
  78         if err != nil {
  79                 log.Fatal("Error reading source configuration: %s", err.Error())
  80         }
  81
  82         if dstConfigFile == "" {
  83                 log.Fatal("-dst-config-file must be specified.")
  84         }
  85         dstConfig, err = readConfigFromFile(dstConfigFile)
  86         if err != nil {
  87                 log.Fatal("Error reading destination configuration: %s", err.Error())
  88         }
  89
  90         // Initialize keep-rsync
  91         err = initializeKeepRsync()
  92         if err != nil {
  93                 log.Fatal("Error configurating keep-rsync: %s", err.Error())
  94         }
  95 }
  96
  97 var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
  98
  99 // Reads config from file
 100 func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
 101         var config arvadosclient.APIConfig
 102
 103         content, err := ioutil.ReadFile(filename)
 104         if err != nil {
 105                 return config, err
 106         }
 107
 108         lines := strings.Split(string(content), "\n")
 109         for _, line := range lines {
 110                 if line == "" {
 111                         continue
 112                 }
 113                 kv := strings.Split(line, "=")
 114
 115                 switch kv[0] {
 116                 case "ARVADOS_API_TOKEN":
 117                         config.APIToken = kv[1]
 118                 case "ARVADOS_API_HOST":
 119                         config.APIHost = kv[1]
 120                 case "ARVADOS_API_HOST_INSECURE":
 121                         config.APIHostInsecure = matchTrue.MatchString(kv[1])
 122                 case "ARVADOS_EXTERNAL_CLIENT":
 123                         config.ExternalClient = matchTrue.MatchString(kv[1])
 124                 }
 125         }
 126         return config, nil
 127 }
 128
 129 // keep-rsync source and destination clients
 130 var (
 131         arvSrc arvadosclient.ArvadosClient
 132         arvDst arvadosclient.ArvadosClient
 133         kcSrc  *keepclient.KeepClient
 134         kcDst  *keepclient.KeepClient
 135 )
 136
 137 // Initializes keep-rsync using the config provided
 138 func initializeKeepRsync() (err error) {
 139         // arvSrc from srcConfig
 140         arvSrc, err = arvadosclient.New(srcConfig)
 141         if err != nil {
 142                 return
 143         }
 144
 145         // arvDst from dstConfig
 146         arvDst, err = arvadosclient.New(dstConfig)
 147         if err != nil {
 148                 return
 149         }
 150
 151         // if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
 152         if srcKeepServicesJSON == "" {
 153                 kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
 154                 if err != nil {
 155                         return
 156                 }
 157         } else {
 158                 kcSrc, err = keepclient.MakeKeepClientFromJSON(&arvSrc, srcKeepServicesJSON)
 159                 if err != nil {
 160                         return
 161                 }
 162         }
 163
 164         // if dstKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
 165         if dstKeepServicesJSON == "" {
 166                 kcDst, err = keepclient.MakeKeepClient(&arvDst)
 167                 if err != nil {
 168                         return
 169                 }
 170         } else {
 171                 kcDst, err = keepclient.MakeKeepClientFromJSON(&arvDst, dstKeepServicesJSON)
 172                 if err != nil {
 173                         return
 174                 }
 175         }
 176         kcDst.Want_replicas = replications
 177
 178         return
 179 }