tools/keep-rsync/keep-rsync.go

   1 package main
   2
   3 import (
   4         "flag"
   5         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   6         "git.curoverse.com/arvados.git/sdk/go/keepclient"
   7         "io/ioutil"
   8         "log"
   9         "regexp"
  10         "strings"
  11 )
  12
  13 // keep-rsync arguments
  14 var (
  15         srcConfig           arvadosclient.APIConfig
  16         dstConfig           arvadosclient.APIConfig
  17         blobSigningKey      string
  18         srcKeepServicesJSON string
  19         dstKeepServicesJSON string
  20         replications        int
  21         prefix              string
  22 )
  23
  24 func main() {
  25         var srcConfigFile string
  26         var dstConfigFile string
  27
  28         flag.StringVar(
  29                 &srcConfigFile,
  30                 "src-config-file",
  31                 "",
  32                 "Source configuration filename with full path that contains "+
  33                         "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
  34                         "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT and ARVADOS_BLOB_SIGNING_KEY.")
  35
  36         flag.StringVar(
  37                 &dstConfigFile,
  38                 "dst-config-file",
  39                 "",
  40                 "Destination configuration filename with full path that contains "+
  41                         "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
  42                         "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT and ARVADOS_BLOB_SIGNING_KEY.")
  43
  44         flag.StringVar(
  45                 &srcKeepServicesJSON,
  46                 "src-keep-services-json",
  47                 "",
  48                 "An optional list of available source keepservices. "+
  49                         "If not provided, this list is obtained from api server configured in src-config-file.")
  50
  51         flag.StringVar(
  52                 &dstKeepServicesJSON,
  53                 "dst-keep-services-json",
  54                 "",
  55                 "An optional list of available destination keepservices. "+
  56                         "If not provided, this list is obtained from api server configured in dst-config-file.")
  57
  58         flag.IntVar(
  59                 &replications,
  60                 "replications",
  61                 0,
  62                 "Number of replications to write to the destination.")
  63
  64         flag.StringVar(
  65                 &prefix,
  66                 "prefix",
  67                 "",
  68                 "Index prefix")
  69
  70         flag.Parse()
  71
  72         var err error
  73
  74         // Load config
  75         if srcConfigFile == "" {
  76                 log.Fatal("-src-config-file must be specified.")
  77         }
  78         srcConfig, err = readConfigFromFile(srcConfigFile)
  79         if err != nil {
  80                 log.Fatal("Error reading source configuration: %s", err.Error())
  81         }
  82
  83         if dstConfigFile == "" {
  84                 log.Fatal("-dst-config-file must be specified.")
  85         }
  86         dstConfig, err = readConfigFromFile(dstConfigFile)
  87         if err != nil {
  88                 log.Fatal("Error reading destination configuration: %s", err.Error())
  89         }
  90
  91         // Initialize keep-rsync
  92         err = initializeKeepRsync()
  93         if err != nil {
  94                 log.Fatal("Error configurating keep-rsync: %s", err.Error())
  95         }
  96 }
  97
  98 var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
  99
 100 // Reads config from file
 101 func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
 102         var config arvadosclient.APIConfig
 103
 104         content, err := ioutil.ReadFile(filename)
 105         if err != nil {
 106                 return config, err
 107         }
 108
 109         lines := strings.Split(string(content), "\n")
 110         for _, line := range lines {
 111                 if line == "" {
 112                         continue
 113                 }
 114                 kv := strings.Split(line, "=")
 115
 116                 switch kv[0] {
 117                 case "ARVADOS_API_TOKEN":
 118                         config.APIToken = kv[1]
 119                 case "ARVADOS_API_HOST":
 120                         config.APIHost = kv[1]
 121                 case "ARVADOS_API_HOST_INSECURE":
 122                         config.APIHostInsecure = matchTrue.MatchString(kv[1])
 123                 case "ARVADOS_EXTERNAL_CLIENT":
 124                         config.ExternalClient = matchTrue.MatchString(kv[1])
 125                 case "ARVADOS_BLOB_SIGNING_KEY":
 126                         blobSigningKey = kv[1]
 127                 }
 128         }
 129         return config, nil
 130 }
 131
 132 // keep-rsync source and destination clients
 133 var (
 134         arvSrc arvadosclient.ArvadosClient
 135         arvDst arvadosclient.ArvadosClient
 136         kcSrc  *keepclient.KeepClient
 137         kcDst  *keepclient.KeepClient
 138 )
 139
 140 // Initializes keep-rsync using the config provided
 141 func initializeKeepRsync() (err error) {
 142         // arvSrc from srcConfig
 143         arvSrc, err = arvadosclient.New(srcConfig)
 144         if err != nil {
 145                 return
 146         }
 147
 148         // arvDst from dstConfig
 149         arvDst, err = arvadosclient.New(dstConfig)
 150         if err != nil {
 151                 return
 152         }
 153
 154         // Get default replications value from destination, if it is not already provided
 155         if replications == 0 {
 156                 value, err := arvDst.Discovery("defaultCollectionReplication")
 157                 if err == nil {
 158                         replications = int(value.(float64))
 159                 } else {
 160                         replications = 2
 161                 }
 162         }
 163
 164         // if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
 165         if srcKeepServicesJSON == "" {
 166                 kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
 167                 if err != nil {
 168                         return
 169                 }
 170         } else {
 171                 kcSrc, err = keepclient.MakeKeepClientFromJSON(&arvSrc, srcKeepServicesJSON)
 172                 if err != nil {
 173                         return
 174                 }
 175         }
 176
 177         // if dstKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
 178         if dstKeepServicesJSON == "" {
 179                 kcDst, err = keepclient.MakeKeepClient(&arvDst)
 180                 if err != nil {
 181                         return
 182                 }
 183         } else {
 184                 kcDst, err = keepclient.MakeKeepClientFromJSON(&arvDst, dstKeepServicesJSON)
 185                 if err != nil {
 186                         return
 187                 }
 188         }
 189         kcDst.Want_replicas = replications
 190
 191         return
 192 }