5 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
6 "git.curoverse.com/arvados.git/sdk/go/keepclient"
13 // keep-rsync arguments
15 srcConfig arvadosclient.APIConfig
16 dstConfig arvadosclient.APIConfig
18 srcKeepServicesJSON string
19 dstKeepServicesJSON string
25 var srcConfigFile string
26 var dstConfigFile string
32 "Source configuration filename with full path that contains "+
33 "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
34 "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT and ARVADOS_BLOB_SIGNING_KEY.")
40 "Destination configuration filename with full path that contains "+
41 "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
42 "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT and ARVADOS_BLOB_SIGNING_KEY.")
46 "src-keep-services-json",
48 "An optional list of available source keepservices. "+
49 "If not provided, this list is obtained from api server configured in src-config-file.")
53 "dst-keep-services-json",
55 "An optional list of available destination keepservices. "+
56 "If not provided, this list is obtained from api server configured in dst-config-file.")
62 "Number of replications to write to the destination.")
75 if srcConfigFile == "" {
76 log.Fatal("-src-config-file must be specified.")
78 srcConfig, err = readConfigFromFile(srcConfigFile)
80 log.Fatal("Error reading source configuration: %s", err.Error())
83 if dstConfigFile == "" {
84 log.Fatal("-dst-config-file must be specified.")
86 dstConfig, err = readConfigFromFile(dstConfigFile)
88 log.Fatal("Error reading destination configuration: %s", err.Error())
91 // Initialize keep-rsync
92 err = initializeKeepRsync()
94 log.Fatal("Error configurating keep-rsync: %s", err.Error())
98 var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
100 // Reads config from file
101 func readConfigFromFile(filename string) (arvadosclient.APIConfig, error) {
102 var config arvadosclient.APIConfig
104 content, err := ioutil.ReadFile(filename)
109 lines := strings.Split(string(content), "\n")
110 for _, line := range lines {
114 kv := strings.Split(line, "=")
117 case "ARVADOS_API_TOKEN":
118 config.APIToken = kv[1]
119 case "ARVADOS_API_HOST":
120 config.APIHost = kv[1]
121 case "ARVADOS_API_HOST_INSECURE":
122 config.APIHostInsecure = matchTrue.MatchString(kv[1])
123 case "ARVADOS_EXTERNAL_CLIENT":
124 config.ExternalClient = matchTrue.MatchString(kv[1])
125 case "ARVADOS_BLOB_SIGNING_KEY":
126 blobSigningKey = kv[1]
132 // keep-rsync source and destination clients
134 arvSrc arvadosclient.ArvadosClient
135 arvDst arvadosclient.ArvadosClient
136 kcSrc *keepclient.KeepClient
137 kcDst *keepclient.KeepClient
140 // Initializes keep-rsync using the config provided
141 func initializeKeepRsync() (err error) {
142 // arvSrc from srcConfig
143 arvSrc, err = arvadosclient.New(srcConfig)
148 // arvDst from dstConfig
149 arvDst, err = arvadosclient.New(dstConfig)
154 // Get default replications value from destination, if it is not already provided
155 if replications == 0 {
156 value, err := arvDst.Discovery("defaultCollectionReplication")
158 replications = int(value.(float64))
164 // if srcKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
165 if srcKeepServicesJSON == "" {
166 kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
171 kcSrc, err = keepclient.MakeKeepClientFromJSON(&arvSrc, srcKeepServicesJSON)
177 // if dstKeepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
178 if dstKeepServicesJSON == "" {
179 kcDst, err = keepclient.MakeKeepClient(&arvDst)
184 kcDst, err = keepclient.MakeKeepClientFromJSON(&arvDst, dstKeepServicesJSON)
189 kcDst.Want_replicas = replications