15 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
16 "git.curoverse.com/arvados.git/sdk/go/keepclient"
20 err := doMain(os.Args[1:])
26 func doMain(args []string) error {
27 flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError)
29 configFile := flags.String(
32 "Configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY for the source.")
34 keepServicesJSON := flags.String(
37 "An optional list of available keepservices. "+
38 "If not provided, this list is obtained from api server configured in config-file.")
40 locatorFile := flags.String(
43 "Filename containing the block hashes to be checked. This is required. "+
44 "This file contains the block hashes one per line.")
46 prefix := flags.String(
49 "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
51 blobSignatureTTLFlag := flags.Duration(
54 "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.")
56 verbose := flags.Bool(
59 "Log progress of each block verification")
61 // Parse args; omit the first arg which is the command name
64 config, blobSigningKey, err := loadConfig(*configFile)
66 return fmt.Errorf("Error loading configuration from file: %s", err.Error())
69 // get list of block locators to be checked
70 blockLocators, err := getBlockLocators(*locatorFile, *prefix)
72 return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error())
76 kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag)
78 return fmt.Errorf("Error configuring keepclient: %s", err.Error())
81 return performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
84 type apiConfig struct {
91 // Load config from given file
92 func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err error) {
94 err = errors.New("Client config file not specified")
98 config, blobSigningKey, err = readConfigFromFile(configFile)
102 // Read config from file
103 func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) {
104 if !strings.Contains(filename, "/") {
105 filename = os.Getenv("HOME") + "/.config/arvados/" + filename + ".conf"
108 content, err := ioutil.ReadFile(filename)
114 lines := strings.Split(string(content), "\n")
115 for _, line := range lines {
120 kv := strings.SplitN(line, "=", 2)
122 key := strings.TrimSpace(kv[0])
123 value := strings.TrimSpace(kv[1])
126 case "ARVADOS_API_TOKEN":
127 config.APIToken = value
128 case "ARVADOS_API_HOST":
129 config.APIHost = value
130 case "ARVADOS_API_HOST_INSECURE":
131 config.APIHostInsecure = arvadosclient.StringBool(value)
132 case "ARVADOS_EXTERNAL_CLIENT":
133 config.ExternalClient = arvadosclient.StringBool(value)
134 case "ARVADOS_BLOB_SIGNING_KEY":
135 blobSigningKey = value
143 // setup keepclient using the config provided
144 func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {
145 arv := arvadosclient.ArvadosClient{
146 ApiToken: config.APIToken,
147 ApiServer: config.APIHost,
148 ApiInsecure: config.APIHostInsecure,
149 Client: &http.Client{Transport: &http.Transport{
150 TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
151 External: config.ExternalClient,
154 // If keepServicesJSON is provided, use it instead of service discovery
155 if keepServicesJSON == "" {
156 kc, err = keepclient.MakeKeepClient(&arv)
161 kc = keepclient.New(&arv)
162 err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
168 // Get if blobSignatureTTL is not provided
169 ttl = blobSignatureTTL
170 if blobSignatureTTL == 0 {
171 value, err := arv.Discovery("blobSignatureTtl")
173 ttl = time.Duration(int(value.(float64))) * time.Second
182 // Get list of unique block locators from the given file
183 func getBlockLocators(locatorFile, prefix string) (locators []string, err error) {
184 if locatorFile == "" {
185 err = errors.New("block-hash-file not specified")
189 content, err := ioutil.ReadFile(locatorFile)
194 locatorMap := make(map[string]bool)
195 for _, line := range strings.Split(string(content), "\n") {
196 line = strings.TrimSpace(line)
197 if line == "" || !strings.HasPrefix(line, prefix) || locatorMap[line] {
200 locators = append(locators, line)
201 locatorMap[line] = true
207 // Get block headers from keep. Log any errors.
208 func performKeepBlockCheck(kc *keepclient.KeepClient, blobSignatureTTL time.Duration, blobSigningKey string, blockLocators []string, verbose bool) error {
209 totalBlocks := len(blockLocators)
212 for _, locator := range blockLocators {
215 log.Printf("Verifying block %d of %d: %v", current, totalBlocks, locator)
217 getLocator := locator
218 if blobSigningKey != "" {
219 expiresAt := time.Now().AddDate(0, 0, 1)
220 getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, blobSignatureTTL, []byte(blobSigningKey))
223 _, _, err := kc.Ask(getLocator)
226 log.Printf("Error verifying block %v: %v", locator, err)
230 log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
232 if notFoundBlocks > 0 {
233 return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)