projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
17610: Check scopes when using a remote token.
[arvados.git]
/
tools
/
keep-block-check
/
keep-block-check.go
diff --git
a/tools/keep-block-check/keep-block-check.go
b/tools/keep-block-check/keep-block-check.go
index ed546f0104db8068fa7d7847cc9b9f2fb718f638..fec699f19f9886e16908d64c4e537e8023eaf0e8 100644
(file)
--- a/
tools/keep-block-check/keep-block-check.go
+++ b/
tools/keep-block-check/keep-block-check.go
@@
-1,3
+1,7
@@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
import (
package main
import (
@@
-5,17
+9,19
@@
import (
"errors"
"flag"
"fmt"
"errors"
"flag"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
"io/ioutil"
"log"
"net/http"
"os"
"io/ioutil"
"log"
"net/http"
"os"
- "regexp"
"strings"
"time"
"strings"
"time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
)
)
+var version = "dev"
+
func main() {
err := doMain(os.Args[1:])
if err != nil {
func main() {
err := doMain(os.Args[1:])
if err != nil {
@@
-48,19
+54,30
@@
func doMain(args []string) error {
"",
"Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
"",
"Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
- blobSign
ingTTL
:= flags.Duration(
- "blob-sign
ing
-ttl",
- 0
*time.Second
,
- "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the
keepservers
.")
+ blobSign
atureTTLFlag
:= flags.Duration(
+ "blob-sign
ature
-ttl",
+ 0,
+ "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the
API server's discovery document
.")
verbose := flags.Bool(
"v",
false,
"Log progress of each block verification")
verbose := flags.Bool(
"v",
false,
"Log progress of each block verification")
+ getVersion := flags.Bool(
+ "version",
+ false,
+ "Print version information and exit.")
+
// Parse args; omit the first arg which is the command name
flags.Parse(args)
// Parse args; omit the first arg which is the command name
flags.Parse(args)
+ // Print version information if requested
+ if *getVersion {
+ fmt.Printf("keep-block-check %s\n", version)
+ os.Exit(0)
+ }
+
config, blobSigningKey, err := loadConfig(*configFile)
if err != nil {
return fmt.Errorf("Error loading configuration from file: %s", err.Error())
config, blobSigningKey, err := loadConfig(*configFile)
if err != nil {
return fmt.Errorf("Error loading configuration from file: %s", err.Error())
@@
-73,12
+90,12
@@
func doMain(args []string) error {
}
// setup keepclient
}
// setup keepclient
- kc,
err := setupKeepClient(config, *keepServicesJSON, *blobSigningTTL
)
+ kc,
blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag
)
if err != nil {
return fmt.Errorf("Error configuring keepclient: %s", err.Error())
}
if err != nil {
return fmt.Errorf("Error configuring keepclient: %s", err.Error())
}
- return performKeepBlockCheck(kc,
*blobSigning
TTL, blobSigningKey, blockLocators, *verbose)
+ return performKeepBlockCheck(kc,
blobSignature
TTL, blobSigningKey, blockLocators, *verbose)
}
type apiConfig struct {
}
type apiConfig struct {
@@
-99,8
+116,6
@@
func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err
return
}
return
}
-var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
-
// Read config from file
func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) {
if !strings.Contains(filename, "/") {
// Read config from file
func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) {
if !strings.Contains(filename, "/") {
@@
-130,9
+145,9
@@
func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
case "ARVADOS_API_HOST":
config.APIHost = value
case "ARVADOS_API_HOST_INSECURE":
case "ARVADOS_API_HOST":
config.APIHost = value
case "ARVADOS_API_HOST_INSECURE":
- config.APIHostInsecure =
matchTrue.MatchString
(value)
+ config.APIHostInsecure =
arvadosclient.StringBool
(value)
case "ARVADOS_EXTERNAL_CLIENT":
case "ARVADOS_EXTERNAL_CLIENT":
- config.ExternalClient =
matchTrue.MatchString
(value)
+ config.ExternalClient =
arvadosclient.StringBool
(value)
case "ARVADOS_BLOB_SIGNING_KEY":
blobSigningKey = value
}
case "ARVADOS_BLOB_SIGNING_KEY":
blobSigningKey = value
}
@@
-143,7
+158,7
@@
func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
}
// setup keepclient using the config provided
}
// setup keepclient using the config provided
-func setupKeepClient(config apiConfig, keepServicesJSON string, blobSign
ingTTL time.Duration) (kc *keepclient.KeepClient
, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, blobSign
atureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration
, err error) {
arv := arvadosclient.ArvadosClient{
ApiToken: config.APIToken,
ApiServer: config.APIHost,
arv := arvadosclient.ArvadosClient{
ApiToken: config.APIToken,
ApiServer: config.APIHost,
@@
-153,7
+168,7
@@
func setupKeepClient(config apiConfig, keepServicesJSON string, blobSigningTTL t
External: config.ExternalClient,
}
External: config.ExternalClient,
}
- //
if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+ //
If keepServicesJSON is provided, use it instead of service discovery
if keepServicesJSON == "" {
kc, err = keepclient.MakeKeepClient(&arv)
if err != nil {
if keepServicesJSON == "" {
kc, err = keepclient.MakeKeepClient(&arv)
if err != nil {
@@
-167,13
+182,14
@@
func setupKeepClient(config apiConfig, keepServicesJSON string, blobSigningTTL t
}
}
}
}
- // Get if blobSigningTTL is not provided
- if blobSigningTTL == 0 {
+ // Get if blobSignatureTTL is not provided
+ ttl = blobSignatureTTL
+ if blobSignatureTTL == 0 {
value, err := arv.Discovery("blobSignatureTtl")
if err == nil {
value, err := arv.Discovery("blobSignatureTtl")
if err == nil {
-
blobSigningTTL
= time.Duration(int(value.(float64))) * time.Second
+
ttl
= time.Duration(int(value.(float64))) * time.Second
} else {
} else {
- return nil, err
+ return nil,
0,
err
}
}
}
}
@@
-206,7
+222,7
@@
func getBlockLocators(locatorFile, prefix string) (locators []string, err error)
}
// Get block headers from keep. Log any errors.
}
// Get block headers from keep. Log any errors.
-func performKeepBlockCheck(kc *keepclient.KeepClient, blobSign
ing
TTL time.Duration, blobSigningKey string, blockLocators []string, verbose bool) error {
+func performKeepBlockCheck(kc *keepclient.KeepClient, blobSign
ature
TTL time.Duration, blobSigningKey string, blockLocators []string, verbose bool) error {
totalBlocks := len(blockLocators)
notFoundBlocks := 0
current := 0
totalBlocks := len(blockLocators)
notFoundBlocks := 0
current := 0
@@
-218,7
+234,7
@@
func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningTTL time.Durati
getLocator := locator
if blobSigningKey != "" {
expiresAt := time.Now().AddDate(0, 0, 1)
getLocator := locator
if blobSigningKey != "" {
expiresAt := time.Now().AddDate(0, 0, 1)
- getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, blobSign
ing
TTL, []byte(blobSigningKey))
+ getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, blobSign
ature
TTL, []byte(blobSigningKey))
}
_, _, err := kc.Ask(getLocator)
}
_, _, err := kc.Ask(getLocator)
@@
-231,7
+247,7
@@
func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningTTL time.Durati
log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
if notFoundBlocks > 0 {
log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
if notFoundBlocks > 0 {
- return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix
.
", notFoundBlocks, totalBlocks)
+ return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix", notFoundBlocks, totalBlocks)
}
return nil
}
return nil