// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
// ARVADOS_API_HOST_INSECURE.
func MakeArvadosClient() (ac ArvadosClient, err error) {
+ config := make(map[string]string)
+ config["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ config["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ config["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+ config["ARVADOS_EXTERNAL_CLIENT"] = os.Getenv("ARVADOS_EXTERNAL_CLIENT")
+
+ return MakeArvadosClientWithConfig(config)
+}
+
+// Create a new ArvadosClient, using the given input parameters.
+func MakeArvadosClientWithConfig(config map[string]string) (ac ArvadosClient, err error) {
var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
- insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
- external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
+
+ insecure := matchTrue.MatchString(config["ARVADOS_API_HOST_INSECURE"])
+ external := matchTrue.MatchString(config["ARVADOS_EXTERNAL_CLIENT"])
ac = ArvadosClient{
- ApiServer: os.Getenv("ARVADOS_API_HOST"),
- ApiToken: os.Getenv("ARVADOS_API_TOKEN"),
+ ApiServer: config["ARVADOS_API_HOST"],
+ ApiToken: config["ARVADOS_API_TOKEN"],
ApiInsecure: insecure,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
}
func StartKeep() {
+ StartKeepAdditional(false)
+}
+
+func StartKeepAdditional(keepExisting bool) {
cwd, _ := os.Getwd()
defer os.Chdir(cwd)
chdirToPythonTests()
cmd := exec.Command("python", "run_test_server.py", "start_keep")
+ if keepExisting {
+ cmd = exec.Command("python", "run_test_server.py", "start_keep", "--keep_existing", "true")
+ }
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatalf("Setting up stderr pipe: %s", err)
return port
def run_keep(blob_signing_key=None, enforce_permissions=False):
- stop_keep()
+ if args.keep_existing is None:
+ stop_keep()
keep_args = {}
if not blob_signing_key:
host=os.environ['ARVADOS_API_HOST'],
token=os.environ['ARVADOS_API_TOKEN'],
insecure=True)
+
for d in api.keep_services().list().execute()['items']:
api.keep_services().delete(uuid=d['uuid']).execute()
for d in api.keep_disks().list().execute()['items']:
api.keep_disks().delete(uuid=d['uuid']).execute()
- for d in range(0, 2):
+ start_index = 0
+ if args.keep_existing is not None:
+ start_index = 2
+ for d in range(start_index, start_index+2):
port = _start_keep(d, keep_args)
svc = api.keep_services().create(body={'keep_service': {
'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
def stop_keep():
_stop_keep(0)
_stop_keep(1)
+ # We may have created 2 additional keep servers when keep_existing is used
+ _stop_keep(2)
+ _stop_keep(3)
def run_keep_proxy():
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
parser = argparse.ArgumentParser()
parser.add_argument('action', type=str, help="one of {}".format(actions))
parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
+ parser.add_argument('--keep_existing', type=str, help="Used to add additional keep servers, without terminating existing servers")
args = parser.parse_args()
if args.action not in actions:
--- /dev/null
+keep-rsync
--- /dev/null
+package main
+
+import (
+ "flag"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "io/ioutil"
+ "log"
+ "strings"
+)
+
+// keep-rsync arguments
+var (
+ srcConfig map[string]string
+ destConfig map[string]string
+ srcKeepServicesJSON string
+ destKeepServicesJSON string
+ replications int
+ prefix string
+)
+
+func main() {
+ var srcConfigFile string
+ var destConfigFile string
+
+ flag.StringVar(
+ &srcConfigFile,
+ "src-config-file",
+ "",
+ "Source configuration filename with full path that contains "+
+ "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+
+ "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
+
+ flag.StringVar(
+ &destConfigFile,
+ "dest-config-file",
+ "",
+ "Destination configuration filename with full path that contains "+
+ "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+
+ "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.")
+
+ flag.StringVar(
+ &srcKeepServicesJSON,
+ "src-keep-services-json",
+ "",
+ "An optional list of available source keepservices. "+
+ "If not provided, this list is obtained from api server configured in src-config-file.")
+
+ flag.StringVar(
+ &destKeepServicesJSON,
+ "dest-keep-services-json",
+ "",
+ "An optional list of available destination keepservices. "+
+ "If not provided, this list is obtained from api server configured in dest-config-file.")
+
+ flag.IntVar(
+ &replications,
+ "replications",
+ 3,
+ "Number of replications to write to the destination.")
+
+ flag.StringVar(
+ &prefix,
+ "prefix",
+ "",
+ "Index prefix")
+
+ flag.Parse()
+
+ var err error
+
+ if srcConfigFile == "" {
+ log.Fatal("-src-config-file must be specified.")
+ }
+ srcConfig, err = readConfigFromFile(srcConfigFile)
+ if err != nil {
+ log.Fatal("Error reading source configuration: %s", err.Error())
+ }
+
+ if destConfigFile == "" {
+ log.Fatal("-dest-config-file must be specified.")
+ }
+ destConfig, err = readConfigFromFile(destConfigFile)
+ if err != nil {
+ log.Fatal("Error reading destination configuration: %s", err.Error())
+ }
+
+ err = initializeKeepRsync()
+ if err != nil {
+ log.Fatal("Error configurating keep-rsync: %s", err.Error())
+ }
+}
+
+// Reads config from file
+func readConfigFromFile(filename string) (map[string]string, error) {
+ content, err := ioutil.ReadFile(filename)
+ if err != nil {
+ return nil, err
+ }
+
+ config := make(map[string]string)
+ lines := strings.Split(string(content), "\n")
+ for _, line := range lines {
+ if line == "" {
+ continue
+ }
+ kv := strings.Split(line, "=")
+ config[kv[0]] = kv[1]
+ }
+ return config, nil
+}
+
+// keep-rsync source and destination clients
+var (
+ arvSrc arvadosclient.ArvadosClient
+ arvDest arvadosclient.ArvadosClient
+ kcSrc *keepclient.KeepClient
+ kcDest *keepclient.KeepClient
+)
+
+// Initializes keep-rsync using the config provided
+func initializeKeepRsync() (err error) {
+ arvSrc, err = arvadosclient.MakeArvadosClientWithConfig(srcConfig)
+ if err != nil {
+ return
+ }
+
+ arvDest, err = arvadosclient.MakeArvadosClientWithConfig(destConfig)
+ if err != nil {
+ return
+ }
+
+ kcSrc, err = keepclient.MakeKeepClient(&arvSrc)
+ if err != nil {
+ return
+ }
+
+ kcDest, err = keepclient.MakeKeepClient(&arvDest)
+
+ return
+}
--- /dev/null
+package main
+
+import (
+ "crypto/md5"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredSuite{})
+
+// Tests that require the Keep server running
+type ServerRequiredSuite struct{}
+
+func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+}
+
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredSuite) TearDownSuite(c *C) {
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
+}
+
+// Testing keep-rsync needs two sets of keep services: src and dest.
+// The test setup hence tweaks keep-rsync initialzation to achieve this.
+// First invoke initializeKeepRsync and then invoke StartKeepAdditional
+// to create the keep servers to be used as destination.
+func setupRsync(c *C) {
+ // srcConfig
+ srcConfig = make(map[string]string)
+ srcConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ srcConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ srcConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+
+ // destConfig
+ destConfig = make(map[string]string)
+ destConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST")
+ destConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN")
+ destConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE")
+
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+
+ // initialize keep-rsync
+ err := initializeKeepRsync()
+ c.Assert(err, Equals, nil)
+
+ // Create two more keep servers to be used as destination
+ arvadostest.StartKeepAdditional(true)
+
+ // load kcDest
+ kcDest, err = keepclient.MakeKeepClient(&arvDest)
+ c.Assert(err, Equals, nil)
+}
+
+// Test readConfigFromFile method
+func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) {
+ // Setup a test config file
+ file, err := ioutil.TempFile(os.TempDir(), "config")
+ c.Assert(err, Equals, nil)
+ defer os.Remove(file.Name())
+
+ fileContent := "ARVADOS_API_HOST=testhost\n"
+ fileContent += "ARVADOS_API_TOKEN=testtoken\n"
+ fileContent += "ARVADOS_API_HOST_INSECURE=true"
+
+ _, err = file.Write([]byte(fileContent))
+
+ // Invoke readConfigFromFile method with this test filename
+ config, err := readConfigFromFile(file.Name())
+ c.Assert(err, Equals, nil)
+ c.Assert(config["ARVADOS_API_HOST"], Equals, "testhost")
+ c.Assert(config["ARVADOS_API_TOKEN"], Equals, "testtoken")
+ c.Assert(config["ARVADOS_API_HOST_INSECURE"], Equals, "true")
+ c.Assert(config["EXTERNAL_CLIENT"], Equals, "")
+}
+
+// Test keep-rsync initialization, with src and dest keep servers.
+// Do a Put and Get in src, both of which should succeed.
+// Do a Get in dest for the same hash, which should raise block not found error.
+func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDestShouldFail(c *C) {
+ setupRsync(c)
+
+ // Put a block in src using kcSrc and Get it
+ data := []byte("test-data")
+ hash := fmt.Sprintf("%x", md5.Sum(data))
+
+ hash2, rep, err := kcSrc.PutB(data)
+ c.Check(hash2, Matches, fmt.Sprintf(`^%s\+9(\+.+)?$`, hash))
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+
+ reader, blocklen, _, err := kcSrc.Get(hash)
+ c.Assert(err, Equals, nil)
+ c.Check(blocklen, Equals, int64(9))
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, data)
+
+ // Get using kcDest should fail with NotFound error
+ _, _, _, err = kcDest.Get(hash)
+ c.Assert(err.Error(), Equals, "Block not found")
+}
projects / arvados.git / commitdiff