From cea3337f1d104150d3314e43bc1c07eef0851bc5 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Sat, 25 Feb 2017 19:32:45 -0500 Subject: [PATCH] multiple servers, add docker-compose test --- build/run-tests.sh | 3 + .../setup_docker_compose_test.go | 23 +++++ .../test-docker-compose/agent.yml | 4 + .../test-docker-compose/docker-compose.yml | 35 ++++++++ .../test-docker-compose/encrypt-key.txt | 1 + .../test-docker-compose/master-token.txt | 1 + lib/setup/consul.go | 83 ++++++++++--------- 7 files changed, 113 insertions(+), 37 deletions(-) create mode 100644 cmd/arvados-admin/setup_docker_compose_test.go create mode 100644 cmd/arvados-admin/test-docker-compose/agent.yml create mode 100644 cmd/arvados-admin/test-docker-compose/docker-compose.yml create mode 100644 cmd/arvados-admin/test-docker-compose/encrypt-key.txt create mode 100644 cmd/arvados-admin/test-docker-compose/master-token.txt diff --git a/build/run-tests.sh b/build/run-tests.sh index aafbeade04..3cbc6b3286 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -196,6 +196,9 @@ sanity_checks() { echo -n 'gitolite: ' which gitolite \ || fatal "No gitolite. Try: apt-get install gitolite3" + echo -n 'docker-compose: ' + which docker-compose \ + || fatal "No docker-compose. Try: sudo curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` --output /usr/local/bin/docker-compose && sudo chmod +x /usr/local/bin/docker-compose" } rotate_logfile() { diff --git a/cmd/arvados-admin/setup_docker_compose_test.go b/cmd/arvados-admin/setup_docker_compose_test.go new file mode 100644 index 0000000000..d949604469 --- /dev/null +++ b/cmd/arvados-admin/setup_docker_compose_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "os/exec" + "testing" +) + +func TestSetupDockerCompose(t *testing.T) { + for _, cmdline := range [][]string{ + {"go", "build"}, + {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "down"}, + {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "up"}, + } { + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + t.Fatal(err) + } + } +} diff --git a/cmd/arvados-admin/test-docker-compose/agent.yml b/cmd/arvados-admin/test-docker-compose/agent.yml new file mode 100644 index 0000000000..46f4de740c --- /dev/null +++ b/cmd/arvados-admin/test-docker-compose/agent.yml @@ -0,0 +1,4 @@ +ControlHosts: + - sys0 + - sys1 + - sys2 diff --git a/cmd/arvados-admin/test-docker-compose/docker-compose.yml b/cmd/arvados-admin/test-docker-compose/docker-compose.yml new file mode 100644 index 0000000000..66378e5f46 --- /dev/null +++ b/cmd/arvados-admin/test-docker-compose/docker-compose.yml @@ -0,0 +1,35 @@ +version: '2' +services: + sys0: + build: ../test-debian8 + cap_add: + - IPC_LOCK + - SYS_ADMIN + volumes: + - ../arvados-admin:/usr/bin/arvados-admin:ro + - ./agent.yml:/etc/arvados/agent/agent.yml:ro + - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro + - ./master-token.txt:/var/lib/arvados/master-token.txt:ro + command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"] + sys1: + build: ../test-debian8 + cap_add: + - IPC_LOCK + - SYS_ADMIN + volumes: + - ../arvados-admin:/usr/bin/arvados-admin:ro + - ./agent.yml:/etc/arvados/agent/agent.yml:ro + - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro + - ./master-token.txt:/var/lib/arvados/master-token.txt:ro + command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && wait"] + sys2: + build: ../test-debian8 + cap_add: + - IPC_LOCK + - SYS_ADMIN + volumes: + - ../arvados-admin:/usr/bin/arvados-admin:ro + - ./agent.yml:/etc/arvados/agent/agent.yml:ro + - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro + - ./master-token.txt:/var/lib/arvados/master-token.txt:ro + command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && wait"] diff --git a/cmd/arvados-admin/test-docker-compose/encrypt-key.txt b/cmd/arvados-admin/test-docker-compose/encrypt-key.txt new file mode 100644 index 0000000000..507ff36ee9 --- /dev/null +++ b/cmd/arvados-admin/test-docker-compose/encrypt-key.txt @@ -0,0 +1 @@ +qigR/fVUccR07/J56MsloA== diff --git a/cmd/arvados-admin/test-docker-compose/master-token.txt b/cmd/arvados-admin/test-docker-compose/master-token.txt new file mode 100644 index 0000000000..f12bf2c174 --- /dev/null +++ b/cmd/arvados-admin/test-docker-compose/master-token.txt @@ -0,0 +1 @@ +2f79a06949ba76666308f5c821f234c9c038664df2b8662b587b9500ef4853a1 \ No newline at end of file diff --git a/lib/setup/consul.go b/lib/setup/consul.go index 0ebff838d8..b1682b2625 100644 --- a/lib/setup/consul.go +++ b/lib/setup/consul.go @@ -16,9 +16,9 @@ import ( func (s *Setup) installConsul() error { prog := s.UsrDir + "/bin/consul" err := (&download{ - URL: "https://releases.hashicorp.com/consul/0.7.4/consul_0.7.4_linux_amd64.zip", + URL: "https://releases.hashicorp.com/consul/0.7.5/consul_0.7.5_linux_amd64.zip", Dest: prog, - Size: 36003597, + Size: 36003713, Mode: 0755, PreloadDir: s.PreloadDir, }).install() @@ -29,43 +29,45 @@ func (s *Setup) installConsul() error { if err := os.MkdirAll(dataDir, 0700); err != nil { return err } - args := []string{"agent"} - { - cf := path.Join(s.DataDir, "consul-encrypt.json") - if _, err := os.Stat(cf); err != nil && !os.IsNotExist(err) { + + keyPath := path.Join(s.DataDir, "encrypt-key.txt") + key, err := ioutil.ReadFile(keyPath) + if os.IsNotExist(err) { + key, err = exec.Command(prog, "keygen").CombinedOutput() + if err != nil { return err - } else if err != nil { - key, err := exec.Command(prog, "keygen").CombinedOutput() - if err != nil { - return err - } - if err = atomicWriteJSON(cf, map[string]interface{}{ - "encrypt": strings.TrimSpace(string(key)), - }, 0400); err != nil { - return err - } } - args = append(args, "-config-file="+cf) + err = atomicWriteFile(keyPath, key, 0400) } - { + if err != nil { + return err + } + encryptKey := strings.TrimSpace(string(key)) + + tokPath := path.Join(s.DataDir, "master-token.txt") + if tok, err := ioutil.ReadFile(tokPath); err != nil { s.masterToken = generateToken() - // os.Setenv("CONSUL_TOKEN", s.masterToken) - err = atomicWriteFile(path.Join(s.DataDir, "master-token.txt"), []byte(s.masterToken), 0600) + err = atomicWriteFile(tokPath, []byte(s.masterToken), 0600) if err != nil { return err } - cf := path.Join(s.DataDir, "consul-config.json") - err = atomicWriteJSON(cf, map[string]interface{}{ - "acl_datacenter": s.ClusterID, - "acl_default_policy": "deny", - "acl_enforce_version_8": true, - "acl_master_token": s.masterToken, - "client_addr": "0.0.0.0", - "bootstrap_expect": len(s.ControlHosts), - "data_dir": dataDir, - "datacenter": s.ClusterID, - "server": true, - "ui": true, + } else { + s.masterToken = string(tok) + } + + cf := path.Join(s.DataDir, "consul-config.json") + { + c := map[string]interface{}{ + "acl_datacenter": s.ClusterID, + "acl_default_policy": "deny", + "acl_master_token": s.masterToken, + "bootstrap_expect": len(s.ControlHosts), + "client_addr": "0.0.0.0", + "data_dir": dataDir, + "datacenter": s.ClusterID, + "encrypt": encryptKey, + "server": true, + "ui": true, "ports": map[string]int{ "dns": s.Ports.ConsulDNS, "http": s.Ports.ConsulHTTP, @@ -75,23 +77,30 @@ func (s *Setup) installConsul() error { "serf_wan": s.Ports.ConsulSerfWAN, "server": s.Ports.ConsulServer, }, - }, 0644) + } + err = atomicWriteJSON(cf, c, 0600) if err != nil { return err } - args = append(args, "-config-file="+cf) } + err = s.installService(daemon{ name: "arvados-consul", prog: prog, - args: args, + args: []string{"agent", "-config-file=" + cf}, noRegister: true, }) if err != nil { return err } + if err = waitCheck(20*time.Second, s.consulCheck); err != nil { + return err + } if len(s.ControlHosts) > 1 { - cmd := exec.Command(prog, append([]string{"join"}, s.ControlHosts...)...) + args := []string{"join"} + args = append(args, fmt.Sprintf("-rpc-addr=127.0.0.1:%d", s.Ports.ConsulRPC)) + args = append(args, s.ControlHosts...) + cmd := exec.Command(prog, args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr err := cmd.Run() @@ -99,7 +108,7 @@ func (s *Setup) installConsul() error { return fmt.Errorf("consul join: %s", err) } } - return waitCheck(20*time.Second, s.consulCheck) + return nil } var consulCfg = api.DefaultConfig() -- 2.30.2