multiple servers, add docker-compose test

diff --git a/build/run-tests.sh b/build/run-tests.sh
index aafbeade04a1be7812a1f2bc93e43eb0c9b01b48..3cbc6b3286dd57c7784c476a609a21739f00ceeb 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -196,6 +196,9 @@ sanity_checks() {
     echo -n 'gitolite: '
     which gitolite \
         || fatal "No gitolite. Try: apt-get install gitolite3"
+    echo -n 'docker-compose: '
+    which docker-compose \
+        || fatal "No docker-compose. Try: sudo curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` --output /usr/local/bin/docker-compose && sudo chmod +x /usr/local/bin/docker-compose"
 }
 
 rotate_logfile() {

diff --git a/cmd/arvados-admin/setup_docker_compose_test.go b/cmd/arvados-admin/setup_docker_compose_test.go
new file mode 100644 (file)
index 0000000..d949604
--- /dev/null
+++ b/cmd/arvados-admin/setup_docker_compose_test.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+       "os"
+       "os/exec"
+       "testing"
+)
+
+func TestSetupDockerCompose(t *testing.T) {
+       for _, cmdline := range [][]string{
+               {"go", "build"},
+               {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "down"},
+               {"docker-compose", "--file", "test-docker-compose/docker-compose.yml", "up"},
+       } {
+               cmd := exec.Command(cmdline[0], cmdline[1:]...)
+               cmd.Stdout = os.Stderr
+               cmd.Stderr = os.Stderr
+               err := cmd.Run()
+               if err != nil {
+                       t.Fatal(err)
+               }
+       }
+}

diff --git a/cmd/arvados-admin/test-docker-compose/agent.yml b/cmd/arvados-admin/test-docker-compose/agent.yml
new file mode 100644 (file)
index 0000000..46f4de7
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/agent.yml
@@ -0,0 +1,4 @@
+ControlHosts:
+  - sys0
+  - sys1
+  - sys2

diff --git a/cmd/arvados-admin/test-docker-compose/docker-compose.yml b/cmd/arvados-admin/test-docker-compose/docker-compose.yml
new file mode 100644 (file)
index 0000000..66378e5
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/docker-compose.yml
@@ -0,0 +1,35 @@
+version: '2'
+services:
+  sys0:
+    build: ../test-debian8
+    cap_add:
+      - IPC_LOCK
+      - SYS_ADMIN
+    volumes:
+      - ../arvados-admin:/usr/bin/arvados-admin:ro
+      - ./agent.yml:/etc/arvados/agent/agent.yml:ro
+      - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
+      - ./master-token.txt:/var/lib/arvados/master-token.txt:ro
+    command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup -unseal=true && wait"]
+  sys1:
+    build: ../test-debian8
+    cap_add:
+      - IPC_LOCK
+      - SYS_ADMIN
+    volumes:
+      - ../arvados-admin:/usr/bin/arvados-admin:ro
+      - ./agent.yml:/etc/arvados/agent/agent.yml:ro
+      - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
+      - ./master-token.txt:/var/lib/arvados/master-token.txt:ro
+    command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && wait"]
+  sys2:
+    build: ../test-debian8
+    cap_add:
+      - IPC_LOCK
+      - SYS_ADMIN
+    volumes:
+      - ../arvados-admin:/usr/bin/arvados-admin:ro
+      - ./agent.yml:/etc/arvados/agent/agent.yml:ro
+      - ./encrypt-key.txt:/var/lib/arvados/encrypt-key.txt:ro
+      - ./master-token.txt:/var/lib/arvados/master-token.txt:ro
+    command: ["bash", "-c", "runsvdir /etc/sv & arvados-admin setup && wait"]

diff --git a/cmd/arvados-admin/test-docker-compose/encrypt-key.txt b/cmd/arvados-admin/test-docker-compose/encrypt-key.txt
new file mode 100644 (file)
index 0000000..507ff36
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/encrypt-key.txt
@@ -0,0 +1 @@
+qigR/fVUccR07/J56MsloA==

diff --git a/cmd/arvados-admin/test-docker-compose/master-token.txt b/cmd/arvados-admin/test-docker-compose/master-token.txt
new file mode 100644 (file)
index 0000000..f12bf2c
--- /dev/null
+++ b/cmd/arvados-admin/test-docker-compose/master-token.txt
@@ -0,0 +1 @@
+2f79a06949ba76666308f5c821f234c9c038664df2b8662b587b9500ef4853a1
\ No newline at end of file

diff --git a/lib/setup/consul.go b/lib/setup/consul.go
index 0ebff838d8dd6abb741778cab9c90312124b3ee2..b1682b26250322db4c456d535a8ba06a7aede4f6 100644 (file)
--- a/lib/setup/consul.go
+++ b/lib/setup/consul.go
@@ -16,9 +16,9 @@ import (
 func (s *Setup) installConsul() error {
        prog := s.UsrDir + "/bin/consul"
        err := (&download{
-               URL:        "https://releases.hashicorp.com/consul/0.7.4/consul_0.7.4_linux_amd64.zip",
+               URL:        "https://releases.hashicorp.com/consul/0.7.5/consul_0.7.5_linux_amd64.zip",
                Dest:       prog,
-               Size:       36003597,
+               Size:       36003713,
                Mode:       0755,
                PreloadDir: s.PreloadDir,
        }).install()
@@ -29,43 +29,45 @@ func (s *Setup) installConsul() error {
        if err := os.MkdirAll(dataDir, 0700); err != nil {
                return err
        }
-       args := []string{"agent"}
-       {
-               cf := path.Join(s.DataDir, "consul-encrypt.json")
-               if _, err := os.Stat(cf); err != nil && !os.IsNotExist(err) {
+
+       keyPath := path.Join(s.DataDir, "encrypt-key.txt")
+       key, err := ioutil.ReadFile(keyPath)
+       if os.IsNotExist(err) {
+               key, err = exec.Command(prog, "keygen").CombinedOutput()
+               if err != nil {
                        return err
-               } else if err != nil {
-                       key, err := exec.Command(prog, "keygen").CombinedOutput()
-                       if err != nil {
-                               return err
-                       }
-                       if err = atomicWriteJSON(cf, map[string]interface{}{
-                               "encrypt": strings.TrimSpace(string(key)),
-                       }, 0400); err != nil {
-                               return err
-                       }
                }
-               args = append(args, "-config-file="+cf)
+               err = atomicWriteFile(keyPath, key, 0400)
        }
-       {
+       if err != nil {
+               return err
+       }
+       encryptKey := strings.TrimSpace(string(key))
+
+       tokPath := path.Join(s.DataDir, "master-token.txt")
+       if tok, err := ioutil.ReadFile(tokPath); err != nil {
                s.masterToken = generateToken()
-               // os.Setenv("CONSUL_TOKEN", s.masterToken)
-               err = atomicWriteFile(path.Join(s.DataDir, "master-token.txt"), []byte(s.masterToken), 0600)
+               err = atomicWriteFile(tokPath, []byte(s.masterToken), 0600)
                if err != nil {
                        return err
                }
-               cf := path.Join(s.DataDir, "consul-config.json")
-               err = atomicWriteJSON(cf, map[string]interface{}{
-                       "acl_datacenter":        s.ClusterID,
-                       "acl_default_policy":    "deny",
-                       "acl_enforce_version_8": true,
-                       "acl_master_token":      s.masterToken,
-                       "client_addr":           "0.0.0.0",
-                       "bootstrap_expect":      len(s.ControlHosts),
-                       "data_dir":              dataDir,
-                       "datacenter":            s.ClusterID,
-                       "server":                true,
-                       "ui":                    true,
+       } else {
+               s.masterToken = string(tok)
+       }
+
+       cf := path.Join(s.DataDir, "consul-config.json")
+       {
+               c := map[string]interface{}{
+                       "acl_datacenter":     s.ClusterID,
+                       "acl_default_policy": "deny",
+                       "acl_master_token":   s.masterToken,
+                       "bootstrap_expect":   len(s.ControlHosts),
+                       "client_addr":        "0.0.0.0",
+                       "data_dir":           dataDir,
+                       "datacenter":         s.ClusterID,
+                       "encrypt":            encryptKey,
+                       "server":             true,
+                       "ui":                 true,
                        "ports": map[string]int{
                                "dns":      s.Ports.ConsulDNS,
                                "http":     s.Ports.ConsulHTTP,
@@ -75,23 +77,30 @@ func (s *Setup) installConsul() error {
                                "serf_wan": s.Ports.ConsulSerfWAN,
                                "server":   s.Ports.ConsulServer,
                        },
-               }, 0644)
+               }
+               err = atomicWriteJSON(cf, c, 0600)
                if err != nil {
                        return err
                }
-               args = append(args, "-config-file="+cf)
        }
+
        err = s.installService(daemon{
                name:       "arvados-consul",
                prog:       prog,
-               args:       args,
+               args:       []string{"agent", "-config-file=" + cf},
                noRegister: true,
        })
        if err != nil {
                return err
        }
+       if err = waitCheck(20*time.Second, s.consulCheck); err != nil {
+               return err
+       }
        if len(s.ControlHosts) > 1 {
-               cmd := exec.Command(prog, append([]string{"join"}, s.ControlHosts...)...)
+               args := []string{"join"}
+               args = append(args, fmt.Sprintf("-rpc-addr=127.0.0.1:%d", s.Ports.ConsulRPC))
+               args = append(args, s.ControlHosts...)
+               cmd := exec.Command(prog, args...)
                cmd.Stdout = os.Stderr
                cmd.Stderr = os.Stderr
                err := cmd.Run()
@@ -99,7 +108,7 @@ func (s *Setup) installConsul() error {
                        return fmt.Errorf("consul join: %s", err)
                }
        }
-       return waitCheck(20*time.Second, s.consulCheck)
+       return nil
 }
 
 var consulCfg = api.DefaultConfig()