From: Tom Clegg <tom@curoverse.com>
Date: Fri, 17 Feb 2017 08:09:34 +0000 (-0500)
Subject: more vault
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/016a995d5a65f9de66c505fe80ab49eebc1a11c4?ds=sidebyside

more vault
---

diff --git a/services/boot/controller.go b/services/boot/controller.go
index 4b3e249ac5..abd07255ea 100644
--- a/services/boot/controller.go
+++ b/services/boot/controller.go
@@ -17,20 +17,16 @@ func (c *controller) Boot(ctx context.Context) error {
 		},
 		Concurrent{
 			postgresql,
-			Concurrent{
-				&download{
-					URL:  "https://releases.hashicorp.com/consul-template/0.18.0/consul-template_0.18.0_linux_amd64.zip",
-					Dest: path.Join(cfg.UsrDir, "bin", "consul-template"),
-					Size: 6912352,
-					Mode: 0755,
-				},
-				consul,
-			},
-			Concurrent{
-				vault,
-				nomad,
+			&download{
+				URL:  "https://releases.hashicorp.com/consul-template/0.18.0/consul-template_0.18.0_linux_amd64.zip",
+				Dest: path.Join(cfg.UsrDir, "bin", "consul-template"),
+				Size: 6912352,
+				Mode: 0755,
 			},
 		},
+		consul,
+		vault,
+		nomad,
 		// Concurrent{
 		// 	dispatchLocal,
 		// 	dispatchSLURM,
diff --git a/services/boot/nomad.go b/services/boot/nomad.go
index 632f993f2c..0dfdd1189a 100644
--- a/services/boot/nomad.go
+++ b/services/boot/nomad.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"sync"
@@ -36,6 +37,11 @@ func (nb *nomadBooter) Boot(ctx context.Context) error {
 		return err
 	}
 
+	masterToken, err := ioutil.ReadFile(cfg.masterTokenFile())
+	if err != nil {
+		return err
+	}
+
 	dataDir := path.Join(cfg.DataDir, "nomad")
 	if err := os.MkdirAll(dataDir, 0700); err != nil {
 		return err
@@ -51,6 +57,7 @@ func (nb *nomadBooter) Boot(ctx context.Context) error {
 		},
 		"consul": map[string]interface{}{
 			"address": fmt.Sprintf("127.0.0.1:%d", cfg.Ports.ConsulHTTP),
+			"token":   string(masterToken),
 		},
 		"data_dir":   dataDir,
 		"datacenter": cfg.SiteID,
diff --git a/services/boot/vault.go b/services/boot/vault.go
index f01dbb626f..4d063e71a3 100644
--- a/services/boot/vault.go
+++ b/services/boot/vault.go
@@ -128,31 +128,54 @@ func (vb *vaultBooter) tryInit(ctx context.Context) error {
 		return fmt.Errorf("vault unseal failed!")
 	}
 
+	// Use master token to create a management token
 	master, err := consul.master(ctx)
 	if err != nil {
 		return err
 	}
-	token, _, err := master.ACL().Create(&consulAPI.ACLEntry{Name: "vault", Type: "management"}, nil)
+	mgmtToken, _, err := master.ACL().Create(&consulAPI.ACLEntry{Name: "vault", Type: "management"}, nil)
 	if err != nil {
 		return err
 	}
-	err = waitCheck(ctx, 30*time.Second, func(context.Context) error {
+	if err = atomicWriteFile(path.Join(cfg.DataDir, "vault-mgmt-token.txt"), []byte(mgmtToken), 0400); err != nil {
+		return err
+	}
+
+	// Mount+configure consul backend
+	if err = waitCheck(ctx, 30*time.Second, func(context.Context) error {
+		// Typically this first fails "500 node not active but
+		// active node not found" but then succeeds.
 		return vault.Sys().Mount("consul", &api.MountInput{Type: "consul"})
-	})
-	if err != nil {
+	}); err != nil {
 		return err
 	}
 	_, err = vault.Logical().Write("consul/config/access", map[string]interface{}{
 		"address": fmt.Sprintf("127.0.0.1:%d", cfg.Ports.ConsulHTTP),
-		"token":   string(token),
+		"token":   string(mgmtToken),
 	})
 	if err != nil {
 		return err
 	}
+
+	// Create a role
 	_, err = vault.Logical().Write("consul/roles/write-all", map[string]interface{}{
 		"policy": base64.StdEncoding.EncodeToString([]byte(`key "" { policy = "write" }`)),
 	})
-	return err
+	if err != nil {
+		return err
+	}
+
+	// Generate a new token with the write-all role
+	secret, err := vault.Logical().Read("consul/creds/write-all")
+	if err != nil {
+		return err
+	}
+	token, ok := secret.Data["token"].(string)
+	if !ok {
+		return fmt.Errorf("secret token broken?? %+v", secret)
+	}
+	log.Printf("Vault supplied token with lease duration %s (renewable=%v): %q", time.Duration(secret.LeaseDuration)*time.Second, secret.Renewable, token)
+	return nil
 }
 
 func (vb *vaultBooter) client(ctx context.Context) (*api.Client, error) {