From: Peter Amstutz <pamstutz@veritasgenetics.com>
Date: Thu, 1 Aug 2019 13:49:58 +0000 (-0400)
Subject: 15467: Adding KeepServices WIP
X-Git-Tag: 2.0.0~229^2~6
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/dbbb649c9b797da8f6805af3a7e74e57b075d294

15467: Adding KeepServices WIP

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>
---

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 2b1da2f2a8..051d478877 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -525,6 +525,9 @@ Clusters:
       SLURM:
         PrioritySpread: 0
         SbatchArgumentsList: []
+        KeepServices:
+          SAMPLE:
+            InternalURLs: {}
         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index cfd77ced23..ce9b2e4d01 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -7,6 +7,7 @@ package config
 import (
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"strings"
 
@@ -197,6 +198,16 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
 		cluster.SystemRootToken = client.AuthToken
 	}
 	cluster.TLS.Insecure = client.Insecure
+	for _, r := range client.KeepServiceURIs {
+		if cluster.Containers.SLURM.KeepServices == nil {
+			cluster.Containers.SLURM.KeepServices = make(map[string]arvados.Service)
+		}
+		if cluster.Containers.SLURM.KeepServices["00000-bi6l4-000000000000000"].InternalURLs == nil {
+			cluster.Containers.SLURM.KeepServices["00000-bi6l4-000000000000000"].InternalURLs = map[arvados.URL]arvados.ServiceInstance{}
+		}
+		p, err := url.Parse(r)
+		cluster.Containers.SLURM.KeepServices["00000-bi6l4-000000000000000"].InternalURLs[arvados.URL(p)] = struct{}{}
+	}
 }
 
 // update config using values from an crunch-dispatch-slurm config file.
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index bee93046eb..f63996437f 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -294,6 +294,7 @@ type ContainersConfig struct {
 	SLURM struct {
 		PrioritySpread      int64
 		SbatchArgumentsList []string
+		KeepServices        map[string]Service
 		Managed             struct {
 			DNSServerConfDir       string
 			DNSServerConfTemplate  string
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 9f69c44460..300e923640 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -132,7 +132,16 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 		if disp.Client.Insecure {
 			os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
 		}
-		os.Setenv("ARVADOS_KEEP_SERVICES", strings.Join(disp.Client.KeepServiceURIs, " "))
+		ks := ""
+		if length(disp.cluster.Containers.SLURM.KeepServices) > 0 {
+			for _, svc := range disp.cluster.Containers.SLURM.KeepServices {
+				for k, _ := range svc.InternalURLs {
+					ks += k
+					ks += " "
+				}
+			}
+		}
+		os.Setenv("ARVADOS_KEEP_SERVICES", ks)
 		os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 	} else {
 		disp.logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 6007c6d4a8..4a0213d477 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -396,6 +396,8 @@ func (s *StubbedSuite) TestLoadLegacyConfig(c *C) {
 Client:
   APIHost: example.com
   AuthToken: abcdefg
+  KeepServices:
+    - https://example.com/keep
 SbatchArguments: ["--foo", "bar"]
 PollPeriod: 12s
 PrioritySpread: 42
@@ -431,4 +433,11 @@ BatchSize: 99
 	c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345))
 	c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second))
 	c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99)
+	c.Check(s.disp.cluster.Containers.SLURM.KeepServices, DeepEquals, map[string]Service{
+		"00000-bi6l4-000000000000000": Service{
+			InternalURLs: map[string]struct{}{
+				"https://example.com/keep": struct{}{},
+			},
+		},
+	})
 }