15467: Adding KeepServices WIP

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 2b1da2f2a86b734d43464867c819a4be81dbbd7c..051d478877637620aceb4cb41035825de3288208 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -525,6 +525,9 @@ Clusters:
       SLURM:
         PrioritySpread: 0
         SbatchArgumentsList: []
       SLURM:
         PrioritySpread: 0
         SbatchArgumentsList: []

+        KeepServices:
+          SAMPLE:
+            InternalURLs: {}

         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config
         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index cfd77ced23bd4e7a228f57a712c99d269ad964fa..ce9b2e4d01d3654c73faf8855acc99c53832cc74 100644 (file)
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -7,6 +7,7 @@ package config
 import (
        "fmt"
        "io/ioutil"
 import (
        "fmt"
        "io/ioutil"

+       "net/url"

        "os"
        "strings"
 
        "os"
        "strings"
 


@@ -197,6 +198,16 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
                cluster.SystemRootToken = client.AuthToken
        }
        cluster.TLS.Insecure = client.Insecure
                cluster.SystemRootToken = client.AuthToken
        }
        cluster.TLS.Insecure = client.Insecure

+       for _, r := range client.KeepServiceURIs {
+               if cluster.Containers.SLURM.KeepServices == nil {
+                       cluster.Containers.SLURM.KeepServices = make(map[string]arvados.Service)
+               }
+               if cluster.Containers.SLURM.KeepServices["00000-bi6l4-000000000000000"].InternalURLs == nil {
+                       cluster.Containers.SLURM.KeepServices["00000-bi6l4-000000000000000"].InternalURLs = map[arvados.URL]arvados.ServiceInstance{}
+               }
+               p, err := url.Parse(r)
+               cluster.Containers.SLURM.KeepServices["00000-bi6l4-000000000000000"].InternalURLs[arvados.URL(p)] = struct{}{}
+       }

 }
 
 // update config using values from an crunch-dispatch-slurm config file.
 }
 
 // update config using values from an crunch-dispatch-slurm config file.

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index bee93046eb8e696f554e31e04a7b6fb0a9fb37dc..f63996437f3ae8989c8511014a2c6ed22b2a5492 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -294,6 +294,7 @@ type ContainersConfig struct {
        SLURM struct {
                PrioritySpread      int64
                SbatchArgumentsList []string
        SLURM struct {
                PrioritySpread      int64
                SbatchArgumentsList []string

+               KeepServices        map[string]Service

                Managed             struct {
                        DNSServerConfDir       string
                        DNSServerConfTemplate  string
                Managed             struct {
                        DNSServerConfDir       string
                        DNSServerConfTemplate  string

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 9f69c44460caf7e03733acf619817f31e3a72327..300e923640bdd0d4cb2e6bb8ecf7a5c83cfa0024 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -132,7 +132,16 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
                if disp.Client.Insecure {
                        os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
                }
                if disp.Client.Insecure {
                        os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
                }

-               os.Setenv("ARVADOS_KEEP_SERVICES", strings.Join(disp.Client.KeepServiceURIs, " "))
+               ks := ""
+               if length(disp.cluster.Containers.SLURM.KeepServices) > 0 {
+                       for _, svc := range disp.cluster.Containers.SLURM.KeepServices {
+                               for k, _ := range svc.InternalURLs {
+                                       ks += k
+                                       ks += " "
+                               }
+                       }
+               }
+               os.Setenv("ARVADOS_KEEP_SERVICES", ks)

                os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
        } else {
                disp.logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
                os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
        } else {
                disp.logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 6007c6d4a80c5e3d151ed96485fdfabf56c92d1b..4a0213d4777ff78dbd70e58c2f5f28d6daf20fd7 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -396,6 +396,8 @@ func (s *StubbedSuite) TestLoadLegacyConfig(c *C) {
 Client:
   APIHost: example.com
   AuthToken: abcdefg
 Client:
   APIHost: example.com
   AuthToken: abcdefg

+  KeepServices:
+    - https://example.com/keep

 SbatchArguments: ["--foo", "bar"]
 PollPeriod: 12s
 PrioritySpread: 42
 SbatchArguments: ["--foo", "bar"]
 PollPeriod: 12s
 PrioritySpread: 42


@@ -431,4 +433,11 @@ BatchSize: 99
        c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345))
        c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second))
        c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99)
        c.Check(s.disp.cluster.Containers.ReserveExtraRAM, Equals, arvados.ByteSize(12345))
        c.Check(s.disp.cluster.Containers.MinRetryPeriod, Equals, arvados.Duration(13*time.Second))
        c.Check(s.disp.cluster.API.MaxItemsPerResponse, Equals, 99)

+       c.Check(s.disp.cluster.Containers.SLURM.KeepServices, DeepEquals, map[string]Service{
+               "00000-bi6l4-000000000000000": Service{
+                       InternalURLs: map[string]struct{}{
+                               "https://example.com/keep": struct{}{},
+                       },
+               },
+       })

 }
 }