17967: Use StorageClasses.*.Default instead of ["default"].

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index ad04807712dd00d7e89e478be1a7eb2c01d0b057..d8e673bd343d772df9e76fcad6c68facefc55855 100644 (file)
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -913,7 +913,7 @@ class ArvPutUploadJob(object):
             self._local_collection = arvados.collection.Collection(
                 self._state['manifest'],
                 replication_desired=self.replication_desired,
-                storage_classes_desired=(self.storage_classes or ['default']),
+                storage_classes_desired=self.storage_classes,
                 put_threads=self.put_threads,
                 api_client=self._api_client,
                 num_retries=self.num_retries)

diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 4e7b64cf5374fb38003d70790aebd8caee0931fb..5edca82a0478c4c4f5ca73c5245d1401e8c714f7 100644 (file)
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -17,7 +17,7 @@ class Collection < ArvadosModel
   # Posgresql JSONB columns should NOT be declared as serialized, Rails 5
   # already know how to properly treat them.
   attribute :properties, :jsonbHash, default: {}
-  attribute :storage_classes_desired, :jsonbArray, default: ["default"]
+  attribute :storage_classes_desired, :jsonbArray, default: Rails.configuration.DefaultStorageClasses
   attribute :storage_classes_confirmed, :jsonbArray, default: []
 
   before_validation :default_empty_manifest
@@ -630,7 +630,7 @@ class Collection < ArvadosModel
   # validation on empty desired storage classes return an error.
   def default_storage_classes
     if self.storage_classes_desired.nil? || self.storage_classes_desired.empty?
-      self.storage_classes_desired = ["default"]
+      self.storage_classes_desired = Rails.configuration.DefaultStorageClasses
     end
     self.storage_classes_confirmed ||= []
   end

diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index af058494b2356628c73d9adb502a325d569e87ed..a880b65ac5d2c7bbd259da943475f074686fcb71 100644 (file)
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -22,7 +22,7 @@ class Container < ArvadosModel
   attribute :secret_mounts, :jsonbHash, default: {}
   attribute :runtime_status, :jsonbHash, default: {}
   attribute :runtime_auth_scopes, :jsonbArray, default: []
-  attribute :output_storage_classes, :jsonbArray, default: ["default"]
+  attribute :output_storage_classes, :jsonbArray, default: Rails.configuration.DefaultStorageClasses
 
   serialize :environment, Hash
   serialize :mounts, Hash

diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index 1de71102c61befff8e0aabef5885e7396405a237..f603d4dd7949ce11d874b2926369c602b41f892e 100644 (file)
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -23,7 +23,7 @@ class ContainerRequest < ArvadosModel
   # already know how to properly treat them.
   attribute :properties, :jsonbHash, default: {}
   attribute :secret_mounts, :jsonbHash, default: {}
-  attribute :output_storage_classes, :jsonbArray, default: ["default"]
+  attribute :output_storage_classes, :jsonbArray, default: Rails.configuration.DefaultStorageClasses
 
   serialize :environment, Hash
   serialize :mounts, Hash

diff --git a/services/api/config/arvados_config.rb b/services/api/config/arvados_config.rb
index a6f1730e86190fc878c4c1ec3fd236af30e7cea3..1b3c96a8adfe55dd58085512969a59844d9a64d6 100644 (file)
--- a/services/api/config/arvados_config.rb
+++ b/services/api/config/arvados_config.rb
@@ -170,6 +170,7 @@ arvcfg.declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
   ConfigLoader.set_cfg cfg, "RemoteClusters", h
 }
 arvcfg.declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
+arvcfg.declare_config "StorageClasses", Hash
 
 dbcfg = ConfigLoader.new
 
@@ -237,6 +238,17 @@ if $arvados_config["Collections"]["DefaultTrashLifetime"] < 86400.seconds then
   raise "default_trash_lifetime is %d, must be at least 86400" % Rails.configuration.Collections.DefaultTrashLifetime
 end
 
+default_storage_classes = []
+$arvados_config["StorageClasses"].each do |cls, cfg|
+  if cfg["Default"]
+    default_storage_classes << cls
+  end
+end
+if default_storage_classes.length == 0
+  default_storage_classes = ["default"]
+end
+$arvados_config["DefaultStorageClasses"] = default_storage_classes.sort
+
 #
 # Special case for test database where there's no database.yml,
 # because the Arvados config.yml doesn't have a concept of multiple

diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index e69d941b1eaf6eadf52b5c48871fea02c0d3f5bb..bb590e13b33f0535d5a7d2610d8902ddab577300 100644 (file)
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -538,10 +538,6 @@ func (bal *Balancer) setupLookupTables() {
                        // effectively read-only.
                        mnt.ReadOnly = mnt.ReadOnly || srv.ReadOnly
 
-                       if len(mnt.StorageClasses) == 0 {
-                               bal.mountsByClass["default"][mnt] = true
-                               continue
-                       }
                        for class := range mnt.StorageClasses {
                                if mbc := bal.mountsByClass[class]; mbc == nil {
                                        bal.classes = append(bal.classes, class)

diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 18a8bdcf47b111b0c6ea6469d58b60d2a10ea5f5..4e2c6803ca81cc593798d8285945aa9f90a1446c 100644 (file)
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -87,20 +87,24 @@ var stubServices = []arvados.KeepService{
 
 var stubMounts = map[string][]arvados.KeepMount{
        "keep0.zzzzz.arvadosapi.com:25107": {{
-               UUID:     "zzzzz-ivpuk-000000000000000",
-               DeviceID: "keep0-vol0",
+               UUID:           "zzzzz-ivpuk-000000000000000",
+               DeviceID:       "keep0-vol0",
+               StorageClasses: map[string]bool{"default": true},
        }},
        "keep1.zzzzz.arvadosapi.com:25107": {{
-               UUID:     "zzzzz-ivpuk-100000000000000",
-               DeviceID: "keep1-vol0",
+               UUID:           "zzzzz-ivpuk-100000000000000",
+               DeviceID:       "keep1-vol0",
+               StorageClasses: map[string]bool{"default": true},
        }},
        "keep2.zzzzz.arvadosapi.com:25107": {{
-               UUID:     "zzzzz-ivpuk-200000000000000",
-               DeviceID: "keep2-vol0",
+               UUID:           "zzzzz-ivpuk-200000000000000",
+               DeviceID:       "keep2-vol0",
+               StorageClasses: map[string]bool{"default": true},
        }},
        "keep3.zzzzz.arvadosapi.com:25107": {{
-               UUID:     "zzzzz-ivpuk-300000000000000",
-               DeviceID: "keep3-vol0",
+               UUID:           "zzzzz-ivpuk-300000000000000",
+               DeviceID:       "keep3-vol0",
+               StorageClasses: map[string]bool{"default": true},
        }},
 }
 

diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 5bc66dbf3fa7b2f23828205b13452b5839d99510..c529ac150e092c37c9f5510d0463be6a0eebe553 100644 (file)
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -85,7 +85,8 @@ func (bal *balancerSuite) SetUpTest(c *check.C) {
                }
                srv.mounts = []*KeepMount{{
                        KeepMount: arvados.KeepMount{
-                               UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
+                               UUID:           fmt.Sprintf("zzzzz-mount-%015x", i),
+                               StorageClasses: map[string]bool{"default": true},
                        },
                        KeepService: srv,
                }}
@@ -166,10 +167,11 @@ func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) {
                srv.mounts[0].KeepMount.DeviceID = fmt.Sprintf("writable-by-srv-%x", i)
                srv.mounts = append(srv.mounts, &KeepMount{
                        KeepMount: arvados.KeepMount{
-                               DeviceID:    fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)),
-                               UUID:        fmt.Sprintf("zzzzz-mount-%015x", i<<16),
-                               ReadOnly:    readonly,
-                               Replication: 1,
+                               DeviceID:       fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)),
+                               UUID:           fmt.Sprintf("zzzzz-mount-%015x", i<<16),
+                               ReadOnly:       readonly,
+                               Replication:    1,
+                               StorageClasses: map[string]bool{"default": true},
                        },
                        KeepService: srv,
                })

diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index a60d17d576ae047b8a3198cd8c755c3828ef9875..2b469a13eb993e0827bac8ae1ebe4db46bc8c4df 100644 (file)
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -252,6 +252,13 @@ func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
                for i, sc := range wantStorageClasses {
                        wantStorageClasses[i] = strings.TrimSpace(sc)
                }
+       } else {
+               // none specified -- use configured default
+               for class, cfg := range rtr.cluster.StorageClasses {
+                       if cfg.Default {
+                               wantStorageClasses = append(wantStorageClasses, class)
+                       }
+               }
        }
 
        buf, err := getBufferWithContext(ctx, bufs, int(req.ContentLength))