From d98a3ad49469f33e01e35776afff55e1452a2321 Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Mon, 25 Oct 2021 13:05:31 -0300 Subject: [PATCH] 17944: Vocabulary loading, monitoring and checking on several object types. Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- .../_metadata_vocabulary_example.liquid | 4 +- lib/controller/federation.go | 2 - lib/controller/federation/conn.go | 4 + lib/controller/handler.go | 1 + lib/controller/handler_test.go | 50 ++++ lib/controller/localdb/collection.go | 12 +- lib/controller/localdb/collection_test.go | 88 ++++++ lib/controller/localdb/conn.go | 121 ++++++++- lib/controller/localdb/container_request.go | 39 +++ .../localdb/container_request_test.go | 167 ++++++++++++ lib/controller/localdb/group.go | 39 +++ lib/controller/localdb/group_test.go | 139 ++++++++++ lib/controller/router/router.go | 7 + lib/controller/rpc/conn.go | 7 + sdk/go/arvados/api.go | 2 + sdk/go/arvados/config.go | 36 +-- sdk/go/arvados/vocabulary.go | 209 +++++++++++++++ sdk/go/arvados/vocabulary_test.go | 252 ++++++++++++++++++ sdk/go/arvadostest/api.go | 4 + 19 files changed, 1155 insertions(+), 28 deletions(-) create mode 100644 lib/controller/localdb/container_request.go create mode 100644 lib/controller/localdb/container_request_test.go create mode 100644 lib/controller/localdb/group.go create mode 100644 lib/controller/localdb/group_test.go create mode 100644 sdk/go/arvados/vocabulary.go create mode 100644 sdk/go/arvados/vocabulary_test.go diff --git a/doc/_includes/_metadata_vocabulary_example.liquid b/doc/_includes/_metadata_vocabulary_example.liquid index 016b48c6ae..fb8e57725b 100644 --- a/doc/_includes/_metadata_vocabulary_example.liquid +++ b/doc/_includes/_metadata_vocabulary_example.liquid @@ -2,9 +2,7 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -{ +{% endcomment %}{ "strict_tags": false, "tags": { "IDTAGANIMALS": { diff --git a/lib/controller/federation.go b/lib/controller/federation.go index 144d41c21b..cd69727ecb 100644 --- a/lib/controller/federation.go +++ b/lib/controller/federation.go @@ -121,8 +121,6 @@ func (h *Handler) setupProxyRemoteCluster(next http.Handler) http.Handler { mux.ServeHTTP(w, req) }) - - return mux } type CurrentUser struct { diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go index aa05cb1e6d..9729416228 100644 --- a/lib/controller/federation/conn.go +++ b/lib/controller/federation/conn.go @@ -192,6 +192,10 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) { return json.RawMessage(buf.Bytes()), err } +func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) { + return conn.chooseBackend(conn.cluster.ClusterID).VocabularyGet(ctx) +} + func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) { if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID { // defer entire login procedure to designated cluster diff --git a/lib/controller/handler.go b/lib/controller/handler.go index a35d003019..51c72b2822 100644 --- a/lib/controller/handler.go +++ b/lib/controller/handler.go @@ -97,6 +97,7 @@ func (h *Handler) setup() { WrapCalls: api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls), }) mux.Handle("/arvados/v1/config", rtr) + mux.Handle("/arvados/v1/vocabulary", rtr) mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/ mux.Handle("/arvados/v1/collections", rtr) mux.Handle("/arvados/v1/collections/", rtr) diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go index 9b71c349a4..c99faba730 100644 --- a/lib/controller/handler_test.go +++ b/lib/controller/handler_test.go @@ -88,6 +88,56 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) { } } +func (s *HandlerSuite) TestVocabularyExport(c *check.C) { + voc := `{ + "strict_tags": false, + "tags": { + "IDTAGIMPORTANCE": { + "strict": false, + "labels": [{"label": "Importance"}], + "values": { + "HIGH": { + "labels": [{"label": "High"}] + }, + "LOW": { + "labels": [{"label": "Low"}] + } + } + } + } + }` + f, err := os.CreateTemp("", "test-vocabulary-*.json") + c.Assert(err, check.IsNil) + defer os.Remove(f.Name()) + _, err = f.WriteString(voc) + c.Assert(err, check.IsNil) + f.Close() + s.cluster.API.VocabularyPath = f.Name() + for _, method := range []string{"GET", "OPTIONS"} { + c.Log(c.TestName()+" ", method) + req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil) + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Log(resp.Body.String()) + if !c.Check(resp.Code, check.Equals, http.StatusOK) { + continue + } + c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`) + c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`) + c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`) + if method == "OPTIONS" { + c.Check(resp.Body.String(), check.HasLen, 0) + continue + } + var expectedVoc, receivedVoc *arvados.Vocabulary + err := json.Unmarshal([]byte(voc), &expectedVoc) + c.Check(err, check.IsNil) + err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc) + c.Check(err, check.IsNil) + c.Check(receivedVoc, check.DeepEquals, expectedVoc) + } +} + func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) { req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil) resp := httptest.NewRecorder() diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go index d81dd812bf..96c89252ec 100644 --- a/lib/controller/localdb/collection.go +++ b/lib/controller/localdb/collection.go @@ -49,8 +49,12 @@ func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions) } // CollectionCreate defers to railsProxy for everything except blob -// signatures. +// signatures and vocabulary checking. func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.Collection{}, err + } if len(opts.Select) > 0 { // We need to know IsTrashed and TrashAt to implement // signing properly, even if the caller doesn't want @@ -66,8 +70,12 @@ func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptio } // CollectionUpdate defers to railsProxy for everything except blob -// signatures. +// signatures and vocabulary checking. func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.Collection{}, err + } if len(opts.Select) > 0 { // We need to know IsTrashed and TrashAt to implement // signing properly, even if the caller doesn't want diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go index 4a44949641..ae996d27b8 100644 --- a/lib/controller/localdb/collection_test.go +++ b/lib/controller/localdb/collection_test.go @@ -48,6 +48,94 @@ func (s *CollectionSuite) TearDownTest(c *check.C) { s.railsSpy.Close() } +func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) { + if testVocabulary == "" { + testVocabulary = `{ + "strict_tags": false, + "tags": { + "IDTAGIMPORTANCES": { + "strict": true, + "labels": [{"label": "Importance"}, {"label": "Priority"}], + "values": { + "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] }, + "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] }, + "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] } + } + } + } + }` + } + voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{}) + c.Assert(err, check.IsNil) + c.Assert(voc.Validate(), check.IsNil) + s.cluster.API.VocabularyPath = "foo" + s.localdb.vocabularyCache = voc +} + +func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) { + s.setUpVocabulary(c, "") + ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) + + tests := []struct { + name string + props map[string]interface{} + success bool + }{ + {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false}, + {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false}, + {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true}, + {"Empty properties", map[string]interface{}{}, true}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + + coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{ + Select: []string{"uuid", "properties"}, + Attrs: map[string]interface{}{ + "properties": tt.props, + }}) + if tt.success { + c.Assert(err, check.IsNil) + c.Assert(coll.Properties, check.DeepEquals, tt.props) + } else { + c.Assert(err, check.NotNil) + } + } +} + +func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) { + s.setUpVocabulary(c, "") + ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) + + tests := []struct { + name string + props map[string]interface{} + success bool + }{ + {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false}, + {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false}, + {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true}, + {"Empty properties", map[string]interface{}{}, true}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{}) + c.Assert(err, check.IsNil) + coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{ + UUID: coll.UUID, + Select: []string{"uuid", "properties"}, + Attrs: map[string]interface{}{ + "properties": tt.props, + }}) + if tt.success { + c.Assert(err, check.IsNil) + c.Assert(coll.Properties, check.DeepEquals, tt.props) + } else { + c.Assert(err, check.NotNil) + } + } +} + func (s *CollectionSuite) TestSignatures(c *check.C) { ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go index a90deded59..0fae35e7d3 100644 --- a/lib/controller/localdb/conn.go +++ b/lib/controller/localdb/conn.go @@ -6,27 +6,33 @@ package localdb import ( "context" + "encoding/json" "fmt" + "os" "strings" "git.arvados.org/arvados.git/lib/controller/railsproxy" "git.arvados.org/arvados.git/lib/controller/rpc" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/fsnotify/fsnotify" + "github.com/sirupsen/logrus" ) type railsProxy = rpc.Conn type Conn struct { - cluster *arvados.Cluster - *railsProxy // handles API methods that aren't defined on Conn itself + cluster *arvados.Cluster + *railsProxy // handles API methods that aren't defined on Conn itself + vocabularyCache *arvados.Vocabulary + reloadVocabulary bool loginController } func NewConn(cluster *arvados.Cluster) *Conn { railsProxy := railsproxy.NewConn(cluster) railsProxy.RedactHostInErrors = true - var conn Conn - conn = Conn{ + conn := Conn{ cluster: cluster, railsProxy: railsProxy, } @@ -34,6 +40,113 @@ func NewConn(cluster *arvados.Cluster) *Conn { return &conn } +func (conn *Conn) checkProperties(ctx context.Context, properties interface{}) error { + if properties == nil { + return nil + } + var props map[string]interface{} + switch properties := properties.(type) { + case string: + err := json.Unmarshal([]byte(properties), &props) + if err != nil { + return err + } + case map[string]interface{}: + props = properties + default: + return fmt.Errorf("unexpected properties type %T", properties) + } + voc, err := conn.VocabularyGet(ctx) + if err != nil { + return err + } + return voc.Check(props) +} + +func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + logger.WithError(err).Error("vocabulary fsnotify setup failed") + return + } + defer watcher.Close() + + err = watcher.Add(vocPath) + if err != nil { + logger.WithError(err).Error("vocabulary file watcher failed") + return + } + + for { + select { + case err, ok := <-watcher.Errors: + if !ok { + return + } + logger.WithError(err).Warn("vocabulary file watcher error") + case _, ok := <-watcher.Events: + if !ok { + return + } + for len(watcher.Events) > 0 { + <-watcher.Events + } + fn() + } + } +} + +func (conn *Conn) loadVocabularyFile() error { + vf, err := os.ReadFile(conn.cluster.API.VocabularyPath) + if err != nil { + return fmt.Errorf("couldn't read vocabulary file %q: %v", conn.cluster.API.VocabularyPath, err) + } + mk := make([]string, 0, len(conn.cluster.Collections.ManagedProperties)) + for k := range conn.cluster.Collections.ManagedProperties { + mk = append(mk, k) + } + voc, err := arvados.NewVocabulary(vf, mk) + if err != nil { + return fmt.Errorf("while loading vocabulary file %q: %s", conn.cluster.API.VocabularyPath, err) + } + err = voc.Validate() + if err != nil { + return fmt.Errorf("while validating vocabulary file %q: %s", conn.cluster.API.VocabularyPath, err) + } + conn.vocabularyCache = voc + return nil +} + +// VocabularyGet refreshes the vocabulary cache if necessary and returns it. +func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) { + if conn.cluster.API.VocabularyPath == "" { + return arvados.Vocabulary{}, nil + } + logger := ctxlog.FromContext(ctx) + if conn.vocabularyCache == nil { + // Initial load of vocabulary file. + err := conn.loadVocabularyFile() + if err != nil { + logger.WithError(err).Error("error loading vocabulary file") + return arvados.Vocabulary{}, err + } + go watchVocabulary(logger, conn.cluster.API.VocabularyPath, func() { + logger.Info("vocabulary file changed, it'll be reloaded next time it's needed") + conn.reloadVocabulary = true + }) + } else if conn.reloadVocabulary { + // Requested reload of vocabulary file. + conn.reloadVocabulary = false + err := conn.loadVocabularyFile() + if err != nil { + logger.WithError(err).Error("error reloading vocabulary file - ignoring") + } else { + logger.Info("vocabulary file reloaded successfully") + } + } + return *conn.vocabularyCache, nil +} + // Logout handles the logout of conn giving to the appropriate loginController func (conn *Conn) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) { return conn.loginController.Logout(ctx, opts) diff --git a/lib/controller/localdb/container_request.go b/lib/controller/localdb/container_request.go new file mode 100644 index 0000000000..5b2ce95da9 --- /dev/null +++ b/lib/controller/localdb/container_request.go @@ -0,0 +1,39 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + + "git.arvados.org/arvados.git/sdk/go/arvados" +) + +// ContainerRequestCreate defers to railsProxy for everything except +// vocabulary checking. +func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.ContainerRequest, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.ContainerRequest{}, err + } + resp, err := conn.railsProxy.ContainerRequestCreate(ctx, opts) + if err != nil { + return resp, err + } + return resp, nil +} + +// ContainerRequestUpdate defers to railsProxy for everything except +// vocabulary checking. +func (conn *Conn) ContainerRequestUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.ContainerRequest, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.ContainerRequest{}, err + } + resp, err := conn.railsProxy.ContainerRequestUpdate(ctx, opts) + if err != nil { + return resp, err + } + return resp, nil +} diff --git a/lib/controller/localdb/container_request_test.go b/lib/controller/localdb/container_request_test.go new file mode 100644 index 0000000000..c231e3ca77 --- /dev/null +++ b/lib/controller/localdb/container_request_test.go @@ -0,0 +1,167 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/auth" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + check "gopkg.in/check.v1" +) + +var _ = check.Suite(&ContainerRequestSuite{}) + +type ContainerRequestSuite struct { + cluster *arvados.Cluster + localdb *Conn + railsSpy *arvadostest.Proxy +} + +func (s *ContainerRequestSuite) TearDownSuite(c *check.C) { + // Undo any changes/additions to the user database so they + // don't affect subsequent tests. + arvadostest.ResetEnv() + c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil) +} + +func (s *ContainerRequestSuite) SetUpTest(c *check.C) { + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() + c.Assert(err, check.IsNil) + s.cluster, err = cfg.GetCluster("") + c.Assert(err, check.IsNil) + s.localdb = NewConn(s.cluster) + s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI) + *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider) +} + +func (s *ContainerRequestSuite) TearDownTest(c *check.C) { + s.railsSpy.Close() +} + +func (s *ContainerRequestSuite) setUpVocabulary(c *check.C, testVocabulary string) { + if testVocabulary == "" { + testVocabulary = `{ + "strict_tags": false, + "tags": { + "IDTAGIMPORTANCES": { + "strict": true, + "labels": [{"label": "Importance"}, {"label": "Priority"}], + "values": { + "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] }, + "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] }, + "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] } + } + } + } + }` + } + voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{}) + c.Assert(err, check.IsNil) + c.Assert(voc.Validate(), check.IsNil) + s.localdb.vocabularyCache = voc + s.cluster.API.VocabularyPath = "foo" +} + +func (s *ContainerRequestSuite) TestCRCreateWithProperties(c *check.C) { + s.setUpVocabulary(c, "") + ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) + + tests := []struct { + name string + props map[string]interface{} + success bool + }{ + {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false}, + {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false}, + {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true}, + {"Empty properties", map[string]interface{}{}, true}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + + cnt, err := s.localdb.ContainerRequestCreate(ctx, arvados.CreateOptions{ + Select: []string{"uuid", "properties"}, + Attrs: map[string]interface{}{ + "command": []string{"echo", "foo"}, + "container_image": "arvados/apitestfixture:latest", + "cwd": "/tmp", + "environment": map[string]string{}, + "mounts": map[string]interface{}{ + "/out": map[string]interface{}{ + "kind": "tmp", + "capacity": 1000000, + }, + }, + "output_path": "/out", + "runtime_constraints": map[string]interface{}{ + "vcpus": 1, + "ram": 2, + }, + "properties": tt.props, + }}) + if tt.success { + c.Assert(err, check.IsNil) + c.Assert(cnt.Properties, check.DeepEquals, tt.props) + } else { + c.Assert(err, check.NotNil) + } + } +} + +func (s *ContainerRequestSuite) TestCRUpdateWithProperties(c *check.C) { + s.setUpVocabulary(c, "") + ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) + + tests := []struct { + name string + props map[string]interface{} + success bool + }{ + {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false}, + {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false}, + {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true}, + {"Empty properties", map[string]interface{}{}, true}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + cnt, err := s.localdb.ContainerRequestCreate(ctx, arvados.CreateOptions{ + Attrs: map[string]interface{}{ + "command": []string{"echo", "foo"}, + "container_image": "arvados/apitestfixture:latest", + "cwd": "/tmp", + "environment": map[string]string{}, + "mounts": map[string]interface{}{ + "/out": map[string]interface{}{ + "kind": "tmp", + "capacity": 1000000, + }, + }, + "output_path": "/out", + "runtime_constraints": map[string]interface{}{ + "vcpus": 1, + "ram": 2, + }, + }, + }) + c.Assert(err, check.IsNil) + cnt, err = s.localdb.ContainerRequestUpdate(ctx, arvados.UpdateOptions{ + UUID: cnt.UUID, + Select: []string{"uuid", "properties"}, + Attrs: map[string]interface{}{ + "properties": tt.props, + }}) + if tt.success { + c.Assert(err, check.IsNil) + c.Assert(cnt.Properties, check.DeepEquals, tt.props) + } else { + c.Assert(err, check.NotNil) + } + } +} diff --git a/lib/controller/localdb/group.go b/lib/controller/localdb/group.go new file mode 100644 index 0000000000..0d77bdbd9c --- /dev/null +++ b/lib/controller/localdb/group.go @@ -0,0 +1,39 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + + "git.arvados.org/arvados.git/sdk/go/arvados" +) + +// GroupCreate defers to railsProxy for everything except vocabulary +// checking. +func (conn *Conn) GroupCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Group, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.Group{}, err + } + resp, err := conn.railsProxy.GroupCreate(ctx, opts) + if err != nil { + return resp, err + } + return resp, nil +} + +// GroupUpdate defers to railsProxy for everything except vocabulary +// checking. +func (conn *Conn) GroupUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Group, error) { + err := conn.checkProperties(ctx, opts.Attrs["properties"]) + if err != nil { + return arvados.Group{}, err + } + resp, err := conn.railsProxy.GroupUpdate(ctx, opts) + if err != nil { + return resp, err + } + return resp, nil +} diff --git a/lib/controller/localdb/group_test.go b/lib/controller/localdb/group_test.go new file mode 100644 index 0000000000..0991f3b721 --- /dev/null +++ b/lib/controller/localdb/group_test.go @@ -0,0 +1,139 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/auth" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + check "gopkg.in/check.v1" +) + +var _ = check.Suite(&GroupSuite{}) + +type GroupSuite struct { + cluster *arvados.Cluster + localdb *Conn + railsSpy *arvadostest.Proxy +} + +func (s *GroupSuite) TearDownSuite(c *check.C) { + // Undo any changes/additions to the user database so they + // don't affect subsequent tests. + arvadostest.ResetEnv() + c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil) +} + +func (s *GroupSuite) SetUpTest(c *check.C) { + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() + c.Assert(err, check.IsNil) + s.cluster, err = cfg.GetCluster("") + c.Assert(err, check.IsNil) + s.localdb = NewConn(s.cluster) + s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI) + *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider) +} + +func (s *GroupSuite) TearDownTest(c *check.C) { + s.railsSpy.Close() +} + +func (s *GroupSuite) setUpVocabulary(c *check.C, testVocabulary string) { + if testVocabulary == "" { + testVocabulary = `{ + "strict_tags": false, + "tags": { + "IDTAGIMPORTANCES": { + "strict": true, + "labels": [{"label": "Importance"}, {"label": "Priority"}], + "values": { + "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] }, + "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] }, + "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] } + } + } + } + }` + } + voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{}) + c.Assert(err, check.IsNil) + c.Assert(voc.Validate(), check.IsNil) + s.localdb.vocabularyCache = voc + s.cluster.API.VocabularyPath = "foo" +} + +func (s *GroupSuite) TestGroupCreateWithProperties(c *check.C) { + s.setUpVocabulary(c, "") + ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) + + tests := []struct { + name string + props map[string]interface{} + success bool + }{ + {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false}, + {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false}, + {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true}, + {"Empty properties", map[string]interface{}{}, true}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + + grp, err := s.localdb.GroupCreate(ctx, arvados.CreateOptions{ + Select: []string{"uuid", "properties"}, + Attrs: map[string]interface{}{ + "group_class": "project", + "properties": tt.props, + }}) + if tt.success { + c.Assert(err, check.IsNil) + c.Assert(grp.Properties, check.DeepEquals, tt.props) + } else { + c.Assert(err, check.NotNil) + } + } +} + +func (s *GroupSuite) TestGroupUpdateWithProperties(c *check.C) { + s.setUpVocabulary(c, "") + ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}}) + + tests := []struct { + name string + props map[string]interface{} + success bool + }{ + {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false}, + {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false}, + {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true}, + {"Empty properties", map[string]interface{}{}, true}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + grp, err := s.localdb.GroupCreate(ctx, arvados.CreateOptions{ + Attrs: map[string]interface{}{ + "group_class": "project", + }, + }) + c.Assert(err, check.IsNil) + grp, err = s.localdb.GroupUpdate(ctx, arvados.UpdateOptions{ + UUID: grp.UUID, + Select: []string{"uuid", "properties"}, + Attrs: map[string]interface{}{ + "properties": tt.props, + }}) + if tt.success { + c.Assert(err, check.IsNil) + c.Assert(grp.Properties, check.DeepEquals, tt.props) + } else { + c.Assert(err, check.NotNil) + } + } +} diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go index 9826c1e744..d04eccf689 100644 --- a/lib/controller/router/router.go +++ b/lib/controller/router/router.go @@ -65,6 +65,13 @@ func (rtr *router) addRoutes() { return rtr.backend.ConfigGet(ctx) }, }, + { + arvados.EndpointVocabularyGet, + func() interface{} { return &struct{}{} }, + func(ctx context.Context, opts interface{}) (interface{}, error) { + return rtr.backend.VocabularyGet(ctx) + }, + }, { arvados.EndpointLogin, func() interface{} { return &arvados.LoginOptions{} }, diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go index 640bbf1c23..1acddfab71 100644 --- a/lib/controller/rpc/conn.go +++ b/lib/controller/rpc/conn.go @@ -178,6 +178,13 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) { return resp, err } +func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) { + ep := arvados.EndpointVocabularyGet + var resp arvados.Vocabulary + err := conn.requestAndDecode(ctx, &resp, ep, nil, nil) + return resp, err +} + func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) { ep := arvados.EndpointLogin var resp arvados.LoginResponse diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go index b429e80084..41727beea8 100644 --- a/sdk/go/arvados/api.go +++ b/sdk/go/arvados/api.go @@ -23,6 +23,7 @@ type APIEndpoint struct { var ( EndpointConfigGet = APIEndpoint{"GET", "arvados/v1/config", ""} + EndpointVocabularyGet = APIEndpoint{"GET", "arvados/v1/vocabulary", ""} EndpointLogin = APIEndpoint{"GET", "login", ""} EndpointLogout = APIEndpoint{"GET", "logout", ""} EndpointCollectionCreate = APIEndpoint{"POST", "arvados/v1/collections", "collection"} @@ -219,6 +220,7 @@ type BlockWriteResponse struct { type API interface { ConfigGet(ctx context.Context) (json.RawMessage, error) + VocabularyGet(ctx context.Context) (Vocabulary, error) Login(ctx context.Context, options LoginOptions) (LoginResponse, error) Logout(ctx context.Context, options LogoutOptions) (LogoutResponse, error) CollectionCreate(ctx context.Context, options CreateOptions) (Collection, error) diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 2df0b90577..8755bbd3e2 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -77,6 +77,12 @@ type UploadDownloadRolePermissions struct { Admin UploadDownloadPermission } +type ManagedProperties map[string]struct { + Value interface{} + Function string + Protected bool +} + type Cluster struct { ClusterID string `json:"-"` ManagementToken string @@ -110,23 +116,19 @@ type Cluster struct { UnloggedAttributes StringSet } Collections struct { - BlobSigning bool - BlobSigningKey string - BlobSigningTTL Duration - BlobTrash bool - BlobTrashLifetime Duration - BlobTrashCheckInterval Duration - BlobTrashConcurrency int - BlobDeleteConcurrency int - BlobReplicateConcurrency int - CollectionVersioning bool - DefaultTrashLifetime Duration - DefaultReplication int - ManagedProperties map[string]struct { - Value interface{} - Function string - Protected bool - } + BlobSigning bool + BlobSigningKey string + BlobSigningTTL Duration + BlobTrash bool + BlobTrashLifetime Duration + BlobTrashCheckInterval Duration + BlobTrashConcurrency int + BlobDeleteConcurrency int + BlobReplicateConcurrency int + CollectionVersioning bool + DefaultTrashLifetime Duration + DefaultReplication int + ManagedProperties ManagedProperties PreserveVersionIfIdle Duration TrashSweepInterval Duration TrustAllContent bool diff --git a/sdk/go/arvados/vocabulary.go b/sdk/go/arvados/vocabulary.go new file mode 100644 index 0000000000..cb1106e9b0 --- /dev/null +++ b/sdk/go/arvados/vocabulary.go @@ -0,0 +1,209 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package arvados + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" +) + +type Vocabulary struct { + reservedTagKeys map[string]bool `json:"-"` + StrictTags bool `json:"strict_tags"` + Tags map[string]VocabularyTag `json:"tags"` +} + +type VocabularyTag struct { + Strict bool `json:"strict"` + Labels []VocabularyLabel `json:"labels"` + Values map[string]VocabularyTagValue `json:"values"` +} + +// Cannot have a constant map in Go, so we have to use a function +func (v *Vocabulary) systemTagKeys() map[string]bool { + return map[string]bool{ + "type": true, + "template_uuid": true, + "groups": true, + "username": true, + "image_timestamp": true, + "docker-image-repo-tag": true, + "filters": true, + "container_request": true, + } +} + +type VocabularyLabel struct { + Label string `json:"label"` +} + +type VocabularyTagValue struct { + Labels []VocabularyLabel `json:"labels"` +} + +func NewVocabulary(data []byte, managedTagKeys []string) (voc *Vocabulary, err error) { + if r := bytes.Compare(data, []byte("")); r == 0 { + return &Vocabulary{}, nil + } + err = json.Unmarshal(data, &voc) + if err != nil { + return nil, fmt.Errorf("invalid JSON format error: %q", err) + } + if reflect.DeepEqual(voc, &Vocabulary{}) { + return nil, fmt.Errorf("JSON data provided doesn't match Vocabulary format: %q", data) + } + voc.reservedTagKeys = make(map[string]bool) + for _, managedKey := range managedTagKeys { + voc.reservedTagKeys[managedKey] = true + } + for systemKey := range voc.systemTagKeys() { + voc.reservedTagKeys[systemKey] = true + } + err = voc.Validate() + if err != nil { + return nil, err + } + return voc, nil +} + +func (v *Vocabulary) Validate() error { + if v == nil { + return nil + } + tagKeys := map[string]bool{} + // Checks for Vocabulary strictness + if v.StrictTags && len(v.Tags) == 0 { + return fmt.Errorf("vocabulary is strict but no tags are defined") + } + // Checks for duplicate tag keys + for key := range v.Tags { + if v.reservedTagKeys[key] { + return fmt.Errorf("tag key %q is reserved", key) + } + if tagKeys[key] { + return fmt.Errorf("duplicate tag key %q", key) + } + tagKeys[key] = true + for _, lbl := range v.Tags[key].Labels { + label := strings.ToLower(lbl.Label) + if tagKeys[label] { + return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", label, key) + } + tagKeys[label] = true + } + // Checks for value strictness + if v.Tags[key].Strict && len(v.Tags[key].Values) == 0 { + return fmt.Errorf("tag key %q is configured as strict but doesn't provide values", key) + } + // Checks for value duplication within a key + tagValues := map[string]bool{} + for val := range v.Tags[key].Values { + if tagValues[val] { + return fmt.Errorf("duplicate tag value %q for tag %q", val, key) + } + tagValues[val] = true + for _, tagLbl := range v.Tags[key].Values[val].Labels { + label := strings.ToLower(tagLbl.Label) + if tagValues[label] { + return fmt.Errorf("tag value label %q for pair (%q:%q) already seen as a value key or label", label, key, val) + } + tagValues[label] = true + } + } + } + return nil +} + +func (v *Vocabulary) getLabelsToKeys() (labels map[string]string) { + if v == nil { + return + } + labels = make(map[string]string) + for key, val := range v.Tags { + for _, lbl := range val.Labels { + label := strings.ToLower(lbl.Label) + labels[label] = key + } + } + return labels +} + +func (v *Vocabulary) getLabelsToValues(key string) (labels map[string]string) { + if v == nil { + return + } + labels = make(map[string]string) + if _, ok := v.Tags[key]; ok { + for val := range v.Tags[key].Values { + for _, tagLbl := range v.Tags[key].Values[val].Labels { + label := strings.ToLower(tagLbl.Label) + labels[label] = val + } + } + } + return labels +} + +func (v *Vocabulary) checkValue(key, val string) error { + if _, ok := v.Tags[key].Values[val]; !ok { + lcVal := strings.ToLower(val) + alias, ok := v.getLabelsToValues(key)[lcVal] + if ok { + return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias) + } else if v.Tags[key].Strict { + return fmt.Errorf("tag value %q for key %q is not listed as valid", val, key) + } + } + return nil +} + +// Check validates the given data against the vocabulary. +func (v *Vocabulary) Check(data map[string]interface{}) error { + if v == nil { + return nil + } + for key, val := range data { + // Checks for key validity + if v.reservedTagKeys[key] { + // Allow reserved keys to be used even if they are not defined in + // the vocabulary no matter its strictness. + continue + } + if _, ok := v.Tags[key]; !ok { + lcKey := strings.ToLower(key) + alias, ok := v.getLabelsToKeys()[lcKey] + if ok { + return fmt.Errorf("tag key %q is not defined but is an alias for %q", key, alias) + } else if v.StrictTags { + return fmt.Errorf("tag key %q is not defined", key) + } + // If the key is not defined, we don't need to check the value + continue + } + // Checks for value validity -- key is defined + switch val := val.(type) { + case string: + return v.checkValue(key, val) + case []interface{}: + for _, singleVal := range val { + switch singleVal := singleVal.(type) { + case string: + err := v.checkValue(key, singleVal) + if err != nil { + return err + } + default: + return fmt.Errorf("tag value %q for key %q is not a valid type (%T)", singleVal, key, singleVal) + } + } + default: + return fmt.Errorf("tag value %q for key %q is not a valid type (%T)", val, key, val) + } + } + return nil +} diff --git a/sdk/go/arvados/vocabulary_test.go b/sdk/go/arvados/vocabulary_test.go new file mode 100644 index 0000000000..b2748c7be7 --- /dev/null +++ b/sdk/go/arvados/vocabulary_test.go @@ -0,0 +1,252 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package arvados + +import ( + "encoding/json" + + check "gopkg.in/check.v1" +) + +type VocabularySuite struct { + testVoc *Vocabulary +} + +var _ = check.Suite(&VocabularySuite{}) + +func (s *VocabularySuite) SetUpTest(c *check.C) { + s.testVoc = &Vocabulary{ + reservedTagKeys: map[string]bool{ + "reservedKey": true, + }, + StrictTags: false, + Tags: map[string]VocabularyTag{ + "IDTAGANIMALS": { + Strict: false, + Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}}, + Values: map[string]VocabularyTagValue{ + "IDVALANIMAL1": { + Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}}, + }, + "IDVALANIMAL2": { + Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}}, + }, + }, + }, + "IDTAGIMPORTANCE": { + Strict: true, + Labels: []VocabularyLabel{{Label: "Importance"}, {Label: "Priority"}}, + Values: map[string]VocabularyTagValue{ + "IDVAL3": { + Labels: []VocabularyLabel{{Label: "Low"}, {Label: "Low priority"}}, + }, + "IDVAL2": { + Labels: []VocabularyLabel{{Label: "Medium"}, {Label: "Medium priority"}}, + }, + "IDVAL1": { + Labels: []VocabularyLabel{{Label: "High"}, {Label: "High priority"}}, + }, + }, + }, + "IDTAGCOMMENT": { + Strict: false, + Labels: []VocabularyLabel{{Label: "Comment"}}, + }, + }, + } + err := s.testVoc.Validate() + c.Assert(err, check.IsNil) +} + +func (s *VocabularySuite) TestCheck(c *check.C) { + tests := []struct { + name string + strictVoc bool + props string + expectSuccess bool + }{ + // Check succeeds + {"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true}, + {"Unknown non-alias key on non-strict vocabulary", false, `{"foo":"bar"}`, true}, + {"Known non-strict key, unknown non-alias value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true}, + {"Undefined but reserved key on strict vocabulary", true, `{"reservedKey":"bar"}`, true}, + {"Known key, list of known values", false, `{"IDTAGANIMALS":["IDVALANIMAL1","IDVALANIMAL2"]}`, true}, + {"Known non-strict key, list of unknown non-alias values", false, `{"IDTAGCOMMENT":["hello world","lorem ipsum"]}`, true}, + // Check fails + {"Unknown non-alias key on strict vocabulary", true, `{"foo":"bar"}`, false}, + {"Known non-strict key, known value alias", false, `{"IDTAGANIMALS":"Loxodonta"}`, false}, + {"Known strict key, unknown non-alias value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false}, + {"Known strict key, known value alias", false, `{"IDTAGIMPORTANCE":"High"}`, false}, + {"Known strict key, list of known alias values", false, `{"IDTAGIMPORTANCE":["Unimportant","High"]}`, false}, + {"Known strict key, list of unknown non-alias values", false, `{"IDTAGIMPORTANCE":["foo","bar"]}`, false}, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + s.testVoc.StrictTags = tt.strictVoc + + var data map[string]interface{} + err := json.Unmarshal([]byte(tt.props), &data) + c.Assert(err, check.IsNil) + err = s.testVoc.Check(data) + if tt.expectSuccess { + c.Assert(err, check.IsNil) + } else { + c.Assert(err, check.NotNil) + } + } +} + +func (s *VocabularySuite) TestNewVocabulary(c *check.C) { + tests := []struct { + name string + data string + isValid bool + errMatches string + expect *Vocabulary + }{ + {"Empty data", "", true, "", &Vocabulary{}}, + {"Invalid JSON", "foo", false, "invalid JSON format.*", nil}, + {"Valid, empty JSON", "{}", false, ".*doesn't match Vocabulary format.*", nil}, + {"Valid JSON, wrong data", `{"foo":"bar"}`, false, ".*doesn't match Vocabulary format.*", nil}, + { + "Simple valid example", + `{"tags":{ + "IDTAGANIMALS":{ + "strict": false, + "labels": [{"label": "Animal"}, {"label": "Creature"}], + "values": { + "IDVALANIMAL1":{"labels":[{"label":"Human"}, {"label":"Homo sapiens"}]}, + "IDVALANIMAL2":{"labels":[{"label":"Elephant"}, {"label":"Loxodonta"}]} + } + } + }}`, + true, "", + &Vocabulary{ + reservedTagKeys: map[string]bool{ + "type": true, + "template_uuid": true, + "groups": true, + "username": true, + "image_timestamp": true, + "docker-image-repo-tag": true, + "filters": true, + "container_request": true, + }, + StrictTags: false, + Tags: map[string]VocabularyTag{ + "IDTAGANIMALS": { + Strict: false, + Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}}, + Values: map[string]VocabularyTagValue{ + "IDVALANIMAL1": { + Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}}, + }, + "IDVALANIMAL2": { + Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}}, + }, + }, + }, + }, + }, + }, + { + "Valid data, but uses reserved key", + `{"tags":{ + "type":{ + "strict": false, + "labels": [{"label": "Type"}] + } + }}`, + false, "tag key.*is reserved", nil, + }, + } + + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + voc, err := NewVocabulary([]byte(tt.data), []string{}) + if tt.isValid { + c.Assert(err, check.IsNil) + } else { + c.Assert(err, check.NotNil) + if tt.errMatches != "" { + c.Assert(err, check.ErrorMatches, tt.errMatches) + } + } + c.Assert(voc, check.DeepEquals, tt.expect) + } +} + +func (s *VocabularySuite) TestValidationErrors(c *check.C) { + tests := []struct { + name string + voc *Vocabulary + errMatches string + }{ + { + "Strict vocabulary, no keys", + &Vocabulary{ + StrictTags: true, + }, + "vocabulary is strict but no tags are defined", + }, + { + "Duplicated tag keys", + &Vocabulary{ + StrictTags: false, + Tags: map[string]VocabularyTag{ + "IDTAGANIMALS": { + Strict: false, + Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}}, + }, + "IDTAGCOMMENT": { + Strict: false, + Labels: []VocabularyLabel{{Label: "Comment"}, {Label: "Animal"}}, + }, + }, + }, + "tag label.*for key.*already seen.*", + }, + { + "Duplicated tag values", + &Vocabulary{ + StrictTags: false, + Tags: map[string]VocabularyTag{ + "IDTAGANIMALS": { + Strict: false, + Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}}, + Values: map[string]VocabularyTagValue{ + "IDVALANIMAL1": { + Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Mammal"}}, + }, + "IDVALANIMAL2": { + Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Mammal"}}, + }, + }, + }, + }, + }, + "tag value label.*for pair.*already seen.*", + }, + { + "Strict key, no values", + &Vocabulary{ + StrictTags: false, + Tags: map[string]VocabularyTag{ + "IDTAGANIMALS": { + Strict: true, + Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}}, + }, + }, + }, + "tag key.*is configured as strict but doesn't provide values", + }, + } + for _, tt := range tests { + c.Log(c.TestName()+" ", tt.name) + err := tt.voc.Validate() + c.Assert(err, check.NotNil) + c.Assert(err, check.ErrorMatches, tt.errMatches) + } +} diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go index 8bf01693c4..2cb35366c0 100644 --- a/sdk/go/arvadostest/api.go +++ b/sdk/go/arvadostest/api.go @@ -33,6 +33,10 @@ func (as *APIStub) ConfigGet(ctx context.Context) (json.RawMessage, error) { as.appendCall(ctx, as.ConfigGet, nil) return nil, as.Error } +func (as *APIStub) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) { + as.appendCall(ctx, as.VocabularyGet, nil) + return arvados.Vocabulary{}, as.Error +} func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) { as.appendCall(ctx, as.Login, options) return arvados.LoginResponse{}, as.Error -- 2.30.2