Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{
+{% endcomment %}{
"strict_tags": false,
"tags": {
"IDTAGANIMALS": {
mux.ServeHTTP(w, req)
})
-
- return mux
}
type CurrentUser struct {
return json.RawMessage(buf.Bytes()), err
}
+func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
+ return conn.chooseBackend(conn.cluster.ClusterID).VocabularyGet(ctx)
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID {
// defer entire login procedure to designated cluster
WrapCalls: api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls),
})
mux.Handle("/arvados/v1/config", rtr)
+ mux.Handle("/arvados/v1/vocabulary", rtr)
mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) // must come before .../users/
mux.Handle("/arvados/v1/collections", rtr)
mux.Handle("/arvados/v1/collections/", rtr)
}
}
+func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
+ voc := `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCE": {
+ "strict": false,
+ "labels": [{"label": "Importance"}],
+ "values": {
+ "HIGH": {
+ "labels": [{"label": "High"}]
+ },
+ "LOW": {
+ "labels": [{"label": "Low"}]
+ }
+ }
+ }
+ }
+ }`
+ f, err := os.CreateTemp("", "test-vocabulary-*.json")
+ c.Assert(err, check.IsNil)
+ defer os.Remove(f.Name())
+ _, err = f.WriteString(voc)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ s.cluster.API.VocabularyPath = f.Name()
+ for _, method := range []string{"GET", "OPTIONS"} {
+ c.Log(c.TestName()+" ", method)
+ req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
+ resp := httptest.NewRecorder()
+ s.handler.ServeHTTP(resp, req)
+ c.Log(resp.Body.String())
+ if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+ continue
+ }
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
+ if method == "OPTIONS" {
+ c.Check(resp.Body.String(), check.HasLen, 0)
+ continue
+ }
+ var expectedVoc, receivedVoc *arvados.Vocabulary
+ err := json.Unmarshal([]byte(voc), &expectedVoc)
+ c.Check(err, check.IsNil)
+ err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
+ c.Check(err, check.IsNil)
+ c.Check(receivedVoc, check.DeepEquals, expectedVoc)
+ }
+}
+
func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
resp := httptest.NewRecorder()
}
// CollectionCreate defers to railsProxy for everything except blob
-// signatures.
+// signatures and vocabulary checking.
func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Collection{}, err
+ }
if len(opts.Select) > 0 {
// We need to know IsTrashed and TrashAt to implement
// signing properly, even if the caller doesn't want
}
// CollectionUpdate defers to railsProxy for everything except blob
-// signatures.
+// signatures and vocabulary checking.
func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Collection{}, err
+ }
if len(opts.Select) > 0 {
// We need to know IsTrashed and TrashAt to implement
// signing properly, even if the caller doesn't want
s.railsSpy.Close()
}
+func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
+ c.Assert(err, check.IsNil)
+ c.Assert(voc.Validate(), check.IsNil)
+ s.cluster.API.VocabularyPath = "foo"
+ s.localdb.vocabularyCache = voc
+}
+
+func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+
+ coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(coll.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
+ c.Assert(err, check.IsNil)
+ coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: coll.UUID,
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(coll.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
func (s *CollectionSuite) TestSignatures(c *check.C) {
ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
import (
"context"
+ "encoding/json"
"fmt"
+ "os"
"strings"
"git.arvados.org/arvados.git/lib/controller/railsproxy"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/fsnotify/fsnotify"
+ "github.com/sirupsen/logrus"
)
type railsProxy = rpc.Conn
type Conn struct {
- cluster *arvados.Cluster
- *railsProxy // handles API methods that aren't defined on Conn itself
+ cluster *arvados.Cluster
+ *railsProxy // handles API methods that aren't defined on Conn itself
+ vocabularyCache *arvados.Vocabulary
+ reloadVocabulary bool
loginController
}
func NewConn(cluster *arvados.Cluster) *Conn {
railsProxy := railsproxy.NewConn(cluster)
railsProxy.RedactHostInErrors = true
- var conn Conn
- conn = Conn{
+ conn := Conn{
cluster: cluster,
railsProxy: railsProxy,
}
return &conn
}
+func (conn *Conn) checkProperties(ctx context.Context, properties interface{}) error {
+ if properties == nil {
+ return nil
+ }
+ var props map[string]interface{}
+ switch properties := properties.(type) {
+ case string:
+ err := json.Unmarshal([]byte(properties), &props)
+ if err != nil {
+ return err
+ }
+ case map[string]interface{}:
+ props = properties
+ default:
+ return fmt.Errorf("unexpected properties type %T", properties)
+ }
+ voc, err := conn.VocabularyGet(ctx)
+ if err != nil {
+ return err
+ }
+ return voc.Check(props)
+}
+
+func watchVocabulary(logger logrus.FieldLogger, vocPath string, fn func()) {
+ watcher, err := fsnotify.NewWatcher()
+ if err != nil {
+ logger.WithError(err).Error("vocabulary fsnotify setup failed")
+ return
+ }
+ defer watcher.Close()
+
+ err = watcher.Add(vocPath)
+ if err != nil {
+ logger.WithError(err).Error("vocabulary file watcher failed")
+ return
+ }
+
+ for {
+ select {
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return
+ }
+ logger.WithError(err).Warn("vocabulary file watcher error")
+ case _, ok := <-watcher.Events:
+ if !ok {
+ return
+ }
+ for len(watcher.Events) > 0 {
+ <-watcher.Events
+ }
+ fn()
+ }
+ }
+}
+
+func (conn *Conn) loadVocabularyFile() error {
+ vf, err := os.ReadFile(conn.cluster.API.VocabularyPath)
+ if err != nil {
+ return fmt.Errorf("couldn't read vocabulary file %q: %v", conn.cluster.API.VocabularyPath, err)
+ }
+ mk := make([]string, 0, len(conn.cluster.Collections.ManagedProperties))
+ for k := range conn.cluster.Collections.ManagedProperties {
+ mk = append(mk, k)
+ }
+ voc, err := arvados.NewVocabulary(vf, mk)
+ if err != nil {
+ return fmt.Errorf("while loading vocabulary file %q: %s", conn.cluster.API.VocabularyPath, err)
+ }
+ err = voc.Validate()
+ if err != nil {
+ return fmt.Errorf("while validating vocabulary file %q: %s", conn.cluster.API.VocabularyPath, err)
+ }
+ conn.vocabularyCache = voc
+ return nil
+}
+
+// VocabularyGet refreshes the vocabulary cache if necessary and returns it.
+func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
+ if conn.cluster.API.VocabularyPath == "" {
+ return arvados.Vocabulary{}, nil
+ }
+ logger := ctxlog.FromContext(ctx)
+ if conn.vocabularyCache == nil {
+ // Initial load of vocabulary file.
+ err := conn.loadVocabularyFile()
+ if err != nil {
+ logger.WithError(err).Error("error loading vocabulary file")
+ return arvados.Vocabulary{}, err
+ }
+ go watchVocabulary(logger, conn.cluster.API.VocabularyPath, func() {
+ logger.Info("vocabulary file changed, it'll be reloaded next time it's needed")
+ conn.reloadVocabulary = true
+ })
+ } else if conn.reloadVocabulary {
+ // Requested reload of vocabulary file.
+ conn.reloadVocabulary = false
+ err := conn.loadVocabularyFile()
+ if err != nil {
+ logger.WithError(err).Error("error reloading vocabulary file - ignoring")
+ } else {
+ logger.Info("vocabulary file reloaded successfully")
+ }
+ }
+ return *conn.vocabularyCache, nil
+}
+
// Logout handles the logout of conn giving to the appropriate loginController
func (conn *Conn) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
return conn.loginController.Logout(ctx, opts)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// ContainerRequestCreate defers to railsProxy for everything except
+// vocabulary checking.
+func (conn *Conn) ContainerRequestCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.ContainerRequest, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.ContainerRequest{}, err
+ }
+ resp, err := conn.railsProxy.ContainerRequestCreate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
+
+// ContainerRequestUpdate defers to railsProxy for everything except
+// vocabulary checking.
+func (conn *Conn) ContainerRequestUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.ContainerRequest, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.ContainerRequest{}, err
+ }
+ resp, err := conn.railsProxy.ContainerRequestUpdate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ContainerRequestSuite{})
+
+type ContainerRequestSuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *ContainerRequestSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *ContainerRequestSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *ContainerRequestSuite) TearDownTest(c *check.C) {
+ s.railsSpy.Close()
+}
+
+func (s *ContainerRequestSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
+ c.Assert(err, check.IsNil)
+ c.Assert(voc.Validate(), check.IsNil)
+ s.localdb.vocabularyCache = voc
+ s.cluster.API.VocabularyPath = "foo"
+}
+
+func (s *ContainerRequestSuite) TestCRCreateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+
+ cnt, err := s.localdb.ContainerRequestCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "command": []string{"echo", "foo"},
+ "container_image": "arvados/apitestfixture:latest",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "mounts": map[string]interface{}{
+ "/out": map[string]interface{}{
+ "kind": "tmp",
+ "capacity": 1000000,
+ },
+ },
+ "output_path": "/out",
+ "runtime_constraints": map[string]interface{}{
+ "vcpus": 1,
+ "ram": 2,
+ },
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(cnt.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *ContainerRequestSuite) TestCRUpdateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ cnt, err := s.localdb.ContainerRequestCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "command": []string{"echo", "foo"},
+ "container_image": "arvados/apitestfixture:latest",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "mounts": map[string]interface{}{
+ "/out": map[string]interface{}{
+ "kind": "tmp",
+ "capacity": 1000000,
+ },
+ },
+ "output_path": "/out",
+ "runtime_constraints": map[string]interface{}{
+ "vcpus": 1,
+ "ram": 2,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+ cnt, err = s.localdb.ContainerRequestUpdate(ctx, arvados.UpdateOptions{
+ UUID: cnt.UUID,
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(cnt.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// GroupCreate defers to railsProxy for everything except vocabulary
+// checking.
+func (conn *Conn) GroupCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Group, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Group{}, err
+ }
+ resp, err := conn.railsProxy.GroupCreate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
+
+// GroupUpdate defers to railsProxy for everything except vocabulary
+// checking.
+func (conn *Conn) GroupUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Group, error) {
+ err := conn.checkProperties(ctx, opts.Attrs["properties"])
+ if err != nil {
+ return arvados.Group{}, err
+ }
+ resp, err := conn.railsProxy.GroupUpdate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ return resp, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&GroupSuite{})
+
+type GroupSuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *GroupSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *GroupSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *GroupSuite) TearDownTest(c *check.C) {
+ s.railsSpy.Close()
+}
+
+func (s *GroupSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
+ c.Assert(err, check.IsNil)
+ c.Assert(voc.Validate(), check.IsNil)
+ s.localdb.vocabularyCache = voc
+ s.cluster.API.VocabularyPath = "foo"
+}
+
+func (s *GroupSuite) TestGroupCreateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+
+ grp, err := s.localdb.GroupCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "group_class": "project",
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(grp.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *GroupSuite) TestGroupUpdateWithProperties(c *check.C) {
+ s.setUpVocabulary(c, "")
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ tests := []struct {
+ name string
+ props map[string]interface{}
+ success bool
+ }{
+ {"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
+ {"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
+ {"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
+ {"Empty properties", map[string]interface{}{}, true},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ grp, err := s.localdb.GroupCreate(ctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "group_class": "project",
+ },
+ })
+ c.Assert(err, check.IsNil)
+ grp, err = s.localdb.GroupUpdate(ctx, arvados.UpdateOptions{
+ UUID: grp.UUID,
+ Select: []string{"uuid", "properties"},
+ Attrs: map[string]interface{}{
+ "properties": tt.props,
+ }})
+ if tt.success {
+ c.Assert(err, check.IsNil)
+ c.Assert(grp.Properties, check.DeepEquals, tt.props)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
return rtr.backend.ConfigGet(ctx)
},
},
+ {
+ arvados.EndpointVocabularyGet,
+ func() interface{} { return &struct{}{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.backend.VocabularyGet(ctx)
+ },
+ },
{
arvados.EndpointLogin,
func() interface{} { return &arvados.LoginOptions{} },
return resp, err
}
+func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
+ ep := arvados.EndpointVocabularyGet
+ var resp arvados.Vocabulary
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, nil)
+ return resp, err
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
ep := arvados.EndpointLogin
var resp arvados.LoginResponse
var (
EndpointConfigGet = APIEndpoint{"GET", "arvados/v1/config", ""}
+ EndpointVocabularyGet = APIEndpoint{"GET", "arvados/v1/vocabulary", ""}
EndpointLogin = APIEndpoint{"GET", "login", ""}
EndpointLogout = APIEndpoint{"GET", "logout", ""}
EndpointCollectionCreate = APIEndpoint{"POST", "arvados/v1/collections", "collection"}
type API interface {
ConfigGet(ctx context.Context) (json.RawMessage, error)
+ VocabularyGet(ctx context.Context) (Vocabulary, error)
Login(ctx context.Context, options LoginOptions) (LoginResponse, error)
Logout(ctx context.Context, options LogoutOptions) (LogoutResponse, error)
CollectionCreate(ctx context.Context, options CreateOptions) (Collection, error)
Admin UploadDownloadPermission
}
+type ManagedProperties map[string]struct {
+ Value interface{}
+ Function string
+ Protected bool
+}
+
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
UnloggedAttributes StringSet
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- BlobTrash bool
- BlobTrashLifetime Duration
- BlobTrashCheckInterval Duration
- BlobTrashConcurrency int
- BlobDeleteConcurrency int
- BlobReplicateConcurrency int
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]struct {
- Value interface{}
- Function string
- Protected bool
- }
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ BlobTrash bool
+ BlobTrashLifetime Duration
+ BlobTrashCheckInterval Duration
+ BlobTrashConcurrency int
+ BlobDeleteConcurrency int
+ BlobReplicateConcurrency int
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties ManagedProperties
PreserveVersionIfIdle Duration
TrashSweepInterval Duration
TrustAllContent bool
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "strings"
+)
+
+type Vocabulary struct {
+ reservedTagKeys map[string]bool `json:"-"`
+ StrictTags bool `json:"strict_tags"`
+ Tags map[string]VocabularyTag `json:"tags"`
+}
+
+type VocabularyTag struct {
+ Strict bool `json:"strict"`
+ Labels []VocabularyLabel `json:"labels"`
+ Values map[string]VocabularyTagValue `json:"values"`
+}
+
+// Cannot have a constant map in Go, so we have to use a function
+func (v *Vocabulary) systemTagKeys() map[string]bool {
+ return map[string]bool{
+ "type": true,
+ "template_uuid": true,
+ "groups": true,
+ "username": true,
+ "image_timestamp": true,
+ "docker-image-repo-tag": true,
+ "filters": true,
+ "container_request": true,
+ }
+}
+
+type VocabularyLabel struct {
+ Label string `json:"label"`
+}
+
+type VocabularyTagValue struct {
+ Labels []VocabularyLabel `json:"labels"`
+}
+
+func NewVocabulary(data []byte, managedTagKeys []string) (voc *Vocabulary, err error) {
+ if r := bytes.Compare(data, []byte("")); r == 0 {
+ return &Vocabulary{}, nil
+ }
+ err = json.Unmarshal(data, &voc)
+ if err != nil {
+ return nil, fmt.Errorf("invalid JSON format error: %q", err)
+ }
+ if reflect.DeepEqual(voc, &Vocabulary{}) {
+ return nil, fmt.Errorf("JSON data provided doesn't match Vocabulary format: %q", data)
+ }
+ voc.reservedTagKeys = make(map[string]bool)
+ for _, managedKey := range managedTagKeys {
+ voc.reservedTagKeys[managedKey] = true
+ }
+ for systemKey := range voc.systemTagKeys() {
+ voc.reservedTagKeys[systemKey] = true
+ }
+ err = voc.Validate()
+ if err != nil {
+ return nil, err
+ }
+ return voc, nil
+}
+
+func (v *Vocabulary) Validate() error {
+ if v == nil {
+ return nil
+ }
+ tagKeys := map[string]bool{}
+ // Checks for Vocabulary strictness
+ if v.StrictTags && len(v.Tags) == 0 {
+ return fmt.Errorf("vocabulary is strict but no tags are defined")
+ }
+ // Checks for duplicate tag keys
+ for key := range v.Tags {
+ if v.reservedTagKeys[key] {
+ return fmt.Errorf("tag key %q is reserved", key)
+ }
+ if tagKeys[key] {
+ return fmt.Errorf("duplicate tag key %q", key)
+ }
+ tagKeys[key] = true
+ for _, lbl := range v.Tags[key].Labels {
+ label := strings.ToLower(lbl.Label)
+ if tagKeys[label] {
+ return fmt.Errorf("tag label %q for key %q already seen as a tag key or label", label, key)
+ }
+ tagKeys[label] = true
+ }
+ // Checks for value strictness
+ if v.Tags[key].Strict && len(v.Tags[key].Values) == 0 {
+ return fmt.Errorf("tag key %q is configured as strict but doesn't provide values", key)
+ }
+ // Checks for value duplication within a key
+ tagValues := map[string]bool{}
+ for val := range v.Tags[key].Values {
+ if tagValues[val] {
+ return fmt.Errorf("duplicate tag value %q for tag %q", val, key)
+ }
+ tagValues[val] = true
+ for _, tagLbl := range v.Tags[key].Values[val].Labels {
+ label := strings.ToLower(tagLbl.Label)
+ if tagValues[label] {
+ return fmt.Errorf("tag value label %q for pair (%q:%q) already seen as a value key or label", label, key, val)
+ }
+ tagValues[label] = true
+ }
+ }
+ }
+ return nil
+}
+
+func (v *Vocabulary) getLabelsToKeys() (labels map[string]string) {
+ if v == nil {
+ return
+ }
+ labels = make(map[string]string)
+ for key, val := range v.Tags {
+ for _, lbl := range val.Labels {
+ label := strings.ToLower(lbl.Label)
+ labels[label] = key
+ }
+ }
+ return labels
+}
+
+func (v *Vocabulary) getLabelsToValues(key string) (labels map[string]string) {
+ if v == nil {
+ return
+ }
+ labels = make(map[string]string)
+ if _, ok := v.Tags[key]; ok {
+ for val := range v.Tags[key].Values {
+ for _, tagLbl := range v.Tags[key].Values[val].Labels {
+ label := strings.ToLower(tagLbl.Label)
+ labels[label] = val
+ }
+ }
+ }
+ return labels
+}
+
+func (v *Vocabulary) checkValue(key, val string) error {
+ if _, ok := v.Tags[key].Values[val]; !ok {
+ lcVal := strings.ToLower(val)
+ alias, ok := v.getLabelsToValues(key)[lcVal]
+ if ok {
+ return fmt.Errorf("tag value %q for key %q is not defined but is an alias for %q", val, key, alias)
+ } else if v.Tags[key].Strict {
+ return fmt.Errorf("tag value %q for key %q is not listed as valid", val, key)
+ }
+ }
+ return nil
+}
+
+// Check validates the given data against the vocabulary.
+func (v *Vocabulary) Check(data map[string]interface{}) error {
+ if v == nil {
+ return nil
+ }
+ for key, val := range data {
+ // Checks for key validity
+ if v.reservedTagKeys[key] {
+ // Allow reserved keys to be used even if they are not defined in
+ // the vocabulary no matter its strictness.
+ continue
+ }
+ if _, ok := v.Tags[key]; !ok {
+ lcKey := strings.ToLower(key)
+ alias, ok := v.getLabelsToKeys()[lcKey]
+ if ok {
+ return fmt.Errorf("tag key %q is not defined but is an alias for %q", key, alias)
+ } else if v.StrictTags {
+ return fmt.Errorf("tag key %q is not defined", key)
+ }
+ // If the key is not defined, we don't need to check the value
+ continue
+ }
+ // Checks for value validity -- key is defined
+ switch val := val.(type) {
+ case string:
+ return v.checkValue(key, val)
+ case []interface{}:
+ for _, singleVal := range val {
+ switch singleVal := singleVal.(type) {
+ case string:
+ err := v.checkValue(key, singleVal)
+ if err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("tag value %q for key %q is not a valid type (%T)", singleVal, key, singleVal)
+ }
+ }
+ default:
+ return fmt.Errorf("tag value %q for key %q is not a valid type (%T)", val, key, val)
+ }
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "encoding/json"
+
+ check "gopkg.in/check.v1"
+)
+
+type VocabularySuite struct {
+ testVoc *Vocabulary
+}
+
+var _ = check.Suite(&VocabularySuite{})
+
+func (s *VocabularySuite) SetUpTest(c *check.C) {
+ s.testVoc = &Vocabulary{
+ reservedTagKeys: map[string]bool{
+ "reservedKey": true,
+ },
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}},
+ },
+ },
+ },
+ "IDTAGIMPORTANCE": {
+ Strict: true,
+ Labels: []VocabularyLabel{{Label: "Importance"}, {Label: "Priority"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVAL3": {
+ Labels: []VocabularyLabel{{Label: "Low"}, {Label: "Low priority"}},
+ },
+ "IDVAL2": {
+ Labels: []VocabularyLabel{{Label: "Medium"}, {Label: "Medium priority"}},
+ },
+ "IDVAL1": {
+ Labels: []VocabularyLabel{{Label: "High"}, {Label: "High priority"}},
+ },
+ },
+ },
+ "IDTAGCOMMENT": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Comment"}},
+ },
+ },
+ }
+ err := s.testVoc.Validate()
+ c.Assert(err, check.IsNil)
+}
+
+func (s *VocabularySuite) TestCheck(c *check.C) {
+ tests := []struct {
+ name string
+ strictVoc bool
+ props string
+ expectSuccess bool
+ }{
+ // Check succeeds
+ {"Known key, known value", false, `{"IDTAGANIMALS":"IDVALANIMAL1"}`, true},
+ {"Unknown non-alias key on non-strict vocabulary", false, `{"foo":"bar"}`, true},
+ {"Known non-strict key, unknown non-alias value", false, `{"IDTAGANIMALS":"IDVALANIMAL3"}`, true},
+ {"Undefined but reserved key on strict vocabulary", true, `{"reservedKey":"bar"}`, true},
+ {"Known key, list of known values", false, `{"IDTAGANIMALS":["IDVALANIMAL1","IDVALANIMAL2"]}`, true},
+ {"Known non-strict key, list of unknown non-alias values", false, `{"IDTAGCOMMENT":["hello world","lorem ipsum"]}`, true},
+ // Check fails
+ {"Unknown non-alias key on strict vocabulary", true, `{"foo":"bar"}`, false},
+ {"Known non-strict key, known value alias", false, `{"IDTAGANIMALS":"Loxodonta"}`, false},
+ {"Known strict key, unknown non-alias value", false, `{"IDTAGIMPORTANCE":"Unimportant"}`, false},
+ {"Known strict key, known value alias", false, `{"IDTAGIMPORTANCE":"High"}`, false},
+ {"Known strict key, list of known alias values", false, `{"IDTAGIMPORTANCE":["Unimportant","High"]}`, false},
+ {"Known strict key, list of unknown non-alias values", false, `{"IDTAGIMPORTANCE":["foo","bar"]}`, false},
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ s.testVoc.StrictTags = tt.strictVoc
+
+ var data map[string]interface{}
+ err := json.Unmarshal([]byte(tt.props), &data)
+ c.Assert(err, check.IsNil)
+ err = s.testVoc.Check(data)
+ if tt.expectSuccess {
+ c.Assert(err, check.IsNil)
+ } else {
+ c.Assert(err, check.NotNil)
+ }
+ }
+}
+
+func (s *VocabularySuite) TestNewVocabulary(c *check.C) {
+ tests := []struct {
+ name string
+ data string
+ isValid bool
+ errMatches string
+ expect *Vocabulary
+ }{
+ {"Empty data", "", true, "", &Vocabulary{}},
+ {"Invalid JSON", "foo", false, "invalid JSON format.*", nil},
+ {"Valid, empty JSON", "{}", false, ".*doesn't match Vocabulary format.*", nil},
+ {"Valid JSON, wrong data", `{"foo":"bar"}`, false, ".*doesn't match Vocabulary format.*", nil},
+ {
+ "Simple valid example",
+ `{"tags":{
+ "IDTAGANIMALS":{
+ "strict": false,
+ "labels": [{"label": "Animal"}, {"label": "Creature"}],
+ "values": {
+ "IDVALANIMAL1":{"labels":[{"label":"Human"}, {"label":"Homo sapiens"}]},
+ "IDVALANIMAL2":{"labels":[{"label":"Elephant"}, {"label":"Loxodonta"}]}
+ }
+ }
+ }}`,
+ true, "",
+ &Vocabulary{
+ reservedTagKeys: map[string]bool{
+ "type": true,
+ "template_uuid": true,
+ "groups": true,
+ "username": true,
+ "image_timestamp": true,
+ "docker-image-repo-tag": true,
+ "filters": true,
+ "container_request": true,
+ },
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Homo sapiens"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Loxodonta"}},
+ },
+ },
+ },
+ },
+ },
+ },
+ {
+ "Valid data, but uses reserved key",
+ `{"tags":{
+ "type":{
+ "strict": false,
+ "labels": [{"label": "Type"}]
+ }
+ }}`,
+ false, "tag key.*is reserved", nil,
+ },
+ }
+
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ voc, err := NewVocabulary([]byte(tt.data), []string{})
+ if tt.isValid {
+ c.Assert(err, check.IsNil)
+ } else {
+ c.Assert(err, check.NotNil)
+ if tt.errMatches != "" {
+ c.Assert(err, check.ErrorMatches, tt.errMatches)
+ }
+ }
+ c.Assert(voc, check.DeepEquals, tt.expect)
+ }
+}
+
+func (s *VocabularySuite) TestValidationErrors(c *check.C) {
+ tests := []struct {
+ name string
+ voc *Vocabulary
+ errMatches string
+ }{
+ {
+ "Strict vocabulary, no keys",
+ &Vocabulary{
+ StrictTags: true,
+ },
+ "vocabulary is strict but no tags are defined",
+ },
+ {
+ "Duplicated tag keys",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ },
+ "IDTAGCOMMENT": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Comment"}, {Label: "Animal"}},
+ },
+ },
+ },
+ "tag label.*for key.*already seen.*",
+ },
+ {
+ "Duplicated tag values",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: false,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ Values: map[string]VocabularyTagValue{
+ "IDVALANIMAL1": {
+ Labels: []VocabularyLabel{{Label: "Human"}, {Label: "Mammal"}},
+ },
+ "IDVALANIMAL2": {
+ Labels: []VocabularyLabel{{Label: "Elephant"}, {Label: "Mammal"}},
+ },
+ },
+ },
+ },
+ },
+ "tag value label.*for pair.*already seen.*",
+ },
+ {
+ "Strict key, no values",
+ &Vocabulary{
+ StrictTags: false,
+ Tags: map[string]VocabularyTag{
+ "IDTAGANIMALS": {
+ Strict: true,
+ Labels: []VocabularyLabel{{Label: "Animal"}, {Label: "Creature"}},
+ },
+ },
+ },
+ "tag key.*is configured as strict but doesn't provide values",
+ },
+ }
+ for _, tt := range tests {
+ c.Log(c.TestName()+" ", tt.name)
+ err := tt.voc.Validate()
+ c.Assert(err, check.NotNil)
+ c.Assert(err, check.ErrorMatches, tt.errMatches)
+ }
+}
as.appendCall(ctx, as.ConfigGet, nil)
return nil, as.Error
}
+func (as *APIStub) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
+ as.appendCall(ctx, as.VocabularyGet, nil)
+ return arvados.Vocabulary{}, as.Error
+}
func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
as.appendCall(ctx, as.Login, options)
return arvados.LoginResponse{}, as.Error
projects / arvados.git / commitdiff