echo >&2 "FATAL: --build-version '$2' is invalid, must match pattern ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+|)(~rc[0-9]+|~dev[0-9]+|)-[0-9]+$"
exit 1
else
+ [[ "$2" =~ (.*)-(.*) ]]
ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
fi
table(table table-bordered table-condensed).
|_. Operator|_. Operand type|_. Description|_. Example|
-|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
+|@=@, @!=@, @<>@|string, number, timestamp, JSON-encoded array, JSON-encoded object, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@
+@["tail_uuid","!=",null]@
+@["storage_classes_desired","=","[\"default\"]"]@|
|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
|@in@, @not in@|array of strings|Set membership|@["script_version","in",["main","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
-|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
+|@exists@|string|Presence of subproperty|@["properties","exists","my_subproperty"]@|
+|@contains@|string, array of strings|Presence of one or more keys or array elements|@["storage_classes_desired", "contains", ["foo", "bar"]]@ (matches both @["foo", "bar"]@ and @["foo", "bar", "baz"]@)
+(note @[..., "contains", "foo"]@ is also accepted, and is equivalent to @[..., "contains", ["foo"]]@)|
h4(#filterexpression). Filtering using boolean expressions
|@like@, @ilike@|string|SQL pattern match, single character match is @_@ and wildcard is @%@, ilike is case-insensitive|@["properties.my_subproperty", "like", "d00220fb%"]@|
|@in@, @not in@|array of strings|Set membership|@["properties.my_subproperty", "in", ["fizz", "buzz"]]@|
|@exists@|boolean|Test if a subproperty is present or not (determined by operand).|@["properties.my_subproperty", "exists", true]@|
-|@contains@|string, number|Filter where subproperty has a value either by exact match or value is element of subproperty list.|@["foo", "contains", "bar"]@ will find both @{"foo": "bar"}@ and @{"foo": ["bar", "baz"]}@.|
+|@contains@|string, number|Filter where subproperty has a value either by exact match or value is element of subproperty list.|@["properties.foo", "contains", "bar"]@ will find both @{"foo": "bar"}@ and @{"foo": ["bar", "baz"]}@.|
Note that exclusion filters @!=@ and @not in@ will return records for which the property is not defined at all. To restrict filtering to records on which the subproperty is defined, combine with an @exists@ filter.
|manifest_text|text|||
|replication_desired|number|Minimum storage replication level desired for each data block referenced by this collection. A value of @null@ signifies that the site default replication level (typically 2) is desired.|@2@|
|replication_confirmed|number|Replication level most recently confirmed by the storage system. This field is null when a collection is first created, and is reset to null when the manifest_text changes in a way that introduces a new data block. An integer value indicates the replication level of the _least replicated_ data block in the collection.|@2@, null|
-|replication_confirmed_at|datetime|When replication_confirmed was confirmed. If replication_confirmed is null, this field is also null.||
+|replication_confirmed_at|datetime|When @replication_confirmed@ was confirmed. If @replication_confirmed@ is null, this field is also null.||
+|storage_classes_desired|list|An optional list of storage class names where the blocks should be saved. If not provided, the cluster's default storage class(es) will be set.|@['archival']@|
+|storage_classes_confirmed|list|Storage classes most recently confirmed by the storage system. This field is an empty list when a collection is first created.|@'archival']@, @[]@|
+|storage_classes_confirmed_at|datetime|When @storage_classes_confirmed@ was confirmed. If @storage_classes_confirmed@ is @[]@, this field is null.||
|trash_at|datetime|If @trash_at@ is non-null and in the past, this collection will be hidden from API calls. May be untrashed.||
|delete_at|datetime|If @delete_at@ is non-null and in the past, the collection may be permanently deleted.||
|is_trashed|boolean|True if @trash_at@ is in the past, false if not.||
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package controller
-
-import (
- "bufio"
- "bytes"
- "context"
- "crypto/md5"
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "strings"
- "sync"
-
- "git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
- "git.arvados.org/arvados.git/sdk/go/keepclient"
-)
-
-func rewriteSignatures(clusterID string, expectHash string,
- resp *http.Response, requestError error) (newResponse *http.Response, err error) {
-
- if requestError != nil {
- return resp, requestError
- }
-
- if resp.StatusCode != http.StatusOK {
- return resp, nil
- }
-
- originalBody := resp.Body
- defer originalBody.Close()
-
- var col arvados.Collection
- err = json.NewDecoder(resp.Body).Decode(&col)
- if err != nil {
- return nil, err
- }
-
- // rewriting signatures will make manifest text 5-10% bigger so calculate
- // capacity accordingly
- updatedManifest := bytes.NewBuffer(make([]byte, 0, int(float64(len(col.ManifestText))*1.1)))
-
- hasher := md5.New()
- mw := io.MultiWriter(hasher, updatedManifest)
- sz := 0
-
- scanner := bufio.NewScanner(strings.NewReader(col.ManifestText))
- scanner.Buffer(make([]byte, 1048576), len(col.ManifestText))
- for scanner.Scan() {
- line := scanner.Text()
- tokens := strings.Split(line, " ")
- if len(tokens) < 3 {
- return nil, fmt.Errorf("Invalid stream (<3 tokens): %q", line)
- }
-
- n, err := mw.Write([]byte(tokens[0]))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- for _, token := range tokens[1:] {
- n, err = mw.Write([]byte(" "))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
-
- m := keepclient.SignedLocatorRe.FindStringSubmatch(token)
- if m != nil {
- // Rewrite the block signature to be a remote signature
- _, err = fmt.Fprintf(updatedManifest, "%s%s%s+R%s-%s%s", m[1], m[2], m[3], clusterID, m[5][2:], m[8])
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
-
- // for hash checking, ignore signatures
- n, err = fmt.Fprintf(hasher, "%s%s", m[1], m[2])
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- } else {
- n, err = mw.Write([]byte(token))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- }
- }
- n, err = mw.Write([]byte("\n"))
- if err != nil {
- return nil, fmt.Errorf("Error updating manifest: %v", err)
- }
- sz += n
- }
-
- // Check that expected hash is consistent with
- // portable_data_hash field of the returned record
- if expectHash == "" {
- expectHash = col.PortableDataHash
- } else if expectHash != col.PortableDataHash {
- return nil, fmt.Errorf("portable_data_hash %q on returned record did not match expected hash %q ", expectHash, col.PortableDataHash)
- }
-
- // Certify that the computed hash of the manifest_text matches our expectation
- sum := hasher.Sum(nil)
- computedHash := fmt.Sprintf("%x+%v", sum, sz)
- if computedHash != expectHash {
- return nil, fmt.Errorf("Computed manifest_text hash %q did not match expected hash %q", computedHash, expectHash)
- }
-
- col.ManifestText = updatedManifest.String()
-
- newbody, err := json.Marshal(col)
- if err != nil {
- return nil, err
- }
-
- buf := bytes.NewBuffer(newbody)
- resp.Body = ioutil.NopCloser(buf)
- resp.ContentLength = int64(buf.Len())
- resp.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
-
- return resp, nil
-}
-
-func filterLocalClusterResponse(resp *http.Response, requestError error) (newResponse *http.Response, err error) {
- if requestError != nil {
- return resp, requestError
- }
-
- if resp.StatusCode == http.StatusNotFound {
- // Suppress returning this result, because we want to
- // search the federation.
- return nil, nil
- }
- return resp, nil
-}
-
-type searchRemoteClusterForPDH struct {
- pdh string
- remoteID string
- mtx *sync.Mutex
- sentResponse *bool
- sharedContext *context.Context
- cancelFunc func()
- errors *[]string
- statusCode *int
-}
-
-func fetchRemoteCollectionByUUID(
- h *genericFederatedRequestHandler,
- effectiveMethod string,
- clusterID *string,
- uuid string,
- remainder string,
- w http.ResponseWriter,
- req *http.Request) bool {
-
- if effectiveMethod != "GET" {
- // Only handle GET requests right now
- return false
- }
-
- if uuid != "" {
- // Collection UUID GET request
- *clusterID = uuid[0:5]
- if *clusterID != "" && *clusterID != h.handler.Cluster.ClusterID {
- // request for remote collection by uuid
- resp, err := h.handler.remoteClusterRequest(*clusterID, req)
- newResponse, err := rewriteSignatures(*clusterID, "", resp, err)
- h.handler.proxy.ForwardResponse(w, newResponse, err)
- return true
- }
- }
-
- return false
-}
-
-func fetchRemoteCollectionByPDH(
- h *genericFederatedRequestHandler,
- effectiveMethod string,
- clusterID *string,
- uuid string,
- remainder string,
- w http.ResponseWriter,
- req *http.Request) bool {
-
- if effectiveMethod != "GET" {
- // Only handle GET requests right now
- return false
- }
-
- m := collectionsByPDHRe.FindStringSubmatch(req.URL.Path)
- if len(m) != 2 {
- return false
- }
-
- // Request for collection by PDH. Search the federation.
-
- // First, query the local cluster.
- resp, err := h.handler.localClusterRequest(req)
- newResp, err := filterLocalClusterResponse(resp, err)
- if newResp != nil || err != nil {
- h.handler.proxy.ForwardResponse(w, newResp, err)
- return true
- }
-
- // Create a goroutine for each cluster in the
- // RemoteClusters map. The first valid result gets
- // returned to the client. When that happens, all
- // other outstanding requests are cancelled
- sharedContext, cancelFunc := context.WithCancel(req.Context())
- defer cancelFunc()
-
- req = req.WithContext(sharedContext)
- wg := sync.WaitGroup{}
- pdh := m[1]
- success := make(chan *http.Response)
- errorChan := make(chan error, len(h.handler.Cluster.RemoteClusters))
-
- acquire, release := semaphore(h.handler.Cluster.API.MaxRequestAmplification)
-
- for remoteID := range h.handler.Cluster.RemoteClusters {
- if remoteID == h.handler.Cluster.ClusterID {
- // No need to query local cluster again
- continue
- }
- if remoteID == "*" {
- // This isn't a real remote cluster: it just sets defaults for unlisted remotes.
- continue
- }
-
- wg.Add(1)
- go func(remote string) {
- defer wg.Done()
- acquire()
- defer release()
- select {
- case <-sharedContext.Done():
- return
- default:
- }
-
- resp, err := h.handler.remoteClusterRequest(remote, req)
- wasSuccess := false
- defer func() {
- if resp != nil && !wasSuccess {
- resp.Body.Close()
- }
- }()
- if err != nil {
- errorChan <- err
- return
- }
- if resp.StatusCode != http.StatusOK {
- errorChan <- HTTPError{resp.Status, resp.StatusCode}
- return
- }
- select {
- case <-sharedContext.Done():
- return
- default:
- }
-
- newResponse, err := rewriteSignatures(remote, pdh, resp, nil)
- if err != nil {
- errorChan <- err
- return
- }
- select {
- case <-sharedContext.Done():
- case success <- newResponse:
- wasSuccess = true
- }
- }(remoteID)
- }
- go func() {
- wg.Wait()
- cancelFunc()
- }()
-
- errorCode := http.StatusNotFound
-
- for {
- select {
- case newResp = <-success:
- h.handler.proxy.ForwardResponse(w, newResp, nil)
- return true
- case <-sharedContext.Done():
- var errors []string
- for len(errorChan) > 0 {
- err := <-errorChan
- if httperr, ok := err.(HTTPError); !ok || httperr.Code != http.StatusNotFound {
- errorCode = http.StatusBadGateway
- }
- errors = append(errors, err.Error())
- }
- httpserver.Errors(w, errors, errorCode)
- return true
- }
- }
-
- // shouldn't ever get here
- return true
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package controller
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io/ioutil"
- "net/http"
- "strings"
-
- "git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/httpserver"
-)
-
-func remoteContainerRequestCreate(
- h *genericFederatedRequestHandler,
- effectiveMethod string,
- clusterID *string,
- uuid string,
- remainder string,
- w http.ResponseWriter,
- req *http.Request) bool {
-
- if effectiveMethod != "POST" || uuid != "" || remainder != "" {
- return false
- }
-
- // First make sure supplied token is valid.
- creds := auth.NewCredentials()
- creds.LoadTokensFromHTTPRequest(req)
-
- currentUser, ok, err := h.handler.validateAPItoken(req, creds.Tokens[0])
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusInternalServerError)
- return true
- } else if !ok {
- httpserver.Error(w, "invalid API token", http.StatusForbidden)
- return true
- }
-
- if *clusterID == "" || *clusterID == h.handler.Cluster.ClusterID {
- // Submitting container request to local cluster. No
- // need to set a runtime_token (rails api will create
- // one when the container runs) or do a remote cluster
- // request.
- return false
- }
-
- if req.Header.Get("Content-Type") != "application/json" {
- httpserver.Error(w, "Expected Content-Type: application/json, got "+req.Header.Get("Content-Type"), http.StatusBadRequest)
- return true
- }
-
- originalBody := req.Body
- defer originalBody.Close()
- var request map[string]interface{}
- err = json.NewDecoder(req.Body).Decode(&request)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusBadRequest)
- return true
- }
-
- crString, ok := request["container_request"].(string)
- if ok {
- var crJSON map[string]interface{}
- err := json.Unmarshal([]byte(crString), &crJSON)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusBadRequest)
- return true
- }
-
- request["container_request"] = crJSON
- }
-
- containerRequest, ok := request["container_request"].(map[string]interface{})
- if !ok {
- // Use toplevel object as the container_request object
- containerRequest = request
- }
-
- // If runtime_token is not set, create a new token
- if _, ok := containerRequest["runtime_token"]; !ok {
- if len(currentUser.Authorization.Scopes) != 1 || currentUser.Authorization.Scopes[0] != "all" {
- httpserver.Error(w, "Token scope is not [all]", http.StatusForbidden)
- return true
- }
-
- if strings.HasPrefix(currentUser.Authorization.UUID, h.handler.Cluster.ClusterID) {
- // Local user, submitting to a remote cluster.
- // Create a new time-limited token.
- newtok, err := h.handler.createAPItoken(req, currentUser.UUID, nil)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusForbidden)
- return true
- }
- containerRequest["runtime_token"] = newtok.TokenV2()
- } else {
- // Remote user. Container request will use the
- // current token, minus the trailing portion
- // (optional container uuid).
- sp := strings.Split(creds.Tokens[0], "/")
- if len(sp) >= 3 {
- containerRequest["runtime_token"] = strings.Join(sp[0:3], "/")
- } else {
- containerRequest["runtime_token"] = creds.Tokens[0]
- }
- }
- }
-
- newbody, err := json.Marshal(request)
- buf := bytes.NewBuffer(newbody)
- req.Body = ioutil.NopCloser(buf)
- req.ContentLength = int64(buf.Len())
- req.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
-
- resp, err := h.handler.remoteClusterRequest(*clusterID, req)
- h.handler.proxy.ForwardResponse(w, resp, err)
- return true
-}
wfHandler := &genericFederatedRequestHandler{next, h, wfRe, nil}
containersHandler := &genericFederatedRequestHandler{next, h, containersRe, nil}
- containerRequestsHandler := &genericFederatedRequestHandler{next, h, containerRequestsRe,
- []federatedRequestDelegate{remoteContainerRequestCreate}}
- collectionsRequestsHandler := &genericFederatedRequestHandler{next, h, collectionsRe,
- []federatedRequestDelegate{fetchRemoteCollectionByUUID, fetchRemoteCollectionByPDH}}
linksRequestsHandler := &genericFederatedRequestHandler{next, h, linksRe, nil}
mux.Handle("/arvados/v1/workflows", wfHandler)
mux.Handle("/arvados/v1/workflows/", wfHandler)
mux.Handle("/arvados/v1/containers", containersHandler)
mux.Handle("/arvados/v1/containers/", containersHandler)
- mux.Handle("/arvados/v1/container_requests", containerRequestsHandler)
- mux.Handle("/arvados/v1/container_requests/", containerRequestsHandler)
- mux.Handle("/arvados/v1/collections", collectionsRequestsHandler)
- mux.Handle("/arvados/v1/collections/", collectionsRequestsHandler)
mux.Handle("/arvados/v1/links", linksRequestsHandler)
mux.Handle("/arvados/v1/links/", linksRequestsHandler)
mux.Handle("/", next)
cluster.API.MaxItemsPerResponse = 1000
cluster.API.MaxRequestAmplification = 4
cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+ cluster.Collections.BlobSigning = true
+ cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+ cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
s.testHandler.Cluster.ClusterID = "zzzzz"
s.testHandler.Cluster.SystemRootToken = arvadostest.SystemRootToken
s.testHandler.Cluster.API.MaxTokenLifetime = arvados.Duration(time.Hour)
- s.testHandler.Cluster.Collections.BlobSigningTTL = arvados.Duration(336 * time.Hour) // For some reason, this was set to 0h
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+)
+
+// CollectionGet defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionGet(ctx context.Context, opts arvados.GetOptions) (arvados.Collection, error) {
+ if len(opts.Select) > 0 {
+ // We need to know IsTrashed and TrashAt to implement
+ // signing properly, even if the caller doesn't want
+ // them.
+ opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+ }
+ resp, err := conn.railsProxy.CollectionGet(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ conn.signCollection(ctx, &resp)
+ return resp, nil
+}
+
+// CollectionList defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionList(ctx context.Context, opts arvados.ListOptions) (arvados.CollectionList, error) {
+ if len(opts.Select) > 0 {
+ // We need to know IsTrashed and TrashAt to implement
+ // signing properly, even if the caller doesn't want
+ // them.
+ opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+ }
+ resp, err := conn.railsProxy.CollectionList(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ for i := range resp.Items {
+ conn.signCollection(ctx, &resp.Items[i])
+ }
+ return resp, nil
+}
+
+// CollectionCreate defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptions) (arvados.Collection, error) {
+ if len(opts.Select) > 0 {
+ // We need to know IsTrashed and TrashAt to implement
+ // signing properly, even if the caller doesn't want
+ // them.
+ opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+ }
+ resp, err := conn.railsProxy.CollectionCreate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ conn.signCollection(ctx, &resp)
+ return resp, nil
+}
+
+// CollectionUpdate defers to railsProxy for everything except blob
+// signatures.
+func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Collection, error) {
+ if len(opts.Select) > 0 {
+ // We need to know IsTrashed and TrashAt to implement
+ // signing properly, even if the caller doesn't want
+ // them.
+ opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
+ }
+ resp, err := conn.railsProxy.CollectionUpdate(ctx, opts)
+ if err != nil {
+ return resp, err
+ }
+ conn.signCollection(ctx, &resp)
+ return resp, nil
+}
+
+func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection) {
+ if coll.IsTrashed || coll.ManifestText == "" || !conn.cluster.Collections.BlobSigning {
+ return
+ }
+ var token string
+ if creds, ok := auth.FromContext(ctx); ok && len(creds.Tokens) > 0 {
+ token = creds.Tokens[0]
+ }
+ if token == "" {
+ return
+ }
+ ttl := conn.cluster.Collections.BlobSigningTTL.Duration()
+ exp := time.Now().Add(ttl)
+ if coll.TrashAt != nil && !coll.TrashAt.IsZero() && coll.TrashAt.Before(exp) {
+ exp = *coll.TrashAt
+ }
+ coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "regexp"
+ "strconv"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CollectionSuite{})
+
+type CollectionSuite struct {
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *CollectionSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *CollectionSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.cluster)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+}
+
+func (s *CollectionSuite) TearDownTest(c *check.C) {
+ s.railsSpy.Close()
+}
+
+func (s *CollectionSuite) TestSignatures(c *check.C) {
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
+ c.Check(err, check.IsNil)
+ c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+ s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
+
+ resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection, Select: []string{"manifest_text"}})
+ c.Check(err, check.IsNil)
+ c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+
+ lresp, err := s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}})
+ c.Check(err, check.IsNil)
+ if c.Check(lresp.Items, check.HasLen, 1) {
+ c.Check(lresp.Items[0].UUID, check.Equals, arvadostest.FooCollection)
+ c.Check(lresp.Items[0].ManifestText, check.Equals, "")
+ c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
+ }
+
+ lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"manifest_text"}})
+ c.Check(err, check.IsNil)
+ if c.Check(lresp.Items, check.HasLen, 1) {
+ c.Check(lresp.Items[0].ManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3\+A[0-9a-f]+@[0-9a-f]+ 0:.*`)
+ c.Check(lresp.Items[0].UnsignedManifestText, check.Equals, "")
+ }
+
+ lresp, err = s.localdb.CollectionList(ctx, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}}, Select: []string{"unsigned_manifest_text"}})
+ c.Check(err, check.IsNil)
+ if c.Check(lresp.Items, check.HasLen, 1) {
+ c.Check(lresp.Items[0].ManifestText, check.Equals, "")
+ c.Check(lresp.Items[0].UnsignedManifestText, check.Matches, `(?ms).* acbd[^ ]*\+3 0:.*`)
+ }
+
+ // early trash date causes lower signature TTL (even if
+ // trash_at and is_trashed fields are unselected)
+ trashed, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+ Select: []string{"uuid", "manifest_text"},
+ Attrs: map[string]interface{}{
+ "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
+ "trash_at": time.Now().UTC().Add(time.Hour),
+ }})
+ c.Assert(err, check.IsNil)
+ s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour)
+ resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
+ c.Assert(err, check.IsNil)
+ s.checkSignatureExpiry(c, resp.ManifestText, time.Hour)
+
+ // distant future trash date does not cause higher signature TTL
+ trashed, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+ UUID: trashed.UUID,
+ Attrs: map[string]interface{}{
+ "trash_at": time.Now().UTC().Add(time.Hour * 24 * 365),
+ }})
+ c.Assert(err, check.IsNil)
+ s.checkSignatureExpiry(c, trashed.ManifestText, time.Hour*24*7*2)
+ resp, err = s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: trashed.UUID})
+ c.Assert(err, check.IsNil)
+ s.checkSignatureExpiry(c, resp.ManifestText, time.Hour*24*7*2)
+
+ // Make sure groups/contents doesn't return manifest_text with
+ // collections (if it did, we'd need to sign it).
+ gresp, err := s.localdb.GroupContents(ctx, arvados.GroupContentsOptions{
+ Limit: -1,
+ Filters: []arvados.Filter{{"uuid", "=", arvadostest.FooCollection}},
+ Select: []string{"uuid", "manifest_text"},
+ })
+ c.Check(err, check.IsNil)
+ if c.Check(gresp.Items, check.HasLen, 1) {
+ c.Check(gresp.Items[0].(map[string]interface{})["uuid"], check.Equals, arvadostest.FooCollection)
+ c.Check(gresp.Items[0].(map[string]interface{})["manifest_text"], check.Equals, nil)
+ }
+}
+
+func (s *CollectionSuite) checkSignatureExpiry(c *check.C, manifestText string, expectedTTL time.Duration) {
+ m := regexp.MustCompile(`@([[:xdigit:]]+)`).FindStringSubmatch(manifestText)
+ c.Assert(m, check.HasLen, 2)
+ sigexp, err := strconv.ParseInt(m[1], 16, 64)
+ c.Assert(err, check.IsNil)
+ expectedExp := time.Now().Add(expectedTTL).Unix()
+ c.Check(sigexp > expectedExp-60, check.Equals, true)
+ c.Check(sigexp <= expectedExp, check.Equals, true)
+}
+
+func (s *CollectionSuite) TestSignaturesDisabled(c *check.C) {
+ s.localdb.cluster.Collections.BlobSigning = false
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+ resp, err := s.localdb.CollectionGet(ctx, arvados.GetOptions{UUID: arvadostest.FooCollection})
+ c.Check(err, check.IsNil)
+ c.Check(resp.ManifestText, check.Matches, `(?ms).* acbd[^ +]*\+3 0:.*`)
+}
"net/http"
"os"
"path/filepath"
+ "time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
PostgreSQL: integrationTestCluster().PostgreSQL,
}}
handler.Cluster.TLS.Insecure = true
+ handler.Cluster.Collections.BlobSigning = true
+ handler.Cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
+ handler.Cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
}
func (s *Suite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
// Get the various arvados, arvadosclient, and keep client objects
type Suite struct{}
func (*Suite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
}
type UpdateOptions struct {
UUID string `json:"uuid"`
Attrs map[string]interface{} `json:"attrs"`
+ Select []string `json:"select"`
BypassFederation bool `json:"bypass_federation"`
}
package arvadostest
import (
- "bufio"
- "bytes"
+ "crypto/tls"
"fmt"
"io/ioutil"
"log"
+ "net/http"
"os"
"os/exec"
"path"
"strconv"
"strings"
+
+ "gopkg.in/check.v1"
)
var authSettings = make(map[string]string)
-// ResetEnv resets test env
+// ResetEnv resets ARVADOS_* env vars to whatever they were the first
+// time this func was called.
+//
+// Call it from your SetUpTest or SetUpSuite func if your tests modify
+// env vars.
func ResetEnv() {
- for k, v := range authSettings {
- os.Setenv(k, v)
- }
-}
-
-// APIHost returns the address:port of the current test server.
-func APIHost() string {
- h := authSettings["ARVADOS_API_HOST"]
- if h == "" {
- log.Fatal("arvadostest.APIHost() was called but authSettings is not populated")
- }
- return h
-}
-
-// ParseAuthSettings parses auth settings from given input
-func ParseAuthSettings(authScript []byte) {
- scanner := bufio.NewScanner(bytes.NewReader(authScript))
- for scanner.Scan() {
- line := scanner.Text()
- if 0 != strings.Index(line, "export ") {
- log.Printf("Ignoring: %v", line)
- continue
+ if len(authSettings) == 0 {
+ for _, e := range os.Environ() {
+ e := strings.SplitN(e, "=", 2)
+ if len(e) == 2 {
+ authSettings[e[0]] = e[1]
+ }
}
- toks := strings.SplitN(strings.Replace(line, "export ", "", 1), "=", 2)
- if len(toks) == 2 {
- authSettings[toks[0]] = toks[1]
- } else {
- log.Fatalf("Could not parse: %v", line)
+ } else {
+ for k, v := range authSettings {
+ os.Setenv(k, v)
}
}
- log.Printf("authSettings: %v", authSettings)
}
var pythonTestDir string
}
}
-// StartAPI starts test API server
-func StartAPI() {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
- chdirToPythonTests()
-
- cmd := exec.Command("python", "run_test_server.py", "start", "--auth", "admin")
- cmd.Stdin = nil
- cmd.Stderr = os.Stderr
-
- authScript, err := cmd.Output()
- if err != nil {
- log.Fatalf("%+v: %s", cmd.Args, err)
- }
- ParseAuthSettings(authScript)
- ResetEnv()
-}
-
-// StopAPI stops test API server
-func StopAPI() {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
- chdirToPythonTests()
-
- cmd := exec.Command("python", "run_test_server.py", "stop")
- bgRun(cmd)
- // Without Wait, "go test" in go1.10.1 tends to hang. https://github.com/golang/go/issues/24050
- cmd.Wait()
+func ResetDB(c *check.C) {
+ hc := http.Client{Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }}
+ req, err := http.NewRequest("POST", "https://"+os.Getenv("ARVADOS_TEST_API_HOST")+"/database/reset", nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "Bearer "+AdminToken)
+ resp, err := hc.Do(req)
+ c.Assert(err, check.IsNil)
+ defer resp.Body.Close()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
}
// StartKeep starts the given number of keep servers,
type suite struct{}
-func (s *suite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
-func (s *suite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func (s *suite) TestTrackContainer(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *ServerRequiredSuite) SetUpTest(c *C) {
:storage_classes_desired:
A list of storage class names where to upload the data. If None,
- the keepstores are expected to store the data into their default
- storage class.
+ the keep client is expected to store the data into the cluster's
+ default storage class(es).
"""
self.hits_counter = Counter()
self.misses_counter = Counter()
self._storage_classes_unsupported_warning = False
+ self._default_classes = []
if local_store:
self.local_store = local_store
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
+ try:
+ self._default_classes = [
+ k for k, v in self.api_client.config()['StorageClasses'].items() if v['Default']]
+ except KeyError:
+ # We're talking to an old cluster
+ pass
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
"failed to read {} after {}".format(loc_s, loop.attempts_str()), service_errors, label="service")
@retry.retry_method
- def put(self, data, copies=2, num_retries=None, request_id=None, classes=[]):
+ def put(self, data, copies=2, num_retries=None, request_id=None, classes=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
be written.
"""
+ classes = classes or self._default_classes
+
if not isinstance(data, bytes):
data = data.encode()
class ApiClientMock(object):
def api_client_mock(self):
- return mock.MagicMock(name='api_client_mock')
+ api_mock = mock.MagicMock(name='api_client_mock')
+ api_mock.config.return_value = {
+ 'StorageClasses': {
+ 'default': {'Default': True}
+ }
+ }
+ return api_mock
def mock_keep_services(self, api_mock=None, status=200, count=12,
service_type='disk',
self.data = b'xyzzy'
self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ def test_multiple_default_storage_classes_req_header(self):
+ api_mock = self.api_client_mock()
+ api_mock.config.return_value = {
+ 'StorageClasses': {
+ 'foo': { 'Default': True },
+ 'bar': { 'Default': True },
+ 'baz': { 'Default': False }
+ }
+ }
+ api_client = self.mock_keep_services(api_mock=api_mock, count=2)
+ keep_client = arvados.KeepClient(api_client=api_client)
+ resp_hdr = {
+ 'x-keep-storage-classes-confirmed': 'foo=1, bar=1',
+ 'x-keep-replicas-stored': 1
+ }
+ with tutil.mock_keep_responses(self.locator, 200, **resp_hdr) as mock:
+ keep_client.put(self.data, copies=1)
+ req_hdr = mock.responses[0]
+ self.assertIn(
+ 'X-Keep-Storage-Classes: bar, foo', req_hdr.getopt(pycurl.HTTPHEADER))
+
def test_storage_classes_req_header(self):
+ self.assertEqual(
+ self.api_client.config()['StorageClasses'],
+ {'default': {'Default': True}})
cases = [
# requested, expected
[['foo'], 'X-Keep-Storage-Classes: foo'],
[['bar', 'foo'], 'X-Keep-Storage-Classes: bar, foo'],
- [[], None],
+ [[], 'X-Keep-Storage-Classes: default'],
+ [None, 'X-Keep-Storage-Classes: default'],
]
for req_classes, expected_header in cases:
headers = {'x-keep-replicas-stored': 1}
- if len(req_classes) > 0:
+ if req_classes is None or len(req_classes) == 0:
+ confirmed_hdr = 'default=1'
+ elif len(req_classes) > 0:
confirmed_hdr = ', '.join(["{}=1".format(cls) for cls in req_classes])
- headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr})
+ headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr})
with tutil.mock_keep_responses(self.locator, 200, **headers) as mock:
self.keep_client.put(self.data, copies=1, classes=req_classes)
- resp = mock.responses[0]
- if expected_header is not None:
- self.assertIn(expected_header, resp.getopt(pycurl.HTTPHEADER))
- else:
- for hdr in resp.getopt(pycurl.HTTPHEADER):
- self.assertNotRegex(hdr, r'^X-Keep-Storage-Classes.*')
+ req_hdr = mock.responses[0]
+ self.assertIn(expected_header, req_hdr.getopt(pycurl.HTTPHEADER))
def test_partial_storage_classes_put(self):
headers = {
return "abc"
elif r == "insecure":
return False
+ elif r == "config":
+ return lambda: {}
else:
raise arvados.errors.KeepReadError()
keep_client = arvados.KeepClient(api_client=ApiMock(),
include_old_versions: params[:include_old_versions],
}
- # It matters which Collection object we pick because we use it to get signed_manifest_text,
- # the value of which is affected by the value of trash_at.
+ # It matters which Collection object we pick because blob
+ # signatures depend on the value of trash_at.
#
- # From postgres doc: "By default, null values sort as if larger than any non-null
- # value; that is, NULLS FIRST is the default for DESC order, and
- # NULLS LAST otherwise."
+ # From postgres doc: "By default, null values sort as if larger
+ # than any non-null value; that is, NULLS FIRST is the default
+ # for DESC order, and NULLS LAST otherwise."
#
# "trash_at desc" sorts null first, then latest to earliest, so
# it will select the Collection object with the longest
@object = {
uuid: c.portable_data_hash,
portable_data_hash: c.portable_data_hash,
- manifest_text: c.signed_manifest_text,
+ manifest_text: c.manifest_text,
}
end
else
t.add :description
t.add :properties
t.add :portable_data_hash
- t.add :signed_manifest_text, as: :manifest_text
t.add :manifest_text, as: :unsigned_manifest_text
+ t.add :manifest_text, as: :manifest_text
t.add :replication_desired
t.add :replication_confirmed
t.add :replication_confirmed_at
def self.attributes_required_columns
super.merge(
- # If we don't list manifest_text explicitly, the
- # params[:select] code gets confused by the way we
- # expose signed_manifest_text as manifest_text in the
- # API response, and never let clients select the
- # manifest_text column.
- #
- # We need trash_at and is_trashed to determine the
- # correct timestamp in signed_manifest_text.
- 'manifest_text' => ['manifest_text', 'trash_at', 'is_trashed'],
+ # If we don't list unsigned_manifest_text explicitly,
+ # the params[:select] code gets confused by the way we
+ # expose manifest_text as unsigned_manifest_text in
+ # the API response, and never let clients select the
+ # unsigned_manifest_text column.
'unsigned_manifest_text' => ['manifest_text'],
'name' => ['name'],
)
end
end
- def signed_manifest_text
+ def signed_manifest_text_only_for_tests
if !has_attribute? :manifest_text
return nil
elsif is_trashed
token = Thread.current[:token]
exp = [db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i,
trash_at].compact.map(&:to_i).min
- self.class.sign_manifest manifest_text, token, exp
+ self.class.sign_manifest_only_for_tests manifest_text, token, exp
end
end
- def self.sign_manifest manifest, token, exp=nil
+ def self.sign_manifest_only_for_tests manifest, token, exp=nil
if exp.nil?
exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
end
raise ArgumentError.new("Invalid operator '#{operator}' (#{operator.class}) in filter")
end
+ operator = operator.downcase
cond_out = []
- if attrs_in == 'any' && (operator.casecmp('ilike').zero? || operator.casecmp('like').zero?) && (operand.is_a? String) && operand.match('^[%].*[%]$')
+ if attrs_in == 'any' && (operator == 'ilike' || operator == 'like') && (operand.is_a? String) && operand.match('^[%].*[%]$')
# Trigram index search
cond_out << model_class.full_text_trgm + " #{operator} ?"
param_out << operand
end
# jsonb search
- case operator.downcase
+ case operator
when '=', '!='
- not_in = if operator.downcase == "!=" then "NOT " else "" end
+ not_in = if operator == "!=" then "NOT " else "" end
cond_out << "#{not_in}(#{attr_table_name}.#{attr} @> ?::jsonb)"
param_out << SafeJSON.dump({proppath => operand})
when 'in'
else
raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
end
- elsif operator.downcase == "exists"
+ elsif operator == "exists"
if col.type != :jsonb
raise ArgumentError.new("Invalid attribute '#{attr}' for operator '#{operator}' in filter")
end
end
cond_out << "#{attr1} #{operator} #{attr2}"
else
- if !attr_model_class.searchable_columns(operator).index attr
+ if !attr_model_class.searchable_columns(operator).index(attr) &&
+ !(col.andand.type == :jsonb && ['contains', '=', '<>', '!='].index(operator))
raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
end
- case operator.downcase
+ case operator
when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
attr_type = attr_model_class.attribute_column(attr).type
operator = '<>' if operator == '!='
end
end
cond_out << cond.join(' OR ')
+ when 'contains'
+ if col.andand.type != :jsonb
+ raise ArgumentError.new("Invalid attribute '#{attr}' for '#{operator}' operator")
+ end
+ if operand == []
+ raise ArgumentError.new("Invalid operand '#{operand.inspect}' for '#{operator}' operator")
+ end
+ operand = [operand] unless operand.is_a? Array
+ operand.each do |op|
+ if !op.is_a?(String)
+ raise ArgumentError.new("Invalid element #{operand.inspect} in operand for #{operator.inspect} operator (operand must be a string or array of strings)")
+ end
+ end
+ # We use jsonb_exists_all(a,b) instead of "a ?& b" because
+ # the pg gem thinks "?" is a bind var. And we use string
+ # interpolation instead of param_out because the pg gem
+ # flattens param_out / doesn't support passing arrays as
+ # bind vars.
+ q = operand.map { |s| ActiveRecord::Base.connection.quote(s) }.join(',')
+ cond_out << "jsonb_exists_all(#{attr_table_name}.#{attr}, array[#{q}])"
else
raise ArgumentError.new("Invalid operator '#{operator}'")
end
end
end
- def assert_unsigned_manifest resp, label=''
- txt = resp['unsigned_manifest_text']
+ def assert_unsigned_manifest txt, label=''
assert_not_nil(txt, "#{label} unsigned_manifest_text was nil")
locs = 0
txt.scan(/ [[:xdigit:]]{32}\S*/) do |tok|
"past version not included on index")
end
- test "collections.get returns signed locators, and no unsigned_manifest_text" do
+ test "collections.get returns unsigned locators, and no unsigned_manifest_text" do
permit_unsigned_manifests
authorize_with :active
get :show, params: {id: collections(:foo_file).uuid}
assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'foo_file'
+ assert_unsigned_manifest json_response["manifest_text"], 'foo_file'
refute_includes json_response, 'unsigned_manifest_text'
end
['v1token', 'v2token'].each do |token_method|
- test "correct signatures are given for #{token_method}" do
- token = api_client_authorizations(:active).send(token_method)
- authorize_with_token token
- get :show, params: {id: collections(:foo_file).uuid}
- assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'foo_file', token: token
- end
-
test "signatures with #{token_method} are accepted" do
token = api_client_authorizations(:active).send(token_method)
signed = Blob.sign_locator(
},
}
assert_response :success
- assert_signed_manifest json_response['manifest_text'], 'updated', token: token
+ assert_unsigned_manifest json_response['manifest_text'], 'updated'
end
end
- test "index with manifest_text selected returns signed locators" do
+ test "index with manifest_text selected returns unsigned locators" do
columns = %w(uuid owner_uuid manifest_text)
authorize_with :active
get :index, params: {select: columns}
json_response["items"].each do |coll|
assert_equal(coll.keys - ['kind'], columns,
"Collections index did not respect selected columns")
- assert_signed_manifest coll['manifest_text'], coll['uuid']
+ assert_unsigned_manifest coll['manifest_text'], coll['uuid']
end
end
json_response["items"].each do |coll|
assert_equal(coll.keys - ['kind'], ['unsigned_manifest_text'],
"Collections index did not respect selected columns")
- locs += assert_unsigned_manifest coll, coll['uuid']
+ assert_nil coll['manifest_text']
+ locs += assert_unsigned_manifest coll['unsigned_manifest_text'], coll['uuid']
end
assert_operator locs, :>, 0, "no locators found in any manifests"
end
assert_not_nil assigns(:object)
resp = assigns(:object)
assert_equal foo_collection[:portable_data_hash], resp[:portable_data_hash]
- assert_signed_manifest resp[:manifest_text]
+ assert_unsigned_manifest resp[:manifest_text]
# The manifest in the response will have had permission hints added.
# Remove any permission hints in the response before comparing it to the source.
authorize_with :active
manifest_text = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:foo.txt\n"
if !unsigned
- manifest_text = Collection.sign_manifest manifest_text, api_token(:active)
+ manifest_text = Collection.sign_manifest_only_for_tests manifest_text, api_token(:active)
end
post :create, params: {
collection: {
assert_not_nil assigns(:object)
resp = JSON.parse(@response.body)
assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
+ # All of the signatures in the output must be valid.
resp['manifest_text'].lines.each do |entry|
m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
- if m
+ if m && m[0].index('+A')
assert Blob.verify_signature m[0], signing_opts
end
end
assert_not_nil assigns(:object)
resp = JSON.parse(@response.body)
assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
+ # All of the signatures in the output must be valid.
resp['manifest_text'].lines.each do |entry|
m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
- if m
+ if m && m[0].index('+A')
assert Blob.verify_signature m[0], signing_opts
end
end
assert_equal manifest_text, stripped_manifest
end
- test "multiple signed locators per line" do
- permit_unsigned_manifests
- authorize_with :active
- locators = %w(
- d41d8cd98f00b204e9800998ecf8427e+0
- acbd18db4cc2f85cedef654fccc4a4d8+3
- ea10d51bcf88862dbcc36eb292017dfd+45)
-
- signing_opts = {
- key: Rails.configuration.Collections.BlobSigningKey,
- api_token: api_token(:active),
- }
-
- unsigned_manifest = [".", *locators, "0:0:foo.txt\n"].join(" ")
- manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
- '+' +
- unsigned_manifest.length.to_s
-
- signed_locators = locators.map { |loc| Blob.sign_locator loc, signing_opts }
- signed_manifest = [".", *signed_locators, "0:0:foo.txt\n"].join(" ")
-
- post :create, params: {
- collection: {
- manifest_text: signed_manifest,
- portable_data_hash: manifest_uuid,
- }
- }
- assert_response :success
- assert_not_nil assigns(:object)
- resp = JSON.parse(@response.body)
- assert_equal manifest_uuid, resp['portable_data_hash']
- # All of the locators in the output must be signed.
- # Each line is of the form "path locator locator ... 0:0:file.txt"
- # entry.split[1..-2] will yield just the tokens in the middle of the line
- returned_locator_count = 0
- resp['manifest_text'].lines.each do |entry|
- entry.split[1..-2].each do |tok|
- returned_locator_count += 1
- assert Blob.verify_signature tok, signing_opts
- end
- end
- assert_equal locators.count, returned_locator_count
- end
-
test 'Reject manifest with unsigned blob' do
permit_unsigned_manifests false
authorize_with :active
}
assert_response 422
end
+
+ ["storage_classes_desired", "storage_classes_confirmed"].each do |attr|
+ test "filter collections by #{attr}" do
+ authorize_with(:active)
+ get :index, params: {
+ filters: [[attr, "=", '["default"]']]
+ }
+ assert_response :success
+ assert_not_equal 0, json_response["items"].length
+ json_response["items"].each do |c|
+ assert_equal ["default"], c[attr]
+ end
+ end
+ end
+>>>>>>> main
end
assert_includes(found, collections(:replication_desired_2_unconfirmed).uuid)
assert_includes(found, collections(:replication_desired_2_confirmed_2).uuid)
end
+
+ [
+ [1, "foo"],
+ [1, ["foo"]],
+ [1, ["bar"]],
+ [1, ["bar", "foo"]],
+ [0, ["foo", "qux"]],
+ [0, ["qux"]],
+ [nil, []],
+ [nil, [[]]],
+ [nil, [["bogus"]]],
+ [nil, [{"foo" => "bar"}]],
+ [nil, {"foo" => "bar"}],
+ ].each do |results, operand|
+ test "storage_classes_desired contains #{operand.inspect}" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with(:active)
+ c = Collection.create!(
+ manifest_text: "",
+ storage_classes_desired: ["foo", "bar", "baz"])
+ get :index, params: {
+ filters: [["storage_classes_desired", "contains", operand]],
+ }
+ if results.nil?
+ assert_response 422
+ next
+ end
+ assert_response :success
+ assert_equal results, json_response["items"].length
+ if results > 0
+ assert_equal c.uuid, json_response["items"][0]["uuid"]
+ end
+ end
+ end
+
+ test "collections properties contains top level key" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with(:active)
+ get :index, params: {
+ filters: [["properties", "contains", "prop1"]],
+ }
+ assert_response :success
+ assert_not_empty json_response["items"]
+ json_response["items"].each do |c|
+ assert c["properties"].has_key?("prop1")
+ end
+ end
end
test "create collection, update manifest, and search with filename" do
# create collection
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
search_using_filter 'my_test_file.txt', 1
# update the collection's manifest text
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_updated_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_updated_test_file.txt\n", api_token(:active))
put "/arvados/v1/collections/#{created['uuid']}",
params: {
format: :json,
test "create and get collection with properties" do
# create collection to be searched for
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
test "create collection and update it with json encoded hash properties" do
# create collection to be searched for
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
Rails.configuration.Collections.CollectionVersioning = true
Rails.configuration.Collections.PreserveVersionIfIdle = -1 # Disable auto versioning
- signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ signed_manifest = Collection.sign_manifest_only_for_tests(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
post "/arvados/v1/collections",
params: {
format: :json,
c = time_block 'read' do
Collection.find_by_uuid(c.uuid)
end
- time_block 'sign' do
- c.signed_manifest_text
- end
- time_block 'sign + render' do
+ time_block 'render' do
c.as_api_response(nil)
end
loc = Blob.sign_locator(Digest::MD5.hexdigest('foo') + '+3',
test "clear replication_confirmed* when introducing a new block in manifest" do
c = collections(:replication_desired_2_confirmed_2)
act_as_user users(:active) do
- assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text)
+ assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text_only_for_tests)
assert_nil c.replication_confirmed
assert_nil c.replication_confirmed_at
end
test "don't clear replication_confirmed* when just renaming a file" do
c = collections(:replication_desired_2_confirmed_2)
act_as_user users(:active) do
- new_manifest = c.signed_manifest_text.sub(':bar', ':foo')
+ new_manifest = c.signed_manifest_text_only_for_tests.sub(':bar', ':foo')
assert c.update_attributes(manifest_text: new_manifest)
assert_equal 2, c.replication_confirmed
assert_not_nil c.replication_confirmed_at
test "don't clear replication_confirmed* when just deleting a data block" do
c = collections(:replication_desired_2_confirmed_2)
act_as_user users(:active) do
- new_manifest = c.signed_manifest_text
+ new_manifest = c.signed_manifest_text_only_for_tests
new_manifest.sub!(/ \S+:bar/, '')
new_manifest.sub!(/ acbd\S+/, '')
# Confirm that we did just remove a block from the manifest (if
# not, this test would pass without testing the relevant case):
- assert_operator new_manifest.length+40, :<, c.signed_manifest_text.length
+ assert_operator new_manifest.length+40, :<, c.signed_manifest_text_only_for_tests.length
assert c.update_attributes(manifest_text: new_manifest)
assert_equal 2, c.replication_confirmed
c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n", name: 'foo')
c.update_attributes! trash_at: (t0 + 1.hours)
c.reload
- sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i
+ sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i
assert_operator sig_exp.to_i, :<=, (t0 + 1.hours).to_i
end
end
c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n",
name: 'foo',
trash_at: db_current_time + 1.years)
- sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i
+ sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i
expect_max_sig_exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i
assert_operator c.trash_at.to_i, :>, expect_max_sig_exp
assert_operator sig_exp.to_i, :<=, expect_max_sig_exp
cluster *arvados.Cluster
}
-func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
-}
-
-func (s *AuthHandlerSuite) TearDownSuite(c *check.C) {
- arvadostest.StopAPI()
-}
-
func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
repoRoot, err := filepath.Abs("../api/tmp/git/test")
cluster *arvados.Cluster
}
-func (s *IntegrationSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
-}
-
-func (s *IntegrationSuite) TearDownSuite(c *check.C) {
- arvadostest.StopAPI()
-}
-
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
func (s *TestSuite) SetUpSuite(c *C) {
initialArgs = os.Args
- arvadostest.StartAPI()
runningCmds = make(map[string]*exec.Cmd)
logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})
}
-func (s *TestSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func (s *TestSuite) SetUpTest(c *C) {
args := []string{"crunch-dispatch-local"}
os.Args = args
}
func (s *IntegrationSuite) SetUpTest(c *C) {
- arvadostest.StartAPI()
+ arvadostest.ResetEnv()
+ arvadostest.ResetDB(c)
os.Setenv("ARVADOS_API_TOKEN", arvadostest.Dispatch1Token)
s.disp = Dispatcher{}
s.disp.cluster = &arvados.Cluster{}
func (s *IntegrationSuite) TearDownTest(c *C) {
arvadostest.ResetEnv()
- arvadostest.StopAPI()
+ arvadostest.ResetDB(c)
}
type slurmFake struct {
c.Skip("-short")
}
arvadostest.ResetEnv()
- arvadostest.StartAPI()
arvadostest.StartKeep(4, true)
arv, err := arvadosclient.MakeArvadosClient()
c.Skip("-short")
}
arvadostest.StopKeep(4)
- arvadostest.StopAPI()
}
func (s *integrationSuite) SetUpTest(c *check.C) {
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
+ arvadostest.ResetDB(c)
arvadostest.StartKeep(2, true)
arv, err := arvadosclient.MakeArvadosClient()
func (s *IntegrationSuite) TearDownSuite(c *check.C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *IntegrationSuite) SetUpTest(c *check.C) {
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
// config.yml defines 4 keepstores
arvadostest.StartKeep(4, false)
}
func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(4)
- arvadostest.StopAPI()
}
func (s *NoKeepServerSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
// We need API to have some keep services listed, but the
// services themselves should be unresponsive.
arvadostest.StartKeep(2, false)
arvadostest.ResetEnv()
}
-func (s *NoKeepServerSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
c.Assert(err, Equals, nil)
}
pullRequest := s.setupPullWorkerIntegrationTest(c, testData, false)
- defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
s.performPullWorkerIntegrationTest(testData, pullRequest, c)
}
pullRequest := s.setupPullWorkerIntegrationTest(c, testData, true)
- defer arvadostest.StopAPI()
defer arvadostest.StopKeep(2)
s.performPullWorkerIntegrationTest(testData, pullRequest, c)
var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
arvadostest.ResetEnv()
}
type ServerRequiredSuite struct{}
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
arvadostest.ResetEnv()
}