lib/controller/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "context"
   9         "crypto/tls"
  10         "encoding/json"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "strings"
  17         "testing"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/sdk/go/arvados"
  21         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  22         "git.arvados.org/arvados.git/sdk/go/auth"
  23         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  24         "git.arvados.org/arvados.git/sdk/go/httpserver"
  25         "github.com/prometheus/client_golang/prometheus"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 func Test(t *testing.T) {
  31         check.TestingT(t)
  32 }
  33
  34 var _ = check.Suite(&HandlerSuite{})
  35
  36 type HandlerSuite struct {
  37         cluster *arvados.Cluster
  38         handler *Handler
  39         ctx     context.Context
  40         cancel  context.CancelFunc
  41 }
  42
  43 func (s *HandlerSuite) SetUpTest(c *check.C) {
  44         s.ctx, s.cancel = context.WithCancel(context.Background())
  45         s.ctx = ctxlog.Context(s.ctx, ctxlog.New(os.Stderr, "json", "debug"))
  46         s.cluster = &arvados.Cluster{
  47                 ClusterID:  "zzzzz",
  48                 PostgreSQL: integrationTestCluster().PostgreSQL,
  49         }
  50         s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
  51         s.cluster.TLS.Insecure = true
  52         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  53         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
  54         s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry()).(*Handler)
  55 }
  56
  57 func (s *HandlerSuite) TearDownTest(c *check.C) {
  58         s.cancel()
  59 }
  60
  61 func (s *HandlerSuite) TestConfigExport(c *check.C) {
  62         s.cluster.ManagementToken = "secret"
  63         s.cluster.SystemRootToken = "secret"
  64         s.cluster.Collections.BlobSigning = true
  65         s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
  66         for _, method := range []string{"GET", "OPTIONS"} {
  67                 req := httptest.NewRequest(method, "/arvados/v1/config", nil)
  68                 resp := httptest.NewRecorder()
  69                 s.handler.ServeHTTP(resp, req)
  70                 c.Log(resp.Body.String())
  71                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
  72                         continue
  73                 }
  74                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
  75                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
  76                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
  77                 if method == "OPTIONS" {
  78                         c.Check(resp.Body.String(), check.HasLen, 0)
  79                         continue
  80                 }
  81                 var cluster arvados.Cluster
  82                 err := json.Unmarshal(resp.Body.Bytes(), &cluster)
  83                 c.Check(err, check.IsNil)
  84                 c.Check(cluster.ManagementToken, check.Equals, "")
  85                 c.Check(cluster.SystemRootToken, check.Equals, "")
  86                 c.Check(cluster.Collections.BlobSigning, check.Equals, true)
  87                 c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
  88         }
  89 }
  90
  91 func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
  92         voc := `{
  93                 "strict_tags": false,
  94                 "tags": {
  95                         "IDTAGIMPORTANCE": {
  96                                 "strict": false,
  97                                 "labels": [{"label": "Importance"}],
  98                                 "values": {
  99                                         "HIGH": {
 100                                                 "labels": [{"label": "High"}]
 101                                         },
 102                                         "LOW": {
 103                                                 "labels": [{"label": "Low"}]
 104                                         }
 105                                 }
 106                         }
 107                 }
 108         }`
 109         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 110         c.Assert(err, check.IsNil)
 111         defer os.Remove(f.Name())
 112         _, err = f.WriteString(voc)
 113         c.Assert(err, check.IsNil)
 114         f.Close()
 115         s.cluster.API.VocabularyPath = f.Name()
 116         for _, method := range []string{"GET", "OPTIONS"} {
 117                 c.Log(c.TestName()+" ", method)
 118                 req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
 119                 resp := httptest.NewRecorder()
 120                 s.handler.ServeHTTP(resp, req)
 121                 c.Log(resp.Body.String())
 122                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 123                         continue
 124                 }
 125                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 126                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 127                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 128                 if method == "OPTIONS" {
 129                         c.Check(resp.Body.String(), check.HasLen, 0)
 130                         continue
 131                 }
 132                 var expectedVoc, receivedVoc *arvados.Vocabulary
 133                 err := json.Unmarshal([]byte(voc), &expectedVoc)
 134                 c.Check(err, check.IsNil)
 135                 err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
 136                 c.Check(err, check.IsNil)
 137                 c.Check(receivedVoc, check.DeepEquals, expectedVoc)
 138         }
 139 }
 140
 141 func (s *HandlerSuite) TestVocabularyFailedCheckStatus(c *check.C) {
 142         voc := `{
 143                 "strict_tags": false,
 144                 "tags": {
 145                         "IDTAGIMPORTANCE": {
 146                                 "strict": true,
 147                                 "labels": [{"label": "Importance"}],
 148                                 "values": {
 149                                         "HIGH": {
 150                                                 "labels": [{"label": "High"}]
 151                                         },
 152                                         "LOW": {
 153                                                 "labels": [{"label": "Low"}]
 154                                         }
 155                                 }
 156                         }
 157                 }
 158         }`
 159         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 160         c.Assert(err, check.IsNil)
 161         defer os.Remove(f.Name())
 162         _, err = f.WriteString(voc)
 163         c.Assert(err, check.IsNil)
 164         f.Close()
 165         s.cluster.API.VocabularyPath = f.Name()
 166
 167         req := httptest.NewRequest("POST", "/arvados/v1/collections",
 168                 strings.NewReader(`{
 169                         "collection": {
 170                                 "properties": {
 171                                         "IDTAGIMPORTANCE": "Critical"
 172                                 }
 173                         }
 174                 }`))
 175         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 176         req.Header.Set("Content-type", "application/json")
 177
 178         resp := httptest.NewRecorder()
 179         s.handler.ServeHTTP(resp, req)
 180         c.Log(resp.Body.String())
 181         c.Assert(resp.Code, check.Equals, http.StatusBadRequest)
 182         var jresp httpserver.ErrorResponse
 183         err = json.Unmarshal(resp.Body.Bytes(), &jresp)
 184         c.Check(err, check.IsNil)
 185         c.Assert(len(jresp.Errors), check.Equals, 1)
 186         c.Check(jresp.Errors[0], check.Matches, `.*tag value.*is not valid for key.*`)
 187 }
 188
 189 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 190         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 191         resp := httptest.NewRecorder()
 192         s.handler.ServeHTTP(resp, req)
 193         c.Check(resp.Code, check.Equals, http.StatusOK)
 194         var dd arvados.DiscoveryDocument
 195         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 196         c.Check(err, check.IsNil)
 197         c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
 198         c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
 199         c.Check(len(dd.Resources), check.Not(check.Equals), 0)
 200         c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
 201 }
 202
 203 func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
 204         s.cluster.API.RequestTimeout = arvados.Duration(time.Nanosecond)
 205         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 206         resp := httptest.NewRecorder()
 207         s.handler.ServeHTTP(resp, req)
 208         c.Check(resp.Code, check.Equals, http.StatusBadGateway)
 209         var jresp httpserver.ErrorResponse
 210         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 211         c.Check(err, check.IsNil)
 212         c.Assert(len(jresp.Errors), check.Equals, 1)
 213         c.Check(jresp.Errors[0], check.Matches, `.*context deadline exceeded.*`)
 214 }
 215
 216 func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
 217         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 218         resp := httptest.NewRecorder()
 219         s.handler.ServeHTTP(resp, req)
 220         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 221         jresp := map[string]interface{}{}
 222         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 223         c.Check(err, check.IsNil)
 224         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 225 }
 226
 227 func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
 228         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 229         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 230         resp := httptest.NewRecorder()
 231         s.handler.ServeHTTP(resp, req)
 232         c.Check(resp.Code, check.Equals, http.StatusOK)
 233         var u arvados.User
 234         err := json.Unmarshal(resp.Body.Bytes(), &u)
 235         c.Check(err, check.IsNil)
 236         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 237 }
 238
 239 func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
 240         req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
 241                 "_method":   {"GET"},
 242                 "api_token": {arvadostest.ActiveToken},
 243         }.Encode()))
 244         req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 245         resp := httptest.NewRecorder()
 246         s.handler.ServeHTTP(resp, req)
 247         c.Check(resp.Code, check.Equals, http.StatusOK)
 248         var u arvados.User
 249         err := json.Unmarshal(resp.Body.Bytes(), &u)
 250         c.Check(err, check.IsNil)
 251         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 252 }
 253
 254 func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
 255         req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
 256         resp := httptest.NewRecorder()
 257         s.handler.ServeHTTP(resp, req)
 258         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 259         jresp := map[string]interface{}{}
 260         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 261         c.Check(err, check.IsNil)
 262         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 263 }
 264
 265 func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
 266         s.cluster.Login.Google.Enable = true
 267         s.cluster.Login.Google.ClientID = "test"
 268         req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
 269         resp := httptest.NewRecorder()
 270         s.handler.ServeHTTP(resp, req)
 271         if !c.Check(resp.Code, check.Equals, http.StatusFound) {
 272                 c.Log(resp.Body.String())
 273         }
 274         c.Check(resp.Header().Get("Location"), check.Equals, "https://example.com/foo")
 275 }
 276
 277 func (s *HandlerSuite) TestValidateV1APIToken(c *check.C) {
 278         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 279         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveToken)
 280         c.Assert(err, check.IsNil)
 281         c.Check(ok, check.Equals, true)
 282         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 283         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 284         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 285         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 286 }
 287
 288 func (s *HandlerSuite) TestValidateV2APIToken(c *check.C) {
 289         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 290         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveTokenV2)
 291         c.Assert(err, check.IsNil)
 292         c.Check(ok, check.Equals, true)
 293         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 294         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 295         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 296         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 297         c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
 298 }
 299
 300 func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
 301         saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
 302         c.Assert(err, check.IsNil)
 303         for _, trial := range []struct {
 304                 code  int
 305                 token string
 306         }{
 307                 {http.StatusOK, saltedToken},
 308                 {http.StatusUnauthorized, "bogus"},
 309         } {
 310                 req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
 311                 req.Header.Set("Authorization", "Bearer "+trial.token)
 312                 resp := httptest.NewRecorder()
 313                 s.handler.ServeHTTP(resp, req)
 314                 if !c.Check(resp.Code, check.Equals, trial.code) {
 315                         c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
 316                 }
 317         }
 318 }
 319
 320 func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
 321         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 322         auth, err := s.handler.createAPItoken(req, arvadostest.ActiveUserUUID, nil)
 323         c.Assert(err, check.IsNil)
 324         c.Check(auth.Scopes, check.DeepEquals, []string{"all"})
 325
 326         user, ok, err := s.handler.validateAPItoken(req, auth.TokenV2())
 327         c.Assert(err, check.IsNil)
 328         c.Check(ok, check.Equals, true)
 329         c.Check(user.Authorization.UUID, check.Equals, auth.UUID)
 330         c.Check(user.Authorization.APIToken, check.Equals, auth.APIToken)
 331         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 332         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 333         c.Check(user.Authorization.TokenV2(), check.Equals, auth.TokenV2())
 334 }
 335
 336 func (s *HandlerSuite) CheckObjectType(c *check.C, url string, token string, skippedFields map[string]bool) {
 337         var proxied, direct map[string]interface{}
 338         var err error
 339
 340         // Get collection from controller
 341         req := httptest.NewRequest("GET", url, nil)
 342         req.Header.Set("Authorization", "Bearer "+token)
 343         resp := httptest.NewRecorder()
 344         s.handler.ServeHTTP(resp, req)
 345         c.Assert(resp.Code, check.Equals, http.StatusOK,
 346                 check.Commentf("Wasn't able to get data from the controller at %q: %q", url, resp.Body.String()))
 347         err = json.Unmarshal(resp.Body.Bytes(), &proxied)
 348         c.Check(err, check.Equals, nil)
 349
 350         // Get collection directly from RailsAPI
 351         client := &http.Client{
 352                 Transport: &http.Transport{
 353                         TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 354                 },
 355         }
 356         resp2, err := client.Get(s.cluster.Services.RailsAPI.ExternalURL.String() + url + "/?api_token=" + token)
 357         c.Check(err, check.Equals, nil)
 358         c.Assert(resp2.StatusCode, check.Equals, http.StatusOK,
 359                 check.Commentf("Wasn't able to get data from the RailsAPI at %q", url))
 360         defer resp2.Body.Close()
 361         db, err := ioutil.ReadAll(resp2.Body)
 362         c.Check(err, check.Equals, nil)
 363         err = json.Unmarshal(db, &direct)
 364         c.Check(err, check.Equals, nil)
 365
 366         // Check that all RailsAPI provided keys exist on the controller response.
 367         for k := range direct {
 368                 if _, ok := skippedFields[k]; ok {
 369                         continue
 370                 } else if val, ok := proxied[k]; ok {
 371                         if direct["kind"] == "arvados#collection" && k == "manifest_text" {
 372                                 // Tokens differ from request to request
 373                                 c.Check(strings.Split(val.(string), "+A")[0], check.Equals, strings.Split(direct[k].(string), "+A")[0])
 374                         } else {
 375                                 c.Check(val, check.DeepEquals, direct[k],
 376                                         check.Commentf("RailsAPI %s key %q's value %q differs from controller's %q.", direct["kind"], k, direct[k], val))
 377                         }
 378                 } else {
 379                         c.Errorf("%s's key %q missing on controller's response.", direct["kind"], k)
 380                 }
 381         }
 382 }
 383
 384 func (s *HandlerSuite) TestGetObjects(c *check.C) {
 385         // Get the 1st keep service's uuid from the running test server.
 386         req := httptest.NewRequest("GET", "/arvados/v1/keep_services/", nil)
 387         req.Header.Set("Authorization", "Bearer "+arvadostest.AdminToken)
 388         resp := httptest.NewRecorder()
 389         s.handler.ServeHTTP(resp, req)
 390         c.Assert(resp.Code, check.Equals, http.StatusOK)
 391         var ksList arvados.KeepServiceList
 392         json.Unmarshal(resp.Body.Bytes(), &ksList)
 393         c.Assert(len(ksList.Items), check.Not(check.Equals), 0)
 394         ksUUID := ksList.Items[0].UUID
 395         // Create a new token for the test user so that we're not comparing
 396         // the ones from the fixtures.
 397         req = httptest.NewRequest("POST", "/arvados/v1/api_client_authorizations",
 398                 strings.NewReader(`{
 399                         "api_client_authorization": {
 400                                 "owner_uuid": "`+arvadostest.AdminUserUUID+`",
 401                                 "created_by_ip_address": "::1",
 402                                 "last_used_by_ip_address": "::1",
 403                                 "default_owner_uuid": "`+arvadostest.AdminUserUUID+`"
 404                         }
 405                 }`))
 406         req.Header.Set("Authorization", "Bearer "+arvadostest.SystemRootToken)
 407         req.Header.Set("Content-type", "application/json")
 408         resp = httptest.NewRecorder()
 409         s.handler.ServeHTTP(resp, req)
 410         c.Assert(resp.Code, check.Equals, http.StatusOK,
 411                 check.Commentf("%s", resp.Body.String()))
 412         var auth arvados.APIClientAuthorization
 413         json.Unmarshal(resp.Body.Bytes(), &auth)
 414         c.Assert(auth.UUID, check.Not(check.Equals), "")
 415
 416         testCases := map[string]map[string]bool{
 417                 "api_clients/" + arvadostest.TrustedWorkbenchAPIClientUUID:     nil,
 418                 "api_client_authorizations/" + auth.UUID:                       {"href": true, "modified_by_client_uuid": true, "modified_by_user_uuid": true},
 419                 "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID:       nil,
 420                 "collections/" + arvadostest.CollectionWithUniqueWordsUUID:     {"href": true},
 421                 "containers/" + arvadostest.RunningContainerUUID:               nil,
 422                 "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil,
 423                 "groups/" + arvadostest.AProjectUUID:                           nil,
 424                 "keep_services/" + ksUUID:                                      nil,
 425                 "links/" + arvadostest.ActiveUserCanReadAllUsersLinkUUID:       nil,
 426                 "logs/" + arvadostest.CrunchstatForRunningJobLogUUID:           nil,
 427                 "nodes/" + arvadostest.IdleNodeUUID:                            nil,
 428                 "repositories/" + arvadostest.ArvadosRepoUUID:                  nil,
 429                 "users/" + arvadostest.ActiveUserUUID:                          {"href": true},
 430                 "virtual_machines/" + arvadostest.TestVMUUID:                   nil,
 431                 "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID:      nil,
 432         }
 433         for url, skippedFields := range testCases {
 434                 c.Logf("Testing %q", url)
 435                 s.CheckObjectType(c, "/arvados/v1/"+url, auth.TokenV2(), skippedFields)
 436         }
 437 }
 438
 439 func (s *HandlerSuite) TestRedactRailsAPIHostFromErrors(c *check.C) {
 440         req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/collections/zzzzz-4zz18-abcdefghijklmno", nil)
 441         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 442         resp := httptest.NewRecorder()
 443         s.handler.ServeHTTP(resp, req)
 444         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 445         var jresp struct {
 446                 Errors []string
 447         }
 448         c.Log(resp.Body.String())
 449         c.Assert(json.NewDecoder(resp.Body).Decode(&jresp), check.IsNil)
 450         c.Assert(jresp.Errors, check.HasLen, 1)
 451         c.Check(jresp.Errors[0], check.Matches, `.*//railsapi\.internal/arvados/v1/collections/.*: 404 Not Found.*`)
 452         c.Check(jresp.Errors[0], check.Not(check.Matches), `(?ms).*127.0.0.1.*`)
 453 }
 454
 455 func (s *HandlerSuite) TestTrashSweep(c *check.C) {
 456         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 457         s.cluster.Collections.TrashSweepInterval = arvados.Duration(time.Second / 10)
 458         s.handler.CheckHealth()
 459         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 460         coll, err := s.handler.federation.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{"name": "test trash sweep"}, EnsureUniqueName: true})
 461         c.Assert(err, check.IsNil)
 462         defer s.handler.federation.CollectionDelete(ctx, arvados.DeleteOptions{UUID: coll.UUID})
 463         db, err := s.handler.db(s.ctx)
 464         c.Assert(err, check.IsNil)
 465         _, err = db.ExecContext(s.ctx, `update collections set trash_at = $1, delete_at = $2 where uuid = $3`, time.Now().UTC().Add(time.Second/10), time.Now().UTC().Add(time.Hour), coll.UUID)
 466         c.Assert(err, check.IsNil)
 467         deadline := time.Now().Add(5 * time.Second)
 468         for {
 469                 if time.Now().After(deadline) {
 470                         c.Log("timed out")
 471                         c.FailNow()
 472                 }
 473                 updated, err := s.handler.federation.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID, IncludeTrash: true})
 474                 c.Assert(err, check.IsNil)
 475                 if updated.IsTrashed {
 476                         break
 477                 }
 478                 time.Sleep(time.Second / 10)
 479         }
 480 }