lib/controller/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/tls"
  11         "encoding/json"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "strings"
  19         "testing"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/auth"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/httpserver"
  27         "github.com/prometheus/client_golang/prometheus"
  28         check "gopkg.in/check.v1"
  29 )
  30
  31 // Gocheck boilerplate
  32 func Test(t *testing.T) {
  33         check.TestingT(t)
  34 }
  35
  36 var _ = check.Suite(&HandlerSuite{})
  37
  38 type HandlerSuite struct {
  39         cluster *arvados.Cluster
  40         handler *Handler
  41         logbuf  *bytes.Buffer
  42         ctx     context.Context
  43         cancel  context.CancelFunc
  44 }
  45
  46 func (s *HandlerSuite) SetUpTest(c *check.C) {
  47         s.logbuf = &bytes.Buffer{}
  48         s.ctx, s.cancel = context.WithCancel(context.Background())
  49         s.ctx = ctxlog.Context(s.ctx, ctxlog.New(io.MultiWriter(os.Stderr, s.logbuf), "json", "debug"))
  50         s.cluster = &arvados.Cluster{
  51                 ClusterID:  "zzzzz",
  52                 PostgreSQL: integrationTestCluster().PostgreSQL,
  53         }
  54         s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
  55         s.cluster.TLS.Insecure = true
  56         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  57         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
  58         s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry()).(*Handler)
  59 }
  60
  61 func (s *HandlerSuite) TearDownTest(c *check.C) {
  62         s.cancel()
  63 }
  64
  65 func (s *HandlerSuite) TestConfigExport(c *check.C) {
  66         s.cluster.ManagementToken = "secret"
  67         s.cluster.SystemRootToken = "secret"
  68         s.cluster.Collections.BlobSigning = true
  69         s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
  70         for _, method := range []string{"GET", "OPTIONS"} {
  71                 req := httptest.NewRequest(method, "/arvados/v1/config", nil)
  72                 resp := httptest.NewRecorder()
  73                 s.handler.ServeHTTP(resp, req)
  74                 c.Log(resp.Body.String())
  75                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
  76                         continue
  77                 }
  78                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
  79                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
  80                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
  81                 if method == "OPTIONS" {
  82                         c.Check(resp.Body.String(), check.HasLen, 0)
  83                         continue
  84                 }
  85                 var cluster arvados.Cluster
  86                 err := json.Unmarshal(resp.Body.Bytes(), &cluster)
  87                 c.Check(err, check.IsNil)
  88                 c.Check(cluster.ManagementToken, check.Equals, "")
  89                 c.Check(cluster.SystemRootToken, check.Equals, "")
  90                 c.Check(cluster.Collections.BlobSigning, check.Equals, true)
  91                 c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
  92         }
  93 }
  94
  95 func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
  96         voc := `{
  97                 "strict_tags": false,
  98                 "tags": {
  99                         "IDTAGIMPORTANCE": {
 100                                 "strict": false,
 101                                 "labels": [{"label": "Importance"}],
 102                                 "values": {
 103                                         "HIGH": {
 104                                                 "labels": [{"label": "High"}]
 105                                         },
 106                                         "LOW": {
 107                                                 "labels": [{"label": "Low"}]
 108                                         }
 109                                 }
 110                         }
 111                 }
 112         }`
 113         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 114         c.Assert(err, check.IsNil)
 115         defer os.Remove(f.Name())
 116         _, err = f.WriteString(voc)
 117         c.Assert(err, check.IsNil)
 118         f.Close()
 119         s.cluster.API.VocabularyPath = f.Name()
 120         for _, method := range []string{"GET", "OPTIONS"} {
 121                 c.Log(c.TestName()+" ", method)
 122                 req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
 123                 resp := httptest.NewRecorder()
 124                 s.handler.ServeHTTP(resp, req)
 125                 c.Log(resp.Body.String())
 126                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 127                         continue
 128                 }
 129                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 130                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 131                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 132                 if method == "OPTIONS" {
 133                         c.Check(resp.Body.String(), check.HasLen, 0)
 134                         continue
 135                 }
 136                 var expectedVoc, receivedVoc *arvados.Vocabulary
 137                 err := json.Unmarshal([]byte(voc), &expectedVoc)
 138                 c.Check(err, check.IsNil)
 139                 err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
 140                 c.Check(err, check.IsNil)
 141                 c.Check(receivedVoc, check.DeepEquals, expectedVoc)
 142         }
 143 }
 144
 145 func (s *HandlerSuite) TestVocabularyFailedCheckStatus(c *check.C) {
 146         voc := `{
 147                 "strict_tags": false,
 148                 "tags": {
 149                         "IDTAGIMPORTANCE": {
 150                                 "strict": true,
 151                                 "labels": [{"label": "Importance"}],
 152                                 "values": {
 153                                         "HIGH": {
 154                                                 "labels": [{"label": "High"}]
 155                                         },
 156                                         "LOW": {
 157                                                 "labels": [{"label": "Low"}]
 158                                         }
 159                                 }
 160                         }
 161                 }
 162         }`
 163         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 164         c.Assert(err, check.IsNil)
 165         defer os.Remove(f.Name())
 166         _, err = f.WriteString(voc)
 167         c.Assert(err, check.IsNil)
 168         f.Close()
 169         s.cluster.API.VocabularyPath = f.Name()
 170
 171         req := httptest.NewRequest("POST", "/arvados/v1/collections",
 172                 strings.NewReader(`{
 173                         "collection": {
 174                                 "properties": {
 175                                         "IDTAGIMPORTANCE": "Critical"
 176                                 }
 177                         }
 178                 }`))
 179         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 180         req.Header.Set("Content-type", "application/json")
 181
 182         resp := httptest.NewRecorder()
 183         s.handler.ServeHTTP(resp, req)
 184         c.Log(resp.Body.String())
 185         c.Assert(resp.Code, check.Equals, http.StatusBadRequest)
 186         var jresp httpserver.ErrorResponse
 187         err = json.Unmarshal(resp.Body.Bytes(), &jresp)
 188         c.Check(err, check.IsNil)
 189         c.Assert(len(jresp.Errors), check.Equals, 1)
 190         c.Check(jresp.Errors[0], check.Matches, `.*tag value.*is not valid for key.*`)
 191 }
 192
 193 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 194         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 195         resp := httptest.NewRecorder()
 196         s.handler.ServeHTTP(resp, req)
 197         c.Check(resp.Code, check.Equals, http.StatusOK)
 198         var dd arvados.DiscoveryDocument
 199         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 200         c.Check(err, check.IsNil)
 201         c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
 202         c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
 203         c.Check(len(dd.Resources), check.Not(check.Equals), 0)
 204         c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
 205 }
 206
 207 func (s *HandlerSuite) TestRequestTimeout(c *check.C) {
 208         s.cluster.API.RequestTimeout = arvados.Duration(time.Nanosecond)
 209         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 210         resp := httptest.NewRecorder()
 211         s.handler.ServeHTTP(resp, req)
 212         c.Check(resp.Code, check.Equals, http.StatusBadGateway)
 213         var jresp httpserver.ErrorResponse
 214         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 215         c.Check(err, check.IsNil)
 216         c.Assert(len(jresp.Errors), check.Equals, 1)
 217         c.Check(jresp.Errors[0], check.Matches, `.*context deadline exceeded.*`)
 218 }
 219
 220 func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
 221         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 222         resp := httptest.NewRecorder()
 223         s.handler.ServeHTTP(resp, req)
 224         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 225         jresp := map[string]interface{}{}
 226         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 227         c.Check(err, check.IsNil)
 228         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 229 }
 230
 231 func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
 232         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 233         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 234         resp := httptest.NewRecorder()
 235         s.handler.ServeHTTP(resp, req)
 236         c.Check(resp.Code, check.Equals, http.StatusOK)
 237         var u arvados.User
 238         err := json.Unmarshal(resp.Body.Bytes(), &u)
 239         c.Check(err, check.IsNil)
 240         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 241 }
 242
 243 func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
 244         req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
 245                 "_method":   {"GET"},
 246                 "api_token": {arvadostest.ActiveToken},
 247         }.Encode()))
 248         req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 249         resp := httptest.NewRecorder()
 250         s.handler.ServeHTTP(resp, req)
 251         c.Check(resp.Code, check.Equals, http.StatusOK)
 252         var u arvados.User
 253         err := json.Unmarshal(resp.Body.Bytes(), &u)
 254         c.Check(err, check.IsNil)
 255         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 256 }
 257
 258 func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
 259         req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
 260         resp := httptest.NewRecorder()
 261         s.handler.ServeHTTP(resp, req)
 262         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 263         jresp := map[string]interface{}{}
 264         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 265         c.Check(err, check.IsNil)
 266         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 267 }
 268
 269 func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
 270         s.cluster.Login.Google.Enable = true
 271         s.cluster.Login.Google.ClientID = "test"
 272         req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
 273         resp := httptest.NewRecorder()
 274         s.handler.ServeHTTP(resp, req)
 275         if !c.Check(resp.Code, check.Equals, http.StatusFound) {
 276                 c.Log(resp.Body.String())
 277         }
 278         c.Check(resp.Header().Get("Location"), check.Equals, "https://example.com/foo")
 279 }
 280
 281 func (s *HandlerSuite) TestValidateV1APIToken(c *check.C) {
 282         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 283         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveToken)
 284         c.Assert(err, check.IsNil)
 285         c.Check(ok, check.Equals, true)
 286         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 287         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 288         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 289         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 290 }
 291
 292 func (s *HandlerSuite) TestValidateV2APIToken(c *check.C) {
 293         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 294         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveTokenV2)
 295         c.Assert(err, check.IsNil)
 296         c.Check(ok, check.Equals, true)
 297         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 298         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 299         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 300         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 301         c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
 302 }
 303
 304 func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
 305         saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
 306         c.Assert(err, check.IsNil)
 307         for _, trial := range []struct {
 308                 code  int
 309                 token string
 310         }{
 311                 {http.StatusOK, saltedToken},
 312                 {http.StatusUnauthorized, "bogus"},
 313         } {
 314                 req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
 315                 req.Header.Set("Authorization", "Bearer "+trial.token)
 316                 resp := httptest.NewRecorder()
 317                 s.handler.ServeHTTP(resp, req)
 318                 if !c.Check(resp.Code, check.Equals, trial.code) {
 319                         c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
 320                 }
 321         }
 322 }
 323
 324 func (s *HandlerSuite) TestLogTokenUUID(c *check.C) {
 325         req := httptest.NewRequest("GET", "https://0.0.0.0/arvados/v1/users/current", nil)
 326         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 327         req = req.WithContext(s.ctx)
 328         resp := httptest.NewRecorder()
 329         httpserver.LogRequests(s.handler).ServeHTTP(resp, req)
 330         c.Check(resp.Code, check.Equals, http.StatusOK)
 331         c.Check(s.logbuf.String(), check.Matches, `(?ms).*"tokenUUIDs":\["`+strings.Split(arvadostest.ActiveTokenV2, "/")[1]+`"\].*`)
 332 }
 333
 334 func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
 335         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 336         auth, err := s.handler.createAPItoken(req, arvadostest.ActiveUserUUID, nil)
 337         c.Assert(err, check.IsNil)
 338         c.Check(auth.Scopes, check.DeepEquals, []string{"all"})
 339
 340         user, ok, err := s.handler.validateAPItoken(req, auth.TokenV2())
 341         c.Assert(err, check.IsNil)
 342         c.Check(ok, check.Equals, true)
 343         c.Check(user.Authorization.UUID, check.Equals, auth.UUID)
 344         c.Check(user.Authorization.APIToken, check.Equals, auth.APIToken)
 345         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 346         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 347         c.Check(user.Authorization.TokenV2(), check.Equals, auth.TokenV2())
 348 }
 349
 350 func (s *HandlerSuite) CheckObjectType(c *check.C, url string, token string, skippedFields map[string]bool) {
 351         var proxied, direct map[string]interface{}
 352         var err error
 353
 354         // Get collection from controller
 355         req := httptest.NewRequest("GET", url, nil)
 356         req.Header.Set("Authorization", "Bearer "+token)
 357         resp := httptest.NewRecorder()
 358         s.handler.ServeHTTP(resp, req)
 359         c.Assert(resp.Code, check.Equals, http.StatusOK,
 360                 check.Commentf("Wasn't able to get data from the controller at %q: %q", url, resp.Body.String()))
 361         err = json.Unmarshal(resp.Body.Bytes(), &proxied)
 362         c.Check(err, check.Equals, nil)
 363
 364         // Get collection directly from RailsAPI
 365         client := &http.Client{
 366                 Transport: &http.Transport{
 367                         TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 368                 },
 369         }
 370         resp2, err := client.Get(s.cluster.Services.RailsAPI.ExternalURL.String() + url + "/?api_token=" + token)
 371         c.Check(err, check.Equals, nil)
 372         c.Assert(resp2.StatusCode, check.Equals, http.StatusOK,
 373                 check.Commentf("Wasn't able to get data from the RailsAPI at %q", url))
 374         defer resp2.Body.Close()
 375         db, err := ioutil.ReadAll(resp2.Body)
 376         c.Check(err, check.Equals, nil)
 377         err = json.Unmarshal(db, &direct)
 378         c.Check(err, check.Equals, nil)
 379
 380         // Check that all RailsAPI provided keys exist on the controller response.
 381         for k := range direct {
 382                 if _, ok := skippedFields[k]; ok {
 383                         continue
 384                 } else if val, ok := proxied[k]; !ok {
 385                         c.Errorf("%s's key %q missing on controller's response.", direct["kind"], k)
 386                 } else if direct["kind"] == "arvados#collection" && k == "manifest_text" {
 387                         // Tokens differ from request to request
 388                         c.Check(strings.Split(val.(string), "+A")[0], check.Equals, strings.Split(direct[k].(string), "+A")[0])
 389                 } else {
 390                         c.Check(val, check.DeepEquals, direct[k],
 391                                 check.Commentf("RailsAPI %s key %q's value %q differs from controller's %q.", direct["kind"], k, direct[k], val))
 392                 }
 393         }
 394 }
 395
 396 func (s *HandlerSuite) TestGetObjects(c *check.C) {
 397         // Get the 1st keep service's uuid from the running test server.
 398         req := httptest.NewRequest("GET", "/arvados/v1/keep_services/", nil)
 399         req.Header.Set("Authorization", "Bearer "+arvadostest.AdminToken)
 400         resp := httptest.NewRecorder()
 401         s.handler.ServeHTTP(resp, req)
 402         c.Assert(resp.Code, check.Equals, http.StatusOK)
 403         var ksList arvados.KeepServiceList
 404         json.Unmarshal(resp.Body.Bytes(), &ksList)
 405         c.Assert(len(ksList.Items), check.Not(check.Equals), 0)
 406         ksUUID := ksList.Items[0].UUID
 407         // Create a new token for the test user so that we're not comparing
 408         // the ones from the fixtures.
 409         req = httptest.NewRequest("POST", "/arvados/v1/api_client_authorizations",
 410                 strings.NewReader(`{
 411                         "api_client_authorization": {
 412                                 "owner_uuid": "`+arvadostest.AdminUserUUID+`",
 413                                 "created_by_ip_address": "::1",
 414                                 "last_used_by_ip_address": "::1",
 415                                 "default_owner_uuid": "`+arvadostest.AdminUserUUID+`"
 416                         }
 417                 }`))
 418         req.Header.Set("Authorization", "Bearer "+arvadostest.SystemRootToken)
 419         req.Header.Set("Content-type", "application/json")
 420         resp = httptest.NewRecorder()
 421         s.handler.ServeHTTP(resp, req)
 422         c.Assert(resp.Code, check.Equals, http.StatusOK,
 423                 check.Commentf("%s", resp.Body.String()))
 424         var auth arvados.APIClientAuthorization
 425         json.Unmarshal(resp.Body.Bytes(), &auth)
 426         c.Assert(auth.UUID, check.Not(check.Equals), "")
 427
 428         testCases := map[string]map[string]bool{
 429                 "api_clients/" + arvadostest.TrustedWorkbenchAPIClientUUID:     nil,
 430                 "api_client_authorizations/" + auth.UUID:                       {"href": true, "modified_by_client_uuid": true, "modified_by_user_uuid": true},
 431                 "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID:       nil,
 432                 "collections/" + arvadostest.CollectionWithUniqueWordsUUID:     {"href": true},
 433                 "containers/" + arvadostest.RunningContainerUUID:               nil,
 434                 "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil,
 435                 "groups/" + arvadostest.AProjectUUID:                           nil,
 436                 "keep_services/" + ksUUID:                                      nil,
 437                 "links/" + arvadostest.ActiveUserCanReadAllUsersLinkUUID:       nil,
 438                 "logs/" + arvadostest.CrunchstatForRunningJobLogUUID:           nil,
 439                 "nodes/" + arvadostest.IdleNodeUUID:                            nil,
 440                 "repositories/" + arvadostest.ArvadosRepoUUID:                  nil,
 441                 "users/" + arvadostest.ActiveUserUUID:                          {"href": true},
 442                 "virtual_machines/" + arvadostest.TestVMUUID:                   nil,
 443                 "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID:      nil,
 444         }
 445         for url, skippedFields := range testCases {
 446                 c.Logf("Testing %q", url)
 447                 s.CheckObjectType(c, "/arvados/v1/"+url, auth.TokenV2(), skippedFields)
 448         }
 449 }
 450
 451 func (s *HandlerSuite) TestRedactRailsAPIHostFromErrors(c *check.C) {
 452         req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/collections/zzzzz-4zz18-abcdefghijklmno", nil)
 453         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 454         resp := httptest.NewRecorder()
 455         s.handler.ServeHTTP(resp, req)
 456         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 457         var jresp struct {
 458                 Errors []string
 459         }
 460         c.Log(resp.Body.String())
 461         c.Assert(json.NewDecoder(resp.Body).Decode(&jresp), check.IsNil)
 462         c.Assert(jresp.Errors, check.HasLen, 1)
 463         c.Check(jresp.Errors[0], check.Matches, `.*//railsapi\.internal/arvados/v1/collections/.*: 404 Not Found.*`)
 464         c.Check(jresp.Errors[0], check.Not(check.Matches), `(?ms).*127.0.0.1.*`)
 465 }
 466
 467 func (s *HandlerSuite) TestTrashSweep(c *check.C) {
 468         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 469         s.cluster.Collections.TrashSweepInterval = arvados.Duration(time.Second / 10)
 470         s.handler.CheckHealth()
 471         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 472         coll, err := s.handler.federation.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{"name": "test trash sweep"}, EnsureUniqueName: true})
 473         c.Assert(err, check.IsNil)
 474         defer s.handler.federation.CollectionDelete(ctx, arvados.DeleteOptions{UUID: coll.UUID})
 475         db, err := s.handler.db(s.ctx)
 476         c.Assert(err, check.IsNil)
 477         _, err = db.ExecContext(s.ctx, `update collections set trash_at = $1, delete_at = $2 where uuid = $3`, time.Now().UTC().Add(time.Second/10), time.Now().UTC().Add(time.Hour), coll.UUID)
 478         c.Assert(err, check.IsNil)
 479         deadline := time.Now().Add(5 * time.Second)
 480         for {
 481                 if time.Now().After(deadline) {
 482                         c.Log("timed out")
 483                         c.FailNow()
 484                 }
 485                 updated, err := s.handler.federation.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID, IncludeTrash: true})
 486                 c.Assert(err, check.IsNil)
 487                 if updated.IsTrashed {
 488                         break
 489                 }
 490                 time.Sleep(time.Second / 10)
 491         }
 492 }