lib/controller/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/tls"
  11         "encoding/json"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "strings"
  19         "testing"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/lib/controller/rpc"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/auth"
  26         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  27         "git.arvados.org/arvados.git/sdk/go/httpserver"
  28         "github.com/prometheus/client_golang/prometheus"
  29         check "gopkg.in/check.v1"
  30 )
  31
  32 // Gocheck boilerplate
  33 func Test(t *testing.T) {
  34         check.TestingT(t)
  35 }
  36
  37 var _ = check.Suite(&HandlerSuite{})
  38
  39 type HandlerSuite struct {
  40         cluster *arvados.Cluster
  41         handler *Handler
  42         logbuf  *bytes.Buffer
  43         ctx     context.Context
  44         cancel  context.CancelFunc
  45 }
  46
  47 func (s *HandlerSuite) SetUpTest(c *check.C) {
  48         s.logbuf = &bytes.Buffer{}
  49         s.ctx, s.cancel = context.WithCancel(context.Background())
  50         s.ctx = ctxlog.Context(s.ctx, ctxlog.New(io.MultiWriter(os.Stderr, s.logbuf), "json", "debug"))
  51         s.cluster = &arvados.Cluster{
  52                 ClusterID:  "zzzzz",
  53                 PostgreSQL: integrationTestCluster().PostgreSQL,
  54         }
  55         s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
  56         s.cluster.TLS.Insecure = true
  57         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  58         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
  59         s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry()).(*Handler)
  60 }
  61
  62 func (s *HandlerSuite) TearDownTest(c *check.C) {
  63         s.cancel()
  64 }
  65
  66 func (s *HandlerSuite) TestConfigExport(c *check.C) {
  67         s.cluster.ManagementToken = "secret"
  68         s.cluster.SystemRootToken = "secret"
  69         s.cluster.Collections.BlobSigning = true
  70         s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
  71         for _, method := range []string{"GET", "OPTIONS"} {
  72                 req := httptest.NewRequest(method, "/arvados/v1/config", nil)
  73                 resp := httptest.NewRecorder()
  74                 s.handler.ServeHTTP(resp, req)
  75                 c.Log(resp.Body.String())
  76                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
  77                         continue
  78                 }
  79                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
  80                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
  81                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
  82                 if method == "OPTIONS" {
  83                         c.Check(resp.Body.String(), check.HasLen, 0)
  84                         continue
  85                 }
  86                 var cluster arvados.Cluster
  87                 err := json.Unmarshal(resp.Body.Bytes(), &cluster)
  88                 c.Check(err, check.IsNil)
  89                 c.Check(cluster.ManagementToken, check.Equals, "")
  90                 c.Check(cluster.SystemRootToken, check.Equals, "")
  91                 c.Check(cluster.Collections.BlobSigning, check.Equals, true)
  92                 c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
  93         }
  94 }
  95
  96 func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
  97         voc := `{
  98                 "strict_tags": false,
  99                 "tags": {
 100                         "IDTAGIMPORTANCE": {
 101                                 "strict": false,
 102                                 "labels": [{"label": "Importance"}],
 103                                 "values": {
 104                                         "HIGH": {
 105                                                 "labels": [{"label": "High"}]
 106                                         },
 107                                         "LOW": {
 108                                                 "labels": [{"label": "Low"}]
 109                                         }
 110                                 }
 111                         }
 112                 }
 113         }`
 114         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 115         c.Assert(err, check.IsNil)
 116         defer os.Remove(f.Name())
 117         _, err = f.WriteString(voc)
 118         c.Assert(err, check.IsNil)
 119         f.Close()
 120         s.cluster.API.VocabularyPath = f.Name()
 121         for _, method := range []string{"GET", "OPTIONS"} {
 122                 c.Log(c.TestName()+" ", method)
 123                 req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
 124                 resp := httptest.NewRecorder()
 125                 s.handler.ServeHTTP(resp, req)
 126                 c.Log(resp.Body.String())
 127                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 128                         continue
 129                 }
 130                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 131                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 132                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 133                 if method == "OPTIONS" {
 134                         c.Check(resp.Body.String(), check.HasLen, 0)
 135                         continue
 136                 }
 137                 var expectedVoc, receivedVoc *arvados.Vocabulary
 138                 err := json.Unmarshal([]byte(voc), &expectedVoc)
 139                 c.Check(err, check.IsNil)
 140                 err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
 141                 c.Check(err, check.IsNil)
 142                 c.Check(receivedVoc, check.DeepEquals, expectedVoc)
 143         }
 144 }
 145
 146 func (s *HandlerSuite) TestVocabularyFailedCheckStatus(c *check.C) {
 147         voc := `{
 148                 "strict_tags": false,
 149                 "tags": {
 150                         "IDTAGIMPORTANCE": {
 151                                 "strict": true,
 152                                 "labels": [{"label": "Importance"}],
 153                                 "values": {
 154                                         "HIGH": {
 155                                                 "labels": [{"label": "High"}]
 156                                         },
 157                                         "LOW": {
 158                                                 "labels": [{"label": "Low"}]
 159                                         }
 160                                 }
 161                         }
 162                 }
 163         }`
 164         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 165         c.Assert(err, check.IsNil)
 166         defer os.Remove(f.Name())
 167         _, err = f.WriteString(voc)
 168         c.Assert(err, check.IsNil)
 169         f.Close()
 170         s.cluster.API.VocabularyPath = f.Name()
 171
 172         req := httptest.NewRequest("POST", "/arvados/v1/collections",
 173                 strings.NewReader(`{
 174                         "collection": {
 175                                 "properties": {
 176                                         "IDTAGIMPORTANCE": "Critical"
 177                                 }
 178                         }
 179                 }`))
 180         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 181         req.Header.Set("Content-type", "application/json")
 182
 183         resp := httptest.NewRecorder()
 184         s.handler.ServeHTTP(resp, req)
 185         c.Log(resp.Body.String())
 186         c.Assert(resp.Code, check.Equals, http.StatusBadRequest)
 187         var jresp httpserver.ErrorResponse
 188         err = json.Unmarshal(resp.Body.Bytes(), &jresp)
 189         c.Check(err, check.IsNil)
 190         c.Assert(len(jresp.Errors), check.Equals, 1)
 191         c.Check(jresp.Errors[0], check.Matches, `.*tag value.*is not valid for key.*`)
 192 }
 193
 194 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 195         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 196         resp := httptest.NewRecorder()
 197         s.handler.ServeHTTP(resp, req)
 198         c.Check(resp.Code, check.Equals, http.StatusOK)
 199         var dd arvados.DiscoveryDocument
 200         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 201         c.Check(err, check.IsNil)
 202         c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
 203         c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
 204         c.Check(len(dd.Resources), check.Not(check.Equals), 0)
 205         c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
 206 }
 207
 208 // Handler should give up and exit early if request context is
 209 // cancelled due to client hangup, httpserver.HandlerWithDeadline,
 210 // etc.
 211 func (s *HandlerSuite) TestRequestCancel(c *check.C) {
 212         ctx, cancel := context.WithCancel(context.Background())
 213         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil).WithContext(ctx)
 214         resp := httptest.NewRecorder()
 215         cancel()
 216         s.handler.ServeHTTP(resp, req)
 217         c.Check(resp.Code, check.Equals, http.StatusBadGateway)
 218         var jresp httpserver.ErrorResponse
 219         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 220         c.Check(err, check.IsNil)
 221         c.Assert(len(jresp.Errors), check.Equals, 1)
 222         c.Check(jresp.Errors[0], check.Matches, `.*context canceled`)
 223 }
 224
 225 func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
 226         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 227         resp := httptest.NewRecorder()
 228         s.handler.ServeHTTP(resp, req)
 229         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 230         jresp := map[string]interface{}{}
 231         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 232         c.Check(err, check.IsNil)
 233         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 234 }
 235
 236 func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
 237         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 238         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 239         resp := httptest.NewRecorder()
 240         s.handler.ServeHTTP(resp, req)
 241         c.Check(resp.Code, check.Equals, http.StatusOK)
 242         var u arvados.User
 243         err := json.Unmarshal(resp.Body.Bytes(), &u)
 244         c.Check(err, check.IsNil)
 245         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 246 }
 247
 248 func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
 249         req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
 250                 "_method":   {"GET"},
 251                 "api_token": {arvadostest.ActiveToken},
 252         }.Encode()))
 253         req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 254         resp := httptest.NewRecorder()
 255         s.handler.ServeHTTP(resp, req)
 256         c.Check(resp.Code, check.Equals, http.StatusOK)
 257         var u arvados.User
 258         err := json.Unmarshal(resp.Body.Bytes(), &u)
 259         c.Check(err, check.IsNil)
 260         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 261 }
 262
 263 func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
 264         req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
 265         resp := httptest.NewRecorder()
 266         s.handler.ServeHTTP(resp, req)
 267         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 268         jresp := map[string]interface{}{}
 269         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 270         c.Check(err, check.IsNil)
 271         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 272 }
 273
 274 func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
 275         s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "wb2.example", Path: "/"}
 276         s.cluster.Login.Google.Enable = true
 277         s.cluster.Login.Google.ClientID = "test"
 278         req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://wb2.example/", nil)
 279         resp := httptest.NewRecorder()
 280         s.handler.ServeHTTP(resp, req)
 281         if !c.Check(resp.Code, check.Equals, http.StatusFound) {
 282                 c.Log(resp.Body.String())
 283         }
 284         c.Check(resp.Header().Get("Location"), check.Equals, "https://wb2.example/")
 285 }
 286
 287 func (s *HandlerSuite) TestValidateV1APIToken(c *check.C) {
 288         c.Assert(s.handler.CheckHealth(), check.IsNil)
 289         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 290         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveToken)
 291         c.Assert(err, check.IsNil)
 292         c.Check(ok, check.Equals, true)
 293         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 294         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 295         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 296         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 297 }
 298
 299 func (s *HandlerSuite) TestValidateV2APIToken(c *check.C) {
 300         c.Assert(s.handler.CheckHealth(), check.IsNil)
 301         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 302         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveTokenV2)
 303         c.Assert(err, check.IsNil)
 304         c.Check(ok, check.Equals, true)
 305         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 306         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 307         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 308         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 309         c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
 310 }
 311
 312 func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
 313         saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
 314         c.Assert(err, check.IsNil)
 315         for _, trial := range []struct {
 316                 code  int
 317                 token string
 318         }{
 319                 {http.StatusOK, saltedToken},
 320                 {http.StatusUnauthorized, "bogus"},
 321         } {
 322                 req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
 323                 req.Header.Set("Authorization", "Bearer "+trial.token)
 324                 resp := httptest.NewRecorder()
 325                 s.handler.ServeHTTP(resp, req)
 326                 if !c.Check(resp.Code, check.Equals, trial.code) {
 327                         c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
 328                 }
 329         }
 330 }
 331
 332 func (s *HandlerSuite) TestLogTokenUUID(c *check.C) {
 333         req := httptest.NewRequest("GET", "https://0.0.0.0/arvados/v1/users/current", nil)
 334         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 335         req = req.WithContext(s.ctx)
 336         resp := httptest.NewRecorder()
 337         httpserver.LogRequests(s.handler).ServeHTTP(resp, req)
 338         c.Check(resp.Code, check.Equals, http.StatusOK)
 339         c.Check(s.logbuf.String(), check.Matches, `(?ms).*"tokenUUIDs":\["`+strings.Split(arvadostest.ActiveTokenV2, "/")[1]+`"\].*`)
 340 }
 341
 342 func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
 343         c.Assert(s.handler.CheckHealth(), check.IsNil)
 344         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 345         auth, err := s.handler.createAPItoken(req, arvadostest.ActiveUserUUID, nil)
 346         c.Assert(err, check.IsNil)
 347         c.Check(auth.Scopes, check.DeepEquals, []string{"all"})
 348
 349         user, ok, err := s.handler.validateAPItoken(req, auth.TokenV2())
 350         c.Assert(err, check.IsNil)
 351         c.Check(ok, check.Equals, true)
 352         c.Check(user.Authorization.UUID, check.Equals, auth.UUID)
 353         c.Check(user.Authorization.APIToken, check.Equals, auth.APIToken)
 354         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 355         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 356         c.Check(user.Authorization.TokenV2(), check.Equals, auth.TokenV2())
 357 }
 358
 359 func (s *HandlerSuite) CheckObjectType(c *check.C, url string, token string, skippedFields map[string]bool) {
 360         var proxied, direct map[string]interface{}
 361         var err error
 362
 363         // Get collection from controller
 364         req := httptest.NewRequest("GET", url, nil)
 365         req.Header.Set("Authorization", "Bearer "+token)
 366         resp := httptest.NewRecorder()
 367         s.handler.ServeHTTP(resp, req)
 368         c.Assert(resp.Code, check.Equals, http.StatusOK,
 369                 check.Commentf("Wasn't able to get data from the controller at %q: %q", url, resp.Body.String()))
 370         err = json.Unmarshal(resp.Body.Bytes(), &proxied)
 371         c.Check(err, check.Equals, nil)
 372
 373         // Get collection directly from RailsAPI
 374         client := &http.Client{
 375                 Transport: &http.Transport{
 376                         TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 377                 },
 378         }
 379         resp2, err := client.Get(s.cluster.Services.RailsAPI.ExternalURL.String() + url + "/?api_token=" + token)
 380         c.Check(err, check.Equals, nil)
 381         c.Assert(resp2.StatusCode, check.Equals, http.StatusOK,
 382                 check.Commentf("Wasn't able to get data from the RailsAPI at %q", url))
 383         defer resp2.Body.Close()
 384         db, err := ioutil.ReadAll(resp2.Body)
 385         c.Check(err, check.Equals, nil)
 386         err = json.Unmarshal(db, &direct)
 387         c.Check(err, check.Equals, nil)
 388
 389         // Check that all RailsAPI provided keys exist on the controller response.
 390         for k := range direct {
 391                 if _, ok := skippedFields[k]; ok {
 392                         continue
 393                 } else if val, ok := proxied[k]; !ok {
 394                         c.Errorf("%s's key %q missing on controller's response.", direct["kind"], k)
 395                 } else if direct["kind"] == "arvados#collection" && k == "manifest_text" {
 396                         // Tokens differ from request to request
 397                         c.Check(strings.Split(val.(string), "+A")[0], check.Equals, strings.Split(direct[k].(string), "+A")[0])
 398                 } else {
 399                         c.Check(val, check.DeepEquals, direct[k],
 400                                 check.Commentf("RailsAPI %s key %q's value %q differs from controller's %q.", direct["kind"], k, direct[k], val))
 401                 }
 402         }
 403 }
 404
 405 func (s *HandlerSuite) TestGetObjects(c *check.C) {
 406         // Get the 1st keep service's uuid from the running test server.
 407         req := httptest.NewRequest("GET", "/arvados/v1/keep_services/", nil)
 408         req.Header.Set("Authorization", "Bearer "+arvadostest.AdminToken)
 409         resp := httptest.NewRecorder()
 410         s.handler.ServeHTTP(resp, req)
 411         c.Assert(resp.Code, check.Equals, http.StatusOK)
 412         var ksList arvados.KeepServiceList
 413         json.Unmarshal(resp.Body.Bytes(), &ksList)
 414         c.Assert(len(ksList.Items), check.Not(check.Equals), 0)
 415         ksUUID := ksList.Items[0].UUID
 416         // Create a new token for the test user so that we're not comparing
 417         // the ones from the fixtures.
 418         req = httptest.NewRequest("POST", "/arvados/v1/api_client_authorizations",
 419                 strings.NewReader(`{
 420                         "api_client_authorization": {
 421                                 "owner_uuid": "`+arvadostest.AdminUserUUID+`",
 422                                 "created_by_ip_address": "::1",
 423                                 "last_used_by_ip_address": "::1",
 424                                 "default_owner_uuid": "`+arvadostest.AdminUserUUID+`"
 425                         }
 426                 }`))
 427         req.Header.Set("Authorization", "Bearer "+arvadostest.SystemRootToken)
 428         req.Header.Set("Content-type", "application/json")
 429         resp = httptest.NewRecorder()
 430         s.handler.ServeHTTP(resp, req)
 431         c.Assert(resp.Code, check.Equals, http.StatusOK,
 432                 check.Commentf("%s", resp.Body.String()))
 433         var auth arvados.APIClientAuthorization
 434         json.Unmarshal(resp.Body.Bytes(), &auth)
 435         c.Assert(auth.UUID, check.Not(check.Equals), "")
 436
 437         testCases := map[string]map[string]bool{
 438                 "api_clients/" + arvadostest.TrustedWorkbenchAPIClientUUID:     nil,
 439                 "api_client_authorizations/" + auth.UUID:                       {"href": true, "modified_by_client_uuid": true, "modified_by_user_uuid": true},
 440                 "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID:       nil,
 441                 "collections/" + arvadostest.CollectionWithUniqueWordsUUID:     {"href": true},
 442                 "containers/" + arvadostest.RunningContainerUUID:               nil,
 443                 "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil,
 444                 "groups/" + arvadostest.AProjectUUID:                           nil,
 445                 "keep_services/" + ksUUID:                                      nil,
 446                 "links/" + arvadostest.ActiveUserCanReadAllUsersLinkUUID:       nil,
 447                 "logs/" + arvadostest.CrunchstatForRunningJobLogUUID:           nil,
 448                 "nodes/" + arvadostest.IdleNodeUUID:                            nil,
 449                 "repositories/" + arvadostest.ArvadosRepoUUID:                  nil,
 450                 "users/" + arvadostest.ActiveUserUUID:                          {"href": true},
 451                 "virtual_machines/" + arvadostest.TestVMUUID:                   nil,
 452                 "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID:      nil,
 453         }
 454         for url, skippedFields := range testCases {
 455                 c.Logf("Testing %q", url)
 456                 s.CheckObjectType(c, "/arvados/v1/"+url, auth.TokenV2(), skippedFields)
 457         }
 458 }
 459
 460 func (s *HandlerSuite) TestRedactRailsAPIHostFromErrors(c *check.C) {
 461         req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/collections/zzzzz-4zz18-abcdefghijklmno", nil)
 462         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 463         resp := httptest.NewRecorder()
 464         s.handler.ServeHTTP(resp, req)
 465         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 466         var jresp struct {
 467                 Errors []string
 468         }
 469         c.Log(resp.Body.String())
 470         c.Assert(json.NewDecoder(resp.Body).Decode(&jresp), check.IsNil)
 471         c.Assert(jresp.Errors, check.HasLen, 1)
 472         c.Check(jresp.Errors[0], check.Matches, `.*//railsapi\.internal/arvados/v1/collections/.*: 404 Not Found.*`)
 473         c.Check(jresp.Errors[0], check.Not(check.Matches), `(?ms).*127.0.0.1.*`)
 474 }
 475
 476 func (s *HandlerSuite) TestTrashSweep(c *check.C) {
 477         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 478         s.cluster.Collections.TrashSweepInterval = arvados.Duration(time.Second / 10)
 479         s.handler.CheckHealth()
 480         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 481         coll, err := s.handler.federation.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{"name": "test trash sweep"}, EnsureUniqueName: true})
 482         c.Assert(err, check.IsNil)
 483         defer s.handler.federation.CollectionDelete(ctx, arvados.DeleteOptions{UUID: coll.UUID})
 484         db, err := s.handler.dbConnector.GetDB(s.ctx)
 485         c.Assert(err, check.IsNil)
 486         _, err = db.ExecContext(s.ctx, `update collections set trash_at = $1, delete_at = $2 where uuid = $3`, time.Now().UTC().Add(time.Second/10), time.Now().UTC().Add(time.Hour), coll.UUID)
 487         c.Assert(err, check.IsNil)
 488         deadline := time.Now().Add(5 * time.Second)
 489         for {
 490                 if time.Now().After(deadline) {
 491                         c.Log("timed out")
 492                         c.FailNow()
 493                 }
 494                 updated, err := s.handler.federation.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID, IncludeTrash: true})
 495                 c.Assert(err, check.IsNil)
 496                 if updated.IsTrashed {
 497                         break
 498                 }
 499                 time.Sleep(time.Second / 10)
 500         }
 501 }
 502
 503 func (s *HandlerSuite) TestContainerLogSweep(c *check.C) {
 504         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 505         s.cluster.Containers.Logging.SweepInterval = arvados.Duration(time.Second / 10)
 506         s.handler.CheckHealth()
 507         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 508         logentry, err := s.handler.federation.LogCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
 509                 "object_uuid": arvadostest.CompletedContainerUUID,
 510                 "event_type":  "stderr",
 511                 "properties": map[string]interface{}{
 512                         "text": "test trash sweep\n",
 513                 },
 514         }})
 515         c.Assert(err, check.IsNil)
 516         defer s.handler.federation.LogDelete(ctx, arvados.DeleteOptions{UUID: logentry.UUID})
 517         deadline := time.Now().Add(5 * time.Second)
 518         for {
 519                 if time.Now().After(deadline) {
 520                         c.Log("timed out")
 521                         c.FailNow()
 522                 }
 523                 logentries, err := s.handler.federation.LogList(ctx, arvados.ListOptions{Filters: []arvados.Filter{{"uuid", "=", logentry.UUID}}, Limit: -1})
 524                 c.Assert(err, check.IsNil)
 525                 if len(logentries.Items) == 0 {
 526                         break
 527                 }
 528                 time.Sleep(time.Second / 10)
 529         }
 530 }
 531
 532 func (s *HandlerSuite) TestLogActivity(c *check.C) {
 533         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 534         s.cluster.Users.ActivityLoggingPeriod = arvados.Duration(24 * time.Hour)
 535         s.handler.CheckHealth()
 536
 537         testServer := newServerFromIntegrationTestEnv(c)
 538         testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.handler))
 539         c.Assert(testServer.Start(), check.IsNil)
 540         defer testServer.Close()
 541
 542         u, _ := url.Parse("http://" + testServer.Addr)
 543         client := rpc.NewConn(s.cluster.ClusterID, u, true, rpc.PassthroughTokenProvider)
 544
 545         starttime := time.Now()
 546         for i := 0; i < 4; i++ {
 547                 for _, token := range []string{
 548                         arvadostest.ActiveTokenV2,
 549                         arvadostest.ActiveToken,
 550                         arvadostest.SpectatorToken,
 551                 } {
 552                         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{token}})
 553                         _, err := client.CollectionList(ctx, arvados.ListOptions{})
 554                         c.Assert(err, check.IsNil)
 555                 }
 556         }
 557         db, err := s.handler.dbConnector.GetDB(s.ctx)
 558         c.Assert(err, check.IsNil)
 559         for _, userUUID := range []string{arvadostest.ActiveUserUUID, arvadostest.SpectatorUserUUID} {
 560                 var rows int
 561                 err = db.QueryRowContext(s.ctx, `select count(uuid) from logs where object_uuid = $1 and event_at > $2`, arvadostest.ActiveUserUUID, starttime.UTC()).Scan(&rows)
 562                 c.Assert(err, check.IsNil)
 563                 c.Check(rows, check.Equals, 1, check.Commentf("expect 1 row for user uuid %s", userUUID))
 564         }
 565 }