lib/controller/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/tls"
  11         "encoding/json"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "strings"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/controller/dblock"
  24         "git.arvados.org/arvados.git/lib/controller/rpc"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/httpserver"
  30         "github.com/prometheus/client_golang/prometheus"
  31         check "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func Test(t *testing.T) {
  36         check.TestingT(t)
  37 }
  38
  39 var _ = check.Suite(&HandlerSuite{})
  40
  41 type HandlerSuite struct {
  42         cluster  *arvados.Cluster
  43         handler  *Handler
  44         railsSpy *arvadostest.Proxy
  45         logbuf   *bytes.Buffer
  46         ctx      context.Context
  47         cancel   context.CancelFunc
  48 }
  49
  50 func (s *HandlerSuite) SetUpTest(c *check.C) {
  51         s.logbuf = &bytes.Buffer{}
  52         s.ctx, s.cancel = context.WithCancel(context.Background())
  53         s.ctx = ctxlog.Context(s.ctx, ctxlog.New(io.MultiWriter(os.Stderr, s.logbuf), "json", "debug"))
  54         s.cluster = &arvados.Cluster{
  55                 ClusterID:  "zzzzz",
  56                 PostgreSQL: integrationTestCluster().PostgreSQL,
  57         }
  58         s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
  59         s.cluster.TLS.Insecure = true
  60         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  61         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
  62         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, s.railsSpy.URL.String())
  63         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
  64         s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry()).(*Handler)
  65 }
  66
  67 func (s *HandlerSuite) TearDownTest(c *check.C) {
  68         s.cancel()
  69
  70         // Wait for dblocks to be released. Otherwise, a subsequent
  71         // test might time out waiting to acquire them.
  72         timeout := time.After(10 * time.Second)
  73         for _, locker := range []*dblock.DBLocker{dblock.TrashSweep, dblock.ContainerLogSweep} {
  74                 ok := make(chan struct{})
  75                 go func() {
  76                         if locker.Lock(context.Background(), s.handler.dbConnector.GetDB) {
  77                                 locker.Unlock()
  78                         }
  79                         close(ok)
  80                 }()
  81                 select {
  82                 case <-timeout:
  83                         c.Log("timed out waiting for dblocks")
  84                         c.Fail()
  85                 case <-ok:
  86                 }
  87         }
  88 }
  89
  90 func (s *HandlerSuite) TestConfigExport(c *check.C) {
  91         s.cluster.ManagementToken = "secret"
  92         s.cluster.SystemRootToken = "secret"
  93         s.cluster.Collections.BlobSigning = true
  94         s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
  95         for _, method := range []string{"GET", "OPTIONS"} {
  96                 req := httptest.NewRequest(method, "/arvados/v1/config", nil)
  97                 resp := httptest.NewRecorder()
  98                 s.handler.ServeHTTP(resp, req)
  99                 c.Log(resp.Body.String())
 100                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 101                         continue
 102                 }
 103                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 104                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 105                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 106                 if method == "OPTIONS" {
 107                         c.Check(resp.Body.String(), check.HasLen, 0)
 108                         continue
 109                 }
 110                 var cluster arvados.Cluster
 111                 err := json.Unmarshal(resp.Body.Bytes(), &cluster)
 112                 c.Check(err, check.IsNil)
 113                 c.Check(cluster.ManagementToken, check.Equals, "")
 114                 c.Check(cluster.SystemRootToken, check.Equals, "")
 115                 c.Check(cluster.Collections.BlobSigning, check.Equals, true)
 116                 c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
 117         }
 118 }
 119
 120 func (s *HandlerSuite) TestDiscoveryDocCache(c *check.C) {
 121         countRailsReqs := func() int {
 122                 s.railsSpy.Wait()
 123                 n := 0
 124                 for _, req := range s.railsSpy.RequestDumps {
 125                         if bytes.Contains(req, []byte("/discovery/v1/apis/arvados/v1/rest")) {
 126                                 n++
 127                         }
 128                 }
 129                 return n
 130         }
 131         getDD := func() int {
 132                 req := httptest.NewRequest(http.MethodGet, "/discovery/v1/apis/arvados/v1/rest", nil)
 133                 resp := httptest.NewRecorder()
 134                 s.handler.ServeHTTP(resp, req)
 135                 if resp.Code == http.StatusOK {
 136                         var dd arvados.DiscoveryDocument
 137                         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 138                         c.Check(err, check.IsNil)
 139                         c.Check(dd.Schemas["Collection"].UUIDPrefix, check.Equals, "4zz18")
 140                 }
 141                 return resp.Code
 142         }
 143         getDDConcurrently := func(n int, expectCode int, checkArgs ...interface{}) *sync.WaitGroup {
 144                 var wg sync.WaitGroup
 145                 for i := 0; i < n; i++ {
 146                         wg.Add(1)
 147                         go func() {
 148                                 defer wg.Done()
 149                                 c.Check(getDD(), check.Equals, append([]interface{}{expectCode}, checkArgs...)...)
 150                         }()
 151                 }
 152                 return &wg
 153         }
 154         clearCache := func() {
 155                 for _, ent := range s.handler.cache {
 156                         ent.refreshLock.Lock()
 157                         ent.mtx.Lock()
 158                         ent.body, ent.header, ent.refreshAfter = nil, nil, time.Time{}
 159                         ent.mtx.Unlock()
 160                         ent.refreshLock.Unlock()
 161                 }
 162         }
 163         waitPendingUpdates := func() {
 164                 for _, ent := range s.handler.cache {
 165                         ent.refreshLock.Lock()
 166                         defer ent.refreshLock.Unlock()
 167                         ent.mtx.Lock()
 168                         defer ent.mtx.Unlock()
 169                 }
 170         }
 171         refreshNow := func() {
 172                 waitPendingUpdates()
 173                 for _, ent := range s.handler.cache {
 174                         ent.refreshAfter = time.Now()
 175                 }
 176         }
 177         expireNow := func() {
 178                 waitPendingUpdates()
 179                 for _, ent := range s.handler.cache {
 180                         ent.expireAfter = time.Now()
 181                 }
 182         }
 183
 184         // Easy path: first req fetches, subsequent reqs use cache.
 185         c.Check(countRailsReqs(), check.Equals, 0)
 186         c.Check(getDD(), check.Equals, http.StatusOK)
 187         c.Check(countRailsReqs(), check.Equals, 1)
 188         c.Check(getDD(), check.Equals, http.StatusOK)
 189         c.Check(countRailsReqs(), check.Equals, 1)
 190         c.Check(getDD(), check.Equals, http.StatusOK)
 191         c.Check(countRailsReqs(), check.Equals, 1)
 192
 193         // To guarantee we have concurrent requests, we set up
 194         // railsSpy to hold up the Handler's outgoing requests until
 195         // we send to (or close) holdReqs.
 196         holdReqs := make(chan struct{})
 197         s.railsSpy.Director = func(*http.Request) {
 198                 <-holdReqs
 199         }
 200
 201         // Race at startup: first req fetches, other concurrent reqs
 202         // wait for the initial fetch to complete, then all return.
 203         clearCache()
 204         reqsBefore := countRailsReqs()
 205         wg := getDDConcurrently(5, http.StatusOK, check.Commentf("race at startup"))
 206         close(holdReqs)
 207         wg.Wait()
 208         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 209
 210         // Race after expiry: concurrent reqs return the cached data
 211         // but initiate a new fetch in the background.
 212         refreshNow()
 213         holdReqs = make(chan struct{})
 214         wg = getDDConcurrently(5, http.StatusOK, check.Commentf("race after expiry"))
 215         reqsBefore = countRailsReqs()
 216         close(holdReqs)
 217         wg.Wait()
 218         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline) && countRailsReqs() < reqsBefore+1; {
 219                 time.Sleep(time.Second / 100)
 220         }
 221         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 222
 223         // Configure railsSpy to return an error or bad content
 224         // depending on flags.
 225         var wantError, wantBadContent bool
 226         s.railsSpy.Director = func(req *http.Request) {
 227                 <-holdReqs
 228                 if wantError {
 229                         // The Passenger server hosting RailsAPI will drop HTTP requests
 230                         // unrecognized names. Make a request with a real method that
 231                         // RailsAPI doesn't implement.
 232                         req.Method = "TRACE"
 233                 } else if wantBadContent {
 234                         req.URL.Path = "/_health/ping"
 235                         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 236                 }
 237         }
 238
 239         // Error at startup (empty cache) => caller gets error, and we
 240         // make an upstream attempt for each incoming request because
 241         // we have nothing better to return
 242         clearCache()
 243         wantError, wantBadContent = true, false
 244         reqsBefore = countRailsReqs()
 245         holdReqs = make(chan struct{})
 246         wg = getDDConcurrently(5, http.StatusBadGateway, check.Commentf("error at startup"))
 247         close(holdReqs)
 248         wg.Wait()
 249         c.Check(countRailsReqs(), check.Equals, reqsBefore+5)
 250
 251         // Response status is OK but body is not a discovery document
 252         wantError, wantBadContent = false, true
 253         reqsBefore = countRailsReqs()
 254         c.Check(getDD(), check.Equals, http.StatusBadGateway)
 255         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 256
 257         // Error condition clears => caller gets OK, cache is warmed
 258         // up
 259         wantError, wantBadContent = false, false
 260         reqsBefore = countRailsReqs()
 261         getDDConcurrently(5, http.StatusOK, check.Commentf("success after errors at startup")).Wait()
 262         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 263
 264         // Error with warm cache => caller gets OK (with no attempt to
 265         // re-fetch)
 266         wantError, wantBadContent = true, false
 267         reqsBefore = countRailsReqs()
 268         getDDConcurrently(5, http.StatusOK, check.Commentf("error with warm cache")).Wait()
 269         c.Check(countRailsReqs(), check.Equals, reqsBefore)
 270
 271         checkBackgroundRefresh := func(reqsExpected int) {
 272                 // There is no guarantee that a background refresh has
 273                 // progressed far enough that we can detect it
 274                 // directly (the first line of refresh() might not
 275                 // have run).  So, to avoid false positives, we just
 276                 // need to poll until it happens.
 277                 for deadline := time.Now().Add(time.Second); countRailsReqs() == reqsBefore && time.Now().Before(deadline); {
 278                         c.Logf("countRailsReqs = %d", countRailsReqs())
 279                         time.Sleep(time.Second / 100)
 280                 }
 281                 // Similarly, to ensure there are no additional
 282                 // refreshes, we just need to wait.
 283                 time.Sleep(time.Second / 2)
 284                 c.Check(countRailsReqs(), check.Equals, reqsExpected)
 285         }
 286
 287         // Error with stale cache => caller gets OK with stale data
 288         // while the re-fetch is attempted in the background
 289         refreshNow()
 290         wantError, wantBadContent = true, false
 291         reqsBefore = countRailsReqs()
 292         holdReqs = make(chan struct{})
 293         getDDConcurrently(5, http.StatusOK, check.Commentf("error with stale cache")).Wait()
 294         close(holdReqs)
 295         // After piling up 5 requests (holdReqs having ensured the
 296         // first update took long enough for the last incoming request
 297         // to arrive) there should be only one attempt to re-fetch.
 298         checkBackgroundRefresh(reqsBefore + 1)
 299
 300         refreshNow()
 301         wantError, wantBadContent = false, false
 302         reqsBefore = countRailsReqs()
 303         holdReqs = make(chan struct{})
 304         getDDConcurrently(5, http.StatusOK, check.Commentf("refresh cache after error condition clears")).Wait()
 305         close(holdReqs)
 306         checkBackgroundRefresh(reqsBefore + 1)
 307
 308         // Make sure expireAfter is getting set
 309         waitPendingUpdates()
 310         exp := s.handler.cache["/discovery/v1/apis/arvados/v1/rest"].expireAfter.Sub(time.Now())
 311         c.Check(exp > cacheTTL, check.Equals, true)
 312         c.Check(exp < cacheExpire, check.Equals, true)
 313
 314         // After the cache *expires* it behaves as if uninitialized:
 315         // each incoming request does a new upstream request until one
 316         // succeeds.
 317         //
 318         // First check failure after expiry:
 319         expireNow()
 320         wantError, wantBadContent = true, false
 321         reqsBefore = countRailsReqs()
 322         holdReqs = make(chan struct{})
 323         wg = getDDConcurrently(5, http.StatusBadGateway, check.Commentf("error after expiry"))
 324         close(holdReqs)
 325         wg.Wait()
 326         c.Check(countRailsReqs(), check.Equals, reqsBefore+5)
 327
 328         // Success after expiry:
 329         wantError, wantBadContent = false, false
 330         reqsBefore = countRailsReqs()
 331         holdReqs = make(chan struct{})
 332         wg = getDDConcurrently(5, http.StatusOK, check.Commentf("success after expiry"))
 333         close(holdReqs)
 334         wg.Wait()
 335         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 336 }
 337
 338 func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
 339         voc := `{
 340                 "strict_tags": false,
 341                 "tags": {
 342                         "IDTAGIMPORTANCE": {
 343                                 "strict": false,
 344                                 "labels": [{"label": "Importance"}],
 345                                 "values": {
 346                                         "HIGH": {
 347                                                 "labels": [{"label": "High"}]
 348                                         },
 349                                         "LOW": {
 350                                                 "labels": [{"label": "Low"}]
 351                                         }
 352                                 }
 353                         }
 354                 }
 355         }`
 356         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 357         c.Assert(err, check.IsNil)
 358         defer os.Remove(f.Name())
 359         _, err = f.WriteString(voc)
 360         c.Assert(err, check.IsNil)
 361         f.Close()
 362         s.cluster.API.VocabularyPath = f.Name()
 363         for _, method := range []string{"GET", "OPTIONS"} {
 364                 c.Log(c.TestName()+" ", method)
 365                 req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
 366                 resp := httptest.NewRecorder()
 367                 s.handler.ServeHTTP(resp, req)
 368                 c.Log(resp.Body.String())
 369                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 370                         continue
 371                 }
 372                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 373                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 374                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 375                 if method == "OPTIONS" {
 376                         c.Check(resp.Body.String(), check.HasLen, 0)
 377                         continue
 378                 }
 379                 var expectedVoc, receivedVoc *arvados.Vocabulary
 380                 err := json.Unmarshal([]byte(voc), &expectedVoc)
 381                 c.Check(err, check.IsNil)
 382                 err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
 383                 c.Check(err, check.IsNil)
 384                 c.Check(receivedVoc, check.DeepEquals, expectedVoc)
 385         }
 386 }
 387
 388 func (s *HandlerSuite) TestVocabularyFailedCheckStatus(c *check.C) {
 389         voc := `{
 390                 "strict_tags": false,
 391                 "tags": {
 392                         "IDTAGIMPORTANCE": {
 393                                 "strict": true,
 394                                 "labels": [{"label": "Importance"}],
 395                                 "values": {
 396                                         "HIGH": {
 397                                                 "labels": [{"label": "High"}]
 398                                         },
 399                                         "LOW": {
 400                                                 "labels": [{"label": "Low"}]
 401                                         }
 402                                 }
 403                         }
 404                 }
 405         }`
 406         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 407         c.Assert(err, check.IsNil)
 408         defer os.Remove(f.Name())
 409         _, err = f.WriteString(voc)
 410         c.Assert(err, check.IsNil)
 411         f.Close()
 412         s.cluster.API.VocabularyPath = f.Name()
 413
 414         req := httptest.NewRequest("POST", "/arvados/v1/collections",
 415                 strings.NewReader(`{
 416                         "collection": {
 417                                 "properties": {
 418                                         "IDTAGIMPORTANCE": "Critical"
 419                                 }
 420                         }
 421                 }`))
 422         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 423         req.Header.Set("Content-type", "application/json")
 424
 425         resp := httptest.NewRecorder()
 426         s.handler.ServeHTTP(resp, req)
 427         c.Log(resp.Body.String())
 428         c.Assert(resp.Code, check.Equals, http.StatusBadRequest)
 429         var jresp httpserver.ErrorResponse
 430         err = json.Unmarshal(resp.Body.Bytes(), &jresp)
 431         c.Check(err, check.IsNil)
 432         c.Assert(len(jresp.Errors), check.Equals, 1)
 433         c.Check(jresp.Errors[0], check.Matches, `.*tag value.*is not valid for key.*`)
 434 }
 435
 436 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 437         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 438         resp := httptest.NewRecorder()
 439         s.handler.ServeHTTP(resp, req)
 440         c.Check(resp.Code, check.Equals, http.StatusOK)
 441         var dd arvados.DiscoveryDocument
 442         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 443         c.Check(err, check.IsNil)
 444         c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
 445         c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
 446         c.Check(len(dd.Resources), check.Not(check.Equals), 0)
 447         c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
 448 }
 449
 450 // Handler should give up and exit early if request context is
 451 // cancelled due to client hangup, httpserver.HandlerWithDeadline,
 452 // etc.
 453 func (s *HandlerSuite) TestRequestCancel(c *check.C) {
 454         ctx, cancel := context.WithCancel(context.Background())
 455         req := httptest.NewRequest("GET", "/static/login_failure", nil).WithContext(ctx)
 456         resp := httptest.NewRecorder()
 457         cancel()
 458         s.handler.ServeHTTP(resp, req)
 459         c.Check(resp.Code, check.Equals, http.StatusBadGateway)
 460         var jresp httpserver.ErrorResponse
 461         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 462         c.Check(err, check.IsNil)
 463         c.Assert(len(jresp.Errors), check.Equals, 1)
 464         c.Check(jresp.Errors[0], check.Matches, `.*context canceled`)
 465 }
 466
 467 func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
 468         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 469         resp := httptest.NewRecorder()
 470         s.handler.ServeHTTP(resp, req)
 471         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 472         jresp := map[string]interface{}{}
 473         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 474         c.Check(err, check.IsNil)
 475         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 476 }
 477
 478 func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
 479         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 480         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 481         resp := httptest.NewRecorder()
 482         s.handler.ServeHTTP(resp, req)
 483         c.Check(resp.Code, check.Equals, http.StatusOK)
 484         var u arvados.User
 485         err := json.Unmarshal(resp.Body.Bytes(), &u)
 486         c.Check(err, check.IsNil)
 487         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 488 }
 489
 490 func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
 491         req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
 492                 "_method":   {"GET"},
 493                 "api_token": {arvadostest.ActiveToken},
 494         }.Encode()))
 495         req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 496         resp := httptest.NewRecorder()
 497         s.handler.ServeHTTP(resp, req)
 498         c.Check(resp.Code, check.Equals, http.StatusOK)
 499         var u arvados.User
 500         err := json.Unmarshal(resp.Body.Bytes(), &u)
 501         c.Check(err, check.IsNil)
 502         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 503 }
 504
 505 func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
 506         req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
 507         resp := httptest.NewRecorder()
 508         s.handler.ServeHTTP(resp, req)
 509         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 510         jresp := map[string]interface{}{}
 511         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 512         c.Check(err, check.IsNil)
 513         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 514 }
 515
 516 func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
 517         s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "wb2.example", Path: "/"}
 518         s.cluster.Login.Google.Enable = true
 519         s.cluster.Login.Google.ClientID = "test"
 520         req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://wb2.example/", nil)
 521         resp := httptest.NewRecorder()
 522         s.handler.ServeHTTP(resp, req)
 523         if !c.Check(resp.Code, check.Equals, http.StatusFound) {
 524                 c.Log(resp.Body.String())
 525         }
 526         c.Check(resp.Header().Get("Location"), check.Equals, "https://wb2.example/")
 527 }
 528
 529 func (s *HandlerSuite) TestValidateV1APIToken(c *check.C) {
 530         c.Assert(s.handler.CheckHealth(), check.IsNil)
 531         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 532         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveToken)
 533         c.Assert(err, check.IsNil)
 534         c.Check(ok, check.Equals, true)
 535         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 536         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 537         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 538         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 539 }
 540
 541 func (s *HandlerSuite) TestValidateV2APIToken(c *check.C) {
 542         c.Assert(s.handler.CheckHealth(), check.IsNil)
 543         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 544         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveTokenV2)
 545         c.Assert(err, check.IsNil)
 546         c.Check(ok, check.Equals, true)
 547         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 548         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 549         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 550         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 551         c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
 552 }
 553
 554 func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
 555         saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
 556         c.Assert(err, check.IsNil)
 557         for _, trial := range []struct {
 558                 code  int
 559                 token string
 560         }{
 561                 {http.StatusOK, saltedToken},
 562                 {http.StatusUnauthorized, "bogus"},
 563         } {
 564                 req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
 565                 req.Header.Set("Authorization", "Bearer "+trial.token)
 566                 resp := httptest.NewRecorder()
 567                 s.handler.ServeHTTP(resp, req)
 568                 if !c.Check(resp.Code, check.Equals, trial.code) {
 569                         c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
 570                 }
 571         }
 572 }
 573
 574 func (s *HandlerSuite) TestLogTokenUUID(c *check.C) {
 575         req := httptest.NewRequest("GET", "https://0.0.0.0/arvados/v1/users/current", nil)
 576         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 577         req = req.WithContext(s.ctx)
 578         resp := httptest.NewRecorder()
 579         httpserver.LogRequests(s.handler).ServeHTTP(resp, req)
 580         c.Check(resp.Code, check.Equals, http.StatusOK)
 581         c.Check(s.logbuf.String(), check.Matches, `(?ms).*"tokenUUIDs":\["`+strings.Split(arvadostest.ActiveTokenV2, "/")[1]+`"\].*`)
 582 }
 583
 584 func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
 585         c.Assert(s.handler.CheckHealth(), check.IsNil)
 586         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 587         auth, err := s.handler.createAPItoken(req, arvadostest.ActiveUserUUID, nil)
 588         c.Assert(err, check.IsNil)
 589         c.Check(auth.Scopes, check.DeepEquals, []string{"all"})
 590
 591         user, ok, err := s.handler.validateAPItoken(req, auth.TokenV2())
 592         c.Assert(err, check.IsNil)
 593         c.Check(ok, check.Equals, true)
 594         c.Check(user.Authorization.UUID, check.Equals, auth.UUID)
 595         c.Check(user.Authorization.APIToken, check.Equals, auth.APIToken)
 596         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 597         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 598         c.Check(user.Authorization.TokenV2(), check.Equals, auth.TokenV2())
 599 }
 600
 601 func (s *HandlerSuite) CheckObjectType(c *check.C, url string, token string, skippedFields map[string]bool) {
 602         var proxied, direct map[string]interface{}
 603         var err error
 604
 605         // Get collection from controller
 606         req := httptest.NewRequest("GET", url, nil)
 607         req.Header.Set("Authorization", "Bearer "+token)
 608         resp := httptest.NewRecorder()
 609         s.handler.ServeHTTP(resp, req)
 610         if !c.Check(resp.Code, check.Equals, http.StatusOK,
 611                 check.Commentf("Wasn't able to get data from the controller at %q: %q", url, resp.Body.String())) {
 612                 return
 613         }
 614         err = json.Unmarshal(resp.Body.Bytes(), &proxied)
 615         c.Check(err, check.Equals, nil)
 616
 617         // Get collection directly from RailsAPI
 618         client := &http.Client{
 619                 Transport: &http.Transport{
 620                         TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 621                 },
 622         }
 623         resp2, err := client.Get(s.cluster.Services.RailsAPI.ExternalURL.String() + url + "/?api_token=" + token)
 624         c.Check(err, check.Equals, nil)
 625         defer resp2.Body.Close()
 626         if !c.Check(resp2.StatusCode, check.Equals, http.StatusOK,
 627                 check.Commentf("Wasn't able to get data from the RailsAPI at %q", url)) {
 628                 return
 629         }
 630         db, err := ioutil.ReadAll(resp2.Body)
 631         c.Check(err, check.Equals, nil)
 632         err = json.Unmarshal(db, &direct)
 633         c.Check(err, check.Equals, nil)
 634
 635         // Check that all RailsAPI provided keys exist on the controller response.
 636         for k := range direct {
 637                 if _, ok := skippedFields[k]; ok {
 638                         continue
 639                 } else if val, ok := proxied[k]; !ok {
 640                         c.Errorf("%s's key %q missing on controller's response.", direct["kind"], k)
 641                 } else if direct["kind"] == "arvados#collection" && k == "manifest_text" {
 642                         // Tokens differ from request to request
 643                         c.Check(strings.Split(val.(string), "+A")[0], check.Equals, strings.Split(direct[k].(string), "+A")[0])
 644                 } else {
 645                         c.Check(val, check.DeepEquals, direct[k],
 646                                 check.Commentf("RailsAPI %s key %q's value %q differs from controller's %q.", direct["kind"], k, direct[k], val))
 647                 }
 648         }
 649
 650         // The "href" field has been removed. We don't particularly
 651         // care whether Rails returns it, as long as controller
 652         // doesn't.
 653         _, hasHref := proxied["href"]
 654         c.Check(hasHref, check.Equals, false)
 655 }
 656
 657 func (s *HandlerSuite) TestGetObjects(c *check.C) {
 658         // Get the 1st keep service's uuid from the running test server.
 659         req := httptest.NewRequest("GET", "/arvados/v1/keep_services/", nil)
 660         req.Header.Set("Authorization", "Bearer "+arvadostest.AdminToken)
 661         resp := httptest.NewRecorder()
 662         s.handler.ServeHTTP(resp, req)
 663         c.Assert(resp.Code, check.Equals, http.StatusOK)
 664         var ksList arvados.KeepServiceList
 665         json.Unmarshal(resp.Body.Bytes(), &ksList)
 666         c.Assert(len(ksList.Items), check.Not(check.Equals), 0)
 667         ksUUID := ksList.Items[0].UUID
 668         // Create a new token for the test user so that we're not comparing
 669         // the ones from the fixtures.
 670         req = httptest.NewRequest("POST", "/arvados/v1/api_client_authorizations",
 671                 strings.NewReader(`{
 672                         "api_client_authorization": {
 673                                 "owner_uuid": "`+arvadostest.AdminUserUUID+`",
 674                                 "created_by_ip_address": "::1",
 675                                 "last_used_by_ip_address": "::1"
 676                         }
 677                 }`))
 678         req.Header.Set("Authorization", "Bearer "+arvadostest.SystemRootToken)
 679         req.Header.Set("Content-type", "application/json")
 680         resp = httptest.NewRecorder()
 681         s.handler.ServeHTTP(resp, req)
 682         c.Assert(resp.Code, check.Equals, http.StatusOK,
 683                 check.Commentf("%s", resp.Body.String()))
 684         var auth arvados.APIClientAuthorization
 685         json.Unmarshal(resp.Body.Bytes(), &auth)
 686         c.Assert(auth.UUID, check.Not(check.Equals), "")
 687
 688         testCases := map[string]map[string]bool{
 689                 "api_client_authorizations/" + auth.UUID:                       {"modified_by_client_uuid": true, "modified_by_user_uuid": true},
 690                 "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID:       nil,
 691                 "collections/" + arvadostest.CollectionWithUniqueWordsUUID:     nil,
 692                 "containers/" + arvadostest.RunningContainerUUID:               nil,
 693                 "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil,
 694                 "groups/" + arvadostest.AProjectUUID:                           nil,
 695                 "keep_services/" + ksUUID:                                      nil,
 696                 "links/" + arvadostest.ActiveUserCanReadAllUsersLinkUUID:       nil,
 697                 "logs/" + arvadostest.CrunchstatForRunningContainerLogUUID:     nil,
 698                 "users/" + arvadostest.ActiveUserUUID:                          nil,
 699                 "virtual_machines/" + arvadostest.TestVMUUID:                   nil,
 700                 "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID:      nil,
 701         }
 702         for url, skippedFields := range testCases {
 703                 c.Logf("Testing %q", url)
 704                 s.CheckObjectType(c, "/arvados/v1/"+url, auth.TokenV2(), skippedFields)
 705         }
 706 }
 707
 708 func (s *HandlerSuite) TestRedactRailsAPIHostFromErrors(c *check.C) {
 709         req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/collections/zzzzz-4zz18-abcdefghijklmno", nil)
 710         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 711         resp := httptest.NewRecorder()
 712         s.handler.ServeHTTP(resp, req)
 713         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 714         var jresp struct {
 715                 Errors []string
 716         }
 717         c.Log(resp.Body.String())
 718         c.Assert(json.NewDecoder(resp.Body).Decode(&jresp), check.IsNil)
 719         c.Assert(jresp.Errors, check.HasLen, 1)
 720         c.Check(jresp.Errors[0], check.Matches, `.*//railsapi\.internal/arvados/v1/collections/.*: 404 Not Found.*`)
 721         c.Check(jresp.Errors[0], check.Not(check.Matches), `(?ms).*127.0.0.1.*`)
 722 }
 723
 724 func (s *HandlerSuite) TestTrashSweep(c *check.C) {
 725         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 726         s.cluster.Collections.TrashSweepInterval = arvados.Duration(time.Second / 10)
 727         s.handler.CheckHealth()
 728         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 729         coll, err := s.handler.federation.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{"name": "test trash sweep"}, EnsureUniqueName: true})
 730         c.Assert(err, check.IsNil)
 731         defer s.handler.federation.CollectionDelete(ctx, arvados.DeleteOptions{UUID: coll.UUID})
 732         db, err := s.handler.dbConnector.GetDB(s.ctx)
 733         c.Assert(err, check.IsNil)
 734         _, err = db.ExecContext(s.ctx, `update collections set trash_at = $1, delete_at = $2 where uuid = $3`, time.Now().UTC().Add(time.Second/10), time.Now().UTC().Add(time.Hour), coll.UUID)
 735         c.Assert(err, check.IsNil)
 736         deadline := time.Now().Add(5 * time.Second)
 737         for {
 738                 if time.Now().After(deadline) {
 739                         c.Log("timed out")
 740                         c.FailNow()
 741                 }
 742                 updated, err := s.handler.federation.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID, IncludeTrash: true})
 743                 c.Assert(err, check.IsNil)
 744                 if updated.IsTrashed {
 745                         break
 746                 }
 747                 time.Sleep(time.Second / 10)
 748         }
 749 }
 750
 751 func (s *HandlerSuite) TestContainerLogSweep(c *check.C) {
 752         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 753         s.cluster.Collections.TrashSweepInterval = arvados.Duration(2 * time.Second)
 754         s.handler.CheckHealth()
 755         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 756         logentry, err := s.handler.federation.LogCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
 757                 "object_uuid": arvadostest.CompletedContainerUUID,
 758                 "event_type":  "stderr",
 759                 "properties": map[string]interface{}{
 760                         "text": "test container log sweep\n",
 761                 },
 762         }})
 763         c.Assert(err, check.IsNil)
 764         defer s.handler.federation.LogDelete(ctx, arvados.DeleteOptions{UUID: logentry.UUID})
 765         deadline := time.Now().Add(5 * time.Second)
 766         for {
 767                 if time.Now().After(deadline) {
 768                         c.Log("timed out")
 769                         c.FailNow()
 770                 }
 771                 logentries, err := s.handler.federation.LogList(ctx, arvados.ListOptions{Filters: []arvados.Filter{{"uuid", "=", logentry.UUID}}, Limit: -1})
 772                 c.Assert(err, check.IsNil)
 773                 if len(logentries.Items) == 0 {
 774                         break
 775                 }
 776                 time.Sleep(time.Second / 10)
 777         }
 778 }
 779
 780 func (s *HandlerSuite) TestLogActivity(c *check.C) {
 781         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 782         s.cluster.Users.ActivityLoggingPeriod = arvados.Duration(24 * time.Hour)
 783         s.handler.CheckHealth()
 784
 785         testServer := newServerFromIntegrationTestEnv(c)
 786         testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.handler))
 787         c.Assert(testServer.Start(), check.IsNil)
 788         defer testServer.Close()
 789
 790         u, _ := url.Parse("http://" + testServer.Addr)
 791         client := rpc.NewConn(s.cluster.ClusterID, u, true, rpc.PassthroughTokenProvider)
 792
 793         starttime := time.Now()
 794         for i := 0; i < 4; i++ {
 795                 for _, token := range []string{
 796                         arvadostest.ActiveTokenV2,
 797                         arvadostest.ActiveToken,
 798                         arvadostest.SpectatorToken,
 799                 } {
 800                         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{token}})
 801                         _, err := client.CollectionList(ctx, arvados.ListOptions{})
 802                         c.Assert(err, check.IsNil)
 803                 }
 804         }
 805         db, err := s.handler.dbConnector.GetDB(s.ctx)
 806         c.Assert(err, check.IsNil)
 807         for _, userUUID := range []string{arvadostest.ActiveUserUUID, arvadostest.SpectatorUserUUID} {
 808                 var rows int
 809                 err = db.QueryRowContext(s.ctx, `select count(uuid) from logs where object_uuid = $1 and event_at > $2`, arvadostest.ActiveUserUUID, starttime.UTC()).Scan(&rows)
 810                 c.Assert(err, check.IsNil)
 811                 c.Check(rows, check.Equals, 1, check.Commentf("expect 1 row for user uuid %s", userUUID))
 812         }
 813 }