lib/controller/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/tls"
  11         "encoding/json"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "strings"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/controller/dblock"
  24         "git.arvados.org/arvados.git/lib/controller/rpc"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/httpserver"
  30         "github.com/prometheus/client_golang/prometheus"
  31         check "gopkg.in/check.v1"
  32 )
  33
  34 // Gocheck boilerplate
  35 func Test(t *testing.T) {
  36         check.TestingT(t)
  37 }
  38
  39 var _ = check.Suite(&HandlerSuite{})
  40
  41 type HandlerSuite struct {
  42         cluster  *arvados.Cluster
  43         handler  *Handler
  44         railsSpy *arvadostest.Proxy
  45         logbuf   *bytes.Buffer
  46         ctx      context.Context
  47         cancel   context.CancelFunc
  48 }
  49
  50 func (s *HandlerSuite) SetUpTest(c *check.C) {
  51         s.logbuf = &bytes.Buffer{}
  52         s.ctx, s.cancel = context.WithCancel(context.Background())
  53         s.ctx = ctxlog.Context(s.ctx, ctxlog.New(io.MultiWriter(os.Stderr, s.logbuf), "json", "debug"))
  54         s.cluster = &arvados.Cluster{
  55                 ClusterID:  "zzzzz",
  56                 PostgreSQL: integrationTestCluster().PostgreSQL,
  57         }
  58         s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
  59         s.cluster.TLS.Insecure = true
  60         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
  61         s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
  62         arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, s.railsSpy.URL.String())
  63         arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
  64         s.handler = newHandler(s.ctx, s.cluster, "", prometheus.NewRegistry()).(*Handler)
  65 }
  66
  67 func (s *HandlerSuite) TearDownTest(c *check.C) {
  68         s.cancel()
  69
  70         // Wait for dblocks to be released. Otherwise, a subsequent
  71         // test might time out waiting to acquire them.
  72         timeout := time.After(10 * time.Second)
  73         for _, locker := range []*dblock.DBLocker{dblock.TrashSweep, dblock.ContainerLogSweep} {
  74                 ok := make(chan struct{})
  75                 go func() {
  76                         if locker.Lock(context.Background(), s.handler.dbConnector.GetDB) {
  77                                 locker.Unlock()
  78                         }
  79                         close(ok)
  80                 }()
  81                 select {
  82                 case <-timeout:
  83                         c.Log("timed out waiting for dblocks")
  84                         c.Fail()
  85                 case <-ok:
  86                 }
  87         }
  88 }
  89
  90 func (s *HandlerSuite) TestConfigExport(c *check.C) {
  91         s.cluster.ManagementToken = "secret"
  92         s.cluster.SystemRootToken = "secret"
  93         s.cluster.Collections.BlobSigning = true
  94         s.cluster.Collections.BlobSigningTTL = arvados.Duration(23 * time.Second)
  95         for _, method := range []string{"GET", "OPTIONS"} {
  96                 req := httptest.NewRequest(method, "/arvados/v1/config", nil)
  97                 resp := httptest.NewRecorder()
  98                 s.handler.ServeHTTP(resp, req)
  99                 c.Log(resp.Body.String())
 100                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 101                         continue
 102                 }
 103                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 104                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 105                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 106                 if method == "OPTIONS" {
 107                         c.Check(resp.Body.String(), check.HasLen, 0)
 108                         continue
 109                 }
 110                 var cluster arvados.Cluster
 111                 err := json.Unmarshal(resp.Body.Bytes(), &cluster)
 112                 c.Check(err, check.IsNil)
 113                 c.Check(cluster.ManagementToken, check.Equals, "")
 114                 c.Check(cluster.SystemRootToken, check.Equals, "")
 115                 c.Check(cluster.Collections.BlobSigning, check.Equals, true)
 116                 c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
 117         }
 118 }
 119
 120 func (s *HandlerSuite) TestDiscoveryDocCache(c *check.C) {
 121         countRailsReqs := func() int {
 122                 s.railsSpy.Wait()
 123                 n := 0
 124                 for _, req := range s.railsSpy.RequestDumps {
 125                         if bytes.Contains(req, []byte("/discovery/v1/apis/arvados/v1/rest")) {
 126                                 n++
 127                         }
 128                 }
 129                 return n
 130         }
 131         getDD := func() int {
 132                 req := httptest.NewRequest(http.MethodGet, "/discovery/v1/apis/arvados/v1/rest", nil)
 133                 resp := httptest.NewRecorder()
 134                 s.handler.ServeHTTP(resp, req)
 135                 if resp.Code == http.StatusOK {
 136                         var dd arvados.DiscoveryDocument
 137                         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 138                         c.Check(err, check.IsNil)
 139                         c.Check(dd.Schemas["Collection"].UUIDPrefix, check.Equals, "4zz18")
 140                 }
 141                 return resp.Code
 142         }
 143         getDDConcurrently := func(n int, expectCode int, checkArgs ...interface{}) *sync.WaitGroup {
 144                 var wg sync.WaitGroup
 145                 for i := 0; i < n; i++ {
 146                         wg.Add(1)
 147                         go func() {
 148                                 defer wg.Done()
 149                                 c.Check(getDD(), check.Equals, append([]interface{}{expectCode}, checkArgs...)...)
 150                         }()
 151                 }
 152                 return &wg
 153         }
 154         clearCache := func() {
 155                 for _, ent := range s.handler.cache {
 156                         ent.refreshLock.Lock()
 157                         ent.mtx.Lock()
 158                         ent.body, ent.header, ent.refreshAfter = nil, nil, time.Time{}
 159                         ent.mtx.Unlock()
 160                         ent.refreshLock.Unlock()
 161                 }
 162         }
 163         waitPendingUpdates := func() {
 164                 for _, ent := range s.handler.cache {
 165                         ent.refreshLock.Lock()
 166                         defer ent.refreshLock.Unlock()
 167                         ent.mtx.Lock()
 168                         defer ent.mtx.Unlock()
 169                 }
 170         }
 171         refreshNow := func() {
 172                 waitPendingUpdates()
 173                 for _, ent := range s.handler.cache {
 174                         ent.refreshAfter = time.Now()
 175                 }
 176         }
 177         expireNow := func() {
 178                 waitPendingUpdates()
 179                 for _, ent := range s.handler.cache {
 180                         ent.expireAfter = time.Now()
 181                 }
 182         }
 183
 184         // Easy path: first req fetches, subsequent reqs use cache.
 185         c.Check(countRailsReqs(), check.Equals, 0)
 186         c.Check(getDD(), check.Equals, http.StatusOK)
 187         c.Check(countRailsReqs(), check.Equals, 1)
 188         c.Check(getDD(), check.Equals, http.StatusOK)
 189         c.Check(countRailsReqs(), check.Equals, 1)
 190         c.Check(getDD(), check.Equals, http.StatusOK)
 191         c.Check(countRailsReqs(), check.Equals, 1)
 192
 193         // To guarantee we have concurrent requests, we set up
 194         // railsSpy to hold up the Handler's outgoing requests until
 195         // we send to (or close) holdReqs.
 196         holdReqs := make(chan struct{})
 197         s.railsSpy.Director = func(*http.Request) {
 198                 <-holdReqs
 199         }
 200
 201         // Race at startup: first req fetches, other concurrent reqs
 202         // wait for the initial fetch to complete, then all return.
 203         clearCache()
 204         reqsBefore := countRailsReqs()
 205         wg := getDDConcurrently(5, http.StatusOK, check.Commentf("race at startup"))
 206         close(holdReqs)
 207         wg.Wait()
 208         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 209
 210         // Race after expiry: concurrent reqs return the cached data
 211         // but initiate a new fetch in the background.
 212         refreshNow()
 213         holdReqs = make(chan struct{})
 214         wg = getDDConcurrently(5, http.StatusOK, check.Commentf("race after expiry"))
 215         reqsBefore = countRailsReqs()
 216         close(holdReqs)
 217         wg.Wait()
 218         for deadline := time.Now().Add(time.Second); time.Now().Before(deadline) && countRailsReqs() < reqsBefore+1; {
 219                 time.Sleep(time.Second / 100)
 220         }
 221         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 222
 223         // Configure railsSpy to return an error or bad content
 224         // depending on flags.
 225         var wantError, wantBadContent bool
 226         s.railsSpy.Director = func(req *http.Request) {
 227                 <-holdReqs
 228                 if wantError {
 229                         req.Method = "MAKE-COFFEE"
 230                 } else if wantBadContent {
 231                         req.URL.Path = "/_health/ping"
 232                         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 233                 }
 234         }
 235
 236         // Error at startup (empty cache) => caller gets error, and we
 237         // make an upstream attempt for each incoming request because
 238         // we have nothing better to return
 239         clearCache()
 240         wantError, wantBadContent = true, false
 241         reqsBefore = countRailsReqs()
 242         holdReqs = make(chan struct{})
 243         wg = getDDConcurrently(5, http.StatusBadGateway, check.Commentf("error at startup"))
 244         close(holdReqs)
 245         wg.Wait()
 246         c.Check(countRailsReqs(), check.Equals, reqsBefore+5)
 247
 248         // Response status is OK but body is not a discovery document
 249         wantError, wantBadContent = false, true
 250         reqsBefore = countRailsReqs()
 251         c.Check(getDD(), check.Equals, http.StatusBadGateway)
 252         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 253
 254         // Error condition clears => caller gets OK, cache is warmed
 255         // up
 256         wantError, wantBadContent = false, false
 257         reqsBefore = countRailsReqs()
 258         getDDConcurrently(5, http.StatusOK, check.Commentf("success after errors at startup")).Wait()
 259         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 260
 261         // Error with warm cache => caller gets OK (with no attempt to
 262         // re-fetch)
 263         wantError, wantBadContent = true, false
 264         reqsBefore = countRailsReqs()
 265         getDDConcurrently(5, http.StatusOK, check.Commentf("error with warm cache")).Wait()
 266         c.Check(countRailsReqs(), check.Equals, reqsBefore)
 267
 268         // Error with stale cache => caller gets OK with stale data
 269         // while the re-fetch is attempted in the background
 270         refreshNow()
 271         wantError, wantBadContent = true, false
 272         reqsBefore = countRailsReqs()
 273         holdReqs = make(chan struct{})
 274         getDDConcurrently(5, http.StatusOK, check.Commentf("error with stale cache")).Wait()
 275         close(holdReqs)
 276         // Only one attempt to re-fetch (holdReqs ensured the first
 277         // update took long enough for the last incoming request to
 278         // arrive)
 279         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 280
 281         refreshNow()
 282         wantError, wantBadContent = false, false
 283         reqsBefore = countRailsReqs()
 284         holdReqs = make(chan struct{})
 285         getDDConcurrently(5, http.StatusOK, check.Commentf("refresh cache after error condition clears")).Wait()
 286         close(holdReqs)
 287         waitPendingUpdates()
 288         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 289
 290         // Make sure expireAfter is getting set
 291         waitPendingUpdates()
 292         exp := s.handler.cache["/discovery/v1/apis/arvados/v1/rest"].expireAfter.Sub(time.Now())
 293         c.Check(exp > cacheTTL, check.Equals, true)
 294         c.Check(exp < cacheExpire, check.Equals, true)
 295
 296         // After the cache *expires* it behaves as if uninitialized:
 297         // each incoming request does a new upstream request until one
 298         // succeeds.
 299         //
 300         // First check failure after expiry:
 301         expireNow()
 302         wantError, wantBadContent = true, false
 303         reqsBefore = countRailsReqs()
 304         holdReqs = make(chan struct{})
 305         wg = getDDConcurrently(5, http.StatusBadGateway, check.Commentf("error after expiry"))
 306         close(holdReqs)
 307         wg.Wait()
 308         c.Check(countRailsReqs(), check.Equals, reqsBefore+5)
 309
 310         // Success after expiry:
 311         wantError, wantBadContent = false, false
 312         reqsBefore = countRailsReqs()
 313         holdReqs = make(chan struct{})
 314         wg = getDDConcurrently(5, http.StatusOK, check.Commentf("success after expiry"))
 315         close(holdReqs)
 316         wg.Wait()
 317         c.Check(countRailsReqs(), check.Equals, reqsBefore+1)
 318 }
 319
 320 func (s *HandlerSuite) TestVocabularyExport(c *check.C) {
 321         voc := `{
 322                 "strict_tags": false,
 323                 "tags": {
 324                         "IDTAGIMPORTANCE": {
 325                                 "strict": false,
 326                                 "labels": [{"label": "Importance"}],
 327                                 "values": {
 328                                         "HIGH": {
 329                                                 "labels": [{"label": "High"}]
 330                                         },
 331                                         "LOW": {
 332                                                 "labels": [{"label": "Low"}]
 333                                         }
 334                                 }
 335                         }
 336                 }
 337         }`
 338         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 339         c.Assert(err, check.IsNil)
 340         defer os.Remove(f.Name())
 341         _, err = f.WriteString(voc)
 342         c.Assert(err, check.IsNil)
 343         f.Close()
 344         s.cluster.API.VocabularyPath = f.Name()
 345         for _, method := range []string{"GET", "OPTIONS"} {
 346                 c.Log(c.TestName()+" ", method)
 347                 req := httptest.NewRequest(method, "/arvados/v1/vocabulary", nil)
 348                 resp := httptest.NewRecorder()
 349                 s.handler.ServeHTTP(resp, req)
 350                 c.Log(resp.Body.String())
 351                 if !c.Check(resp.Code, check.Equals, http.StatusOK) {
 352                         continue
 353                 }
 354                 c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 355                 c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 356                 c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
 357                 if method == "OPTIONS" {
 358                         c.Check(resp.Body.String(), check.HasLen, 0)
 359                         continue
 360                 }
 361                 var expectedVoc, receivedVoc *arvados.Vocabulary
 362                 err := json.Unmarshal([]byte(voc), &expectedVoc)
 363                 c.Check(err, check.IsNil)
 364                 err = json.Unmarshal(resp.Body.Bytes(), &receivedVoc)
 365                 c.Check(err, check.IsNil)
 366                 c.Check(receivedVoc, check.DeepEquals, expectedVoc)
 367         }
 368 }
 369
 370 func (s *HandlerSuite) TestVocabularyFailedCheckStatus(c *check.C) {
 371         voc := `{
 372                 "strict_tags": false,
 373                 "tags": {
 374                         "IDTAGIMPORTANCE": {
 375                                 "strict": true,
 376                                 "labels": [{"label": "Importance"}],
 377                                 "values": {
 378                                         "HIGH": {
 379                                                 "labels": [{"label": "High"}]
 380                                         },
 381                                         "LOW": {
 382                                                 "labels": [{"label": "Low"}]
 383                                         }
 384                                 }
 385                         }
 386                 }
 387         }`
 388         f, err := os.CreateTemp("", "test-vocabulary-*.json")
 389         c.Assert(err, check.IsNil)
 390         defer os.Remove(f.Name())
 391         _, err = f.WriteString(voc)
 392         c.Assert(err, check.IsNil)
 393         f.Close()
 394         s.cluster.API.VocabularyPath = f.Name()
 395
 396         req := httptest.NewRequest("POST", "/arvados/v1/collections",
 397                 strings.NewReader(`{
 398                         "collection": {
 399                                 "properties": {
 400                                         "IDTAGIMPORTANCE": "Critical"
 401                                 }
 402                         }
 403                 }`))
 404         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 405         req.Header.Set("Content-type", "application/json")
 406
 407         resp := httptest.NewRecorder()
 408         s.handler.ServeHTTP(resp, req)
 409         c.Log(resp.Body.String())
 410         c.Assert(resp.Code, check.Equals, http.StatusBadRequest)
 411         var jresp httpserver.ErrorResponse
 412         err = json.Unmarshal(resp.Body.Bytes(), &jresp)
 413         c.Check(err, check.IsNil)
 414         c.Assert(len(jresp.Errors), check.Equals, 1)
 415         c.Check(jresp.Errors[0], check.Matches, `.*tag value.*is not valid for key.*`)
 416 }
 417
 418 func (s *HandlerSuite) TestProxyDiscoveryDoc(c *check.C) {
 419         req := httptest.NewRequest("GET", "/discovery/v1/apis/arvados/v1/rest", nil)
 420         resp := httptest.NewRecorder()
 421         s.handler.ServeHTTP(resp, req)
 422         c.Check(resp.Code, check.Equals, http.StatusOK)
 423         var dd arvados.DiscoveryDocument
 424         err := json.Unmarshal(resp.Body.Bytes(), &dd)
 425         c.Check(err, check.IsNil)
 426         c.Check(dd.BlobSignatureTTL, check.Not(check.Equals), int64(0))
 427         c.Check(dd.BlobSignatureTTL > 0, check.Equals, true)
 428         c.Check(len(dd.Resources), check.Not(check.Equals), 0)
 429         c.Check(len(dd.Schemas), check.Not(check.Equals), 0)
 430 }
 431
 432 // Handler should give up and exit early if request context is
 433 // cancelled due to client hangup, httpserver.HandlerWithDeadline,
 434 // etc.
 435 func (s *HandlerSuite) TestRequestCancel(c *check.C) {
 436         ctx, cancel := context.WithCancel(context.Background())
 437         req := httptest.NewRequest("GET", "/static/login_failure", nil).WithContext(ctx)
 438         resp := httptest.NewRecorder()
 439         cancel()
 440         s.handler.ServeHTTP(resp, req)
 441         c.Check(resp.Code, check.Equals, http.StatusBadGateway)
 442         var jresp httpserver.ErrorResponse
 443         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 444         c.Check(err, check.IsNil)
 445         c.Assert(len(jresp.Errors), check.Equals, 1)
 446         c.Check(jresp.Errors[0], check.Matches, `.*context canceled`)
 447 }
 448
 449 func (s *HandlerSuite) TestProxyWithoutToken(c *check.C) {
 450         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 451         resp := httptest.NewRecorder()
 452         s.handler.ServeHTTP(resp, req)
 453         c.Check(resp.Code, check.Equals, http.StatusUnauthorized)
 454         jresp := map[string]interface{}{}
 455         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 456         c.Check(err, check.IsNil)
 457         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 458 }
 459
 460 func (s *HandlerSuite) TestProxyWithToken(c *check.C) {
 461         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 462         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 463         resp := httptest.NewRecorder()
 464         s.handler.ServeHTTP(resp, req)
 465         c.Check(resp.Code, check.Equals, http.StatusOK)
 466         var u arvados.User
 467         err := json.Unmarshal(resp.Body.Bytes(), &u)
 468         c.Check(err, check.IsNil)
 469         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 470 }
 471
 472 func (s *HandlerSuite) TestProxyWithTokenInRequestBody(c *check.C) {
 473         req := httptest.NewRequest("POST", "/arvados/v1/users/current", strings.NewReader(url.Values{
 474                 "_method":   {"GET"},
 475                 "api_token": {arvadostest.ActiveToken},
 476         }.Encode()))
 477         req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 478         resp := httptest.NewRecorder()
 479         s.handler.ServeHTTP(resp, req)
 480         c.Check(resp.Code, check.Equals, http.StatusOK)
 481         var u arvados.User
 482         err := json.Unmarshal(resp.Body.Bytes(), &u)
 483         c.Check(err, check.IsNil)
 484         c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
 485 }
 486
 487 func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
 488         req := httptest.NewRequest("GET", "/arvados/v1/xyzzy", nil)
 489         resp := httptest.NewRecorder()
 490         s.handler.ServeHTTP(resp, req)
 491         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 492         jresp := map[string]interface{}{}
 493         err := json.Unmarshal(resp.Body.Bytes(), &jresp)
 494         c.Check(err, check.IsNil)
 495         c.Check(jresp["errors"], check.FitsTypeOf, []interface{}{})
 496 }
 497
 498 func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
 499         s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "wb2.example", Path: "/"}
 500         s.cluster.Login.Google.Enable = true
 501         s.cluster.Login.Google.ClientID = "test"
 502         req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://wb2.example/", nil)
 503         resp := httptest.NewRecorder()
 504         s.handler.ServeHTTP(resp, req)
 505         if !c.Check(resp.Code, check.Equals, http.StatusFound) {
 506                 c.Log(resp.Body.String())
 507         }
 508         c.Check(resp.Header().Get("Location"), check.Equals, "https://wb2.example/")
 509 }
 510
 511 func (s *HandlerSuite) TestValidateV1APIToken(c *check.C) {
 512         c.Assert(s.handler.CheckHealth(), check.IsNil)
 513         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 514         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveToken)
 515         c.Assert(err, check.IsNil)
 516         c.Check(ok, check.Equals, true)
 517         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 518         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 519         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 520         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 521 }
 522
 523 func (s *HandlerSuite) TestValidateV2APIToken(c *check.C) {
 524         c.Assert(s.handler.CheckHealth(), check.IsNil)
 525         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 526         user, ok, err := s.handler.validateAPItoken(req, arvadostest.ActiveTokenV2)
 527         c.Assert(err, check.IsNil)
 528         c.Check(ok, check.Equals, true)
 529         c.Check(user.Authorization.UUID, check.Equals, arvadostest.ActiveTokenUUID)
 530         c.Check(user.Authorization.APIToken, check.Equals, arvadostest.ActiveToken)
 531         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 532         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 533         c.Check(user.Authorization.TokenV2(), check.Equals, arvadostest.ActiveTokenV2)
 534 }
 535
 536 func (s *HandlerSuite) TestValidateRemoteToken(c *check.C) {
 537         saltedToken, err := auth.SaltToken(arvadostest.ActiveTokenV2, "abcde")
 538         c.Assert(err, check.IsNil)
 539         for _, trial := range []struct {
 540                 code  int
 541                 token string
 542         }{
 543                 {http.StatusOK, saltedToken},
 544                 {http.StatusUnauthorized, "bogus"},
 545         } {
 546                 req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/users/current?remote=abcde", nil)
 547                 req.Header.Set("Authorization", "Bearer "+trial.token)
 548                 resp := httptest.NewRecorder()
 549                 s.handler.ServeHTTP(resp, req)
 550                 if !c.Check(resp.Code, check.Equals, trial.code) {
 551                         c.Logf("HTTP %d: %s", resp.Code, resp.Body.String())
 552                 }
 553         }
 554 }
 555
 556 func (s *HandlerSuite) TestLogTokenUUID(c *check.C) {
 557         req := httptest.NewRequest("GET", "https://0.0.0.0/arvados/v1/users/current", nil)
 558         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 559         req = req.WithContext(s.ctx)
 560         resp := httptest.NewRecorder()
 561         httpserver.LogRequests(s.handler).ServeHTTP(resp, req)
 562         c.Check(resp.Code, check.Equals, http.StatusOK)
 563         c.Check(s.logbuf.String(), check.Matches, `(?ms).*"tokenUUIDs":\["`+strings.Split(arvadostest.ActiveTokenV2, "/")[1]+`"\].*`)
 564 }
 565
 566 func (s *HandlerSuite) TestCreateAPIToken(c *check.C) {
 567         c.Assert(s.handler.CheckHealth(), check.IsNil)
 568         req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
 569         auth, err := s.handler.createAPItoken(req, arvadostest.ActiveUserUUID, nil)
 570         c.Assert(err, check.IsNil)
 571         c.Check(auth.Scopes, check.DeepEquals, []string{"all"})
 572
 573         user, ok, err := s.handler.validateAPItoken(req, auth.TokenV2())
 574         c.Assert(err, check.IsNil)
 575         c.Check(ok, check.Equals, true)
 576         c.Check(user.Authorization.UUID, check.Equals, auth.UUID)
 577         c.Check(user.Authorization.APIToken, check.Equals, auth.APIToken)
 578         c.Check(user.Authorization.Scopes, check.DeepEquals, []string{"all"})
 579         c.Check(user.UUID, check.Equals, arvadostest.ActiveUserUUID)
 580         c.Check(user.Authorization.TokenV2(), check.Equals, auth.TokenV2())
 581 }
 582
 583 func (s *HandlerSuite) CheckObjectType(c *check.C, url string, token string, skippedFields map[string]bool) {
 584         var proxied, direct map[string]interface{}
 585         var err error
 586
 587         // Get collection from controller
 588         req := httptest.NewRequest("GET", url, nil)
 589         req.Header.Set("Authorization", "Bearer "+token)
 590         resp := httptest.NewRecorder()
 591         s.handler.ServeHTTP(resp, req)
 592         if !c.Check(resp.Code, check.Equals, http.StatusOK,
 593                 check.Commentf("Wasn't able to get data from the controller at %q: %q", url, resp.Body.String())) {
 594                 return
 595         }
 596         err = json.Unmarshal(resp.Body.Bytes(), &proxied)
 597         c.Check(err, check.Equals, nil)
 598
 599         // Get collection directly from RailsAPI
 600         client := &http.Client{
 601                 Transport: &http.Transport{
 602                         TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 603                 },
 604         }
 605         resp2, err := client.Get(s.cluster.Services.RailsAPI.ExternalURL.String() + url + "/?api_token=" + token)
 606         c.Check(err, check.Equals, nil)
 607         defer resp2.Body.Close()
 608         if !c.Check(resp2.StatusCode, check.Equals, http.StatusOK,
 609                 check.Commentf("Wasn't able to get data from the RailsAPI at %q", url)) {
 610                 return
 611         }
 612         db, err := ioutil.ReadAll(resp2.Body)
 613         c.Check(err, check.Equals, nil)
 614         err = json.Unmarshal(db, &direct)
 615         c.Check(err, check.Equals, nil)
 616
 617         // Check that all RailsAPI provided keys exist on the controller response.
 618         for k := range direct {
 619                 if _, ok := skippedFields[k]; ok {
 620                         continue
 621                 } else if val, ok := proxied[k]; !ok {
 622                         c.Errorf("%s's key %q missing on controller's response.", direct["kind"], k)
 623                 } else if direct["kind"] == "arvados#collection" && k == "manifest_text" {
 624                         // Tokens differ from request to request
 625                         c.Check(strings.Split(val.(string), "+A")[0], check.Equals, strings.Split(direct[k].(string), "+A")[0])
 626                 } else {
 627                         c.Check(val, check.DeepEquals, direct[k],
 628                                 check.Commentf("RailsAPI %s key %q's value %q differs from controller's %q.", direct["kind"], k, direct[k], val))
 629                 }
 630         }
 631 }
 632
 633 func (s *HandlerSuite) TestGetObjects(c *check.C) {
 634         // Get the 1st keep service's uuid from the running test server.
 635         req := httptest.NewRequest("GET", "/arvados/v1/keep_services/", nil)
 636         req.Header.Set("Authorization", "Bearer "+arvadostest.AdminToken)
 637         resp := httptest.NewRecorder()
 638         s.handler.ServeHTTP(resp, req)
 639         c.Assert(resp.Code, check.Equals, http.StatusOK)
 640         var ksList arvados.KeepServiceList
 641         json.Unmarshal(resp.Body.Bytes(), &ksList)
 642         c.Assert(len(ksList.Items), check.Not(check.Equals), 0)
 643         ksUUID := ksList.Items[0].UUID
 644         // Create a new token for the test user so that we're not comparing
 645         // the ones from the fixtures.
 646         req = httptest.NewRequest("POST", "/arvados/v1/api_client_authorizations",
 647                 strings.NewReader(`{
 648                         "api_client_authorization": {
 649                                 "owner_uuid": "`+arvadostest.AdminUserUUID+`",
 650                                 "created_by_ip_address": "::1",
 651                                 "last_used_by_ip_address": "::1"
 652                         }
 653                 }`))
 654         req.Header.Set("Authorization", "Bearer "+arvadostest.SystemRootToken)
 655         req.Header.Set("Content-type", "application/json")
 656         resp = httptest.NewRecorder()
 657         s.handler.ServeHTTP(resp, req)
 658         c.Assert(resp.Code, check.Equals, http.StatusOK,
 659                 check.Commentf("%s", resp.Body.String()))
 660         var auth arvados.APIClientAuthorization
 661         json.Unmarshal(resp.Body.Bytes(), &auth)
 662         c.Assert(auth.UUID, check.Not(check.Equals), "")
 663
 664         testCases := map[string]map[string]bool{
 665                 "api_clients/" + arvadostest.TrustedWorkbenchAPIClientUUID:     nil,
 666                 "api_client_authorizations/" + auth.UUID:                       {"href": true, "modified_by_client_uuid": true, "modified_by_user_uuid": true},
 667                 "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID:       {"href": true},
 668                 "collections/" + arvadostest.CollectionWithUniqueWordsUUID:     {"href": true},
 669                 "containers/" + arvadostest.RunningContainerUUID:               nil,
 670                 "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil,
 671                 "groups/" + arvadostest.AProjectUUID:                           nil,
 672                 "keep_services/" + ksUUID:                                      nil,
 673                 "links/" + arvadostest.ActiveUserCanReadAllUsersLinkUUID:       nil,
 674                 "logs/" + arvadostest.CrunchstatForRunningContainerLogUUID:     nil,
 675                 "users/" + arvadostest.ActiveUserUUID:                          {"href": true},
 676                 "virtual_machines/" + arvadostest.TestVMUUID:                   nil,
 677                 "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID:      nil,
 678         }
 679         for url, skippedFields := range testCases {
 680                 c.Logf("Testing %q", url)
 681                 s.CheckObjectType(c, "/arvados/v1/"+url, auth.TokenV2(), skippedFields)
 682         }
 683 }
 684
 685 func (s *HandlerSuite) TestRedactRailsAPIHostFromErrors(c *check.C) {
 686         req := httptest.NewRequest("GET", "https://0.0.0.0:1/arvados/v1/collections/zzzzz-4zz18-abcdefghijklmno", nil)
 687         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 688         resp := httptest.NewRecorder()
 689         s.handler.ServeHTTP(resp, req)
 690         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 691         var jresp struct {
 692                 Errors []string
 693         }
 694         c.Log(resp.Body.String())
 695         c.Assert(json.NewDecoder(resp.Body).Decode(&jresp), check.IsNil)
 696         c.Assert(jresp.Errors, check.HasLen, 1)
 697         c.Check(jresp.Errors[0], check.Matches, `.*//railsapi\.internal/arvados/v1/collections/.*: 404 Not Found.*`)
 698         c.Check(jresp.Errors[0], check.Not(check.Matches), `(?ms).*127.0.0.1.*`)
 699 }
 700
 701 func (s *HandlerSuite) TestTrashSweep(c *check.C) {
 702         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 703         s.cluster.Collections.TrashSweepInterval = arvados.Duration(time.Second / 10)
 704         s.handler.CheckHealth()
 705         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 706         coll, err := s.handler.federation.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{"name": "test trash sweep"}, EnsureUniqueName: true})
 707         c.Assert(err, check.IsNil)
 708         defer s.handler.federation.CollectionDelete(ctx, arvados.DeleteOptions{UUID: coll.UUID})
 709         db, err := s.handler.dbConnector.GetDB(s.ctx)
 710         c.Assert(err, check.IsNil)
 711         _, err = db.ExecContext(s.ctx, `update collections set trash_at = $1, delete_at = $2 where uuid = $3`, time.Now().UTC().Add(time.Second/10), time.Now().UTC().Add(time.Hour), coll.UUID)
 712         c.Assert(err, check.IsNil)
 713         deadline := time.Now().Add(5 * time.Second)
 714         for {
 715                 if time.Now().After(deadline) {
 716                         c.Log("timed out")
 717                         c.FailNow()
 718                 }
 719                 updated, err := s.handler.federation.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID, IncludeTrash: true})
 720                 c.Assert(err, check.IsNil)
 721                 if updated.IsTrashed {
 722                         break
 723                 }
 724                 time.Sleep(time.Second / 10)
 725         }
 726 }
 727
 728 func (s *HandlerSuite) TestContainerLogSweep(c *check.C) {
 729         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 730         s.cluster.Collections.TrashSweepInterval = arvados.Duration(2 * time.Second)
 731         s.handler.CheckHealth()
 732         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 733         logentry, err := s.handler.federation.LogCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
 734                 "object_uuid": arvadostest.CompletedContainerUUID,
 735                 "event_type":  "stderr",
 736                 "properties": map[string]interface{}{
 737                         "text": "test container log sweep\n",
 738                 },
 739         }})
 740         c.Assert(err, check.IsNil)
 741         defer s.handler.federation.LogDelete(ctx, arvados.DeleteOptions{UUID: logentry.UUID})
 742         deadline := time.Now().Add(5 * time.Second)
 743         for {
 744                 if time.Now().After(deadline) {
 745                         c.Log("timed out")
 746                         c.FailNow()
 747                 }
 748                 logentries, err := s.handler.federation.LogList(ctx, arvados.ListOptions{Filters: []arvados.Filter{{"uuid", "=", logentry.UUID}}, Limit: -1})
 749                 c.Assert(err, check.IsNil)
 750                 if len(logentries.Items) == 0 {
 751                         break
 752                 }
 753                 time.Sleep(time.Second / 10)
 754         }
 755 }
 756
 757 func (s *HandlerSuite) TestLogActivity(c *check.C) {
 758         s.cluster.SystemRootToken = arvadostest.SystemRootToken
 759         s.cluster.Users.ActivityLoggingPeriod = arvados.Duration(24 * time.Hour)
 760         s.handler.CheckHealth()
 761
 762         testServer := newServerFromIntegrationTestEnv(c)
 763         testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.handler))
 764         c.Assert(testServer.Start(), check.IsNil)
 765         defer testServer.Close()
 766
 767         u, _ := url.Parse("http://" + testServer.Addr)
 768         client := rpc.NewConn(s.cluster.ClusterID, u, true, rpc.PassthroughTokenProvider)
 769
 770         starttime := time.Now()
 771         for i := 0; i < 4; i++ {
 772                 for _, token := range []string{
 773                         arvadostest.ActiveTokenV2,
 774                         arvadostest.ActiveToken,
 775                         arvadostest.SpectatorToken,
 776                 } {
 777                         ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{token}})
 778                         _, err := client.CollectionList(ctx, arvados.ListOptions{})
 779                         c.Assert(err, check.IsNil)
 780                 }
 781         }
 782         db, err := s.handler.dbConnector.GetDB(s.ctx)
 783         c.Assert(err, check.IsNil)
 784         for _, userUUID := range []string{arvadostest.ActiveUserUUID, arvadostest.SpectatorUserUUID} {
 785                 var rows int
 786                 err = db.QueryRowContext(s.ctx, `select count(uuid) from logs where object_uuid = $1 and event_at > $2`, arvadostest.ActiveUserUUID, starttime.UTC()).Scan(&rows)
 787                 c.Assert(err, check.IsNil)
 788                 c.Check(rows, check.Equals, 1, check.Commentf("expect 1 row for user uuid %s", userUUID))
 789         }
 790 }