lib/controller/integration_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "database/sql"
  11         "encoding/json"
  12         "fmt"
  13         "io"
  14         "io/fs"
  15         "io/ioutil"
  16         "math"
  17         "net"
  18         "net/http"
  19         "os"
  20         "os/exec"
  21         "strconv"
  22         "strings"
  23         "sync"
  24         "sync/atomic"
  25         "time"
  26
  27         "git.arvados.org/arvados.git/lib/boot"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  30         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  31         "git.arvados.org/arvados.git/sdk/go/httpserver"
  32         "git.arvados.org/arvados.git/sdk/go/keepclient"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&IntegrationSuite{})
  37
  38 type IntegrationSuite struct {
  39         super        *boot.Supervisor
  40         oidcprovider *arvadostest.OIDCProvider
  41 }
  42
  43 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
  44         if output, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
  45                 // See TestContainerHTTPProxy for details about why we
  46                 // depend on docker instead of singularity.
  47                 c.Fatalf("this test suite depends on docker, which does not appear to be working. `docker info` reports:\n\n%s", output)
  48         }
  49         s.oidcprovider = arvadostest.NewOIDCProvider(c)
  50         s.oidcprovider.AuthEmail = "user@example.com"
  51         s.oidcprovider.AuthEmailVerified = true
  52         s.oidcprovider.AuthName = "Example User"
  53         s.oidcprovider.ValidClientID = "clientid"
  54         s.oidcprovider.ValidClientSecret = "clientsecret"
  55
  56         hostport := map[string]string{}
  57         for _, id := range []string{"z1111", "z2222", "z3333"} {
  58                 hostport[id] = func() string {
  59                         // TODO: Instead of expecting random ports on
  60                         // 127.0.0.11, 22, 33 to be race-safe, try
  61                         // different 127.x.y.z until finding one that
  62                         // isn't in use.
  63                         ln, err := net.Listen("tcp", ":0")
  64                         c.Assert(err, check.IsNil)
  65                         ln.Close()
  66                         _, port, err := net.SplitHostPort(ln.Addr().String())
  67                         c.Assert(err, check.IsNil)
  68                         return "127.0.0." + id[3:] + ":" + port
  69                 }()
  70         }
  71         yaml := "Clusters:\n"
  72         for id := range hostport {
  73                 yaml += `
  74   ` + id + `:
  75     Services:
  76       Controller:
  77         ExternalURL: https://` + hostport[id] + `
  78       ContainerWebServices:
  79         ExternalURL: https://` + hostport[id] + `
  80         ExternalPortMin: 2000
  81         ExternalPortMax: 2099
  82     TLS:
  83       Insecure: true
  84     SystemLogs:
  85       Format: text
  86     API:
  87       MaxConcurrentRequests: 128
  88     Containers:
  89       CloudVMs:
  90         Enable: true
  91         Driver: loopback
  92         BootProbeCommand: "rm -f /var/lock/crunch-run-broken"
  93         ProbeInterval: 1s
  94         PollInterval: 5s
  95         SyncInterval: 10s
  96         TimeoutIdle: 1s
  97         TimeoutBooting: 2s
  98       # Note RuntimeEngine: singularity would almost work, except see
  99       # comment on TestContainerHTTPProxy()
 100       RuntimeEngine: docker
 101       CrunchRunArgumentsList: ["--broken-node-hook", "true"]
 102     RemoteClusters:
 103       z1111:
 104         Host: ` + hostport["z1111"] + `
 105         Scheme: https
 106         Insecure: true
 107         Proxy: true
 108         ActivateUsers: true
 109 `
 110                 if id != "z2222" {
 111                         yaml += `      z2222:
 112         Host: ` + hostport["z2222"] + `
 113         Scheme: https
 114         Insecure: true
 115         Proxy: true
 116         ActivateUsers: true
 117 `
 118                 }
 119                 if id != "z3333" {
 120                         yaml += `      z3333:
 121         Host: ` + hostport["z3333"] + `
 122         Scheme: https
 123         Insecure: true
 124         Proxy: true
 125         ActivateUsers: true
 126 `
 127                 }
 128                 if id == "z1111" {
 129                         yaml += `
 130     Login:
 131       LoginCluster: z1111
 132       OpenIDConnect:
 133         Enable: true
 134         Issuer: ` + s.oidcprovider.Issuer.URL + `
 135         ClientID: ` + s.oidcprovider.ValidClientID + `
 136         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
 137         EmailClaim: email
 138         EmailVerifiedClaim: email_verified
 139         AcceptAccessToken: true
 140         AcceptAccessTokenScope: ""
 141 `
 142                 } else {
 143                         yaml += `
 144     Login:
 145       LoginCluster: z1111
 146 `
 147                 }
 148         }
 149         s.super = &boot.Supervisor{
 150                 ClusterType:          "test",
 151                 ConfigYAML:           yaml,
 152                 Stderr:               ctxlog.LogWriter(c.Log),
 153                 NoWorkbench1:         true,
 154                 NoWorkbench2:         true,
 155                 OwnTemporaryDatabase: true,
 156         }
 157
 158         // Give up if startup takes longer than 3m
 159         timeout := time.AfterFunc(3*time.Minute, s.super.Stop)
 160         defer timeout.Stop()
 161         s.super.Start(context.Background())
 162         ok := s.super.WaitReady()
 163         c.Assert(ok, check.Equals, true)
 164 }
 165
 166 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 167         if s.super != nil {
 168                 s.super.Stop()
 169                 s.super.Wait()
 170         }
 171 }
 172
 173 func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) {
 174         conn := s.super.Conn("z1111")
 175         rootctx, _, _ := s.super.RootClients("z1111")
 176         userctx, _, kc, _ := s.super.UserClients("z1111", rootctx, c, conn, s.oidcprovider.AuthEmail, true)
 177         c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true)
 178         coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{})
 179         c.Assert(err, check.IsNil)
 180         c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses)
 181 }
 182
 183 func (s *IntegrationSuite) createTestCollectionManifest(c *check.C, ac *arvados.Client, kc *keepclient.KeepClient, content string) string {
 184         fs, err := (&arvados.Collection{}).FileSystem(ac, kc)
 185         c.Assert(err, check.IsNil)
 186         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
 187         c.Assert(err, check.IsNil)
 188         _, err = io.WriteString(f, content)
 189         c.Assert(err, check.IsNil)
 190         err = f.Close()
 191         c.Assert(err, check.IsNil)
 192         mtxt, err := fs.MarshalManifest(".")
 193         c.Assert(err, check.IsNil)
 194         return mtxt
 195 }
 196
 197 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
 198         conn1 := s.super.Conn("z1111")
 199         rootctx1, _, _ := s.super.RootClients("z1111")
 200         conn3 := s.super.Conn("z3333")
 201         userctx1, ac1, kc1, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 202
 203         // Create the collection to find its PDH (but don't save it
 204         // anywhere yet)
 205         mtxt := s.createTestCollectionManifest(c, ac1, kc1, c.TestName())
 206         pdh := arvados.PortableDataHash(mtxt)
 207
 208         // Looking up the PDH before saving returns 404 if cycle
 209         // detection is working.
 210         _, err := conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
 211         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
 212
 213         // Save the collection on cluster z1111.
 214         _, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
 215                 "manifest_text": mtxt,
 216         }})
 217         c.Assert(err, check.IsNil)
 218
 219         // Retrieve the collection from cluster z3333.
 220         coll2, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
 221         c.Check(err, check.IsNil)
 222         c.Check(coll2.PortableDataHash, check.Equals, pdh)
 223 }
 224
 225 func (s *IntegrationSuite) TestFederation_Write1Read2(c *check.C) {
 226         s.testFederationCollectionAccess(c, "z1111", "z2222")
 227 }
 228
 229 func (s *IntegrationSuite) TestFederation_Write2Read1(c *check.C) {
 230         s.testFederationCollectionAccess(c, "z2222", "z1111")
 231 }
 232
 233 func (s *IntegrationSuite) TestFederation_Write2Read3(c *check.C) {
 234         s.testFederationCollectionAccess(c, "z2222", "z3333")
 235 }
 236
 237 func (s *IntegrationSuite) testFederationCollectionAccess(c *check.C, writeCluster, readCluster string) {
 238         conn1 := s.super.Conn("z1111")
 239         rootctx1, _, _ := s.super.RootClients("z1111")
 240         _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 241
 242         connW := s.super.Conn(writeCluster)
 243         userctxW, acW, kcW := s.super.ClientsWithToken(writeCluster, ac1.AuthToken)
 244         kcW.DiskCacheSize = keepclient.DiskCacheDisabled
 245         connR := s.super.Conn(readCluster)
 246         userctxR, acR, kcR := s.super.ClientsWithToken(readCluster, ac1.AuthToken)
 247         kcR.DiskCacheSize = keepclient.DiskCacheDisabled
 248
 249         filedata := fmt.Sprintf("%s: write to %s, read from %s", c.TestName(), writeCluster, readCluster)
 250         mtxt := s.createTestCollectionManifest(c, acW, kcW, filedata)
 251         collW, err := connW.CollectionCreate(userctxW, arvados.CreateOptions{Attrs: map[string]interface{}{
 252                 "manifest_text": mtxt,
 253         }})
 254         c.Assert(err, check.IsNil)
 255
 256         collR, err := connR.CollectionGet(userctxR, arvados.GetOptions{UUID: collW.UUID})
 257         if !c.Check(err, check.IsNil) {
 258                 return
 259         }
 260         fsR, err := collR.FileSystem(acR, kcR)
 261         if !c.Check(err, check.IsNil) {
 262                 return
 263         }
 264         buf, err := fs.ReadFile(arvados.FS(fsR), "test.txt")
 265         if !c.Check(err, check.IsNil) {
 266                 return
 267         }
 268         c.Check(string(buf), check.Equals, filedata)
 269 }
 270
 271 // Tests bug #18004
 272 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) {
 273         conn1 := s.super.Conn("z1111")
 274         rootctx1, _, _ := s.super.RootClients("z1111")
 275         rootctx2, _, _ := s.super.RootClients("z2222")
 276         conn2 := s.super.Conn("z2222")
 277         userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user2@example.com", true)
 278
 279         var wg1, wg2 sync.WaitGroup
 280         creqs := 100
 281
 282         // Make concurrent requests to z2222 with a local token to make sure more
 283         // than one worker is listening.
 284         wg1.Add(1)
 285         for i := 0; i < creqs; i++ {
 286                 wg2.Add(1)
 287                 go func() {
 288                         defer wg2.Done()
 289                         wg1.Wait()
 290                         _, err := conn2.UserGetCurrent(rootctx2, arvados.GetOptions{})
 291                         c.Check(err, check.IsNil, check.Commentf("warm up phase failed"))
 292                 }()
 293         }
 294         wg1.Done()
 295         wg2.Wait()
 296
 297         // Real test pass -- use a new remote token than the one used in the warm-up
 298         // phase.
 299         wg1.Add(1)
 300         for i := 0; i < creqs; i++ {
 301                 wg2.Add(1)
 302                 go func() {
 303                         defer wg2.Done()
 304                         wg1.Wait()
 305                         // Retrieve the remote collection from cluster z2222.
 306                         _, err := conn2.UserGetCurrent(userctx1, arvados.GetOptions{})
 307                         c.Check(err, check.IsNil, check.Commentf("testing phase failed"))
 308                 }()
 309         }
 310         wg1.Done()
 311         wg2.Wait()
 312 }
 313
 314 // After using a token issued by z1111 to call the Logout endpoint on
 315 // z2222, the token should be expired and rejected by both z1111 and
 316 // z2222.
 317 func (s *IntegrationSuite) TestLogoutUsingLoginCluster(c *check.C) {
 318         conn1 := s.super.Conn("z1111")
 319         conn2 := s.super.Conn("z2222")
 320         rootctx1, _, _ := s.super.RootClients("z1111")
 321         _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user1@example.com", true)
 322         userctx2, ac2, _ := s.super.ClientsWithToken("z2222", ac1.AuthToken)
 323         c.Assert(ac2.AuthToken, check.Matches, `^v2/z1111-.*`)
 324         _, err := conn1.CollectionCreate(userctx2, arvados.CreateOptions{})
 325         c.Assert(err, check.IsNil)
 326         _, err = conn2.CollectionCreate(userctx2, arvados.CreateOptions{})
 327         c.Assert(err, check.IsNil)
 328
 329         _, err = conn2.Logout(userctx2, arvados.LogoutOptions{})
 330         c.Assert(err, check.IsNil)
 331
 332         _, err = conn1.CollectionCreate(userctx2, arvados.CreateOptions{})
 333         se, ok := err.(httpserver.HTTPStatusError)
 334         if c.Check(ok, check.Equals, true, check.Commentf("after logging out, token should have been rejected by login cluster")) {
 335                 c.Check(se.HTTPStatus(), check.Equals, 401)
 336         }
 337
 338         _, err = conn2.CollectionCreate(userctx2, arvados.CreateOptions{})
 339         se, ok = err.(httpserver.HTTPStatusError)
 340         if c.Check(ok, check.Equals, true, check.Commentf("after logging out, token should have been rejected by remote cluster")) {
 341                 c.Check(se.HTTPStatus(), check.Equals, 401)
 342         }
 343
 344 }
 345
 346 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
 347         if _, err := exec.LookPath("s3cmd"); err != nil {
 348                 c.Skip("s3cmd not in PATH")
 349                 return
 350         }
 351
 352         testText := "IntegrationSuite.TestS3WithFederatedToken"
 353
 354         conn1 := s.super.Conn("z1111")
 355         rootctx1, _, _ := s.super.RootClients("z1111")
 356         userctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 357         conn3 := s.super.Conn("z3333")
 358
 359         createColl := func(clusterID string) arvados.Collection {
 360                 _, ac, kc := s.super.ClientsWithToken(clusterID, ac1.AuthToken)
 361                 var coll arvados.Collection
 362                 fs, err := coll.FileSystem(ac, kc)
 363                 c.Assert(err, check.IsNil)
 364                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
 365                 c.Assert(err, check.IsNil)
 366                 _, err = io.WriteString(f, testText)
 367                 c.Assert(err, check.IsNil)
 368                 err = f.Close()
 369                 c.Assert(err, check.IsNil)
 370                 mtxt, err := fs.MarshalManifest(".")
 371                 c.Assert(err, check.IsNil)
 372                 coll, err = s.super.Conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
 373                         "manifest_text": mtxt,
 374                 }})
 375                 c.Assert(err, check.IsNil)
 376                 return coll
 377         }
 378
 379         for _, trial := range []struct {
 380                 clusterID string // create the collection on this cluster (then use z3333 to access it)
 381                 token     string
 382         }{
 383                 // Try the hardest test first: z3333 hasn't seen
 384                 // z1111's token yet, and we're just passing the
 385                 // opaque secret part, so z3333 has to guess that it
 386                 // belongs to z1111.
 387                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
 388                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
 389                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
 390                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
 391         } {
 392                 c.Logf("================ %v", trial)
 393                 coll := createColl(trial.clusterID)
 394
 395                 cfgjson, err := conn3.ConfigGet(userctx1)
 396                 c.Assert(err, check.IsNil)
 397                 var cluster arvados.Cluster
 398                 err = json.Unmarshal(cfgjson, &cluster)
 399                 c.Assert(err, check.IsNil)
 400
 401                 c.Logf("TokenV2 is %s", ac1.AuthToken)
 402                 host := cluster.Services.WebDAV.ExternalURL.Host
 403                 s3args := []string{
 404                         "--ssl", "--no-check-certificate",
 405                         "--host=" + host, "--host-bucket=" + host,
 406                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
 407                 }
 408                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
 409                 c.Check(err, check.IsNil)
 410                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
 411
 412                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
 413                 // Command fails because we don't return Etag header.
 414                 flen := strconv.Itoa(len(testText))
 415                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
 416         }
 417 }
 418
 419 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
 420         conn1 := s.super.Conn("z1111")
 421         conn3 := s.super.Conn("z3333")
 422         rootctx1, rootac1, rootkc1 := s.super.RootClients("z1111")
 423         anonctx3, anonac3, _ := s.super.AnonymousClients("z3333")
 424
 425         // Make sure anonymous token was set
 426         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
 427
 428         // Create the collection to find its PDH (but don't save it
 429         // anywhere yet)
 430         var coll1 arvados.Collection
 431         fs1, err := coll1.FileSystem(rootac1, rootkc1)
 432         c.Assert(err, check.IsNil)
 433         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
 434         c.Assert(err, check.IsNil)
 435         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
 436         c.Assert(err, check.IsNil)
 437         err = f.Close()
 438         c.Assert(err, check.IsNil)
 439         mtxt, err := fs1.MarshalManifest(".")
 440         c.Assert(err, check.IsNil)
 441         pdh := arvados.PortableDataHash(mtxt)
 442
 443         // Save the collection on cluster z1111.
 444         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
 445                 "manifest_text": mtxt,
 446         }})
 447         c.Assert(err, check.IsNil)
 448
 449         // Share it with the anonymous users group.
 450         var outLink arvados.Link
 451         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
 452                 map[string]interface{}{"link": map[string]interface{}{
 453                         "link_class": "permission",
 454                         "name":       "can_read",
 455                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
 456                         "head_uuid":  coll1.UUID,
 457                 },
 458                 })
 459         c.Check(err, check.IsNil)
 460
 461         // Current user should be z3 anonymous user
 462         outUser, err := anonac3.CurrentUser()
 463         c.Check(err, check.IsNil)
 464         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
 465
 466         // Get the token uuid
 467         var outAuth arvados.APIClientAuthorization
 468         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
 469         c.Check(err, check.IsNil)
 470
 471         // Make a v2 token of the z3 anonymous user, and use it on z1
 472         _, anonac1, _ := s.super.ClientsWithToken("z1111", outAuth.TokenV2())
 473         outUser2, err := anonac1.CurrentUser()
 474         c.Check(err, check.IsNil)
 475         // z3 anonymous user will be mapped to the z1 anonymous user
 476         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
 477
 478         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
 479         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
 480         c.Check(err, check.IsNil)
 481         c.Check(coll.PortableDataHash, check.Equals, pdh)
 482 }
 483
 484 // z3333 should forward the locally-issued anonymous user token to its login
 485 // cluster z1111. That is no problem because the login cluster controller will
 486 // map any anonymous user token to its local anonymous user.
 487 //
 488 // This needs to work because wb1 has a tendency to slap the local anonymous
 489 // user token on every request as a reader_token, which gets folded into the
 490 // request token list controller.
 491 //
 492 // Use a z1111 user token and the anonymous token from z3333 passed in as a
 493 // reader_token to do a request on z3333, asking for the z1111 anonymous user
 494 // object. The request will be forwarded to the z1111 cluster. The presence of
 495 // the z3333 anonymous user token should not prohibit the request from being
 496 // forwarded.
 497 func (s *IntegrationSuite) TestForwardAnonymousTokenToLoginCluster(c *check.C) {
 498         conn1 := s.super.Conn("z1111")
 499
 500         rootctx1, _, _ := s.super.RootClients("z1111")
 501         _, anonac3, _ := s.super.AnonymousClients("z3333")
 502
 503         // Make a user connection to z3333 (using a z1111 user, because that's the login cluster)
 504         _, userac1, _, _ := s.super.UserClients("z3333", rootctx1, c, conn1, "user@example.com", true)
 505
 506         // Get the anonymous user token for z3333
 507         var anon3Auth arvados.APIClientAuthorization
 508         err := anonac3.RequestAndDecode(&anon3Auth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
 509         c.Check(err, check.IsNil)
 510
 511         var userList arvados.UserList
 512         where := make(map[string]string)
 513         where["uuid"] = "z1111-tpzed-anonymouspublic"
 514         err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
 515                 map[string]interface{}{
 516                         "reader_tokens": []string{anon3Auth.TokenV2()},
 517                         "where":         where,
 518                 },
 519         )
 520         // The local z3333 anonymous token must be allowed to be forwarded to the login cluster
 521         c.Check(err, check.IsNil)
 522
 523         userac1.AuthToken = "v2/z1111-gj3su-asdfasdfasdfasd/this-token-does-not-validate-so-anonymous-token-will-be-used-instead"
 524         err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
 525                 map[string]interface{}{
 526                         "reader_tokens": []string{anon3Auth.TokenV2()},
 527                         "where":         where,
 528                 },
 529         )
 530         c.Check(err, check.IsNil)
 531 }
 532
 533 // Get a token from the login cluster (z1111), use it to submit a
 534 // container request on z2222.
 535 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
 536         conn1 := s.super.Conn("z1111")
 537         rootctx1, _, _ := s.super.RootClients("z1111")
 538         _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 539
 540         // Use ac2 to get the discovery doc with a blank token, so the
 541         // SDK doesn't magically pass the z1111 token to z2222 before
 542         // we're ready to start our test.
 543         _, ac2, _ := s.super.ClientsWithToken("z2222", "")
 544         var dd map[string]interface{}
 545         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
 546         c.Assert(err, check.IsNil)
 547
 548         var (
 549                 body bytes.Buffer
 550                 req  *http.Request
 551                 resp *http.Response
 552                 u    arvados.User
 553                 cr   arvados.ContainerRequest
 554         )
 555         json.NewEncoder(&body).Encode(map[string]interface{}{
 556                 "container_request": map[string]interface{}{
 557                         "command":         []string{"echo"},
 558                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 559                         "cwd":             "/",
 560                         "output_path":     "/",
 561                 },
 562         })
 563         ac2.AuthToken = ac1.AuthToken
 564
 565         c.Log("...post CR with good (but not yet cached) token")
 566         cr = arvados.ContainerRequest{}
 567         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
 568         c.Assert(err, check.IsNil)
 569         req.Header.Set("Content-Type", "application/json")
 570         err = ac2.DoAndDecode(&cr, req)
 571         c.Assert(err, check.IsNil)
 572         c.Logf("err == %#v", err)
 573
 574         c.Log("...get user with good token")
 575         u = arvados.User{}
 576         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
 577         c.Assert(err, check.IsNil)
 578         err = ac2.DoAndDecode(&u, req)
 579         c.Check(err, check.IsNil)
 580         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
 581
 582         c.Log("...post CR with good cached token")
 583         cr = arvados.ContainerRequest{}
 584         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
 585         c.Assert(err, check.IsNil)
 586         req.Header.Set("Content-Type", "application/json")
 587         err = ac2.DoAndDecode(&cr, req)
 588         c.Check(err, check.IsNil)
 589         c.Check(cr.UUID, check.Matches, "z2222-.*")
 590
 591         c.Log("...post with good cached token ('Bearer ...')")
 592         cr = arvados.ContainerRequest{}
 593         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
 594         c.Assert(err, check.IsNil)
 595         req.Header.Set("Content-Type", "application/json")
 596         req.Header.Set("Authorization", "Bearer "+ac2.AuthToken)
 597         resp, err = arvados.InsecureHTTPClient.Do(req)
 598         c.Assert(err, check.IsNil)
 599         defer resp.Body.Close()
 600         err = json.NewDecoder(resp.Body).Decode(&cr)
 601         c.Check(err, check.IsNil)
 602         c.Check(cr.UUID, check.Matches, "z2222-.*")
 603 }
 604
 605 func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) {
 606         conn1 := s.super.Conn("z1111")
 607         rootctx1, _, _ := s.super.RootClients("z1111")
 608         _, ac1, _, au := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
 609
 610         tests := []struct {
 611                 name         string
 612                 token        string
 613                 expectedCode int
 614         }{
 615                 {"Good token", ac1.AuthToken, http.StatusOK},
 616                 {"Bogus token", "abcdef", http.StatusUnauthorized},
 617                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
 618                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
 619         }
 620
 621         body, _ := json.Marshal(map[string]interface{}{
 622                 "container_request": map[string]interface{}{
 623                         "command":         []string{"echo"},
 624                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 625                         "cwd":             "/",
 626                         "output_path":     "/",
 627                 },
 628         })
 629
 630         for _, tt := range tests {
 631                 c.Log(c.TestName() + " " + tt.name)
 632                 ac1.AuthToken = tt.token
 633                 req, err := http.NewRequest("POST", "https://"+ac1.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body))
 634                 c.Assert(err, check.IsNil)
 635                 req.Header.Set("Content-Type", "application/json")
 636                 resp, err := ac1.Do(req)
 637                 if c.Check(err, check.IsNil) {
 638                         c.Assert(resp.StatusCode, check.Equals, tt.expectedCode)
 639                         resp.Body.Close()
 640                 }
 641         }
 642 }
 643
 644 func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
 645         conn1 := s.super.Conn("z1111")
 646         rootctx1, _, _ := s.super.RootClients("z1111")
 647         userctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
 648
 649         coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
 650         c.Check(err, check.IsNil)
 651
 652         tests := []struct {
 653                 path            string
 654                 reqIdProvided   bool
 655                 notFoundRequest bool
 656         }{
 657                 {"/arvados/v1/collections", false, false},
 658                 {"/arvados/v1/collections", true, false},
 659                 {"/arvados/v1/nonexistant", false, true},
 660                 {"/arvados/v1/nonexistant", true, true},
 661                 {"/arvados/v1/collections/" + coll.UUID, false, false},
 662                 {"/arvados/v1/collections/" + coll.UUID, true, false},
 663                 // new code path (lib/controller/router etc) - single-cluster request
 664                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
 665                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
 666                 // new code path (lib/controller/router etc) - federated request
 667                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", false, true},
 668                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", true, true},
 669                 // old code path (proxyRailsAPI) - single-cluster request
 670                 {"/arvados/v1/containers/z1111-dz642-0123456789abcde", false, true},
 671                 {"/arvados/v1/containers/z1111-dz642-0123456789abcde", true, true},
 672                 // old code path (setupProxyRemoteCluster) - federated request
 673                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", false, true},
 674                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", true, true},
 675         }
 676
 677         for _, tt := range tests {
 678                 c.Log(c.TestName() + " " + tt.path)
 679                 req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
 680                 c.Assert(err, check.IsNil)
 681                 customReqId := "abcdeG"
 682                 if !tt.reqIdProvided {
 683                         c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
 684                 } else {
 685                         req.Header.Set("X-Request-Id", customReqId)
 686                 }
 687                 resp, err := ac1.Do(req)
 688                 c.Assert(err, check.IsNil)
 689                 if tt.notFoundRequest {
 690                         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 691                 } else {
 692                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 693                 }
 694                 respHdr := resp.Header.Get("X-Request-Id")
 695                 if tt.reqIdProvided {
 696                         c.Check(respHdr, check.Equals, customReqId)
 697                 } else {
 698                         c.Check(respHdr, check.Matches, `req-[0-9a-zA-Z]{20}`)
 699                 }
 700                 if tt.notFoundRequest {
 701                         var jresp httpserver.ErrorResponse
 702                         err := json.NewDecoder(resp.Body).Decode(&jresp)
 703                         c.Check(err, check.IsNil)
 704                         if c.Check(jresp.Errors, check.HasLen, 1) {
 705                                 c.Check(jresp.Errors[0], check.Matches, `.*\(`+respHdr+`\).*`)
 706                         }
 707                 }
 708                 resp.Body.Close()
 709         }
 710 }
 711
 712 // We test the direct access to the database
 713 // normally an integration test would not have a database access, but in this case we need
 714 // to test tokens that are secret, so there is no API response that will give them back
 715 func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) {
 716         ctx := context.Background()
 717         db, err := sql.Open("postgres", s.super.Cluster(clusterID).PostgreSQL.Connection.String())
 718         c.Assert(err, check.IsNil)
 719
 720         conn, err := db.Conn(ctx)
 721         c.Assert(err, check.IsNil)
 722
 723         rows, err := conn.ExecContext(ctx, `SELECT 1`)
 724         c.Assert(err, check.IsNil)
 725         n, err := rows.RowsAffected()
 726         c.Assert(err, check.IsNil)
 727         c.Assert(n, check.Equals, int64(1))
 728         return db, conn
 729 }
 730
 731 // TestRuntimeTokenInCR will test several different tokens in the runtime attribute
 732 // and check the expected results accessing directly to the database if needed.
 733 func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) {
 734         db, dbconn := s.dbConn(c, "z1111")
 735         defer db.Close()
 736         defer dbconn.Close()
 737         conn1 := s.super.Conn("z1111")
 738         rootctx1, _, _ := s.super.RootClients("z1111")
 739         userctx1, ac1, _, au := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
 740
 741         tests := []struct {
 742                 name                 string
 743                 token                string
 744                 expectAToGetAValidCR bool
 745                 expectedToken        *string
 746         }{
 747                 {"Good token z1111 user", ac1.AuthToken, true, &ac1.AuthToken},
 748                 {"Bogus token", "abcdef", false, nil},
 749                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", false, nil},
 750                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", false, nil},
 751         }
 752
 753         for _, tt := range tests {
 754                 c.Log(c.TestName() + " " + tt.name)
 755
 756                 rq := map[string]interface{}{
 757                         "command":         []string{"echo"},
 758                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 759                         "cwd":             "/",
 760                         "output_path":     "/",
 761                         "runtime_token":   tt.token,
 762                 }
 763                 cr, err := conn1.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: rq})
 764                 if tt.expectAToGetAValidCR {
 765                         c.Check(err, check.IsNil)
 766                         c.Check(cr, check.NotNil)
 767                         c.Check(cr.UUID, check.Not(check.Equals), "")
 768                 }
 769
 770                 if tt.expectedToken == nil {
 771                         continue
 772                 }
 773
 774                 c.Logf("cr.UUID: %s", cr.UUID)
 775                 row := dbconn.QueryRowContext(rootctx1, `SELECT runtime_token from container_requests where uuid=$1`, cr.UUID)
 776                 c.Check(row, check.NotNil)
 777                 var token sql.NullString
 778                 row.Scan(&token)
 779                 if c.Check(token.Valid, check.Equals, true) {
 780                         c.Check(token.String, check.Equals, *tt.expectedToken)
 781                 }
 782         }
 783 }
 784
 785 // TestIntermediateCluster will send a container request to
 786 // one cluster with another cluster as the destination
 787 // and check the tokens are being handled properly
 788 func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) {
 789         conn1 := s.super.Conn("z1111")
 790         rootctx1, _, _ := s.super.RootClients("z1111")
 791         uctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
 792
 793         tests := []struct {
 794                 name                 string
 795                 token                string
 796                 expectedRuntimeToken string
 797                 expectedUUIDprefix   string
 798         }{
 799                 {"Good token z1111 user sending a CR to z2222", ac1.AuthToken, "", "z2222-xvhdp-"},
 800         }
 801
 802         for _, tt := range tests {
 803                 c.Log(c.TestName() + " " + tt.name)
 804                 rq := map[string]interface{}{
 805                         "command":         []string{"echo"},
 806                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
 807                         "cwd":             "/",
 808                         "output_path":     "/",
 809                         "runtime_token":   tt.token,
 810                 }
 811                 cr, err := conn1.ContainerRequestCreate(uctx1, arvados.CreateOptions{ClusterID: "z2222", Attrs: rq})
 812
 813                 c.Check(err, check.IsNil)
 814                 c.Check(strings.HasPrefix(cr.UUID, tt.expectedUUIDprefix), check.Equals, true)
 815                 c.Check(cr.RuntimeToken, check.Equals, tt.expectedRuntimeToken)
 816         }
 817 }
 818
 819 // Test for #17785
 820 func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) {
 821         rootctx1, rootclnt1, _ := s.super.RootClients("z1111")
 822         conn1 := s.super.Conn("z1111")
 823
 824         // Make sure LoginCluster is properly configured
 825         for _, cls := range []string{"z1111", "z3333"} {
 826                 c.Check(
 827                         s.super.Cluster(cls).Login.LoginCluster,
 828                         check.Equals, "z1111",
 829                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 830         }
 831         // Get user's UUID & attempt to create a token for it on the remote cluster
 832         _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1,
 833                 "user@example.com", true)
 834         _, rootclnt3, _ := s.super.ClientsWithToken("z3333", rootclnt1.AuthToken)
 835         var resp arvados.APIClientAuthorization
 836         err := rootclnt3.RequestAndDecode(
 837                 &resp, "POST", "arvados/v1/api_client_authorizations", nil,
 838                 map[string]interface{}{
 839                         "api_client_authorization": map[string]string{
 840                                 "owner_uuid": user.UUID,
 841                         },
 842                 },
 843         )
 844         c.Assert(err, check.IsNil)
 845         newTok := resp.TokenV2()
 846         c.Assert(newTok, check.Not(check.Equals), "")
 847
 848         // Confirm the token is from z1111
 849         c.Assert(strings.HasPrefix(newTok, "v2/z1111-gj3su-"), check.Equals, true)
 850
 851         // Confirm the token works and is from the correct user
 852         _, rootclnt3bis, _ := s.super.ClientsWithToken("z3333", newTok)
 853         var curUser arvados.User
 854         err = rootclnt3bis.RequestAndDecode(
 855                 &curUser, "GET", "arvados/v1/users/current", nil, nil,
 856         )
 857         c.Assert(err, check.IsNil)
 858         c.Assert(curUser.UUID, check.Equals, user.UUID)
 859
 860         // Request the ApiClientAuthorization list using the new token
 861         _, userClient, _ := s.super.ClientsWithToken("z3333", newTok)
 862         var acaLst arvados.APIClientAuthorizationList
 863         err = userClient.RequestAndDecode(
 864                 &acaLst, "GET", "arvados/v1/api_client_authorizations", nil, nil,
 865         )
 866         c.Assert(err, check.IsNil)
 867 }
 868
 869 // Test for bug #18076
 870 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
 871         rootctx1, _, _ := s.super.RootClients("z1111")
 872         conn1 := s.super.Conn("z1111")
 873         conn3 := s.super.Conn("z3333")
 874
 875         // Make sure LoginCluster is properly configured
 876         for _, cls := range []string{"z1111", "z3333"} {
 877                 c.Check(
 878                         s.super.Cluster(cls).Login.LoginCluster,
 879                         check.Equals, "z1111",
 880                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 881         }
 882
 883         // Create some users, request them on the federated cluster so they're cached.
 884         var users []arvados.User
 885         for userNr := 0; userNr < 2; userNr++ {
 886                 _, _, _, user := s.super.UserClients("z1111",
 887                         rootctx1,
 888                         c,
 889                         conn1,
 890                         fmt.Sprintf("user0%d@example.com", userNr),
 891                         true)
 892                 c.Assert(user.Username, check.Not(check.Equals), "")
 893                 users = append(users, user)
 894
 895                 lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 896                 c.Assert(err, check.Equals, nil)
 897                 userFound := false
 898                 for _, fedUser := range lst.Items {
 899                         if fedUser.UUID == user.UUID {
 900                                 c.Assert(fedUser.Username, check.Equals, user.Username)
 901                                 userFound = true
 902                                 break
 903                         }
 904                 }
 905                 c.Assert(userFound, check.Equals, true)
 906         }
 907
 908         // Swap the usernames
 909         _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
 910                 UUID: users[0].UUID,
 911                 Attrs: map[string]interface{}{
 912                         "username": "",
 913                 },
 914         })
 915         c.Assert(err, check.Equals, nil)
 916         _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
 917                 UUID: users[1].UUID,
 918                 Attrs: map[string]interface{}{
 919                         "username": users[0].Username,
 920                 },
 921         })
 922         c.Assert(err, check.Equals, nil)
 923         _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
 924                 UUID: users[0].UUID,
 925                 Attrs: map[string]interface{}{
 926                         "username": users[1].Username,
 927                 },
 928         })
 929         c.Assert(err, check.Equals, nil)
 930
 931         // Re-request the list on the federated cluster & check for updates
 932         lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 933         c.Assert(err, check.Equals, nil)
 934         var user0Found, user1Found bool
 935         for _, user := range lst.Items {
 936                 if user.UUID == users[0].UUID {
 937                         user0Found = true
 938                         c.Assert(user.Username, check.Equals, users[1].Username)
 939                 } else if user.UUID == users[1].UUID {
 940                         user1Found = true
 941                         c.Assert(user.Username, check.Equals, users[0].Username)
 942                 }
 943         }
 944         c.Assert(user0Found, check.Equals, true)
 945         c.Assert(user1Found, check.Equals, true)
 946 }
 947
 948 // Test for bug #16263
 949 func (s *IntegrationSuite) TestListUsers(c *check.C) {
 950         rootctx1, _, _ := s.super.RootClients("z1111")
 951         conn1 := s.super.Conn("z1111")
 952         conn3 := s.super.Conn("z3333")
 953         userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
 954
 955         // Make sure LoginCluster is properly configured
 956         for _, cls := range []string{"z1111", "z2222", "z3333"} {
 957                 c.Check(
 958                         s.super.Cluster(cls).Login.LoginCluster,
 959                         check.Equals, "z1111",
 960                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 961         }
 962         // Make sure z1111 has users with NULL usernames
 963         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
 964                 Limit: math.MaxInt64, // check that large limit works (see #16263)
 965         })
 966         nullUsername := false
 967         c.Assert(err, check.IsNil)
 968         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
 969         for _, user := range lst.Items {
 970                 if user.Username == "" {
 971                         nullUsername = true
 972                         break
 973                 }
 974         }
 975         c.Assert(nullUsername, check.Equals, true)
 976
 977         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
 978         c.Assert(err, check.IsNil)
 979         c.Check(user1.IsActive, check.Equals, true)
 980
 981         // Ask for the user list on z3333 using z1111's system root token
 982         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
 983         c.Assert(err, check.IsNil)
 984         found := false
 985         for _, user := range lst.Items {
 986                 if user.UUID == user1.UUID {
 987                         c.Check(user.IsActive, check.Equals, true)
 988                         found = true
 989                         break
 990                 }
 991         }
 992         c.Check(found, check.Equals, true)
 993
 994         // Deactivate user acct on z1111
 995         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
 996         c.Assert(err, check.IsNil)
 997
 998         // Get user list from z3333, check the returned z1111 user is
 999         // deactivated
1000         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
1001         c.Assert(err, check.IsNil)
1002         found = false
1003         for _, user := range lst.Items {
1004                 if user.UUID == user1.UUID {
1005                         c.Check(user.IsActive, check.Equals, false)
1006                         found = true
1007                         break
1008                 }
1009         }
1010         c.Check(found, check.Equals, true)
1011
1012         // Deactivated user no longer has working token
1013         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
1014         c.Assert(err, check.ErrorMatches, `.*401 Unauthorized.*`)
1015 }
1016
1017 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
1018         conn1 := s.super.Conn("z1111")
1019         conn3 := s.super.Conn("z3333")
1020         rootctx1, rootac1, _ := s.super.RootClients("z1111")
1021
1022         // Create user on LoginCluster z1111
1023         _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
1024
1025         // Make a new root token (because rootClients() uses SystemRootToken)
1026         var outAuth arvados.APIClientAuthorization
1027         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
1028         c.Check(err, check.IsNil)
1029
1030         // Make a v2 root token to communicate with z3333
1031         rootctx3, rootac3, _ := s.super.ClientsWithToken("z3333", outAuth.TokenV2())
1032
1033         // Create VM on z3333
1034         var outVM arvados.VirtualMachine
1035         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
1036                 map[string]interface{}{"virtual_machine": map[string]interface{}{
1037                         "hostname": "example",
1038                 },
1039                 })
1040         c.Assert(err, check.IsNil)
1041         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
1042
1043         // Make sure z3333 user list is up to date
1044         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
1045         c.Check(err, check.IsNil)
1046
1047         // Try to set up user on z3333 with the VM
1048         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
1049         c.Check(err, check.IsNil)
1050
1051         var outLinks arvados.LinkList
1052         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
1053                 arvados.ListOptions{
1054                         Limit: 1000,
1055                         Filters: []arvados.Filter{
1056                                 {
1057                                         Attr:     "tail_uuid",
1058                                         Operator: "=",
1059                                         Operand:  user.UUID,
1060                                 },
1061                                 {
1062                                         Attr:     "head_uuid",
1063                                         Operator: "=",
1064                                         Operand:  outVM.UUID,
1065                                 },
1066                                 {
1067                                         Attr:     "name",
1068                                         Operator: "=",
1069                                         Operand:  "can_login",
1070                                 },
1071                                 {
1072                                         Attr:     "link_class",
1073                                         Operator: "=",
1074                                         Operand:  "permission",
1075                                 }}})
1076         c.Check(err, check.IsNil)
1077
1078         c.Check(len(outLinks.Items), check.Equals, 1)
1079 }
1080
1081 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
1082         conn1 := s.super.Conn("z1111")
1083         rootctx1, _, _ := s.super.RootClients("z1111")
1084         s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
1085
1086         accesstoken := s.oidcprovider.ValidAccessToken()
1087
1088         for _, clusterID := range []string{"z1111", "z2222"} {
1089
1090                 var coll arvados.Collection
1091
1092                 // Write some file data and create a collection
1093                 {
1094                         c.Logf("save collection to %s", clusterID)
1095
1096                         conn := s.super.Conn(clusterID)
1097                         ctx, ac, kc := s.super.ClientsWithToken(clusterID, accesstoken)
1098
1099                         fs, err := coll.FileSystem(ac, kc)
1100                         c.Assert(err, check.IsNil)
1101                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
1102                         c.Assert(err, check.IsNil)
1103                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
1104                         c.Assert(err, check.IsNil)
1105                         err = f.Close()
1106                         c.Assert(err, check.IsNil)
1107                         mtxt, err := fs.MarshalManifest(".")
1108                         c.Assert(err, check.IsNil)
1109                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
1110                                 "manifest_text": mtxt,
1111                         }})
1112                         c.Assert(err, check.IsNil)
1113                 }
1114
1115                 // Read the collection & file data -- both from the
1116                 // cluster where it was created, and from the other
1117                 // cluster.
1118                 for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
1119                         c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
1120
1121                         conn := s.super.Conn(readClusterID)
1122                         ctx, ac, kc := s.super.ClientsWithToken(readClusterID, accesstoken)
1123
1124                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
1125                         c.Assert(err, check.IsNil)
1126                         c.Check(user.FullName, check.Equals, "Example User")
1127                         readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
1128                         c.Assert(err, check.IsNil)
1129                         c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
1130                         fs, err := readcoll.FileSystem(ac, kc)
1131                         c.Assert(err, check.IsNil)
1132                         f, err := fs.Open("test.txt")
1133                         c.Assert(err, check.IsNil)
1134                         buf, err := ioutil.ReadAll(f)
1135                         c.Assert(err, check.IsNil)
1136                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
1137                 }
1138         }
1139 }
1140
1141 // z3333 should not forward a locally-issued container runtime token,
1142 // associated with a z1111 user, to its login cluster z1111. z1111
1143 // would only call back to z3333 and then reject the response because
1144 // the user ID does not match the token prefix. See
1145 // dev.arvados.org/issues/18346
1146 func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) {
1147         db3, db3conn := s.dbConn(c, "z3333")
1148         defer db3.Close()
1149         defer db3conn.Close()
1150         rootctx1, _, _ := s.super.RootClients("z1111")
1151         rootctx3, _, _ := s.super.RootClients("z3333")
1152         conn1 := s.super.Conn("z1111")
1153         conn3 := s.super.Conn("z3333")
1154         userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
1155
1156         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
1157         c.Assert(err, check.IsNil)
1158         c.Logf("user1 %+v", user1)
1159
1160         imageColl, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1161                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n",
1162         }})
1163         c.Assert(err, check.IsNil)
1164         c.Logf("imageColl %+v", imageColl)
1165
1166         cr, err := conn3.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1167                 "state":           "Committed",
1168                 "command":         []string{"echo"},
1169                 "container_image": imageColl.PortableDataHash,
1170                 "cwd":             "/",
1171                 "output_path":     "/",
1172                 "priority":        1,
1173                 "runtime_constraints": arvados.RuntimeConstraints{
1174                         VCPUs: 1,
1175                         RAM:   1000000000,
1176                 },
1177         }})
1178         c.Assert(err, check.IsNil)
1179         c.Logf("container request %+v", cr)
1180         ctr, err := conn3.ContainerLock(rootctx3, arvados.GetOptions{UUID: cr.ContainerUUID})
1181         c.Assert(err, check.IsNil)
1182         c.Logf("container %+v", ctr)
1183
1184         // We could use conn3.ContainerAuth() here, but that API
1185         // hasn't been added to sdk/go/arvados/api.go yet.
1186         row := db3conn.QueryRowContext(context.Background(), `SELECT api_token from api_client_authorizations where uuid=$1`, ctr.AuthUUID)
1187         c.Check(row, check.NotNil)
1188         var val sql.NullString
1189         row.Scan(&val)
1190         c.Assert(val.Valid, check.Equals, true)
1191         runtimeToken := "v2/" + ctr.AuthUUID + "/" + val.String
1192         ctrctx, _, _ := s.super.ClientsWithToken("z3333", runtimeToken)
1193         c.Logf("container runtime token %+v", runtimeToken)
1194
1195         _, err = conn3.UserGet(ctrctx, arvados.GetOptions{UUID: user1.UUID})
1196         c.Assert(err, check.NotNil)
1197         c.Check(err, check.ErrorMatches, `request failed: .* 401 Unauthorized: cannot use a locally issued token to forward a request to our login cluster \(z1111\)`)
1198         c.Check(err, check.Not(check.ErrorMatches), `(?ms).*127\.0\.0\.11.*`)
1199 }
1200
1201 func (s *IntegrationSuite) TestRunTrivialContainer(c *check.C) {
1202         outcoll, _ := s.runContainer(c, "z1111", "", map[string]interface{}{
1203                 "command":             []string{"sh", "-c", "touch \"/out/hello world\" /out/ohai"},
1204                 "container_image":     "busybox:uclibc",
1205                 "cwd":                 "/tmp",
1206                 "environment":         map[string]string{},
1207                 "mounts":              map[string]arvados.Mount{"/out": {Kind: "tmp", Capacity: 10000}},
1208                 "output_path":         "/out",
1209                 "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26},
1210                 "priority":            1,
1211                 "state":               arvados.ContainerRequestStateCommitted,
1212         }, 0, nil)
1213         c.Check(outcoll.ManifestText, check.Matches, `\. d41d8.* 0:0:hello\\040world 0:0:ohai\n`)
1214         c.Check(outcoll.PortableDataHash, check.Equals, "8fa5dee9231a724d7cf377c5a2f4907c+65")
1215 }
1216
1217 // Note that unlike the rest of the suite, this test requires
1218 // {RuntimeEngine: docker} because:
1219 //
1220 //   - the singularity driver relies on ARVADOS_TEST_PRIVESC=sudo to
1221 //     connect to ports inside the container
1222 //
1223 //   - non-passwordless sudo does not work in crunch-run running under
1224 //     the loopback driver
1225 func (s *IntegrationSuite) TestContainerHTTPProxy(c *check.C) {
1226         ctrport := "12345"
1227         var success bool
1228         var done atomic.Bool
1229         var ctrLatest atomic.Value
1230         defer func() { done.Store(true) }()
1231         go func() {
1232                 for range time.NewTicker(time.Second).C {
1233                         if done.Load() {
1234                                 break
1235                         }
1236                         ctr, ok := ctrLatest.Load().(arvados.Container)
1237                         if !ok {
1238                                 c.Log("waiting for onStateUpdate...")
1239                                 continue
1240                         }
1241                         if ctr.State != arvados.ContainerStateRunning {
1242                                 continue
1243                         }
1244                         ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second))
1245                         defer cancel()
1246                         exturl := ctr.PublishedPorts[ctrport].InitialURL + "test-path"
1247                         c.Logf("poll: %s", exturl)
1248                         req, err := http.NewRequestWithContext(ctx, http.MethodGet, exturl, nil)
1249                         c.Assert(err, check.IsNil)
1250                         resp, err := arvados.InsecureHTTPClient.Do(req)
1251                         if err != nil {
1252                                 c.Logf("poll: Do: %v", err)
1253                                 continue
1254                         }
1255                         body, err := ioutil.ReadAll(resp.Body)
1256                         if err != nil {
1257                                 c.Logf("poll: error reading response body: %v", err)
1258                                 continue
1259                         }
1260                         if resp.StatusCode != http.StatusOK {
1261                                 c.Logf("poll: resp.Status %q body %q", resp.Status, body)
1262                                 continue
1263                         }
1264                         if len(body) == 0 {
1265                                 c.Log("poll: empty response body")
1266                                 continue
1267                         }
1268                         success = true
1269                         c.Logf("poll: %s, resp.Body %q", resp.Status, body)
1270                         break
1271                 }
1272         }()
1273         outcoll, _ := s.runContainer(c, "z1111", "", map[string]interface{}{
1274                 // Note "$req" and "$hdr" each have a trailing \r here
1275                 "command":             []string{"sh", "-c", `nc -l -p ` + ctrport + ` -w 30 -e sh -c 'read req; echo >&2 "$req"; hdr=zzz; while [ "${#hdr}" -gt 1 ]; do read hdr; echo >&2 "${hdr}"; done; printf "HTTP/1.0 200 OK\\r\\n\\r\\n%s\\n" "$req"' && touch /out/OK`},
1276                 "container_image":     "busybox:uclibc",
1277                 "cwd":                 "/tmp",
1278                 "environment":         map[string]string{},
1279                 "mounts":              map[string]arvados.Mount{"/out": {Kind: "tmp", Capacity: 10000}},
1280                 "output_path":         "/out",
1281                 "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26, API: true},
1282                 "priority":            1,
1283                 "state":               arvados.ContainerRequestStateCommitted,
1284                 "published_ports": map[string]arvados.RequestPublishedPort{
1285                         ctrport: arvados.RequestPublishedPort{
1286                                 Access: arvados.PublishedPortAccessPublic,
1287                                 Label:  "testport",
1288                         }},
1289         }, 0, func(ctr arvados.Container) { ctrLatest.Store(ctr) })
1290         c.Check(outcoll.ManifestText, check.Matches, `\. d41d8.* 0:0:OK\n`)
1291         c.Check(outcoll.PortableDataHash, check.Equals, "c1c354b852802ee13ad87da784b9c28c+44")
1292         c.Check(success, check.Equals, true)
1293 }
1294 func (s *IntegrationSuite) TestContainerInputOnDifferentCluster(c *check.C) {
1295         conn := s.super.Conn("z1111")
1296         rootctx, _, _ := s.super.RootClients("z1111")
1297         userctx, ac, _, _ := s.super.UserClients("z1111", rootctx, c, conn, s.oidcprovider.AuthEmail, true)
1298         z1coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{Attrs: map[string]interface{}{
1299                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:ocelot\n",
1300         }})
1301         c.Assert(err, check.IsNil)
1302
1303         outcoll, logcfs := s.runContainer(c, "z2222", ac.AuthToken, map[string]interface{}{
1304                 "command":         []string{"ls", "/in"},
1305                 "container_image": "busybox:uclibc",
1306                 "cwd":             "/tmp",
1307                 "environment":     map[string]string{},
1308                 "mounts": map[string]arvados.Mount{
1309                         "/in":  {Kind: "collection", PortableDataHash: z1coll.PortableDataHash},
1310                         "/out": {Kind: "tmp", Capacity: 10000},
1311                 },
1312                 "output_path":         "/out",
1313                 "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26},
1314                 "priority":            1,
1315                 "state":               arvados.ContainerRequestStateCommitted,
1316                 "container_count_max": 1,
1317         }, -1, nil)
1318         if outcoll.UUID == "" {
1319                 arvmountlog, err := fs.ReadFile(arvados.FS(logcfs), "/arv-mount.txt")
1320                 c.Check(err, check.IsNil)
1321                 c.Check(string(arvmountlog), check.Matches, `(?ms).*cannot use a locally issued token to forward a request to our login cluster \(z1111\).*`)
1322                 c.Skip("this use case is not supported yet")
1323         }
1324         stdout, err := fs.ReadFile(arvados.FS(logcfs), "/stdout.txt")
1325         c.Check(err, check.IsNil)
1326         c.Check(string(stdout), check.Equals, "ocelot\n")
1327 }
1328
1329 func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, token string, ctrSpec map[string]interface{}, expectExitCode int, onStateUpdate func(arvados.Container)) (outcoll arvados.Collection, logcfs arvados.CollectionFileSystem) {
1330         conn := s.super.Conn(clusterID)
1331         rootctx, _, _ := s.super.RootClients(clusterID)
1332         if token == "" {
1333                 _, ac, _, _ := s.super.UserClients(clusterID, rootctx, c, conn, s.oidcprovider.AuthEmail, true)
1334                 token = ac.AuthToken
1335         }
1336         _, ac, kc := s.super.ClientsWithToken(clusterID, token)
1337
1338         c.Log("[docker load]")
1339         out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput()
1340         c.Logf("[docker load output] %s", out)
1341         c.Check(err, check.IsNil)
1342
1343         c.Log("[arv-keepdocker]")
1344         akd := exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc")
1345         akd.Env = append(os.Environ(), "ARVADOS_API_HOST="+ac.APIHost, "ARVADOS_API_HOST_INSECURE=1", "ARVADOS_API_TOKEN="+ac.AuthToken)
1346         out, err = akd.CombinedOutput()
1347         c.Logf("[arv-keepdocker output]\n%s", out)
1348         c.Check(err, check.IsNil)
1349
1350         var cr arvados.ContainerRequest
1351         err = ac.RequestAndDecode(&cr, "POST", "/arvados/v1/container_requests", nil, map[string]interface{}{
1352                 "container_request": ctrSpec,
1353         })
1354         c.Assert(err, check.IsNil)
1355
1356         showlogs := func(collectionID string) arvados.CollectionFileSystem {
1357                 var logcoll arvados.Collection
1358                 err = ac.RequestAndDecode(&logcoll, "GET", "/arvados/v1/collections/"+collectionID, nil, nil)
1359                 c.Assert(err, check.IsNil)
1360                 cfs, err := logcoll.FileSystem(ac, kc)
1361                 c.Assert(err, check.IsNil)
1362                 fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
1363                         if d.IsDir() || strings.HasPrefix(path, "/log for container") {
1364                                 return nil
1365                         }
1366                         f, err := cfs.Open(path)
1367                         c.Assert(err, check.IsNil)
1368                         defer f.Close()
1369                         buf, err := ioutil.ReadAll(f)
1370                         c.Assert(err, check.IsNil)
1371                         c.Logf("=== %s\n%s\n", path, buf)
1372                         return nil
1373                 })
1374                 return cfs
1375         }
1376
1377         checkwebdavlogs := func(cr arvados.ContainerRequest) {
1378                 req, err := http.NewRequest("OPTIONS", "https://"+ac.APIHost+"/arvados/v1/container_requests/"+cr.UUID+"/log/"+cr.ContainerUUID+"/", nil)
1379                 c.Assert(err, check.IsNil)
1380                 req.Header.Set("Origin", "http://example.example")
1381                 resp, err := ac.Do(req)
1382                 c.Assert(err, check.IsNil)
1383                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
1384                 // Check for duplicate headers -- must use Header[], not Header.Get()
1385                 c.Check(resp.Header["Access-Control-Allow-Origin"], check.DeepEquals, []string{"*"})
1386         }
1387
1388         var ctr arvados.Container
1389         var lastState arvados.ContainerState
1390         var status, lastStatus arvados.ContainerStatus
1391         var allStatus string
1392         checkstatus := func() {
1393                 err := ac.RequestAndDecode(&status, "GET", "/arvados/v1/container_requests/"+cr.UUID+"/container_status", nil, nil)
1394                 c.Assert(err, check.IsNil)
1395                 if status != lastStatus {
1396                         c.Logf("container status: %s, %s", status.State, status.SchedulingStatus)
1397                         allStatus += fmt.Sprintf("%s, %s\n", status.State, status.SchedulingStatus)
1398                         lastStatus = status
1399                 }
1400         }
1401         deadline := time.Now().Add(time.Minute)
1402         for cr.State != arvados.ContainerRequestStateFinal || (lastStatus.State != arvados.ContainerStateComplete && lastStatus.State != arvados.ContainerStateCancelled) {
1403                 err = ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil)
1404                 c.Assert(err, check.IsNil)
1405                 checkstatus()
1406                 err = ac.RequestAndDecode(&ctr, "GET", "/arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
1407                 if err != nil {
1408                         c.Logf("error getting container state: %s", err)
1409                 } else if ctr.State != lastState {
1410                         c.Logf("container state changed to %q", ctr.State)
1411                         lastState = ctr.State
1412                         if onStateUpdate != nil {
1413                                 onStateUpdate(ctr)
1414                         }
1415                 } else {
1416                         if time.Now().After(deadline) {
1417                                 c.Errorf("timed out, container state is %q", cr.State)
1418                                 if ctr.Log == "" {
1419                                         c.Logf("=== NO LOG COLLECTION saved for container")
1420                                 } else {
1421                                         showlogs(ctr.Log)
1422                                 }
1423                                 c.FailNow()
1424                         }
1425                         time.Sleep(time.Second / 2)
1426                 }
1427         }
1428         checkstatus()
1429         c.Logf("cr.CumulativeCost == %f", cr.CumulativeCost)
1430         c.Check(cr.CumulativeCost, check.Not(check.Equals), 0.0)
1431         if expectExitCode >= 0 {
1432                 c.Check(ctr.State, check.Equals, arvados.ContainerStateComplete)
1433                 c.Check(ctr.ExitCode, check.Equals, expectExitCode)
1434                 err = ac.RequestAndDecode(&outcoll, "GET", "/arvados/v1/collections/"+cr.OutputUUID, nil, nil)
1435                 c.Assert(err, check.IsNil)
1436                 c.Check(allStatus, check.Matches, `(Queued, Waiting in queue\.\n)?`+
1437                         // Occasionally the dispatcher will
1438                         // unlock/retry, and we get state/status from
1439                         // database/dispatcher via separate API calls,
1440                         // so we can also see "Queued, preparing
1441                         // runtime environment".
1442                         `((Queued|Locked), (Waiting .*|Container is allocated to an instance and preparing to run\.)\n)*`+
1443                         `(Running, \n)?`+
1444                         `Complete, \n`)
1445         }
1446         logcfs = showlogs(cr.LogUUID)
1447         checkwebdavlogs(cr)
1448         return outcoll, logcfs
1449 }
1450
1451 func (s *IntegrationSuite) TestCUDAContainerReuse(c *check.C) {
1452         // Check that the legacy "CUDA" API still works.
1453
1454         conn1 := s.super.Conn("z1111")
1455         rootctx1, _, _ := s.super.RootClients("z1111")
1456         _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
1457
1458         crInput := map[string]interface{}{
1459                 "command":         []string{"echo", "hello", "/bin/sh", "-c", "'cat' '/keep/fa7aeb5140e2848d39b416daeef4ffc5+45/foobar' '/keep/fa7aeb5140e2848d39b416daeef4ffc5+45/baz' '|' 'gzip' '>' '/dev/null'"},
1460                 "cwd":             "test",
1461                 "environment":     map[string]interface{}{},
1462                 "output_path":     "test",
1463                 "output_glob":     []string{},
1464                 "container_image": "fa3c1a9cb6783f85f2ecda037e07b8c3+167",
1465                 "mounts":          map[string]interface{}{},
1466                 "runtime_constraints": map[string]interface{}{
1467                         "cuda": map[string]interface{}{
1468                                 "device_count":        1,
1469                                 "driver_version":      "11.0",
1470                                 "hardware_capability": "9.0",
1471                         },
1472                         "ram":   12000000000,
1473                         "vcpus": 4,
1474                 },
1475                 "state": "Committed",
1476         }
1477
1478         var outCR arvados.ContainerRequest
1479         err := ac1.RequestAndDecode(&outCR, "POST", "/arvados/v1/container_requests", nil,
1480                 map[string]interface{}{"container_request": crInput})
1481         c.Check(err, check.IsNil)
1482
1483         c.Check(outCR.RuntimeConstraints.GPU.Stack, check.Equals, "cuda")
1484         c.Check(outCR.RuntimeConstraints.GPU.DriverVersion, check.Equals, "11.0")
1485         c.Check(outCR.RuntimeConstraints.GPU.HardwareTarget, check.DeepEquals, []string{"9.0"})
1486         c.Check(outCR.RuntimeConstraints.GPU.DeviceCount, check.Equals, 1)
1487         c.Check(outCR.RuntimeConstraints.GPU.VRAM, check.Equals, int64(0))
1488
1489         var outCR2 arvados.ContainerRequest
1490         err = ac1.RequestAndDecode(&outCR2, "POST", "/arvados/v1/container_requests", nil,
1491                 map[string]interface{}{"container_request": crInput})
1492         c.Check(err, check.IsNil)
1493
1494         c.Check(outCR.ContainerUUID, check.Equals, outCR2.ContainerUUID)
1495 }
1496
1497 func (s *IntegrationSuite) TestGPUContainerReuse(c *check.C) {
1498         // Test container reuse using the "GPU" API
1499         conn1 := s.super.Conn("z1111")
1500         rootctx1, _, _ := s.super.RootClients("z1111")
1501         _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
1502
1503         crInput := map[string]interface{}{
1504                 "command":         []string{"echo", "hello", "/bin/sh", "-c", "'cat' '/keep/fa7aeb5140e2848d39b416daeef4ffc5+45/foobar' '/keep/fa7aeb5140e2848d39b416daeef4ffc5+45/baz' '|' 'gzip' '>' '/dev/null'"},
1505                 "cwd":             "test",
1506                 "environment":     map[string]interface{}{},
1507                 "output_path":     "test",
1508                 "output_glob":     []string{},
1509                 "container_image": "fa3c1a9cb6783f85f2ecda037e07b8c3+167",
1510                 "mounts":          map[string]interface{}{},
1511                 "runtime_constraints": map[string]interface{}{
1512                         "gpu": map[string]interface{}{
1513                                 "stack":           "cuda",
1514                                 "device_count":    1,
1515                                 "driver_version":  "11.0",
1516                                 "hardware_target": []string{"9.0"},
1517                                 "vram":            8000000000,
1518                         },
1519                         "ram":   12000000000,
1520                         "vcpus": 4,
1521                 },
1522                 "state": "Committed",
1523         }
1524
1525         var outCR arvados.ContainerRequest
1526         err := ac1.RequestAndDecode(&outCR, "POST", "/arvados/v1/container_requests", nil,
1527                 map[string]interface{}{"container_request": crInput})
1528         c.Check(err, check.IsNil)
1529
1530         c.Check(outCR.RuntimeConstraints.GPU.Stack, check.Equals, "cuda")
1531         c.Check(outCR.RuntimeConstraints.GPU.DriverVersion, check.Equals, "11.0")
1532         c.Check(outCR.RuntimeConstraints.GPU.HardwareTarget, check.DeepEquals, []string{"9.0"})
1533         c.Check(outCR.RuntimeConstraints.GPU.DeviceCount, check.Equals, 1)
1534         c.Check(outCR.RuntimeConstraints.GPU.VRAM, check.Equals, int64(8000000000))
1535
1536         var outCR2 arvados.ContainerRequest
1537         err = ac1.RequestAndDecode(&outCR2, "POST", "/arvados/v1/container_requests", nil,
1538                 map[string]interface{}{"container_request": crInput})
1539         c.Check(err, check.IsNil)
1540
1541         c.Check(outCR.ContainerUUID, check.Equals, outCR2.ContainerUUID)
1542 }