Merge branch '20200-log-limit' refs #20200
[arvados.git] / lib / controller / integration_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "bytes"
9         "context"
10         "database/sql"
11         "encoding/json"
12         "fmt"
13         "io"
14         "io/fs"
15         "io/ioutil"
16         "math"
17         "net"
18         "net/http"
19         "os"
20         "os/exec"
21         "strconv"
22         "strings"
23         "sync"
24         "time"
25
26         "git.arvados.org/arvados.git/lib/boot"
27         "git.arvados.org/arvados.git/sdk/go/arvados"
28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
30         "git.arvados.org/arvados.git/sdk/go/httpserver"
31         check "gopkg.in/check.v1"
32 )
33
34 var _ = check.Suite(&IntegrationSuite{})
35
36 type IntegrationSuite struct {
37         super        *boot.Supervisor
38         oidcprovider *arvadostest.OIDCProvider
39 }
40
41 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
42         s.oidcprovider = arvadostest.NewOIDCProvider(c)
43         s.oidcprovider.AuthEmail = "user@example.com"
44         s.oidcprovider.AuthEmailVerified = true
45         s.oidcprovider.AuthName = "Example User"
46         s.oidcprovider.ValidClientID = "clientid"
47         s.oidcprovider.ValidClientSecret = "clientsecret"
48
49         hostport := map[string]string{}
50         for _, id := range []string{"z1111", "z2222", "z3333"} {
51                 hostport[id] = func() string {
52                         // TODO: Instead of expecting random ports on
53                         // 127.0.0.11, 22, 33 to be race-safe, try
54                         // different 127.x.y.z until finding one that
55                         // isn't in use.
56                         ln, err := net.Listen("tcp", ":0")
57                         c.Assert(err, check.IsNil)
58                         ln.Close()
59                         _, port, err := net.SplitHostPort(ln.Addr().String())
60                         c.Assert(err, check.IsNil)
61                         return "127.0.0." + id[3:] + ":" + port
62                 }()
63         }
64         yaml := "Clusters:\n"
65         for id := range hostport {
66                 yaml += `
67   ` + id + `:
68     Services:
69       Controller:
70         ExternalURL: https://` + hostport[id] + `
71     TLS:
72       Insecure: true
73     SystemLogs:
74       Format: text
75     API:
76       MaxConcurrentRequests: 128
77     Containers:
78       CloudVMs:
79         Enable: true
80         Driver: loopback
81         BootProbeCommand: "rm -f /var/lock/crunch-run-broken"
82         ProbeInterval: 1s
83         PollInterval: 5s
84         SyncInterval: 10s
85         TimeoutIdle: 1s
86         TimeoutBooting: 2s
87       RuntimeEngine: singularity
88       CrunchRunArgumentsList: ["--broken-node-hook", "true"]
89     RemoteClusters:
90       z1111:
91         Host: ` + hostport["z1111"] + `
92         Scheme: https
93         Insecure: true
94         Proxy: true
95         ActivateUsers: true
96 `
97                 if id != "z2222" {
98                         yaml += `      z2222:
99         Host: ` + hostport["z2222"] + `
100         Scheme: https
101         Insecure: true
102         Proxy: true
103         ActivateUsers: true
104 `
105                 }
106                 if id != "z3333" {
107                         yaml += `      z3333:
108         Host: ` + hostport["z3333"] + `
109         Scheme: https
110         Insecure: true
111         Proxy: true
112         ActivateUsers: true
113 `
114                 }
115                 if id == "z1111" {
116                         yaml += `
117     Login:
118       LoginCluster: z1111
119       OpenIDConnect:
120         Enable: true
121         Issuer: ` + s.oidcprovider.Issuer.URL + `
122         ClientID: ` + s.oidcprovider.ValidClientID + `
123         ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
124         EmailClaim: email
125         EmailVerifiedClaim: email_verified
126         AcceptAccessToken: true
127         AcceptAccessTokenScope: ""
128 `
129                 } else {
130                         yaml += `
131     Login:
132       LoginCluster: z1111
133 `
134                 }
135         }
136         s.super = &boot.Supervisor{
137                 ClusterType:          "test",
138                 ConfigYAML:           yaml,
139                 Stderr:               ctxlog.LogWriter(c.Log),
140                 NoWorkbench1:         true,
141                 NoWorkbench2:         true,
142                 OwnTemporaryDatabase: true,
143         }
144
145         // Give up if startup takes longer than 3m
146         timeout := time.AfterFunc(3*time.Minute, s.super.Stop)
147         defer timeout.Stop()
148         s.super.Start(context.Background())
149         ok := s.super.WaitReady()
150         c.Assert(ok, check.Equals, true)
151 }
152
153 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
154         if s.super != nil {
155                 s.super.Stop()
156                 s.super.Wait()
157         }
158 }
159
160 func (s *IntegrationSuite) TestDefaultStorageClassesOnCollections(c *check.C) {
161         conn := s.super.Conn("z1111")
162         rootctx, _, _ := s.super.RootClients("z1111")
163         userctx, _, kc, _ := s.super.UserClients("z1111", rootctx, c, conn, s.oidcprovider.AuthEmail, true)
164         c.Assert(len(kc.DefaultStorageClasses) > 0, check.Equals, true)
165         coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{})
166         c.Assert(err, check.IsNil)
167         c.Assert(coll.StorageClassesDesired, check.DeepEquals, kc.DefaultStorageClasses)
168 }
169
170 func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
171         conn1 := s.super.Conn("z1111")
172         rootctx1, _, _ := s.super.RootClients("z1111")
173         conn3 := s.super.Conn("z3333")
174         userctx1, ac1, kc1, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
175
176         // Create the collection to find its PDH (but don't save it
177         // anywhere yet)
178         var coll1 arvados.Collection
179         fs1, err := coll1.FileSystem(ac1, kc1)
180         c.Assert(err, check.IsNil)
181         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
182         c.Assert(err, check.IsNil)
183         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
184         c.Assert(err, check.IsNil)
185         err = f.Close()
186         c.Assert(err, check.IsNil)
187         mtxt, err := fs1.MarshalManifest(".")
188         c.Assert(err, check.IsNil)
189         pdh := arvados.PortableDataHash(mtxt)
190
191         // Looking up the PDH before saving returns 404 if cycle
192         // detection is working.
193         _, err = conn1.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
194         c.Assert(err, check.ErrorMatches, `.*404 Not Found.*`)
195
196         // Save the collection on cluster z1111.
197         coll1, err = conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
198                 "manifest_text": mtxt,
199         }})
200         c.Assert(err, check.IsNil)
201
202         // Retrieve the collection from cluster z3333.
203         coll, err := conn3.CollectionGet(userctx1, arvados.GetOptions{UUID: pdh})
204         c.Check(err, check.IsNil)
205         c.Check(coll.PortableDataHash, check.Equals, pdh)
206 }
207
208 // Tests bug #18004
209 func (s *IntegrationSuite) TestRemoteUserAndTokenCacheRace(c *check.C) {
210         conn1 := s.super.Conn("z1111")
211         rootctx1, _, _ := s.super.RootClients("z1111")
212         rootctx2, _, _ := s.super.RootClients("z2222")
213         conn2 := s.super.Conn("z2222")
214         userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user2@example.com", true)
215
216         var wg1, wg2 sync.WaitGroup
217         creqs := 100
218
219         // Make concurrent requests to z2222 with a local token to make sure more
220         // than one worker is listening.
221         wg1.Add(1)
222         for i := 0; i < creqs; i++ {
223                 wg2.Add(1)
224                 go func() {
225                         defer wg2.Done()
226                         wg1.Wait()
227                         _, err := conn2.UserGetCurrent(rootctx2, arvados.GetOptions{})
228                         c.Check(err, check.IsNil, check.Commentf("warm up phase failed"))
229                 }()
230         }
231         wg1.Done()
232         wg2.Wait()
233
234         // Real test pass -- use a new remote token than the one used in the warm-up
235         // phase.
236         wg1.Add(1)
237         for i := 0; i < creqs; i++ {
238                 wg2.Add(1)
239                 go func() {
240                         defer wg2.Done()
241                         wg1.Wait()
242                         // Retrieve the remote collection from cluster z2222.
243                         _, err := conn2.UserGetCurrent(userctx1, arvados.GetOptions{})
244                         c.Check(err, check.IsNil, check.Commentf("testing phase failed"))
245                 }()
246         }
247         wg1.Done()
248         wg2.Wait()
249 }
250
251 func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
252         if _, err := exec.LookPath("s3cmd"); err != nil {
253                 c.Skip("s3cmd not in PATH")
254                 return
255         }
256
257         testText := "IntegrationSuite.TestS3WithFederatedToken"
258
259         conn1 := s.super.Conn("z1111")
260         rootctx1, _, _ := s.super.RootClients("z1111")
261         userctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
262         conn3 := s.super.Conn("z3333")
263
264         createColl := func(clusterID string) arvados.Collection {
265                 _, ac, kc := s.super.ClientsWithToken(clusterID, ac1.AuthToken)
266                 var coll arvados.Collection
267                 fs, err := coll.FileSystem(ac, kc)
268                 c.Assert(err, check.IsNil)
269                 f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
270                 c.Assert(err, check.IsNil)
271                 _, err = io.WriteString(f, testText)
272                 c.Assert(err, check.IsNil)
273                 err = f.Close()
274                 c.Assert(err, check.IsNil)
275                 mtxt, err := fs.MarshalManifest(".")
276                 c.Assert(err, check.IsNil)
277                 coll, err = s.super.Conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
278                         "manifest_text": mtxt,
279                 }})
280                 c.Assert(err, check.IsNil)
281                 return coll
282         }
283
284         for _, trial := range []struct {
285                 clusterID string // create the collection on this cluster (then use z3333 to access it)
286                 token     string
287         }{
288                 // Try the hardest test first: z3333 hasn't seen
289                 // z1111's token yet, and we're just passing the
290                 // opaque secret part, so z3333 has to guess that it
291                 // belongs to z1111.
292                 {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
293                 {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
294                 {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
295                 {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
296         } {
297                 c.Logf("================ %v", trial)
298                 coll := createColl(trial.clusterID)
299
300                 cfgjson, err := conn3.ConfigGet(userctx1)
301                 c.Assert(err, check.IsNil)
302                 var cluster arvados.Cluster
303                 err = json.Unmarshal(cfgjson, &cluster)
304                 c.Assert(err, check.IsNil)
305
306                 c.Logf("TokenV2 is %s", ac1.AuthToken)
307                 host := cluster.Services.WebDAV.ExternalURL.Host
308                 s3args := []string{
309                         "--ssl", "--no-check-certificate",
310                         "--host=" + host, "--host-bucket=" + host,
311                         "--access_key=" + trial.token, "--secret_key=" + trial.token,
312                 }
313                 buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
314                 c.Check(err, check.IsNil)
315                 c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
316
317                 buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
318                 // Command fails because we don't return Etag header.
319                 flen := strconv.Itoa(len(testText))
320                 c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
321         }
322 }
323
324 func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
325         conn1 := s.super.Conn("z1111")
326         conn3 := s.super.Conn("z3333")
327         rootctx1, rootac1, rootkc1 := s.super.RootClients("z1111")
328         anonctx3, anonac3, _ := s.super.AnonymousClients("z3333")
329
330         // Make sure anonymous token was set
331         c.Assert(anonac3.AuthToken, check.Not(check.Equals), "")
332
333         // Create the collection to find its PDH (but don't save it
334         // anywhere yet)
335         var coll1 arvados.Collection
336         fs1, err := coll1.FileSystem(rootac1, rootkc1)
337         c.Assert(err, check.IsNil)
338         f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
339         c.Assert(err, check.IsNil)
340         _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous")
341         c.Assert(err, check.IsNil)
342         err = f.Close()
343         c.Assert(err, check.IsNil)
344         mtxt, err := fs1.MarshalManifest(".")
345         c.Assert(err, check.IsNil)
346         pdh := arvados.PortableDataHash(mtxt)
347
348         // Save the collection on cluster z1111.
349         coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
350                 "manifest_text": mtxt,
351         }})
352         c.Assert(err, check.IsNil)
353
354         // Share it with the anonymous users group.
355         var outLink arvados.Link
356         err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil,
357                 map[string]interface{}{"link": map[string]interface{}{
358                         "link_class": "permission",
359                         "name":       "can_read",
360                         "tail_uuid":  "z1111-j7d0g-anonymouspublic",
361                         "head_uuid":  coll1.UUID,
362                 },
363                 })
364         c.Check(err, check.IsNil)
365
366         // Current user should be z3 anonymous user
367         outUser, err := anonac3.CurrentUser()
368         c.Check(err, check.IsNil)
369         c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
370
371         // Get the token uuid
372         var outAuth arvados.APIClientAuthorization
373         err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
374         c.Check(err, check.IsNil)
375
376         // Make a v2 token of the z3 anonymous user, and use it on z1
377         _, anonac1, _ := s.super.ClientsWithToken("z1111", outAuth.TokenV2())
378         outUser2, err := anonac1.CurrentUser()
379         c.Check(err, check.IsNil)
380         // z3 anonymous user will be mapped to the z1 anonymous user
381         c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic")
382
383         // Retrieve the collection (which is on z1) using anonymous from cluster z3333.
384         coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID})
385         c.Check(err, check.IsNil)
386         c.Check(coll.PortableDataHash, check.Equals, pdh)
387 }
388
389 // z3333 should forward the locally-issued anonymous user token to its login
390 // cluster z1111. That is no problem because the login cluster controller will
391 // map any anonymous user token to its local anonymous user.
392 //
393 // This needs to work because wb1 has a tendency to slap the local anonymous
394 // user token on every request as a reader_token, which gets folded into the
395 // request token list controller.
396 //
397 // Use a z1111 user token and the anonymous token from z3333 passed in as a
398 // reader_token to do a request on z3333, asking for the z1111 anonymous user
399 // object. The request will be forwarded to the z1111 cluster. The presence of
400 // the z3333 anonymous user token should not prohibit the request from being
401 // forwarded.
402 func (s *IntegrationSuite) TestForwardAnonymousTokenToLoginCluster(c *check.C) {
403         conn1 := s.super.Conn("z1111")
404
405         rootctx1, _, _ := s.super.RootClients("z1111")
406         _, anonac3, _ := s.super.AnonymousClients("z3333")
407
408         // Make a user connection to z3333 (using a z1111 user, because that's the login cluster)
409         _, userac1, _, _ := s.super.UserClients("z3333", rootctx1, c, conn1, "user@example.com", true)
410
411         // Get the anonymous user token for z3333
412         var anon3Auth arvados.APIClientAuthorization
413         err := anonac3.RequestAndDecode(&anon3Auth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
414         c.Check(err, check.IsNil)
415
416         var userList arvados.UserList
417         where := make(map[string]string)
418         where["uuid"] = "z1111-tpzed-anonymouspublic"
419         err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
420                 map[string]interface{}{
421                         "reader_tokens": []string{anon3Auth.TokenV2()},
422                         "where":         where,
423                 },
424         )
425         // The local z3333 anonymous token must be allowed to be forwarded to the login cluster
426         c.Check(err, check.IsNil)
427
428         userac1.AuthToken = "v2/z1111-gj3su-asdfasdfasdfasd/this-token-does-not-validate-so-anonymous-token-will-be-used-instead"
429         err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
430                 map[string]interface{}{
431                         "reader_tokens": []string{anon3Auth.TokenV2()},
432                         "where":         where,
433                 },
434         )
435         c.Check(err, check.IsNil)
436 }
437
438 // Get a token from the login cluster (z1111), use it to submit a
439 // container request on z2222.
440 func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
441         conn1 := s.super.Conn("z1111")
442         rootctx1, _, _ := s.super.RootClients("z1111")
443         _, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
444
445         // Use ac2 to get the discovery doc with a blank token, so the
446         // SDK doesn't magically pass the z1111 token to z2222 before
447         // we're ready to start our test.
448         _, ac2, _ := s.super.ClientsWithToken("z2222", "")
449         var dd map[string]interface{}
450         err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
451         c.Assert(err, check.IsNil)
452
453         var (
454                 body bytes.Buffer
455                 req  *http.Request
456                 resp *http.Response
457                 u    arvados.User
458                 cr   arvados.ContainerRequest
459         )
460         json.NewEncoder(&body).Encode(map[string]interface{}{
461                 "container_request": map[string]interface{}{
462                         "command":         []string{"echo"},
463                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
464                         "cwd":             "/",
465                         "output_path":     "/",
466                 },
467         })
468         ac2.AuthToken = ac1.AuthToken
469
470         c.Log("...post CR with good (but not yet cached) token")
471         cr = arvados.ContainerRequest{}
472         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
473         c.Assert(err, check.IsNil)
474         req.Header.Set("Content-Type", "application/json")
475         err = ac2.DoAndDecode(&cr, req)
476         c.Assert(err, check.IsNil)
477         c.Logf("err == %#v", err)
478
479         c.Log("...get user with good token")
480         u = arvados.User{}
481         req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
482         c.Assert(err, check.IsNil)
483         err = ac2.DoAndDecode(&u, req)
484         c.Check(err, check.IsNil)
485         c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
486
487         c.Log("...post CR with good cached token")
488         cr = arvados.ContainerRequest{}
489         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
490         c.Assert(err, check.IsNil)
491         req.Header.Set("Content-Type", "application/json")
492         err = ac2.DoAndDecode(&cr, req)
493         c.Check(err, check.IsNil)
494         c.Check(cr.UUID, check.Matches, "z2222-.*")
495
496         c.Log("...post with good cached token ('OAuth2 ...')")
497         cr = arvados.ContainerRequest{}
498         req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
499         c.Assert(err, check.IsNil)
500         req.Header.Set("Content-Type", "application/json")
501         req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
502         resp, err = arvados.InsecureHTTPClient.Do(req)
503         c.Assert(err, check.IsNil)
504         err = json.NewDecoder(resp.Body).Decode(&cr)
505         c.Check(err, check.IsNil)
506         c.Check(cr.UUID, check.Matches, "z2222-.*")
507 }
508
509 func (s *IntegrationSuite) TestCreateContainerRequestWithBadToken(c *check.C) {
510         conn1 := s.super.Conn("z1111")
511         rootctx1, _, _ := s.super.RootClients("z1111")
512         _, ac1, _, au := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
513
514         tests := []struct {
515                 name         string
516                 token        string
517                 expectedCode int
518         }{
519                 {"Good token", ac1.AuthToken, http.StatusOK},
520                 {"Bogus token", "abcdef", http.StatusUnauthorized},
521                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
522                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", http.StatusUnauthorized},
523         }
524
525         body, _ := json.Marshal(map[string]interface{}{
526                 "container_request": map[string]interface{}{
527                         "command":         []string{"echo"},
528                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
529                         "cwd":             "/",
530                         "output_path":     "/",
531                 },
532         })
533
534         for _, tt := range tests {
535                 c.Log(c.TestName() + " " + tt.name)
536                 ac1.AuthToken = tt.token
537                 req, err := http.NewRequest("POST", "https://"+ac1.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body))
538                 c.Assert(err, check.IsNil)
539                 req.Header.Set("Content-Type", "application/json")
540                 resp, err := ac1.Do(req)
541                 c.Assert(err, check.IsNil)
542                 c.Assert(resp.StatusCode, check.Equals, tt.expectedCode)
543         }
544 }
545
546 func (s *IntegrationSuite) TestRequestIDHeader(c *check.C) {
547         conn1 := s.super.Conn("z1111")
548         rootctx1, _, _ := s.super.RootClients("z1111")
549         userctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
550
551         coll, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{})
552         c.Check(err, check.IsNil)
553         specimen, err := conn1.SpecimenCreate(userctx1, arvados.CreateOptions{})
554         c.Check(err, check.IsNil)
555
556         tests := []struct {
557                 path            string
558                 reqIdProvided   bool
559                 notFoundRequest bool
560         }{
561                 {"/arvados/v1/collections", false, false},
562                 {"/arvados/v1/collections", true, false},
563                 {"/arvados/v1/nonexistant", false, true},
564                 {"/arvados/v1/nonexistant", true, true},
565                 {"/arvados/v1/collections/" + coll.UUID, false, false},
566                 {"/arvados/v1/collections/" + coll.UUID, true, false},
567                 {"/arvados/v1/specimens/" + specimen.UUID, false, false},
568                 {"/arvados/v1/specimens/" + specimen.UUID, true, false},
569                 // new code path (lib/controller/router etc) - single-cluster request
570                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", false, true},
571                 {"/arvados/v1/collections/z1111-4zz18-0123456789abcde", true, true},
572                 // new code path (lib/controller/router etc) - federated request
573                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", false, true},
574                 {"/arvados/v1/collections/z2222-4zz18-0123456789abcde", true, true},
575                 // old code path (proxyRailsAPI) - single-cluster request
576                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", false, true},
577                 {"/arvados/v1/specimens/z1111-j58dm-0123456789abcde", true, true},
578                 // old code path (setupProxyRemoteCluster) - federated request
579                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", false, true},
580                 {"/arvados/v1/workflows/z2222-7fd4e-0123456789abcde", true, true},
581         }
582
583         for _, tt := range tests {
584                 c.Log(c.TestName() + " " + tt.path)
585                 req, err := http.NewRequest("GET", "https://"+ac1.APIHost+tt.path, nil)
586                 c.Assert(err, check.IsNil)
587                 customReqId := "abcdeG"
588                 if !tt.reqIdProvided {
589                         c.Assert(req.Header.Get("X-Request-Id"), check.Equals, "")
590                 } else {
591                         req.Header.Set("X-Request-Id", customReqId)
592                 }
593                 resp, err := ac1.Do(req)
594                 c.Assert(err, check.IsNil)
595                 if tt.notFoundRequest {
596                         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
597                 } else {
598                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
599                 }
600                 respHdr := resp.Header.Get("X-Request-Id")
601                 if tt.reqIdProvided {
602                         c.Check(respHdr, check.Equals, customReqId)
603                 } else {
604                         c.Check(respHdr, check.Matches, `req-[0-9a-zA-Z]{20}`)
605                 }
606                 if tt.notFoundRequest {
607                         var jresp httpserver.ErrorResponse
608                         err := json.NewDecoder(resp.Body).Decode(&jresp)
609                         c.Check(err, check.IsNil)
610                         c.Assert(jresp.Errors, check.HasLen, 1)
611                         c.Check(jresp.Errors[0], check.Matches, `.*\(`+respHdr+`\).*`)
612                 }
613         }
614 }
615
616 // We test the direct access to the database
617 // normally an integration test would not have a database access, but in this case we need
618 // to test tokens that are secret, so there is no API response that will give them back
619 func (s *IntegrationSuite) dbConn(c *check.C, clusterID string) (*sql.DB, *sql.Conn) {
620         ctx := context.Background()
621         db, err := sql.Open("postgres", s.super.Cluster(clusterID).PostgreSQL.Connection.String())
622         c.Assert(err, check.IsNil)
623
624         conn, err := db.Conn(ctx)
625         c.Assert(err, check.IsNil)
626
627         rows, err := conn.ExecContext(ctx, `SELECT 1`)
628         c.Assert(err, check.IsNil)
629         n, err := rows.RowsAffected()
630         c.Assert(err, check.IsNil)
631         c.Assert(n, check.Equals, int64(1))
632         return db, conn
633 }
634
635 // TestRuntimeTokenInCR will test several different tokens in the runtime attribute
636 // and check the expected results accessing directly to the database if needed.
637 func (s *IntegrationSuite) TestRuntimeTokenInCR(c *check.C) {
638         db, dbconn := s.dbConn(c, "z1111")
639         defer db.Close()
640         defer dbconn.Close()
641         conn1 := s.super.Conn("z1111")
642         rootctx1, _, _ := s.super.RootClients("z1111")
643         userctx1, ac1, _, au := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
644
645         tests := []struct {
646                 name                 string
647                 token                string
648                 expectAToGetAValidCR bool
649                 expectedToken        *string
650         }{
651                 {"Good token z1111 user", ac1.AuthToken, true, &ac1.AuthToken},
652                 {"Bogus token", "abcdef", false, nil},
653                 {"v1-looking token", "badtoken00badtoken00badtoken00badtoken00b", false, nil},
654                 {"v2-looking token", "v2/" + au.UUID + "/badtoken00badtoken00badtoken00badtoken00b", false, nil},
655         }
656
657         for _, tt := range tests {
658                 c.Log(c.TestName() + " " + tt.name)
659
660                 rq := map[string]interface{}{
661                         "command":         []string{"echo"},
662                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
663                         "cwd":             "/",
664                         "output_path":     "/",
665                         "runtime_token":   tt.token,
666                 }
667                 cr, err := conn1.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: rq})
668                 if tt.expectAToGetAValidCR {
669                         c.Check(err, check.IsNil)
670                         c.Check(cr, check.NotNil)
671                         c.Check(cr.UUID, check.Not(check.Equals), "")
672                 }
673
674                 if tt.expectedToken == nil {
675                         continue
676                 }
677
678                 c.Logf("cr.UUID: %s", cr.UUID)
679                 row := dbconn.QueryRowContext(rootctx1, `SELECT runtime_token from container_requests where uuid=$1`, cr.UUID)
680                 c.Check(row, check.NotNil)
681                 var token sql.NullString
682                 row.Scan(&token)
683                 if c.Check(token.Valid, check.Equals, true) {
684                         c.Check(token.String, check.Equals, *tt.expectedToken)
685                 }
686         }
687 }
688
689 // TestIntermediateCluster will send a container request to
690 // one cluster with another cluster as the destination
691 // and check the tokens are being handled properly
692 func (s *IntegrationSuite) TestIntermediateCluster(c *check.C) {
693         conn1 := s.super.Conn("z1111")
694         rootctx1, _, _ := s.super.RootClients("z1111")
695         uctx1, ac1, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
696
697         tests := []struct {
698                 name                 string
699                 token                string
700                 expectedRuntimeToken string
701                 expectedUUIDprefix   string
702         }{
703                 {"Good token z1111 user sending a CR to z2222", ac1.AuthToken, "", "z2222-xvhdp-"},
704         }
705
706         for _, tt := range tests {
707                 c.Log(c.TestName() + " " + tt.name)
708                 rq := map[string]interface{}{
709                         "command":         []string{"echo"},
710                         "container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
711                         "cwd":             "/",
712                         "output_path":     "/",
713                         "runtime_token":   tt.token,
714                 }
715                 cr, err := conn1.ContainerRequestCreate(uctx1, arvados.CreateOptions{ClusterID: "z2222", Attrs: rq})
716
717                 c.Check(err, check.IsNil)
718                 c.Check(strings.HasPrefix(cr.UUID, tt.expectedUUIDprefix), check.Equals, true)
719                 c.Check(cr.RuntimeToken, check.Equals, tt.expectedRuntimeToken)
720         }
721 }
722
723 // Test for #17785
724 func (s *IntegrationSuite) TestFederatedApiClientAuthHandling(c *check.C) {
725         rootctx1, rootclnt1, _ := s.super.RootClients("z1111")
726         conn1 := s.super.Conn("z1111")
727
728         // Make sure LoginCluster is properly configured
729         for _, cls := range []string{"z1111", "z3333"} {
730                 c.Check(
731                         s.super.Cluster(cls).Login.LoginCluster,
732                         check.Equals, "z1111",
733                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
734         }
735         // Get user's UUID & attempt to create a token for it on the remote cluster
736         _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1,
737                 "user@example.com", true)
738         _, rootclnt3, _ := s.super.ClientsWithToken("z3333", rootclnt1.AuthToken)
739         var resp arvados.APIClientAuthorization
740         err := rootclnt3.RequestAndDecode(
741                 &resp, "POST", "arvados/v1/api_client_authorizations", nil,
742                 map[string]interface{}{
743                         "api_client_authorization": map[string]string{
744                                 "owner_uuid": user.UUID,
745                         },
746                 },
747         )
748         c.Assert(err, check.IsNil)
749         c.Assert(resp.APIClientID, check.Not(check.Equals), 0)
750         newTok := resp.TokenV2()
751         c.Assert(newTok, check.Not(check.Equals), "")
752
753         // Confirm the token is from z1111
754         c.Assert(strings.HasPrefix(newTok, "v2/z1111-gj3su-"), check.Equals, true)
755
756         // Confirm the token works and is from the correct user
757         _, rootclnt3bis, _ := s.super.ClientsWithToken("z3333", newTok)
758         var curUser arvados.User
759         err = rootclnt3bis.RequestAndDecode(
760                 &curUser, "GET", "arvados/v1/users/current", nil, nil,
761         )
762         c.Assert(err, check.IsNil)
763         c.Assert(curUser.UUID, check.Equals, user.UUID)
764
765         // Request the ApiClientAuthorization list using the new token
766         _, userClient, _ := s.super.ClientsWithToken("z3333", newTok)
767         var acaLst arvados.APIClientAuthorizationList
768         err = userClient.RequestAndDecode(
769                 &acaLst, "GET", "arvados/v1/api_client_authorizations", nil, nil,
770         )
771         c.Assert(err, check.IsNil)
772 }
773
774 // Test for bug #18076
775 func (s *IntegrationSuite) TestStaleCachedUserRecord(c *check.C) {
776         rootctx1, _, _ := s.super.RootClients("z1111")
777         _, rootclnt3, _ := s.super.RootClients("z3333")
778         conn1 := s.super.Conn("z1111")
779         conn3 := s.super.Conn("z3333")
780
781         // Make sure LoginCluster is properly configured
782         for _, cls := range []string{"z1111", "z3333"} {
783                 c.Check(
784                         s.super.Cluster(cls).Login.LoginCluster,
785                         check.Equals, "z1111",
786                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
787         }
788
789         for testCaseNr, testCase := range []struct {
790                 name           string
791                 withRepository bool
792         }{
793                 {"User without local repository", false},
794                 {"User with local repository", true},
795         } {
796                 c.Log(c.TestName() + " " + testCase.name)
797                 // Create some users, request them on the federated cluster so they're cached.
798                 var users []arvados.User
799                 for userNr := 0; userNr < 2; userNr++ {
800                         _, _, _, user := s.super.UserClients("z1111",
801                                 rootctx1,
802                                 c,
803                                 conn1,
804                                 fmt.Sprintf("user%d%d@example.com", testCaseNr, userNr),
805                                 true)
806                         c.Assert(user.Username, check.Not(check.Equals), "")
807                         users = append(users, user)
808
809                         lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
810                         c.Assert(err, check.Equals, nil)
811                         userFound := false
812                         for _, fedUser := range lst.Items {
813                                 if fedUser.UUID == user.UUID {
814                                         c.Assert(fedUser.Username, check.Equals, user.Username)
815                                         userFound = true
816                                         break
817                                 }
818                         }
819                         c.Assert(userFound, check.Equals, true)
820
821                         if testCase.withRepository {
822                                 var repo interface{}
823                                 err = rootclnt3.RequestAndDecode(
824                                         &repo, "POST", "arvados/v1/repositories", nil,
825                                         map[string]interface{}{
826                                                 "repository": map[string]string{
827                                                         "name":       fmt.Sprintf("%s/test", user.Username),
828                                                         "owner_uuid": user.UUID,
829                                                 },
830                                         },
831                                 )
832                                 c.Assert(err, check.IsNil)
833                         }
834                 }
835
836                 // Swap the usernames
837                 _, err := conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
838                         UUID: users[0].UUID,
839                         Attrs: map[string]interface{}{
840                                 "username": "",
841                         },
842                 })
843                 c.Assert(err, check.Equals, nil)
844                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
845                         UUID: users[1].UUID,
846                         Attrs: map[string]interface{}{
847                                 "username": users[0].Username,
848                         },
849                 })
850                 c.Assert(err, check.Equals, nil)
851                 _, err = conn1.UserUpdate(rootctx1, arvados.UpdateOptions{
852                         UUID: users[0].UUID,
853                         Attrs: map[string]interface{}{
854                                 "username": users[1].Username,
855                         },
856                 })
857                 c.Assert(err, check.Equals, nil)
858
859                 // Re-request the list on the federated cluster & check for updates
860                 lst, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
861                 c.Assert(err, check.Equals, nil)
862                 var user0Found, user1Found bool
863                 for _, user := range lst.Items {
864                         if user.UUID == users[0].UUID {
865                                 user0Found = true
866                                 c.Assert(user.Username, check.Equals, users[1].Username)
867                         } else if user.UUID == users[1].UUID {
868                                 user1Found = true
869                                 c.Assert(user.Username, check.Equals, users[0].Username)
870                         }
871                 }
872                 c.Assert(user0Found, check.Equals, true)
873                 c.Assert(user1Found, check.Equals, true)
874         }
875 }
876
877 // Test for bug #16263
878 func (s *IntegrationSuite) TestListUsers(c *check.C) {
879         rootctx1, _, _ := s.super.RootClients("z1111")
880         conn1 := s.super.Conn("z1111")
881         conn3 := s.super.Conn("z3333")
882         userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
883
884         // Make sure LoginCluster is properly configured
885         for _, cls := range []string{"z1111", "z2222", "z3333"} {
886                 c.Check(
887                         s.super.Cluster(cls).Login.LoginCluster,
888                         check.Equals, "z1111",
889                         check.Commentf("incorrect LoginCluster config on cluster %q", cls))
890         }
891         // Make sure z1111 has users with NULL usernames
892         lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
893                 Limit: math.MaxInt64, // check that large limit works (see #16263)
894         })
895         nullUsername := false
896         c.Assert(err, check.IsNil)
897         c.Assert(len(lst.Items), check.Not(check.Equals), 0)
898         for _, user := range lst.Items {
899                 if user.Username == "" {
900                         nullUsername = true
901                         break
902                 }
903         }
904         c.Assert(nullUsername, check.Equals, true)
905
906         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
907         c.Assert(err, check.IsNil)
908         c.Check(user1.IsActive, check.Equals, true)
909
910         // Ask for the user list on z3333 using z1111's system root token
911         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
912         c.Assert(err, check.IsNil)
913         found := false
914         for _, user := range lst.Items {
915                 if user.UUID == user1.UUID {
916                         c.Check(user.IsActive, check.Equals, true)
917                         found = true
918                         break
919                 }
920         }
921         c.Check(found, check.Equals, true)
922
923         // Deactivate user acct on z1111
924         _, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
925         c.Assert(err, check.IsNil)
926
927         // Get user list from z3333, check the returned z1111 user is
928         // deactivated
929         lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
930         c.Assert(err, check.IsNil)
931         found = false
932         for _, user := range lst.Items {
933                 if user.UUID == user1.UUID {
934                         c.Check(user.IsActive, check.Equals, false)
935                         found = true
936                         break
937                 }
938         }
939         c.Check(found, check.Equals, true)
940
941         // Deactivated user no longer has working token
942         user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
943         c.Assert(err, check.ErrorMatches, `.*401 Unauthorized.*`)
944 }
945
946 func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) {
947         conn1 := s.super.Conn("z1111")
948         conn3 := s.super.Conn("z3333")
949         rootctx1, rootac1, _ := s.super.RootClients("z1111")
950
951         // Create user on LoginCluster z1111
952         _, _, _, user := s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
953
954         // Make a new root token (because rootClients() uses SystemRootToken)
955         var outAuth arvados.APIClientAuthorization
956         err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil)
957         c.Check(err, check.IsNil)
958
959         // Make a v2 root token to communicate with z3333
960         rootctx3, rootac3, _ := s.super.ClientsWithToken("z3333", outAuth.TokenV2())
961
962         // Create VM on z3333
963         var outVM arvados.VirtualMachine
964         err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil,
965                 map[string]interface{}{"virtual_machine": map[string]interface{}{
966                         "hostname": "example",
967                 },
968                 })
969         c.Check(outVM.UUID[0:5], check.Equals, "z3333")
970         c.Check(err, check.IsNil)
971
972         // Make sure z3333 user list is up to date
973         _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000})
974         c.Check(err, check.IsNil)
975
976         // Try to set up user on z3333 with the VM
977         _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID})
978         c.Check(err, check.IsNil)
979
980         var outLinks arvados.LinkList
981         err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil,
982                 arvados.ListOptions{
983                         Limit: 1000,
984                         Filters: []arvados.Filter{
985                                 {
986                                         Attr:     "tail_uuid",
987                                         Operator: "=",
988                                         Operand:  user.UUID,
989                                 },
990                                 {
991                                         Attr:     "head_uuid",
992                                         Operator: "=",
993                                         Operand:  outVM.UUID,
994                                 },
995                                 {
996                                         Attr:     "name",
997                                         Operator: "=",
998                                         Operand:  "can_login",
999                                 },
1000                                 {
1001                                         Attr:     "link_class",
1002                                         Operator: "=",
1003                                         Operand:  "permission",
1004                                 }}})
1005         c.Check(err, check.IsNil)
1006
1007         c.Check(len(outLinks.Items), check.Equals, 1)
1008 }
1009
1010 func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
1011         conn1 := s.super.Conn("z1111")
1012         rootctx1, _, _ := s.super.RootClients("z1111")
1013         s.super.UserClients("z1111", rootctx1, c, conn1, s.oidcprovider.AuthEmail, true)
1014
1015         accesstoken := s.oidcprovider.ValidAccessToken()
1016
1017         for _, clusterID := range []string{"z1111", "z2222"} {
1018
1019                 var coll arvados.Collection
1020
1021                 // Write some file data and create a collection
1022                 {
1023                         c.Logf("save collection to %s", clusterID)
1024
1025                         conn := s.super.Conn(clusterID)
1026                         ctx, ac, kc := s.super.ClientsWithToken(clusterID, accesstoken)
1027
1028                         fs, err := coll.FileSystem(ac, kc)
1029                         c.Assert(err, check.IsNil)
1030                         f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
1031                         c.Assert(err, check.IsNil)
1032                         _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
1033                         c.Assert(err, check.IsNil)
1034                         err = f.Close()
1035                         c.Assert(err, check.IsNil)
1036                         mtxt, err := fs.MarshalManifest(".")
1037                         c.Assert(err, check.IsNil)
1038                         coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
1039                                 "manifest_text": mtxt,
1040                         }})
1041                         c.Assert(err, check.IsNil)
1042                 }
1043
1044                 // Read the collection & file data -- both from the
1045                 // cluster where it was created, and from the other
1046                 // cluster.
1047                 for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
1048                         c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
1049
1050                         conn := s.super.Conn(readClusterID)
1051                         ctx, ac, kc := s.super.ClientsWithToken(readClusterID, accesstoken)
1052
1053                         user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
1054                         c.Assert(err, check.IsNil)
1055                         c.Check(user.FullName, check.Equals, "Example User")
1056                         readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
1057                         c.Assert(err, check.IsNil)
1058                         c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
1059                         fs, err := readcoll.FileSystem(ac, kc)
1060                         c.Assert(err, check.IsNil)
1061                         f, err := fs.Open("test.txt")
1062                         c.Assert(err, check.IsNil)
1063                         buf, err := ioutil.ReadAll(f)
1064                         c.Assert(err, check.IsNil)
1065                         c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
1066                 }
1067         }
1068 }
1069
1070 // z3333 should not forward a locally-issued container runtime token,
1071 // associated with a z1111 user, to its login cluster z1111. z1111
1072 // would only call back to z3333 and then reject the response because
1073 // the user ID does not match the token prefix. See
1074 // dev.arvados.org/issues/18346
1075 func (s *IntegrationSuite) TestForwardRuntimeTokenToLoginCluster(c *check.C) {
1076         db3, db3conn := s.dbConn(c, "z3333")
1077         defer db3.Close()
1078         defer db3conn.Close()
1079         rootctx1, _, _ := s.super.RootClients("z1111")
1080         rootctx3, _, _ := s.super.RootClients("z3333")
1081         conn1 := s.super.Conn("z1111")
1082         conn3 := s.super.Conn("z3333")
1083         userctx1, _, _, _ := s.super.UserClients("z1111", rootctx1, c, conn1, "user@example.com", true)
1084
1085         user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
1086         c.Assert(err, check.IsNil)
1087         c.Logf("user1 %+v", user1)
1088
1089         imageColl, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1090                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n",
1091         }})
1092         c.Assert(err, check.IsNil)
1093         c.Logf("imageColl %+v", imageColl)
1094
1095         cr, err := conn3.ContainerRequestCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
1096                 "state":           "Committed",
1097                 "command":         []string{"echo"},
1098                 "container_image": imageColl.PortableDataHash,
1099                 "cwd":             "/",
1100                 "output_path":     "/",
1101                 "priority":        1,
1102                 "runtime_constraints": arvados.RuntimeConstraints{
1103                         VCPUs: 1,
1104                         RAM:   1000000000,
1105                 },
1106         }})
1107         c.Assert(err, check.IsNil)
1108         c.Logf("container request %+v", cr)
1109         ctr, err := conn3.ContainerLock(rootctx3, arvados.GetOptions{UUID: cr.ContainerUUID})
1110         c.Assert(err, check.IsNil)
1111         c.Logf("container %+v", ctr)
1112
1113         // We could use conn3.ContainerAuth() here, but that API
1114         // hasn't been added to sdk/go/arvados/api.go yet.
1115         row := db3conn.QueryRowContext(context.Background(), `SELECT api_token from api_client_authorizations where uuid=$1`, ctr.AuthUUID)
1116         c.Check(row, check.NotNil)
1117         var val sql.NullString
1118         row.Scan(&val)
1119         c.Assert(val.Valid, check.Equals, true)
1120         runtimeToken := "v2/" + ctr.AuthUUID + "/" + val.String
1121         ctrctx, _, _ := s.super.ClientsWithToken("z3333", runtimeToken)
1122         c.Logf("container runtime token %+v", runtimeToken)
1123
1124         _, err = conn3.UserGet(ctrctx, arvados.GetOptions{UUID: user1.UUID})
1125         c.Assert(err, check.NotNil)
1126         c.Check(err, check.ErrorMatches, `request failed: .* 401 Unauthorized: cannot use a locally issued token to forward a request to our login cluster \(z1111\)`)
1127         c.Check(err, check.Not(check.ErrorMatches), `(?ms).*127\.0\.0\.11.*`)
1128 }
1129
1130 func (s *IntegrationSuite) TestRunTrivialContainer(c *check.C) {
1131         outcoll, _ := s.runContainer(c, "z1111", "", map[string]interface{}{
1132                 "command":             []string{"sh", "-c", "touch \"/out/hello world\" /out/ohai"},
1133                 "container_image":     "busybox:uclibc",
1134                 "cwd":                 "/tmp",
1135                 "environment":         map[string]string{},
1136                 "mounts":              map[string]arvados.Mount{"/out": {Kind: "tmp", Capacity: 10000}},
1137                 "output_path":         "/out",
1138                 "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26},
1139                 "priority":            1,
1140                 "state":               arvados.ContainerRequestStateCommitted,
1141         }, 0)
1142         c.Check(outcoll.ManifestText, check.Matches, `\. d41d8.* 0:0:hello\\040world 0:0:ohai\n`)
1143         c.Check(outcoll.PortableDataHash, check.Equals, "8fa5dee9231a724d7cf377c5a2f4907c+65")
1144 }
1145
1146 func (s *IntegrationSuite) TestContainerInputOnDifferentCluster(c *check.C) {
1147         conn := s.super.Conn("z1111")
1148         rootctx, _, _ := s.super.RootClients("z1111")
1149         userctx, ac, _, _ := s.super.UserClients("z1111", rootctx, c, conn, s.oidcprovider.AuthEmail, true)
1150         z1coll, err := conn.CollectionCreate(userctx, arvados.CreateOptions{Attrs: map[string]interface{}{
1151                 "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:ocelot\n",
1152         }})
1153         c.Assert(err, check.IsNil)
1154
1155         outcoll, logcfs := s.runContainer(c, "z2222", ac.AuthToken, map[string]interface{}{
1156                 "command":         []string{"ls", "/in"},
1157                 "container_image": "busybox:uclibc",
1158                 "cwd":             "/tmp",
1159                 "environment":     map[string]string{},
1160                 "mounts": map[string]arvados.Mount{
1161                         "/in":  {Kind: "collection", PortableDataHash: z1coll.PortableDataHash},
1162                         "/out": {Kind: "tmp", Capacity: 10000},
1163                 },
1164                 "output_path":         "/out",
1165                 "runtime_constraints": arvados.RuntimeConstraints{RAM: 100000000, VCPUs: 1, KeepCacheRAM: 1 << 26},
1166                 "priority":            1,
1167                 "state":               arvados.ContainerRequestStateCommitted,
1168                 "container_count_max": 1,
1169         }, -1)
1170         if outcoll.UUID == "" {
1171                 arvmountlog, err := fs.ReadFile(arvados.FS(logcfs), "/arv-mount.txt")
1172                 c.Check(err, check.IsNil)
1173                 c.Check(string(arvmountlog), check.Matches, `(?ms).*cannot use a locally issued token to forward a request to our login cluster \(z1111\).*`)
1174                 c.Skip("this use case is not supported yet")
1175         }
1176         stdout, err := fs.ReadFile(arvados.FS(logcfs), "/stdout.txt")
1177         c.Check(err, check.IsNil)
1178         c.Check(string(stdout), check.Equals, "ocelot\n")
1179 }
1180
1181 func (s *IntegrationSuite) runContainer(c *check.C, clusterID string, token string, ctrSpec map[string]interface{}, expectExitCode int) (outcoll arvados.Collection, logcfs arvados.CollectionFileSystem) {
1182         conn := s.super.Conn(clusterID)
1183         rootctx, _, _ := s.super.RootClients(clusterID)
1184         if token == "" {
1185                 _, ac, _, _ := s.super.UserClients(clusterID, rootctx, c, conn, s.oidcprovider.AuthEmail, true)
1186                 token = ac.AuthToken
1187         }
1188         _, ac, kc := s.super.ClientsWithToken(clusterID, token)
1189
1190         c.Log("[docker load]")
1191         out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput()
1192         c.Logf("[docker load output] %s", out)
1193         c.Check(err, check.IsNil)
1194
1195         c.Log("[arv-keepdocker]")
1196         akd := exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc")
1197         akd.Env = append(os.Environ(), "ARVADOS_API_HOST="+ac.APIHost, "ARVADOS_API_HOST_INSECURE=1", "ARVADOS_API_TOKEN="+ac.AuthToken)
1198         out, err = akd.CombinedOutput()
1199         c.Logf("[arv-keepdocker output]\n%s", out)
1200         c.Check(err, check.IsNil)
1201
1202         var cr arvados.ContainerRequest
1203         err = ac.RequestAndDecode(&cr, "POST", "/arvados/v1/container_requests", nil, map[string]interface{}{
1204                 "container_request": ctrSpec,
1205         })
1206         c.Assert(err, check.IsNil)
1207
1208         showlogs := func(collectionID string) arvados.CollectionFileSystem {
1209                 var logcoll arvados.Collection
1210                 err = ac.RequestAndDecode(&logcoll, "GET", "/arvados/v1/collections/"+collectionID, nil, nil)
1211                 c.Assert(err, check.IsNil)
1212                 cfs, err := logcoll.FileSystem(ac, kc)
1213                 c.Assert(err, check.IsNil)
1214                 fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
1215                         if d.IsDir() || strings.HasPrefix(path, "/log for container") {
1216                                 return nil
1217                         }
1218                         f, err := cfs.Open(path)
1219                         c.Assert(err, check.IsNil)
1220                         defer f.Close()
1221                         buf, err := ioutil.ReadAll(f)
1222                         c.Assert(err, check.IsNil)
1223                         c.Logf("=== %s\n%s\n", path, buf)
1224                         return nil
1225                 })
1226                 return cfs
1227         }
1228
1229         var ctr arvados.Container
1230         var lastState arvados.ContainerState
1231         deadline := time.Now().Add(time.Minute)
1232         for cr.State != arvados.ContainerRequestStateFinal {
1233                 err = ac.RequestAndDecode(&cr, "GET", "/arvados/v1/container_requests/"+cr.UUID, nil, nil)
1234                 c.Assert(err, check.IsNil)
1235                 err = ac.RequestAndDecode(&ctr, "GET", "/arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
1236                 if err != nil {
1237                         c.Logf("error getting container state: %s", err)
1238                 } else if ctr.State != lastState {
1239                         c.Logf("container state changed to %q", ctr.State)
1240                         lastState = ctr.State
1241                 } else {
1242                         if time.Now().After(deadline) {
1243                                 c.Errorf("timed out, container state is %q", cr.State)
1244                                 showlogs(ctr.Log)
1245                                 c.FailNow()
1246                         }
1247                         time.Sleep(time.Second / 2)
1248                 }
1249         }
1250         c.Logf("cr.CumulativeCost == %f", cr.CumulativeCost)
1251         c.Check(cr.CumulativeCost, check.Not(check.Equals), 0.0)
1252         if expectExitCode >= 0 {
1253                 c.Check(ctr.State, check.Equals, arvados.ContainerStateComplete)
1254                 c.Check(ctr.ExitCode, check.Equals, expectExitCode)
1255                 err = ac.RequestAndDecode(&outcoll, "GET", "/arvados/v1/collections/"+cr.OutputUUID, nil, nil)
1256                 c.Assert(err, check.IsNil)
1257         }
1258         logcfs = showlogs(cr.LogUUID)
1259         return outcoll, logcfs
1260 }