X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c9b8b9b9c78a77dd30b828914c8bee9fa8dcbb90..7415e01eb23989648fb4850b5a0569796973445f:/cmd/arvados-client/container_gateway_test.go diff --git a/cmd/arvados-client/container_gateway_test.go b/cmd/arvados-client/container_gateway_test.go index 743b91d69b..016b793f3f 100644 --- a/cmd/arvados-client/container_gateway_test.go +++ b/cmd/arvados-client/container_gateway_test.go @@ -10,6 +10,7 @@ import ( "crypto/hmac" "crypto/sha256" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -24,34 +25,48 @@ import ( "git.arvados.org/arvados.git/lib/controller/rpc" "git.arvados.org/arvados.git/lib/crunchrun" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/httpserver" + "git.arvados.org/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) -func (s *ClientSuite) TestShellGatewayNotAvailable(c *check.C) { - var stdout, stderr bytes.Buffer - cmd := exec.Command("go", "run", ".", "shell", arvadostest.QueuedContainerUUID, "-o", "controlpath=none", "echo", "ok") - cmd.Env = append(cmd.Env, os.Environ()...) - cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - c.Check(cmd.Run(), check.NotNil) - c.Log(stderr.String()) - c.Check(stderr.String(), check.Matches, `(?ms).*container is not running yet \(state is "Queued"\).*`) +var _ = check.Suite(&shellSuite{}) + +type shellSuite struct { + gobindir string + homedir string + runningUUID string } -func (s *ClientSuite) TestShellGateway(c *check.C) { - defer func() { - c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil) - }() - uuid := arvadostest.QueuedContainerUUID +func (s *shellSuite) SetUpSuite(c *check.C) { + tmpdir := c.MkDir() + s.gobindir = tmpdir + "/bin" + c.Check(os.Mkdir(s.gobindir, 0777), check.IsNil) + s.homedir = tmpdir + "/home" + c.Check(os.Mkdir(s.homedir, 0777), check.IsNil) + + // We explicitly build a client binary in our tempdir here, + // instead of using "go run .", because (a) we're going to + // invoke the same binary several times, and (b) we're going + // to change $HOME to a temp dir in some of the tests, which + // would force "go run ." to recompile the world instead of + // using the cached object files in the real $HOME. + c.Logf("building arvados-client binary in %s", s.gobindir) + cmd := exec.Command("go", "install", ".") + cmd.Env = append(os.Environ(), "GOBIN="+s.gobindir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + c.Assert(cmd.Run(), check.IsNil) + + s.runningUUID = arvadostest.RunningContainerUUID h := hmac.New(sha256.New, []byte(arvadostest.SystemRootToken)) - fmt.Fprint(h, uuid) + fmt.Fprint(h, s.runningUUID) authSecret := fmt.Sprintf("%x", h.Sum(nil)) gw := crunchrun.Gateway{ - ContainerUUID: uuid, + ContainerUUID: s.runningUUID, Address: "0.0.0.0:0", AuthSecret: authSecret, Log: ctxlog.TestLogger(c), @@ -72,32 +87,78 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { func(context.Context) ([]string, error) { return []string{arvadostest.SystemRootToken}, nil }) - _, err = rpcconn.ContainerUpdate(context.TODO(), arvados.UpdateOptions{UUID: uuid, Attrs: map[string]interface{}{ - "state": arvados.ContainerStateLocked, - }}) - c.Assert(err, check.IsNil) - _, err = rpcconn.ContainerUpdate(context.TODO(), arvados.UpdateOptions{UUID: uuid, Attrs: map[string]interface{}{ - "state": arvados.ContainerStateRunning, + _, err = rpcconn.ContainerUpdate(context.TODO(), arvados.UpdateOptions{UUID: s.runningUUID, Attrs: map[string]interface{}{ "gateway_address": gw.Address, }}) c.Assert(err, check.IsNil) +} +func (s *shellSuite) TearDownSuite(c *check.C) { + c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil) +} + +func (s *shellSuite) TestShellGatewayNotAvailable(c *check.C) { var stdout, stderr bytes.Buffer - cmd := exec.Command("go", "run", ".", "shell", uuid, "-o", "controlpath=none", "-o", "userknownhostsfile="+c.MkDir()+"/known_hosts", "echo", "ok") + cmd := exec.Command(s.gobindir+"/arvados-client", "shell", arvadostest.QueuedContainerUUID, "-o", "controlpath=none", "echo", "ok") cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2) cmd.Stdout = &stdout cmd.Stderr = &stderr + c.Check(cmd.Run(), check.NotNil) + c.Log(stderr.String()) + c.Check(stderr.String(), check.Matches, `(?ms).*container is not running yet \(state is "Queued"\).*`) +} + +func (s *shellSuite) TestShellGatewayUsingEnvVars(c *check.C) { + s.testShellGateway(c, false) +} +func (s *shellSuite) TestShellGatewayUsingSettingsConf(c *check.C) { + s.testShellGateway(c, true) +} +func (s *shellSuite) testShellGateway(c *check.C, useSettingsConf bool) { + var stdout, stderr bytes.Buffer + cmd := exec.Command( + s.gobindir+"/arvados-client", "shell", s.runningUUID, + "-o", "controlpath=none", + "-o", "userknownhostsfile="+s.homedir+"/known_hosts", + "echo", "ok") + if useSettingsConf { + settings := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\nARVADOS_API_TOKEN=" + arvadostest.ActiveTokenV2 + "\nARVADOS_API_HOST_INSECURE=true\n" + err := os.MkdirAll(s.homedir+"/.config/arvados", 0777) + c.Assert(err, check.IsNil) + err = os.WriteFile(s.homedir+"/.config/arvados/settings.conf", []byte(settings), 0777) + c.Assert(err, check.IsNil) + for _, kv := range os.Environ() { + if !strings.HasPrefix(kv, "ARVADOS_") && !strings.HasPrefix(kv, "HOME=") { + cmd.Env = append(cmd.Env, kv) + } + } + cmd.Env = append(cmd.Env, "HOME="+s.homedir) + } else { + err := os.Remove(s.homedir + "/.config/arvados/settings.conf") + if !os.IsNotExist(err) { + c.Assert(err, check.IsNil) + } + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2) + } + cmd.Stdout = &stdout + cmd.Stderr = &stderr stdin, err := cmd.StdinPipe() c.Assert(err, check.IsNil) go fmt.Fprintln(stdin, "data appears on stdin, but stdin does not close; cmd should exit anyway, not hang") - time.AfterFunc(5*time.Second, func() { + timeout := time.AfterFunc(5*time.Second, func() { c.Errorf("timed out -- remote end is probably hung waiting for us to close stdin") stdin.Close() }) + c.Logf("cmd.Args: %s", cmd.Args) c.Check(cmd.Run(), check.IsNil) + timeout.Stop() c.Check(stdout.String(), check.Equals, "ok\n") +} +func (s *shellSuite) TestShellGatewayPortForwarding(c *check.C) { + c.Log("setting up an http server") // Set up an http server, and try using "arvados-client shell" // to forward traffic to it. httpTarget := &httpserver.Server{} @@ -109,7 +170,7 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { w.WriteHeader(http.StatusNotFound) } }) - err = httpTarget.Start() + err := httpTarget.Start() c.Assert(err, check.IsNil) ln, err := net.Listen("tcp", ":0") @@ -117,22 +178,22 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { _, forwardedPort, _ := net.SplitHostPort(ln.Addr().String()) ln.Close() - stdout.Reset() - stderr.Reset() + c.Log("connecting") + var stdout, stderr bytes.Buffer ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second)) defer cancel() - cmd = exec.CommandContext(ctx, - "go", "run", ".", "shell", uuid, + cmd := exec.CommandContext(ctx, + s.gobindir+"/arvados-client", "shell", s.runningUUID, "-L", forwardedPort+":"+httpTarget.Addr, "-o", "controlpath=none", - "-o", "userknownhostsfile="+c.MkDir()+"/known_hosts", + "-o", "userknownhostsfile="+s.homedir+"/known_hosts", "-N", ) - c.Logf("cmd.Args: %s", cmd.Args) cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2) cmd.Stdout = &stdout cmd.Stderr = &stderr + c.Logf("cmd.Args: %s", cmd.Args) cmd.Start() forwardedURL := fmt.Sprintf("http://localhost:%s/foo", forwardedPort) @@ -178,3 +239,150 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { } wg.Wait() } + +var _ = check.Suite(&logsSuite{}) + +type logsSuite struct{} + +func (s *logsSuite) TestContainerRequestLog(c *check.C) { + arvadostest.StartKeep(2, true) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(30*time.Second)) + defer cancel() + + rpcconn := rpc.NewConn("", + &url.URL{ + Scheme: "https", + Host: os.Getenv("ARVADOS_API_HOST"), + }, + true, + func(context.Context) ([]string, error) { + return []string{arvadostest.SystemRootToken}, nil + }) + imageColl, err := rpcconn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n", + }}) + c.Assert(err, check.IsNil) + c.Logf("imageColl %+v", imageColl) + cr, err := rpcconn.ContainerRequestCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "state": "Committed", + "command": []string{"echo", fmt.Sprintf("%d", time.Now().Unix())}, + "container_image": imageColl.PortableDataHash, + "cwd": "/", + "output_path": "/", + "priority": 1, + "runtime_constraints": arvados.RuntimeConstraints{ + VCPUs: 1, + RAM: 1000000000, + }, + "container_count_max": 1, + }}) + c.Assert(err, check.IsNil) + h := hmac.New(sha256.New, []byte(arvadostest.SystemRootToken)) + fmt.Fprint(h, cr.ContainerUUID) + authSecret := fmt.Sprintf("%x", h.Sum(nil)) + + coll := arvados.Collection{} + client := arvados.NewClientFromEnv() + ac, err := arvadosclient.New(client) + c.Assert(err, check.IsNil) + kc, err := keepclient.MakeKeepClient(ac) + c.Assert(err, check.IsNil) + cfs, err := coll.FileSystem(client, kc) + c.Assert(err, check.IsNil) + + c.Log("running logs command on queued container") + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "go", "run", ".", "logs", "-f", "-poll=250ms", cr.UUID) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.SystemRootToken) + cmd.Stdout = io.MultiWriter(&stdout, os.Stderr) + cmd.Stderr = io.MultiWriter(&stderr, os.Stderr) + err = cmd.Start() + c.Assert(err, check.Equals, nil) + + c.Log("changing container state to Locked") + _, err = rpcconn.ContainerUpdate(ctx, arvados.UpdateOptions{UUID: cr.ContainerUUID, Attrs: map[string]interface{}{ + "state": arvados.ContainerStateLocked, + }}) + c.Assert(err, check.IsNil) + c.Log("starting gateway") + gw := crunchrun.Gateway{ + ContainerUUID: cr.ContainerUUID, + Address: "0.0.0.0:0", + AuthSecret: authSecret, + Log: ctxlog.TestLogger(c), + Target: crunchrun.GatewayTargetStub{}, + LogCollection: cfs, + } + err = gw.Start() + c.Assert(err, check.IsNil) + c.Log("updating container gateway address") + _, err = rpcconn.ContainerUpdate(ctx, arvados.UpdateOptions{UUID: cr.ContainerUUID, Attrs: map[string]interface{}{ + "gateway_address": gw.Address, + "state": arvados.ContainerStateRunning, + }}) + c.Assert(err, check.IsNil) + + const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + fCrunchrun, err := cfs.OpenFile("crunch-run.txt", os.O_CREATE|os.O_WRONLY, 0777) + c.Assert(err, check.IsNil) + _, err = fmt.Fprintf(fCrunchrun, "%s line 1 of crunch-run.txt\n", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + fStderr, err := cfs.OpenFile("stderr.txt", os.O_CREATE|os.O_WRONLY, 0777) + c.Assert(err, check.IsNil) + _, err = fmt.Fprintf(fStderr, "%s line 1 of stderr\n", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + + { + // Without "-f", just show the existing logs and + // exit. Timeout needs to be long enough for "go run". + ctxNoFollow, cancel := context.WithDeadline(ctx, time.Now().Add(time.Second*5)) + defer cancel() + cmdNoFollow := exec.CommandContext(ctxNoFollow, "go", "run", ".", "logs", "-poll=250ms", cr.UUID) + buf, err := cmdNoFollow.CombinedOutput() + c.Check(err, check.IsNil) + c.Check(string(buf), check.Matches, `(?ms).*line 1 of stderr\n`) + } + + time.Sleep(time.Second * 2) + _, err = fmt.Fprintf(fCrunchrun, "%s line 2 of crunch-run.txt", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + _, err = fmt.Fprintf(fStderr, "%s --end--", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !strings.Contains(stdout.String(), "--end--"); time.Sleep(time.Second / 10) { + } + c.Check(stdout.String(), check.Matches, `(?ms).*stderr\.txt +20\S+Z --end--\n.*`) + + mtxt, err := cfs.MarshalManifest(".") + c.Assert(err, check.IsNil) + savedLog, err := rpcconn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": mtxt, + }}) + c.Assert(err, check.IsNil) + _, err = rpcconn.ContainerUpdate(ctx, arvados.UpdateOptions{UUID: cr.ContainerUUID, Attrs: map[string]interface{}{ + "state": arvados.ContainerStateComplete, + "log": savedLog.PortableDataHash, + "output": "d41d8cd98f00b204e9800998ecf8427e+0", + "exit_code": 0, + }}) + c.Assert(err, check.IsNil) + + err = cmd.Wait() + c.Check(err, check.IsNil) + // Ensure controller doesn't cheat by fetching data from the + // gateway after the container is complete. + gw.LogCollection = nil + + c.Logf("re-running logs command on completed container") + { + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(time.Second*5)) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "run", ".", "logs", "-f", cr.UUID) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.SystemRootToken) + buf, err := cmd.CombinedOutput() + c.Check(err, check.Equals, nil) + c.Check(string(buf), check.Matches, `(?ms).*--end--\n`) + } +}