X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3cdc942884fe529b2ba973a20c92029aa55e6735..50e724b9f45eb25825486e17839e4d2867343caf:/cmd/arvados-client/container_gateway_test.go diff --git a/cmd/arvados-client/container_gateway_test.go b/cmd/arvados-client/container_gateway_test.go index 62303eab63..761a7c3ef5 100644 --- a/cmd/arvados-client/container_gateway_test.go +++ b/cmd/arvados-client/container_gateway_test.go @@ -10,6 +10,7 @@ import ( "crypto/hmac" "crypto/sha256" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -18,13 +19,17 @@ import ( "os/exec" "strings" "sync" + "syscall" "time" "git.arvados.org/arvados.git/lib/controller/rpc" "git.arvados.org/arvados.git/lib/crunchrun" "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/httpserver" + "git.arvados.org/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) @@ -48,16 +53,15 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { h := hmac.New(sha256.New, []byte(arvadostest.SystemRootToken)) fmt.Fprint(h, uuid) authSecret := fmt.Sprintf("%x", h.Sum(nil)) - dcid := "theperthcountyconspiracy" gw := crunchrun.Gateway{ - DockerContainerID: &dcid, - ContainerUUID: uuid, - Address: "0.0.0.0:0", - AuthSecret: authSecret, + ContainerUUID: uuid, + Address: "0.0.0.0:0", + AuthSecret: authSecret, + Log: ctxlog.TestLogger(c), // Just forward connections to localhost instead of a // container, so we can test without running a // container. - ContainerIPAddress: func() (string, error) { return "0.0.0.0", nil }, + Target: crunchrun.GatewayTargetStub{}, } err := gw.Start() c.Assert(err, check.IsNil) @@ -87,9 +91,15 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2) cmd.Stdout = &stdout cmd.Stderr = &stderr - c.Check(cmd.Run(), check.NotNil) - c.Log(stderr.String()) - c.Check(stderr.String(), check.Matches, `(?ms).*(No such container: theperthcountyconspiracy|exec: \"docker\": executable file not found in \$PATH).*`) + stdin, err := cmd.StdinPipe() + c.Assert(err, check.IsNil) + go fmt.Fprintln(stdin, "data appears on stdin, but stdin does not close; cmd should exit anyway, not hang") + time.AfterFunc(5*time.Second, func() { + c.Errorf("timed out -- remote end is probably hung waiting for us to close stdin") + stdin.Close() + }) + c.Check(cmd.Run(), check.IsNil) + c.Check(stdout.String(), check.Equals, "ok\n") // Set up an http server, and try using "arvados-client shell" // to forward traffic to it. @@ -126,7 +136,7 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2) cmd.Stdout = &stdout cmd.Stderr = &stderr - go cmd.Run() + cmd.Start() forwardedURL := fmt.Sprintf("http://localhost:%s/foo", forwardedPort) @@ -136,7 +146,14 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { if !strings.Contains(err.Error(), "connect") { c.Fatal(err) } else if ctx.Err() != nil { - c.Fatal("timed out") + if cmd.Process.Signal(syscall.Signal(0)) != nil { + c.Error("OpenSSH exited") + } else { + c.Errorf("timed out trying to connect: %s", err) + } + c.Logf("OpenSSH stdout:\n%s", stdout.String()) + c.Logf("OpenSSH stderr:\n%s", stderr.String()) + c.FailNow() } // Retry until OpenSSH starts listening continue @@ -164,3 +181,146 @@ func (s *ClientSuite) TestShellGateway(c *check.C) { } wg.Wait() } + +func (s *ClientSuite) TestContainerRequestLog(c *check.C) { + arvadostest.StartKeep(2, true) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(30*time.Second)) + defer cancel() + + rpcconn := rpc.NewConn("", + &url.URL{ + Scheme: "https", + Host: os.Getenv("ARVADOS_API_HOST"), + }, + true, + func(context.Context) ([]string, error) { + return []string{arvadostest.SystemRootToken}, nil + }) + imageColl, err := rpcconn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar\n", + }}) + c.Assert(err, check.IsNil) + c.Logf("imageColl %+v", imageColl) + cr, err := rpcconn.ContainerRequestCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "state": "Committed", + "command": []string{"echo", fmt.Sprintf("%d", time.Now().Unix())}, + "container_image": imageColl.PortableDataHash, + "cwd": "/", + "output_path": "/", + "priority": 1, + "runtime_constraints": arvados.RuntimeConstraints{ + VCPUs: 1, + RAM: 1000000000, + }, + "container_count_max": 1, + }}) + c.Assert(err, check.IsNil) + h := hmac.New(sha256.New, []byte(arvadostest.SystemRootToken)) + fmt.Fprint(h, cr.ContainerUUID) + authSecret := fmt.Sprintf("%x", h.Sum(nil)) + + coll := arvados.Collection{} + client := arvados.NewClientFromEnv() + ac, err := arvadosclient.New(client) + c.Assert(err, check.IsNil) + kc, err := keepclient.MakeKeepClient(ac) + c.Assert(err, check.IsNil) + cfs, err := coll.FileSystem(client, kc) + c.Assert(err, check.IsNil) + + c.Log("running logs command on queued container") + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "go", "run", ".", "logs", "-f", "-poll=250ms", cr.UUID) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.SystemRootToken) + cmd.Stdout = io.MultiWriter(&stdout, os.Stderr) + cmd.Stderr = io.MultiWriter(&stderr, os.Stderr) + err = cmd.Start() + c.Assert(err, check.Equals, nil) + + c.Log("changing container state to Locked") + _, err = rpcconn.ContainerUpdate(ctx, arvados.UpdateOptions{UUID: cr.ContainerUUID, Attrs: map[string]interface{}{ + "state": arvados.ContainerStateLocked, + }}) + c.Assert(err, check.IsNil) + c.Log("starting gateway") + gw := crunchrun.Gateway{ + ContainerUUID: cr.ContainerUUID, + Address: "0.0.0.0:0", + AuthSecret: authSecret, + Log: ctxlog.TestLogger(c), + Target: crunchrun.GatewayTargetStub{}, + LogCollection: cfs, + } + err = gw.Start() + c.Assert(err, check.IsNil) + c.Log("updating container gateway address") + _, err = rpcconn.ContainerUpdate(ctx, arvados.UpdateOptions{UUID: cr.ContainerUUID, Attrs: map[string]interface{}{ + "gateway_address": gw.Address, + "state": arvados.ContainerStateRunning, + }}) + c.Assert(err, check.IsNil) + + const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + fCrunchrun, err := cfs.OpenFile("crunch-run.txt", os.O_CREATE|os.O_WRONLY, 0777) + c.Assert(err, check.IsNil) + _, err = fmt.Fprintf(fCrunchrun, "%s line 1 of crunch-run.txt\n", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + fStderr, err := cfs.OpenFile("stderr.txt", os.O_CREATE|os.O_WRONLY, 0777) + c.Assert(err, check.IsNil) + _, err = fmt.Fprintf(fStderr, "%s line 1 of stderr\n", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + + { + // Without "-f", just show the existing logs and + // exit. Timeout needs to be long enough for "go run". + ctxNoFollow, cancel := context.WithDeadline(ctx, time.Now().Add(time.Second*5)) + defer cancel() + cmdNoFollow := exec.CommandContext(ctxNoFollow, "go", "run", ".", "logs", "-poll=250ms", cr.UUID) + buf, err := cmdNoFollow.CombinedOutput() + c.Check(err, check.IsNil) + c.Check(string(buf), check.Matches, `(?ms).*line 1 of stderr\n`) + } + + time.Sleep(time.Second * 2) + _, err = fmt.Fprintf(fCrunchrun, "%s line 2 of crunch-run.txt", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + _, err = fmt.Fprintf(fStderr, "%s --end--", time.Now().UTC().Format(rfc3339NanoFixed)) + c.Assert(err, check.IsNil) + + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !strings.Contains(stdout.String(), "--end--"); time.Sleep(time.Second / 10) { + } + c.Check(stdout.String(), check.Matches, `(?ms).*stderr\.txt +20\S+Z --end--\n.*`) + + mtxt, err := cfs.MarshalManifest(".") + c.Assert(err, check.IsNil) + savedLog, err := rpcconn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": mtxt, + }}) + c.Assert(err, check.IsNil) + _, err = rpcconn.ContainerUpdate(ctx, arvados.UpdateOptions{UUID: cr.ContainerUUID, Attrs: map[string]interface{}{ + "state": arvados.ContainerStateComplete, + "log": savedLog.PortableDataHash, + "output": "d41d8cd98f00b204e9800998ecf8427e+0", + "exit_code": 0, + }}) + c.Assert(err, check.IsNil) + + err = cmd.Wait() + c.Check(err, check.IsNil) + // Ensure controller doesn't cheat by fetching data from the + // gateway after the container is complete. + gw.LogCollection = nil + + c.Logf("re-running logs command on completed container") + { + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(time.Second*5)) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "run", ".", "logs", "-f", cr.UUID) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.SystemRootToken) + buf, err := cmd.CombinedOutput() + c.Check(err, check.Equals, nil) + c.Check(string(buf), check.Matches, `(?ms).*--end--\n`) + } +}