X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/fbc6c71ef67461cde5ba914e61f3f3b8740a4045..390af6a13f7c8974329aecc2f23fbfa81f8e298b:/services/crunch-run/crunchrun_test.go diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go index d1d2cee138..7224c4f1b3 100644 --- a/services/crunch-run/crunchrun_test.go +++ b/services/crunch-run/crunchrun_test.go @@ -6,23 +6,25 @@ import ( "encoding/json" "errors" "fmt" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/keepclient" - "git.curoverse.com/arvados.git/sdk/go/manifest" - "github.com/curoverse/dockerclient" - . "gopkg.in/check.v1" "io" "io/ioutil" "os" "os/exec" "path/filepath" + "runtime/pprof" "sort" "strings" "sync" "syscall" "testing" "time" + + "git.curoverse.com/arvados.git/sdk/go/arvados" + "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.curoverse.com/arvados.git/sdk/go/manifest" + "github.com/curoverse/dockerclient" + . "gopkg.in/check.v1" ) // Gocheck boilerplate @@ -182,6 +184,21 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par } } +func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string, + parameters arvadosclient.Dict) (reader io.ReadCloser, err error) { + j := []byte(`{ + "command": ["sleep", "1"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {} + }`) + return ioutil.NopCloser(bytes.NewReader(j)), nil +} + func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error { if resourceType == "collections" { if uuid == hwPDH { @@ -320,6 +337,11 @@ func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parame return errors.New("ArvError") } +func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string, + parameters arvadosclient.Dict) (reader io.ReadCloser, err error) { + return nil, errors.New("ArvError") +} + func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error { return errors.New("ArvError") } @@ -483,6 +505,7 @@ func (s *TestSuite) TestCommitLogs(c *C) { c.Check(err, IsNil) c.Check(api.Calls, Equals, 2) + c.Check(api.Content[1]["ensure_unique_name"], Equals, true) c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n") c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60") @@ -523,7 +546,7 @@ func (s *TestSuite) TestUpdateContainerCancelled(c *C) { api := &ArvTestClient{} kc := &KeepTestClient{} cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") - cr.Cancelled = true + cr.cCancelled = true cr.finalState = "Cancelled" err := cr.UpdateContainerFinal() @@ -648,6 +671,56 @@ func (s *TestSuite) TestCrunchstat(c *C) { c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`) } +func (s *TestSuite) TestNodeInfoLog(c *C) { + api, _, _ := FullRunHelper(c, `{ + "command": ["sleep", "1"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {} + }`, nil, func(t *TestDockerClient) { + time.Sleep(time.Second) + t.logWriter.Close() + t.finish <- dockerclient.WaitResult{} + }) + + c.Check(api.CalledWith("container.exit_code", 0), NotNil) + c.Check(api.CalledWith("container.state", "Complete"), NotNil) + + c.Assert(api.Logs["node-info"], NotNil) + c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`) + c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`) + c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`) + c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`) + c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`) +} + +func (s *TestSuite) TestContainerRecordLog(c *C) { + api, _, _ := FullRunHelper(c, `{ + "command": ["sleep", "1"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {} + }`, nil, func(t *TestDockerClient) { + time.Sleep(time.Second) + t.logWriter.Close() + t.finish <- dockerclient.WaitResult{} + }) + + c.Check(api.CalledWith("container.exit_code", 0), NotNil) + c.Check(api.CalledWith("container.state", "Complete"), NotNil) + + c.Assert(api.Logs["container"], NotNil) + c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`) +} + func (s *TestSuite) TestFullRunStderr(c *C) { api, _, _ := FullRunHelper(c, `{ "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"], @@ -717,7 +790,28 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) { c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true) } -func (s *TestSuite) TestCancel(c *C) { +func (s *TestSuite) TestStopOnSignal(c *C) { + s.testStopContainer(c, func(cr *ContainerRunner) { + go func() { + for !cr.cStarted { + time.Sleep(time.Millisecond) + } + cr.SigChan <- syscall.SIGINT + }() + }) +} + +func (s *TestSuite) TestStopOnArvMountDeath(c *C) { + s.testStopContainer(c, func(cr *ContainerRunner) { + cr.ArvMountExit = make(chan error) + go func() { + cr.ArvMountExit <- exec.Command("true").Run() + close(cr.ArvMountExit) + }() + }) +} + +func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) { record := `{ "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"], "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", @@ -744,30 +838,28 @@ func (s *TestSuite) TestCancel(c *C) { api := &ArvTestClient{Container: rec} cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") - am := &ArvMountCmdLine{} - cr.RunArvMount = am.ArvMountTest + cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil } + setup(cr) + done := make(chan error) go func() { - for cr.ContainerID == "" { - time.Sleep(time.Millisecond) - } - cr.SigChan <- syscall.SIGINT + done <- cr.Run() }() - - err = cr.Run() - - c.Check(err, IsNil) - if err != nil { - for k, v := range api.Logs { - c.Log(k) - c.Log(v.String()) - } + select { + case <-time.After(20 * time.Second): + pprof.Lookup("goroutine").WriteTo(os.Stderr, 1) + c.Fatal("timed out") + case err = <-done: + c.Check(err, IsNil) + } + for k, v := range api.Logs { + c.Log(k) + c.Log(v.String()) } c.Check(api.CalledWith("container.log", nil), NotNil) c.Check(api.CalledWith("container.state", "Cancelled"), NotNil) c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true) - } func (s *TestSuite) TestFullRunSetEnv(c *C) { @@ -1219,6 +1311,7 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { c.Check(api.CalledWith("container.state", "Complete"), NotNil) for _, v := range api.Content { if v["collection"] != nil { + c.Check(v["ensure_unique_name"], Equals, true) collection := v["collection"].(arvadosclient.Dict) if strings.Index(collection["name"].(string), "output") == 0 { manifest := collection["manifest_text"].(string)