X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3568f12c9b36cbd11c3cc9d229919a073c0fb43b..90944740f40ab0dbfc4bdfc1b16accfbf6559e4f:/services/crunch-run/crunchrun_test.go diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go index 0e48a50aa1..60729c019b 100644 --- a/services/crunch-run/crunchrun_test.go +++ b/services/crunch-run/crunchrun_test.go @@ -178,12 +178,15 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, return body, err } -func (t *TestDockerClient) ContainerList(ctx context.Context, options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) { +func (t *TestDockerClient) ContainerInspect(ctx context.Context, id string) (c dockertypes.ContainerJSON, err error) { + c.ContainerJSONBase = &dockertypes.ContainerJSONBase{} + c.ID = "abcde" if t.ctrExited { - return nil, nil + c.State = &dockertypes.ContainerState{Status: "exited", Dead: true} } else { - return []dockertypes.Container{{ID: "abcde"}}, nil + c.State = &dockertypes.ContainerState{Status: "running", Pid: 1234, Running: true} } + return } func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) { @@ -372,6 +375,10 @@ call: return nil } +func (client *KeepTestClient) LocalLocator(locator string) (string, error) { + return locator, nil +} + func (client *KeepTestClient) PutB(buf []byte) (string, int, error) { client.Content = buf return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil @@ -435,10 +442,14 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s } func (s *TestSuite) TestLoadImage(c *C) { + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, + &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) + kc := &KeepTestClient{} defer kc.Close() - cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") - c.Assert(err, IsNil) + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = kc _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) c.Check(err, IsNil) @@ -485,6 +496,9 @@ func (ArvErrorTestClient) Create(resourceType string, } func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error { + if method == "GET" && resourceType == "containers" && action == "auth" { + return nil + } return errors.New("ArvError") } @@ -517,6 +531,10 @@ func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) { return "", 0, errors.New("KeepError") } +func (*KeepErrorTestClient) LocalLocator(string) (string, error) { + return "", errors.New("KeepError") +} + type KeepReadErrorTestClient struct { KeepTestClient } @@ -545,8 +563,12 @@ func (s *TestSuite) TestLoadImageArvError(c *C) { // (1) Arvados error kc := &KeepTestClient{} defer kc.Close() - cr, err := NewContainerRunner(s.client, ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + cr, err := NewContainerRunner(s.client, &ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Assert(err, IsNil) + + cr.ContainerArvClient = &ArvErrorTestClient{} + cr.ContainerKeepClient = &KeepTestClient{} + cr.Container.ContainerImage = hwPDH err = cr.LoadImage() @@ -555,8 +577,13 @@ func (s *TestSuite) TestLoadImageArvError(c *C) { func (s *TestSuite) TestLoadImageKeepError(c *C) { // (2) Keep error - cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepErrorTestClient{} + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Assert(err, IsNil) + + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepErrorTestClient{} + cr.Container.ContainerImage = hwPDH err = cr.LoadImage() @@ -566,19 +593,26 @@ func (s *TestSuite) TestLoadImageKeepError(c *C) { func (s *TestSuite) TestLoadImageCollectionError(c *C) { // (3) Collection doesn't contain image - cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepReadErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepReadErrorTestClient{} + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Assert(err, IsNil) cr.Container.ContainerImage = otherPDH + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepReadErrorTestClient{} + err = cr.LoadImage() c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar") } func (s *TestSuite) TestLoadImageKeepReadError(c *C) { // (4) Collection doesn't contain image - cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepReadErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepReadErrorTestClient{} + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Assert(err, IsNil) cr.Container.ContainerImage = hwPDH + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepReadErrorTestClient{} err = cr.LoadImage() c.Check(err, NotNil) @@ -626,6 +660,9 @@ func (s *TestSuite) TestRunContainer(c *C) { cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Assert(err, IsNil) + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepTestClient{} + var logs TestLogs cr.NewLogWriter = logs.NewTestLoggingWriter cr.Container.ContainerImage = hwPDH @@ -748,6 +785,7 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi c.Assert(err, IsNil) s.runner = cr cr.statInterval = 100 * time.Millisecond + cr.containerWatchdogInterval = time.Second am := &ArvMountCmdLine{} cr.RunArvMount = am.ArvMountTest @@ -768,8 +806,8 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi } return d, err } - cr.MkArvClient = func(token string) (IArvadosClient, error) { - return &ArvTestClient{secretMounts: secretMounts}, nil + cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) { + return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil } if extraMounts != nil && len(extraMounts) > 0 { @@ -787,7 +825,15 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi } if exitCode != 2 { c.Check(api.WasSetRunning, Equals, true) - c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil) + var lastupdate arvadosclient.Dict + for _, content := range api.Content { + if content["container"] != nil { + lastupdate = content["container"].(arvadosclient.Dict) + } + } + if lastupdate["log"] == nil { + c.Errorf("no container update with non-nil log -- updates were: %v", api.Content) + } } if err != nil { @@ -809,7 +855,8 @@ func (s *TestSuite) TestFullRunHello(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -821,6 +868,28 @@ func (s *TestSuite) TestFullRunHello(c *C) { } +func (s *TestSuite) TestRunAlreadyRunning(c *C) { + var ran bool + api, _, _ := s.fullRunHelper(c, `{ + "command": ["sleep", "3"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {}, + "scheduling_parameters":{"max_run_time": 1}, + "state": "Running" +}`, nil, 2, func(t *TestDockerClient) { + ran = true + }) + + c.Check(api.CalledWith("container.state", "Cancelled"), IsNil) + c.Check(api.CalledWith("container.state", "Complete"), IsNil) + c.Check(ran, Equals, false) +} + func (s *TestSuite) TestRunTimeExceeded(c *C) { api, _, _ := s.fullRunHelper(c, `{ "command": ["sleep", "3"], @@ -830,8 +899,9 @@ func (s *TestSuite) TestRunTimeExceeded(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {}, - "scheduling_parameters":{"max_run_time": 1} + "runtime_constraints": {}, + "scheduling_parameters":{"max_run_time": 1}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(3 * time.Second) t.logWriter.Close() @@ -848,17 +918,16 @@ func (s *TestSuite) TestContainerWaitFails(c *C) { "cwd": ".", "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", - "priority": 1 + "priority": 1, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { - s.runner.containerWaitGracePeriod = time.Second t.ctrExited = true time.Sleep(10 * time.Second) t.logWriter.Close() }) c.Check(api.CalledWith("container.state", "Cancelled"), NotNil) - c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*container.*no longer exists.*") - c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*docker client never returned status.*") + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*") } func (s *TestSuite) TestCrunchstat(c *C) { @@ -870,7 +939,8 @@ func (s *TestSuite) TestCrunchstat(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(time.Second) t.logWriter.Close() @@ -903,7 +973,8 @@ func (s *TestSuite) TestNodeInfoLog(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(time.Second) @@ -937,7 +1008,8 @@ func (s *TestSuite) TestContainerRecordLog(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(time.Second) @@ -960,7 +1032,8 @@ func (s *TestSuite) TestFullRunStderr(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 1, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello\n")) t.logWriter.Write(dockerLog(2, "world\n")) @@ -985,7 +1058,8 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.cwd+"\n")) t.logWriter.Close() @@ -1006,7 +1080,8 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.cwd+"\n")) t.logWriter.Close() @@ -1047,7 +1122,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` rec := arvados.Container{} @@ -1067,8 +1143,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) { cr, err := NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") c.Assert(err, IsNil) cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil } - cr.MkArvClient = func(token string) (IArvadosClient, error) { - return &ArvTestClient{}, nil + cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) { + return &ArvTestClient{}, &KeepTestClient{}, nil, nil } setup(cr) @@ -1102,7 +1178,8 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n")) t.logWriter.Close() @@ -1140,6 +1217,8 @@ func (s *TestSuite) TestSetupMounts(c *C) { c.Assert(err, IsNil) am := &ArvMountCmdLine{} cr.RunArvMount = am.ArvMountTest + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepTestClient{} realTemp, err := ioutil.TempDir("", "crunchrun_test1-") c.Assert(err, IsNil) @@ -1522,17 +1601,18 @@ func (s *TestSuite) TestStdout(c *C) { "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` - api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { + api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n")) t.logWriter.Close() }) c.Check(api.CalledWith("container.exit_code", 0), NotNil) c.Check(api.CalledWith("container.state", "Complete"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) } // Used by the TestStdoutWithWrongPath*() @@ -1551,8 +1631,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc c.Assert(err, IsNil) am := &ArvMountCmdLine{} cr.RunArvMount = am.ArvMountTest - cr.MkArvClient = func(token string) (IArvadosClient, error) { - return &ArvTestClient{}, nil + cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) { + return &ArvTestClient{}, &KeepTestClient{}, nil, nil } err = cr.Run() @@ -1562,7 +1642,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc func (s *TestSuite) TestStdoutWithWrongPath(c *C) { _, _, err := s.stdoutErrorRunHelper(c, `{ "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} }, - "output_path": "/tmp" + "output_path": "/tmp", + "state": "Locked" }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) @@ -1572,7 +1653,8 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) { func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) { _, _, err := s.stdoutErrorRunHelper(c, `{ "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} }, - "output_path": "/tmp" + "output_path": "/tmp", + "state": "Locked" }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) @@ -1582,7 +1664,8 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) { func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) { _, _, err := s.stdoutErrorRunHelper(c, `{ "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} }, - "output_path": "/tmp" + "output_path": "/tmp", + "state": "Locked" }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) @@ -1600,7 +1683,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {"API": true} + "runtime_constraints": {"API": true}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n")) t.logWriter.Close() @@ -1623,7 +1707,8 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {"API": true} + "runtime_constraints": {"API": true}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122" t.logWriter.Close() @@ -1650,19 +1735,20 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"} - api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { + api, cr, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n")) t.logWriter.Close() }) c.Check(api.CalledWith("container.exit_code", 0), NotNil) c.Check(api.CalledWith("container.state", "Complete"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) } func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { @@ -1681,7 +1767,8 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{ @@ -1735,7 +1822,8 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest( }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{ @@ -1770,11 +1858,12 @@ func (s *TestSuite) TestOutputError(c *C) { "cwd": "/bin", "environment": {"FROBIZ": "bilbo"}, "mounts": { - "/tmp": {"kind": "tmp"} - }, + "/tmp": {"kind": "tmp"} + }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{} @@ -1800,7 +1889,8 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{ @@ -1839,7 +1929,8 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { @@ -1862,7 +1953,7 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) { } func (s *TestSuite) TestStderrMount(c *C) { - api, _, _ := s.fullRunHelper(c, `{ + api, cr, _ := s.fullRunHelper(c, `{ "command": ["/bin/sh", "-c", "echo hello;exit 1"], "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": ".", @@ -1872,7 +1963,8 @@ func (s *TestSuite) TestStderrMount(c *C) { "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}}, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 1, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello\n")) t.logWriter.Write(dockerLog(2, "oops\n")) @@ -1884,7 +1976,7 @@ func (s *TestSuite) TestStderrMount(c *C) { c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1) c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil) - c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil) } func (s *TestSuite) TestNumberRoundTrip(c *C) { @@ -1922,7 +2014,8 @@ exec echo killme "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 2, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1947,7 +2040,8 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 2, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1955,7 +2049,7 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) { c.Check(api.CalledWith("container.state", "Queued"), NotNil) c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*") - c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*") + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*") } func (s *TestSuite) TestFullBrokenDocker3(c *C) { @@ -1970,7 +2064,8 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 3, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1992,7 +2087,8 @@ func (s *TestSuite) TestBadCommand1(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 4, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -2014,7 +2110,8 @@ func (s *TestSuite) TestBadCommand2(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 5, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -2036,7 +2133,8 @@ func (s *TestSuite) TestBadCommand3(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 6, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -2060,10 +2158,11 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` - api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { + api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf") c.Check(err, IsNil) c.Check(content, DeepEquals, []byte("mypassword")) @@ -2072,8 +2171,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) { c.Check(api.CalledWith("container.exit_code", 0), NotNil) c.Check(api.CalledWith("container.state", "Complete"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", ""), IsNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil) // under secret mounts, not captured in output helperRecord = `{ @@ -2088,10 +2187,11 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` - api, _, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { + api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf") c.Check(err, IsNil) c.Check(content, DeepEquals, []byte("mypassword")) @@ -2100,8 +2200,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) { c.Check(api.CalledWith("container.exit_code", 0), NotNil) c.Check(api.CalledWith("container.state", "Complete"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil) - c.Check(api.CalledWith("collection.manifest_text", ""), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil) } type FakeProcess struct { @@ -2111,41 +2211,3 @@ type FakeProcess struct { func (fp FakeProcess) CmdlineSlice() ([]string, error) { return fp.cmdLine, nil } - -func (s *TestSuite) helpCheckContainerd(c *C, lp func() ([]PsProcess, error)) error { - kc := &KeepTestClient{} - defer kc.Close() - cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") - cr.checkContainerd = time.Duration(100 * time.Millisecond) - c.Assert(err, IsNil) - cr.ListProcesses = lp - - s.docker.fn = func(t *TestDockerClient) { - time.Sleep(1 * time.Second) - t.logWriter.Close() - } - - err = cr.CreateContainer() - c.Check(err, IsNil) - - err = cr.StartContainer() - c.Check(err, IsNil) - - err = cr.WaitFinish() - return err - -} - -func (s *TestSuite) TestCheckContainerdPresent(c *C) { - err := s.helpCheckContainerd(c, func() ([]PsProcess, error) { - return []PsProcess{FakeProcess{[]string{"docker-containerd"}}}, nil - }) - c.Check(err, IsNil) -} - -func (s *TestSuite) TestCheckContainerdMissing(c *C) { - err := s.helpCheckContainerd(c, func() ([]PsProcess, error) { - return []PsProcess{FakeProcess{[]string{"abc"}}}, nil - }) - c.Check(err, ErrorMatches, `'containerd' not found in process list.`) -}