13996: Adjust config:dump to dump active config

[arvados.git] / services / crunch-run / crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 7b324ce9a7dd1cb47d620657095f8cd1d7b04811..60729c019b1a1c508cacceb5b4e7d08e8d300bc5 100644 (file)
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -825,7 +825,15 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
        }
        if exitCode != 2 {
                c.Check(api.WasSetRunning, Equals, true)
-               c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil)
+               var lastupdate arvadosclient.Dict
+               for _, content := range api.Content {
+                       if content["container"] != nil {
+                               lastupdate = content["container"].(arvadosclient.Dict)
+                       }
+               }
+               if lastupdate["log"] == nil {
+                       c.Errorf("no container update with non-nil log -- updates were: %v", api.Content)
+               }
        }
 
        if err != nil {
@@ -847,7 +855,8 @@ func (s *TestSuite) TestFullRunHello(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-       "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -859,6 +868,28 @@ func (s *TestSuite) TestFullRunHello(c *C) {
 
 }
 
+func (s *TestSuite) TestRunAlreadyRunning(c *C) {
+       var ran bool
+       api, _, _ := s.fullRunHelper(c, `{
+    "command": ["sleep", "3"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"} },
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Running"
+}`, nil, 2, func(t *TestDockerClient) {
+               ran = true
+       })
+
+       c.Check(api.CalledWith("container.state", "Cancelled"), IsNil)
+       c.Check(api.CalledWith("container.state", "Complete"), IsNil)
+       c.Check(ran, Equals, false)
+}
+
 func (s *TestSuite) TestRunTimeExceeded(c *C) {
        api, _, _ := s.fullRunHelper(c, `{
     "command": ["sleep", "3"],
@@ -868,8 +899,9 @@ func (s *TestSuite) TestRunTimeExceeded(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-       "runtime_constraints": {},
-       "scheduling_parameters":{"max_run_time": 1}
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                time.Sleep(3 * time.Second)
                t.logWriter.Close()
@@ -886,7 +918,8 @@ func (s *TestSuite) TestContainerWaitFails(c *C) {
     "cwd": ".",
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
-    "priority": 1
+    "priority": 1,
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.ctrExited = true
                time.Sleep(10 * time.Second)
@@ -906,7 +939,8 @@ func (s *TestSuite) TestCrunchstat(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0, func(t *TestDockerClient) {
                time.Sleep(time.Second)
                t.logWriter.Close()
@@ -939,7 +973,8 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0,
                func(t *TestDockerClient) {
                        time.Sleep(time.Second)
@@ -973,7 +1008,8 @@ func (s *TestSuite) TestContainerRecordLog(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0,
                func(t *TestDockerClient) {
                        time.Sleep(time.Second)
@@ -996,7 +1032,8 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello\n"))
                t.logWriter.Write(dockerLog(2, "world\n"))
@@ -1021,7 +1058,8 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
                t.logWriter.Close()
@@ -1042,7 +1080,8 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
                t.logWriter.Close()
@@ -1083,7 +1122,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`
 
        rec := arvados.Container{}
@@ -1138,7 +1178,8 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
@@ -1560,7 +1601,8 @@ func (s *TestSuite) TestStdout(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -1600,7 +1642,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1610,7 +1653,8 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1620,7 +1664,8 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1638,7 +1683,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
                t.logWriter.Close()
@@ -1661,7 +1707,8 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
                t.logWriter.Close()
@@ -1688,7 +1735,8 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
@@ -1719,7 +1767,8 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1773,7 +1822,8 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1808,11 +1858,12 @@ func (s *TestSuite) TestOutputError(c *C) {
                "cwd": "/bin",
                "environment": {"FROBIZ": "bilbo"},
                "mounts": {
-        "/tmp": {"kind": "tmp"}
-    },
+                       "/tmp": {"kind": "tmp"}
+               },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{}
@@ -1838,7 +1889,8 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1877,7 +1929,8 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -1910,7 +1963,8 @@ func (s *TestSuite) TestStderrMount(c *C) {
                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello\n"))
                t.logWriter.Write(dockerLog(2, "oops\n"))
@@ -1960,7 +2014,8 @@ exec echo killme
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1985,7 +2040,8 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1993,7 +2049,7 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
 
        c.Check(api.CalledWith("container.state", "Queued"), NotNil)
        c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
-       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*")
+       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
 }
 
 func (s *TestSuite) TestFullBrokenDocker3(c *C) {
@@ -2008,7 +2064,8 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 3, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2030,7 +2087,8 @@ func (s *TestSuite) TestBadCommand1(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 4, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2052,7 +2110,8 @@ func (s *TestSuite) TestBadCommand2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 5, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2074,7 +2133,8 @@ func (s *TestSuite) TestBadCommand3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 6, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2098,7 +2158,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -2126,7 +2187,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -2149,41 +2211,3 @@ type FakeProcess struct {
 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
        return fp.cmdLine, nil
 }
-
-func (s *TestSuite) helpCheckContainerd(c *C, lp func() ([]PsProcess, error)) error {
-       kc := &KeepTestClient{}
-       defer kc.Close()
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
-       cr.checkContainerd = time.Duration(100 * time.Millisecond)
-       c.Assert(err, IsNil)
-       cr.ListProcesses = lp
-
-       s.docker.fn = func(t *TestDockerClient) {
-               time.Sleep(1 * time.Second)
-               t.logWriter.Close()
-       }
-
-       err = cr.CreateContainer()
-       c.Check(err, IsNil)
-
-       err = cr.StartContainer()
-       c.Check(err, IsNil)
-
-       err = cr.WaitFinish()
-       return err
-
-}
-
-func (s *TestSuite) TestCheckContainerdPresent(c *C) {
-       err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-               return []PsProcess{FakeProcess{[]string{"docker-containerd"}}}, nil
-       })
-       c.Check(err, IsNil)
-}
-
-func (s *TestSuite) TestCheckContainerdMissing(c *C) {
-       err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-               return []PsProcess{FakeProcess{[]string{"abc"}}}, nil
-       })
-       c.Check(err, ErrorMatches, `'containerd' not found in process list.`)
-}