X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/664cbad59d3fff30efb7e19c73fa57120a7672b0..4a919918a4ce37b5290793f02fa959db1c073590:/services/crunch-run/crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 7b324ce9a7..60729c019b 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -825,7 +825,15 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
 	}
 	if exitCode != 2 {
 		c.Check(api.WasSetRunning, Equals, true)
-		c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil)
+		var lastupdate arvadosclient.Dict
+		for _, content := range api.Content {
+			if content["container"] != nil {
+				lastupdate = content["container"].(arvadosclient.Dict)
+			}
+		}
+		if lastupdate["log"] == nil {
+			c.Errorf("no container update with non-nil log -- updates were: %v", api.Content)
+		}
 	}
 
 	if err != nil {
@@ -847,7 +855,8 @@ func (s *TestSuite) TestFullRunHello(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-	"runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -859,6 +868,28 @@ func (s *TestSuite) TestFullRunHello(c *C) {
 
 }
 
+func (s *TestSuite) TestRunAlreadyRunning(c *C) {
+	var ran bool
+	api, _, _ := s.fullRunHelper(c, `{
+    "command": ["sleep", "3"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"} },
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Running"
+}`, nil, 2, func(t *TestDockerClient) {
+		ran = true
+	})
+
+	c.Check(api.CalledWith("container.state", "Cancelled"), IsNil)
+	c.Check(api.CalledWith("container.state", "Complete"), IsNil)
+	c.Check(ran, Equals, false)
+}
+
 func (s *TestSuite) TestRunTimeExceeded(c *C) {
 	api, _, _ := s.fullRunHelper(c, `{
     "command": ["sleep", "3"],
@@ -868,8 +899,9 @@ func (s *TestSuite) TestRunTimeExceeded(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-	"runtime_constraints": {},
-	"scheduling_parameters":{"max_run_time": 1}
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		time.Sleep(3 * time.Second)
 		t.logWriter.Close()
@@ -886,7 +918,8 @@ func (s *TestSuite) TestContainerWaitFails(c *C) {
     "cwd": ".",
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
-    "priority": 1
+    "priority": 1,
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.ctrExited = true
 		time.Sleep(10 * time.Second)
@@ -906,7 +939,8 @@ func (s *TestSuite) TestCrunchstat(c *C) {
 		"mounts": {"/tmp": {"kind": "tmp"} },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`, nil, 0, func(t *TestDockerClient) {
 		time.Sleep(time.Second)
 		t.logWriter.Close()
@@ -939,7 +973,8 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
 		"mounts": {"/tmp": {"kind": "tmp"} },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`, nil, 0,
 		func(t *TestDockerClient) {
 			time.Sleep(time.Second)
@@ -973,7 +1008,8 @@ func (s *TestSuite) TestContainerRecordLog(c *C) {
 		"mounts": {"/tmp": {"kind": "tmp"} },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`, nil, 0,
 		func(t *TestDockerClient) {
 			time.Sleep(time.Second)
@@ -996,7 +1032,8 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello\n"))
 		t.logWriter.Write(dockerLog(2, "world\n"))
@@ -1021,7 +1058,8 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 		t.logWriter.Close()
@@ -1042,7 +1080,8 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 		t.logWriter.Close()
@@ -1083,7 +1122,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`
 
 	rec := arvados.Container{}
@@ -1138,7 +1178,8 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
@@ -1560,7 +1601,8 @@ func (s *TestSuite) TestStdout(c *C) {
 		"mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -1600,7 +1642,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
 	_, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
 	c.Check(err, NotNil)
@@ -1610,7 +1653,8 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
 	_, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
 	c.Check(err, NotNil)
@@ -1620,7 +1664,8 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
 	_, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
 	c.Check(err, NotNil)
@@ -1638,7 +1683,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
 		t.logWriter.Close()
@@ -1661,7 +1707,8 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
 		t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
 		t.logWriter.Close()
@@ -1688,7 +1735,8 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
     },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
@@ -1719,7 +1767,8 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
     },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	extraMounts := []string{
@@ -1773,7 +1822,8 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
     },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	extraMounts := []string{
@@ -1808,11 +1858,12 @@ func (s *TestSuite) TestOutputError(c *C) {
 		"cwd": "/bin",
 		"environment": {"FROBIZ": "bilbo"},
 		"mounts": {
-        "/tmp": {"kind": "tmp"}
-    },
+			"/tmp": {"kind": "tmp"}
+		},
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	extraMounts := []string{}
@@ -1838,7 +1889,8 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
     },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	extraMounts := []string{
@@ -1877,7 +1929,8 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
     },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -1910,7 +1963,8 @@ func (s *TestSuite) TestStderrMount(c *C) {
                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello\n"))
 		t.logWriter.Write(dockerLog(2, "oops\n"))
@@ -1960,7 +2014,8 @@ exec echo killme
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -1985,7 +2040,8 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -1993,7 +2049,7 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
 
 	c.Check(api.CalledWith("container.state", "Queued"), NotNil)
 	c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
-	c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*")
+	c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
 }
 
 func (s *TestSuite) TestFullBrokenDocker3(c *C) {
@@ -2008,7 +2064,8 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 3, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -2030,7 +2087,8 @@ func (s *TestSuite) TestBadCommand1(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 4, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -2052,7 +2110,8 @@ func (s *TestSuite) TestBadCommand2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 5, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -2074,7 +2133,8 @@ func (s *TestSuite) TestBadCommand3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 6, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
@@ -2098,7 +2158,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -2126,7 +2187,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
 		"output_path": "/tmp",
 		"priority": 1,
-		"runtime_constraints": {}
+		"runtime_constraints": {},
+		"state": "Locked"
 	}`
 
 	api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -2149,41 +2211,3 @@ type FakeProcess struct {
 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
 	return fp.cmdLine, nil
 }
-
-func (s *TestSuite) helpCheckContainerd(c *C, lp func() ([]PsProcess, error)) error {
-	kc := &KeepTestClient{}
-	defer kc.Close()
-	cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
-	cr.checkContainerd = time.Duration(100 * time.Millisecond)
-	c.Assert(err, IsNil)
-	cr.ListProcesses = lp
-
-	s.docker.fn = func(t *TestDockerClient) {
-		time.Sleep(1 * time.Second)
-		t.logWriter.Close()
-	}
-
-	err = cr.CreateContainer()
-	c.Check(err, IsNil)
-
-	err = cr.StartContainer()
-	c.Check(err, IsNil)
-
-	err = cr.WaitFinish()
-	return err
-
-}
-
-func (s *TestSuite) TestCheckContainerdPresent(c *C) {
-	err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-		return []PsProcess{FakeProcess{[]string{"docker-containerd"}}}, nil
-	})
-	c.Check(err, IsNil)
-}
-
-func (s *TestSuite) TestCheckContainerdMissing(c *C) {
-	err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-		return []PsProcess{FakeProcess{[]string{"abc"}}}, nil
-	})
-	c.Check(err, ErrorMatches, `'containerd' not found in process list.`)
-}