13996: Adjust config:dump to dump active config
[arvados.git] / services / crunch-run / crunchrun_test.go
index 0e48a50aa196d41908d0b9e14eb844193ff3bbc7..60729c019b1a1c508cacceb5b4e7d08e8d300bc5 100644 (file)
@@ -178,12 +178,15 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string,
        return body, err
 }
 
-func (t *TestDockerClient) ContainerList(ctx context.Context, options dockertypes.ContainerListOptions) ([]dockertypes.Container, error) {
+func (t *TestDockerClient) ContainerInspect(ctx context.Context, id string) (c dockertypes.ContainerJSON, err error) {
+       c.ContainerJSONBase = &dockertypes.ContainerJSONBase{}
+       c.ID = "abcde"
        if t.ctrExited {
-               return nil, nil
+               c.State = &dockertypes.ContainerState{Status: "exited", Dead: true}
        } else {
-               return []dockertypes.Container{{ID: "abcde"}}, nil
+               c.State = &dockertypes.ContainerState{Status: "running", Pid: 1234, Running: true}
        }
+       return
 }
 
 func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
@@ -372,6 +375,10 @@ call:
        return nil
 }
 
+func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
+       return locator, nil
+}
+
 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
        client.Content = buf
        return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
@@ -435,10 +442,14 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s
 }
 
 func (s *TestSuite) TestLoadImage(c *C) {
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{},
+               &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       c.Assert(err, IsNil)
+
        kc := &KeepTestClient{}
        defer kc.Close()
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
-       c.Assert(err, IsNil)
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = kc
 
        _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
        c.Check(err, IsNil)
@@ -485,6 +496,9 @@ func (ArvErrorTestClient) Create(resourceType string,
 }
 
 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
+       if method == "GET" && resourceType == "containers" && action == "auth" {
+               return nil
+       }
        return errors.New("ArvError")
 }
 
@@ -517,6 +531,10 @@ func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
        return "", 0, errors.New("KeepError")
 }
 
+func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
+       return "", errors.New("KeepError")
+}
+
 type KeepReadErrorTestClient struct {
        KeepTestClient
 }
@@ -545,8 +563,12 @@ func (s *TestSuite) TestLoadImageArvError(c *C) {
        // (1) Arvados error
        kc := &KeepTestClient{}
        defer kc.Close()
-       cr, err := NewContainerRunner(s.client, ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       cr, err := NewContainerRunner(s.client, &ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
+
+       cr.ContainerArvClient = &ArvErrorTestClient{}
+       cr.ContainerKeepClient = &KeepTestClient{}
+
        cr.Container.ContainerImage = hwPDH
 
        err = cr.LoadImage()
@@ -555,8 +577,13 @@ func (s *TestSuite) TestLoadImageArvError(c *C) {
 
 func (s *TestSuite) TestLoadImageKeepError(c *C) {
        // (2) Keep error
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       kc := &KeepErrorTestClient{}
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
+
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepErrorTestClient{}
+
        cr.Container.ContainerImage = hwPDH
 
        err = cr.LoadImage()
@@ -566,19 +593,26 @@ func (s *TestSuite) TestLoadImageKeepError(c *C) {
 
 func (s *TestSuite) TestLoadImageCollectionError(c *C) {
        // (3) Collection doesn't contain image
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepReadErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       kc := &KeepReadErrorTestClient{}
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = otherPDH
 
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepReadErrorTestClient{}
+
        err = cr.LoadImage()
        c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
 }
 
 func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
        // (4) Collection doesn't contain image
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, &KeepReadErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       kc := &KeepReadErrorTestClient{}
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = hwPDH
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepReadErrorTestClient{}
 
        err = cr.LoadImage()
        c.Check(err, NotNil)
@@ -626,6 +660,9 @@ func (s *TestSuite) TestRunContainer(c *C) {
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
 
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepTestClient{}
+
        var logs TestLogs
        cr.NewLogWriter = logs.NewTestLoggingWriter
        cr.Container.ContainerImage = hwPDH
@@ -748,6 +785,7 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
        c.Assert(err, IsNil)
        s.runner = cr
        cr.statInterval = 100 * time.Millisecond
+       cr.containerWatchdogInterval = time.Second
        am := &ArvMountCmdLine{}
        cr.RunArvMount = am.ArvMountTest
 
@@ -768,8 +806,8 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
                }
                return d, err
        }
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
-               return &ArvTestClient{secretMounts: secretMounts}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+               return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil
        }
 
        if extraMounts != nil && len(extraMounts) > 0 {
@@ -787,7 +825,15 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
        }
        if exitCode != 2 {
                c.Check(api.WasSetRunning, Equals, true)
-               c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil)
+               var lastupdate arvadosclient.Dict
+               for _, content := range api.Content {
+                       if content["container"] != nil {
+                               lastupdate = content["container"].(arvadosclient.Dict)
+                       }
+               }
+               if lastupdate["log"] == nil {
+                       c.Errorf("no container update with non-nil log -- updates were: %v", api.Content)
+               }
        }
 
        if err != nil {
@@ -809,7 +855,8 @@ func (s *TestSuite) TestFullRunHello(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-       "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -821,6 +868,28 @@ func (s *TestSuite) TestFullRunHello(c *C) {
 
 }
 
+func (s *TestSuite) TestRunAlreadyRunning(c *C) {
+       var ran bool
+       api, _, _ := s.fullRunHelper(c, `{
+    "command": ["sleep", "3"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"} },
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Running"
+}`, nil, 2, func(t *TestDockerClient) {
+               ran = true
+       })
+
+       c.Check(api.CalledWith("container.state", "Cancelled"), IsNil)
+       c.Check(api.CalledWith("container.state", "Complete"), IsNil)
+       c.Check(ran, Equals, false)
+}
+
 func (s *TestSuite) TestRunTimeExceeded(c *C) {
        api, _, _ := s.fullRunHelper(c, `{
     "command": ["sleep", "3"],
@@ -830,8 +899,9 @@ func (s *TestSuite) TestRunTimeExceeded(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-       "runtime_constraints": {},
-       "scheduling_parameters":{"max_run_time": 1}
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                time.Sleep(3 * time.Second)
                t.logWriter.Close()
@@ -848,17 +918,16 @@ func (s *TestSuite) TestContainerWaitFails(c *C) {
     "cwd": ".",
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
-    "priority": 1
+    "priority": 1,
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
-               s.runner.containerWaitGracePeriod = time.Second
                t.ctrExited = true
                time.Sleep(10 * time.Second)
                t.logWriter.Close()
        })
 
        c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
-       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*container.*no longer exists.*")
-       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*docker client never returned status.*")
+       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
 }
 
 func (s *TestSuite) TestCrunchstat(c *C) {
@@ -870,7 +939,8 @@ func (s *TestSuite) TestCrunchstat(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0, func(t *TestDockerClient) {
                time.Sleep(time.Second)
                t.logWriter.Close()
@@ -903,7 +973,8 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0,
                func(t *TestDockerClient) {
                        time.Sleep(time.Second)
@@ -937,7 +1008,8 @@ func (s *TestSuite) TestContainerRecordLog(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0,
                func(t *TestDockerClient) {
                        time.Sleep(time.Second)
@@ -960,7 +1032,8 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello\n"))
                t.logWriter.Write(dockerLog(2, "world\n"))
@@ -985,7 +1058,8 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
                t.logWriter.Close()
@@ -1006,7 +1080,8 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
                t.logWriter.Close()
@@ -1047,7 +1122,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`
 
        rec := arvados.Container{}
@@ -1067,8 +1143,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
        cr, err := NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
-               return &ArvTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+               return &ArvTestClient{}, &KeepTestClient{}, nil, nil
        }
        setup(cr)
 
@@ -1102,7 +1178,8 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
@@ -1140,6 +1217,8 @@ func (s *TestSuite) TestSetupMounts(c *C) {
        c.Assert(err, IsNil)
        am := &ArvMountCmdLine{}
        cr.RunArvMount = am.ArvMountTest
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepTestClient{}
 
        realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
        c.Assert(err, IsNil)
@@ -1522,17 +1601,18 @@ func (s *TestSuite) TestStdout(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
-       api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+       api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
        })
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
 }
 
 // Used by the TestStdoutWithWrongPath*()
@@ -1551,8 +1631,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
        c.Assert(err, IsNil)
        am := &ArvMountCmdLine{}
        cr.RunArvMount = am.ArvMountTest
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
-               return &ArvTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+               return &ArvTestClient{}, &KeepTestClient{}, nil, nil
        }
 
        err = cr.Run()
@@ -1562,7 +1642,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1572,7 +1653,8 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1582,7 +1664,8 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1600,7 +1683,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
                t.logWriter.Close()
@@ -1623,7 +1707,8 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
                t.logWriter.Close()
@@ -1650,19 +1735,20 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
 
-       api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+       api, cr, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
        })
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
 }
 
 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
@@ -1681,7 +1767,8 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1735,7 +1822,8 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1770,11 +1858,12 @@ func (s *TestSuite) TestOutputError(c *C) {
                "cwd": "/bin",
                "environment": {"FROBIZ": "bilbo"},
                "mounts": {
-        "/tmp": {"kind": "tmp"}
-    },
+                       "/tmp": {"kind": "tmp"}
+               },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{}
@@ -1800,7 +1889,8 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1839,7 +1929,8 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -1862,7 +1953,7 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
 }
 
 func (s *TestSuite) TestStderrMount(c *C) {
-       api, _, _ := s.fullRunHelper(c, `{
+       api, cr, _ := s.fullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": ".",
@@ -1872,7 +1963,8 @@ func (s *TestSuite) TestStderrMount(c *C) {
                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello\n"))
                t.logWriter.Write(dockerLog(2, "oops\n"))
@@ -1884,7 +1976,7 @@ func (s *TestSuite) TestStderrMount(c *C) {
        c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
        c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 
-       c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
 }
 
 func (s *TestSuite) TestNumberRoundTrip(c *C) {
@@ -1922,7 +2014,8 @@ exec echo killme
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1947,7 +2040,8 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1955,7 +2049,7 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
 
        c.Check(api.CalledWith("container.state", "Queued"), NotNil)
        c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
-       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*")
+       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
 }
 
 func (s *TestSuite) TestFullBrokenDocker3(c *C) {
@@ -1970,7 +2064,8 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 3, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1992,7 +2087,8 @@ func (s *TestSuite) TestBadCommand1(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 4, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2014,7 +2110,8 @@ func (s *TestSuite) TestBadCommand2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 5, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2036,7 +2133,8 @@ func (s *TestSuite) TestBadCommand3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 6, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2060,10 +2158,11 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
-       api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+       api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
                content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf")
                c.Check(err, IsNil)
                c.Check(content, DeepEquals, []byte("mypassword"))
@@ -2072,8 +2171,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", ""), IsNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
 
        // under secret mounts, not captured in output
        helperRecord = `{
@@ -2088,10 +2187,11 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
-       api, _, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+       api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
                content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf")
                c.Check(err, IsNil)
                c.Check(content, DeepEquals, []byte("mypassword"))
@@ -2100,8 +2200,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
-       c.Check(api.CalledWith("collection.manifest_text", ""), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
 }
 
 type FakeProcess struct {
@@ -2111,41 +2211,3 @@ type FakeProcess struct {
 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
        return fp.cmdLine, nil
 }
-
-func (s *TestSuite) helpCheckContainerd(c *C, lp func() ([]PsProcess, error)) error {
-       kc := &KeepTestClient{}
-       defer kc.Close()
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
-       cr.checkContainerd = time.Duration(100 * time.Millisecond)
-       c.Assert(err, IsNil)
-       cr.ListProcesses = lp
-
-       s.docker.fn = func(t *TestDockerClient) {
-               time.Sleep(1 * time.Second)
-               t.logWriter.Close()
-       }
-
-       err = cr.CreateContainer()
-       c.Check(err, IsNil)
-
-       err = cr.StartContainer()
-       c.Check(err, IsNil)
-
-       err = cr.WaitFinish()
-       return err
-
-}
-
-func (s *TestSuite) TestCheckContainerdPresent(c *C) {
-       err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-               return []PsProcess{FakeProcess{[]string{"docker-containerd"}}}, nil
-       })
-       c.Check(err, IsNil)
-}
-
-func (s *TestSuite) TestCheckContainerdMissing(c *C) {
-       err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-               return []PsProcess{FakeProcess{[]string{"abc"}}}, nil
-       })
-       c.Check(err, ErrorMatches, `'containerd' not found in process list.`)
-}