15924: Change import paths to git.arvados.org.

[arvados.git] / services / crunch-run / crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 2f254b5bd71fa556869392b58f0b8a102f22e3ce..636926a8d0a06054c3cca22fe2ab72e2d4df67d4 100644 (file)
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -24,10 +24,10 @@ import (
        "testing"
        "time"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-       "git.curoverse.com/arvados.git/sdk/go/manifest"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/manifest"
        "golang.org/x/net/context"
 
        dockertypes "github.com/docker/docker/api/types"
@@ -375,6 +375,10 @@ call:
        return nil
 }
 
+func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
+       return locator, nil
+}
+
 func (client *KeepTestClient) PutB(buf []byte) (string, int, error) {
        client.Content = buf
        return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil
@@ -438,14 +442,14 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s
 }
 
 func (s *TestSuite) TestLoadImage(c *C) {
-       kc := &KeepTestClient{}
-       defer kc.Close()
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+       cr, err := NewContainerRunner(s.client, &ArvTestClient{},
+               &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
 
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, kc, nil
-       }
+       kc := &KeepTestClient{}
+       defer kc.Close()
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = kc
 
        _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
        c.Check(err, IsNil)
@@ -527,6 +531,10 @@ func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) {
        return "", 0, errors.New("KeepError")
 }
 
+func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
+       return "", errors.New("KeepError")
+}
+
 type KeepReadErrorTestClient struct {
        KeepTestClient
 }
@@ -558,10 +566,10 @@ func (s *TestSuite) TestLoadImageArvError(c *C) {
        cr, err := NewContainerRunner(s.client, &ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
 
+       cr.ContainerArvClient = &ArvErrorTestClient{}
+       cr.ContainerKeepClient = &KeepTestClient{}
+
        cr.Container.ContainerImage = hwPDH
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvErrorTestClient{}, &KeepTestClient{}, nil
-       }
 
        err = cr.LoadImage()
        c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
@@ -572,10 +580,11 @@ func (s *TestSuite) TestLoadImageKeepError(c *C) {
        kc := &KeepErrorTestClient{}
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
+
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepErrorTestClient{}
+
        cr.Container.ContainerImage = hwPDH
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, kc, nil
-       }
 
        err = cr.LoadImage()
        c.Assert(err, NotNil)
@@ -588,9 +597,9 @@ func (s *TestSuite) TestLoadImageCollectionError(c *C) {
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = otherPDH
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, kc, nil
-       }
+
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepReadErrorTestClient{}
 
        err = cr.LoadImage()
        c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
@@ -602,9 +611,8 @@ func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.Container.ContainerImage = hwPDH
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, kc, nil
-       }
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepReadErrorTestClient{}
 
        err = cr.LoadImage()
        c.Check(err, NotNil)
@@ -652,9 +660,8 @@ func (s *TestSuite) TestRunContainer(c *C) {
        cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
 
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, kc, nil
-       }
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepTestClient{}
 
        var logs TestLogs
        cr.NewLogWriter = logs.NewTestLoggingWriter
@@ -799,8 +806,8 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
                }
                return d, err
        }
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+               return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil
        }
 
        if extraMounts != nil && len(extraMounts) > 0 {
@@ -818,7 +825,15 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
        }
        if exitCode != 2 {
                c.Check(api.WasSetRunning, Equals, true)
-               c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil)
+               var lastupdate arvadosclient.Dict
+               for _, content := range api.Content {
+                       if content["container"] != nil {
+                               lastupdate = content["container"].(arvadosclient.Dict)
+                       }
+               }
+               if lastupdate["log"] == nil {
+                       c.Errorf("no container update with non-nil log -- updates were: %v", api.Content)
+               }
        }
 
        if err != nil {
@@ -840,7 +855,8 @@ func (s *TestSuite) TestFullRunHello(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-       "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -852,6 +868,28 @@ func (s *TestSuite) TestFullRunHello(c *C) {
 
 }
 
+func (s *TestSuite) TestRunAlreadyRunning(c *C) {
+       var ran bool
+       api, _, _ := s.fullRunHelper(c, `{
+    "command": ["sleep", "3"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"} },
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Running"
+}`, nil, 2, func(t *TestDockerClient) {
+               ran = true
+       })
+
+       c.Check(api.CalledWith("container.state", "Cancelled"), IsNil)
+       c.Check(api.CalledWith("container.state", "Complete"), IsNil)
+       c.Check(ran, Equals, false)
+}
+
 func (s *TestSuite) TestRunTimeExceeded(c *C) {
        api, _, _ := s.fullRunHelper(c, `{
     "command": ["sleep", "3"],
@@ -861,8 +899,9 @@ func (s *TestSuite) TestRunTimeExceeded(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-       "runtime_constraints": {},
-       "scheduling_parameters":{"max_run_time": 1}
+    "runtime_constraints": {},
+    "scheduling_parameters":{"max_run_time": 1},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                time.Sleep(3 * time.Second)
                t.logWriter.Close()
@@ -879,7 +918,8 @@ func (s *TestSuite) TestContainerWaitFails(c *C) {
     "cwd": ".",
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
-    "priority": 1
+    "priority": 1,
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.ctrExited = true
                time.Sleep(10 * time.Second)
@@ -899,7 +939,8 @@ func (s *TestSuite) TestCrunchstat(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0, func(t *TestDockerClient) {
                time.Sleep(time.Second)
                t.logWriter.Close()
@@ -932,7 +973,8 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0,
                func(t *TestDockerClient) {
                        time.Sleep(time.Second)
@@ -966,7 +1008,8 @@ func (s *TestSuite) TestContainerRecordLog(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`, nil, 0,
                func(t *TestDockerClient) {
                        time.Sleep(time.Second)
@@ -989,7 +1032,8 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello\n"))
                t.logWriter.Write(dockerLog(2, "world\n"))
@@ -1014,7 +1058,8 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
                t.logWriter.Close()
@@ -1035,7 +1080,8 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
                t.logWriter.Close()
@@ -1076,7 +1122,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`
 
        rec := arvados.Container{}
@@ -1096,8 +1143,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
        cr, err := NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
        c.Assert(err, IsNil)
        cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, &KeepTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+               return &ArvTestClient{}, &KeepTestClient{}, nil, nil
        }
        setup(cr)
 
@@ -1131,7 +1178,8 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
@@ -1169,6 +1217,8 @@ func (s *TestSuite) TestSetupMounts(c *C) {
        c.Assert(err, IsNil)
        am := &ArvMountCmdLine{}
        cr.RunArvMount = am.ArvMountTest
+       cr.ContainerArvClient = &ArvTestClient{}
+       cr.ContainerKeepClient = &KeepTestClient{}
 
        realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
        c.Assert(err, IsNil)
@@ -1551,17 +1601,18 @@ func (s *TestSuite) TestStdout(c *C) {
                "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
-       api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+       api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
        })
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
 }
 
 // Used by the TestStdoutWithWrongPath*()
@@ -1580,8 +1631,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
        c.Assert(err, IsNil)
        am := &ArvMountCmdLine{}
        cr.RunArvMount = am.ArvMountTest
-       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, error) {
-               return &ArvTestClient{}, &KeepTestClient{}, nil
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
+               return &ArvTestClient{}, &KeepTestClient{}, nil, nil
        }
 
        err = cr.Run()
@@ -1591,7 +1642,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc
 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1601,7 +1653,8 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1611,7 +1664,8 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
        _, _, err := s.stdoutErrorRunHelper(c, `{
     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
-    "output_path": "/tmp"
+    "output_path": "/tmp",
+    "state": "Locked"
 }`, func(t *TestDockerClient) {})
 
        c.Check(err, NotNil)
@@ -1629,7 +1683,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
                t.logWriter.Close()
@@ -1652,7 +1707,8 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {"API": true}
+    "runtime_constraints": {"API": true},
+    "state": "Locked"
 }`, nil, 0, func(t *TestDockerClient) {
                t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
                t.logWriter.Close()
@@ -1679,19 +1735,20 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
 
-       api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+       api, cr, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
                t.logWriter.Close()
        })
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
 }
 
 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
@@ -1710,7 +1767,8 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1764,7 +1822,8 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1799,11 +1858,12 @@ func (s *TestSuite) TestOutputError(c *C) {
                "cwd": "/bin",
                "environment": {"FROBIZ": "bilbo"},
                "mounts": {
-        "/tmp": {"kind": "tmp"}
-    },
+                       "/tmp": {"kind": "tmp"}
+               },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{}
@@ -1829,7 +1889,8 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        extraMounts := []string{
@@ -1868,7 +1929,8 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
     },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
        api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
@@ -1891,7 +1953,7 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
 }
 
 func (s *TestSuite) TestStderrMount(c *C) {
-       api, _, _ := s.fullRunHelper(c, `{
+       api, cr, _ := s.fullRunHelper(c, `{
     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
     "cwd": ".",
@@ -1901,7 +1963,8 @@ func (s *TestSuite) TestStderrMount(c *C) {
                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 1, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello\n"))
                t.logWriter.Write(dockerLog(2, "oops\n"))
@@ -1913,7 +1976,7 @@ func (s *TestSuite) TestStderrMount(c *C) {
        c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
        c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 
-       c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
 }
 
 func (s *TestSuite) TestNumberRoundTrip(c *C) {
@@ -1951,7 +2014,8 @@ exec echo killme
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1976,7 +2040,8 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 2, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -1984,7 +2049,7 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) {
 
        c.Check(api.CalledWith("container.state", "Queued"), NotNil)
        c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
-       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*")
+       c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
 }
 
 func (s *TestSuite) TestFullBrokenDocker3(c *C) {
@@ -1999,7 +2064,8 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 3, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2021,7 +2087,8 @@ func (s *TestSuite) TestBadCommand1(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 4, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2043,7 +2110,8 @@ func (s *TestSuite) TestBadCommand2(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 5, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2065,7 +2133,8 @@ func (s *TestSuite) TestBadCommand3(c *C) {
     "mounts": {"/tmp": {"kind": "tmp"} },
     "output_path": "/tmp",
     "priority": 1,
-    "runtime_constraints": {}
+    "runtime_constraints": {},
+    "state": "Locked"
 }`, nil, 6, func(t *TestDockerClient) {
                t.logWriter.Write(dockerLog(1, "hello world\n"))
                t.logWriter.Close()
@@ -2089,10 +2158,11 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
-       api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+       api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
                content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf")
                c.Check(err, IsNil)
                c.Check(content, DeepEquals, []byte("mypassword"))
@@ -2101,8 +2171,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", ""), IsNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
 
        // under secret mounts, not captured in output
        helperRecord = `{
@@ -2117,10 +2187,11 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
                 },
                "output_path": "/tmp",
                "priority": 1,
-               "runtime_constraints": {}
+               "runtime_constraints": {},
+               "state": "Locked"
        }`
 
-       api, _, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+       api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
                content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf")
                c.Check(err, IsNil)
                c.Check(content, DeepEquals, []byte("mypassword"))
@@ -2129,8 +2200,8 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
 
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
-       c.Check(api.CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
-       c.Check(api.CalledWith("collection.manifest_text", ""), NotNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
+       c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
 }
 
 type FakeProcess struct {
@@ -2140,41 +2211,3 @@ type FakeProcess struct {
 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
        return fp.cmdLine, nil
 }
-
-func (s *TestSuite) helpCheckContainerd(c *C, lp func() ([]PsProcess, error)) error {
-       kc := &KeepTestClient{}
-       defer kc.Close()
-       cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
-       cr.checkContainerd = time.Duration(100 * time.Millisecond)
-       c.Assert(err, IsNil)
-       cr.ListProcesses = lp
-
-       s.docker.fn = func(t *TestDockerClient) {
-               time.Sleep(1 * time.Second)
-               t.logWriter.Close()
-       }
-
-       err = cr.CreateContainer()
-       c.Check(err, IsNil)
-
-       err = cr.StartContainer()
-       c.Check(err, IsNil)
-
-       err = cr.WaitFinish()
-       return err
-
-}
-
-func (s *TestSuite) TestCheckContainerdPresent(c *C) {
-       err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-               return []PsProcess{FakeProcess{[]string{"docker-containerd"}}}, nil
-       })
-       c.Check(err, IsNil)
-}
-
-func (s *TestSuite) TestCheckContainerdMissing(c *C) {
-       err := s.helpCheckContainerd(c, func() ([]PsProcess, error) {
-               return []PsProcess{FakeProcess{[]string{"abc"}}}, nil
-       })
-       c.Check(err, ErrorMatches, `'containerd' not found in process list.`)
-}