X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5976c751048100e813edd638632927794df072f2..cd4f5cfb00a253726a0c9087721273fd9b142be1:/services/crunch-run/crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 9b12dd96fd..d3c9990770 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1,13 +1,20 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
+	"bufio"
 	"bytes"
+	"context"
 	"crypto/md5"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -21,9 +28,11 @@ import (
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"git.curoverse.com/arvados.git/sdk/go/manifest"
-	"github.com/curoverse/dockerclient"
+
+	dockertypes "github.com/docker/docker/api/types"
+	dockercontainer "github.com/docker/docker/api/types/container"
+	dockernetwork "github.com/docker/docker/api/types/network"
 	. "gopkg.in/check.v1"
 )
 
@@ -45,6 +54,7 @@ type ArvTestClient struct {
 	Logs map[string]*bytes.Buffer
 	sync.Mutex
 	WasSetRunning bool
+	callraw       bool
 }
 
 type KeepTestClient struct {
@@ -59,7 +69,11 @@ var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea
 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 
-var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
+var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
+./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
+./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
+`
+
 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 
 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
@@ -73,55 +87,50 @@ type TestDockerClient struct {
 	logReader   io.ReadCloser
 	logWriter   io.WriteCloser
 	fn          func(t *TestDockerClient)
-	finish      chan dockerclient.WaitResult
+	finish      int
 	stop        chan bool
 	cwd         string
 	env         []string
 	api         *ArvTestClient
+	realTemp    string
 }
 
-func NewTestDockerClient() *TestDockerClient {
+func NewTestDockerClient(exitCode int) *TestDockerClient {
 	t := &TestDockerClient{}
 	t.logReader, t.logWriter = io.Pipe()
-	t.finish = make(chan dockerclient.WaitResult)
-	t.stop = make(chan bool)
+	t.finish = exitCode
+	t.stop = make(chan bool, 1)
 	t.cwd = "/"
 	return t
 }
 
-func (t *TestDockerClient) StopContainer(id string, timeout int) error {
-	t.stop <- true
-	return nil
+type MockConn struct {
+	net.Conn
 }
 
-func (t *TestDockerClient) InspectImage(id string) (*dockerclient.ImageInfo, error) {
-	if t.imageLoaded == id {
-		return &dockerclient.ImageInfo{}, nil
-	} else {
-		return nil, errors.New("")
-	}
+func (m *MockConn) Write(b []byte) (int, error) {
+	return len(b), nil
 }
 
-func (t *TestDockerClient) LoadImage(reader io.Reader) error {
-	_, err := io.Copy(ioutil.Discard, reader)
-	if err != nil {
-		return err
-	} else {
-		t.imageLoaded = hwImageId
-		return nil
-	}
+func NewMockConn() *MockConn {
+	c := &MockConn{}
+	return c
+}
+
+func (t *TestDockerClient) ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error) {
+	return dockertypes.HijackedResponse{Conn: NewMockConn(), Reader: bufio.NewReader(t.logReader)}, nil
 }
 
-func (t *TestDockerClient) CreateContainer(config *dockerclient.ContainerConfig, name string, authConfig *dockerclient.AuthConfig) (string, error) {
+func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig, networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error) {
 	if config.WorkingDir != "" {
 		t.cwd = config.WorkingDir
 	}
 	t.env = config.Env
-	return "abcde", nil
+	return dockercontainer.ContainerCreateCreatedBody{ID: "abcde"}, nil
 }
 
-func (t *TestDockerClient) StartContainer(id string, config *dockerclient.HostConfig) error {
-	if id == "abcde" {
+func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
+	if container == "abcde" {
 		go t.fn(t)
 		return nil
 	} else {
@@ -129,15 +138,41 @@ func (t *TestDockerClient) StartContainer(id string, config *dockerclient.HostCo
 	}
 }
 
-func (t *TestDockerClient) AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error) {
-	return t.logReader, nil
+func (t *TestDockerClient) ContainerStop(ctx context.Context, container string, timeout *time.Duration) error {
+	t.stop <- true
+	return nil
 }
 
-func (t *TestDockerClient) Wait(id string) <-chan dockerclient.WaitResult {
-	return t.finish
+func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error) {
+	body := make(chan dockercontainer.ContainerWaitOKBody)
+	err := make(chan error)
+	go func() {
+		body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.finish)}
+		close(body)
+		close(err)
+	}()
+	return body, err
 }
 
-func (*TestDockerClient) RemoveImage(name string, force bool) ([]*dockerclient.ImageDelete, error) {
+func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
+	if t.imageLoaded == image {
+		return dockertypes.ImageInspect{}, nil, nil
+	} else {
+		return dockertypes.ImageInspect{}, nil, errors.New("")
+	}
+}
+
+func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
+	_, err := io.Copy(ioutil.Discard, input)
+	if err != nil {
+		return dockertypes.ImageLoadResponse{}, err
+	} else {
+		t.imageLoaded = hwImageId
+		return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil
+	}
+}
+
+func (*TestDockerClient) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) {
 	return nil, nil
 }
 
@@ -186,17 +221,22 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par
 
 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 	parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
-	j := []byte(`{
-		"command": ["sleep", "1"],
-		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
-		"cwd": ".",
-		"environment": {},
-		"mounts": {"/tmp": {"kind": "tmp"} },
-		"output_path": "/tmp",
-		"priority": 1,
-		"runtime_constraints": {}
-	}`)
-	return ioutil.NopCloser(bytes.NewReader(j)), nil
+	var j []byte
+	if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
+		j, err = json.Marshal(client.Container)
+	} else {
+		j = []byte(`{
+			"command": ["sleep", "1"],
+			"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+			"cwd": ".",
+			"environment": {},
+			"mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
+			"output_path": "/tmp",
+			"priority": 1,
+			"runtime_constraints": {}
+		}`)
+	}
+	return ioutil.NopCloser(bytes.NewReader(j)), err
 }
 
 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
@@ -230,7 +270,16 @@ func (client *ArvTestClient) Update(resourceType string, uuid string, parameters
 	return nil
 }
 
-var discoveryMap = map[string]interface{}{"defaultTrashLifetime": float64(1209600)}
+var discoveryMap = map[string]interface{}{
+	"defaultTrashLifetime":               float64(1209600),
+	"crunchLimitLogBytesPerJob":          float64(67108864),
+	"crunchLogThrottleBytes":             float64(65536),
+	"crunchLogThrottlePeriod":            float64(60),
+	"crunchLogThrottleLines":             float64(1024),
+	"crunchLogPartialLineThrottlePeriod": float64(5),
+	"crunchLogBytesPerEvent":             float64(4096),
+	"crunchLogSecondsBetweenEvents":      float64(1),
+}
 
 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 	return discoveryMap[key], nil
@@ -263,12 +312,15 @@ func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error
 	return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil
 }
 
+func (*KeepTestClient) ClearBlockCache() {
+}
+
 type FileWrapper struct {
 	io.ReadCloser
-	len uint64
+	len int64
 }
 
-func (fw FileWrapper) Len() uint64 {
+func (fw FileWrapper) Size() int64 {
 	return fw.len
 }
 
@@ -276,23 +328,27 @@ func (fw FileWrapper) Seek(int64, int) (int64, error) {
 	return 0, errors.New("not implemented")
 }
 
-func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 	if filename == hwImageId+".tar" {
 		rdr := ioutil.NopCloser(&bytes.Buffer{})
 		client.Called = true
 		return FileWrapper{rdr, 1321984}, nil
+	} else if filename == "/file1_in_main.txt" {
+		rdr := ioutil.NopCloser(strings.NewReader("foo"))
+		client.Called = true
+		return FileWrapper{rdr, 3}, nil
 	}
 	return nil, nil
 }
 
 func (s *TestSuite) TestLoadImage(c *C) {
 	kc := &KeepTestClient{}
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(0)
 	cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 
-	_, err := cr.Docker.RemoveImage(hwImageId, true)
+	_, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 
-	_, err = cr.Docker.InspectImage(hwImageId)
+	_, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
 	c.Check(err, NotNil)
 
 	cr.Container.ContainerImage = hwPDH
@@ -305,13 +361,13 @@ func (s *TestSuite) TestLoadImage(c *C) {
 
 	c.Check(err, IsNil)
 	defer func() {
-		cr.Docker.RemoveImage(hwImageId, true)
+		cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 	}()
 
 	c.Check(kc.Called, Equals, true)
 	c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 
-	_, err = cr.Docker.InspectImage(hwImageId)
+	_, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
 	c.Check(err, IsNil)
 
 	// (2) Test using image that's already loaded
@@ -360,16 +416,22 @@ func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 	return "", 0, errors.New("KeepError")
 }
 
-func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 	return nil, errors.New("KeepError")
 }
 
+func (KeepErrorTestClient) ClearBlockCache() {
+}
+
 type KeepReadErrorTestClient struct{}
 
 func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 	return "", 0, nil
 }
 
+func (KeepReadErrorTestClient) ClearBlockCache() {
+}
+
 type ErrorReader struct{}
 
 func (ErrorReader) Read(p []byte) (n int, err error) {
@@ -380,7 +442,7 @@ func (ErrorReader) Close() error {
 	return nil
 }
 
-func (ErrorReader) Len() uint64 {
+func (ErrorReader) Size() int64 {
 	return 0
 }
 
@@ -388,7 +450,7 @@ func (ErrorReader) Seek(int64, int) (int64, error) {
 	return 0, errors.New("ErrorReader")
 }
 
-func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 	return ErrorReader{}, nil
 }
 
@@ -403,7 +465,7 @@ func (s *TestSuite) TestLoadImageArvError(c *C) {
 
 func (s *TestSuite) TestLoadImageKeepError(c *C) {
 	// (2) Keep error
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(0)
 	cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 	cr.Container.ContainerImage = hwPDH
 
@@ -422,7 +484,7 @@ func (s *TestSuite) TestLoadImageCollectionError(c *C) {
 
 func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
 	// (4) Collection doesn't contain image
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(0)
 	cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 	cr.Container.ContainerImage = hwPDH
 
@@ -463,11 +525,10 @@ func dockerLog(fd byte, msg string) []byte {
 }
 
 func (s *TestSuite) TestRunContainer(c *C) {
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(0)
 	docker.fn = func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "Hello world\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{}
 	}
 	cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 
@@ -559,14 +620,14 @@ func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 
 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 // dress rehearsal of the Run() function, starting from a JSON container record.
-func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
+func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
 	rec := arvados.Container{}
 	err := json.Unmarshal([]byte(record), &rec)
 	c.Check(err, IsNil)
 
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(exitCode)
 	docker.fn = fn
-	docker.RemoveImage(hwImageId, true)
+	docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 
 	api = &ArvTestClient{Container: rec}
 	docker.api = api
@@ -579,6 +640,8 @@ func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDoc
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(realTemp)
 
+	docker.realTemp = realTemp
+
 	tempcount := 0
 	cr.MkTempDir = func(_ string, prefix string) (string, error) {
 		tempcount++
@@ -601,7 +664,9 @@ func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDoc
 	}
 
 	err = cr.Run()
-	c.Check(err, IsNil)
+	if api.CalledWith("container.state", "Complete") != nil {
+		c.Check(err, IsNil)
+	}
 	c.Check(api.WasSetRunning, Equals, true)
 
 	c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
@@ -626,10 +691,9 @@ func (s *TestSuite) TestFullRunHello(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello world\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -648,10 +712,9 @@ func (s *TestSuite) TestCrunchstat(c *C) {
 		"output_path": "/tmp",
 		"priority": 1,
 		"runtime_constraints": {}
-	}`, nil, func(t *TestDockerClient) {
+	}`, nil, 0, func(t *TestDockerClient) {
 		time.Sleep(time.Second)
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -681,11 +744,11 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
 		"output_path": "/tmp",
 		"priority": 1,
 		"runtime_constraints": {}
-	}`, nil, func(t *TestDockerClient) {
-		time.Sleep(time.Second)
-		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{}
-	})
+	}`, nil, 0,
+		func(t *TestDockerClient) {
+			time.Sleep(time.Second)
+			t.logWriter.Close()
+		})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
@@ -695,6 +758,7 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
 	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
 	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
 	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
+	c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
 }
 
 func (s *TestSuite) TestContainerRecordLog(c *C) {
@@ -707,11 +771,11 @@ func (s *TestSuite) TestContainerRecordLog(c *C) {
 		"output_path": "/tmp",
 		"priority": 1,
 		"runtime_constraints": {}
-	}`, nil, func(t *TestDockerClient) {
-		time.Sleep(time.Second)
-		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{}
-	})
+	}`, nil, 0,
+		func(t *TestDockerClient) {
+			time.Sleep(time.Second)
+			t.logWriter.Close()
+		})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
@@ -730,11 +794,10 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 1, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, "hello\n"))
 		t.logWriter.Write(dockerLog(2, "world\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 1}
 	})
 
 	final := api.CalledWith("container.state", "Complete")
@@ -756,10 +819,9 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -778,10 +840,9 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -826,14 +887,13 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
 	err := json.Unmarshal([]byte(record), &rec)
 	c.Check(err, IsNil)
 
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(0)
 	docker.fn = func(t *TestDockerClient) {
 		<-t.stop
 		t.logWriter.Write(dockerLog(1, "foo\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	}
-	docker.RemoveImage(hwImageId, true)
+	docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 
 	api := &ArvTestClient{Container: rec}
 	cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
@@ -871,10 +931,9 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -952,6 +1011,22 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 		checkEmpty()
 	}
 
+	{
+		i = 0
+		cr.ArvMountPoint = ""
+		cr.Container.Mounts = make(map[string]arvados.Mount)
+		cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
+		cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
+		cr.OutputPath = "/out"
+
+		err := cr.SetupMounts()
+		c.Check(err, IsNil)
+		c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+		c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/out", realTemp + "/3:/tmp"})
+		cr.CleanupDirs()
+		checkEmpty()
+	}
+
 	{
 		i = 0
 		cr.ArvMountPoint = ""
@@ -1041,7 +1116,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 	}{
 		{in: "foo", out: `"foo"`},
 		{in: nil, out: `null`},
-		{in: map[string]int{"foo": 123}, out: `{"foo":123}`},
+		{in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
 	} {
 		i = 0
 		cr.ArvMountPoint = ""
@@ -1115,6 +1190,22 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 		cr.CleanupDirs()
 		checkEmpty()
 	}
+
+	// Only mount point of kind 'collection' is allowed for stdin
+	{
+		i = 0
+		cr.ArvMountPoint = ""
+		cr.Container.Mounts = make(map[string]arvados.Mount)
+		cr.Container.Mounts = map[string]arvados.Mount{
+			"stdin": {Kind: "tmp"},
+		}
+
+		err := cr.SetupMounts()
+		c.Check(err, NotNil)
+		c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+		cr.CleanupDirs()
+		checkEmpty()
+	}
 }
 
 func (s *TestSuite) TestStdout(c *C) {
@@ -1129,10 +1220,9 @@ func (s *TestSuite) TestStdout(c *C) {
 		"runtime_constraints": {}
 	}`
 
-	api, _, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
+	api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -1146,9 +1236,9 @@ func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (ap
 	err = json.Unmarshal([]byte(record), &rec)
 	c.Check(err, IsNil)
 
-	docker := NewTestDockerClient()
+	docker := NewTestDockerClient(0)
 	docker.fn = fn
-	docker.RemoveImage(hwImageId, true)
+	docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 
 	api = &ArvTestClient{Container: rec}
 	cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
@@ -1201,10 +1291,9 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {"API": true}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -1225,10 +1314,9 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) {
     "output_path": "/tmp",
     "priority": 1,
     "runtime_constraints": {"API": true}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
 		t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -1257,10 +1345,9 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
 
 	extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
 
-	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -1293,10 +1380,9 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
 		"a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
 	}
 
-	api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+	api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
@@ -1346,10 +1432,9 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
 		"b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
 	}
 
-	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
 		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 		t.logWriter.Close()
-		t.finish <- dockerclient.WaitResult{ExitCode: 0}
 	})
 
 	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
@@ -1367,3 +1452,284 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
 		}
 	}
 }
+
+func (s *TestSuite) TestOutputSymlinkToInput(c *C) {
+	helperRecord := `{
+		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": "/bin",
+		"environment": {"FROBIZ": "bilbo"},
+		"mounts": {
+        "/tmp": {"kind": "tmp"},
+        "/keep/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
+        "/keep/foo2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"}
+    },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`
+
+	extraMounts := []string{
+		"a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
+	}
+
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+		os.Symlink("/keep/foo/sub1file2", t.realTemp+"/2/baz")
+		os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz2")
+		os.Symlink("/keep/foo2/subdir1", t.realTemp+"/2/baz3")
+		os.Mkdir(t.realTemp+"/2/baz4", 0700)
+		os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz4/baz5")
+		t.logWriter.Close()
+	})
+
+	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+	for _, v := range api.Content {
+		if v["collection"] != nil {
+			collection := v["collection"].(arvadosclient.Dict)
+			if strings.Index(collection["name"].(string), "output") == 0 {
+				manifest := collection["manifest_text"].(string)
+				c.Check(manifest, Equals, `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz 9:18:baz2
+./baz3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
+./baz3/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
+./baz4 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz5
+`)
+			}
+		}
+	}
+}
+
+func (s *TestSuite) TestOutputError(c *C) {
+	helperRecord := `{
+		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": "/bin",
+		"environment": {"FROBIZ": "bilbo"},
+		"mounts": {
+        "/tmp": {"kind": "tmp"}
+    },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`
+
+	extraMounts := []string{}
+
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+		os.Symlink("/etc/hosts", t.realTemp+"/2/baz")
+		t.logWriter.Close()
+	})
+
+	c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
+}
+
+func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
+	helperRecord := `{
+		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": "/bin",
+		"environment": {"FROBIZ": "bilbo"},
+		"mounts": {
+        "/tmp": {"kind": "tmp"}
+    },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`
+
+	extraMounts := []string{}
+
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+		rf, _ := os.Create(t.realTemp + "/2/realfile")
+		rf.Write([]byte("foo"))
+		rf.Close()
+
+		os.Mkdir(t.realTemp+"/2/realdir", 0700)
+		rf, _ = os.Create(t.realTemp + "/2/realdir/subfile")
+		rf.Write([]byte("bar"))
+		rf.Close()
+
+		os.Symlink("/tmp/realfile", t.realTemp+"/2/file1")
+		os.Symlink("realfile", t.realTemp+"/2/file2")
+		os.Symlink("/tmp/file1", t.realTemp+"/2/file3")
+		os.Symlink("file2", t.realTemp+"/2/file4")
+		os.Symlink("realdir", t.realTemp+"/2/dir1")
+		os.Symlink("/tmp/realdir", t.realTemp+"/2/dir2")
+		t.logWriter.Close()
+	})
+
+	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+	for _, v := range api.Content {
+		if v["collection"] != nil {
+			collection := v["collection"].(arvadosclient.Dict)
+			if strings.Index(collection["name"].(string), "output") == 0 {
+				manifest := collection["manifest_text"].(string)
+				c.Check(manifest, Equals,
+					`. 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile
+./dir1 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./dir2 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./realdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+`)
+			}
+		}
+	}
+}
+
+func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
+	helperRecord := `{
+		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": "/bin",
+		"environment": {"FROBIZ": "bilbo"},
+		"mounts": {
+        "/tmp": {"kind": "tmp"},
+        "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
+        "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
+    },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`
+
+	extraMounts := []string{
+		"b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
+	}
+
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
+		t.logWriter.Close()
+	})
+
+	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+	for _, v := range api.Content {
+		if v["collection"] != nil {
+			collection := v["collection"].(arvadosclient.Dict)
+			if strings.Index(collection["name"].(string), "output") == 0 {
+				manifest := collection["manifest_text"].(string)
+				c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+`)
+			}
+		}
+	}
+}
+
+func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
+	helperRecord := `{
+		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": "/bin",
+		"environment": {"FROBIZ": "bilbo"},
+		"mounts": {
+        "/tmp": {"kind": "tmp"},
+        "stdin": {"kind": "json", "content": "foo"},
+        "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
+    },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`
+
+	api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+		t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
+		t.logWriter.Close()
+	})
+
+	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+	for _, v := range api.Content {
+		if v["collection"] != nil {
+			collection := v["collection"].(arvadosclient.Dict)
+			if strings.Index(collection["name"].(string), "output") == 0 {
+				manifest := collection["manifest_text"].(string)
+				c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+`)
+			}
+		}
+	}
+}
+
+func (s *TestSuite) TestStderrMount(c *C) {
+	api, _, _ := FullRunHelper(c, `{
+    "command": ["/bin/sh", "-c", "echo hello;exit 1"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"},
+               "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
+               "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {}
+}`, nil, 1, func(t *TestDockerClient) {
+		t.logWriter.Write(dockerLog(1, "hello\n"))
+		t.logWriter.Write(dockerLog(2, "oops\n"))
+		t.logWriter.Close()
+	})
+
+	final := api.CalledWith("container.state", "Complete")
+	c.Assert(final, NotNil)
+	c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
+	c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
+
+	c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
+}
+
+func (s *TestSuite) TestNumberRoundTrip(c *C) {
+	cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+	cr.fetchContainerRecord()
+
+	jsondata, err := json.Marshal(cr.Container.Mounts["/json"].Content)
+
+	c.Check(err, IsNil)
+	c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
+}
+
+func (s *TestSuite) TestEvalSymlinks(c *C) {
+	cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+
+	realTemp, err := ioutil.TempDir("", "crunchrun_test-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(realTemp)
+
+	cr.HostOutputDir = realTemp
+
+	// Absolute path outside output dir
+	os.Symlink("/etc/passwd", realTemp+"/p1")
+
+	// Relative outside output dir
+	os.Symlink("../zip", realTemp+"/p2")
+
+	// Circular references
+	os.Symlink("p4", realTemp+"/p3")
+	os.Symlink("p5", realTemp+"/p4")
+	os.Symlink("p3", realTemp+"/p5")
+
+	// Target doesn't exist
+	os.Symlink("p99", realTemp+"/p6")
+
+	for _, v := range []string{"p1", "p2", "p3", "p4", "p5"} {
+		info, err := os.Lstat(realTemp + "/" + v)
+		_, _, _, err = cr.derefOutputSymlink(realTemp+"/"+v, info)
+		c.Assert(err, NotNil)
+	}
+}
+
+func (s *TestSuite) TestEvalSymlinkDir(c *C) {
+	cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+
+	realTemp, err := ioutil.TempDir("", "crunchrun_test-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(realTemp)
+
+	cr.HostOutputDir = realTemp
+
+	// Absolute path outside output dir
+	os.Symlink(".", realTemp+"/loop")
+
+	v := "loop"
+	info, err := os.Lstat(realTemp + "/" + v)
+	_, err = cr.UploadOutputFile(realTemp+"/"+v, info, err, []string{}, nil, "", "", 0)
+	c.Assert(err, NotNil)
+}