X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b3f88934721fc97973b40786cae97e55e84b31d5..289d2cf581b59632369087388f6163f3979c5e86:/services/crunch-run/crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 8cefbedf19..d3c9990770 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
@@ -24,7 +28,6 @@ import (
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
 	"git.curoverse.com/arvados.git/sdk/go/manifest"
 
 	dockertypes "github.com/docker/docker/api/types"
@@ -51,6 +54,7 @@ type ArvTestClient struct {
 	Logs map[string]*bytes.Buffer
 	sync.Mutex
 	WasSetRunning bool
+	callraw       bool
 }
 
 type KeepTestClient struct {
@@ -95,7 +99,7 @@ func NewTestDockerClient(exitCode int) *TestDockerClient {
 	t := &TestDockerClient{}
 	t.logReader, t.logWriter = io.Pipe()
 	t.finish = exitCode
-	t.stop = make(chan bool)
+	t.stop = make(chan bool, 1)
 	t.cwd = "/"
 	return t
 }
@@ -217,17 +221,22 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par
 
 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 	parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
-	j := []byte(`{
-		"command": ["sleep", "1"],
-		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
-		"cwd": ".",
-		"environment": {},
-		"mounts": {"/tmp": {"kind": "tmp"} },
-		"output_path": "/tmp",
-		"priority": 1,
-		"runtime_constraints": {}
-	}`)
-	return ioutil.NopCloser(bytes.NewReader(j)), nil
+	var j []byte
+	if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
+		j, err = json.Marshal(client.Container)
+	} else {
+		j = []byte(`{
+			"command": ["sleep", "1"],
+			"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+			"cwd": ".",
+			"environment": {},
+			"mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
+			"output_path": "/tmp",
+			"priority": 1,
+			"runtime_constraints": {}
+		}`)
+	}
+	return ioutil.NopCloser(bytes.NewReader(j)), err
 }
 
 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
@@ -303,12 +312,15 @@ func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error
 	return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil
 }
 
+func (*KeepTestClient) ClearBlockCache() {
+}
+
 type FileWrapper struct {
 	io.ReadCloser
-	len uint64
+	len int64
 }
 
-func (fw FileWrapper) Len() uint64 {
+func (fw FileWrapper) Size() int64 {
 	return fw.len
 }
 
@@ -316,7 +328,7 @@ func (fw FileWrapper) Seek(int64, int) (int64, error) {
 	return 0, errors.New("not implemented")
 }
 
-func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 	if filename == hwImageId+".tar" {
 		rdr := ioutil.NopCloser(&bytes.Buffer{})
 		client.Called = true
@@ -404,16 +416,22 @@ func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 	return "", 0, errors.New("KeepError")
 }
 
-func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 	return nil, errors.New("KeepError")
 }
 
+func (KeepErrorTestClient) ClearBlockCache() {
+}
+
 type KeepReadErrorTestClient struct{}
 
 func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 	return "", 0, nil
 }
 
+func (KeepReadErrorTestClient) ClearBlockCache() {
+}
+
 type ErrorReader struct{}
 
 func (ErrorReader) Read(p []byte) (n int, err error) {
@@ -424,7 +442,7 @@ func (ErrorReader) Close() error {
 	return nil
 }
 
-func (ErrorReader) Len() uint64 {
+func (ErrorReader) Size() int64 {
 	return 0
 }
 
@@ -432,7 +450,7 @@ func (ErrorReader) Seek(int64, int) (int64, error) {
 	return 0, errors.New("ErrorReader")
 }
 
-func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 	return ErrorReader{}, nil
 }
 
@@ -1098,7 +1116,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 	}{
 		{in: "foo", out: `"foo"`},
 		{in: nil, out: `null`},
-		{in: map[string]int{"foo": 123}, out: `{"foo":123}`},
+		{in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
 	} {
 		i = 0
 		cr.ArvMountPoint = ""
@@ -1505,6 +1523,59 @@ func (s *TestSuite) TestOutputError(c *C) {
 	c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
 }
 
+func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
+	helperRecord := `{
+		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
+		"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+		"cwd": "/bin",
+		"environment": {"FROBIZ": "bilbo"},
+		"mounts": {
+        "/tmp": {"kind": "tmp"}
+    },
+		"output_path": "/tmp",
+		"priority": 1,
+		"runtime_constraints": {}
+	}`
+
+	extraMounts := []string{}
+
+	api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+		rf, _ := os.Create(t.realTemp + "/2/realfile")
+		rf.Write([]byte("foo"))
+		rf.Close()
+
+		os.Mkdir(t.realTemp+"/2/realdir", 0700)
+		rf, _ = os.Create(t.realTemp + "/2/realdir/subfile")
+		rf.Write([]byte("bar"))
+		rf.Close()
+
+		os.Symlink("/tmp/realfile", t.realTemp+"/2/file1")
+		os.Symlink("realfile", t.realTemp+"/2/file2")
+		os.Symlink("/tmp/file1", t.realTemp+"/2/file3")
+		os.Symlink("file2", t.realTemp+"/2/file4")
+		os.Symlink("realdir", t.realTemp+"/2/dir1")
+		os.Symlink("/tmp/realdir", t.realTemp+"/2/dir2")
+		t.logWriter.Close()
+	})
+
+	c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+	c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+	for _, v := range api.Content {
+		if v["collection"] != nil {
+			collection := v["collection"].(arvadosclient.Dict)
+			if strings.Index(collection["name"].(string), "output") == 0 {
+				manifest := collection["manifest_text"].(string)
+				c.Check(manifest, Equals,
+					`. 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile
+./dir1 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./dir2 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+./realdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
+`)
+			}
+		}
+	}
+}
+
 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
 	helperRecord := `{
 		"command": ["/bin/sh", "-c", "echo $FROBIZ"],
@@ -1604,3 +1675,61 @@ func (s *TestSuite) TestStderrMount(c *C) {
 
 	c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
 }
+
+func (s *TestSuite) TestNumberRoundTrip(c *C) {
+	cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+	cr.fetchContainerRecord()
+
+	jsondata, err := json.Marshal(cr.Container.Mounts["/json"].Content)
+
+	c.Check(err, IsNil)
+	c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
+}
+
+func (s *TestSuite) TestEvalSymlinks(c *C) {
+	cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+
+	realTemp, err := ioutil.TempDir("", "crunchrun_test-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(realTemp)
+
+	cr.HostOutputDir = realTemp
+
+	// Absolute path outside output dir
+	os.Symlink("/etc/passwd", realTemp+"/p1")
+
+	// Relative outside output dir
+	os.Symlink("../zip", realTemp+"/p2")
+
+	// Circular references
+	os.Symlink("p4", realTemp+"/p3")
+	os.Symlink("p5", realTemp+"/p4")
+	os.Symlink("p3", realTemp+"/p5")
+
+	// Target doesn't exist
+	os.Symlink("p99", realTemp+"/p6")
+
+	for _, v := range []string{"p1", "p2", "p3", "p4", "p5"} {
+		info, err := os.Lstat(realTemp + "/" + v)
+		_, _, _, err = cr.derefOutputSymlink(realTemp+"/"+v, info)
+		c.Assert(err, NotNil)
+	}
+}
+
+func (s *TestSuite) TestEvalSymlinkDir(c *C) {
+	cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+
+	realTemp, err := ioutil.TempDir("", "crunchrun_test-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(realTemp)
+
+	cr.HostOutputDir = realTemp
+
+	// Absolute path outside output dir
+	os.Symlink(".", realTemp+"/loop")
+
+	v := "loop"
+	info, err := os.Lstat(realTemp + "/" + v)
+	_, err = cr.UploadOutputFile(realTemp+"/"+v, info, err, []string{}, nil, "", "", 0)
+	c.Assert(err, NotNil)
+}