11789: Merge branch 'master' into 11789-arvput-exclude-flag

[arvados.git] / services / crunch-run / crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 2b2c232cf4aeb7e298f7be65380bab87108ee7d4..9fdc689e79d06cba29d257a93bf9bc58e8b163c2 100644 (file)
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
@@ -10,7 +14,6 @@ import (
        "fmt"
        "io"
        "io/ioutil"
-       "log"
        "net"
        "os"
        "os/exec"
@@ -25,7 +28,6 @@ import (
 
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
        "git.curoverse.com/arvados.git/sdk/go/manifest"
 
        dockertypes "github.com/docker/docker/api/types"
@@ -66,7 +68,11 @@ var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea
 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
 
-var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
+var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
+./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
+./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
+`
+
 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
 
 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
@@ -136,8 +142,15 @@ func (t *TestDockerClient) ContainerStop(ctx context.Context, container string,
        return nil
 }
 
-func (t *TestDockerClient) ContainerWait(ctx context.Context, container string) (int64, error) {
-       return int64(t.finish), nil
+func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error) {
+       body := make(chan dockercontainer.ContainerWaitOKBody)
+       err := make(chan error)
+       go func() {
+               body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.finish)}
+               close(body)
+               close(err)
+       }()
+       return body, err
 }
 
 func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
@@ -295,10 +308,10 @@ func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error
 
 type FileWrapper struct {
        io.ReadCloser
-       len uint64
+       len int64
 }
 
-func (fw FileWrapper) Len() uint64 {
+func (fw FileWrapper) Size() int64 {
        return fw.len
 }
 
@@ -306,7 +319,7 @@ func (fw FileWrapper) Seek(int64, int) (int64, error) {
        return 0, errors.New("not implemented")
 }
 
-func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
        if filename == hwImageId+".tar" {
                rdr := ioutil.NopCloser(&bytes.Buffer{})
                client.Called = true
@@ -394,7 +407,7 @@ func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
        return "", 0, errors.New("KeepError")
 }
 
-func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
        return nil, errors.New("KeepError")
 }
 
@@ -414,7 +427,7 @@ func (ErrorReader) Close() error {
        return nil
 }
 
-func (ErrorReader) Len() uint64 {
+func (ErrorReader) Size() int64 {
        return 0
 }
 
@@ -422,7 +435,7 @@ func (ErrorReader) Seek(int64, int) (int64, error) {
        return 0, errors.New("ErrorReader")
 }
 
-func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
+func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
        return ErrorReader{}, nil
 }
 
@@ -636,7 +649,9 @@ func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn f
        }
 
        err = cr.Run()
-       c.Check(err, IsNil)
+       if api.CalledWith("container.state", "Complete") != nil {
+               c.Check(err, IsNil)
+       }
        c.Check(api.WasSetRunning, Equals, true)
 
        c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
@@ -1444,9 +1459,11 @@ func (s *TestSuite) TestOutputSymlinkToInput(c *C) {
        }
 
        api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
-               log.Printf("realtemp is %v", t.realTemp)
                os.Symlink("/keep/foo/sub1file2", t.realTemp+"/2/baz")
                os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz2")
+               os.Symlink("/keep/foo2/subdir1", t.realTemp+"/2/baz3")
+               os.Mkdir(t.realTemp+"/2/baz4", 0700)
+               os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz4/baz5")
                t.logWriter.Close()
        })
 
@@ -1458,12 +1475,39 @@ func (s *TestSuite) TestOutputSymlinkToInput(c *C) {
                        if strings.Index(collection["name"].(string), "output") == 0 {
                                manifest := collection["manifest_text"].(string)
                                c.Check(manifest, Equals, `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz 9:18:baz2
+./baz3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
+./baz3/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
+./baz4 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz5
 `)
                        }
                }
        }
 }
 
+func (s *TestSuite) TestOutputError(c *C) {
+       helperRecord := `{
+               "command": ["/bin/sh", "-c", "echo $FROBIZ"],
+               "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+               "cwd": "/bin",
+               "environment": {"FROBIZ": "bilbo"},
+               "mounts": {
+        "/tmp": {"kind": "tmp"}
+    },
+               "output_path": "/tmp",
+               "priority": 1,
+               "runtime_constraints": {}
+       }`
+
+       extraMounts := []string{}
+
+       api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+               os.Symlink("/etc/hosts", t.realTemp+"/2/baz")
+               t.logWriter.Close()
+       })
+
+       c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
+}
+
 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
        helperRecord := `{
                "command": ["/bin/sh", "-c", "echo $FROBIZ"],