X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ef2889f20ad25c30abb05c53cc5e08e2c2ed97d3..d97388bdbfeb6a43cb86996012a1db0ba4a8871f:/services/crunch-run/crunchrun_test.go diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go index 22989bb2ec..17e5e14581 100644 --- a/services/crunch-run/crunchrun_test.go +++ b/services/crunch-run/crunchrun_test.go @@ -16,7 +16,6 @@ import ( "net" "os" "os/exec" - "path/filepath" "runtime/pprof" "sort" "strings" @@ -46,10 +45,13 @@ func TestCrunchExec(t *testing.T) { var _ = Suite(&TestSuite{}) type TestSuite struct { + client *arvados.Client docker *TestDockerClient + runner *ContainerRunner } func (s *TestSuite) SetUpTest(c *C) { + s.client = arvados.NewClientFromEnv() s.docker = NewTestDockerClient() } @@ -58,7 +60,8 @@ type ArvTestClient struct { Calls int Content []arvadosclient.Dict arvados.Container - Logs map[string]*bytes.Buffer + secretMounts []byte + Logs map[string]*bytes.Buffer sync.Mutex WasSetRunning bool callraw bool @@ -101,6 +104,7 @@ type TestDockerClient struct { api *ArvTestClient realTemp string calledWait bool + ctrExited bool } func NewTestDockerClient() *TestDockerClient { @@ -174,6 +178,17 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, return body, err } +func (t *TestDockerClient) ContainerInspect(ctx context.Context, id string) (c dockertypes.ContainerJSON, err error) { + c.ContainerJSONBase = &dockertypes.ContainerJSONBase{} + c.ID = "abcde" + if t.ctrExited { + c.State = &dockertypes.ContainerState{Status: "exited", Dead: true} + } else { + c.State = &dockertypes.ContainerState{Status: "running", Pid: 1234, Running: true} + } + return +} + func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) { if t.exitCode == 2 { return dockertypes.ImageInspect{}, nil, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?") @@ -228,6 +243,7 @@ func (client *ArvTestClient) Create(resourceType string, mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string) outmap := output.(*arvados.Collection) outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt)) + outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt))) } return nil @@ -241,6 +257,12 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par "uuid": "`+fakeAuthUUID+`", "api_token": "`+fakeAuthToken+`" }`), output) + case method == "GET" && resourceType == "containers" && action == "secret_mounts": + if client.secretMounts != nil { + return json.Unmarshal(client.secretMounts, output) + } else { + return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output) + } default: return fmt.Errorf("Not found") } @@ -308,6 +330,10 @@ func (client *ArvTestClient) Update(resourceType string, uuid string, parameters if parameters["container"].(arvadosclient.Dict)["state"] == "Running" { client.WasSetRunning = true } + } else if resourceType == "collections" { + mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string) + output.(*arvados.Collection).UUID = uuid + output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt))) } return nil } @@ -349,12 +375,24 @@ call: return nil } -func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) { +func (client *KeepTestClient) LocalLocator(locator string) (string, error) { + return locator, nil +} + +func (client *KeepTestClient) PutB(buf []byte) (string, int, error) { client.Content = buf - return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil + return fmt.Sprintf("%x+%d", md5.Sum(buf), len(buf)), len(buf), nil +} + +func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) { + return 0, errors.New("not implemented") } -func (*KeepTestClient) ClearBlockCache() { +func (client *KeepTestClient) ClearBlockCache() { +} + +func (client *KeepTestClient) Close() { + client.Content = nil } type FileWrapper struct { @@ -386,6 +424,10 @@ func (fw FileWrapper) Write([]byte) (int, error) { return 0, errors.New("not implemented") } +func (fw FileWrapper) Sync() error { + return errors.New("not implemented") +} + func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) { if filename == hwImageId+".tar" { rdr := ioutil.NopCloser(&bytes.Buffer{}) @@ -400,10 +442,17 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s } func (s *TestSuite) TestLoadImage(c *C) { + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, + &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) + kc := &KeepTestClient{} - cr := NewContainerRunner(&ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + defer kc.Close() + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = kc - _, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + c.Check(err, IsNil) _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId) c.Check(err, NotNil) @@ -447,6 +496,9 @@ func (ArvErrorTestClient) Create(resourceType string, } func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error { + if method == "GET" && resourceType == "containers" && action == "auth" { + return nil + } return errors.New("ArvError") } @@ -467,26 +519,28 @@ func (ArvErrorTestClient) Discovery(key string) (interface{}, error) { return discoveryMap[key], nil } -type KeepErrorTestClient struct{} - -func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) { - return "", 0, errors.New("KeepError") +type KeepErrorTestClient struct { + KeepTestClient } -func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) { +func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) { return nil, errors.New("KeepError") } -func (KeepErrorTestClient) ClearBlockCache() { +func (*KeepErrorTestClient) PutB(buf []byte) (string, int, error) { + return "", 0, errors.New("KeepError") } -type KeepReadErrorTestClient struct{} +func (*KeepErrorTestClient) LocalLocator(string) (string, error) { + return "", errors.New("KeepError") +} -func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) { - return "", 0, nil +type KeepReadErrorTestClient struct { + KeepTestClient } -func (KeepReadErrorTestClient) ClearBlockCache() { +func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) { + return 0, errors.New("KeepError") } type ErrorReader struct { @@ -507,37 +561,60 @@ func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename func (s *TestSuite) TestLoadImageArvError(c *C) { // (1) Arvados error - cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepTestClient{} + defer kc.Close() + cr, err := NewContainerRunner(s.client, &ArvErrorTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) + + cr.ContainerArvClient = &ArvErrorTestClient{} + cr.ContainerKeepClient = &KeepTestClient{} + cr.Container.ContainerImage = hwPDH - err := cr.LoadImage() + err = cr.LoadImage() c.Check(err.Error(), Equals, "While getting container image collection: ArvError") } func (s *TestSuite) TestLoadImageKeepError(c *C) { // (2) Keep error - cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepErrorTestClient{} + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) + + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepErrorTestClient{} + cr.Container.ContainerImage = hwPDH - err := cr.LoadImage() + err = cr.LoadImage() + c.Assert(err, NotNil) c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError") } func (s *TestSuite) TestLoadImageCollectionError(c *C) { // (3) Collection doesn't contain image - cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepReadErrorTestClient{} + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.Container.ContainerImage = otherPDH - err := cr.LoadImage() + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepReadErrorTestClient{} + + err = cr.LoadImage() c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar") } func (s *TestSuite) TestLoadImageKeepReadError(c *C) { // (4) Collection doesn't contain image - cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepReadErrorTestClient{} + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.Container.ContainerImage = hwPDH + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepReadErrorTestClient{} - err := cr.LoadImage() + err = cr.LoadImage() c.Check(err, NotNil) } @@ -554,14 +631,14 @@ type TestLogs struct { Stderr ClosableBuffer } -func (tl *TestLogs) NewTestLoggingWriter(logstr string) io.WriteCloser { +func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) { if logstr == "stdout" { - return &tl.Stdout + return &tl.Stdout, nil } if logstr == "stderr" { - return &tl.Stderr + return &tl.Stderr, nil } - return nil + return nil, errors.New("???") } func dockerLog(fd byte, msg string) []byte { @@ -578,13 +655,19 @@ func (s *TestSuite) TestRunContainer(c *C) { t.logWriter.Write(dockerLog(1, "Hello world\n")) t.logWriter.Close() } - cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepTestClient{} + defer kc.Close() + cr, err := NewContainerRunner(s.client, &ArvTestClient{}, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) + + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepTestClient{} var logs TestLogs cr.NewLogWriter = logs.NewTestLoggingWriter cr.Container.ContainerImage = hwPDH cr.Container.Command = []string{"./hw"} - err := cr.LoadImage() + err = cr.LoadImage() c.Check(err, IsNil) err = cr.CreateContainer() @@ -603,14 +686,16 @@ func (s *TestSuite) TestRunContainer(c *C) { func (s *TestSuite) TestCommitLogs(c *C) { api := &ArvTestClient{} kc := &KeepTestClient{} - cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + defer kc.Close() + cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp cr.CrunchLog.Print("Hello world!") cr.CrunchLog.Print("Goodbye") cr.finalState = "Complete" - err := cr.CommitLogs() + err = cr.CommitLogs() c.Check(err, IsNil) c.Check(api.Calls, Equals, 2) @@ -623,9 +708,11 @@ func (s *TestSuite) TestCommitLogs(c *C) { func (s *TestSuite) TestUpdateContainerRunning(c *C) { api := &ArvTestClient{} kc := &KeepTestClient{} - cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + defer kc.Close() + cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) - err := cr.UpdateContainerRunning() + err = cr.UpdateContainerRunning() c.Check(err, IsNil) c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running") @@ -634,7 +721,9 @@ func (s *TestSuite) TestUpdateContainerRunning(c *C) { func (s *TestSuite) TestUpdateContainerComplete(c *C) { api := &ArvTestClient{} kc := &KeepTestClient{} - cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + defer kc.Close() + cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.LogsPDH = new(string) *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60" @@ -643,7 +732,7 @@ func (s *TestSuite) TestUpdateContainerComplete(c *C) { *cr.ExitCode = 42 cr.finalState = "Complete" - err := cr.UpdateContainerFinal() + err = cr.UpdateContainerFinal() c.Check(err, IsNil) c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH) @@ -654,11 +743,13 @@ func (s *TestSuite) TestUpdateContainerComplete(c *C) { func (s *TestSuite) TestUpdateContainerCancelled(c *C) { api := &ArvTestClient{} kc := &KeepTestClient{} - cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + defer kc.Close() + cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.cCancelled = true cr.finalState = "Cancelled" - err := cr.UpdateContainerFinal() + err = cr.UpdateContainerFinal() c.Check(err, IsNil) c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil) @@ -673,14 +764,28 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi err := json.Unmarshal([]byte(record), &rec) c.Check(err, IsNil) + var sm struct { + SecretMounts map[string]arvados.Mount `json:"secret_mounts"` + } + err = json.Unmarshal([]byte(record), &sm) + c.Check(err, IsNil) + secretMounts, err := json.Marshal(sm) + c.Logf("%s %q", sm, secretMounts) + c.Check(err, IsNil) + s.docker.exitCode = exitCode s.docker.fn = fn s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) api = &ArvTestClient{Container: rec} s.docker.api = api - cr = NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepTestClient{} + defer kc.Close() + cr, err = NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) + s.runner = cr cr.statInterval = 100 * time.Millisecond + cr.containerWatchdogInterval = time.Second am := &ArvMountCmdLine{} cr.RunArvMount = am.ArvMountTest @@ -701,6 +806,9 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi } return d, err } + cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) { + return &ArvTestClient{secretMounts: secretMounts}, &KeepTestClient{}, nil, nil + } if extraMounts != nil && len(extraMounts) > 0 { err := cr.SetupArvMountPoint("keep") @@ -717,7 +825,15 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi } if exitCode != 2 { c.Check(api.WasSetRunning, Equals, true) - c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil) + var lastupdate arvadosclient.Dict + for _, content := range api.Content { + if content["container"] != nil { + lastupdate = content["container"].(arvadosclient.Dict) + } + } + if lastupdate["log"] == nil { + c.Errorf("no container update with non-nil log -- updates were: %v", api.Content) + } } if err != nil { @@ -739,7 +855,8 @@ func (s *TestSuite) TestFullRunHello(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -751,6 +868,68 @@ func (s *TestSuite) TestFullRunHello(c *C) { } +func (s *TestSuite) TestRunAlreadyRunning(c *C) { + var ran bool + api, _, _ := s.fullRunHelper(c, `{ + "command": ["sleep", "3"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {}, + "scheduling_parameters":{"max_run_time": 1}, + "state": "Running" +}`, nil, 2, func(t *TestDockerClient) { + ran = true + }) + + c.Check(api.CalledWith("container.state", "Cancelled"), IsNil) + c.Check(api.CalledWith("container.state", "Complete"), IsNil) + c.Check(ran, Equals, false) +} + +func (s *TestSuite) TestRunTimeExceeded(c *C) { + api, _, _ := s.fullRunHelper(c, `{ + "command": ["sleep", "3"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {}, + "scheduling_parameters":{"max_run_time": 1}, + "state": "Locked" +}`, nil, 0, func(t *TestDockerClient) { + time.Sleep(3 * time.Second) + t.logWriter.Close() + }) + + c.Check(api.CalledWith("container.state", "Cancelled"), NotNil) + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*") +} + +func (s *TestSuite) TestContainerWaitFails(c *C) { + api, _, _ := s.fullRunHelper(c, `{ + "command": ["sleep", "3"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "state": "Locked" +}`, nil, 0, func(t *TestDockerClient) { + t.ctrExited = true + time.Sleep(10 * time.Second) + t.logWriter.Close() + }) + + c.Check(api.CalledWith("container.state", "Cancelled"), NotNil) + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*") +} + func (s *TestSuite) TestCrunchstat(c *C) { api, _, _ := s.fullRunHelper(c, `{ "command": ["sleep", "1"], @@ -760,7 +939,8 @@ func (s *TestSuite) TestCrunchstat(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(time.Second) t.logWriter.Close() @@ -793,7 +973,8 @@ func (s *TestSuite) TestNodeInfoLog(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(time.Second) @@ -827,7 +1008,8 @@ func (s *TestSuite) TestContainerRecordLog(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { time.Sleep(time.Second) @@ -850,7 +1032,8 @@ func (s *TestSuite) TestFullRunStderr(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 1, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello\n")) t.logWriter.Write(dockerLog(2, "world\n")) @@ -875,7 +1058,8 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.cwd+"\n")) t.logWriter.Close() @@ -896,7 +1080,8 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.cwd+"\n")) t.logWriter.Close() @@ -937,7 +1122,8 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` rec := arvados.Container{} @@ -952,8 +1138,14 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) { s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) api := &ArvTestClient{Container: rec} - cr := NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepTestClient{} + defer kc.Close() + cr, err := NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil } + cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) { + return &ArvTestClient{}, &KeepTestClient{}, nil, nil + } setup(cr) done := make(chan error) @@ -986,7 +1178,8 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n")) t.logWriter.Close() @@ -1019,9 +1212,13 @@ func stubCert(temp string) string { func (s *TestSuite) TestSetupMounts(c *C) { api := &ArvTestClient{} kc := &KeepTestClient{} - cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + defer kc.Close() + cr, err := NewContainerRunner(s.client, api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) am := &ArvMountCmdLine{} cr.RunArvMount = am.ArvMountTest + cr.ContainerArvClient = &ArvTestClient{} + cr.ContainerKeepClient = &KeepTestClient{} realTemp, err := ioutil.TempDir("", "crunchrun_test1-") c.Assert(err, IsNil) @@ -1029,6 +1226,8 @@ func (s *TestSuite) TestSetupMounts(c *C) { c.Assert(err, IsNil) stubCertPath := stubCert(certTemp) + cr.parentTemp = realTemp + defer os.RemoveAll(realTemp) defer os.RemoveAll(certTemp) @@ -1045,11 +1244,12 @@ func (s *TestSuite) TestSetupMounts(c *C) { } checkEmpty := func() { - filepath.Walk(realTemp, func(path string, _ os.FileInfo, err error) error { - c.Check(path, Equals, realTemp) - c.Check(err, IsNil) - return nil - }) + // Should be deleted. + _, err := os.Stat(realTemp) + c.Assert(os.IsNotExist(err), Equals, true) + + // Now recreate it for the next test. + c.Assert(os.Mkdir(realTemp, 0777), IsNil) } { @@ -1057,14 +1257,14 @@ func (s *TestSuite) TestSetupMounts(c *C) { cr.ArvMountPoint = "" cr.Container.Mounts = make(map[string]arvados.Mount) cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"} - cr.OutputPath = "/tmp" + cr.Container.OutputPath = "/tmp" cr.statInterval = 5 * time.Second err := cr.SetupMounts() c.Check(err, IsNil) c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--crunchstat-interval=5", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) - c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"}) + c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/tmp"}) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1076,14 +1276,14 @@ func (s *TestSuite) TestSetupMounts(c *C) { cr.Container.Mounts = make(map[string]arvados.Mount) cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"} cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"} - cr.OutputPath = "/out" + cr.Container.OutputPath = "/out" err := cr.SetupMounts() c.Check(err, IsNil) c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--crunchstat-interval=5", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) - c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/out", realTemp + "/3:/tmp"}) + c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/out", realTemp + "/tmp3:/tmp"}) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1094,7 +1294,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { cr.ArvMountPoint = "" cr.Container.Mounts = make(map[string]arvados.Mount) cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"} - cr.OutputPath = "/tmp" + cr.Container.OutputPath = "/tmp" apiflag := true cr.Container.RuntimeConstraints.API = &apiflag @@ -1104,7 +1304,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--crunchstat-interval=5", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) - c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"}) + c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"}) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1118,7 +1318,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { cr.Container.Mounts = map[string]arvados.Mount{ "/keeptmp": {Kind: "collection", Writable: true}, } - cr.OutputPath = "/keeptmp" + cr.Container.OutputPath = "/keeptmp" os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm) @@ -1140,7 +1340,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"}, "/keepout": {Kind: "collection", Writable: true}, } - cr.OutputPath = "/keepout" + cr.Container.OutputPath = "/keepout" os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm) os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm) @@ -1166,7 +1366,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"}, "/keepout": {Kind: "collection", Writable: true}, } - cr.OutputPath = "/keepout" + cr.Container.OutputPath = "/keepout" os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm) os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm) @@ -1200,8 +1400,8 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) sort.StringSlice(cr.Binds).Sort() - c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2/mountdata.json:/mnt/test.json:ro"}) - content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json") + c.Check(cr.Binds, DeepEquals, []string{realTemp + "/json2/mountdata.json:/mnt/test.json:ro"}) + content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json") c.Check(err, IsNil) c.Check(content, DeepEquals, []byte(test.out)) os.RemoveAll(cr.ArvMountPoint) @@ -1209,6 +1409,35 @@ func (s *TestSuite) TestSetupMounts(c *C) { checkEmpty() } + for _, test := range []struct { + in interface{} + out string + }{ + {in: "foo", out: `foo`}, + {in: nil, out: "error"}, + {in: map[string]int64{"foo": 123456789123456789}, out: "error"}, + } { + i = 0 + cr.ArvMountPoint = "" + cr.Container.Mounts = map[string]arvados.Mount{ + "/mnt/test.txt": {Kind: "text", Content: test.in}, + } + err := cr.SetupMounts() + if test.out == "error" { + c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string") + } else { + c.Check(err, IsNil) + sort.StringSlice(cr.Binds).Sort() + c.Check(cr.Binds, DeepEquals, []string{realTemp + "/text2/mountdata.text:/mnt/test.txt:ro"}) + content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text") + c.Check(err, IsNil) + c.Check(content, DeepEquals, []byte(test.out)) + } + os.RemoveAll(cr.ArvMountPoint) + cr.CleanupDirs() + checkEmpty() + } + // Read-only mount points are allowed underneath output_dir mount point { i = 0 @@ -1218,7 +1447,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { "/tmp": {Kind: "tmp"}, "/tmp/foo": {Kind: "collection"}, } - cr.OutputPath = "/tmp" + cr.Container.OutputPath = "/tmp" os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm) @@ -1227,26 +1456,42 @@ func (s *TestSuite) TestSetupMounts(c *C) { c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--crunchstat-interval=5", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) - c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"}) + c.Check(cr.Binds, DeepEquals, []string{realTemp + "/tmp2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"}) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } - // Writable mount points are not allowed underneath output_dir mount point + // Writable mount points copied to output_dir mount point { i = 0 cr.ArvMountPoint = "" cr.Container.Mounts = make(map[string]arvados.Mount) cr.Container.Mounts = map[string]arvados.Mount{ - "/tmp": {Kind: "tmp"}, - "/tmp/foo": {Kind: "collection", Writable: true}, + "/tmp": {Kind: "tmp"}, + "/tmp/foo": {Kind: "collection", + PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53", + Writable: true}, + "/tmp/bar": {Kind: "collection", + PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53", + Path: "baz", + Writable: true}, } - cr.OutputPath = "/tmp" + cr.Container.OutputPath = "/tmp" + + os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm) + os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm) + + rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux") + rf.Write([]byte("bar")) + rf.Close() err := cr.SetupMounts() - c.Check(err, NotNil) - c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`) + c.Check(err, IsNil) + _, err = os.Stat(cr.HostOutputDir + "/foo") + c.Check(err, IsNil) + _, err = os.Stat(cr.HostOutputDir + "/bar/quux") + c.Check(err, IsNil) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1259,13 +1504,13 @@ func (s *TestSuite) TestSetupMounts(c *C) { cr.Container.Mounts = make(map[string]arvados.Mount) cr.Container.Mounts = map[string]arvados.Mount{ "/tmp": {Kind: "tmp"}, - "/tmp/foo": {Kind: "json"}, + "/tmp/foo": {Kind: "tmp"}, } - cr.OutputPath = "/tmp" + cr.Container.OutputPath = "/tmp" err := cr.SetupMounts() c.Check(err, NotNil) - c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`) + c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1309,7 +1554,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { Path: "/", }, } - cr.OutputPath = "/tmp" + cr.Container.OutputPath = "/tmp" err := cr.SetupMounts() c.Check(err, IsNil) @@ -1356,17 +1601,18 @@ func (s *TestSuite) TestStdout(c *C) { "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` - api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { + api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n")) t.logWriter.Close() }) c.Check(api.CalledWith("container.exit_code", 0), NotNil) c.Check(api.CalledWith("container.state", "Complete"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) } // Used by the TestStdoutWithWrongPath*() @@ -1379,9 +1625,15 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) api = &ArvTestClient{Container: rec} - cr = NewContainerRunner(api, &KeepTestClient{}, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepTestClient{} + defer kc.Close() + cr, err = NewContainerRunner(s.client, api, kc, s.docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) am := &ArvMountCmdLine{} cr.RunArvMount = am.ArvMountTest + cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) { + return &ArvTestClient{}, &KeepTestClient{}, nil, nil + } err = cr.Run() return @@ -1390,7 +1642,8 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc func (s *TestSuite) TestStdoutWithWrongPath(c *C) { _, _, err := s.stdoutErrorRunHelper(c, `{ "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} }, - "output_path": "/tmp" + "output_path": "/tmp", + "state": "Locked" }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) @@ -1400,7 +1653,8 @@ func (s *TestSuite) TestStdoutWithWrongPath(c *C) { func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) { _, _, err := s.stdoutErrorRunHelper(c, `{ "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} }, - "output_path": "/tmp" + "output_path": "/tmp", + "state": "Locked" }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) @@ -1410,7 +1664,8 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) { func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) { _, _, err := s.stdoutErrorRunHelper(c, `{ "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} }, - "output_path": "/tmp" + "output_path": "/tmp", + "state": "Locked" }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) @@ -1418,8 +1673,8 @@ func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) { } func (s *TestSuite) TestFullRunWithAPI(c *C) { + defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST")) os.Setenv("ARVADOS_API_HOST", "test.arvados.org") - defer os.Unsetenv("ARVADOS_API_HOST") api, _, _ := s.fullRunHelper(c, `{ "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"], "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", @@ -1428,7 +1683,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {"API": true} + "runtime_constraints": {"API": true}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n")) t.logWriter.Close() @@ -1441,8 +1697,8 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) { } func (s *TestSuite) TestFullRunSetOutput(c *C) { + defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST")) os.Setenv("ARVADOS_API_HOST", "test.arvados.org") - defer os.Unsetenv("ARVADOS_API_HOST") api, _, _ := s.fullRunHelper(c, `{ "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"], "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", @@ -1451,7 +1707,8 @@ func (s *TestSuite) TestFullRunSetOutput(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {"API": true} + "runtime_constraints": {"API": true}, + "state": "Locked" }`, nil, 0, func(t *TestDockerClient) { t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122" t.logWriter.Close() @@ -1478,19 +1735,20 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"} - api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { + api, cr, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n")) t.logWriter.Close() }) c.Check(api.CalledWith("container.exit_code", 0), NotNil) c.Check(api.CalledWith("container.state", "Complete"), NotNil) - c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil) } func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { @@ -1509,7 +1767,8 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{ @@ -1523,7 +1782,7 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { t.logWriter.Close() }) - c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp", + c.Check(runner.Binds, DeepEquals, []string{realtemp + "/tmp2:/tmp", realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro", realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro", realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro", @@ -1540,7 +1799,7 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) { manifest := collection["manifest_text"].(string) c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out -./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 9:18:bar 9:18:sub1file2 +./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt @@ -1558,12 +1817,13 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest( "environment": {"FROBIZ": "bilbo"}, "mounts": { "/tmp": {"kind": "tmp"}, - "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"}, + "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{ @@ -1591,52 +1851,6 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest( } } -func (s *TestSuite) TestOutputSymlinkToInput(c *C) { - helperRecord := `{ - "command": ["/bin/sh", "-c", "echo $FROBIZ"], - "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", - "cwd": "/bin", - "environment": {"FROBIZ": "bilbo"}, - "mounts": { - "/tmp": {"kind": "tmp"}, - "/keep/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"}, - "/keep/foo2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"} - }, - "output_path": "/tmp", - "priority": 1, - "runtime_constraints": {} - }` - - extraMounts := []string{ - "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", - } - - api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { - os.Symlink("/keep/foo/sub1file2", t.realTemp+"/2/baz") - os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz2") - os.Symlink("/keep/foo2/subdir1", t.realTemp+"/2/baz3") - os.Mkdir(t.realTemp+"/2/baz4", 0700) - os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz4/baz5") - t.logWriter.Close() - }) - - c.Check(api.CalledWith("container.exit_code", 0), NotNil) - c.Check(api.CalledWith("container.state", "Complete"), NotNil) - for _, v := range api.Content { - if v["collection"] != nil { - collection := v["collection"].(arvadosclient.Dict) - if strings.Index(collection["name"].(string), "output") == 0 { - manifest := collection["manifest_text"].(string) - c.Check(manifest, Equals, `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz 9:18:baz2 -./baz3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt -./baz3/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt -./baz4 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz5 -`) - } - } - } -} - func (s *TestSuite) TestOutputError(c *C) { helperRecord := `{ "command": ["/bin/sh", "-c", "echo $FROBIZ"], @@ -1644,76 +1858,24 @@ func (s *TestSuite) TestOutputError(c *C) { "cwd": "/bin", "environment": {"FROBIZ": "bilbo"}, "mounts": { - "/tmp": {"kind": "tmp"} - }, + "/tmp": {"kind": "tmp"} + }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{} api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { - os.Symlink("/etc/hosts", t.realTemp+"/2/baz") + os.Symlink("/etc/hosts", t.realTemp+"/tmp2/baz") t.logWriter.Close() }) c.Check(api.CalledWith("container.state", "Cancelled"), NotNil) } -func (s *TestSuite) TestOutputSymlinkToOutput(c *C) { - helperRecord := `{ - "command": ["/bin/sh", "-c", "echo $FROBIZ"], - "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", - "cwd": "/bin", - "environment": {"FROBIZ": "bilbo"}, - "mounts": { - "/tmp": {"kind": "tmp"} - }, - "output_path": "/tmp", - "priority": 1, - "runtime_constraints": {} - }` - - extraMounts := []string{} - - api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) { - rf, _ := os.Create(t.realTemp + "/2/realfile") - rf.Write([]byte("foo")) - rf.Close() - - os.Mkdir(t.realTemp+"/2/realdir", 0700) - rf, _ = os.Create(t.realTemp + "/2/realdir/subfile") - rf.Write([]byte("bar")) - rf.Close() - - os.Symlink("/tmp/realfile", t.realTemp+"/2/file1") - os.Symlink("realfile", t.realTemp+"/2/file2") - os.Symlink("/tmp/file1", t.realTemp+"/2/file3") - os.Symlink("file2", t.realTemp+"/2/file4") - os.Symlink("realdir", t.realTemp+"/2/dir1") - os.Symlink("/tmp/realdir", t.realTemp+"/2/dir2") - t.logWriter.Close() - }) - - c.Check(api.CalledWith("container.exit_code", 0), NotNil) - c.Check(api.CalledWith("container.state", "Complete"), NotNil) - for _, v := range api.Content { - if v["collection"] != nil { - collection := v["collection"].(arvadosclient.Dict) - if strings.Index(collection["name"].(string), "output") == 0 { - manifest := collection["manifest_text"].(string) - c.Check(manifest, Equals, - `. 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile -./dir1 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile -./dir2 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile -./realdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile -`) - } - } - } -} - func (s *TestSuite) TestStdinCollectionMountPoint(c *C) { helperRecord := `{ "command": ["/bin/sh", "-c", "echo $FROBIZ"], @@ -1727,7 +1889,8 @@ func (s *TestSuite) TestStdinCollectionMountPoint(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` extraMounts := []string{ @@ -1766,7 +1929,8 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) { }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }` api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { @@ -1789,7 +1953,7 @@ func (s *TestSuite) TestStdinJsonMountPoint(c *C) { } func (s *TestSuite) TestStderrMount(c *C) { - api, _, _ := s.fullRunHelper(c, `{ + api, cr, _ := s.fullRunHelper(c, `{ "command": ["/bin/sh", "-c", "echo hello;exit 1"], "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": ".", @@ -1799,7 +1963,8 @@ func (s *TestSuite) TestStderrMount(c *C) { "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}}, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 1, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello\n")) t.logWriter.Write(dockerLog(2, "oops\n")) @@ -1811,11 +1976,14 @@ func (s *TestSuite) TestStderrMount(c *C) { c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1) c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil) - c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil) } func (s *TestSuite) TestNumberRoundTrip(c *C) { - cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + kc := &KeepTestClient{} + defer kc.Close() + cr, err := NewContainerRunner(s.client, &ArvTestClient{callraw: true}, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") + c.Assert(err, IsNil) cr.fetchContainerRecord() jsondata, err := json.Marshal(cr.Container.Mounts["/json"].Content) @@ -1824,54 +1992,6 @@ func (s *TestSuite) TestNumberRoundTrip(c *C) { c.Check(string(jsondata), Equals, `{"number":123456789123456789}`) } -func (s *TestSuite) TestEvalSymlinks(c *C) { - cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") - - realTemp, err := ioutil.TempDir("", "crunchrun_test-") - c.Assert(err, IsNil) - defer os.RemoveAll(realTemp) - - cr.HostOutputDir = realTemp - - // Absolute path outside output dir - os.Symlink("/etc/passwd", realTemp+"/p1") - - // Relative outside output dir - os.Symlink("../zip", realTemp+"/p2") - - // Circular references - os.Symlink("p4", realTemp+"/p3") - os.Symlink("p5", realTemp+"/p4") - os.Symlink("p3", realTemp+"/p5") - - // Target doesn't exist - os.Symlink("p99", realTemp+"/p6") - - for _, v := range []string{"p1", "p2", "p3", "p4", "p5"} { - info, err := os.Lstat(realTemp + "/" + v) - _, _, _, err = cr.derefOutputSymlink(realTemp+"/"+v, info) - c.Assert(err, NotNil) - } -} - -func (s *TestSuite) TestEvalSymlinkDir(c *C) { - cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz") - - realTemp, err := ioutil.TempDir("", "crunchrun_test-") - c.Assert(err, IsNil) - defer os.RemoveAll(realTemp) - - cr.HostOutputDir = realTemp - - // Absolute path outside output dir - os.Symlink(".", realTemp+"/loop") - - v := "loop" - info, err := os.Lstat(realTemp + "/" + v) - _, err = cr.UploadOutputFile(realTemp+"/"+v, info, err, []string{}, nil, "", "", 0) - c.Assert(err, NotNil) -} - func (s *TestSuite) TestFullBrokenDocker1(c *C) { tf, err := ioutil.TempFile("", "brokenNodeHook-") c.Assert(err, IsNil) @@ -1894,7 +2014,8 @@ exec echo killme "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 2, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1919,7 +2040,8 @@ func (s *TestSuite) TestFullBrokenDocker2(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 2, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1942,7 +2064,8 @@ func (s *TestSuite) TestFullBrokenDocker3(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 3, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1964,7 +2087,8 @@ func (s *TestSuite) TestBadCommand1(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 4, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -1986,7 +2110,8 @@ func (s *TestSuite) TestBadCommand2(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 5, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -2008,7 +2133,8 @@ func (s *TestSuite) TestBadCommand3(c *C) { "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, - "runtime_constraints": {} + "runtime_constraints": {}, + "state": "Locked" }`, nil, 6, func(t *TestDockerClient) { t.logWriter.Write(dockerLog(1, "hello world\n")) t.logWriter.Close() @@ -2017,3 +2143,71 @@ func (s *TestSuite) TestBadCommand3(c *C) { c.Check(api.CalledWith("container.state", "Cancelled"), NotNil) c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*") } + +func (s *TestSuite) TestSecretTextMountPoint(c *C) { + // under normal mounts, gets captured in output, oops + helperRecord := `{ + "command": ["true"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": "/bin", + "mounts": { + "/tmp": {"kind": "tmp"}, + "/tmp/secret.conf": {"kind": "text", "content": "mypassword"} + }, + "secret_mounts": { + }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {}, + "state": "Locked" + }` + + api, cr, _ := s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { + content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf") + c.Check(err, IsNil) + c.Check(content, DeepEquals, []byte("mypassword")) + t.logWriter.Close() + }) + + c.Check(api.CalledWith("container.exit_code", 0), NotNil) + c.Check(api.CalledWith("container.state", "Complete"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil) + + // under secret mounts, not captured in output + helperRecord = `{ + "command": ["true"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": "/bin", + "mounts": { + "/tmp": {"kind": "tmp"} + }, + "secret_mounts": { + "/tmp/secret.conf": {"kind": "text", "content": "mypassword"} + }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {}, + "state": "Locked" + }` + + api, cr, _ = s.fullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) { + content, err := ioutil.ReadFile(t.realTemp + "/tmp2/secret.conf") + c.Check(err, IsNil) + c.Check(content, DeepEquals, []byte("mypassword")) + t.logWriter.Close() + }) + + c.Check(api.CalledWith("container.exit_code", 0), NotNil) + c.Check(api.CalledWith("container.state", "Complete"), NotNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil) + c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil) +} + +type FakeProcess struct { + cmdLine []string +} + +func (fp FakeProcess) CmdlineSlice() ([]string, error) { + return fp.cmdLine, nil +}