From faf5784b78844e5da0b4c63860323eb1ee06bd2f Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Sat, 13 Feb 2016 21:49:55 -0500 Subject: [PATCH] 8015: Fix existing tests, still need tests for new functionality --- services/crunch-run/crunchrun.go | 66 +++++++++++++++------------ services/crunch-run/crunchrun_test.go | 22 ++++----- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go index 5f63f5d75c..d197fa95f7 100644 --- a/services/crunch-run/crunchrun.go +++ b/services/crunch-run/crunchrun.go @@ -101,7 +101,8 @@ type ContainerRunner struct { ArvMount *exec.Cmd ArvMountPoint string HostOutputDir string - OutputPDH string + Binds []string + OutputPDH *string CancelLock sync.Mutex Cancelled bool SigChan chan os.Signal @@ -179,7 +180,7 @@ func (runner *ContainerRunner) LoadImage() (err error) { return nil } -func (runner *ContainerRunner) SetupMounts(hostConfig *dockerclient.HostConfig) (err error) { +func (runner *ContainerRunner) SetupMounts() (err error) { runner.ArvMountPoint, err = ioutil.TempDir("", "keep") if err != nil { return fmt.Errorf("While creating keep mount temp dir: %v", err) @@ -217,9 +218,9 @@ func (runner *ContainerRunner) SetupMounts(hostConfig *dockerclient.HostConfig) if bind == runner.ContainerRecord.OutputPath { runner.HostOutputDir = src } - hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:%s", src, bind)) + runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind)) } else { - hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:%s:ro", src, bind)) + runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind)) } collectionPaths = append(collectionPaths, src) } else if mnt.Kind == "tmp" { @@ -229,9 +230,9 @@ func (runner *ContainerRunner) SetupMounts(hostConfig *dockerclient.HostConfig) return fmt.Errorf("While creating mount temp dir: %v", err) } - hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:%s", runner.HostOutputDir, bind)) + runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", runner.HostOutputDir, bind)) } else { - hostConfig.Binds = append(hostConfig.Binds, bind) + runner.Binds = append(runner.Binds, bind) } } else { return fmt.Errorf("Unknown mount kind '%s'", mnt.Kind) @@ -284,16 +285,12 @@ func (runner *ContainerRunner) StartContainer() (err error) { for k, v := range runner.ContainerRecord.Environment { runner.ContainerConfig.Env = append(runner.ContainerConfig.Env, k+"="+v) } + runner.ContainerConfig.NetworkDisabled = true runner.ContainerID, err = runner.Docker.CreateContainer(&runner.ContainerConfig, "", nil) if err != nil { return fmt.Errorf("While creating container: %v", err) } - hostConfig := &dockerclient.HostConfig{} - - err = runner.SetupMounts(hostConfig) - if err != nil { - return fmt.Errorf("While setting up mounts: %v", err) - } + hostConfig := &dockerclient.HostConfig{Binds: runner.Binds} runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID) err = runner.Docker.StartContainer(runner.ContainerID, hostConfig) @@ -351,7 +348,7 @@ func (runner *ContainerRunner) WaitFinish() error { } // HandleOutput sets the output and unmounts the FUSE mount. -func (runner *ContainerRunner) HandleOutput() error { +func (runner *ContainerRunner) CaptureOutput() error { if runner.ArvMount != nil { defer func() { umount := exec.Command("fusermount", "-z", "-u", runner.ArvMountPoint) @@ -363,6 +360,10 @@ func (runner *ContainerRunner) HandleOutput() error { return nil } + if runner.HostOutputDir == "" { + return nil + } + _, err := os.Stat(runner.HostOutputDir) if err != nil { return fmt.Errorf("While checking host output path: %v", err) @@ -403,7 +404,8 @@ func (runner *ContainerRunner) HandleOutput() error { return fmt.Errorf("While creating output collection: %v", err) } - runner.OutputPDH = response.PortableDataHash + runner.OutputPDH = new(string) + *runner.OutputPDH = response.PortableDataHash return nil } @@ -425,17 +427,17 @@ func (runner *ContainerRunner) CommitLogs() error { return fmt.Errorf("While creating log manifest: %v", err) } - response := make(map[string]string) + var response CollectionRecord err = runner.ArvClient.Create("collections", arvadosclient.Dict{"name": "logs for " + runner.ContainerRecord.UUID, "manifest_text": mt}, - response) + &response) if err != nil { return fmt.Errorf("While creating log collection: %v", err) } runner.LogsPDH = new(string) - *runner.LogsPDH = response["portable_data_hash"] + *runner.LogsPDH = response.PortableDataHash return nil } @@ -456,9 +458,11 @@ func (runner *ContainerRunner) UpdateContainerRecordComplete() error { if runner.ExitCode != nil { update["exit_code"] = *runner.ExitCode } + if runner.OutputPDH != nil { + update["output"] = runner.OutputPDH + } update["state"] = runner.finalState - update["output"] = runner.OutputPDH return runner.ArvClient.Update("containers", runner.ContainerRecord.UUID, update, nil) } @@ -485,19 +489,19 @@ func (runner *ContainerRunner) Run(containerUUID string) (err error) { runner.finalState = "Complete" } - // (6) handle output - outputerr := runner.HandleOutput() + // (7) capture output + outputerr := runner.CaptureOutput() if outputerr != nil { runner.CrunchLog.Print(outputerr) } - // (7) write logs + // (8) write logs logerr := runner.CommitLogs() if logerr != nil { runner.CrunchLog.Print(logerr) } - // (8) update container record with results + // (9) update container record with results updateerr := runner.UpdateContainerRecordComplete() if updateerr != nil { runner.CrunchLog.Print(updateerr) @@ -523,19 +527,25 @@ func (runner *ContainerRunner) Run(containerUUID string) (err error) { return fmt.Errorf("While getting container record: %v", err) } - // (0) setup signal handling + // (1) setup signal handling err = runner.SetupSignals() if err != nil { return fmt.Errorf("While setting up signal handling: %v", err) } - // (1) check for and/or load image + // (2) check for and/or load image err = runner.LoadImage() if err != nil { return fmt.Errorf("While loading container image: %v", err) } - // (2) start container + // (3) set up FUSE mount and binds + err = runner.SetupMounts() + if err != nil { + return fmt.Errorf("While setting up mounts: %v", err) + } + + // (3) create and start container err = runner.StartContainer() if err != nil { if err == ErrCancelled { @@ -544,19 +554,19 @@ func (runner *ContainerRunner) Run(containerUUID string) (err error) { return } - // (3) update container record state + // (4) update container record state err = runner.UpdateContainerRecordRunning() if err != nil { runner.CrunchLog.Print(err) } - // (4) attach container logs + // (5) attach container logs runerr = runner.AttachLogs() if runerr != nil { runner.CrunchLog.Print(runerr) } - // (5) wait for container to finish + // (6) wait for container to finish waiterr = runner.WaitFinish() return diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go index 3c4dbf0891..eaa16d13bc 100644 --- a/services/crunch-run/crunchrun_test.go +++ b/services/crunch-run/crunchrun_test.go @@ -151,8 +151,8 @@ func (this *ArvTestClient) Create(resourceType string, if resourceType == "collections" && output != nil { mt := parameters["manifest_text"].(string) - outmap := output.(map[string]string) - outmap["portable_data_hash"] = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt)) + outmap := output.(*CollectionRecord) + outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt)) } return nil @@ -301,7 +301,7 @@ func (s *TestSuite) TestLoadImageArvError(c *C) { cr.ContainerRecord.ContainerImage = hwPDH err := cr.LoadImage() - c.Check(err.Error(), Equals, "ArvError") + c.Check(err.Error(), Equals, "While getting container image collection: ArvError") } func (s *TestSuite) TestLoadImageKeepError(c *C) { @@ -311,7 +311,7 @@ func (s *TestSuite) TestLoadImageKeepError(c *C) { cr.ContainerRecord.ContainerImage = hwPDH err := cr.LoadImage() - c.Check(err.Error(), Equals, "KeepError") + c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError") } func (s *TestSuite) TestLoadImageCollectionError(c *C) { @@ -320,7 +320,7 @@ func (s *TestSuite) TestLoadImageCollectionError(c *C) { cr.ContainerRecord.ContainerImage = otherPDH err := cr.LoadImage() - c.Check(err.Error(), Equals, "First file in the collection does not end in .tar") + c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar") } func (s *TestSuite) TestLoadImageKeepReadError(c *C) { @@ -490,7 +490,7 @@ func (s *TestSuite) TestFullRunHello(c *C) { "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": ".", "environment": {}, - "mounts": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, "runtime_constraints": {} @@ -514,7 +514,7 @@ func (s *TestSuite) TestFullRunStderr(c *C) { "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": ".", "environment": {}, - "mounts": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, "runtime_constraints": {} @@ -540,7 +540,7 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) { "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": ".", "environment": {}, - "mounts": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, "runtime_constraints": {} @@ -563,7 +563,7 @@ func (s *TestSuite) TestFullRunSetCwd(c *C) { "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": "/bin", "environment": {}, - "mounts": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, "runtime_constraints": {} @@ -586,7 +586,7 @@ func (s *TestSuite) TestCancel(c *C) { "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": ".", "environment": {}, - "mounts": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, "runtime_constraints": {} @@ -641,7 +641,7 @@ func (s *TestSuite) TestFullRunSetEnv(c *C) { "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", "cwd": "/bin", "environment": {"FROBIZ": "bilbo"}, - "mounts": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, "output_path": "/tmp", "priority": 1, "runtime_constraints": {} -- 2.39.5