X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/11eda09c7340da2c0a01259e3711758782c9b1c9..02d7f4e820b65d83a5e3709dc14b6d72b9f5ab49:/services/crunch-run/crunchrun.go diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go index 5437e3653a..098c53f8a6 100644 --- a/services/crunch-run/crunchrun.go +++ b/services/crunch-run/crunchrun.go @@ -32,6 +32,7 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/keepclient" "git.curoverse.com/arvados.git/sdk/go/manifest" + "github.com/shirou/gopsutil/process" "golang.org/x/net/context" dockertypes "github.com/docker/docker/api/types" @@ -57,13 +58,14 @@ var ErrCancelled = errors.New("Cancelled") // IKeepClient is the minimal Keep API methods used by crunch-run. type IKeepClient interface { - PutHB(hash string, buf []byte) (string, int, error) + PutB(buf []byte) (string, int, error) + ReadAt(locator string, p []byte, off int) (int, error) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) ClearBlockCache() } // NewLogWriter is a factory function to create a new log writer. -type NewLogWriter func(name string) io.WriteCloser +type NewLogWriter func(name string) (io.WriteCloser, error) type RunArvMount func(args []string, tok string) (*exec.Cmd, error) @@ -82,10 +84,15 @@ type ThinDockerClient interface { ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) } +type PsProcess interface { + CmdlineSlice() ([]string, error) +} + // ContainerRunner is the main stateful struct used for a single execution of a // container. type ContainerRunner struct { Docker ThinDockerClient + client *arvados.Client ArvClient IArvadosClient Kc IKeepClient arvados.Container @@ -99,25 +106,31 @@ type ContainerRunner struct { CrunchLog *ThrottledLogger Stdout io.WriteCloser Stderr io.WriteCloser - LogCollection *CollectionWriter + LogCollection arvados.CollectionFileSystem LogsPDH *string RunArvMount MkTempDir - ArvMount *exec.Cmd - ArvMountPoint string - HostOutputDir string - CleanupTempDir []string - Binds []string - Volumes map[string]struct{} - OutputPDH *string - SigChan chan os.Signal - ArvMountExit chan error - finalState string - - statLogger io.WriteCloser - statReporter *crunchstat.Reporter - statInterval time.Duration - cgroupRoot string + ArvMount *exec.Cmd + ArvMountPoint string + HostOutputDir string + Binds []string + Volumes map[string]struct{} + OutputPDH *string + SigChan chan os.Signal + ArvMountExit chan error + SecretMounts map[string]arvados.Mount + MkArvClient func(token string) (IArvadosClient, error) + finalState string + parentTemp string + + ListProcesses func() ([]PsProcess, error) + + statLogger io.WriteCloser + statReporter *crunchstat.Reporter + hoststatLogger io.WriteCloser + hoststatReporter *crunchstat.Reporter + statInterval time.Duration + cgroupRoot string // What we expect the container's cgroup parent to be. expectCgroupParent string // What we tell docker to use as the container's cgroup @@ -135,9 +148,10 @@ type ContainerRunner struct { cStateLock sync.Mutex cCancelled bool // StopContainer() invoked - enableNetwork string // one of "default" or "always" - networkMode string // passed through to HostConfig.NetworkMode - arvMountLog *ThrottledLogger + enableNetwork string // one of "default" or "always" + networkMode string // passed through to HostConfig.NetworkMode + arvMountLog *ThrottledLogger + checkContainerd time.Duration } // setupSignals sets up signal handling to gracefully terminate the underlying @@ -150,16 +164,18 @@ func (runner *ContainerRunner) setupSignals() { go func(sig chan os.Signal) { for s := range sig { - runner.CrunchLog.Printf("caught signal: %v", s) - runner.stop() + runner.stop(s) } }(runner.SigChan) } // stop the underlying Docker container. -func (runner *ContainerRunner) stop() { +func (runner *ContainerRunner) stop(sig os.Signal) { runner.cStateLock.Lock() defer runner.cStateLock.Unlock() + if sig != nil { + runner.CrunchLog.Printf("caught signal: %v", sig) + } if runner.ContainerID == "" { return } @@ -171,35 +187,34 @@ func (runner *ContainerRunner) stop() { } } -func (runner *ContainerRunner) stopSignals() { - if runner.SigChan != nil { - signal.Stop(runner.SigChan) - } -} - var errorBlacklist = []string{ "(?ms).*[Cc]annot connect to the Docker daemon.*", "(?ms).*oci runtime error.*starting container process.*container init.*mounting.*to rootfs.*no such file or directory.*", + "(?ms).*grpc: the connection is unavailable.*", } var brokenNodeHook *string = flag.String("broken-node-hook", "", "Script to run if node is detected to be broken (for example, Docker daemon is not running)") +func (runner *ContainerRunner) runBrokenNodeHook() { + if *brokenNodeHook == "" { + runner.CrunchLog.Printf("No broken node hook provided, cannot mark node as broken.") + } else { + runner.CrunchLog.Printf("Running broken node hook %q", *brokenNodeHook) + // run killme script + c := exec.Command(*brokenNodeHook) + c.Stdout = runner.CrunchLog + c.Stderr = runner.CrunchLog + err := c.Run() + if err != nil { + runner.CrunchLog.Printf("Error running broken node hook: %v", err) + } + } +} + func (runner *ContainerRunner) checkBrokenNode(goterr error) bool { for _, d := range errorBlacklist { if m, e := regexp.MatchString(d, goterr.Error()); m && e == nil { runner.CrunchLog.Printf("Error suggests node is unable to run containers: %v", goterr) - if *brokenNodeHook == "" { - runner.CrunchLog.Printf("No broken node hook provided, cannot mark node as broken.") - } else { - runner.CrunchLog.Printf("Running broken node hook %q", *brokenNodeHook) - // run killme script - c := exec.Command(*brokenNodeHook) - c.Stdout = runner.CrunchLog - c.Stderr = runner.CrunchLog - err := c.Run() - if err != nil { - runner.CrunchLog.Printf("Error running broken node hook: %v", err) - } - } + runner.runBrokenNodeHook() return true } } @@ -275,7 +290,11 @@ func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string, token string) ( } c.Env = append(c.Env, "ARVADOS_API_TOKEN="+token) - runner.arvMountLog = NewThrottledLogger(runner.NewLogWriter("arv-mount")) + w, err := runner.NewLogWriter("arv-mount") + if err != nil { + return nil, err + } + runner.arvMountLog = NewThrottledLogger(w) c.Stdout = runner.arvMountLog c.Stderr = runner.arvMountLog @@ -325,7 +344,7 @@ func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string, token string) ( func (runner *ContainerRunner) SetupArvMountPoint(prefix string) (err error) { if runner.ArvMountPoint == "" { - runner.ArvMountPoint, err = runner.MkTempDir("", prefix) + runner.ArvMountPoint, err = runner.MkTempDir(runner.parentTemp, prefix) } return } @@ -336,13 +355,13 @@ func copyfile(src string, dst string) (err error) { return } - os.MkdirAll(path.Dir(dst), 0770) + os.MkdirAll(path.Dir(dst), 0777) dstfile, err := os.Create(dst) if err != nil { return } - _, err = io.Copy(srcfile, dstfile) + _, err = io.Copy(dstfile, srcfile) if err != nil { return } @@ -398,10 +417,24 @@ func (runner *ContainerRunner) SetupMounts() (err error) { for bind := range runner.Container.Mounts { binds = append(binds, bind) } + for bind := range runner.SecretMounts { + if _, ok := runner.Container.Mounts[bind]; ok { + return fmt.Errorf("Secret mount %q conflicts with regular mount", bind) + } + if runner.SecretMounts[bind].Kind != "json" && + runner.SecretMounts[bind].Kind != "text" { + return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.", + bind, runner.SecretMounts[bind].Kind) + } + binds = append(binds, bind) + } sort.Strings(binds) for _, bind := range binds { - mnt := runner.Container.Mounts[bind] + mnt, ok := runner.Container.Mounts[bind] + if !ok { + mnt = runner.SecretMounts[bind] + } if bind == "stdout" || bind == "stderr" { // Is it a "file" mount kind? if mnt.Kind != "file" { @@ -430,8 +463,8 @@ func (runner *ContainerRunner) SetupMounts() (err error) { } if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" { - if mnt.Kind != "collection" { - return fmt.Errorf("Only mount points of kind 'collection' are supported underneath the output_path: %v", bind) + if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" { + return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind) } } @@ -488,7 +521,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { case mnt.Kind == "tmp": var tmpdir string - tmpdir, err = runner.MkTempDir("", "") + tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp") if err != nil { return fmt.Errorf("While creating mount temp dir: %v", err) } @@ -500,40 +533,46 @@ func (runner *ContainerRunner) SetupMounts() (err error) { if staterr != nil { return fmt.Errorf("While Chmod temp dir: %v", err) } - runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir) runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind)) if bind == runner.Container.OutputPath { runner.HostOutputDir = tmpdir } - case mnt.Kind == "json": - jsondata, err := json.Marshal(mnt.Content) - if err != nil { - return fmt.Errorf("encoding json data: %v", err) + case mnt.Kind == "json" || mnt.Kind == "text": + var filedata []byte + if mnt.Kind == "json" { + filedata, err = json.Marshal(mnt.Content) + if err != nil { + return fmt.Errorf("encoding json data: %v", err) + } + } else { + text, ok := mnt.Content.(string) + if !ok { + return fmt.Errorf("content for mount %q must be a string", bind) + } + filedata = []byte(text) } - // Create a tempdir with a single file - // (instead of just a tempfile): this way we - // can ensure the file is world-readable - // inside the container, without having to - // make it world-readable on the docker host. - tmpdir, err := runner.MkTempDir("", "") + + tmpdir, err := runner.MkTempDir(runner.parentTemp, mnt.Kind) if err != nil { return fmt.Errorf("creating temp dir: %v", err) } - runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir) - tmpfn := filepath.Join(tmpdir, "mountdata.json") - err = ioutil.WriteFile(tmpfn, jsondata, 0644) + tmpfn := filepath.Join(tmpdir, "mountdata."+mnt.Kind) + err = ioutil.WriteFile(tmpfn, filedata, 0444) if err != nil { return fmt.Errorf("writing temp file: %v", err) } - runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind)) + if strings.HasPrefix(bind, runner.Container.OutputPath+"/") { + copyFiles = append(copyFiles, copyFile{tmpfn, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]}) + } else { + runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind)) + } case mnt.Kind == "git_tree": - tmpdir, err := runner.MkTempDir("", "") + tmpdir, err := runner.MkTempDir(runner.parentTemp, "git_tree") if err != nil { return fmt.Errorf("creating temp dir: %v", err) } - runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir) err = gitMount(mnt).extractTree(runner.ArvClient, tmpdir, token) if err != nil { return err @@ -576,21 +615,40 @@ func (runner *ContainerRunner) SetupMounts() (err error) { } for _, cp := range copyFiles { - dir, err := os.Stat(cp.src) - if err == nil { - if dir.IsDir() { - err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error { - if walkinfo.Mode().IsRegular() { - return copyfile(walkpath, path.Join(cp.bind, walkpath[len(cp.src):])) + st, err := os.Stat(cp.src) + if err != nil { + return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err) + } + if st.IsDir() { + err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error { + if walkerr != nil { + return walkerr + } + target := path.Join(cp.bind, walkpath[len(cp.src):]) + if walkinfo.Mode().IsRegular() { + copyerr := copyfile(walkpath, target) + if copyerr != nil { + return copyerr } - return nil - }) - } else { - err = copyfile(cp.src, cp.bind) + return os.Chmod(target, walkinfo.Mode()|0777) + } else if walkinfo.Mode().IsDir() { + mkerr := os.MkdirAll(target, 0777) + if mkerr != nil { + return mkerr + } + return os.Chmod(target, walkinfo.Mode()|os.ModeSetgid|0777) + } else { + return fmt.Errorf("Source %q is not a regular file or directory", cp.src) + } + }) + } else if st.Mode().IsRegular() { + err = copyfile(cp.src, cp.bind) + if err == nil { + err = os.Chmod(cp.bind, st.Mode()|0777) } } if err != nil { - return fmt.Errorf("While staging writable files: %v", err) + return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err) } } @@ -600,53 +658,84 @@ func (runner *ContainerRunner) SetupMounts() (err error) { func (runner *ContainerRunner) ProcessDockerAttach(containerReader io.Reader) { // Handle docker log protocol // https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container + defer close(runner.loggingDone) header := make([]byte, 8) - for { - _, readerr := io.ReadAtLeast(containerReader, header, 8) - - if readerr == nil { - readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24) - if header[0] == 1 { - // stdout - _, readerr = io.CopyN(runner.Stdout, containerReader, readsize) - } else { - // stderr - _, readerr = io.CopyN(runner.Stderr, containerReader, readsize) + var err error + for err == nil { + _, err = io.ReadAtLeast(containerReader, header, 8) + if err != nil { + if err == io.EOF { + err = nil } + break } + readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24) + if header[0] == 1 { + // stdout + _, err = io.CopyN(runner.Stdout, containerReader, readsize) + } else { + // stderr + _, err = io.CopyN(runner.Stderr, containerReader, readsize) + } + } - if readerr != nil { - if readerr != io.EOF { - runner.CrunchLog.Printf("While reading docker logs: %v", readerr) - } - - closeerr := runner.Stdout.Close() - if closeerr != nil { - runner.CrunchLog.Printf("While closing stdout logs: %v", closeerr) - } + if err != nil { + runner.CrunchLog.Printf("error reading docker logs: %v", err) + } - closeerr = runner.Stderr.Close() - if closeerr != nil { - runner.CrunchLog.Printf("While closing stderr logs: %v", closeerr) - } + err = runner.Stdout.Close() + if err != nil { + runner.CrunchLog.Printf("error closing stdout logs: %v", err) + } - if runner.statReporter != nil { - runner.statReporter.Stop() - closeerr = runner.statLogger.Close() - if closeerr != nil { - runner.CrunchLog.Printf("While closing crunchstat logs: %v", closeerr) - } - } + err = runner.Stderr.Close() + if err != nil { + runner.CrunchLog.Printf("error closing stderr logs: %v", err) + } - close(runner.loggingDone) - return + if runner.statReporter != nil { + runner.statReporter.Stop() + err = runner.statLogger.Close() + if err != nil { + runner.CrunchLog.Printf("error closing crunchstat logs: %v", err) } } } -func (runner *ContainerRunner) StartCrunchstat() { - runner.statLogger = NewThrottledLogger(runner.NewLogWriter("crunchstat")) +func (runner *ContainerRunner) stopHoststat() error { + if runner.hoststatReporter == nil { + return nil + } + runner.hoststatReporter.Stop() + err := runner.hoststatLogger.Close() + if err != nil { + return fmt.Errorf("error closing hoststat logs: %v", err) + } + return nil +} + +func (runner *ContainerRunner) startHoststat() error { + w, err := runner.NewLogWriter("hoststat") + if err != nil { + return err + } + runner.hoststatLogger = NewThrottledLogger(w) + runner.hoststatReporter = &crunchstat.Reporter{ + Logger: log.New(runner.hoststatLogger, "", 0), + CgroupRoot: runner.cgroupRoot, + PollPeriod: runner.statInterval, + } + runner.hoststatReporter.Start() + return nil +} + +func (runner *ContainerRunner) startCrunchstat() error { + w, err := runner.NewLogWriter("crunchstat") + if err != nil { + return err + } + runner.statLogger = NewThrottledLogger(w) runner.statReporter = &crunchstat.Reporter{ CID: runner.ContainerID, Logger: log.New(runner.statLogger, "", 0), @@ -655,6 +744,7 @@ func (runner *ContainerRunner) StartCrunchstat() { PollPeriod: runner.statInterval, } runner.statReporter.Start() + return nil } type infoCommand struct { @@ -668,7 +758,10 @@ type infoCommand struct { // might differ from what's described in the node record (see // LogNodeRecord). func (runner *ContainerRunner) LogHostInfo() (err error) { - w := runner.NewLogWriter("node-info") + w, err := runner.NewLogWriter("node-info") + if err != nil { + return + } commands := []infoCommand{ { @@ -741,11 +834,15 @@ func (runner *ContainerRunner) LogNodeRecord() error { } func (runner *ContainerRunner) logAPIResponse(label, path string, params map[string]interface{}, munge func(interface{})) (logged bool, err error) { + writer, err := runner.LogCollection.OpenFile(label+".json", os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return false, err + } w := &ArvLogWriter{ ArvClient: runner.ArvClient, UUID: runner.Container.UUID, loggingStream: label, - writeCloser: runner.LogCollection.Open(label + ".json"), + writeCloser: writer, } reader, err := runner.ArvClient.CallRaw("GET", path, "", "", arvadosclient.Dict(params)) @@ -832,8 +929,10 @@ func (runner *ContainerRunner) AttachStreams() (err error) { return err } runner.Stdout = stdoutFile + } else if w, err := runner.NewLogWriter("stdout"); err != nil { + return err } else { - runner.Stdout = NewThrottledLogger(runner.NewLogWriter("stdout")) + runner.Stdout = NewThrottledLogger(w) } if stderrMnt, ok := runner.Container.Mounts["stderr"]; ok { @@ -842,8 +941,10 @@ func (runner *ContainerRunner) AttachStreams() (err error) { return err } runner.Stderr = stderrFile + } else if w, err := runner.NewLogWriter("stderr"); err != nil { + return err } else { - runner.Stderr = NewThrottledLogger(runner.NewLogWriter("stderr")) + runner.Stderr = NewThrottledLogger(w) } if stdinRdr != nil { @@ -851,7 +952,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) { _, err := io.Copy(response.Conn, stdinRdr) if err != nil { runner.CrunchLog.Print("While writing stdin collection to docker container %q", err) - runner.stop() + runner.stop(nil) } stdinRdr.Close() response.CloseWrite() @@ -861,7 +962,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) { _, err := io.Copy(response.Conn, bytes.NewReader(stdinJson)) if err != nil { runner.CrunchLog.Print("While writing stdin json to docker container %q", err) - runner.stop() + runner.stop(nil) } response.CloseWrite() }() @@ -912,6 +1013,11 @@ func (runner *ContainerRunner) CreateContainer() error { runner.ContainerConfig.Volumes = runner.Volumes + maxRAM := int64(runner.Container.RuntimeConstraints.RAM) + if maxRAM < 4*1024*1024 { + // Docker daemon won't let you set a limit less than 4 MiB + maxRAM = 4 * 1024 * 1024 + } runner.HostConfig = dockercontainer.HostConfig{ Binds: runner.Binds, LogConfig: dockercontainer.LogConfig{ @@ -919,6 +1025,10 @@ func (runner *ContainerRunner) CreateContainer() error { }, Resources: dockercontainer.Resources{ CgroupParent: runner.setCgroupParent, + NanoCPUs: int64(runner.Container.RuntimeConstraints.VCPUs) * 1000000000, + Memory: maxRAM, // RAM + MemorySwap: maxRAM, // RAM+swap + KernelMemory: maxRAM, // kernel portion }, } @@ -978,13 +1088,60 @@ func (runner *ContainerRunner) StartContainer() error { return nil } +// checkContainerd checks if "containerd" is present in the process list. +func (runner *ContainerRunner) CheckContainerd() error { + if runner.checkContainerd == 0 { + return nil + } + p, _ := runner.ListProcesses() + for _, i := range p { + e, _ := i.CmdlineSlice() + if len(e) > 0 { + if strings.Index(e[0], "containerd") > -1 { + return nil + } + } + } + + // Not found + runner.runBrokenNodeHook() + runner.stop(nil) + return fmt.Errorf("'containerd' not found in process list.") +} + // WaitFinish waits for the container to terminate, capture the exit code, and // close the stdout/stderr logging. func (runner *ContainerRunner) WaitFinish() error { + var runTimeExceeded <-chan time.Time runner.CrunchLog.Print("Waiting for container to finish") waitOk, waitErr := runner.Docker.ContainerWait(context.TODO(), runner.ContainerID, dockercontainer.WaitConditionNotRunning) arvMountExit := runner.ArvMountExit + if timeout := runner.Container.SchedulingParameters.MaxRunTime; timeout > 0 { + runTimeExceeded = time.After(time.Duration(timeout) * time.Second) + } + + containerdGone := make(chan error) + defer close(containerdGone) + if runner.checkContainerd > 0 { + go func() { + ticker := time.NewTicker(time.Duration(runner.checkContainerd)) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if ck := runner.CheckContainerd(); ck != nil { + containerdGone <- ck + return + } + case <-containerdGone: + // Channel closed, quit goroutine + return + } + } + }() + } + for { select { case waitBody := <-waitOk: @@ -1001,189 +1158,24 @@ func (runner *ContainerRunner) WaitFinish() error { case <-arvMountExit: runner.CrunchLog.Printf("arv-mount exited while container is still running. Stopping container.") - runner.stop() + runner.stop(nil) // arvMountExit will always be ready now that // it's closed, but that doesn't interest us. arvMountExit = nil - } - } -} - -var ErrNotInOutputDir = fmt.Errorf("Must point to path within the output directory") - -func (runner *ContainerRunner) derefOutputSymlink(path string, startinfo os.FileInfo) (tgt string, readlinktgt string, info os.FileInfo, err error) { - // Follow symlinks if necessary - info = startinfo - tgt = path - readlinktgt = "" - nextlink := path - for followed := 0; info.Mode()&os.ModeSymlink != 0; followed++ { - if followed >= limitFollowSymlinks { - // Got stuck in a loop or just a pathological number of links, give up. - err = fmt.Errorf("Followed more than %v symlinks from path %q", limitFollowSymlinks, path) - return - } - - readlinktgt, err = os.Readlink(nextlink) - if err != nil { - return - } - - tgt = readlinktgt - if !strings.HasPrefix(tgt, "/") { - // Relative symlink, resolve it to host path - tgt = filepath.Join(filepath.Dir(path), tgt) - } - if strings.HasPrefix(tgt, runner.Container.OutputPath+"/") && !strings.HasPrefix(tgt, runner.HostOutputDir+"/") { - // Absolute symlink to container output path, adjust it to host output path. - tgt = filepath.Join(runner.HostOutputDir, tgt[len(runner.Container.OutputPath):]) - } - if !strings.HasPrefix(tgt, runner.HostOutputDir+"/") { - // After dereferencing, symlink target must either be - // within output directory, or must point to a - // collection mount. - err = ErrNotInOutputDir - return - } - - info, err = os.Lstat(tgt) - if err != nil { - // tgt - err = fmt.Errorf("Symlink in output %q points to invalid location %q: %v", - path[len(runner.HostOutputDir):], readlinktgt, err) - return - } - - nextlink = tgt - } - return -} + case <-runTimeExceeded: + runner.CrunchLog.Printf("maximum run time exceeded. Stopping container.") + runner.stop(nil) + runTimeExceeded = nil -var limitFollowSymlinks = 10 - -// UploadFile uploads files within the output directory, with special handling -// for symlinks. If the symlink leads to a keep mount, copy the manifest text -// from the keep mount into the output manifestText. Ensure that whether -// symlinks are relative or absolute, every symlink target (even targets that -// are symlinks themselves) must point to a path in either the output directory -// or a collection mount. -// -// Assumes initial value of "path" is absolute, and located within runner.HostOutputDir. -func (runner *ContainerRunner) UploadOutputFile( - path string, - info os.FileInfo, - infoerr error, - binds []string, - walkUpload *WalkUpload, - relocateFrom string, - relocateTo string, - followed int) (manifestText string, err error) { - - if infoerr != nil { - return "", infoerr - } - - if info.Mode().IsDir() { - // if empty, need to create a .keep file - dir, direrr := os.Open(path) - if direrr != nil { - return "", direrr - } - defer dir.Close() - names, eof := dir.Readdirnames(1) - if len(names) == 0 && eof == io.EOF && path != runner.HostOutputDir { - containerPath := runner.OutputPath + path[len(runner.HostOutputDir):] - for _, bind := range binds { - mnt := runner.Container.Mounts[bind] - // Check if there is a bind for this - // directory, in which case assume we don't need .keep - if (containerPath == bind || strings.HasPrefix(containerPath, bind+"/")) && mnt.PortableDataHash != "d41d8cd98f00b204e9800998ecf8427e+0" { - return - } - } - outputSuffix := path[len(runner.HostOutputDir)+1:] - return fmt.Sprintf("./%v d41d8cd98f00b204e9800998ecf8427e+0 0:0:.keep\n", outputSuffix), nil - } - return - } - - if followed >= limitFollowSymlinks { - // Got stuck in a loop or just a pathological number of - // directory links, give up. - err = fmt.Errorf("Followed more than %v symlinks from path %q", limitFollowSymlinks, path) - return - } - - // "path" is the actual path we are visiting - // "tgt" is the target of "path" (a non-symlink) after following symlinks - // "relocated" is the path in the output manifest where the file should be placed, - // but has HostOutputDir as a prefix. - - // The destination path in the output manifest may need to be - // logically relocated to some other path in order to appear - // in the correct location as a result of following a symlink. - // Remove the relocateFrom prefix and replace it with - // relocateTo. - relocated := relocateTo + path[len(relocateFrom):] - - tgt, readlinktgt, info, derefErr := runner.derefOutputSymlink(path, info) - if derefErr != nil && derefErr != ErrNotInOutputDir { - return "", derefErr - } - - // go through mounts and try reverse map to collection reference - for _, bind := range binds { - mnt := runner.Container.Mounts[bind] - if tgt == bind || strings.HasPrefix(tgt, bind+"/") { - // get path relative to bind - targetSuffix := tgt[len(bind):] - - // Copy mount and adjust the path to add path relative to the bind - adjustedMount := mnt - adjustedMount.Path = filepath.Join(adjustedMount.Path, targetSuffix) - - // Terminates in this keep mount, so add the - // manifest text at appropriate location. - outputSuffix := relocated[len(runner.HostOutputDir):] - manifestText, err = runner.getCollectionManifestForPath(adjustedMount, outputSuffix) - return + case err := <-containerdGone: + return err } } - - // If target is not a collection mount, it must be located within the - // output directory, otherwise it is an error. - if derefErr == ErrNotInOutputDir { - err = fmt.Errorf("Symlink in output %q points to invalid location %q, must point to path within the output directory.", - path[len(runner.HostOutputDir):], readlinktgt) - return - } - - if info.Mode().IsRegular() { - return "", walkUpload.UploadFile(relocated, tgt) - } - - if info.Mode().IsDir() { - // Symlink leads to directory. Walk() doesn't follow - // directory symlinks, so we walk the target directory - // instead. Within the walk, file paths are relocated - // so they appear under the original symlink path. - err = filepath.Walk(tgt, func(walkpath string, walkinfo os.FileInfo, walkerr error) error { - var m string - m, walkerr = runner.UploadOutputFile(walkpath, walkinfo, walkerr, - binds, walkUpload, tgt, relocated, followed+1) - if walkerr == nil { - manifestText = manifestText + m - } - return walkerr - }) - return - } - - return } -// HandleOutput sets the output, unmounts the FUSE mount, and deletes temporary directories +// CaptureOutput saves data from the container's output directory if +// needed, and updates the container output accordingly. func (runner *ContainerRunner) CaptureOutput() error { if wantAPI := runner.Container.RuntimeConstraints.API; wantAPI != nil && *wantAPI { // Output may have been set directly by the container, so @@ -1200,153 +1192,36 @@ func (runner *ContainerRunner) CaptureOutput() error { } } - if runner.HostOutputDir == "" { - return nil - } - - _, err := os.Stat(runner.HostOutputDir) + txt, err := (&copier{ + client: runner.client, + arvClient: runner.ArvClient, + keepClient: runner.Kc, + hostOutputDir: runner.HostOutputDir, + ctrOutputDir: runner.Container.OutputPath, + binds: runner.Binds, + mounts: runner.Container.Mounts, + secretMounts: runner.SecretMounts, + logger: runner.CrunchLog, + }).Copy() if err != nil { - return fmt.Errorf("While checking host output path: %v", err) - } - - // Pre-populate output from the configured mount points - var binds []string - for bind, mnt := range runner.Container.Mounts { - if mnt.Kind == "collection" { - binds = append(binds, bind) - } - } - sort.Strings(binds) - - var manifestText string - - collectionMetafile := fmt.Sprintf("%s/.arvados#collection", runner.HostOutputDir) - _, err = os.Stat(collectionMetafile) - if err != nil { - // Regular directory - - cw := CollectionWriter{0, runner.Kc, nil, nil, sync.Mutex{}} - walkUpload := cw.BeginUpload(runner.HostOutputDir, runner.CrunchLog.Logger) - - var m string - err = filepath.Walk(runner.HostOutputDir, func(path string, info os.FileInfo, err error) error { - m, err = runner.UploadOutputFile(path, info, err, binds, walkUpload, "", "", 0) - if err == nil { - manifestText = manifestText + m - } - return err - }) - - cw.EndUpload(walkUpload) - - if err != nil { - return fmt.Errorf("While uploading output files: %v", err) - } - - m, err = cw.ManifestText() - manifestText = manifestText + m - if err != nil { - return fmt.Errorf("While uploading output files: %v", err) - } - } else { - // FUSE mount directory - file, openerr := os.Open(collectionMetafile) - if openerr != nil { - return fmt.Errorf("While opening FUSE metafile: %v", err) - } - defer file.Close() - - var rec arvados.Collection - err = json.NewDecoder(file).Decode(&rec) - if err != nil { - return fmt.Errorf("While reading FUSE metafile: %v", err) - } - manifestText = rec.ManifestText - } - - for _, bind := range binds { - mnt := runner.Container.Mounts[bind] - - bindSuffix := strings.TrimPrefix(bind, runner.Container.OutputPath) - - if bindSuffix == bind || len(bindSuffix) <= 0 { - // either does not start with OutputPath or is OutputPath itself - continue - } - - if mnt.ExcludeFromOutput == true { - continue - } - - // append to manifest_text - m, err := runner.getCollectionManifestForPath(mnt, bindSuffix) - if err != nil { - return err - } - - manifestText = manifestText + m - } - - // Save output - var response arvados.Collection - manifest := manifest.Manifest{Text: manifestText} - manifestText = manifest.Extract(".", ".").Text - err = runner.ArvClient.Create("collections", - arvadosclient.Dict{ - "ensure_unique_name": true, - "collection": arvadosclient.Dict{ - "is_trashed": true, - "name": "output for " + runner.Container.UUID, - "manifest_text": manifestText}}, - &response) + return err + } + var resp arvados.Collection + err = runner.ArvClient.Create("collections", arvadosclient.Dict{ + "ensure_unique_name": true, + "collection": arvadosclient.Dict{ + "is_trashed": true, + "name": "output for " + runner.Container.UUID, + "manifest_text": txt, + }, + }, &resp) if err != nil { - return fmt.Errorf("While creating output collection: %v", err) + return fmt.Errorf("error creating output collection: %v", err) } - runner.OutputPDH = &response.PortableDataHash + runner.OutputPDH = &resp.PortableDataHash return nil } -var outputCollections = make(map[string]arvados.Collection) - -// Fetch the collection for the mnt.PortableDataHash -// Return the manifest_text fragment corresponding to the specified mnt.Path -// after making any required updates. -// Ex: -// If mnt.Path is not specified, -// return the entire manifest_text after replacing any "." with bindSuffix -// If mnt.Path corresponds to one stream, -// return the manifest_text for that stream after replacing that stream name with bindSuffix -// Otherwise, check if a filename in any one stream is being sought. Return the manifest_text -// for that stream after replacing stream name with bindSuffix minus the last word -// and the file name with last word of the bindSuffix -// Allowed path examples: -// "path":"/" -// "path":"/subdir1" -// "path":"/subdir1/subdir2" -// "path":"/subdir/filename" etc -func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, bindSuffix string) (string, error) { - collection := outputCollections[mnt.PortableDataHash] - if collection.PortableDataHash == "" { - err := runner.ArvClient.Get("collections", mnt.PortableDataHash, nil, &collection) - if err != nil { - return "", fmt.Errorf("While getting collection for %v: %v", mnt.PortableDataHash, err) - } - outputCollections[mnt.PortableDataHash] = collection - } - - if collection.ManifestText == "" { - runner.CrunchLog.Printf("No manifest text for collection %v", collection.PortableDataHash) - return "", nil - } - - mft := manifest.Manifest{Text: collection.ManifestText} - extracted := mft.Extract(mnt.Path, bindSuffix) - if extracted.Err != nil { - return "", fmt.Errorf("Error parsing manifest for %v: %v", mnt.PortableDataHash, extracted.Err.Error()) - } - return extracted.Text, nil -} - func (runner *ContainerRunner) CleanupDirs() { if runner.ArvMount != nil { var delay int64 = 8 @@ -1396,29 +1271,37 @@ func (runner *ContainerRunner) CleanupDirs() { } } - for _, tmpdir := range runner.CleanupTempDir { - if rmerr := os.RemoveAll(tmpdir); rmerr != nil { - runner.CrunchLog.Printf("While cleaning up temporary directory %s: %v", tmpdir, rmerr) - } + if rmerr := os.RemoveAll(runner.parentTemp); rmerr != nil { + runner.CrunchLog.Printf("While cleaning up temporary directory %s: %v", runner.parentTemp, rmerr) } } // CommitLogs posts the collection containing the final container logs. func (runner *ContainerRunner) CommitLogs() error { - runner.CrunchLog.Print(runner.finalState) + func() { + // Hold cStateLock to prevent races on CrunchLog (e.g., stop()). + runner.cStateLock.Lock() + defer runner.cStateLock.Unlock() - if runner.arvMountLog != nil { - runner.arvMountLog.Close() - } - runner.CrunchLog.Close() + runner.CrunchLog.Print(runner.finalState) - // Closing CrunchLog above allows them to be committed to Keep at this - // point, but re-open crunch log with ArvClient in case there are any - // other further errors (such as failing to write the log to Keep!) - // while shutting down - runner.CrunchLog = NewThrottledLogger(&ArvLogWriter{ArvClient: runner.ArvClient, - UUID: runner.Container.UUID, loggingStream: "crunch-run", writeCloser: nil}) - runner.CrunchLog.Immediate = log.New(os.Stderr, runner.Container.UUID+" ", 0) + if runner.arvMountLog != nil { + runner.arvMountLog.Close() + } + runner.CrunchLog.Close() + + // Closing CrunchLog above allows them to be committed to Keep at this + // point, but re-open crunch log with ArvClient in case there are any + // other further errors (such as failing to write the log to Keep!) + // while shutting down + runner.CrunchLog = NewThrottledLogger(&ArvLogWriter{ + ArvClient: runner.ArvClient, + UUID: runner.Container.UUID, + loggingStream: "crunch-run", + writeCloser: nil, + }) + runner.CrunchLog.Immediate = log.New(os.Stderr, runner.Container.UUID+" ", 0) + }() if runner.LogsPDH != nil { // If we have already assigned something to LogsPDH, @@ -1429,7 +1312,7 @@ func (runner *ContainerRunner) CommitLogs() error { return nil } - mt, err := runner.LogCollection.ManifestText() + mt, err := runner.LogCollection.MarshalManifest(".") if err != nil { return fmt.Errorf("While creating log manifest: %v", err) } @@ -1504,12 +1387,17 @@ func (runner *ContainerRunner) IsCancelled() bool { } // NewArvLogWriter creates an ArvLogWriter -func (runner *ContainerRunner) NewArvLogWriter(name string) io.WriteCloser { +func (runner *ContainerRunner) NewArvLogWriter(name string) (io.WriteCloser, error) { + writer, err := runner.LogCollection.OpenFile(name+".txt", os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return nil, err + } return &ArvLogWriter{ ArvClient: runner.ArvClient, UUID: runner.Container.UUID, loggingStream: name, - writeCloser: runner.LogCollection.Open(name + ".txt")} + writeCloser: writer, + }, nil } // Run the full container lifecycle. @@ -1527,7 +1415,6 @@ func (runner *ContainerRunner) Run() (err error) { runner.finalState = "Queued" defer func() { - runner.stopSignals() runner.CleanupDirs() runner.CrunchLog.Printf("crunch-run finished") @@ -1569,6 +1456,7 @@ func (runner *ContainerRunner) Run() (err error) { } checkErr(runner.CaptureOutput()) + checkErr(runner.stopHoststat()) checkErr(runner.CommitLogs()) checkErr(runner.UpdateContainerFinal()) }() @@ -1577,9 +1465,17 @@ func (runner *ContainerRunner) Run() (err error) { if err != nil { return } - - // setup signal handling runner.setupSignals() + err = runner.startHoststat() + if err != nil { + return + } + + // Sanity check that containerd is running. + err = runner.CheckContainerd() + if err != nil { + return + } // check for and/or load image err = runner.LoadImage() @@ -1628,7 +1524,10 @@ func (runner *ContainerRunner) Run() (err error) { } runner.finalState = "Cancelled" - runner.StartCrunchstat() + err = runner.startCrunchstat() + if err != nil { + return + } err = runner.StartContainer() if err != nil { @@ -1658,27 +1557,80 @@ func (runner *ContainerRunner) fetchContainerRecord() error { if err != nil { return fmt.Errorf("error decoding container record: %v", err) } + + var sm struct { + SecretMounts map[string]arvados.Mount `json:"secret_mounts"` + } + + containerToken, err := runner.ContainerToken() + if err != nil { + return fmt.Errorf("error getting container token: %v", err) + } + + containerClient, err := runner.MkArvClient(containerToken) + if err != nil { + return fmt.Errorf("error creating container API client: %v", err) + } + + err = containerClient.Call("GET", "containers", runner.Container.UUID, "secret_mounts", nil, &sm) + if err != nil { + if apierr, ok := err.(arvadosclient.APIServerError); !ok || apierr.HttpStatusCode != 404 { + return fmt.Errorf("error fetching secret_mounts: %v", err) + } + // ok && apierr.HttpStatusCode == 404, which means + // secret_mounts isn't supported by this API server. + } + runner.SecretMounts = sm.SecretMounts + return nil } // NewContainerRunner creates a new container runner. -func NewContainerRunner(api IArvadosClient, - kc IKeepClient, - docker ThinDockerClient, - containerUUID string) *ContainerRunner { - - cr := &ContainerRunner{ArvClient: api, Kc: kc, Docker: docker} +func NewContainerRunner(client *arvados.Client, api IArvadosClient, kc IKeepClient, docker ThinDockerClient, containerUUID string) (*ContainerRunner, error) { + cr := &ContainerRunner{ + client: client, + ArvClient: api, + Kc: kc, + Docker: docker, + } cr.NewLogWriter = cr.NewArvLogWriter cr.RunArvMount = cr.ArvMountCmd cr.MkTempDir = ioutil.TempDir - cr.LogCollection = &CollectionWriter{0, kc, nil, nil, sync.Mutex{}} + cr.ListProcesses = func() ([]PsProcess, error) { + pr, err := process.Processes() + if err != nil { + return nil, err + } + ps := make([]PsProcess, len(pr)) + for i, j := range pr { + ps[i] = j + } + return ps, nil + } + cr.MkArvClient = func(token string) (IArvadosClient, error) { + cl, err := arvadosclient.MakeArvadosClient() + if err != nil { + return nil, err + } + cl.ApiToken = token + return cl, nil + } + var err error + cr.LogCollection, err = (&arvados.Collection{}).FileSystem(cr.client, cr.Kc) + if err != nil { + return nil, err + } cr.Container.UUID = containerUUID - cr.CrunchLog = NewThrottledLogger(cr.NewLogWriter("crunch-run")) + w, err := cr.NewLogWriter("crunch-run") + if err != nil { + return nil, err + } + cr.CrunchLog = NewThrottledLogger(w) cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0) loadLogThrottleParams(api) - return cr + return cr, nil } func main() { @@ -1697,6 +1649,7 @@ func main() { `) memprofile := flag.String("memprofile", "", "write memory profile to `file` after running container") getVersion := flag.Bool("version", false, "Print version information and exit.") + checkContainerd := flag.Duration("check-containerd", 60*time.Second, "Periodic check if (docker-)containerd is running (use 0s to disable).") flag.Parse() // Print version information if requested @@ -1730,7 +1683,10 @@ func main() { // minimum version we want to support. docker, dockererr := dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil) - cr := NewContainerRunner(api, kc, docker, containerId) + cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, docker, containerId) + if err != nil { + log.Fatal(err) + } if dockererr != nil { cr.CrunchLog.Printf("%s: %v", containerId, dockererr) cr.checkBrokenNode(dockererr) @@ -1738,11 +1694,18 @@ func main() { os.Exit(1) } + parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerId+".") + if tmperr != nil { + log.Fatalf("%s: %v", containerId, tmperr) + } + + cr.parentTemp = parentTemp cr.statInterval = *statInterval cr.cgroupRoot = *cgroupRoot cr.expectCgroupParent = *cgroupParent cr.enableNetwork = *enableNetwork cr.networkMode = *networkMode + cr.checkContainerd = *checkContainerd if *cgroupParentSubsystem != "" { p := findCgroup(*cgroupParentSubsystem) cr.setCgroupParent = p @@ -1754,15 +1717,15 @@ func main() { if *memprofile != "" { f, err := os.Create(*memprofile) if err != nil { - log.Printf("could not create memory profile: ", err) + log.Printf("could not create memory profile: %s", err) } runtime.GC() // get up-to-date statistics if err := pprof.WriteHeapProfile(f); err != nil { - log.Printf("could not write memory profile: ", err) + log.Printf("could not write memory profile: %s", err) } closeerr := f.Close() if closeerr != nil { - log.Printf("closing memprofile file: ", err) + log.Printf("closing memprofile file: %s", err) } }