14360: Merge branch 'master'
authorTom Clegg <tclegg@veritasgenetics.com>
Thu, 15 Nov 2018 21:24:48 +0000 (16:24 -0500)
committerTom Clegg <tclegg@veritasgenetics.com>
Thu, 15 Nov 2018 21:24:48 +0000 (16:24 -0500)
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

1  2 
build/run-tests.sh
sdk/go/arvados/container.go
services/crunch-run/crunchrun.go
vendor/vendor.json

diff --combined build/run-tests.sh
index 1b42ea21c4cc0ae7ae44c84ad2cb8ecc52325c77,cd44347cbabed54d1d19dc8373383160fde7dc1a..cb44372566f8eb36ec6163f9108c97e464d6fca8
@@@ -77,10 -77,6 +77,10 @@@ lib/cm
  lib/controller
  lib/crunchstat
  lib/dispatchcloud
 +lib/dispatchcloud/container
 +lib/dispatchcloud/scheduler
 +lib/dispatchcloud/ssh_executor
 +lib/dispatchcloud/worker
  services/api
  services/arv-git-httpd
  services/crunchstat
@@@ -693,7 -689,7 +693,7 @@@ do_test_once() 
          # before trying "go test". Otherwise, coverage-reporting
          # mode makes Go show the wrong line numbers when reporting
          # compilation errors.
-         go get -t "git.curoverse.com/arvados.git/$1" && \
+         go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
              cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
              [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
              if [[ -n "${testargs[$1]}" ]]
@@@ -761,7 -757,7 +761,7 @@@ do_install_once() 
      timer_reset
      if [[ "$2" == "go" ]]
      then
-         go get -t "git.curoverse.com/arvados.git/$1"
+         go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
      elif [[ "$2" == "pip" ]]
      then
          # $3 can name a path directory for us to use, including trailing
@@@ -930,10 -926,6 +930,10 @@@ gostuff=
      lib/controller
      lib/crunchstat
      lib/dispatchcloud
 +    lib/dispatchcloud/container
 +    lib/dispatchcloud/scheduler
 +    lib/dispatchcloud/ssh_executor
 +    lib/dispatchcloud/worker
      sdk/go/arvados
      sdk/go/arvadosclient
      sdk/go/auth
index def4e33cbb5f771cfbe6edbcec0d9704bab518af,b70b4ac917672f363096a810cd35e3689f5132f9..02a0d76decbad272baee737282b5087a72a33c60
@@@ -18,11 -18,10 +18,11 @@@ type Container struct 
        Mounts               map[string]Mount     `json:"mounts"`
        Output               string               `json:"output"`
        OutputPath           string               `json:"output_path"`
 -      Priority             int                  `json:"priority"`
 +      Priority             int64                `json:"priority"`
        RuntimeConstraints   RuntimeConstraints   `json:"runtime_constraints"`
        State                ContainerState       `json:"state"`
        SchedulingParameters SchedulingParameters `json:"scheduling_parameters"`
 +      ExitCode             int                  `json:"exit_code"`
  }
  
  // Container is an arvados#container resource.
@@@ -57,6 -56,7 +57,7 @@@ type ContainerRequest struct 
        UseExisting             bool                   `json:"use_existing"`
        LogUUID                 string                 `json:"log_uuid"`
        OutputUUID              string                 `json:"output_uuid"`
+       RuntimeToken            string                 `json:"runtime_token"`
  }
  
  // Mount is special behavior to attach to a filesystem path or device.
index 27fb8367f50e05764d45c77dc362497f7f702847,e5a1b94706c781cdc62e805e972c5f46ac0bba24..1c6c58009fac71bd5fa5015a4922821937e61d5a
@@@ -32,7 -32,6 +32,6 @@@ import 
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/keepclient"
        "git.curoverse.com/arvados.git/sdk/go/manifest"
-       "github.com/shirou/gopsutil/process"
        "golang.org/x/net/context"
  
        dockertypes "github.com/docker/docker/api/types"
@@@ -61,6 -60,7 +60,7 @@@ type IKeepClient interface 
        PutB(buf []byte) (string, int, error)
        ReadAt(locator string, p []byte, off int) (int, error)
        ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error)
+       LocalLocator(locator string) (string, error)
        ClearBlockCache()
  }
  
@@@ -79,6 -79,7 +79,7 @@@ type ThinDockerClient interface 
        ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error
        ContainerRemove(ctx context.Context, container string, options dockertypes.ContainerRemoveOptions) error
        ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error)
+       ContainerInspect(ctx context.Context, id string) (dockertypes.ContainerJSON, error)
        ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error)
        ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error)
        ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error)
@@@ -91,10 -92,30 +92,30 @@@ type PsProcess interface 
  // ContainerRunner is the main stateful struct used for a single execution of a
  // container.
  type ContainerRunner struct {
-       Docker          ThinDockerClient
-       client          *arvados.Client
-       ArvClient       IArvadosClient
-       Kc              IKeepClient
+       Docker ThinDockerClient
+       // Dispatcher client is initialized with the Dispatcher token.
+       // This is a priviledged token used to manage container status
+       // and logs.
+       //
+       // We have both dispatcherClient and DispatcherArvClient
+       // because there are two different incompatible Arvados Go
+       // SDKs and we have to use both (hopefully this gets fixed in
+       // #14467)
+       dispatcherClient     *arvados.Client
+       DispatcherArvClient  IArvadosClient
+       DispatcherKeepClient IKeepClient
+       // Container client is initialized with the Container token
+       // This token controls the permissions of the container, and
+       // must be used for operations such as reading collections.
+       //
+       // Same comment as above applies to
+       // containerClient/ContainerArvClient.
+       containerClient     *arvados.Client
+       ContainerArvClient  IArvadosClient
+       ContainerKeepClient IKeepClient
        Container       arvados.Container
        ContainerConfig dockercontainer.Config
        HostConfig      dockercontainer.HostConfig
        SigChan         chan os.Signal
        ArvMountExit    chan error
        SecretMounts    map[string]arvados.Mount
-       MkArvClient     func(token string) (IArvadosClient, error)
+       MkArvClient     func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error)
        finalState      string
        parentTemp      string
  
-       ListProcesses func() ([]PsProcess, error)
        statLogger       io.WriteCloser
        statReporter     *crunchstat.Reporter
        hoststatLogger   io.WriteCloser
  
        cStateLock sync.Mutex
        cCancelled bool // StopContainer() invoked
+       cRemoved   bool // docker confirmed the container no longer exists
+       enableNetwork string // one of "default" or "always"
+       networkMode   string // passed through to HostConfig.NetworkMode
+       arvMountLog   *ThrottledLogger
  
-       enableNetwork   string // one of "default" or "always"
-       networkMode     string // passed through to HostConfig.NetworkMode
-       arvMountLog     *ThrottledLogger
-       checkContainerd time.Duration
+       containerWatchdogInterval time.Duration
  }
  
  // setupSignals sets up signal handling to gracefully terminate the underlying
@@@ -187,6 -208,9 +208,9 @@@ func (runner *ContainerRunner) stop(si
        if err != nil {
                runner.CrunchLog.Printf("error removing container: %s", err)
        }
+       if err == nil || strings.Contains(err.Error(), "No such container: "+runner.ContainerID) {
+               runner.cRemoved = true
+       }
  }
  
  var errorBlacklist = []string{
@@@ -231,7 -255,7 +255,7 @@@ func (runner *ContainerRunner) LoadImag
        runner.CrunchLog.Printf("Fetching Docker image from collection '%s'", runner.Container.ContainerImage)
  
        var collection arvados.Collection
-       err = runner.ArvClient.Get("collections", runner.Container.ContainerImage, nil, &collection)
+       err = runner.ContainerArvClient.Get("collections", runner.Container.ContainerImage, nil, &collection)
        if err != nil {
                return fmt.Errorf("While getting container image collection: %v", err)
        }
                runner.CrunchLog.Print("Loading Docker image from keep")
  
                var readCloser io.ReadCloser
-               readCloser, err = runner.Kc.ManifestFileReader(manifest, img)
+               readCloser, err = runner.ContainerKeepClient.ManifestFileReader(manifest, img)
                if err != nil {
                        return fmt.Errorf("While creating ManifestFileReader for container image: %v", err)
                }
  
        runner.ContainerConfig.Image = imageID
  
-       runner.Kc.ClearBlockCache()
+       runner.ContainerKeepClient.ClearBlockCache()
  
        return nil
  }
@@@ -575,7 -599,7 +599,7 @@@ func (runner *ContainerRunner) SetupMou
                        if err != nil {
                                return fmt.Errorf("creating temp dir: %v", err)
                        }
-                       err = gitMount(mnt).extractTree(runner.ArvClient, tmpdir, token)
+                       err = gitMount(mnt).extractTree(runner.ContainerArvClient, tmpdir, token)
                        if err != nil {
                                return err
                        }
@@@ -842,13 -866,13 +866,13 @@@ func (runner *ContainerRunner) logAPIRe
                return false, err
        }
        w := &ArvLogWriter{
-               ArvClient:     runner.ArvClient,
+               ArvClient:     runner.DispatcherArvClient,
                UUID:          runner.Container.UUID,
                loggingStream: label,
                writeCloser:   writer,
        }
  
-       reader, err := runner.ArvClient.CallRaw("GET", path, "", "", arvadosclient.Dict(params))
+       reader, err := runner.DispatcherArvClient.CallRaw("GET", path, "", "", arvadosclient.Dict(params))
        if err != nil {
                return false, fmt.Errorf("error getting %s record: %v", label, err)
        }
@@@ -898,12 -922,14 +922,14 @@@ func (runner *ContainerRunner) AttachSt
                        if collId == "" {
                                collId = stdinMnt.PortableDataHash
                        }
-                       err = runner.ArvClient.Get("collections", collId, nil, &stdinColl)
+                       err = runner.ContainerArvClient.Get("collections", collId, nil, &stdinColl)
                        if err != nil {
-                               return fmt.Errorf("While getting stding collection: %v", err)
+                               return fmt.Errorf("While getting stdin collection: %v", err)
                        }
  
-                       stdinRdr, err = runner.Kc.ManifestFileReader(manifest.Manifest{Text: stdinColl.ManifestText}, stdinMnt.Path)
+                       stdinRdr, err = runner.ContainerKeepClient.ManifestFileReader(
+                               manifest.Manifest{Text: stdinColl.ManifestText},
+                               stdinMnt.Path)
                        if os.IsNotExist(err) {
                                return fmt.Errorf("stdin collection path not found: %v", stdinMnt.Path)
                        } else if err != nil {
@@@ -1091,27 -1117,6 +1117,6 @@@ func (runner *ContainerRunner) StartCon
        return nil
  }
  
- // checkContainerd checks if "containerd" is present in the process list.
- func (runner *ContainerRunner) CheckContainerd() error {
-       if runner.checkContainerd == 0 {
-               return nil
-       }
-       p, _ := runner.ListProcesses()
-       for _, i := range p {
-               e, _ := i.CmdlineSlice()
-               if len(e) > 0 {
-                       if strings.Index(e[0], "containerd") > -1 {
-                               return nil
-                       }
-               }
-       }
-       // Not found
-       runner.runBrokenNodeHook()
-       runner.stop(nil)
-       return fmt.Errorf("'containerd' not found in process list.")
- }
  // WaitFinish waits for the container to terminate, capture the exit code, and
  // close the stdout/stderr logging.
  func (runner *ContainerRunner) WaitFinish() error {
                runTimeExceeded = time.After(time.Duration(timeout) * time.Second)
        }
  
-       containerdGone := make(chan error)
-       defer close(containerdGone)
-       if runner.checkContainerd > 0 {
-               go func() {
-                       ticker := time.NewTicker(time.Duration(runner.checkContainerd))
-                       defer ticker.Stop()
-                       for {
-                               select {
-                               case <-ticker.C:
-                                       if ck := runner.CheckContainerd(); ck != nil {
-                                               containerdGone <- ck
-                                               return
-                                       }
-                               case <-containerdGone:
-                                       // Channel closed, quit goroutine
-                                       return
-                               }
+       containerGone := make(chan struct{})
+       go func() {
+               defer close(containerGone)
+               if runner.containerWatchdogInterval < 1 {
+                       runner.containerWatchdogInterval = time.Minute
+               }
+               for range time.NewTicker(runner.containerWatchdogInterval).C {
+                       ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(runner.containerWatchdogInterval))
+                       ctr, err := runner.Docker.ContainerInspect(ctx, runner.ContainerID)
+                       cancel()
+                       runner.cStateLock.Lock()
+                       done := runner.cRemoved || runner.ExitCode != nil
+                       runner.cStateLock.Unlock()
+                       if done {
+                               return
+                       } else if err != nil {
+                               runner.CrunchLog.Printf("Error inspecting container: %s", err)
+                               runner.checkBrokenNode(err)
+                               return
+                       } else if ctr.State == nil || !(ctr.State.Running || ctr.State.Status == "created") {
+                               runner.CrunchLog.Printf("Container is not running: State=%v", ctr.State)
+                               return
                        }
-               }()
-       }
+               }
+       }()
  
        for {
                select {
                        runner.stop(nil)
                        runTimeExceeded = nil
  
-               case err := <-containerdGone:
-                       return err
+               case <-containerGone:
+                       return errors.New("docker client never returned status")
                }
        }
  }
@@@ -1213,7 -1223,7 +1223,7 @@@ func (runner *ContainerRunner) updateLo
                }
  
                var updated arvados.Container
-               err = runner.ArvClient.Update("containers", runner.Container.UUID, arvadosclient.Dict{
+               err = runner.DispatcherArvClient.Update("containers", runner.Container.UUID, arvadosclient.Dict{
                        "container": arvadosclient.Dict{"log": saved.PortableDataHash},
                }, &updated)
                if err != nil {
@@@ -1231,7 -1241,7 +1241,7 @@@ func (runner *ContainerRunner) CaptureO
        if wantAPI := runner.Container.RuntimeConstraints.API; wantAPI != nil && *wantAPI {
                // Output may have been set directly by the container, so
                // refresh the container record to check.
-               err := runner.ArvClient.Get("containers", runner.Container.UUID,
+               err := runner.DispatcherArvClient.Get("containers", runner.Container.UUID,
                        nil, &runner.Container)
                if err != nil {
                        return err
        }
  
        txt, err := (&copier{
-               client:        runner.client,
-               arvClient:     runner.ArvClient,
-               keepClient:    runner.Kc,
+               client:        runner.containerClient,
+               arvClient:     runner.ContainerArvClient,
+               keepClient:    runner.ContainerKeepClient,
                hostOutputDir: runner.HostOutputDir,
                ctrOutputDir:  runner.Container.OutputPath,
                binds:         runner.Binds,
        if err != nil {
                return err
        }
+       if n := len(regexp.MustCompile(` [0-9a-f]+\+\S*\+R`).FindAllStringIndex(txt, -1)); n > 0 {
+               runner.CrunchLog.Printf("Copying %d data blocks from remote input collections...", n)
+               fs, err := (&arvados.Collection{ManifestText: txt}).FileSystem(runner.containerClient, runner.ContainerKeepClient)
+               if err != nil {
+                       return err
+               }
+               txt, err = fs.MarshalManifest(".")
+               if err != nil {
+                       return err
+               }
+       }
        var resp arvados.Collection
-       err = runner.ArvClient.Create("collections", arvadosclient.Dict{
+       err = runner.ContainerArvClient.Create("collections", arvadosclient.Dict{
                "ensure_unique_name": true,
                "collection": arvadosclient.Dict{
                        "is_trashed":    true,
@@@ -1346,7 -1367,7 +1367,7 @@@ func (runner *ContainerRunner) CommitLo
                // other further errors (such as failing to write the log to Keep!)
                // while shutting down
                runner.CrunchLog = NewThrottledLogger(&ArvLogWriter{
-                       ArvClient:     runner.ArvClient,
+                       ArvClient:     runner.DispatcherArvClient,
                        UUID:          runner.Container.UUID,
                        loggingStream: "crunch-run",
                        writeCloser:   nil,
@@@ -1398,9 -1419,9 +1419,9 @@@ func (runner *ContainerRunner) saveLogC
        reqBody := arvadosclient.Dict{"collection": updates}
        if runner.logUUID == "" {
                reqBody["ensure_unique_name"] = true
-               err = runner.ArvClient.Create("collections", reqBody, &response)
+               err = runner.DispatcherArvClient.Create("collections", reqBody, &response)
        } else {
-               err = runner.ArvClient.Update("collections", runner.logUUID, reqBody, &response)
+               err = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
        }
        if err != nil {
                return
@@@ -1416,7 -1437,7 +1437,7 @@@ func (runner *ContainerRunner) UpdateCo
        if runner.cCancelled {
                return ErrCancelled
        }
-       return runner.ArvClient.Update("containers", runner.Container.UUID,
+       return runner.DispatcherArvClient.Update("containers", runner.Container.UUID,
                arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running"}}, nil)
  }
  
@@@ -1428,7 -1449,7 +1449,7 @@@ func (runner *ContainerRunner) Containe
        }
  
        var auth arvados.APIClientAuthorization
-       err := runner.ArvClient.Call("GET", "containers", runner.Container.UUID, "auth", nil, &auth)
+       err := runner.DispatcherArvClient.Call("GET", "containers", runner.Container.UUID, "auth", nil, &auth)
        if err != nil {
                return "", err
        }
@@@ -1452,7 -1473,7 +1473,7 @@@ func (runner *ContainerRunner) UpdateCo
                        update["output"] = *runner.OutputPDH
                }
        }
-       return runner.ArvClient.Update("containers", runner.Container.UUID, arvadosclient.Dict{"container": update}, nil)
+       return runner.DispatcherArvClient.Update("containers", runner.Container.UUID, arvadosclient.Dict{"container": update}, nil)
  }
  
  // IsCancelled returns the value of Cancelled, with goroutine safety.
@@@ -1469,7 -1490,7 +1490,7 @@@ func (runner *ContainerRunner) NewArvLo
                return nil, err
        }
        return &ArvLogWriter{
-               ArvClient:     runner.ArvClient,
+               ArvClient:     runner.DispatcherArvClient,
                UUID:          runner.Container.UUID,
                loggingStream: name,
                writeCloser:   writer,
@@@ -1547,12 -1568,6 +1568,6 @@@ func (runner *ContainerRunner) Run() (e
                return
        }
  
-       // Sanity check that containerd is running.
-       err = runner.CheckContainerd()
-       if err != nil {
-               return
-       }
        // check for and/or load image
        err = runner.LoadImage()
        if err != nil {
  // Fetch the current container record (uuid = runner.Container.UUID)
  // into runner.Container.
  func (runner *ContainerRunner) fetchContainerRecord() error {
-       reader, err := runner.ArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil)
+       reader, err := runner.DispatcherArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil)
        if err != nil {
                return fmt.Errorf("error fetching container record: %v", err)
        }
                return fmt.Errorf("error getting container token: %v", err)
        }
  
-       containerClient, err := runner.MkArvClient(containerToken)
+       runner.ContainerArvClient, runner.ContainerKeepClient,
+               runner.containerClient, err = runner.MkArvClient(containerToken)
        if err != nil {
                return fmt.Errorf("error creating container API client: %v", err)
        }
  
-       err = containerClient.Call("GET", "containers", runner.Container.UUID, "secret_mounts", nil, &sm)
+       err = runner.ContainerArvClient.Call("GET", "containers", runner.Container.UUID, "secret_mounts", nil, &sm)
        if err != nil {
                if apierr, ok := err.(arvadosclient.APIServerError); !ok || apierr.HttpStatusCode != 404 {
                        return fmt.Errorf("error fetching secret_mounts: %v", err)
  }
  
  // NewContainerRunner creates a new container runner.
- func NewContainerRunner(client *arvados.Client, api IArvadosClient, kc IKeepClient, docker ThinDockerClient, containerUUID string) (*ContainerRunner, error) {
+ func NewContainerRunner(dispatcherClient *arvados.Client,
+       dispatcherArvClient IArvadosClient,
+       dispatcherKeepClient IKeepClient,
+       docker ThinDockerClient,
+       containerUUID string) (*ContainerRunner, error) {
        cr := &ContainerRunner{
-               client:    client,
-               ArvClient: api,
-               Kc:        kc,
-               Docker:    docker,
+               dispatcherClient:     dispatcherClient,
+               DispatcherArvClient:  dispatcherArvClient,
+               DispatcherKeepClient: dispatcherKeepClient,
+               Docker:               docker,
        }
        cr.NewLogWriter = cr.NewArvLogWriter
        cr.RunArvMount = cr.ArvMountCmd
        cr.MkTempDir = ioutil.TempDir
-       cr.ListProcesses = func() ([]PsProcess, error) {
-               pr, err := process.Processes()
-               if err != nil {
-                       return nil, err
-               }
-               ps := make([]PsProcess, len(pr))
-               for i, j := range pr {
-                       ps[i] = j
-               }
-               return ps, nil
-       }
-       cr.MkArvClient = func(token string) (IArvadosClient, error) {
+       cr.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
                cl, err := arvadosclient.MakeArvadosClient()
                if err != nil {
-                       return nil, err
+                       return nil, nil, nil, err
                }
                cl.ApiToken = token
-               return cl, nil
+               kc, err := keepclient.MakeKeepClient(cl)
+               if err != nil {
+                       return nil, nil, nil, err
+               }
+               c2 := arvados.NewClientFromEnv()
+               c2.AuthToken = token
+               return cl, kc, c2, nil
        }
        var err error
-       cr.LogCollection, err = (&arvados.Collection{}).FileSystem(cr.client, cr.Kc)
+       cr.LogCollection, err = (&arvados.Collection{}).FileSystem(cr.dispatcherClient, cr.DispatcherKeepClient)
        if err != nil {
                return nil, err
        }
        cr.CrunchLog = NewThrottledLogger(w)
        cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0)
  
-       loadLogThrottleParams(api)
+       loadLogThrottleParams(dispatcherArvClient)
        go cr.updateLogs()
  
        return cr, nil
@@@ -1716,10 -1732,6 +1732,10 @@@ func main() 
        cgroupParent := flag.String("cgroup-parent", "docker", "name of container's parent cgroup (ignored if -cgroup-parent-subsystem is used)")
        cgroupParentSubsystem := flag.String("cgroup-parent-subsystem", "", "use current cgroup for given subsystem as parent cgroup for container")
        caCertsPath := flag.String("ca-certs", "", "Path to TLS root certificates")
 +      detach := flag.Bool("detach", false, "Detach from parent process and run in the background")
 +      sleep := flag.Duration("sleep", 0, "Delay before starting (testing use only)")
 +      kill := flag.Int("kill", -1, "Send signal to an existing crunch-run process for given UUID")
 +      list := flag.Bool("list", false, "List UUIDs of existing crunch-run processes")
        enableNetwork := flag.String("container-enable-networking", "default",
                `Specify if networking should be enabled for container.  One of 'default', 'always':
        default: only enable networking if container requests it.
        `)
        memprofile := flag.String("memprofile", "", "write memory profile to `file` after running container")
        getVersion := flag.Bool("version", false, "Print version information and exit.")
-       checkContainerd := flag.Duration("check-containerd", 60*time.Second, "Periodic check if (docker-)containerd is running (use 0s to disable).")
+       flag.Duration("check-containerd", 0, "Ignored. Exists for compatibility with older versions.")
 +
 +      detached := false
 +      if len(os.Args) > 1 && os.Args[1] == "-detached" {
 +              // This process was invoked by a parent process, which
 +              // has passed along its own arguments, including
 +              // -detach, after the leading -detached flag.  Strip
 +              // the leading -detached flag (it's not recognized by
 +              // flag.Parse()) ... and remember not to detach all
 +              // over again in this process.
 +              os.Args = append([]string{os.Args[0]}, os.Args[2:]...)
 +              detached = true
 +      }
++
        flag.Parse()
  
 +      switch {
 +      case *detach && !detached:
 +              os.Exit(Detach(flag.Arg(0), os.Args, os.Stdout, os.Stderr))
 +      case *kill >= 0:
 +              os.Exit(KillProcess(flag.Arg(0), syscall.Signal(*kill), os.Stdout, os.Stderr))
 +      case *list:
 +              os.Exit(ListProcesses(os.Stdout, os.Stderr))
 +      }
 +
        // Print version information if requested
        if *getVersion {
                fmt.Printf("crunch-run %s\n", version)
        }
  
        log.Printf("crunch-run %s started", version)
 +      time.Sleep(*sleep)
  
        containerId := flag.Arg(0)
  
        cr.expectCgroupParent = *cgroupParent
        cr.enableNetwork = *enableNetwork
        cr.networkMode = *networkMode
-       cr.checkContainerd = *checkContainerd
        if *cgroupParentSubsystem != "" {
                p := findCgroup(*cgroupParentSubsystem)
                cr.setCgroupParent = p
diff --combined vendor/vendor.json
index aee25beabdf98e6ccd86bbb816e49ec19d41f781,c0e6fee5dbe63ed053c70fcc1ac0875f13813381..ec296d21d1c25a92c14639c36658c4b0aa10bce5
@@@ -3,24 -3,24 +3,24 @@@
        "ignore": "test",
        "package": [
                {
-                       "checksumSHA1": "jf7K+UTQNIzRdlG5F4zX/8b++/E=",
+                       "checksumSHA1": "j4je0EzPGzjb6INLY1BHZ+hyMjc=",
                        "origin": "github.com/curoverse/goamz/aws",
                        "path": "github.com/AdRoll/goamz/aws",
-                       "revision": "21e563311c2dc5ac53464a2c31cb91fb833c6cb9",
+                       "revision": "888b4804f2653cd35ebcc95f046079e63b5b2799",
                        "revisionTime": "2017-07-27T13:52:37Z"
                },
                {
-                       "checksumSHA1": "9nUwQXI+pNxZo6bnR7NslpMpfPI=",
+                       "checksumSHA1": "0+n3cT6e7sQCCbBAH8zg6neiHTk=",
                        "origin": "github.com/curoverse/goamz/s3",
                        "path": "github.com/AdRoll/goamz/s3",
-                       "revision": "21e563311c2dc5ac53464a2c31cb91fb833c6cb9",
+                       "revision": "888b4804f2653cd35ebcc95f046079e63b5b2799",
                        "revisionTime": "2017-07-27T13:52:37Z"
                },
                {
                        "checksumSHA1": "tvxbsTkdjB0C/uxEglqD6JfVnMg=",
                        "origin": "github.com/curoverse/goamz/s3/s3test",
                        "path": "github.com/AdRoll/goamz/s3/s3test",
-                       "revision": "21e563311c2dc5ac53464a2c31cb91fb833c6cb9",
+                       "revision": "888b4804f2653cd35ebcc95f046079e63b5b2799",
                        "revisionTime": "2017-07-27T13:52:37Z"
                },
                {
                        "revision": "d682213848ed68c0a260ca37d6dd5ace8423f5ba",
                        "revisionTime": "2017-12-05T20:32:29Z"
                },
-               {
-                       "checksumSHA1": "st4vb0GmDeoKbsfxdpNZ2MPl76M=",
-                       "path": "github.com/StackExchange/wmi",
-                       "revision": "cdffdb33acae0e14efff2628f9bae377b597840e",
-                       "revisionTime": "2018-04-12T20:51:11Z"
-               },
                {
                        "checksumSHA1": "spyv5/YFBjYyZLZa1U2LBfDR8PM=",
                        "path": "github.com/beorn7/perks/quantile",
                        "revision": "0ca9ea5df5451ffdf184b4428c902747c2c11cd7",
                        "revisionTime": "2017-03-27T23:54:44Z"
                },
-               {
-                       "checksumSHA1": "Kqv7bA4oJG0nPwQvGWDwGGaKONo=",
-                       "path": "github.com/go-ole/go-ole",
-                       "revision": "7a0fa49edf48165190530c675167e2f319a05268",
-                       "revisionTime": "2018-06-25T08:58:08Z"
-               },
-               {
-                       "checksumSHA1": "PArleDBtadu2qO4hJwHR8a3IOTA=",
-                       "path": "github.com/go-ole/go-ole/oleutil",
-                       "revision": "7a0fa49edf48165190530c675167e2f319a05268",
-                       "revisionTime": "2018-06-25T08:58:08Z"
-               },
                {
                        "checksumSHA1": "8UEp6v0Dczw/SlasE0DivB0mAHA=",
                        "path": "github.com/gogo/protobuf/jsonpb",
                        "revision": "d14ea06fba99483203c19d92cfcd13ebe73135f4",
                        "revisionTime": "2015-07-11T00:45:18Z"
                },
+               {
+                       "checksumSHA1": "khL6oKjx81rAZKW+36050b7f5As=",
+                       "path": "github.com/jmcvetta/randutil",
+                       "revision": "2bb1b664bcff821e02b2a0644cd29c7e824d54f8",
+                       "revisionTime": "2015-08-17T12:26:01Z"
+               },
                {
                        "checksumSHA1": "oX6jFQD74oOApvDIhOzW2dXpg5Q=",
                        "path": "github.com/kevinburke/ssh_config",
                        "revision": "b8bc1bf767474819792c23f32d8286a45736f1c6",
                        "revisionTime": "2016-12-03T19:45:07Z"
                },
 +              {
 +                      "checksumSHA1": "ewGq4nGalpCQOHcmBTdAEQx1wW0=",
 +                      "path": "github.com/mitchellh/mapstructure",
 +                      "revision": "bb74f1db0675b241733089d5a1faa5dd8b0ef57b",
 +                      "revisionTime": "2018-05-11T14:21:26Z"
 +              },
                {
                        "checksumSHA1": "OFNit1Qx2DdWhotfREKodDNUwCM=",
                        "path": "github.com/opencontainers/go-digest",
                        "revision": "1744e2970ca51c86172c8190fadad617561ed6e7",
                        "revisionTime": "2017-11-10T11:01:46Z"
                },
-               {
-                       "checksumSHA1": "q14d3C3xvWevU3dSv4P5K0+OSD0=",
-                       "path": "github.com/shirou/gopsutil/cpu",
-                       "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
-                       "revisionTime": "2018-07-05T13:28:12Z"
-               },
-               {
-                       "checksumSHA1": "LZ9GloiGLTISmQ4dalK2XspH6Wo=",
-                       "path": "github.com/shirou/gopsutil/host",
-                       "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
-                       "revisionTime": "2018-07-05T13:28:12Z"
-               },
-               {
-                       "checksumSHA1": "cyoqI0gryzjxGTkaAfyUqMiuUR0=",
-                       "path": "github.com/shirou/gopsutil/internal/common",
-                       "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
-                       "revisionTime": "2018-07-05T13:28:12Z"
-               },
-               {
-                       "checksumSHA1": "vEQLjAO5T5K9zXblEMYdoaBZzj0=",
-                       "path": "github.com/shirou/gopsutil/mem",
-                       "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
-                       "revisionTime": "2018-07-05T13:28:12Z"
-               },
-               {
-                       "checksumSHA1": "KMWFRa0DVpabo9d8euB4RYjUBQE=",
-                       "path": "github.com/shirou/gopsutil/net",
-                       "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
-                       "revisionTime": "2018-07-05T13:28:12Z"
-               },
-               {
-                       "checksumSHA1": "fbO7c1gv1kSvWKOb/+5HUWFkBaA=",
-                       "path": "github.com/shirou/gopsutil/process",
-                       "revision": "63728fcf6b24475ecfea044e22242447666c2f52",
-                       "revisionTime": "2018-07-05T13:28:12Z"
-               },
-               {
-                       "checksumSHA1": "Nve7SpDmjsv6+rhkXAkfg/UQx94=",
-                       "path": "github.com/shirou/w32",
-                       "revision": "bb4de0191aa41b5507caa14b0650cdbddcd9280b",
-                       "revisionTime": "2016-09-30T03:27:40Z"
-               },
                {
                        "checksumSHA1": "8QeSG127zQqbA+YfkO1WkKx/iUI=",
                        "path": "github.com/src-d/gcfg",