20319: Move /containers/*/log to /container_requests/*/log.

[arvados.git] / lib / crunchrun / docker.go

diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go
index eee8f1d76a7c4dd86bcf63e8958b38042a432139..8d8cdfc8ba620a4317d4f48bf0f654c04cf58bef 100644 (file)
--- a/lib/crunchrun/docker.go
+++ b/lib/crunchrun/docker.go
@@ -4,6 +4,7 @@
 package crunchrun
 
 import (
+       "context"
        "fmt"
        "io"
        "io/ioutil"
@@ -17,12 +18,24 @@ import (
        dockertypes "github.com/docker/docker/api/types"
        dockercontainer "github.com/docker/docker/api/types/container"
        dockerclient "github.com/docker/docker/client"
-       "golang.org/x/net/context"
 )
 
 // Docker daemon won't let you set a limit less than ~10 MiB
 const minDockerRAM = int64(16 * 1024 * 1024)
 
+// DockerAPIVersion is the API version we use to communicate with the
+// docker service.  The oldest OS we support is Ubuntu 18.04 (bionic)
+// which originally shipped docker 1.17.12 / API 1.35 so there is no
+// reason to use an older API version.  See
+// https://dev.arvados.org/issues/15370#note-38 and
+// https://docs.docker.com/engine/api/.
+const DockerAPIVersion = "1.35"
+
+// Number of consecutive "inspect container" failures before
+// concluding Docker is unresponsive, giving up, and cancelling the
+// container.
+const dockerWatchdogThreshold = 3
+
 type dockerExecutor struct {
        containerUUID    string
        logf             func(string, ...interface{})
@@ -37,7 +50,7 @@ type dockerExecutor struct {
 func newDockerExecutor(containerUUID string, logf func(string, ...interface{}), watchdogInterval time.Duration) (*dockerExecutor, error) {
        // API version 1.21 corresponds to Docker 1.9, which is
        // currently the minimum version we want to support.
-       client, err := dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil)
+       client, err := dockerclient.NewClient(dockerclient.DefaultDockerHost, DockerAPIVersion, nil, nil)
        if watchdogInterval < 1 {
                watchdogInterval = time.Minute
        }
@@ -217,17 +230,17 @@ func (e *dockerExecutor) Wait(ctx context.Context) (int, error) {
                                // kill it.
                                return
                        } else if err != nil {
-                               e.logf("Error inspecting container: %s", err)
-                               watchdogErr <- err
-                               return
+                               watchdogErr <- fmt.Errorf("error inspecting container: %s", err)
                        } else if ctr.State == nil || !(ctr.State.Running || ctr.State.Status == "created") {
-                               watchdogErr <- fmt.Errorf("Container is not running: State=%v", ctr.State)
-                               return
+                               watchdogErr <- fmt.Errorf("container is not running: State=%v", ctr.State)
+                       } else {
+                               watchdogErr <- nil
                        }
                }
        }()
 
        waitOk, waitErr := e.dockerclient.ContainerWait(ctx, e.containerID, dockercontainer.WaitConditionNotRunning)
+       errors := 0
        for {
                select {
                case waitBody := <-waitOk:
@@ -242,7 +255,16 @@ func (e *dockerExecutor) Wait(ctx context.Context) (int, error) {
                        return -1, ctx.Err()
 
                case err := <-watchdogErr:
-                       return -1, err
+                       if err == nil {
+                               errors = 0
+                       } else {
+                               e.logf("docker watchdog: %s", err)
+                               errors++
+                               if errors >= dockerWatchdogThreshold {
+                                       e.logf("docker watchdog: giving up")
+                                       return -1, err
+                               }
+                       }
                }
        }
 }