// Copyright (C) The Arvados Authors. All rights reserved.
//
// SPDX-License-Identifier: AGPL-3.0

package diagnostics

import (
	"archive/tar"
	"bytes"
	"context"
	"crypto/sha256"
	_ "embed"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"git.arvados.org/arvados.git/lib/cmd"
	"git.arvados.org/arvados.git/lib/config"
	"git.arvados.org/arvados.git/sdk/go/arvados"
	"git.arvados.org/arvados.git/sdk/go/ctxlog"
	"git.arvados.org/arvados.git/sdk/go/health"
	"github.com/sirupsen/logrus"
)

type Command struct{}

func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
	var diag diagnoser
	f := flag.NewFlagSet(prog, flag.ContinueOnError)
	f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "`name` of project to find/create in home project and use for temporary/test objects")
	f.StringVar(&diag.logLevel, "log-level", "info", "logging `level` (debug, info, warning, error)")
	f.StringVar(&diag.dockerImage, "docker-image", "", "`image` (tag or portable data hash) to use when running a test container, or \"hello-world\" to use embedded hello-world image (default: build a custom image containing this executable, and run diagnostics inside the container too)")
	f.StringVar(&diag.dockerImageFrom, "docker-image-from", "debian:stable-slim", "`base` image to use when building a custom image (see https://doc.arvados.org/main/admin/diagnostics.html#container-options)")
	f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
	f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
	f.BoolVar(&diag.verbose, "v", false, "verbose: include more information in report")
	f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
	f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
	if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
		return code
	}
	diag.stdout = stdout
	diag.stderr = stderr
	diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
	diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
	diag.runtests()
	if len(diag.errors) == 0 {
		diag.logger.Info("--- no errors ---")
		return 0
	} else {
		if diag.logger.Level > logrus.ErrorLevel {
			fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n")
			for _, e := range diag.errors {
				diag.logger.Error(e)
			}
		}
		return 1
	}
}

// docker save hello-world > hello-world.tar
//
//go:embed hello-world.tar
var HelloWorldDockerImage []byte

type diagnoser struct {
	stdout          io.Writer
	stderr          io.Writer
	logLevel        string
	priority        int
	projectName     string
	dockerImage     string
	dockerImageFrom string
	checkInternal   bool
	checkExternal   bool
	verbose         bool
	timeout         time.Duration
	logger          *logrus.Logger
	errors          []string
	done            map[int]bool
}

func (diag *diagnoser) debugf(f string, args ...interface{}) {
	diag.logger.Debugf("  ... "+f, args...)
}

func (diag *diagnoser) infof(f string, args ...interface{}) {
	diag.logger.Infof("  ... "+f, args...)
}

func (diag *diagnoser) verbosef(f string, args ...interface{}) {
	if diag.verbose {
		diag.logger.Infof("  ... "+f, args...)
	}
}

func (diag *diagnoser) warnf(f string, args ...interface{}) {
	diag.logger.Warnf("  ... "+f, args...)
}

func (diag *diagnoser) errorf(f string, args ...interface{}) {
	diag.logger.Errorf(f, args...)
	diag.errors = append(diag.errors, fmt.Sprintf(f, args...))
}

// Run the given func, logging appropriate messages before and after,
// adding timing info, etc.
//
// The id argument should be unique among tests, and shouldn't change
// when other tests are added/removed.
func (diag *diagnoser) dotest(id int, title string, fn func() error) {
	if diag.done == nil {
		diag.done = map[int]bool{}
	} else if diag.done[id] {
		diag.errorf("(bug) reused test id %d", id)
	}
	diag.done[id] = true

	diag.logger.Infof("%4d: %s", id, title)
	t0 := time.Now()
	err := fn()
	elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
	if err != nil {
		diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
	} else {
		diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
	}
}

func (diag *diagnoser) runtests() {
	client := arvados.NewClientFromEnv()
	// Disable auto-retry, use context instead
	client.Timeout = 0

	if client.APIHost == "" || client.AuthToken == "" {
		diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
		return
	}

	hostname, err := os.Hostname()
	if err != nil {
		diag.warnf("error getting hostname: %s")
	} else {
		diag.verbosef("hostname = %s", hostname)
	}

	diag.dotest(5, "running health check (same as `arvados-server check`)", func() error {
		ldr := config.NewLoader(&bytes.Buffer{}, ctxlog.New(&bytes.Buffer{}, "text", "info"))
		ldr.SetupFlags(flag.NewFlagSet("diagnostics", flag.ContinueOnError))
		cfg, err := ldr.Load()
		if err != nil {
			diag.infof("skipping because config could not be loaded: %s", err)
			return nil
		}
		cluster, err := cfg.GetCluster("")
		if err != nil {
			return err
		}
		if cluster.SystemRootToken != os.Getenv("ARVADOS_API_TOKEN") {
			return fmt.Errorf("diagnostics usage error: %s is readable but SystemRootToken does not match $ARVADOS_API_TOKEN (to fix, either run 'arvados-client sudo diagnostics' to load everything from config file, or set ARVADOS_CONFIG=- to load nothing from config file)", ldr.Path)
		}
		agg := &health.Aggregator{Cluster: cluster}
		resp := agg.ClusterHealth()
		for _, e := range resp.Errors {
			diag.errorf("health check: %s", e)
		}
		if len(resp.Errors) > 0 {
			diag.infof("consider running `arvados-server check -yaml` for a comprehensive report")
		}
		diag.verbosef("reported clock skew = %v", resp.ClockSkew)
		reported := map[string]bool{}
		for _, result := range resp.Checks {
			version := strings.SplitN(result.Metrics.Version, " (go", 2)[0]
			if version != "" && !reported[version] {
				diag.verbosef("arvados version = %s", version)
				reported[version] = true
			}
		}
		reported = map[string]bool{}
		for _, result := range resp.Checks {
			if result.Server != "" && !reported[result.Server] {
				diag.verbosef("http frontend version = %s", result.Server)
				reported[result.Server] = true
			}
		}
		reported = map[string]bool{}
		for _, result := range resp.Checks {
			if sha := result.ConfigSourceSHA256; sha != "" && !reported[sha] {
				diag.verbosef("config file sha256 = %s", sha)
				reported[sha] = true
			}
		}
		return nil
	})

	var dd arvados.DiscoveryDocument
	ddpath := "discovery/v1/apis/arvados/v1/rest"
	diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
		if err != nil {
			return err
		}
		diag.verbosef("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
		return nil
	})

	var cluster arvados.Cluster
	cfgpath := "arvados/v1/config"
	cfgOK := false
	diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
		if err != nil {
			return err
		}
		diag.verbosef("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
		cfgOK = true
		return nil
	})

	var user arvados.User
	diag.dotest(30, "getting current user record", func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
		if err != nil {
			return err
		}
		diag.verbosef("user uuid = %s", user.UUID)
		return nil
	})

	if !cfgOK {
		diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
		return
	}

	// uncomment to create some spurious errors
	// cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"

	// TODO: detect routing errors here, like finding wb2 at the
	// wb1 address.
	for i, svc := range []*arvados.Service{
		&cluster.Services.Keepproxy,
		&cluster.Services.WebDAV,
		&cluster.Services.WebDAVDownload,
		&cluster.Services.Websocket,
		&cluster.Services.Workbench1,
		&cluster.Services.Workbench2,
	} {
		diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
			defer cancel()
			u := svc.ExternalURL
			if strings.HasPrefix(u.Scheme, "ws") {
				// We can do a real websocket test elsewhere,
				// but for now we'll just check the https
				// connection.
				u.Scheme = "http" + u.Scheme[2:]
			}
			if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
				u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
			}
			req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
			if err != nil {
				return err
			}
			resp, err := http.DefaultClient.Do(req)
			if err != nil {
				return err
			}
			resp.Body.Close()
			return nil
		})
	}

	for i, url := range []string{
		cluster.Services.Controller.ExternalURL.String(),
		cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0",
		cluster.Services.WebDAVDownload.ExternalURL.String(),
	} {
		diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
			defer cancel()
			req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
			if err != nil {
				return err
			}
			req.Header.Set("Origin", "https://example.com")
			resp, err := http.DefaultClient.Do(req)
			if err != nil {
				return err
			}
			if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" {
				return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr)
			}
			return nil
		})
	}

	var keeplist arvados.KeepServiceList
	diag.dotest(60, "checking internal/external client detection", func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
		if err != nil {
			return fmt.Errorf("error getting keep services list: %s", err)
		} else if len(keeplist.Items) == 0 {
			return fmt.Errorf("controller did not return any keep services")
		}
		found := map[string]int{}
		for _, ks := range keeplist.Items {
			found[ks.ServiceType]++
		}
		isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
		isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
		if isExternal {
			diag.infof("controller returned only proxy services, this host is treated as \"external\"")
		} else if isInternal {
			diag.infof("controller returned only non-proxy services, this host is treated as \"internal\"")
		}
		if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
			return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
		}
		return nil
	})

	for i, ks := range keeplist.Items {
		u := url.URL{
			Scheme: "http",
			Host:   net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)),
			Path:   "/",
		}
		if ks.ServiceSSLFlag {
			u.Scheme = "https"
		}
		diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error {
			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
			defer cancel()
			req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil)
			if err != nil {
				return err
			}
			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
			resp, err := http.DefaultClient.Do(req)
			if err != nil {
				return err
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("reading response body: %s", err)
			}
			loc := strings.TrimSpace(string(body))
			if !strings.HasPrefix(loc, "d41d8") {
				return fmt.Errorf("unexpected response from write: %q", body)
			}

			req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil)
			if err != nil {
				return err
			}
			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
			resp, err = http.DefaultClient.Do(req)
			if err != nil {
				return err
			}
			defer resp.Body.Close()
			body, err = ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("reading response body: %s", err)
			}
			if len(body) != 0 {
				return fmt.Errorf("unexpected response from read: %q", body)
			}

			return nil
		})
	}

	var project arvados.Group
	diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		var grplist arvados.GroupList
		err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
			Filters: []arvados.Filter{
				{"name", "=", diag.projectName},
				{"group_class", "=", "project"},
				{"owner_uuid", "=", user.UUID}},
			Limit: 999999})
		if err != nil {
			return fmt.Errorf("list groups: %s", err)
		}
		if len(grplist.Items) > 0 {
			project = grplist.Items[0]
			diag.verbosef("using existing project, uuid = %s", project.UUID)
			return nil
		}
		diag.debugf("list groups: ok, no results")
		err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
			"name":        diag.projectName,
			"group_class": "project",
		}})
		if err != nil {
			return fmt.Errorf("create project: %s", err)
		}
		diag.verbosef("created project, uuid = %s", project.UUID)
		return nil
	})

	var collection arvados.Collection
	diag.dotest(90, "creating temporary collection", func() error {
		if project.UUID == "" {
			return fmt.Errorf("skipping, no project to work in")
		}
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
			"ensure_unique_name": true,
			"collection": map[string]interface{}{
				"owner_uuid": project.UUID,
				"name":       "test collection",
				"trash_at":   time.Now().Add(time.Hour)}})
		if err != nil {
			return err
		}
		diag.verbosef("ok, uuid = %s", collection.UUID)
		return nil
	})

	if collection.UUID != "" {
		defer func() {
			diag.dotest(9990, "deleting temporary collection", func() error {
				ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
				defer cancel()
				return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
			})
		}()
	}

	tempdir, err := ioutil.TempDir("", "arvados-diagnostics")
	if err != nil {
		diag.errorf("error creating temp dir: %s", err)
		return
	}
	defer os.RemoveAll(tempdir)

	var imageSHA2 string
	var dockerImageData []byte
	if diag.dockerImage != "" || diag.priority < 1 {
		// We won't be using the self-built docker image, so
		// don't build it.  But we will write the embedded
		// "hello-world" image to our test collection to test
		// upload/download, whether or not we're using it as a
		// docker image.
		dockerImageData = HelloWorldDockerImage

		if diag.priority > 0 {
			imageSHA2, err = getSHA2FromImageData(dockerImageData)
			if err != nil {
				diag.errorf("internal error/bug: %s", err)
				return
			}
		}
	} else if selfbin, err := os.Readlink("/proc/self/exe"); err != nil {
		diag.errorf("readlink /proc/self/exe: %s", err)
		return
	} else if selfbindata, err := os.ReadFile(selfbin); err != nil {
		diag.errorf("error reading %s: %s", selfbin, err)
		return
	} else {
		selfbinSha := fmt.Sprintf("%x", sha256.Sum256(selfbindata))
		tag := "arvados-client-diagnostics:" + selfbinSha[:9]
		err := os.WriteFile(tempdir+"/arvados-client", selfbindata, 0777)
		if err != nil {
			diag.errorf("error writing %s: %s", tempdir+"/arvados-client", err)
			return
		}

		dockerfile := "FROM " + diag.dockerImageFrom + "\n"
		dockerfile += "RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends libfuse2 ca-certificates && apt-get clean\n"
		dockerfile += "COPY /arvados-client /arvados-client\n"
		cmd := exec.Command("docker", "build", "--tag", tag, "-f", "-", tempdir)
		cmd.Stdin = strings.NewReader(dockerfile)
		cmd.Stdout = diag.stderr
		cmd.Stderr = diag.stderr
		err = cmd.Run()
		if err != nil {
			diag.errorf("error building docker image: %s", err)
			return
		}
		checkversion, err := exec.Command("docker", "run", tag, "/arvados-client", "version").CombinedOutput()
		if err != nil {
			diag.errorf("docker image does not seem to work: %s", err)
			return
		}
		diag.infof("arvados-client version: %s", checkversion)

		buf, err := exec.Command("docker", "inspect", "--format={{.Id}}", tag).Output()
		if err != nil {
			diag.errorf("docker inspect --format={{.Id}} %s: %s", tag, err)
			return
		}
		imageSHA2 = min64HexDigits.FindString(string(buf))
		if len(imageSHA2) != 64 {
			diag.errorf("docker inspect --format={{.Id}} output %q does not seem to contain sha256 digest", buf)
			return
		}

		buf, err = exec.Command("docker", "save", tag).Output()
		if err != nil {
			diag.errorf("docker save %s: %s", tag, err)
			return
		}
		diag.infof("docker image size is %d", len(buf))
		dockerImageData = buf
	}

	tarfilename := "sha256:" + imageSHA2 + ".tar"

	diag.dotest(100, "uploading file via webdav", func() error {
		timeout := diag.timeout
		if len(dockerImageData) > 10<<20 && timeout < time.Minute {
			// Extend the normal http timeout if we're
			// uploading a substantial docker image.
			timeout = time.Minute
		}
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(timeout))
		defer cancel()
		if collection.UUID == "" {
			return fmt.Errorf("skipping, no test collection")
		}
		t0 := time.Now()
		req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/"+tarfilename, bytes.NewReader(dockerImageData))
		if err != nil {
			return fmt.Errorf("BUG? http.NewRequest: %s", err)
		}
		req.Header.Set("Authorization", "Bearer "+client.AuthToken)
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return fmt.Errorf("error performing http request: %s", err)
		}
		resp.Body.Close()
		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("status %s", resp.Status)
		}
		diag.verbosef("upload ok, status %s, %f MB/s", resp.Status, float64(len(dockerImageData))/time.Since(t0).Seconds()/1000000)
		err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
		if err != nil {
			return fmt.Errorf("get updated collection: %s", err)
		}
		diag.verbosef("upload pdh %s", collection.PortableDataHash)
		return nil
	})

	davurl := cluster.Services.WebDAV.ExternalURL
	davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
	diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
		if davurl.Host == "" {
			return fmt.Errorf("host missing - content previews will not work")
		}
		if !davWildcard && !cluster.Collections.TrustAllContent {
			diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
		}
		return nil
	})

	for i, trial := range []struct {
		needcoll     bool
		needWildcard bool
		status       int
		fileurl      string
	}{
		{false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
		{false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + tarfilename},
		{false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
		{false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/" + tarfilename},
		{true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + tarfilename},
		{true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/" + tarfilename},
	} {
		diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
			if trial.needWildcard && !davWildcard {
				diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
				return nil
			}
			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
			defer cancel()
			if trial.needcoll && collection.UUID == "" {
				return fmt.Errorf("skipping, no test collection")
			}
			req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
			if err != nil {
				return err
			}
			req.Header.Set("Authorization", "Bearer "+client.AuthToken)
			resp, err := http.DefaultClient.Do(req)
			if err != nil {
				return err
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("reading response: %s", err)
			}
			if resp.StatusCode != trial.status {
				return fmt.Errorf("unexpected response status: %s", resp.Status)
			}
			if trial.status == http.StatusOK && !bytes.Equal(body, dockerImageData) {
				excerpt := body
				if len(excerpt) > 128 {
					excerpt = append([]byte(nil), body[:128]...)
					excerpt = append(excerpt, []byte("[...]")...)
				}
				return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt)
			}
			return nil
		})
	}

	var vm arvados.VirtualMachine
	diag.dotest(130, "getting list of virtual machines", func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		var vmlist arvados.VirtualMachineList
		err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
		if err != nil {
			return err
		}
		if len(vmlist.Items) < 1 {
			diag.warnf("no VMs found")
		} else {
			vm = vmlist.Items[0]
		}
		return nil
	})

	diag.dotest(150, "connecting to webshell service", func() error {
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()
		if vm.UUID == "" {
			diag.warnf("skipping, no vm available")
			return nil
		}
		u := cluster.Services.WebShell.ExternalURL
		webshellurl := u.String() + vm.Hostname + "?"
		if strings.HasPrefix(u.Host, "*") {
			u.Host = vm.Hostname + u.Host[1:]
			webshellurl = u.String() + "?"
		}
		diag.debugf("url %s", webshellurl)
		req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{
			"width":   {"80"},
			"height":  {"25"},
			"session": {"xyzzy"},
			"rooturl": {webshellurl},
		}.Encode()))
		if err != nil {
			return err
		}
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		diag.debugf("response status %s", resp.Status)
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("reading response: %s", err)
		}
		diag.debugf("response body %q", body)
		// We don't speak the protocol, so we get a 400 error
		// from the webshell server even if everything is
		// OK. Anything else (404, 502, ???) indicates a
		// problem.
		if resp.StatusCode != http.StatusBadRequest {
			return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body)
		}
		return nil
	})

	diag.dotest(160, "running a container", func() error {
		if diag.priority < 1 {
			diag.infof("skipping (use priority > 0 if you want to run a container)")
			return nil
		}
		if project.UUID == "" {
			return fmt.Errorf("skipping, no project to work in")
		}

		timestamp := time.Now().Format(time.RFC3339)

		var ctrCommand []string
		switch diag.dockerImage {
		case "":
			if collection.UUID == "" {
				return fmt.Errorf("skipping, no test collection to use as docker image")
			}
			diag.dockerImage = collection.PortableDataHash
			ctrCommand = []string{"/arvados-client", "diagnostics",
				"-priority=0", // don't run a container
				"-log-level=" + diag.logLevel,
				"-internal-client=true"}
		case "hello-world":
			if collection.UUID == "" {
				return fmt.Errorf("skipping, no test collection to use as docker image")
			}
			diag.dockerImage = collection.PortableDataHash
			ctrCommand = []string{"/hello"}
		default:
			ctrCommand = []string{"echo", timestamp}
		}

		var cr arvados.ContainerRequest
		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
		defer cancel()

		err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
			"owner_uuid":      project.UUID,
			"name":            fmt.Sprintf("diagnostics container request %s", timestamp),
			"container_image": diag.dockerImage,
			"command":         ctrCommand,
			"use_existing":    false,
			"output_path":     "/mnt/output",
			"output_name":     fmt.Sprintf("diagnostics output %s", timestamp),
			"priority":        diag.priority,
			"state":           arvados.ContainerRequestStateCommitted,
			"mounts": map[string]map[string]interface{}{
				"/mnt/output": {
					"kind":     "collection",
					"writable": true,
				},
			},
			"runtime_constraints": arvados.RuntimeConstraints{
				API:          true,
				VCPUs:        1,
				RAM:          128 << 20,
				KeepCacheRAM: 64 << 20,
			},
		}})
		if err != nil {
			return err
		}
		diag.infof("container request uuid = %s", cr.UUID)
		diag.verbosef("container uuid = %s", cr.ContainerUUID)

		timeout := 10 * time.Minute
		diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
		deadline := time.Now().Add(timeout)

		var c arvados.Container
		for ; cr.State != arvados.ContainerRequestStateFinal && time.Now().Before(deadline); time.Sleep(2 * time.Second) {
			ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
			defer cancel()

			crStateWas := cr.State
			err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
			if err != nil {
				return err
			}
			if cr.State != crStateWas {
				diag.debugf("container request state = %s", cr.State)
			}

			cStateWas := c.State
			err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
			if err != nil {
				return err
			}
			if c.State != cStateWas {
				diag.debugf("container state = %s", c.State)
			}

			cancel()
		}

		if cr.State != arvados.ContainerRequestStateFinal {
			err := client.RequestAndDecodeContext(context.Background(), &cr, "PATCH", "arvados/v1/container_requests/"+cr.UUID, nil, map[string]interface{}{
				"container_request": map[string]interface{}{
					"priority": 0,
				}})
			if err != nil {
				diag.infof("error canceling container request %s: %s", cr.UUID, err)
			} else {
				diag.debugf("canceled container request %s", cr.UUID)
			}
			return fmt.Errorf("timed out waiting for container to finish; container request %s state was %q, container %s state was %q", cr.UUID, cr.State, c.UUID, c.State)
		}
		if c.State != arvados.ContainerStateComplete {
			return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
		}
		if c.ExitCode != 0 {
			return fmt.Errorf("container exited %d", c.ExitCode)
		}
		return nil
	})
}

func getSHA2FromImageData(dockerImageData []byte) (string, error) {
	tr := tar.NewReader(bytes.NewReader(dockerImageData))
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return "", fmt.Errorf("cannot find manifest.json in docker image tar file")
		}
		if err != nil {
			return "", fmt.Errorf("cannot read docker image tar file: %s", err)
		}
		if hdr.Name != "manifest.json" {
			continue
		}
		var manifest []struct {
			Config string
		}
		err = json.NewDecoder(tr).Decode(&manifest)
		if err != nil {
			return "", fmt.Errorf("cannot read manifest.json from docker image tar file: %s", err)
		}
		if len(manifest) == 0 {
			return "", fmt.Errorf("manifest.json is empty")
		}
		s := min64HexDigits.FindString(manifest[0].Config)
		if len(s) != 64 {
			return "", fmt.Errorf("found manifest.json but .[0].Config %q does not seem to contain sha256 digest", manifest[0].Config)
		}
		return s, nil
	}
}

var min64HexDigits = regexp.MustCompile(`[0-9a-f]{64,}`)