package diagnostics
import (
+ "archive/tar"
"bytes"
"context"
+ "crypto/sha256"
+ _ "embed"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
+ "os"
+ "os/exec"
"strings"
"time"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/health"
"github.com/sirupsen/logrus"
)
func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var diag diagnoser
f := flag.NewFlagSet(prog, flag.ContinueOnError)
- f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
- f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
+ f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "`name` of project to find/create in home project and use for temporary/test objects")
+ f.StringVar(&diag.logLevel, "log-level", "info", "logging `level` (debug, info, warning, error)")
+ f.StringVar(&diag.dockerImage, "docker-image", "", "`image` (tag or portable data hash) to use when running a test container, or \"hello-world\" to use embedded hello-world image (default: build a custom image containing this executable, and run diagnostics inside the container too)")
+ f.StringVar(&diag.dockerImageFrom, "docker-image-from", "debian:stable-slim", "`base` image to use when building a custom image (see https://doc.arvados.org/main/admin/diagnostics.html#container-options)")
f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
+ f.BoolVar(&diag.verbose, "v", false, "verbose: include more information in report")
f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
return code
}
+ diag.stdout = stdout
+ diag.stderr = stderr
diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
diag.runtests()
}
}
+// docker save hello-world > hello-world.tar
+//
+//go:embed hello-world.tar
+var HelloWorldDockerImage []byte
+
type diagnoser struct {
- stdout io.Writer
- stderr io.Writer
- logLevel string
- priority int
- projectName string
- checkInternal bool
- checkExternal bool
- timeout time.Duration
- logger *logrus.Logger
- errors []string
- done map[int]bool
+ stdout io.Writer
+ stderr io.Writer
+ logLevel string
+ priority int
+ projectName string
+ dockerImage string
+ dockerImageFrom string
+ checkInternal bool
+ checkExternal bool
+ verbose bool
+ timeout time.Duration
+ logger *logrus.Logger
+ errors []string
+ done map[int]bool
}
func (diag *diagnoser) debugf(f string, args ...interface{}) {
diag.logger.Infof(" ... "+f, args...)
}
+func (diag *diagnoser) verbosef(f string, args ...interface{}) {
+ if diag.verbose {
+ diag.logger.Infof(" ... "+f, args...)
+ }
+}
+
func (diag *diagnoser) warnf(f string, args ...interface{}) {
diag.logger.Warnf(" ... "+f, args...)
}
func (diag *diagnoser) runtests() {
client := arvados.NewClientFromEnv()
+ // Disable auto-retry, use context instead
+ client.Timeout = 0
if client.APIHost == "" || client.AuthToken == "" {
diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests")
return
}
+ hostname, err := os.Hostname()
+ if err != nil {
+ diag.warnf("error getting hostname: %s")
+ } else {
+ diag.verbosef("hostname = %s", hostname)
+ }
+
+ diag.dotest(5, "running health check (same as `arvados-server check`)", func() error {
+ ldr := config.NewLoader(&bytes.Buffer{}, ctxlog.New(&bytes.Buffer{}, "text", "info"))
+ ldr.SetupFlags(flag.NewFlagSet("diagnostics", flag.ContinueOnError))
+ cfg, err := ldr.Load()
+ if err != nil {
+ diag.infof("skipping because config could not be loaded: %s", err)
+ return nil
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+ if cluster.SystemRootToken != os.Getenv("ARVADOS_API_TOKEN") {
+ return fmt.Errorf("diagnostics usage error: %s is readable but SystemRootToken does not match $ARVADOS_API_TOKEN (to fix, either run 'arvados-client sudo diagnostics' to load everything from config file, or set ARVADOS_CONFIG=- to load nothing from config file)", ldr.Path)
+ }
+ agg := &health.Aggregator{Cluster: cluster}
+ resp := agg.ClusterHealth()
+ for _, e := range resp.Errors {
+ diag.errorf("health check: %s", e)
+ }
+ if len(resp.Errors) > 0 {
+ diag.infof("consider running `arvados-server check -yaml` for a comprehensive report")
+ }
+ diag.verbosef("reported clock skew = %v", resp.ClockSkew)
+ reported := map[string]bool{}
+ for _, result := range resp.Checks {
+ version := strings.SplitN(result.Metrics.Version, " (go", 2)[0]
+ if version != "" && !reported[version] {
+ diag.verbosef("arvados version = %s", version)
+ reported[version] = true
+ }
+ }
+ reported = map[string]bool{}
+ for _, result := range resp.Checks {
+ if result.Server != "" && !reported[result.Server] {
+ diag.verbosef("http frontend version = %s", result.Server)
+ reported[result.Server] = true
+ }
+ }
+ reported = map[string]bool{}
+ for _, result := range resp.Checks {
+ if sha := result.ConfigSourceSHA256; sha != "" && !reported[sha] {
+ diag.verbosef("config file sha256 = %s", sha)
+ reported[sha] = true
+ }
+ }
+ return nil
+ })
+
var dd arvados.DiscoveryDocument
ddpath := "discovery/v1/apis/arvados/v1/rest"
diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
if err != nil {
return err
}
- diag.debugf("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
+ diag.verbosef("BlobSignatureTTL = %d", dd.BlobSignatureTTL)
return nil
})
var cluster arvados.Cluster
cfgpath := "arvados/v1/config"
+ cfgOK := false
diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if err != nil {
return err
}
- diag.debugf("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
+ diag.verbosef("Collections.BlobSigning = %v", cluster.Collections.BlobSigning)
+ cfgOK = true
return nil
})
if err != nil {
return err
}
- diag.debugf("user uuid = %s", user.UUID)
+ diag.verbosef("user uuid = %s", user.UUID)
return nil
})
+ if !cfgOK {
+ diag.errorf("cannot proceed without cluster config -- aborting without running any further tests")
+ return
+ }
+
// uncomment to create some spurious errors
// cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9"
isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
if isExternal {
- diag.debugf("controller returned only proxy services, this host is treated as \"external\"")
+ diag.infof("controller returned only proxy services, this host is treated as \"external\"")
} else if isInternal {
- diag.debugf("controller returned only non-proxy services, this host is treated as \"internal\"")
+ diag.infof("controller returned only non-proxy services, this host is treated as \"internal\"")
}
if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
}
if len(grplist.Items) > 0 {
project = grplist.Items[0]
- diag.debugf("using existing project, uuid = %s", project.UUID)
+ diag.verbosef("using existing project, uuid = %s", project.UUID)
return nil
}
diag.debugf("list groups: ok, no results")
if err != nil {
return fmt.Errorf("create project: %s", err)
}
- diag.debugf("created project, uuid = %s", project.UUID)
+ diag.verbosef("created project, uuid = %s", project.UUID)
return nil
})
if err != nil {
return err
}
- diag.debugf("ok, uuid = %s", collection.UUID)
+ diag.verbosef("ok, uuid = %s", collection.UUID)
return nil
})
}()
}
+ tempdir, err := ioutil.TempDir("", "arvados-diagnostics")
+ if err != nil {
+ diag.errorf("error creating temp dir: %s", err)
+ return
+ }
+ defer os.RemoveAll(tempdir)
+
+ var dockerImageData []byte
+ if diag.dockerImage != "" || diag.priority < 1 {
+ // We won't be using the self-built docker image, so
+ // don't build it. But we will write the embedded
+ // "hello-world" image to our test collection to test
+ // upload/download, whether or not we're using it as a
+ // docker image.
+ dockerImageData = HelloWorldDockerImage
+ } else if selfbin, err := os.Readlink("/proc/self/exe"); err != nil {
+ diag.errorf("readlink /proc/self/exe: %s", err)
+ return
+ } else if selfbindata, err := os.ReadFile(selfbin); err != nil {
+ diag.errorf("error reading %s: %s", selfbin, err)
+ return
+ } else {
+ selfbinSha := fmt.Sprintf("%x", sha256.Sum256(selfbindata))
+ tag := "arvados-client-diagnostics:" + selfbinSha[:9]
+ err := os.WriteFile(tempdir+"/arvados-client", selfbindata, 0777)
+ if err != nil {
+ diag.errorf("error writing %s: %s", tempdir+"/arvados-client", err)
+ return
+ }
+
+ dockerfile := "FROM " + diag.dockerImageFrom + "\n"
+ dockerfile += "RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends libfuse2 ca-certificates && apt-get clean\n"
+ dockerfile += "COPY /arvados-client /arvados-client\n"
+ cmd := exec.Command("docker", "build", "--tag", tag, "-f", "-", tempdir)
+ cmd.Stdin = strings.NewReader(dockerfile)
+ cmd.Stdout = diag.stderr
+ cmd.Stderr = diag.stderr
+ err = cmd.Run()
+ if err != nil {
+ diag.errorf("error building docker image: %s", err)
+ return
+ }
+ checkversion, err := exec.Command("docker", "run", tag, "/arvados-client", "version").CombinedOutput()
+ if err != nil {
+ diag.errorf("docker image does not seem to work: %s", err)
+ return
+ }
+ diag.infof("arvados-client version: %s", checkversion)
+
+ buf, err := exec.Command("docker", "save", tag).Output()
+ if err != nil {
+ diag.errorf("docker save %s: %s", tag, err)
+ return
+ }
+ diag.infof("docker image size is %d", len(buf))
+ dockerImageData = buf
+ }
+
+ // Read image tarball to find image ID, so we can upload it as
+ // "sha256:{...}.tar"
+ var imageSHA2 string
+ {
+ tr := tar.NewReader(bytes.NewReader(dockerImageData))
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ diag.errorf("internal error/bug: cannot read docker image tar file: %s", err)
+ return
+ }
+ if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name {
+ imageSHA2 = s
+ }
+ }
+ if imageSHA2 == "" {
+ diag.errorf("internal error/bug: cannot find {sha256}.json file in docker image tar file")
+ return
+ }
+ }
+ tarfilename := "sha256:" + imageSHA2 + ".tar"
+
diag.dotest(100, "uploading file via webdav", func() error {
- ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ timeout := diag.timeout
+ if len(dockerImageData) > 10<<20 && timeout < time.Minute {
+ // Extend the normal http timeout if we're
+ // uploading a substantial docker image.
+ timeout = time.Minute
+ }
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(timeout))
defer cancel()
if collection.UUID == "" {
return fmt.Errorf("skipping, no test collection")
}
- req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
+ t0 := time.Now()
+ req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/"+tarfilename, bytes.NewReader(dockerImageData))
if err != nil {
return fmt.Errorf("BUG? http.NewRequest: %s", err)
}
if resp.StatusCode != http.StatusCreated {
return fmt.Errorf("status %s", resp.Status)
}
- diag.debugf("ok, status %s", resp.Status)
+ diag.verbosef("upload ok, status %s, %f MB/s", resp.Status, float64(len(dockerImageData))/time.Since(t0).Seconds()/1000000)
err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
if err != nil {
return fmt.Errorf("get updated collection: %s", err)
}
- diag.debugf("ok, pdh %s", collection.PortableDataHash)
+ diag.verbosef("upload pdh %s", collection.PortableDataHash)
return nil
})
davurl := cluster.Services.WebDAV.ExternalURL
+ davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
if davurl.Host == "" {
return fmt.Errorf("host missing - content previews will not work")
}
- if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
+ if !davWildcard && !cluster.Collections.TrustAllContent {
diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
}
return nil
})
for i, trial := range []struct {
- needcoll bool
- status int
- fileurl string
+ needcoll bool
+ needWildcard bool
+ status int
+ fileurl string
}{
- {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
- {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
- {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
- {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
- {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
- {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+ {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+ {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + tarfilename},
+ {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+ {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/" + tarfilename},
+ {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + tarfilename},
+ {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/" + tarfilename},
} {
diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+ if trial.needWildcard && !davWildcard {
+ diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
+ return nil
+ }
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if trial.needcoll && collection.UUID == "" {
if resp.StatusCode != trial.status {
return fmt.Errorf("unexpected response status: %s", resp.Status)
}
- if trial.status == http.StatusOK && string(body) != "testfiledata" {
- return fmt.Errorf("unexpected response content: %q", body)
+ if trial.status == http.StatusOK && !bytes.Equal(body, dockerImageData) {
+ excerpt := body
+ if len(excerpt) > 128 {
+ excerpt = append([]byte(nil), body[:128]...)
+ excerpt = append(excerpt, []byte("[...]")...)
+ }
+ return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt)
}
return nil
})
return err
}
if len(vmlist.Items) < 1 {
- return fmt.Errorf("no VMs found")
- }
- vm = vmlist.Items[0]
- return nil
- })
-
- diag.dotest(140, "getting workbench1 webshell page", func() error {
- ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
- defer cancel()
- if vm.UUID == "" {
- return fmt.Errorf("skipping, no vm available")
- }
- webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
- diag.debugf("url %s", webshelltermurl)
- req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
- if err != nil {
- return err
- }
- req.Header.Set("Authorization", "Bearer "+client.AuthToken)
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- body, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return fmt.Errorf("reading response: %s", err)
- }
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("unexpected response status: %s %q", resp.Status, body)
+ diag.warnf("no VMs found")
+ } else {
+ vm = vmlist.Items[0]
}
return nil
})
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if vm.UUID == "" {
- return fmt.Errorf("skipping, no vm available")
+ diag.warnf("skipping, no vm available")
+ return nil
}
u := cluster.Services.WebShell.ExternalURL
webshellurl := u.String() + vm.Hostname + "?"
return fmt.Errorf("skipping, no project to work in")
}
+ timestamp := time.Now().Format(time.RFC3339)
+
+ var ctrCommand []string
+ switch diag.dockerImage {
+ case "":
+ if collection.UUID == "" {
+ return fmt.Errorf("skipping, no test collection to use as docker image")
+ }
+ diag.dockerImage = collection.PortableDataHash
+ ctrCommand = []string{"/arvados-client", "diagnostics",
+ "-priority=0", // don't run a container
+ "-log-level=" + diag.logLevel,
+ "-internal-client=true"}
+ case "hello-world":
+ if collection.UUID == "" {
+ return fmt.Errorf("skipping, no test collection to use as docker image")
+ }
+ diag.dockerImage = collection.PortableDataHash
+ ctrCommand = []string{"/hello"}
+ default:
+ ctrCommand = []string{"echo", timestamp}
+ }
+
var cr arvados.ContainerRequest
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
- timestamp := time.Now().Format(time.RFC3339)
err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
"owner_uuid": project.UUID,
"name": fmt.Sprintf("diagnostics container request %s", timestamp),
- "container_image": "arvados/jobs",
- "command": []string{"echo", timestamp},
+ "container_image": diag.dockerImage,
+ "command": ctrCommand,
"use_existing": false,
"output_path": "/mnt/output",
"output_name": fmt.Sprintf("diagnostics output %s", timestamp),
},
},
"runtime_constraints": arvados.RuntimeConstraints{
+ API: true,
VCPUs: 1,
- RAM: 1 << 26,
- KeepCacheRAM: 1 << 26,
+ RAM: 128 << 20,
+ KeepCacheRAM: 64 << 20,
},
}})
if err != nil {
return err
}
- diag.debugf("container request uuid = %s", cr.UUID)
- diag.debugf("container uuid = %s", cr.ContainerUUID)
+ diag.infof("container request uuid = %s", cr.UUID)
+ diag.verbosef("container uuid = %s", cr.ContainerUUID)
timeout := 10 * time.Minute
diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
- ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
- defer cancel()
+ deadline := time.Now().Add(timeout)
var c arvados.Container
- for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
- ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
+ for ; cr.State != arvados.ContainerRequestStateFinal && time.Now().Before(deadline); time.Sleep(2 * time.Second) {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
crStateWas := cr.State
if c.State != cStateWas {
diag.debugf("container state = %s", c.State)
}
+
+ cancel()
}
+ if cr.State != arvados.ContainerRequestStateFinal {
+ err := client.RequestAndDecodeContext(context.Background(), &cr, "PATCH", "arvados/v1/container_requests/"+cr.UUID, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "priority": 0,
+ }})
+ if err != nil {
+ diag.infof("error canceling container request %s: %s", cr.UUID, err)
+ } else {
+ diag.debugf("canceled container request %s", cr.UUID)
+ }
+ return fmt.Errorf("timed out waiting for container to finish; container request %s state was %q, container %s state was %q", cr.UUID, cr.State, c.UUID, c.State)
+ }
if c.State != arvados.ContainerStateComplete {
return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
- } else if c.ExitCode != 0 {
+ }
+ if c.ExitCode != 0 {
return fmt.Errorf("container exited %d", c.ExitCode)
}
return nil