X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/751cdf33d5519c393323baa15233ca55b7c7a9a1..60cd2ac359e15c65cd5714658f49b624eccab0c8:/lib/diagnostics/cmd.go diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go index 26bbdfe1d3..30dd6ee83b 100644 --- a/lib/diagnostics/cmd.go +++ b/lib/diagnostics/cmd.go @@ -5,92 +5,245 @@ package diagnostics import ( + "archive/tar" "bytes" + "context" + "crypto/sha256" + _ "embed" "flag" "fmt" "io" "io/ioutil" + "net" "net/http" + "net/url" + "os" + "os/exec" "strings" "time" + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/health" "github.com/sirupsen/logrus" ) -type Command struct { - projectName string -} +type Command struct{} -func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + var diag diagnoser f := flag.NewFlagSet(prog, flag.ContinueOnError) - f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects") - loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)") - checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client") - checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client") - err := f.Parse(args) - if err == flag.ErrHelp { + f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "`name` of project to find/create in home project and use for temporary/test objects") + f.StringVar(&diag.logLevel, "log-level", "info", "logging `level` (debug, info, warning, error)") + f.StringVar(&diag.dockerImage, "docker-image", "", "`image` (tag or portable data hash) to use when running a test container, or \"hello-world\" to use embedded hello-world image (default: build a custom image containing this executable, and run diagnostics inside the container too)") + f.StringVar(&diag.dockerImageFrom, "docker-image-from", "debian:stable-slim", "`base` image to use when building a custom image (see https://doc.arvados.org/main/admin/diagnostics.html#container-options)") + f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client") + f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client") + f.BoolVar(&diag.verbose, "v", false, "verbose: include more information in report") + f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)") + f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests") + if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok { + return code + } + diag.stdout = stdout + diag.stderr = stderr + diag.logger = ctxlog.New(stdout, "text", diag.logLevel) + diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true}) + diag.runtests() + if len(diag.errors) == 0 { + diag.logger.Info("--- no errors ---") return 0 - } else if err != nil { - fmt.Fprintln(stderr, err) - return 2 + } else { + if diag.logger.Level > logrus.ErrorLevel { + fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n") + for _, e := range diag.errors { + diag.logger.Error(e) + } + } + return 1 } +} - logger := ctxlog.New(stdout, "text", *loglevel) - logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}) +// docker save hello-world > hello-world.tar +// +//go:embed hello-world.tar +var HelloWorldDockerImage []byte - infof := logger.Infof - warnf := logger.Warnf - var errors []string - errorf := func(f string, args ...interface{}) { - logger.Errorf(f, args...) - errors = append(errors, fmt.Sprintf(f, args...)) +type diagnoser struct { + stdout io.Writer + stderr io.Writer + logLevel string + priority int + projectName string + dockerImage string + dockerImageFrom string + checkInternal bool + checkExternal bool + verbose bool + timeout time.Duration + logger *logrus.Logger + errors []string + done map[int]bool +} + +func (diag *diagnoser) debugf(f string, args ...interface{}) { + diag.logger.Debugf(" ... "+f, args...) +} + +func (diag *diagnoser) infof(f string, args ...interface{}) { + diag.logger.Infof(" ... "+f, args...) +} + +func (diag *diagnoser) verbosef(f string, args ...interface{}) { + if diag.verbose { + diag.logger.Infof(" ... "+f, args...) } - defer func() { - if len(errors) == 0 { - logger.Info("--- no errors ---") - } else { - fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n") - for _, e := range errors { - logger.Error(e) - } - } - }() +} - client := arvados.NewClientFromEnv() +func (diag *diagnoser) warnf(f string, args ...interface{}) { + diag.logger.Warnf(" ... "+f, args...) +} - var dd arvados.DiscoveryDocument - ddpath := "discovery/v1/apis/arvados/v1/rest" - testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath) - logger.Info(testname) - err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil) +func (diag *diagnoser) errorf(f string, args ...interface{}) { + diag.logger.Errorf(f, args...) + diag.errors = append(diag.errors, fmt.Sprintf(f, args...)) +} + +// Run the given func, logging appropriate messages before and after, +// adding timing info, etc. +// +// The id argument should be unique among tests, and shouldn't change +// when other tests are added/removed. +func (diag *diagnoser) dotest(id int, title string, fn func() error) { + if diag.done == nil { + diag.done = map[int]bool{} + } else if diag.done[id] { + diag.errorf("(bug) reused test id %d", id) + } + diag.done[id] = true + + diag.logger.Infof("%4d: %s", id, title) + t0 := time.Now() + err := fn() + elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond) if err != nil { - errorf("%s: %s", testname, err) + diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err) } else { - infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL) + diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed) + } +} + +func (diag *diagnoser) runtests() { + client := arvados.NewClientFromEnv() + // Disable auto-retry, use context instead + client.Timeout = 0 + + if client.APIHost == "" || client.AuthToken == "" { + diag.errorf("ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set -- aborting without running any tests") + return } - var cluster arvados.Cluster - cfgpath := "arvados/v1/config" - testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath) - logger.Info(testname) - err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil) + hostname, err := os.Hostname() if err != nil { - errorf("%s: %s", testname, err) + diag.warnf("error getting hostname: %s") } else { - infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning) + diag.verbosef("hostname = %s", hostname) } + diag.dotest(5, "running health check (same as `arvados-server check`)", func() error { + ldr := config.NewLoader(&bytes.Buffer{}, ctxlog.New(&bytes.Buffer{}, "text", "info")) + ldr.SetupFlags(flag.NewFlagSet("diagnostics", flag.ContinueOnError)) + cfg, err := ldr.Load() + if err != nil { + diag.infof("skipping because config could not be loaded: %s", err) + return nil + } + cluster, err := cfg.GetCluster("") + if err != nil { + return err + } + if cluster.SystemRootToken != os.Getenv("ARVADOS_API_TOKEN") { + return fmt.Errorf("diagnostics usage error: %s is readable but SystemRootToken does not match $ARVADOS_API_TOKEN (to fix, either run 'arvados-client sudo diagnostics' to load everything from config file, or set ARVADOS_CONFIG=- to load nothing from config file)", ldr.Path) + } + agg := &health.Aggregator{Cluster: cluster} + resp := agg.ClusterHealth() + for _, e := range resp.Errors { + diag.errorf("health check: %s", e) + } + if len(resp.Errors) > 0 { + diag.infof("consider running `arvados-server check -yaml` for a comprehensive report") + } + diag.verbosef("reported clock skew = %v", resp.ClockSkew) + reported := map[string]bool{} + for _, result := range resp.Checks { + version := strings.SplitN(result.Metrics.Version, " (go", 2)[0] + if version != "" && !reported[version] { + diag.verbosef("arvados version = %s", version) + reported[version] = true + } + } + reported = map[string]bool{} + for _, result := range resp.Checks { + if result.Server != "" && !reported[result.Server] { + diag.verbosef("http frontend version = %s", result.Server) + reported[result.Server] = true + } + } + reported = map[string]bool{} + for _, result := range resp.Checks { + if sha := result.ConfigSourceSHA256; sha != "" && !reported[sha] { + diag.verbosef("config file sha256 = %s", sha) + reported[sha] = true + } + } + return nil + }) + + var dd arvados.DiscoveryDocument + ddpath := "discovery/v1/apis/arvados/v1/rest" + diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil) + if err != nil { + return err + } + diag.verbosef("BlobSignatureTTL = %d", dd.BlobSignatureTTL) + return nil + }) + + var cluster arvados.Cluster + cfgpath := "arvados/v1/config" + cfgOK := false + diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil) + if err != nil { + return err + } + diag.verbosef("Collections.BlobSigning = %v", cluster.Collections.BlobSigning) + cfgOK = true + return nil + }) + var user arvados.User - testname = "getting current user record" - logger.Info(testname) - err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil) - if err != nil { - errorf("%s: %s", testname, err) - return 2 - } else { - infof("%s: ok, uuid = %s", testname, user.UUID) + diag.dotest(30, "getting current user record", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil) + if err != nil { + return err + } + diag.verbosef("user uuid = %s", user.UUID) + return nil + }) + + if !cfgOK { + diag.errorf("cannot proceed without cluster config -- aborting without running any further tests") + return } // uncomment to create some spurious errors @@ -98,7 +251,7 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo // TODO: detect routing errors here, like finding wb2 at the // wb1 address. - for _, svc := range []*arvados.Service{ + for i, svc := range []*arvados.Service{ &cluster.Services.Keepproxy, &cluster.Services.WebDAV, &cluster.Services.WebDAVDownload, @@ -106,223 +259,554 @@ func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdo &cluster.Services.Workbench1, &cluster.Services.Workbench2, } { - testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL) - logger.Info(testname) - u := svc.ExternalURL - if strings.HasPrefix(u.Scheme, "ws") { - // We can do a real websocket test elsewhere, - // but for now we'll just check the https - // connection. - u.Scheme = "http" + u.Scheme[2:] - } - if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") { - u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:] - } - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { - errorf("%s: %s", testname, err) - continue - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - errorf("%s: %s", testname, err) - continue - } - resp.Body.Close() - infof("%s: ok", testname) + diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + u := svc.ExternalURL + if strings.HasPrefix(u.Scheme, "ws") { + // We can do a real websocket test elsewhere, + // but for now we'll just check the https + // connection. + u.Scheme = "http" + u.Scheme[2:] + } + if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") { + u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:] + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + resp.Body.Close() + return nil + }) } - for _, url := range []string{ + for i, url := range []string{ cluster.Services.Controller.ExternalURL.String(), cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0", cluster.Services.WebDAVDownload.ExternalURL.String(), } { - testname = fmt.Sprintf("checking CORS headers at %s", url) - logger.Info(testname) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - errorf("%s: %s", testname, err) - continue - } - req.Header.Set("Origin", "https://example.com") - resp, err := http.DefaultClient.Do(req) - if err != nil { - errorf("%s: %s", testname, err) - continue - } - if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" { - warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr) - } else { - infof("%s: ok", testname) - } + diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return err + } + req.Header.Set("Origin", "https://example.com") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" { + return fmt.Errorf("expected \"Access-Control-Allow-Origin: *\", got %q", hdr) + } + return nil + }) } var keeplist arvados.KeepServiceList - testname = "checking internal/external client detection" - logger.Info(testname) - err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1}) - if err != nil { - errorf("%s: error getting keep services list: %s", testname, err) - } else if len(keeplist.Items) == 0 { - errorf("%s: controller did not return any keep services", testname) - } else { + diag.dotest(60, "checking internal/external client detection", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999}) + if err != nil { + return fmt.Errorf("error getting keep services list: %s", err) + } else if len(keeplist.Items) == 0 { + return fmt.Errorf("controller did not return any keep services") + } found := map[string]int{} for _, ks := range keeplist.Items { found[ks.ServiceType]++ } - infof := infof isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0 isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items) - if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) { - infof = errorf - } if isExternal { - infof("%s: controller returned only proxy services, this host is considered \"external\"", testname) + diag.infof("controller returned only proxy services, this host is treated as \"external\"") } else if isInternal { - infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname) - } else { - errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found) + diag.infof("controller returned only non-proxy services, this host is treated as \"internal\"") + } + if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) { + return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal) + } + return nil + }) + + for i, ks := range keeplist.Items { + u := url.URL{ + Scheme: "http", + Host: net.JoinHostPort(ks.ServiceHost, fmt.Sprintf("%d", ks.ServicePort)), + Path: "/", } + if ks.ServiceSSLFlag { + u.Scheme = "https" + } + diag.dotest(61+i, fmt.Sprintf("reading+writing via keep service at %s", u.String()), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "PUT", u.String()+"d41d8cd98f00b204e9800998ecf8427e", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %s", err) + } + loc := strings.TrimSpace(string(body)) + if !strings.HasPrefix(loc, "d41d8") { + return fmt.Errorf("unexpected response from write: %q", body) + } + + req, err = http.NewRequestWithContext(ctx, "GET", u.String()+loc, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + resp, err = http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %s", err) + } + if len(body) != 0 { + return fmt.Errorf("unexpected response from read: %q", body) + } + + return nil + }) } var project arvados.Group - var grplist arvados.GroupList - testname = fmt.Sprintf("finding/creating %q project", diag.projectName) - logger.Info(testname) - err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{ - Filters: []arvados.Filter{ - {"name", "=", diag.projectName}, - {"group_class", "=", "project"}, - {"owner_uuid", "=", user.UUID}}, - Limit: -1}) - if err != nil { - errorf("%s: list groups: %s", testname, err) - } else if len(grplist.Items) < 1 { - infof("%s: list groups: ok, no results", testname) - err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{ + diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + var grplist arvados.GroupList + err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{ + Filters: []arvados.Filter{ + {"name", "=", diag.projectName}, + {"group_class", "=", "project"}, + {"owner_uuid", "=", user.UUID}}, + Limit: 999999}) + if err != nil { + return fmt.Errorf("list groups: %s", err) + } + if len(grplist.Items) > 0 { + project = grplist.Items[0] + diag.verbosef("using existing project, uuid = %s", project.UUID) + return nil + } + diag.debugf("list groups: ok, no results") + err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{ "name": diag.projectName, "group_class": "project", }}) if err != nil { - errorf("%s: create project: %s", testname, err) - } else { - infof("%s: created project, uuid = %s", testname, project.UUID) + return fmt.Errorf("create project: %s", err) } - } else { - project = grplist.Items[0] - infof("%s: ok, using existing project, uuid = %s", testname, project.UUID) - } + diag.verbosef("created project, uuid = %s", project.UUID) + return nil + }) - testname = "creating temporary collection" - logger.Info(testname) var collection arvados.Collection - err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ - "ensure_unique_name": true, - "collection": map[string]interface{}{ - "name": "test collection", - "trash_at": time.Now().Add(time.Hour)}}) + diag.dotest(90, "creating temporary collection", func() error { + if project.UUID == "" { + return fmt.Errorf("skipping, no project to work in") + } + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "ensure_unique_name": true, + "collection": map[string]interface{}{ + "owner_uuid": project.UUID, + "name": "test collection", + "trash_at": time.Now().Add(time.Hour)}}) + if err != nil { + return err + } + diag.verbosef("ok, uuid = %s", collection.UUID) + return nil + }) + + if collection.UUID != "" { + defer func() { + diag.dotest(9990, "deleting temporary collection", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil) + }) + }() + } + + tempdir, err := ioutil.TempDir("", "arvados-diagnostics") if err != nil { - errorf("%s: %s", testname, err) + diag.errorf("error creating temp dir: %s", err) + return + } + defer os.RemoveAll(tempdir) + + var dockerImageData []byte + if diag.dockerImage != "" || diag.priority < 1 { + // We won't be using the self-built docker image, so + // don't build it. But we will write the embedded + // "hello-world" image to our test collection to test + // upload/download, whether or not we're using it as a + // docker image. + dockerImageData = HelloWorldDockerImage + } else if selfbin, err := os.Readlink("/proc/self/exe"); err != nil { + diag.errorf("readlink /proc/self/exe: %s", err) + return + } else if selfbindata, err := os.ReadFile(selfbin); err != nil { + diag.errorf("error reading %s: %s", selfbin, err) + return } else { - infof("%s: ok, uuid = %s", testname, collection.UUID) - defer func() { - testname := "deleting temporary collection" - logger.Info(testname) - err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil) + selfbinSha := fmt.Sprintf("%x", sha256.Sum256(selfbindata)) + tag := "arvados-client-diagnostics:" + selfbinSha[:9] + err := os.WriteFile(tempdir+"/arvados-client", selfbindata, 0777) + if err != nil { + diag.errorf("error writing %s: %s", tempdir+"/arvados-client", err) + return + } + + dockerfile := "FROM " + diag.dockerImageFrom + "\n" + dockerfile += "RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends libfuse2 ca-certificates && apt-get clean\n" + dockerfile += "COPY /arvados-client /arvados-client\n" + cmd := exec.Command("docker", "build", "--tag", tag, "-f", "-", tempdir) + cmd.Stdin = strings.NewReader(dockerfile) + cmd.Stdout = diag.stderr + cmd.Stderr = diag.stderr + err = cmd.Run() + if err != nil { + diag.errorf("error building docker image: %s", err) + return + } + checkversion, err := exec.Command("docker", "run", tag, "/arvados-client", "version").CombinedOutput() + if err != nil { + diag.errorf("docker image does not seem to work: %s", err) + return + } + diag.infof("arvados-client version: %s", checkversion) + + buf, err := exec.Command("docker", "save", tag).Output() + if err != nil { + diag.errorf("docker save %s: %s", tag, err) + return + } + diag.infof("docker image size is %d", len(buf)) + dockerImageData = buf + } + + // Read image tarball to find image ID, so we can upload it as + // "sha256:{...}.tar" + var imageSHA2 string + { + tr := tar.NewReader(bytes.NewReader(dockerImageData)) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } if err != nil { - errorf("%s: %s", testname, err) - } else { - infof("%s: ok", testname) + diag.errorf("internal error/bug: cannot read docker image tar file: %s", err) + return } - }() + if s := strings.TrimSuffix(hdr.Name, ".json"); len(s) == 64 && s != hdr.Name { + imageSHA2 = s + } + } + if imageSHA2 == "" { + diag.errorf("internal error/bug: cannot find {sha256}.json file in docker image tar file") + return + } } + tarfilename := "sha256:" + imageSHA2 + ".tar" - testname = "uploading file via webdav" - logger.Info(testname) - func() { + diag.dotest(100, "uploading file via webdav", func() error { + timeout := diag.timeout + if len(dockerImageData) > 10<<20 && timeout < time.Minute { + // Extend the normal http timeout if we're + // uploading a substantial docker image. + timeout = time.Minute + } + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(timeout)) + defer cancel() if collection.UUID == "" { - infof("%s: skipping, no test collection", testname) - return + return fmt.Errorf("skipping, no test collection") } - req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata")) + t0 := time.Now() + req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/"+tarfilename, bytes.NewReader(dockerImageData)) if err != nil { - errorf("%s: BUG? http.NewRequest: %s", testname, err) - return + return fmt.Errorf("BUG? http.NewRequest: %s", err) } req.Header.Set("Authorization", "Bearer "+client.AuthToken) resp, err := http.DefaultClient.Do(req) if err != nil { - errorf("%s: error performing http request: %s", testname, err) - return + return fmt.Errorf("error performing http request: %s", err) } resp.Body.Close() if resp.StatusCode != http.StatusCreated { - errorf("%s: status %s", testname, resp.Status) - return + return fmt.Errorf("status %s", resp.Status) } - infof("%s: ok, status %s", testname, resp.Status) - err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil) + diag.verbosef("upload ok, status %s, %f MB/s", resp.Status, float64(len(dockerImageData))/time.Since(t0).Seconds()/1000000) + err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil) if err != nil { - errorf("%s: get updated collection: %s", testname, err) - return + return fmt.Errorf("get updated collection: %s", err) } - infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash) - }() + diag.verbosef("upload pdh %s", collection.PortableDataHash) + return nil + }) davurl := cluster.Services.WebDAV.ExternalURL - testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl) - logger.Info(testname) - if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") { - infof("%s: looks ok", testname) - } else if davurl.Host == "" { - warnf("%s: host missing - content previews will not work", testname) - } else { - warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname) - } + davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") + diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error { + if davurl.Host == "" { + return fmt.Errorf("host missing - content previews will not work") + } + if !davWildcard && !cluster.Collections.TrustAllContent { + diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work") + } + return nil + }) - for _, trial := range []struct { - status int - fileurl string + for i, trial := range []struct { + needcoll bool + needWildcard bool + status int + fileurl string }{ - {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, - {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"}, - {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, - {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"}, - {http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"}, - {http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"}, + {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, + {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + tarfilename}, + {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, + {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/" + tarfilename}, + {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + tarfilename}, + {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/" + tarfilename}, } { - func() { - testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl) - logger.Info(testname) - if collection.UUID == "" { - errorf("%s: skipping, no test collection", testname) - return + diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error { + if trial.needWildcard && !davWildcard { + diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard") + return nil } - req, err := http.NewRequest("GET", trial.fileurl, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + if trial.needcoll && collection.UUID == "" { + return fmt.Errorf("skipping, no test collection") + } + req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil) if err != nil { - errorf("%s: %s", testname, err) - return + return err } req.Header.Set("Authorization", "Bearer "+client.AuthToken) resp, err := http.DefaultClient.Do(req) if err != nil { - errorf("%s: %s", testname, err) - return + return err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - errorf("%s: error reading response: %s", testname, err) + return fmt.Errorf("reading response: %s", err) } if resp.StatusCode != trial.status { - errorf("%s: unexpected response status: %s", testname, resp.Status) - } else if trial.status == http.StatusOK && string(body) != "testfiledata" { - errorf("%s: unexpected response content: %q", testname, body) - } else { - infof("%s: ok", testname) + return fmt.Errorf("unexpected response status: %s", resp.Status) } - }() + if trial.status == http.StatusOK && !bytes.Equal(body, dockerImageData) { + excerpt := body + if len(excerpt) > 128 { + excerpt = append([]byte(nil), body[:128]...) + excerpt = append(excerpt, []byte("[...]")...) + } + return fmt.Errorf("unexpected response content: len %d, %q", len(body), excerpt) + } + return nil + }) } - return 0 + var vm arvados.VirtualMachine + diag.dotest(130, "getting list of virtual machines", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + var vmlist arvados.VirtualMachineList + err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999}) + if err != nil { + return err + } + if len(vmlist.Items) < 1 { + diag.warnf("no VMs found") + } else { + vm = vmlist.Items[0] + } + return nil + }) + + diag.dotest(150, "connecting to webshell service", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + if vm.UUID == "" { + diag.warnf("skipping, no vm available") + return nil + } + u := cluster.Services.WebShell.ExternalURL + webshellurl := u.String() + vm.Hostname + "?" + if strings.HasPrefix(u.Host, "*") { + u.Host = vm.Hostname + u.Host[1:] + webshellurl = u.String() + "?" + } + diag.debugf("url %s", webshellurl) + req, err := http.NewRequestWithContext(ctx, "POST", webshellurl, bytes.NewBufferString(url.Values{ + "width": {"80"}, + "height": {"25"}, + "session": {"xyzzy"}, + "rooturl": {webshellurl}, + }.Encode())) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + diag.debugf("response status %s", resp.Status) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %s", err) + } + diag.debugf("response body %q", body) + // We don't speak the protocol, so we get a 400 error + // from the webshell server even if everything is + // OK. Anything else (404, 502, ???) indicates a + // problem. + if resp.StatusCode != http.StatusBadRequest { + return fmt.Errorf("unexpected response status: %s, %q", resp.Status, body) + } + return nil + }) + + diag.dotest(160, "running a container", func() error { + if diag.priority < 1 { + diag.infof("skipping (use priority > 0 if you want to run a container)") + return nil + } + if project.UUID == "" { + return fmt.Errorf("skipping, no project to work in") + } + + timestamp := time.Now().Format(time.RFC3339) + + var ctrCommand []string + switch diag.dockerImage { + case "": + if collection.UUID == "" { + return fmt.Errorf("skipping, no test collection to use as docker image") + } + diag.dockerImage = collection.PortableDataHash + ctrCommand = []string{"/arvados-client", "diagnostics", + "-priority=0", // don't run a container + "-log-level=" + diag.logLevel, + "-internal-client=true"} + case "hello-world": + if collection.UUID == "" { + return fmt.Errorf("skipping, no test collection to use as docker image") + } + diag.dockerImage = collection.PortableDataHash + ctrCommand = []string{"/hello"} + default: + ctrCommand = []string{"echo", timestamp} + } + + var cr arvados.ContainerRequest + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + + err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{ + "owner_uuid": project.UUID, + "name": fmt.Sprintf("diagnostics container request %s", timestamp), + "container_image": diag.dockerImage, + "command": ctrCommand, + "use_existing": false, + "output_path": "/mnt/output", + "output_name": fmt.Sprintf("diagnostics output %s", timestamp), + "priority": diag.priority, + "state": arvados.ContainerRequestStateCommitted, + "mounts": map[string]map[string]interface{}{ + "/mnt/output": { + "kind": "collection", + "writable": true, + }, + }, + "runtime_constraints": arvados.RuntimeConstraints{ + API: true, + VCPUs: 1, + RAM: 128 << 20, + KeepCacheRAM: 64 << 20, + }, + }}) + if err != nil { + return err + } + diag.infof("container request uuid = %s", cr.UUID) + diag.verbosef("container uuid = %s", cr.ContainerUUID) + + timeout := 10 * time.Minute + diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout)) + deadline := time.Now().Add(timeout) + + var c arvados.Container + for ; cr.State != arvados.ContainerRequestStateFinal && time.Now().Before(deadline); time.Sleep(2 * time.Second) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + + crStateWas := cr.State + err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil) + if err != nil { + return err + } + if cr.State != crStateWas { + diag.debugf("container request state = %s", cr.State) + } + + cStateWas := c.State + err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil) + if err != nil { + return err + } + if c.State != cStateWas { + diag.debugf("container state = %s", c.State) + } + + cancel() + } + + if cr.State != arvados.ContainerRequestStateFinal { + err := client.RequestAndDecodeContext(context.Background(), &cr, "PATCH", "arvados/v1/container_requests/"+cr.UUID, nil, map[string]interface{}{ + "container_request": map[string]interface{}{ + "priority": 0, + }}) + if err != nil { + diag.infof("error canceling container request %s: %s", cr.UUID, err) + } else { + diag.debugf("canceled container request %s", cr.UUID) + } + return fmt.Errorf("timed out waiting for container to finish; container request %s state was %q, container %s state was %q", cr.UUID, cr.State, c.UUID, c.State) + } + if c.State != arvados.ContainerStateComplete { + return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State) + } + if c.ExitCode != 0 { + return fmt.Errorf("container exited %d", c.ExitCode) + } + return nil + }) }