From 751cdf33d5519c393323baa15233ca55b7c7a9a1 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Mon, 31 May 2021 16:52:36 -0400 Subject: [PATCH] 17609: Add diagnostics command. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- cmd/arvados-client/cmd.go | 2 + lib/cmd/cmd.go | 8 + lib/costanalyzer/cmd.go | 11 +- lib/diagnostics/cmd.go | 328 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+), 9 deletions(-) create mode 100644 lib/diagnostics/cmd.go diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go index aefcce79a4..cb15462119 100644 --- a/cmd/arvados-client/cmd.go +++ b/cmd/arvados-client/cmd.go @@ -11,6 +11,7 @@ import ( "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/lib/costanalyzer" "git.arvados.org/arvados.git/lib/deduplicationreport" + "git.arvados.org/arvados.git/lib/diagnostics" "git.arvados.org/arvados.git/lib/mount" ) @@ -59,6 +60,7 @@ var ( "costanalyzer": costanalyzer.Command, "shell": shellCommand{}, "connect-ssh": connectSSHCommand{}, + "diagnostics": diagnostics.Command{}, }) ) diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go index b7d918739b..63d7576b4c 100644 --- a/lib/cmd/cmd.go +++ b/lib/cmd/cmd.go @@ -16,6 +16,8 @@ import ( "runtime" "sort" "strings" + + "github.com/sirupsen/logrus" ) type Handler interface { @@ -153,3 +155,9 @@ func SubcommandToFront(args []string, flagset FlagSet) []string { copy(newargs[flagargs+1:], args[flagargs+1:]) return newargs } + +type NoPrefixFormatter struct{} + +func (NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) { + return []byte(entry.Message + "\n"), nil +} diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go index 9b0685225b..633e95e29f 100644 --- a/lib/costanalyzer/cmd.go +++ b/lib/costanalyzer/cmd.go @@ -7,33 +7,26 @@ package costanalyzer import ( "io" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/ctxlog" - "github.com/sirupsen/logrus" ) var Command command type command struct{} -type NoPrefixFormatter struct{} - -func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) { - return []byte(entry.Message), nil -} - // RunCommand implements the subcommand "costanalyzer ..." func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { var err error logger := ctxlog.New(stderr, "text", "info") + logger.SetFormatter(cmd.NoPrefixFormatter{}) defer func() { if err != nil { logger.Error("\n" + err.Error() + "\n") } }() - logger.SetFormatter(new(NoPrefixFormatter)) - loader := config.NewLoader(stdin, logger) loader.SkipLegacy = true diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go new file mode 100644 index 0000000000..26bbdfe1d3 --- /dev/null +++ b/lib/diagnostics/cmd.go @@ -0,0 +1,328 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package diagnostics + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + "time" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/sirupsen/logrus" +) + +type Command struct { + projectName string +} + +func (diag Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + f := flag.NewFlagSet(prog, flag.ContinueOnError) + f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects") + loglevel := f.String("log-level", "info", "logging level (debug, info, warning, error)") + checkInternal := f.Bool("internal-client", false, "check that this host is considered an \"internal\" client") + checkExternal := f.Bool("external-client", false, "check that this host is considered an \"external\" client") + err := f.Parse(args) + if err == flag.ErrHelp { + return 0 + } else if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + logger := ctxlog.New(stdout, "text", *loglevel) + logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}) + + infof := logger.Infof + warnf := logger.Warnf + var errors []string + errorf := func(f string, args ...interface{}) { + logger.Errorf(f, args...) + errors = append(errors, fmt.Sprintf(f, args...)) + } + defer func() { + if len(errors) == 0 { + logger.Info("--- no errors ---") + } else { + fmt.Fprint(stdout, "\n--- cut here --- error summary ---\n\n") + for _, e := range errors { + logger.Error(e) + } + } + }() + + client := arvados.NewClientFromEnv() + + var dd arvados.DiscoveryDocument + ddpath := "discovery/v1/apis/arvados/v1/rest" + testname := fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath) + logger.Info(testname) + err = client.RequestAndDecode(&dd, "GET", ddpath, nil, nil) + if err != nil { + errorf("%s: %s", testname, err) + } else { + infof("%s: ok, BlobSignatureTTL is %d", testname, dd.BlobSignatureTTL) + } + + var cluster arvados.Cluster + cfgpath := "arvados/v1/config" + testname = fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath) + logger.Info(testname) + err = client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil) + if err != nil { + errorf("%s: %s", testname, err) + } else { + infof("%s: ok, Collections.BlobSigning = %v", testname, cluster.Collections.BlobSigning) + } + + var user arvados.User + testname = "getting current user record" + logger.Info(testname) + err = client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil) + if err != nil { + errorf("%s: %s", testname, err) + return 2 + } else { + infof("%s: ok, uuid = %s", testname, user.UUID) + } + + // uncomment to create some spurious errors + // cluster.Services.WebDAVDownload.ExternalURL.Host = "0.0.0.0:9" + + // TODO: detect routing errors here, like finding wb2 at the + // wb1 address. + for _, svc := range []*arvados.Service{ + &cluster.Services.Keepproxy, + &cluster.Services.WebDAV, + &cluster.Services.WebDAVDownload, + &cluster.Services.Websocket, + &cluster.Services.Workbench1, + &cluster.Services.Workbench2, + } { + testname = fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL) + logger.Info(testname) + u := svc.ExternalURL + if strings.HasPrefix(u.Scheme, "ws") { + // We can do a real websocket test elsewhere, + // but for now we'll just check the https + // connection. + u.Scheme = "http" + u.Scheme[2:] + } + if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") { + u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:] + } + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + errorf("%s: %s", testname, err) + continue + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + errorf("%s: %s", testname, err) + continue + } + resp.Body.Close() + infof("%s: ok", testname) + } + + for _, url := range []string{ + cluster.Services.Controller.ExternalURL.String(), + cluster.Services.Keepproxy.ExternalURL.String() + "d41d8cd98f00b204e9800998ecf8427e+0", + cluster.Services.WebDAVDownload.ExternalURL.String(), + } { + testname = fmt.Sprintf("checking CORS headers at %s", url) + logger.Info(testname) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + errorf("%s: %s", testname, err) + continue + } + req.Header.Set("Origin", "https://example.com") + resp, err := http.DefaultClient.Do(req) + if err != nil { + errorf("%s: %s", testname, err) + continue + } + if hdr := resp.Header.Get("Access-Control-Allow-Origin"); hdr != "*" { + warnf("%s: expected \"Access-Control-Allow-Origin: *\", got %q", testname, hdr) + } else { + infof("%s: ok", testname) + } + } + + var keeplist arvados.KeepServiceList + testname = "checking internal/external client detection" + logger.Info(testname) + err = client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1}) + if err != nil { + errorf("%s: error getting keep services list: %s", testname, err) + } else if len(keeplist.Items) == 0 { + errorf("%s: controller did not return any keep services", testname) + } else { + found := map[string]int{} + for _, ks := range keeplist.Items { + found[ks.ServiceType]++ + } + infof := infof + isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0 + isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items) + if (*checkInternal && !isInternal) || (*checkExternal && !isExternal) { + infof = errorf + } + if isExternal { + infof("%s: controller returned only proxy services, this host is considered \"external\"", testname) + } else if isInternal { + infof("%s: controller returned only non-proxy services, this host is considered \"internal\"", testname) + } else { + errorf("%s: controller returned both proxy and non-proxy services: %v", testname, found) + } + } + + var project arvados.Group + var grplist arvados.GroupList + testname = fmt.Sprintf("finding/creating %q project", diag.projectName) + logger.Info(testname) + err = client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{ + Filters: []arvados.Filter{ + {"name", "=", diag.projectName}, + {"group_class", "=", "project"}, + {"owner_uuid", "=", user.UUID}}, + Limit: -1}) + if err != nil { + errorf("%s: list groups: %s", testname, err) + } else if len(grplist.Items) < 1 { + infof("%s: list groups: ok, no results", testname) + err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{ + "name": diag.projectName, + "group_class": "project", + }}) + if err != nil { + errorf("%s: create project: %s", testname, err) + } else { + infof("%s: created project, uuid = %s", testname, project.UUID) + } + } else { + project = grplist.Items[0] + infof("%s: ok, using existing project, uuid = %s", testname, project.UUID) + } + + testname = "creating temporary collection" + logger.Info(testname) + var collection arvados.Collection + err = client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "ensure_unique_name": true, + "collection": map[string]interface{}{ + "name": "test collection", + "trash_at": time.Now().Add(time.Hour)}}) + if err != nil { + errorf("%s: %s", testname, err) + } else { + infof("%s: ok, uuid = %s", testname, collection.UUID) + defer func() { + testname := "deleting temporary collection" + logger.Info(testname) + err := client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil) + if err != nil { + errorf("%s: %s", testname, err) + } else { + infof("%s: ok", testname) + } + }() + } + + testname = "uploading file via webdav" + logger.Info(testname) + func() { + if collection.UUID == "" { + infof("%s: skipping, no test collection", testname) + return + } + req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata")) + if err != nil { + errorf("%s: BUG? http.NewRequest: %s", testname, err) + return + } + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + errorf("%s: error performing http request: %s", testname, err) + return + } + resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + errorf("%s: status %s", testname, resp.Status) + return + } + infof("%s: ok, status %s", testname, resp.Status) + err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil) + if err != nil { + errorf("%s: get updated collection: %s", testname, err) + return + } + infof("%s: get updated collection: ok, pdh %s", testname, collection.PortableDataHash) + }() + + davurl := cluster.Services.WebDAV.ExternalURL + testname = fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl) + logger.Info(testname) + if strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") { + infof("%s: looks ok", testname) + } else if davurl.Host == "" { + warnf("%s: host missing - content previews will not work", testname) + } else { + warnf("%s: host has no leading wildcard - content previews will not work unless TrustAllContent==true", testname) + } + + for _, trial := range []struct { + status int + fileurl string + }{ + {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, + {http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"}, + {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, + {http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"}, + {http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"}, + {http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"}, + } { + func() { + testname := fmt.Sprintf("downloading from webdav (%s)", trial.fileurl) + logger.Info(testname) + if collection.UUID == "" { + errorf("%s: skipping, no test collection", testname) + return + } + req, err := http.NewRequest("GET", trial.fileurl, nil) + if err != nil { + errorf("%s: %s", testname, err) + return + } + req.Header.Set("Authorization", "Bearer "+client.AuthToken) + resp, err := http.DefaultClient.Do(req) + if err != nil { + errorf("%s: %s", testname, err) + return + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + errorf("%s: error reading response: %s", testname, err) + } + if resp.StatusCode != trial.status { + errorf("%s: unexpected response status: %s", testname, resp.Status) + } else if trial.status == http.StatusOK && string(body) != "testfiledata" { + errorf("%s: unexpected response content: %q", testname, body) + } else { + infof("%s: ok", testname) + } + }() + } + + return 0 +} -- 2.30.2