X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/6560ade0a1fca34182814e0881b8b543216ad328..37b0b06bd0ed9250f3221ad50ef6a83e3a342a17:/lib/diagnostics/cmd.go diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go index 7021bb0ee3..6dc5f679b5 100644 --- a/lib/diagnostics/cmd.go +++ b/lib/diagnostics/cmd.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/sirupsen/logrus" @@ -24,23 +25,21 @@ import ( type Command struct{} -func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { var diag diagnoser f := flag.NewFlagSet(prog, flag.ContinueOnError) f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects") f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)") + f.StringVar(&diag.dockerImage, "docker-image", "alpine:latest", "image to use when running a test container") f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client") f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client") + f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)") f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests") - err := f.Parse(args) - if err == flag.ErrHelp { - return 0 - } else if err != nil { - fmt.Fprintln(stderr, err) - return 2 + if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok { + return code } diag.logger = ctxlog.New(stdout, "text", diag.logLevel) - diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}) + diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true}) diag.runtests() if len(diag.errors) == 0 { diag.logger.Info("--- no errors ---") @@ -60,7 +59,9 @@ type diagnoser struct { stdout io.Writer stderr io.Writer logLevel string + priority int projectName string + dockerImage string checkInternal bool checkExternal bool timeout time.Duration @@ -70,15 +71,15 @@ type diagnoser struct { } func (diag *diagnoser) debugf(f string, args ...interface{}) { - diag.logger.Debugf(f, args...) + diag.logger.Debugf(" ... "+f, args...) } func (diag *diagnoser) infof(f string, args ...interface{}) { - diag.logger.Infof(f, args...) + diag.logger.Infof(" ... "+f, args...) } func (diag *diagnoser) warnf(f string, args ...interface{}) { - diag.logger.Warnf(f, args...) + diag.logger.Warnf(" ... "+f, args...) } func (diag *diagnoser) errorf(f string, args ...interface{}) { @@ -99,14 +100,15 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) { } diag.done[id] = true - diag.infof("%d %s", id, title) + diag.logger.Infof("%4d: %s", id, title) t0 := time.Now() err := fn() - elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond) + elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond) if err != nil { - diag.errorf("%s (%s): %s", title, elapsed, err) + diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err) + } else { + diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed) } - diag.debugf("%d %s (%s): ok", id, title, elapsed) } func (diag *diagnoser) runtests() { @@ -120,7 +122,9 @@ func (diag *diagnoser) runtests() { var dd arvados.DiscoveryDocument ddpath := "discovery/v1/apis/arvados/v1/rest" diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error { - err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil) if err != nil { return err } @@ -131,7 +135,9 @@ func (diag *diagnoser) runtests() { var cluster arvados.Cluster cfgpath := "arvados/v1/config" diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error { - err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil) if err != nil { return err } @@ -141,7 +147,9 @@ func (diag *diagnoser) runtests() { var user arvados.User diag.dotest(30, "getting current user record", func() error { - err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil) if err != nil { return err } @@ -163,6 +171,8 @@ func (diag *diagnoser) runtests() { &cluster.Services.Workbench2, } { diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() u := svc.ExternalURL if strings.HasPrefix(u.Scheme, "ws") { // We can do a real websocket test elsewhere, @@ -173,7 +183,7 @@ func (diag *diagnoser) runtests() { if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") { u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:] } - req, err := http.NewRequest(http.MethodGet, u.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } @@ -192,7 +202,9 @@ func (diag *diagnoser) runtests() { cluster.Services.WebDAVDownload.ExternalURL.String(), } { diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error { - req, err := http.NewRequest("GET", url, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return err } @@ -210,7 +222,9 @@ func (diag *diagnoser) runtests() { var keeplist arvados.KeepServiceList diag.dotest(60, "checking internal/external client detection", func() error { - err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999}) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999}) if err != nil { return fmt.Errorf("error getting keep services list: %s", err) } else if len(keeplist.Items) == 0 { @@ -288,8 +302,10 @@ func (diag *diagnoser) runtests() { var project arvados.Group diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() var grplist arvados.GroupList - err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{ + err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{ Filters: []arvados.Filter{ {"name", "=", diag.projectName}, {"group_class", "=", "project"}, @@ -304,7 +320,7 @@ func (diag *diagnoser) runtests() { return nil } diag.debugf("list groups: ok, no results") - err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{ + err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{ "name": diag.projectName, "group_class": "project", }}) @@ -317,11 +333,17 @@ func (diag *diagnoser) runtests() { var collection arvados.Collection diag.dotest(90, "creating temporary collection", func() error { - err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + if project.UUID == "" { + return fmt.Errorf("skipping, no project to work in") + } + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ "ensure_unique_name": true, "collection": map[string]interface{}{ - "name": "test collection", - "trash_at": time.Now().Add(time.Hour)}}) + "owner_uuid": project.UUID, + "name": "test collection", + "trash_at": time.Now().Add(time.Hour)}}) if err != nil { return err } @@ -332,16 +354,20 @@ func (diag *diagnoser) runtests() { if collection.UUID != "" { defer func() { diag.dotest(9990, "deleting temporary collection", func() error { - return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil) }) }() } diag.dotest(100, "uploading file via webdav", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() if collection.UUID == "" { return fmt.Errorf("skipping, no test collection") } - req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata")) + req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata")) if err != nil { return fmt.Errorf("BUG? http.NewRequest: %s", err) } @@ -355,7 +381,7 @@ func (diag *diagnoser) runtests() { return fmt.Errorf("status %s", resp.Status) } diag.debugf("ok, status %s", resp.Status) - err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil) + err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil) if err != nil { return fmt.Errorf("get updated collection: %s", err) } @@ -364,33 +390,41 @@ func (diag *diagnoser) runtests() { }) davurl := cluster.Services.WebDAV.ExternalURL + davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.") diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error { if davurl.Host == "" { return fmt.Errorf("host missing - content previews will not work") } - if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent { + if !davWildcard && !cluster.Collections.TrustAllContent { diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work") } return nil }) for i, trial := range []struct { - needcoll bool - status int - fileurl string + needcoll bool + needWildcard bool + status int + fileurl string }{ - {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, - {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"}, - {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, - {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"}, - {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"}, - {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"}, + {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"}, + {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"}, + {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"}, + {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"}, + {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"}, + {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"}, } { diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error { + if trial.needWildcard && !davWildcard { + diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard") + return nil + } + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() if trial.needcoll && collection.UUID == "" { return fmt.Errorf("skipping, no test collection") } - req, err := http.NewRequest("GET", trial.fileurl, nil) + req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil) if err != nil { return err } @@ -416,8 +450,10 @@ func (diag *diagnoser) runtests() { var vm arvados.VirtualMachine diag.dotest(130, "getting list of virtual machines", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() var vmlist arvados.VirtualMachineList - err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999}) + err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999}) if err != nil { return err } @@ -429,12 +465,14 @@ func (diag *diagnoser) runtests() { }) diag.dotest(140, "getting workbench1 webshell page", func() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() if vm.UUID == "" { return fmt.Errorf("skipping, no vm available") } webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername" diag.debugf("url %s", webshelltermurl) - req, err := http.NewRequest("GET", webshelltermurl, nil) + req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil) if err != nil { return err } @@ -497,4 +535,83 @@ func (diag *diagnoser) runtests() { } return nil }) + + diag.dotest(160, "running a container", func() error { + if diag.priority < 1 { + diag.infof("skipping (use priority > 0 if you want to run a container)") + return nil + } + if project.UUID == "" { + return fmt.Errorf("skipping, no project to work in") + } + + var cr arvados.ContainerRequest + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + + timestamp := time.Now().Format(time.RFC3339) + err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{ + "owner_uuid": project.UUID, + "name": fmt.Sprintf("diagnostics container request %s", timestamp), + "container_image": diag.dockerImage, + "command": []string{"echo", timestamp}, + "use_existing": false, + "output_path": "/mnt/output", + "output_name": fmt.Sprintf("diagnostics output %s", timestamp), + "priority": diag.priority, + "state": arvados.ContainerRequestStateCommitted, + "mounts": map[string]map[string]interface{}{ + "/mnt/output": { + "kind": "collection", + "writable": true, + }, + }, + "runtime_constraints": arvados.RuntimeConstraints{ + VCPUs: 1, + RAM: 1 << 26, + KeepCacheRAM: 1 << 26, + }, + }}) + if err != nil { + return err + } + diag.debugf("container request uuid = %s", cr.UUID) + diag.debugf("container uuid = %s", cr.ContainerUUID) + + timeout := 10 * time.Minute + diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout)) + ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout)) + defer cancel() + + var c arvados.Container + for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) { + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout)) + defer cancel() + + crStateWas := cr.State + err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil) + if err != nil { + return err + } + if cr.State != crStateWas { + diag.debugf("container request state = %s", cr.State) + } + + cStateWas := c.State + err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil) + if err != nil { + return err + } + if c.State != cStateWas { + diag.debugf("container state = %s", c.State) + } + } + + if c.State != arvados.ContainerStateComplete { + return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State) + } else if c.ExitCode != 0 { + return fmt.Errorf("container exited %d", c.ExitCode) + } + return nil + }) }