"strings"
"time"
+ "git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/sirupsen/logrus"
type Command struct{}
-func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var diag diagnoser
f := flag.NewFlagSet(prog, flag.ContinueOnError)
f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
+ f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests")
- err := f.Parse(args)
- if err == flag.ErrHelp {
- return 0
- } else if err != nil {
- fmt.Fprintln(stderr, err)
- return 2
+ if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok {
+ return code
}
diag.logger = ctxlog.New(stdout, "text", diag.logLevel)
- diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true})
+ diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true})
diag.runtests()
if len(diag.errors) == 0 {
diag.logger.Info("--- no errors ---")
stdout io.Writer
stderr io.Writer
logLevel string
+ priority int
projectName string
checkInternal bool
checkExternal bool
}
func (diag *diagnoser) debugf(f string, args ...interface{}) {
- diag.logger.Debugf(f, args...)
+ diag.logger.Debugf(" ... "+f, args...)
}
func (diag *diagnoser) infof(f string, args ...interface{}) {
- diag.logger.Infof(f, args...)
+ diag.logger.Infof(" ... "+f, args...)
}
func (diag *diagnoser) warnf(f string, args ...interface{}) {
- diag.logger.Warnf(f, args...)
+ diag.logger.Warnf(" ... "+f, args...)
}
func (diag *diagnoser) errorf(f string, args ...interface{}) {
}
diag.done[id] = true
- diag.infof("%d %s", id, title)
+ diag.logger.Infof("%4d: %s", id, title)
t0 := time.Now()
err := fn()
- elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond)
+ elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond)
if err != nil {
- diag.errorf("%s (%s): %s", title, elapsed, err)
+ diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err)
+ } else {
+ diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed)
}
- diag.debugf("%d %s (%s): ok", id, title, elapsed)
}
func (diag *diagnoser) runtests() {
var dd arvados.DiscoveryDocument
ddpath := "discovery/v1/apis/arvados/v1/rest"
diag.dotest(10, fmt.Sprintf("getting discovery document from https://%s/%s", client.APIHost, ddpath), func() error {
- err := client.RequestAndDecode(&dd, "GET", ddpath, nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &dd, "GET", ddpath, nil, nil)
if err != nil {
return err
}
var cluster arvados.Cluster
cfgpath := "arvados/v1/config"
diag.dotest(20, fmt.Sprintf("getting exported config from https://%s/%s", client.APIHost, cfgpath), func() error {
- err := client.RequestAndDecode(&cluster, "GET", cfgpath, nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &cluster, "GET", cfgpath, nil, nil)
if err != nil {
return err
}
var user arvados.User
diag.dotest(30, "getting current user record", func() error {
- err := client.RequestAndDecode(&user, "GET", "arvados/v1/users/current", nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
if err != nil {
return err
}
&cluster.Services.Workbench2,
} {
diag.dotest(40+i, fmt.Sprintf("connecting to service endpoint %s", svc.ExternalURL), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
u := svc.ExternalURL
if strings.HasPrefix(u.Scheme, "ws") {
// We can do a real websocket test elsewhere,
if svc == &cluster.Services.WebDAV && strings.HasPrefix(u.Host, "*") {
u.Host = "d41d8cd98f00b204e9800998ecf8427e-0" + u.Host[1:]
}
- req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return err
}
cluster.Services.WebDAVDownload.ExternalURL.String(),
} {
diag.dotest(50+i, fmt.Sprintf("checking CORS headers at %s", url), func() error {
- req, err := http.NewRequest("GET", url, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return err
}
var keeplist arvados.KeepServiceList
diag.dotest(60, "checking internal/external client detection", func() error {
- err := client.RequestAndDecode(&keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: -1})
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &keeplist, "GET", "arvados/v1/keep_services/accessible", nil, arvados.ListOptions{Limit: 999999})
if err != nil {
return fmt.Errorf("error getting keep services list: %s", err)
} else if len(keeplist.Items) == 0 {
var project arvados.Group
diag.dotest(80, fmt.Sprintf("finding/creating %q project", diag.projectName), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
var grplist arvados.GroupList
- err := client.RequestAndDecode(&grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
+ err := client.RequestAndDecodeContext(ctx, &grplist, "GET", "arvados/v1/groups", nil, arvados.ListOptions{
Filters: []arvados.Filter{
{"name", "=", diag.projectName},
{"group_class", "=", "project"},
{"owner_uuid", "=", user.UUID}},
- Limit: -1})
+ Limit: 999999})
if err != nil {
return fmt.Errorf("list groups: %s", err)
}
return nil
}
diag.debugf("list groups: ok, no results")
- err = client.RequestAndDecode(&project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
+ err = client.RequestAndDecodeContext(ctx, &project, "POST", "arvados/v1/groups", nil, map[string]interface{}{"group": map[string]interface{}{
"name": diag.projectName,
"group_class": "project",
}})
var collection arvados.Collection
diag.dotest(90, "creating temporary collection", func() error {
- err := client.RequestAndDecode(&collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ if project.UUID == "" {
+ return fmt.Errorf("skipping, no project to work in")
+ }
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
"ensure_unique_name": true,
"collection": map[string]interface{}{
- "name": "test collection",
- "trash_at": time.Now().Add(time.Hour)}})
+ "owner_uuid": project.UUID,
+ "name": "test collection",
+ "trash_at": time.Now().Add(time.Hour)}})
if err != nil {
return err
}
if collection.UUID != "" {
defer func() {
diag.dotest(9990, "deleting temporary collection", func() error {
- return client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+ return client.RequestAndDecodeContext(ctx, nil, "DELETE", "arvados/v1/collections/"+collection.UUID, nil, nil)
})
}()
}
diag.dotest(100, "uploading file via webdav", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
if collection.UUID == "" {
return fmt.Errorf("skipping, no test collection")
}
- req, err := http.NewRequest("PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
+ req, err := http.NewRequestWithContext(ctx, "PUT", cluster.Services.WebDAVDownload.ExternalURL.String()+"c="+collection.UUID+"/testfile", bytes.NewBufferString("testfiledata"))
if err != nil {
return fmt.Errorf("BUG? http.NewRequest: %s", err)
}
return fmt.Errorf("status %s", resp.Status)
}
diag.debugf("ok, status %s", resp.Status)
- err = client.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
+ err = client.RequestAndDecodeContext(ctx, &collection, "GET", "arvados/v1/collections/"+collection.UUID, nil, nil)
if err != nil {
return fmt.Errorf("get updated collection: %s", err)
}
{true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
} {
diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
if trial.needcoll && collection.UUID == "" {
return fmt.Errorf("skipping, no test collection")
}
- req, err := http.NewRequest("GET", trial.fileurl, nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", trial.fileurl, nil)
if err != nil {
return err
}
var vm arvados.VirtualMachine
diag.dotest(130, "getting list of virtual machines", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
var vmlist arvados.VirtualMachineList
- err := client.RequestAndDecode(&vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
+ err := client.RequestAndDecodeContext(ctx, &vmlist, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{Limit: 999999})
if err != nil {
return err
}
})
diag.dotest(140, "getting workbench1 webshell page", func() error {
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
if vm.UUID == "" {
return fmt.Errorf("skipping, no vm available")
}
webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
diag.debugf("url %s", webshelltermurl)
- req, err := http.NewRequest("GET", webshelltermurl, nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", webshelltermurl, nil)
if err != nil {
return err
}
}
return nil
})
+
+ diag.dotest(160, "running a container", func() error {
+ if diag.priority < 1 {
+ diag.infof("skipping (use priority > 0 if you want to run a container)")
+ return nil
+ }
+ if project.UUID == "" {
+ return fmt.Errorf("skipping, no project to work in")
+ }
+
+ var cr arvados.ContainerRequest
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
+ defer cancel()
+
+ timestamp := time.Now().Format(time.RFC3339)
+ err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
+ "owner_uuid": project.UUID,
+ "name": fmt.Sprintf("diagnostics container request %s", timestamp),
+ "container_image": "arvados/jobs",
+ "command": []string{"echo", timestamp},
+ "use_existing": false,
+ "output_path": "/mnt/output",
+ "output_name": fmt.Sprintf("diagnostics output %s", timestamp),
+ "priority": diag.priority,
+ "state": arvados.ContainerRequestStateCommitted,
+ "mounts": map[string]map[string]interface{}{
+ "/mnt/output": {
+ "kind": "collection",
+ "writable": true,
+ },
+ },
+ "runtime_constraints": arvados.RuntimeConstraints{
+ VCPUs: 1,
+ RAM: 1 << 26,
+ KeepCacheRAM: 1 << 26,
+ },
+ }})
+ if err != nil {
+ return err
+ }
+ diag.debugf("container request uuid = %s", cr.UUID)
+ diag.debugf("container uuid = %s", cr.ContainerUUID)
+
+ timeout := 10 * time.Minute
+ diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
+ ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
+ defer cancel()
+
+ var c arvados.Container
+ for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
+ ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
+ defer cancel()
+
+ crStateWas := cr.State
+ err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil)
+ if err != nil {
+ return err
+ }
+ if cr.State != crStateWas {
+ diag.debugf("container request state = %s", cr.State)
+ }
+
+ cStateWas := c.State
+ err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil)
+ if err != nil {
+ return err
+ }
+ if c.State != cStateWas {
+ diag.debugf("container state = %s", c.State)
+ }
+ }
+
+ if c.State != arvados.ContainerStateComplete {
+ return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
+ } else if c.ExitCode != 0 {
+ return fmt.Errorf("container exited %d", c.ExitCode)
+ }
+ return nil
+ })
}
projects / arvados.git / blobdiff