X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dd7268b61d18f37036f40479c67b4b9f446eb2a0..37d9f94b06ff367a3514b58ec6f0e4d4d0116030:/lib/diagnostics/cmd.go diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go index 54c78d07cf..71fe1c5dc6 100644 --- a/lib/diagnostics/cmd.go +++ b/lib/diagnostics/cmd.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/sirupsen/logrus" @@ -24,23 +25,20 @@ import ( type Command struct{} -func (cmd Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { var diag diagnoser f := flag.NewFlagSet(prog, flag.ContinueOnError) f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects") f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)") f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client") f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client") + f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)") f.DurationVar(&diag.timeout, "timeout", 10*time.Second, "timeout for http requests") - err := f.Parse(args) - if err == flag.ErrHelp { - return 0 - } else if err != nil { - fmt.Fprintln(stderr, err) - return 2 + if ok, code := cmd.ParseFlags(f, prog, args, "", stderr); !ok { + return code } diag.logger = ctxlog.New(stdout, "text", diag.logLevel) - diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}) + diag.logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true, PadLevelText: true}) diag.runtests() if len(diag.errors) == 0 { diag.logger.Info("--- no errors ---") @@ -60,6 +58,7 @@ type diagnoser struct { stdout io.Writer stderr io.Writer logLevel string + priority int projectName string checkInternal bool checkExternal bool @@ -70,15 +69,15 @@ type diagnoser struct { } func (diag *diagnoser) debugf(f string, args ...interface{}) { - diag.logger.Debugf(f, args...) + diag.logger.Debugf(" ... "+f, args...) } func (diag *diagnoser) infof(f string, args ...interface{}) { - diag.logger.Infof(f, args...) + diag.logger.Infof(" ... "+f, args...) } func (diag *diagnoser) warnf(f string, args ...interface{}) { - diag.logger.Warnf(f, args...) + diag.logger.Warnf(" ... "+f, args...) } func (diag *diagnoser) errorf(f string, args ...interface{}) { @@ -99,14 +98,15 @@ func (diag *diagnoser) dotest(id int, title string, fn func() error) { } diag.done[id] = true - diag.infof("%4d %s", id, title) + diag.logger.Infof("%4d: %s", id, title) t0 := time.Now() err := fn() - elapsed := fmt.Sprintf("%.0dms", time.Now().Sub(t0)/time.Millisecond) + elapsed := fmt.Sprintf("%d ms", time.Now().Sub(t0)/time.Millisecond) if err != nil { - diag.errorf("%s (%s): %s", title, elapsed, err) + diag.errorf("%4d: %s (%s): %s", id, title, elapsed, err) + } else { + diag.logger.Debugf("%4d: %s (%s): ok", id, title, elapsed) } - diag.debugf("%4d %s (%s): ok", id, title, elapsed) } func (diag *diagnoser) runtests() { @@ -331,13 +331,17 @@ func (diag *diagnoser) runtests() { var collection arvados.Collection diag.dotest(90, "creating temporary collection", func() error { + if project.UUID == "" { + return fmt.Errorf("skipping, no project to work in") + } ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) defer cancel() err := client.RequestAndDecodeContext(ctx, &collection, "POST", "arvados/v1/collections", nil, map[string]interface{}{ "ensure_unique_name": true, "collection": map[string]interface{}{ - "name": "test collection", - "trash_at": time.Now().Add(time.Hour)}}) + "owner_uuid": project.UUID, + "name": "test collection", + "trash_at": time.Now().Add(time.Hour)}}) if err != nil { return err } @@ -523,4 +527,83 @@ func (diag *diagnoser) runtests() { } return nil }) + + diag.dotest(160, "running a container", func() error { + if diag.priority < 1 { + diag.infof("skipping (use priority > 0 if you want to run a container)") + return nil + } + if project.UUID == "" { + return fmt.Errorf("skipping, no project to work in") + } + + var cr arvados.ContainerRequest + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout)) + defer cancel() + + timestamp := time.Now().Format(time.RFC3339) + err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{ + "owner_uuid": project.UUID, + "name": fmt.Sprintf("diagnostics container request %s", timestamp), + "container_image": "arvados/jobs", + "command": []string{"echo", timestamp}, + "use_existing": false, + "output_path": "/mnt/output", + "output_name": fmt.Sprintf("diagnostics output %s", timestamp), + "priority": diag.priority, + "state": arvados.ContainerRequestStateCommitted, + "mounts": map[string]map[string]interface{}{ + "/mnt/output": { + "kind": "collection", + "writable": true, + }, + }, + "runtime_constraints": arvados.RuntimeConstraints{ + VCPUs: 1, + RAM: 1 << 26, + KeepCacheRAM: 1 << 26, + }, + }}) + if err != nil { + return err + } + diag.debugf("container request uuid = %s", cr.UUID) + diag.debugf("container uuid = %s", cr.ContainerUUID) + + timeout := 10 * time.Minute + diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout)) + ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout)) + defer cancel() + + var c arvados.Container + for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) { + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout)) + defer cancel() + + crStateWas := cr.State + err := client.RequestAndDecodeContext(ctx, &cr, "GET", "arvados/v1/container_requests/"+cr.UUID, nil, nil) + if err != nil { + return err + } + if cr.State != crStateWas { + diag.debugf("container request state = %s", cr.State) + } + + cStateWas := c.State + err = client.RequestAndDecodeContext(ctx, &c, "GET", "arvados/v1/containers/"+cr.ContainerUUID, nil, nil) + if err != nil { + return err + } + if c.State != cStateWas { + diag.debugf("container state = %s", c.State) + } + } + + if c.State != arvados.ContainerStateComplete { + return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State) + } else if c.ExitCode != 0 { + return fmt.Errorf("container exited %d", c.ExitCode) + } + return nil + }) }