Merge branch 'main' into 18842-arv-mount-disk-config

[arvados.git] / lib / diagnostics / cmd.go

diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index c944db66607d58a612cf2febdc17816477786d42..ed963e1ef75b42439ed1e23fef7d11e9a62a695c 100644 (file)
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -157,8 +157,7 @@ func (diag *diagnoser) runtests() {
                        return err
                }
                if cluster.SystemRootToken != os.Getenv("ARVADOS_API_TOKEN") {
-                       diag.infof("skipping because SystemRootToken read from %s does not match $ARVADOS_API_TOKEN (consider using 'arvados-client sudo diagnostics' to load endpoint and token from config file instead of environment)", ldr.Path)
-                       return nil
+                       return fmt.Errorf("diagnostics usage error: %s is readable but SystemRootToken does not match $ARVADOS_API_TOKEN (to fix, either run 'arvados-client sudo diagnostics' to load everything from config file, or set ARVADOS_CONFIG=- to load nothing from config file)", ldr.Path)
                }
                agg := &health.Aggregator{Cluster: cluster}
                resp := agg.ClusterHealth()
@@ -319,9 +318,9 @@ func (diag *diagnoser) runtests() {
                isInternal := found["proxy"] == 0 && len(keeplist.Items) > 0
                isExternal := found["proxy"] > 0 && found["proxy"] == len(keeplist.Items)
                if isExternal {
-                       diag.verbosef("controller returned only proxy services, this host is treated as \"external\"")
+                       diag.infof("controller returned only proxy services, this host is treated as \"external\"")
                } else if isInternal {
-                       diag.verbosef("controller returned only non-proxy services, this host is treated as \"internal\"")
+                       diag.infof("controller returned only non-proxy services, this host is treated as \"internal\"")
                }
                if (diag.checkInternal && !isInternal) || (diag.checkExternal && !isExternal) {
                        return fmt.Errorf("expecting internal=%v external=%v, but found internal=%v external=%v", diag.checkInternal, diag.checkExternal, isInternal, isExternal)
@@ -704,12 +703,11 @@ func (diag *diagnoser) runtests() {
 
                timeout := 10 * time.Minute
                diag.infof("container request submitted, waiting up to %v for container to run", arvados.Duration(timeout))
-               ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(timeout))
-               defer cancel()
+               deadline := time.Now().Add(timeout)
 
                var c arvados.Container
-               for ; cr.State != arvados.ContainerRequestStateFinal; time.Sleep(2 * time.Second) {
-                       ctx, cancel := context.WithDeadline(ctx, time.Now().Add(diag.timeout))
+               for ; cr.State != arvados.ContainerRequestStateFinal && time.Now().Before(deadline); time.Sleep(2 * time.Second) {
+                       ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
                        defer cancel()
 
                        crStateWas := cr.State
@@ -729,11 +727,26 @@ func (diag *diagnoser) runtests() {
                        if c.State != cStateWas {
                                diag.debugf("container state = %s", c.State)
                        }
+
+                       cancel()
                }
 
+               if cr.State != arvados.ContainerRequestStateFinal {
+                       err := client.RequestAndDecodeContext(context.Background(), &cr, "PATCH", "arvados/v1/container_requests/"+cr.UUID, nil, map[string]interface{}{
+                               "container_request": map[string]interface{}{
+                                       "priority": 0,
+                               }})
+                       if err != nil {
+                               diag.infof("error canceling container request %s: %s", cr.UUID, err)
+                       } else {
+                               diag.debugf("canceled container request %s", cr.UUID)
+                       }
+                       return fmt.Errorf("timed out waiting for container to finish; container request %s state was %q, container %s state was %q", cr.UUID, cr.State, c.UUID, c.State)
+               }
                if c.State != arvados.ContainerStateComplete {
                        return fmt.Errorf("container request %s is final but container %s did not complete: container state = %q", cr.UUID, cr.ContainerUUID, c.State)
-               } else if c.ExitCode != 0 {
+               }
+               if c.ExitCode != 0 {
                        return fmt.Errorf("container exited %d", c.ExitCode)
                }
                return nil