17344: Reset "broken node" flag on loopback instance create/reset.

[arvados.git] / lib / cloud / loopback / loopback.go

diff --git a/lib/cloud/loopback/loopback.go b/lib/cloud/loopback/loopback.go
index 6ad4f876d99cb908c6d95fca7ebb0ecf6d03492e..9b71c9f162b9083731f5e3113755fefdae0c31a5 100644 (file)
--- a/lib/cloud/loopback/loopback.go
+++ b/lib/cloud/loopback/loopback.go
@@ -11,6 +11,7 @@ import (
        "encoding/json"
        "errors"
        "io"
+       "os"
        "os/exec"
        "os/user"
        "strings"
@@ -58,6 +59,14 @@ func (is *instanceSet) Create(it arvados.InstanceType, _ cloud.ImageID, tags clo
        if len(is.instances) > 0 {
                return nil, errQuota
        }
+       // A crunch-run process running in a previous instance may
+       // have marked the node as broken. In the loopback scenario a
+       // destroy+create cycle doesn't fix whatever was broken -- but
+       // nothing else will either, so the best we can do is remove
+       // the "broken" flag and try again.
+       if err := os.Remove("/var/lock/crunch-run-broken"); err != nil && !errors.Is(err, os.ErrNotExist) {
+               return nil, err
+       }
        u, err := user.Current()
        if err != nil {
                return nil, err