14328: Check broken-node blacklist when watchdog hits errors.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 7f5a85e808b08750f98f7d9b031b1863f5e950cf..39cf4408203dfda83463b7a3518a243cb4314e3b 100644 (file)
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1143,6 +1143,9 @@ func (runner *ContainerRunner) WaitFinish() error {
                        ctrs, err := runner.Docker.ContainerList(context.Background(), dockertypes.ContainerListOptions{})
                        if err != nil {
                                runner.CrunchLog.Printf("error checking container list: %s", err)
+                               if runner.checkBrokenNode(err) {
+                                       return
+                               }
                                continue polling
                        }
                        for _, ctr := range ctrs {