Modifies sanity check to query for the total expected number of collections

[arvados.git] / services / datamanager / collection / collection.go

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 724e3ec5dbfb0897c3689c0588546092372c9134..42e0cc429054fbb8da0831f117881682f4be225f 100644 (file)
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -227,14 +227,26 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
                }
        }
 
-       if totalCollections < initialNumberOfCollectionsAvailable {
-               err = fmt.Errorf("Initially there were %d collections available, "+
-                               "but we only retrieved %d. Refusing to continue as "+
-                               "this could indicate an otherwise undetected "+
-                               "failure, though it is also possible that "+
-                               "collections were deleted by another process "+
-                               "while datamanager was running.",
-                               initialNumberOfCollectionsAvailable, totalCollections)
+       // Make one final API request to verify that we have processed all collections available up to the latest modification date
+       sdkParams["filters"].([][]string)[0][1] = "<="
+       sdkParams["limit"] = 0
+       err = params.Client.List("collections", sdkParams, &collections)
+       if err != nil {
+               return
+       }
+       finalNumberOfCollectionsAvailable, err :=
+               util.NumberItemsAvailable(params.Client, "collections")
+       if err != nil {
+               return
+       }
+       if totalCollections < finalNumberOfCollectionsAvailable {
+               err = fmt.Errorf("API server indicates a total of %d collections "+
+                               "available up to %v, but we only retrieved %d. "+
+                               "Refusing to continue as this could indicate an "+
+                               "otherwise undetected failure.",
+                               finalNumberOfCollectionsAvailable, 
+                               sdkParams["filters"].([][]string)[0][2],
+                               totalCollections)
                return
        }