X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0662b235357dd40b5d27efd06b60044ddcec06f6..45ee6988e990ec0c808d3a64b7b0c4a5f864a5ec:/services/datamanager/collection/collection.go

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 1229f2917e..42e0cc4290 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -227,6 +227,29 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
 		}
 	}
 
+	// Make one final API request to verify that we have processed all collections available up to the latest modification date
+	sdkParams["filters"].([][]string)[0][1] = "<="
+	sdkParams["limit"] = 0
+	err = params.Client.List("collections", sdkParams, &collections)
+	if err != nil {
+		return
+	}
+	finalNumberOfCollectionsAvailable, err :=
+		util.NumberItemsAvailable(params.Client, "collections")
+	if err != nil {
+		return
+	}
+	if totalCollections < finalNumberOfCollectionsAvailable {
+		err = fmt.Errorf("API server indicates a total of %d collections "+
+				"available up to %v, but we only retrieved %d. "+
+				"Refusing to continue as this could indicate an "+
+				"otherwise undetected failure.",
+				finalNumberOfCollectionsAvailable, 
+				sdkParams["filters"].([][]string)[0][2],
+				totalCollections)
+		return
+	}
+
 	// Write the heap profile for examining memory usage
 	err = WriteHeapProfile()