X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/15623db9b12d8ac1b0b44a3eb53358896254b28a..166f27869b2f530aa3dd05b02cdc46f787dd8216:/services/keep-balance/collection.go

diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index 1e5fa57978..c4ddc90c41 100644
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"time"
 
-	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
 )
 
 func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int, error) {
@@ -43,6 +43,18 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
 		return err
 	}
 
+	// Note the obvious way to get all collections (sorting by
+	// UUID) would be much easier, but would lose data: If a
+	// client were to move files from collection with uuid="zzz"
+	// to a collection with uuid="aaa" around the time when we
+	// were fetching the "mmm" page, we would never see those
+	// files' block IDs at all -- even if the client is careful to
+	// save "aaa" before saving "zzz".
+	//
+	// Instead, we get pages in modified_at order. Collections
+	// that are modified during the run will be re-fetched in a
+	// subsequent page.
+
 	limit := pageSize
 	if limit <= 0 {
 		// Use the maximum page size the server allows
@@ -68,7 +80,7 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
 			return err
 		}
 		for _, coll := range page.Items {
-			if last.ModifiedAt != nil && *last.ModifiedAt == *coll.ModifiedAt && last.UUID >= coll.UUID {
+			if last.ModifiedAt == coll.ModifiedAt && last.UUID >= coll.UUID {
 				continue
 			}
 			callCount++
@@ -80,9 +92,9 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
 		}
 		if len(page.Items) == 0 && !gettingExactTimestamp {
 			break
-		} else if last.ModifiedAt == nil {
+		} else if last.ModifiedAt.IsZero() {
 			return fmt.Errorf("BUG: Last collection on the page (%s) has no modified_at timestamp; cannot make progress", last.UUID)
-		} else if len(page.Items) > 0 && *last.ModifiedAt == filterTime {
+		} else if len(page.Items) > 0 && last.ModifiedAt == filterTime {
 			// If we requested time>=X and never got a
 			// time>X then we might not have received all
 			// items with time==X yet. Switch to
@@ -123,7 +135,7 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
 			// avoiding that would add overhead in the
 			// overwhelmingly common cases, so we don't
 			// bother.
-			filterTime = *last.ModifiedAt
+			filterTime = last.ModifiedAt
 			params.Filters = []arvados.Filter{{
 				Attr:     "modified_at",
 				Operator: ">=",