From: Tom Clegg Date: Mon, 22 Apr 2019 14:50:59 +0000 (-0400) Subject: 15112: Note rationale for non-obvious paging approach. X-Git-Tag: 1.4.0~61^2 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/62b9c297c3336b2bc605f70a756c56b69285e730 15112: Note rationale for non-obvious paging approach. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go index 1e5fa57978..534928bc82 100644 --- a/services/keep-balance/collection.go +++ b/services/keep-balance/collection.go @@ -43,6 +43,18 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) return err } + // Note the obvious way to get all collections (sorting by + // UUID) would be much easier, but would lose data: If a + // client were to move files from collection with uuid="zzz" + // to a collection with uuid="aaa" around the time when we + // were fetching the "mmm" page, we would never see those + // files' block IDs at all -- even if the client is careful to + // save "aaa" before saving "zzz". + // + // Instead, we get pages in modified_at order. Collections + // that are modified during the run will be re-fetched in a + // subsequent page. + limit := pageSize if limit <= 0 { // Use the maximum page size the server allows