7 "git.curoverse.com/arvados.git/sdk/go/arvados"
10 func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int, error) {
11 var page arvados.CollectionList
14 err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
15 return page.ItemsAvailable, err
18 // EachCollection calls f once for every readable
19 // collection. EachCollection stops if it encounters an error, such as
20 // f returning a non-nil error.
22 // The progress function is called periodically with done (number of
23 // times f has been called) and total (number of times f is expected
26 // If pageSize > 0 it is used as the maximum page size in each API
27 // call; otherwise the maximum allowed page size is requested.
28 func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
30 progress = func(_, _ int) {}
33 expectCount, err := countCollections(c, arvados.ResourceListParams{})
40 // Use the maximum page size the server allows
43 params := arvados.ResourceListParams{
45 Order: "modified_at, uuid",
46 Select: []string{"uuid", "manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
48 var last arvados.Collection
49 var filterTime time.Time
52 progress(callCount, expectCount)
53 var page arvados.CollectionList
54 err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
58 for _, coll := range page.Items {
59 if last.ModifiedAt != nil && *last.ModifiedAt == *coll.ModifiedAt && last.UUID >= coll.UUID {
69 if last.ModifiedAt == nil || *last.ModifiedAt == filterTime {
70 if page.ItemsAvailable > len(page.Items) {
71 // TODO: use "mtime=X && UUID>Y"
72 // filters to get all collections with
73 // this timestamp, then use "mtime>X"
74 // to get the next timestamp.
75 return fmt.Errorf("BUG: Received an entire page with the same modified_at timestamp (%v), cannot make progress", filterTime)
79 filterTime = *last.ModifiedAt
80 params.Filters = []arvados.Filter{{
90 progress(callCount, expectCount)
92 if checkCount, err := countCollections(c, arvados.ResourceListParams{Filters: []arvados.Filter{{
95 Operand: filterTime}}}); err != nil {
97 } else if callCount < checkCount {
98 return fmt.Errorf("Retrieved %d collections with modtime <= T=%q, but server now reports there are %d collections with modtime <= T", callCount, filterTime, checkCount)