+ if params.Logger != nil {
+ params.Logger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ collectionInfo := logger.GetOrCreateMap(p, "collection_info")
+ collectionInfo["num_collections_at_start"] = initialNumberOfCollectionsAvailable
+ collectionInfo["batch_size"] = params.BatchSize
+ collectionInfo["default_replication_level"] = defaultReplicationLevel
+ })
+ }
+
+ // These values are just for getting the loop to run the first time,
+ // afterwards they'll be set to real values.
+ previousTotalCollections := -1
+ totalCollections := 0
+ for totalCollections > previousTotalCollections {
+ // We're still finding new collections
+
+ // Write the heap profile for examining memory usage
+ err = WriteHeapProfile()
+ if err != nil {
+ return
+ }
+
+ // Get next batch of collections.
+ var collections SdkCollectionList
+ err = params.Client.List("collections", sdkParams, &collections)
+ if err != nil {
+ return
+ }
+
+ // Process collection and update our date filter.
+ var latestModificationDate time.Time
+ latestModificationDate, err = ProcessCollections(params.Logger,
+ collections.Items,
+ defaultReplicationLevel,
+ results.UUIDToCollection)
+ if err != nil {
+ return
+ }
+ sdkParams["filters"].([][]string)[0][2] = latestModificationDate.Format(time.RFC3339)
+
+ // update counts
+ previousTotalCollections = totalCollections
+ totalCollections = len(results.UUIDToCollection)
+
+ log.Printf("%d collections read, %d new in last batch, "+
+ "%s latest modified date, %.0f %d %d avg,max,total manifest size",
+ totalCollections,
+ totalCollections-previousTotalCollections,
+ sdkParams["filters"].([][]string)[0][2],
+ float32(totalManifestSize)/float32(totalCollections),
+ maxManifestSize, totalManifestSize)