- if collectionChannel, err := IterateSdkListItems(collections); err != nil {
- log.Fatalf("Error trying to iterate collections returned by SDK: %v", err)
- } else {
- index := 0
- results.UuidToCollection = make(map[string]Collection)
- for item_map := range collectionChannel {
- index += 1
- if m := params.LogEveryNthCollectionProcessed; m >0 && (index % m) == 0 {
- log.Printf("Processing collection #%d", index)
+ // Write the heap profile for examining memory usage
+ WriteHeapProfile()
+
+ return
+}
+
+// StrCopy returns a newly allocated string.
+// It is useful to copy slices so that the garbage collector can reuse
+// the memory of the longer strings they came from.
+func StrCopy(s string) string {
+ return string([]byte(s))
+}
+
+// ProcessCollections read from api server
+func ProcessCollections(arvLogger *logger.Logger,
+ receivedCollections []SdkCollectionInfo,
+ defaultReplicationLevel int,
+ UUIDToCollection map[string]Collection) (latestModificationDate time.Time, err error) {
+ for _, sdkCollection := range receivedCollections {
+ collection := Collection{UUID: StrCopy(sdkCollection.UUID),
+ OwnerUUID: StrCopy(sdkCollection.OwnerUUID),
+ ReplicationLevel: sdkCollection.Redundancy,
+ BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
+
+ if sdkCollection.ModifiedAt.IsZero() {
+ return latestModificationDate, fmt.Errorf(
+ "Arvados SDK collection returned with unexpected zero "+
+ "modification date. This probably means that either we failed to "+
+ "parse the modification date or the API server has changed how "+
+ "it returns modification dates: %+v",
+ collection)
+ }
+
+ if sdkCollection.ModifiedAt.After(latestModificationDate) {
+ latestModificationDate = sdkCollection.ModifiedAt
+ }
+
+ if collection.ReplicationLevel == 0 {
+ collection.ReplicationLevel = defaultReplicationLevel
+ }
+
+ manifest := manifest.Manifest{sdkCollection.ManifestText}
+ manifestSize := uint64(len(sdkCollection.ManifestText))
+
+ if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
+ totalManifestSize += manifestSize
+ }
+ if manifestSize > maxManifestSize {
+ maxManifestSize = manifestSize
+ }
+
+ blockChannel := manifest.BlockIterWithDuplicates()
+ for block := range blockChannel {
+ if storedSize, stored := collection.BlockDigestToSize[block.Digest]; stored && storedSize != block.Size {
+ err = fmt.Errorf(
+ "Collection %s contains multiple sizes (%d and %d) for block %s",
+ collection.UUID,
+ storedSize,
+ block.Size,
+ block.Digest)
+ return