7253: update BlockIterWithDuplicates to return any errors through Manifest, rather...

[arvados.git] / services / datamanager / collection / collection.go

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 463c8473a1bc6b924520e887248730d1cccc8a9e..df6852687a621fccbb9606b99ae30cfa81bf3dca 100644 (file)
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -10,7 +10,6 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/logger"
        "git.curoverse.com/arvados.git/sdk/go/manifest"
        "git.curoverse.com/arvados.git/sdk/go/util"
-       "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
        "log"
        "os"
        "runtime/pprof"
@@ -42,6 +41,7 @@ type ReadCollections struct {
        CollectionUUIDToIndex     map[string]int
        CollectionIndexToUUID     []string
        BlockToCollectionIndices  map[blockdigest.DigestWithSize][]int
+       Err                       error
 }
 
 // GetCollectionsParams params
@@ -96,11 +96,10 @@ func WriteHeapProfile() error {
 }
 
 // GetCollectionsAndSummarize gets collections from api and summarizes
-func GetCollectionsAndSummarize(arvLogger *logger.Logger, params GetCollectionsParams) (results ReadCollections) {
+func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections) {
        results, err := GetCollections(params)
        if err != nil {
-               loggerutil.LogErrorMessage(arvLogger, fmt.Sprintf("Error during GetCollections with params %v: %v", params, err))
-               results = ReadCollections{}
+               results.Err = err
                return
        }
 
@@ -125,6 +124,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
        if &params.Client == nil {
                err = fmt.Errorf("params.Client passed to GetCollections() should " +
                        "contain a valid ArvadosClient, but instead it is nil.")
+               return
        }
 
        fieldsWanted := []string{"manifest_text",
@@ -274,7 +274,7 @@ func ProcessCollections(arvLogger *logger.Logger,
                        collection.ReplicationLevel = defaultReplicationLevel
                }
 
-               manifest := manifest.Manifest{sdkCollection.ManifestText}
+               manifest := manifest.Manifest{Text: sdkCollection.ManifestText}
                manifestSize := uint64(len(sdkCollection.ManifestText))
 
                if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
@@ -297,6 +297,11 @@ func ProcessCollections(arvLogger *logger.Logger,
                        }
                        collection.BlockDigestToSize[block.Digest] = block.Size
                }
+               if manifest.Err != nil {
+                       err = manifest.Err
+                       return
+               }
+
                collection.TotalSize = 0
                for _, size := range collection.BlockDigestToSize {
                        collection.TotalSize += size