7253: update BlockIterWithDuplicates to return any errors through Manifest, rather...

[arvados.git] / services / datamanager / collection / collection.go

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 55b0b3711a258ef4aa89de27fd585f11577749b5..df6852687a621fccbb9606b99ae30cfa81bf3dca 100644 (file)
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -10,7 +10,6 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/logger"
        "git.curoverse.com/arvados.git/sdk/go/manifest"
        "git.curoverse.com/arvados.git/sdk/go/util"
-       "git.curoverse.com/arvados.git/services/datamanager/loggerutil"
        "log"
        "os"
        "runtime/pprof"
@@ -18,7 +17,7 @@ import (
 )
 
 var (
-       heapProfileFilename string
+       HeapProfileFilename string
        // globals for debugging
        totalManifestSize uint64
        maxManifestSize   uint64
@@ -42,6 +41,7 @@ type ReadCollections struct {
        CollectionUUIDToIndex     map[string]int
        CollectionIndexToUUID     []string
        BlockToCollectionIndices  map[blockdigest.DigestWithSize][]int
+       Err                       error
 }
 
 // GetCollectionsParams params
@@ -67,7 +67,7 @@ type SdkCollectionList struct {
 }
 
 func init() {
-       flag.StringVar(&heapProfileFilename,
+       flag.StringVar(&HeapProfileFilename,
                "heap-profile",
                "",
                "File to write the heap profiles to. Leave blank to skip profiling.")
@@ -80,8 +80,8 @@ func init() {
 // Otherwise we would see cumulative numbers as explained here:
 // https://groups.google.com/d/msg/golang-nuts/ZyHciRglQYc/2nh4Ndu2fZcJ
 func WriteHeapProfile() error {
-       if heapProfileFilename != "" {
-               heapProfile, err := os.Create(heapProfileFilename)
+       if HeapProfileFilename != "" {
+               heapProfile, err := os.Create(HeapProfileFilename)
                if err != nil {
                        return err
                }
@@ -96,10 +96,11 @@ func WriteHeapProfile() error {
 }
 
 // GetCollectionsAndSummarize gets collections from api and summarizes
-func GetCollectionsAndSummarize(arvLogger *logger.Logger, params GetCollectionsParams) (results ReadCollections) {
+func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections) {
        results, err := GetCollections(params)
        if err != nil {
-               loggerutil.LogErrorMessage(arvLogger, fmt.Sprintf("Error during GetCollections with params %v: %v", params, err))
+               results.Err = err
+               return
        }
 
        results.Summarize(params.Logger)
@@ -123,6 +124,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
        if &params.Client == nil {
                err = fmt.Errorf("params.Client passed to GetCollections() should " +
                        "contain a valid ArvadosClient, but instead it is nil.")
+               return
        }
 
        fieldsWanted := []string{"manifest_text",
@@ -232,7 +234,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
        }
 
        // Write the heap profile for examining memory usage
-       WriteHeapProfile()
+       err = WriteHeapProfile()
 
        return
 }
@@ -272,7 +274,7 @@ func ProcessCollections(arvLogger *logger.Logger,
                        collection.ReplicationLevel = defaultReplicationLevel
                }
 
-               manifest := manifest.Manifest{sdkCollection.ManifestText}
+               manifest := manifest.Manifest{Text: sdkCollection.ManifestText}
                manifestSize := uint64(len(sdkCollection.ManifestText))
 
                if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
@@ -295,6 +297,11 @@ func ProcessCollections(arvLogger *logger.Logger,
                        }
                        collection.BlockDigestToSize[block.Digest] = block.Size
                }
+               if manifest.Err != nil {
+                       err = manifest.Err
+                       return
+               }
+
                collection.TotalSize = 0
                for _, size := range collection.BlockDigestToSize {
                        collection.TotalSize += size