X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/dfb9a5e285e4275c26b6f959a04babd5a8ad836e..76e2a1fc805e80f6882117848d99782b569b281c:/services/datamanager/collection/collection.go

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 55b0b3711a..33970d8bc4 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -10,7 +10,6 @@ import (
 	"git.curoverse.com/arvados.git/sdk/go/logger"
 	"git.curoverse.com/arvados.git/sdk/go/manifest"
 	"git.curoverse.com/arvados.git/sdk/go/util"
-	"git.curoverse.com/arvados.git/services/datamanager/loggerutil"
 	"log"
 	"os"
 	"runtime/pprof"
@@ -18,7 +17,7 @@ import (
 )
 
 var (
-	heapProfileFilename string
+	HeapProfileFilename string
 	// globals for debugging
 	totalManifestSize uint64
 	maxManifestSize   uint64
@@ -42,6 +41,7 @@ type ReadCollections struct {
 	CollectionUUIDToIndex     map[string]int
 	CollectionIndexToUUID     []string
 	BlockToCollectionIndices  map[blockdigest.DigestWithSize][]int
+	Err                       error
 }
 
 // GetCollectionsParams params
@@ -67,7 +67,7 @@ type SdkCollectionList struct {
 }
 
 func init() {
-	flag.StringVar(&heapProfileFilename,
+	flag.StringVar(&HeapProfileFilename,
 		"heap-profile",
 		"",
 		"File to write the heap profiles to. Leave blank to skip profiling.")
@@ -80,8 +80,8 @@ func init() {
 // Otherwise we would see cumulative numbers as explained here:
 // https://groups.google.com/d/msg/golang-nuts/ZyHciRglQYc/2nh4Ndu2fZcJ
 func WriteHeapProfile() error {
-	if heapProfileFilename != "" {
-		heapProfile, err := os.Create(heapProfileFilename)
+	if HeapProfileFilename != "" {
+		heapProfile, err := os.Create(HeapProfileFilename)
 		if err != nil {
 			return err
 		}
@@ -96,10 +96,11 @@ func WriteHeapProfile() error {
 }
 
 // GetCollectionsAndSummarize gets collections from api and summarizes
-func GetCollectionsAndSummarize(arvLogger *logger.Logger, params GetCollectionsParams) (results ReadCollections) {
+func GetCollectionsAndSummarize(params GetCollectionsParams) (results ReadCollections) {
 	results, err := GetCollections(params)
 	if err != nil {
-		loggerutil.LogErrorMessage(arvLogger, fmt.Sprintf("Error during GetCollections with params %v: %v", params, err))
+		results.Err = err
+		return
 	}
 
 	results.Summarize(params.Logger)
@@ -123,6 +124,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
 	if &params.Client == nil {
 		err = fmt.Errorf("params.Client passed to GetCollections() should " +
 			"contain a valid ArvadosClient, but instead it is nil.")
+		return
 	}
 
 	fieldsWanted := []string{"manifest_text",
@@ -232,7 +234,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
 	}
 
 	// Write the heap profile for examining memory usage
-	WriteHeapProfile()
+	err = WriteHeapProfile()
 
 	return
 }
@@ -272,7 +274,7 @@ func ProcessCollections(arvLogger *logger.Logger,
 			collection.ReplicationLevel = defaultReplicationLevel
 		}
 
-		manifest := manifest.Manifest{sdkCollection.ManifestText}
+		manifest := manifest.Manifest{Text: sdkCollection.ManifestText}
 		manifestSize := uint64(len(sdkCollection.ManifestText))
 
 		if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
@@ -285,16 +287,20 @@ func ProcessCollections(arvLogger *logger.Logger,
 		blockChannel := manifest.BlockIterWithDuplicates()
 		for block := range blockChannel {
 			if storedSize, stored := collection.BlockDigestToSize[block.Digest]; stored && storedSize != block.Size {
-				err = fmt.Errorf(
+				log.Printf(
 					"Collection %s contains multiple sizes (%d and %d) for block %s",
 					collection.UUID,
 					storedSize,
 					block.Size,
 					block.Digest)
-				return
 			}
 			collection.BlockDigestToSize[block.Digest] = block.Size
 		}
+		if manifest.Err != nil {
+			err = manifest.Err
+			return
+		}
+
 		collection.TotalSize = 0
 		for _, size := range collection.BlockDigestToSize {
 			collection.TotalSize += size