X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0b0b4c7b23e96a6efb3cfd88b0ba7224158e9544..c92af4dedd8f2bcc0dfcc2f6e052e850341f8a5e:/lib/costanalyzer/costanalyzer.go

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index edaaa5bd17..dfe3d584ce 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -9,9 +9,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"git.arvados.org/arvados.git/sdk/go/arvados"
-	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
-	"git.arvados.org/arvados.git/sdk/go/keepclient"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -20,6 +17,9 @@ import (
 	"strings"
 	"time"
 
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
 	"github.com/sirupsen/logrus"
 )
 
@@ -169,7 +169,7 @@ Options:
 	}
 	logger.SetLevel(lvl)
 	if !c.cache {
-		logger.Debug("Caching disabled\n")
+		logger.Debug("Caching disabled")
 	}
 	return
 }
@@ -249,12 +249,12 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string, object in
 	case *arvados.ContainerRequest:
 		if v.State == arvados.ContainerRequestStateFinal {
 			reload = false
-			logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+			logger.Debugf("Loaded object %s from local cache (%s)", uuid, file)
 		}
 	case *arvados.Container:
 		if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
 			reload = false
-			logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+			logger.Debugf("Loaded object %s from local cache (%s)", uuid, file)
 		}
 	}
 	return
@@ -384,7 +384,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
 	}
 	if value, ok := childCrs["items"]; ok {
-		logger.Infof("Collecting top level container requests in project %s\n", uuid)
+		logger.Infof("Collecting top level container requests in project %s", uuid)
 		items := value.([]interface{})
 		for _, item := range items {
 			itemMap := item.(map[string]interface{})
@@ -397,7 +397,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 			}
 		}
 	} else {
-		logger.Infof("No top level container requests found in project %s\n", uuid)
+		logger.Infof("No top level container requests found in project %s", uuid)
 	}
 	return
 }
@@ -410,7 +410,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 	var tmpCsv string
 	var tmpTotalCost float64
 	var totalCost float64
-	fmt.Printf("Processing %s\n", uuid)
+	logger.Debugf("Processing %s", uuid)
 
 	var crUUID = uuid
 	if strings.Contains(uuid, "-4zz18-") {
@@ -438,7 +438,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 	}
 	if len(cr.ContainerUUID) == 0 {
 		// Nothing to do! E.g. a CR in 'Uncommitted' state.
-		logger.Infof("No container associated with container request %s, skipping\n", crUUID)
+		logger.Infof("No container associated with container request %s, skipping", crUUID)
 		return nil, nil
 	}
 	var container arvados.Container
@@ -473,14 +473,20 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
 	}
 	logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
-	for _, cr2 := range childCrs.Items {
-		logger.Info(".")
+	progressTicker := time.NewTicker(5 * time.Second)
+	defer progressTicker.Stop()
+	for i, cr2 := range childCrs.Items {
+		select {
+		case <-progressTicker.C:
+			logger.Infof("... %d of %d", i+1, len(childCrs.Items))
+		default:
+		}
 		node, err := getNode(arv, ac, kc, cr2)
 		if err != nil {
 			logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
 			continue
 		}
-		logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
+		logger.Debug("Child container: " + cr2.ContainerUUID)
 		var c2 arvados.Container
 		err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
 		if err != nil {
@@ -491,7 +497,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		csv += tmpCsv
 		totalCost += tmpTotalCost
 	}
-	logger.Info(" done\n")
+	logger.Debug("Done collecting child containers")
 
 	csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
 
@@ -502,7 +508,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 		if err != nil {
 			return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
 		}
-		logger.Infof("\nUUID report in %s\n\n", fName)
+		logger.Infof("\nUUID report in %s", fName)
 	}
 
 	return
@@ -562,7 +568,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
 
 				err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
 				if err != nil {
-					logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+					logger.Errorf("Error getting container request list from Arvados API: %s", err)
 					break
 				}
 				if len(list.Items) == 0 {
@@ -581,7 +587,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
 	cost := make(map[string]float64)
 
 	for uuid := range uuidChannel {
-		fmt.Printf("Considering %s\n", uuid)
+		logger.Debugf("Considering %s", uuid)
 		if strings.Contains(uuid, "-j7d0g-") {
 			// This is a project (group)
 			cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
@@ -611,14 +617,14 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
 			// keep going.
 			logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
 		} else {
-			logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
+			logger.Errorf("this argument does not look like a uuid: %s", uuid)
 			exitcode = 3
 			return
 		}
 	}
 
 	if len(cost) == 0 {
-		logger.Info("Nothing to do!\n")
+		logger.Info("Nothing to do!")
 		return
 	}
 
@@ -646,7 +652,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger
 			exitcode = 1
 			return
 		}
-		logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+		logger.Infof("Aggregate cost accounting for all supplied uuids in %s", aFile)
 	}
 
 	// Output the total dollar amount on stdout