17717: address review comments.

[arvados.git] / lib / costanalyzer / costanalyzer.go

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 4284542b869a07d713d698fbb7eaa1a243e77ec5..6087c88e411ab7a766b93c088ee878325f1a9f45 100644 (file)
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -24,6 +24,8 @@ import (
        "github.com/sirupsen/logrus"
 )
 
        "github.com/sirupsen/logrus"
 )
 

+const timestampFormat = "2006-01-02T15:04:05"
+

 type nodeInfo struct {
        // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
        Properties struct {
 type nodeInfo struct {
        // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
        Properties struct {


@@ -35,6 +37,7 @@ type nodeInfo struct {
        // Modern
        ProviderType string
        Price        float64
        // Modern
        ProviderType string
        Price        float64

+       Preemptible  bool

 }
 
 type arrayFlags []string
 }
 
 type arrayFlags []string


@@ -44,55 +47,84 @@ func (i *arrayFlags) String() string {
 }
 
 func (i *arrayFlags) Set(value string) error {
 }
 
 func (i *arrayFlags) Set(value string) error {

-       *i = append(*i, value)
+       for _, s := range strings.Split(value, ",") {
+               *i = append(*i, s)
+       }

        return nil
 }
 
        return nil
 }
 

-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, begin time.Time, end time.Time, err error) {
+       var beginStr, endStr string

        flags := flag.NewFlagSet("", flag.ContinueOnError)
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `
 Usage:
        flags := flag.NewFlagSet("", flag.ContinueOnError)
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `
 Usage:

-  %s [options ...]
-
-       This program analyzes the cost of Arvados container requests. For each uuid
-       supplied, it creates a CSV report that lists all the containers used to
-       fulfill the container request, together with the machine type and cost of
-       each container.
-
-       When supplied with the uuid of a container request, it will calculate the
-       cost of that container request and all its children. When suplied with a
-       project uuid or when supplied with multiple container request uuids, it will
-       create a CSV report for each supplied uuid, as well as a CSV file with
-       aggregate cost accounting for all supplied uuids. The aggregate cost report
-       takes container reuse into account: if a container was reused between several
-       container requests, its cost will only be counted once.
-
-       To get the node costs, the progam queries the Arvados API for current cost
-       data for each node type used. This means that the reported cost always
-       reflects the cost data as currently defined in the Arvados API configuration
-       file.
+  %s [options ...] [UUID ...]
+
+       This program analyzes the cost of Arvados container requests and calculates
+       the total cost across all requests. At least one UUID or a timestamp range
+       must be specified.
+
+       When the '-output' option is specified, a set of CSV files with cost details
+       will be written to the provided directory. Each file is a CSV report that lists
+       all the containers used to fulfill the container request, together with the
+       machine type and cost of each container.
+
+       When supplied with the UUID of a container request, it will calculate the
+       cost of that container request and all its children.
+
+       When supplied with the UUID of a collection, it will see if there is a
+       container_request UUID in the properties of the collection, and if so, it
+       will calculate the cost of that container request and all its children.
+
+       When supplied with a project UUID or when supplied with multiple container
+       request or collection UUIDs, it will calculate the total cost for all
+       supplied UUIDs.
+
+       When supplied with a 'begin' and 'end' timestamp (format:
+       %s), it will calculate the cost for all top-level container
+       requests whose containers finished during the specified interval.
+
+       The total cost calculation takes container reuse into account: if a container
+       was reused between several container requests, its cost will only be counted
+       once.

 
        Caveats:
 
        Caveats:

-       - the Arvados API configuration cost data may be out of sync with the cloud
-       provider.
-       - when generating reports for older container requests, the cost data in the
-       Arvados API configuration file may have changed since the container request
-       was fulfilled. This program uses the cost data stored at the time of the
+
+       - This program uses the cost data from config.yml at the time of the

        execution of the container, stored in the 'node.json' file in its log
        execution of the container, stored in the 'node.json' file in its log

-       collection.
+       collection. If the cost data was not correctly configured at the time the
+       container was executed, the output from this program will be incorrect.
+
+       - If a container was run on a preemptible ("spot") instance, the cost data
+       reported by this program may be wildly inaccurate, because it does not have
+       access to the spot pricing in effect for the node then the container ran. The
+       UUID report file that is generated when the '-output' option is specified has
+       a column that indicates the preemptible state of the instance that ran the
+       container.

 
 

-       In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+       - This program does not take into account overhead costs like the time spent
+       starting and stopping compute nodes that run containers, the cost of the
+       permanent cloud nodes that provide the Arvados services, the cost of data
+       stored in Arvados, etc.
+
+       - When provided with a project UUID, subprojects will not be considered.
+
+       In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and

        ARVADOS_API_TOKEN environment variables must be set.
 
        ARVADOS_API_TOKEN environment variables must be set.
 

+       This program prints the total dollar amount from the aggregate cost
+       accounting across all provided UUIDs on stdout.
+

 Options:
 Options:

-`, prog)
+`, prog, timestampFormat)

                flags.PrintDefaults()
        }
        loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
                flags.PrintDefaults()
        }
        loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")

-       flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports (required)")
-       flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once. (required)")
+       flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
+       flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+       flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))

        flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
        err = flags.Parse(args)
        if err == flag.ErrHelp {
        flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
        err = flags.Parse(args)
        if err == flag.ErrHelp {


@@ -103,17 +135,30 @@ Options:
                exitCode = 2
                return
        }
                exitCode = 2
                return
        }

+       uuids = flags.Args()

 
 

-       if len(uuids) < 1 {
+       if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {

                flags.Usage()
                flags.Usage()

-               err = fmt.Errorf("Error: no uuid(s) provided")
+               err = fmt.Errorf("When specifying a date range, both begin and end must be specified")

                exitCode = 2
                return
        }
 
                exitCode = 2
                return
        }
 

-       if resultsDir == "" {
+       if len(beginStr) != 0 {
+               var errB, errE error
+               begin, errB = time.Parse(timestampFormat, beginStr)
+               end, errE = time.Parse(timestampFormat, endStr)
+               if (errB != nil) || (errE != nil) {
+                       flags.Usage()
+                       err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE)
+                       exitCode = 2
+                       return
+               }
+       }
+
+       if (len(uuids) < 1) && (len(beginStr) == 0) {

                flags.Usage()
                flags.Usage()

-               err = fmt.Errorf("Error: output directory must be specified")
+               err = fmt.Errorf("error: no uuid(s) provided")

                exitCode = 2
                return
        }
                exitCode = 2
                return
        }


@@ -174,14 +219,14 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container
                size = node.ProviderType
        }
        cost = delta.Seconds() / 3600 * price
                size = node.ProviderType
        }
        cost = delta.Seconds() / 3600 * price

-       csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+       csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"

        return
 }
 
 func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
        reload = true
        return
 }
 
 func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
        reload = true

-       if strings.Contains(uuid, "-j7d0g-") {
-               // We do not cache projects, they have no final state
+       if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+               // We do not cache projects or collections, they have no final state

                return
        }
        // See if we have a cached copy of this object
                return
        }
        // See if we have a cached copy of this object


@@ -251,6 +296,8 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
                err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
        } else if strings.Contains(uuid, "-dz642-") {
                err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
                err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
        } else if strings.Contains(uuid, "-dz642-") {
                err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)

+       } else if strings.Contains(uuid, "-4zz18-") {
+               err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)

        } else {
                err = fmt.Errorf("unsupported object type with UUID %q:\n  %s", uuid, err)
                return
        } else {
                err = fmt.Errorf("unsupported object type with UUID %q:\n  %s", uuid, err)
                return


@@ -276,7 +323,7 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
 
 func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
        if cr.LogUUID == "" {
 
 func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
        if cr.LogUUID == "" {

-               err = errors.New("No log collection")
+               err = errors.New("no log collection")

                return
        }
 
                return
        }
 


@@ -309,7 +356,6 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien
 }
 
 func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
 }
 
 func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {

-

        cost = make(map[string]float64)
 
        var project arvados.Group
        cost = make(map[string]float64)
 
        var project arvados.Group


@@ -361,26 +407,51 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 
        cost = make(map[string]float64)
 
 
        cost = make(map[string]float64)
 

-       csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+       csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"

        var tmpCsv string
        var tmpTotalCost float64
        var totalCost float64
        var tmpCsv string
        var tmpTotalCost float64
        var totalCost float64

+       fmt.Printf("Processing %s\n", uuid)
+
+       var crUUID = uuid
+       if strings.Contains(uuid, "-4zz18-") {
+               // This is a collection, find the associated container request (if any)
+               var c arvados.Collection
+               err = loadObject(logger, ac, uuid, uuid, cache, &c)
+               if err != nil {
+                       return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+               }
+               value, ok := c.Properties["container_request"]
+               if !ok {
+                       return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+               }
+               crUUID, ok = value.(string)
+               if !ok {
+                       return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
+               }
+       }

 
        // This is a container request, find the container
        var cr arvados.ContainerRequest
 
        // This is a container request, find the container
        var cr arvados.ContainerRequest

-       err = loadObject(logger, ac, uuid, uuid, cache, &cr)
+       err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)

        if err != nil {
                return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
        }
        if err != nil {
                return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
        }

+       if len(cr.ContainerUUID) == 0 {
+               // Nothing to do! E.g. a CR in 'Uncommitted' state.
+               logger.Infof("No container associated with container request %s, skipping\n", crUUID)
+               return nil, nil
+       }

        var container arvados.Container
        var container arvados.Container

-       err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+       err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)

        if err != nil {
                return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
        }
 
        topNode, err := getNode(arv, ac, kc, cr)
        if err != nil {
        if err != nil {
                return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
        }
 
        topNode, err := getNode(arv, ac, kc, cr)
        if err != nil {

-               return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+               logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+               return nil, nil

        }
        tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
        csv += tmpCsv
        }
        tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
        csv += tmpCsv


@@ -402,16 +473,17 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
        if err != nil {
                return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
        }
        if err != nil {
                return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
        }

-       logger.Infof("Collecting child containers for container request %s", uuid)
+       logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)

        for _, cr2 := range childCrs.Items {
                logger.Info(".")
                node, err := getNode(arv, ac, kc, cr2)
                if err != nil {
        for _, cr2 := range childCrs.Items {
                logger.Info(".")
                node, err := getNode(arv, ac, kc, cr2)
                if err != nil {

-                       return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+                       logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+                       continue

                }
                logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
                var c2 arvados.Container
                }
                logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
                var c2 arvados.Container

-               err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+               err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)

                if err != nil {
                        return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
                }
                if err != nil {
                        return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
                }


@@ -424,28 +496,34 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 
        csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
 
 
        csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
 

-       // Write the resulting CSV file
-       fName := resultsDir + "/" + uuid + ".csv"
-       err = ioutil.WriteFile(fName, []byte(csv), 0644)
-       if err != nil {
-               return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+       if resultsDir != "" {
+               // Write the resulting CSV file
+               fName := resultsDir + "/" + crUUID + ".csv"
+               err = ioutil.WriteFile(fName, []byte(csv), 0644)
+               if err != nil {
+                       return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+               }
+               logger.Infof("\nUUID report in %s\n\n", fName)

        }
        }

-       logger.Infof("\nUUID report in %s\n\n", fName)

 
        return
 }
 
 func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
 
        return
 }
 
 func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {

-       exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+       exitcode, uuids, resultsDir, cache, begin, end, err := parseFlags(prog, args, loader, logger, stderr)

        if exitcode != 0 {
                return
        }
        if exitcode != 0 {
                return
        }

-       err = ensureDirectory(logger, resultsDir)
-       if err != nil {
-               exitcode = 3
-               return
+       if resultsDir != "" {
+               err = ensureDirectory(logger, resultsDir)
+               if err != nil {
+                       exitcode = 3
+                       return
+               }

        }
 
        }
 

+       uuidChannel := make(chan string)
+

        // Arvados Client setup
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {
        // Arvados Client setup
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {


@@ -462,8 +540,48 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
 
        ac := arvados.NewClientFromEnv()
 
 
        ac := arvados.NewClientFromEnv()
 

+       // Populate uuidChannel with the requested uuid list
+       go func() {
+               defer close(uuidChannel)
+               for _, uuid := range uuids {
+                       uuidChannel <- uuid
+               }
+
+               if !begin.IsZero() {
+                       initialParams := arvados.ResourceListParams{
+                               Filters: []arvados.Filter{{"container.finished_at", ">=", begin}, {"container.finished_at", "<", end}, {"requesting_container_uuid", "=", nil}},
+                               Order:   "created_at",
+                       }
+                       params := initialParams
+                       for {
+                               // This list variable must be a new one declared
+                               // inside the loop: otherwise, items in the API
+                               // response would get deep-merged into the items
+                               // loaded in previous iterations.
+                               var list arvados.ContainerRequestList
+
+                               err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+                               if err != nil {
+                                       logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+                                       break
+                               }
+                               if len(list.Items) == 0 {
+                                       break
+                               }
+
+                               for _, i := range list.Items {
+                                       uuidChannel <- i.UUID
+                               }
+                               params.Offset += len(list.Items)
+                       }
+
+               }
+       }()
+

        cost := make(map[string]float64)
        cost := make(map[string]float64)

-       for _, uuid := range uuids {
+
+       for uuid := range uuidChannel {
+               fmt.Printf("Considering %s\n", uuid)

                if strings.Contains(uuid, "-j7d0g-") {
                        // This is a project (group)
                        cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
                if strings.Contains(uuid, "-j7d0g-") {
                        // This is a project (group)
                        cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)


@@ -474,12 +592,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                        for k, v := range cost {
                                cost[k] = v
                        }
                        for k, v := range cost {
                                cost[k] = v
                        }

-               } else if strings.Contains(uuid, "-xvhdp-") {
+               } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {

                        // This is a container request
                        var crCsv map[string]float64
                        crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
                        if err != nil {
                        // This is a container request
                        var crCsv map[string]float64
                        crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
                        if err != nil {

-                               err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+                               err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())

                                exitcode = 2
                                return
                        }
                                exitcode = 2
                                return
                        }


@@ -491,7 +609,11 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                        // It is identified by the user uuid. As such, cost analysis for the
                        // "Home" project is not supported by this program. Skip this uuid, but
                        // keep going.
                        // It is identified by the user uuid. As such, cost analysis for the
                        // "Home" project is not supported by this program. Skip this uuid, but
                        // keep going.

-                       logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+                       logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
+               } else {
+                       logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
+                       exitcode = 3
+                       return

                }
        }
 
                }
        }
 


@@ -515,14 +637,20 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
 
        csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
 
 
        csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
 

-       // Write the resulting CSV file
-       aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
-       err = ioutil.WriteFile(aFile, []byte(csv), 0644)
-       if err != nil {
-               err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
-               exitcode = 1
-               return
+       if resultsDir != "" {
+               // Write the resulting CSV file
+               aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+               err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+               if err != nil {
+                       err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
+                       exitcode = 1
+                       return
+               }
+               logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)

        }
        }

-       logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+
+       // Output the total dollar amount on stdout
+       fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
+

        return
 }
        return
 }