20831: Remove debug print and add error check

[arvados.git] / lib / costanalyzer / costanalyzer.go

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index ccd9520bc27dabc274636471248ae0d5717b7cda..e68e2cb8c11115330001e67bca71d4310f07b1a7 100644 (file)
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -5,16 +5,10 @@
 package costanalyzer
 
 import (
 package costanalyzer
 
 import (

-       "bytes"
-       "context"

        "encoding/json"
        "errors"
        "flag"
        "fmt"
        "encoding/json"
        "errors"
        "flag"
        "fmt"

-       "git.arvados.org/arvados.git/lib/config"
-       "git.arvados.org/arvados.git/sdk/go/arvados"
-       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
-       "git.arvados.org/arvados.git/sdk/go/keepclient"

        "io"
        "io/ioutil"
        "net/http"
        "io"
        "io/ioutil"
        "net/http"


@@ -23,57 +17,41 @@ import (
        "strings"
        "time"
 
        "strings"
        "time"
 

+       "git.arvados.org/arvados.git/lib/cmd"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"

        "github.com/sirupsen/logrus"
 )
 
        "github.com/sirupsen/logrus"
 )
 

-// LegacyNodeInfo is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
-// Example:
-// {
-//    "total_cpu_cores":2,
-//    "total_scratch_mb":33770,
-//    "cloud_node":
-//      {
-//        "price":0.1,
-//        "size":"m4.large"
-//      },
-//     "total_ram_mb":7986
-// }
-type LegacyNodeInfo struct {
-       CPUCores  int64           `json:"total_cpu_cores"`
-       ScratchMb int64           `json:"total_scratch_mb"`
-       RAMMb     int64           `json:"total_ram_mb"`
-       CloudNode LegacyCloudNode `json:"cloud_node"`
-}
+const timestampFormat = "2006-01-02T15:04:05"

 
 

-// LegacyCloudNode is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
-type LegacyCloudNode struct {
-       Price float64 `json:"price"`
-       Size  string  `json:"size"`
-}
+var pagesize = 1000

 
 

-// Node is a struct for records created by Arvados Dispatch Cloud (Arvados >= 2.0.0)
-// Example:
-// {
-//    "Name": "Standard_D1_v2",
-//    "ProviderType": "Standard_D1_v2",
-//    "VCPUs": 1,
-//    "RAM": 3584000000,
-//    "Scratch": 50000000000,
-//    "IncludedScratch": 50000000000,
-//    "AddedScratch": 0,
-//    "Price": 0.057,
-//    "Preemptible": false
-//}
-type Node struct {
-       VCPUs        int64
-       Scratch      int64
-       RAM          int64
-       Price        float64
-       Name         string
+type nodeInfo struct {
+       // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
+       Properties struct {
+               CloudNode struct {
+                       Price float64
+                       Size  string
+               } `json:"cloud_node"`
+       }
+       // Modern

        ProviderType string
        ProviderType string

+       Price        float64

        Preemptible  bool
 }
 
        Preemptible  bool
 }
 

+type consumption struct {
+       cost     float64
+       duration float64
+}
+
+func (c *consumption) Add(n consumption) {
+       c.cost += n.cost
+       c.duration += n.duration
+}
+

 type arrayFlags []string
 
 func (i *arrayFlags) String() string {
 type arrayFlags []string
 
 func (i *arrayFlags) String() string {


@@ -81,79 +59,119 @@ func (i *arrayFlags) String() string {
 }
 
 func (i *arrayFlags) Set(value string) error {
 }
 
 func (i *arrayFlags) Set(value string) error {

-       *i = append(*i, value)
+       for _, s := range strings.Split(value, ",") {
+               *i = append(*i, s)
+       }

        return nil
 }
 
        return nil
 }
 

-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool) {
+func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (ok bool, exitCode int) {
+       var beginStr, endStr string

        flags := flag.NewFlagSet("", flag.ContinueOnError)
        flags := flag.NewFlagSet("", flag.ContinueOnError)

-       flags.SetOutput(stderr)

        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `
 Usage:
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `
 Usage:

-  %s [options ...]
-
-       This program analyzes the cost of Arvados container requests. For each uuid
-       supplied, it creates a CSV report that lists all the containers used to
-       fulfill the container request, together with the machine type and cost of
-       each container.
-
-       When supplied with the uuid of a container request, it will calculate the
-       cost of that container request and all its children. When suplied with a
-       project uuid or when supplied with multiple container request uuids, it will
-       create a CSV report for each supplied uuid, as well as a CSV file with
-       aggregate cost accounting for all supplied uuids. The aggregate cost report
-       takes container reuse into account: if a container was reused between several
-       container requests, its cost will only be counted once.
-
-       To get the node costs, the progam queries the Arvados API for current cost
-       data for each node type used. This means that the reported cost always
-       reflects the cost data as currently defined in the Arvados API configuration
-       file.
+  %s [options ...] [UUID ...]
+
+       This program analyzes the cost of Arvados container requests and calculates
+       the total cost across all requests. At least one UUID or a timestamp range
+       must be specified.
+
+       When the '-output' option is specified, a set of CSV files with cost details
+       will be written to the provided directory. Each file is a CSV report that lists
+       all the containers used to fulfill the container request, together with the
+       machine type and cost of each container.
+
+       When supplied with the UUID of a container request, it will calculate the
+       cost of that container request and all its children.
+
+       When supplied with the UUID of a collection, it will see if there is a
+       container_request UUID in the properties of the collection, and if so, it
+       will calculate the cost of that container request and all its children.
+
+       When supplied with a project UUID or when supplied with multiple container
+       request or collection UUIDs, it will calculate the total cost for all
+       supplied UUIDs.
+
+       When supplied with a 'begin' and 'end' timestamp (format:
+       %s), it will calculate the cost for all top-level container
+       requests whose containers finished during the specified interval.
+
+       The total cost calculation takes container reuse into account: if a container
+       was reused between several container requests, its cost will only be counted
+       once.

 
        Caveats:
 
        Caveats:

-       - the Arvados API configuration cost data may be out of sync with the cloud
-       provider.
-       - when generating reports for older container requests, the cost data in the
-       Arvados API configuration file may have changed since the container request
-       was fulfilled. This program uses the cost data stored at the time of the
+
+       - This program uses the cost data from config.yml at the time of the

        execution of the container, stored in the 'node.json' file in its log
        execution of the container, stored in the 'node.json' file in its log

-       collection.
+       collection. If the cost data was not correctly configured at the time the
+       container was executed, the output from this program will be incorrect.
+
+       - If a container was run on a preemptible ("spot") instance, the cost data
+       reported by this program may be wildly inaccurate, because it does not have
+       access to the spot pricing in effect for the node then the container ran. The
+       UUID report file that is generated when the '-output' option is specified has
+       a column that indicates the preemptible state of the instance that ran the
+       container.
+
+       - This program does not take into account overhead costs like the time spent
+       starting and stopping compute nodes that run containers, the cost of the
+       permanent cloud nodes that provide the Arvados services, the cost of data
+       stored in Arvados, etc.

 
 

-       In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+       - When provided with a project UUID, subprojects will not be considered.
+
+       In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and

        ARVADOS_API_TOKEN environment variables must be set.
 
        ARVADOS_API_TOKEN environment variables must be set.
 

+       This program prints the total dollar amount from the aggregate cost
+       accounting across all provided UUIDs on stdout.
+

 Options:
 Options:

-`, prog)
+`, prog, timestampFormat)

                flags.PrintDefaults()
        }
                flags.PrintDefaults()
        }

-       loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
-       resultsDir = *flags.String("output", "results", "output directory for the CSV reports")
-       flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.")
-       flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
-       err := flags.Parse(args)
-       if err == flag.ErrHelp {
-               exitCode = 1
-               return
-       } else if err != nil {
-               exitCode = 2
-               return
+       loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
+       flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports")
+       flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+       flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))
+       flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects")
+       if ok, code := cmd.ParseFlags(flags, prog, args, "[uuid ...]", stderr); !ok {
+               return false, code

        }
        }

+       c.uuids = flags.Args()

 
 

-       if len(uuids) < 1 {
-               logger.Errorf("Error: no uuid(s) provided")
-               flags.Usage()
-               exitCode = 2
-               return
+       if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {
+               fmt.Fprintf(stderr, "When specifying a date range, both begin and end must be specified (try -help)\n")
+               return false, 2
+       }
+
+       if len(beginStr) != 0 {
+               var errB, errE error
+               c.begin, errB = time.Parse(timestampFormat, beginStr)
+               c.end, errE = time.Parse(timestampFormat, endStr)
+               if (errB != nil) || (errE != nil) {
+                       fmt.Fprintf(stderr, "When specifying a date range, both begin and end must be of the format %s %+v, %+v\n", timestampFormat, errB, errE)
+                       return false, 2
+               }
+       }
+
+       if (len(c.uuids) < 1) && (len(beginStr) == 0) {
+               fmt.Fprintf(stderr, "error: no uuid(s) provided (try -help)\n")
+               return false, 2

        }
 
        lvl, err := logrus.ParseLevel(*loglevel)
        if err != nil {
        }
 
        lvl, err := logrus.ParseLevel(*loglevel)
        if err != nil {

-               exitCode = 2
-               return
+               fmt.Fprintf(stderr, "invalid argument to -log-level: %s\n", err)
+               return false, 2

        }
        logger.SetLevel(lvl)
        }
        logger.SetLevel(lvl)

-       return
+       if !c.cache {
+               logger.Debug("Caching disabled")
+       }
+       return true, 0

 }
 
 func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
 }
 
 func ensureDirectory(logger *logrus.Logger, dir string) (err error) {


@@ -161,425 +179,498 @@ func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
        if os.IsNotExist(err) {
                err = os.MkdirAll(dir, 0700)
                if err != nil {
        if os.IsNotExist(err) {
                err = os.MkdirAll(dir, 0700)
                if err != nil {

-                       return fmt.Errorf("Error creating directory %s: %s\n", dir, err.Error())
+                       return fmt.Errorf("error creating directory %s: %s", dir, err.Error())

                }
        } else {
                if !statData.IsDir() {
                }
        } else {
                if !statData.IsDir() {

-                       return fmt.Errorf("The path %s is not a directory\n", dir)
+                       return fmt.Errorf("the path %s is not a directory", dir)

                }
        }
        return
 }
 
                }
        }
        return
 }
 

-func addContainerLine(logger *logrus.Logger, node interface{}, cr, container map[string]interface{}) (csv string, cost float64) {
-       csv = cr["uuid"].(string) + ","
-       csv += cr["name"].(string) + ","
-       csv += container["uuid"].(string) + ","
-       csv += container["state"].(string) + ","
-       if container["started_at"] != nil {
-               csv += container["started_at"].(string) + ","
+func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (string, consumption) {
+       var csv string
+       var containerConsumption consumption
+       csv = cr.UUID + ","
+       csv += cr.Name + ","
+       csv += container.UUID + ","
+       csv += string(container.State) + ","
+       if container.StartedAt != nil {
+               csv += container.StartedAt.String() + ","

        } else {
                csv += ","
        }
 
        var delta time.Duration
        } else {
                csv += ","
        }
 
        var delta time.Duration

-       if container["finished_at"] != nil {
-               csv += container["finished_at"].(string) + ","
-               finishedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["finished_at"].(string))
-               if err != nil {
-                       fmt.Println(err)
-               }
-               startedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["started_at"].(string))
-               if err != nil {
-                       fmt.Println(err)
-               }
-               delta = finishedTimestamp.Sub(startedTimestamp)
-               csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
+       if container.FinishedAt != nil {
+               csv += container.FinishedAt.String() + ","
+               delta = container.FinishedAt.Sub(*container.StartedAt)
+               csv += strconv.FormatFloat(delta.Seconds(), 'f', 3, 64) + ","

        } else {
                csv += ",,"
        }
        var price float64
        var size string
        } else {
                csv += ",,"
        }
        var price float64
        var size string

-       switch n := node.(type) {
-       case Node:
-               price = n.Price
-               size = n.ProviderType
-       case LegacyNodeInfo:
-               price = n.CloudNode.Price
-               size = n.CloudNode.Size
-       default:
-               logger.Warn("WARNING: unknown node type found!")
-       }
-       cost = delta.Seconds() / 3600 * price
-       csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
-       return
+       if node.Properties.CloudNode.Price != 0 {
+               price = node.Properties.CloudNode.Price
+               size = node.Properties.CloudNode.Size
+       } else {
+               price = node.Price
+               size = node.ProviderType
+       }
+       containerConsumption.cost = delta.Seconds() / 3600 * price
+       containerConsumption.duration = delta.Seconds()
+       csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(containerConsumption.cost, 'f', 8, 64) + "\n"
+       return csv, containerConsumption

 }
 
 }
 

-func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object map[string]interface{}) {
+func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {

        reload = true
        reload = true

+       if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+               // We do not cache projects or collections, they have no final state
+               return
+       }

        // See if we have a cached copy of this object
        // See if we have a cached copy of this object

-       if _, err := os.Stat(file); err == nil {
-               data, err := ioutil.ReadFile(file)
-               if err != nil {
-                       logger.Errorf("error reading %q: %s", file, err)
-                       return
-               }
-               err = json.Unmarshal(data, &object)
-               if err != nil {
-                       logger.Errorf("failed to unmarshal json: %s: %s", data, err)
-                       return
-               }
+       _, err := os.Stat(file)
+       if err != nil {
+               return
+       }
+       data, err := ioutil.ReadFile(file)
+       if err != nil {
+               logger.Errorf("error reading %q: %s", file, err)
+               return
+       }
+       err = json.Unmarshal(data, &object)
+       if err != nil {
+               logger.Errorf("failed to unmarshal json: %s: %s", data, err)
+               return
+       }

 
 

-               // See if it is in a final state, if that makes sense
-               // Projects (j7d0g) do not have state so they should always be reloaded
-               if !strings.Contains(uuid, "-j7d0g-") {
-                       if object["state"].(string) == "Complete" || object["state"].(string) == "Failed" {
-                               reload = false
-                               logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
-                               return
-                       }
+       // See if it is in a final state, if that makes sense
+       switch v := object.(type) {
+       case *arvados.ContainerRequest:
+               if v.State == arvados.ContainerRequestStateFinal {
+                       reload = false
+                       logger.Debugf("Loaded object %s from local cache (%s)", uuid, file)
+               }
+       case *arvados.Container:
+               if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
+                       reload = false
+                       logger.Debugf("Loaded object %s from local cache (%s)", uuid, file)

                }
        }
        return
 }
 
 // Load an Arvados object.
                }
        }
        return
 }
 
 // Load an Arvados object.

-func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string, cache bool) (object map[string]interface{}, err error) {
-       err = ensureDirectory(logger, path)
-       if err != nil {
-               return
-       }
-
-       file := path + "/" + uuid + ".json"
+func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) {
+       file := uuid + ".json"

 
        var reload bool
 
        var reload bool

+       var cacheDir string
+

        if !cache {
                reload = true
        } else {
        if !cache {
                reload = true
        } else {

-               reload, object = loadCachedObject(logger, file, uuid)
+               homeDir, err := os.UserHomeDir()
+               if err != nil {
+                       reload = true
+                       logger.Info("Unable to determine current user home directory, not using cache")
+               } else {
+                       cacheDir = homeDir + "/.cache/arvados/costanalyzer/"
+                       err = ensureDirectory(logger, cacheDir)
+                       if err != nil {
+                               reload = true
+                               logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error())
+                       } else {
+                               reload = loadCachedObject(logger, cacheDir+file, uuid, object)
+                       }
+               }

        }
        if !reload {
                return
        }
 
        if strings.Contains(uuid, "-j7d0g-") {
        }
        if !reload {
                return
        }
 
        if strings.Contains(uuid, "-j7d0g-") {

-               err = arv.Get("groups", uuid, nil, &object)
+               err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil)

        } else if strings.Contains(uuid, "-xvhdp-") {
        } else if strings.Contains(uuid, "-xvhdp-") {

-               err = arv.Get("container_requests", uuid, nil, &object)
+               err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)

        } else if strings.Contains(uuid, "-dz642-") {
        } else if strings.Contains(uuid, "-dz642-") {

-               err = arv.Get("containers", uuid, nil, &object)
+               err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+       } else if strings.Contains(uuid, "-4zz18-") {
+               err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)

        } else {
        } else {

-               err = arv.Get("jobs", uuid, nil, &object)
+               err = fmt.Errorf("unsupported object type with UUID %q:\n  %s", uuid, err)
+               return

        }
        if err != nil {
        }
        if err != nil {

-               err = fmt.Errorf("Error loading object with UUID %q:\n  %s\n", uuid, err)
+               err = fmt.Errorf("error loading object with UUID %q:\n  %s", uuid, err)

                return
        }
        encoded, err := json.MarshalIndent(object, "", " ")
        if err != nil {
                return
        }
        encoded, err := json.MarshalIndent(object, "", " ")
        if err != nil {

-               err = fmt.Errorf("Error marshaling object with UUID %q:\n  %s\n", uuid, err)
+               err = fmt.Errorf("error marshaling object with UUID %q:\n  %s", uuid, err)

                return
        }
                return
        }

-       err = ioutil.WriteFile(file, encoded, 0644)
-       if err != nil {
-               err = fmt.Errorf("Error writing file %s:\n  %s\n", file, err)
-               return
+       if cacheDir != "" {
+               err = ioutil.WriteFile(cacheDir+file, encoded, 0644)
+               if err != nil {
+                       err = fmt.Errorf("error writing file %s:\n  %s", file, err)
+                       return
+               }

        }
        return
 }
 
        }
        return
 }
 

-func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, itemMap map[string]interface{}) (node interface{}, err error) {
-       logUuid, ok := itemMap["log_uuid"]
-       if !ok {
-               err = errors.New("No log collection")
+func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
+       if cr.LogUUID == "" {
+               err = errors.New("no log collection")

                return
        }
 
        var collection arvados.Collection
                return
        }
 
        var collection arvados.Collection

-       err = arv.Get("collections", logUuid.(string), nil, &collection)
+       err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)

        if err != nil {
        if err != nil {

-               err = fmt.Errorf("Error getting collection: %s", err)
+               err = fmt.Errorf("error getting collection: %s", err)

                return
        }
 
        var fs arvados.CollectionFileSystem
        fs, err = collection.FileSystem(ac, kc)
        if err != nil {
                return
        }
 
        var fs arvados.CollectionFileSystem
        fs, err = collection.FileSystem(ac, kc)
        if err != nil {

-               err = fmt.Errorf("Error opening collection as filesystem: %s", err)
+               err = fmt.Errorf("error opening collection as filesystem: %s", err)

                return
        }
        var f http.File
        f, err = fs.Open("node.json")
        if err != nil {
                return
        }
        var f http.File
        f, err = fs.Open("node.json")
        if err != nil {

-               err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", logUuid.(string), err)
+               err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err)

                return
        }
 
                return
        }
 

-       var nodeDict map[string]interface{}
-       buf := new(bytes.Buffer)
-       _, err = buf.ReadFrom(f)
+       err = json.NewDecoder(f).Decode(&node)

        if err != nil {
        if err != nil {

-               err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", logUuid.(string), err)
+               err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err)

                return
        }
                return
        }

-       contents := buf.String()
-       f.Close()
+       return
+}

 
 

-       err = json.Unmarshal([]byte(contents), &nodeDict)
-       if err != nil {
-               err = fmt.Errorf("Error unmarshalling: %s", err)
-               return
-       }
-       if val, ok := nodeDict["properties"]; ok {
-               var encoded []byte
-               encoded, err = json.MarshalIndent(val, "", " ")
-               if err != nil {
-                       err = fmt.Errorf("Error marshalling: %s", err)
-                       return
+func getContainerRequests(ac *arvados.Client, filters []arvados.Filter) ([]arvados.ContainerRequest, error) {
+       var allItems []arvados.ContainerRequest
+       for {
+               pagefilters := append([]arvados.Filter(nil), filters...)
+               if len(allItems) > 0 {
+                       pagefilters = append(pagefilters, arvados.Filter{
+                               Attr:     "uuid",
+                               Operator: ">",
+                               Operand:  allItems[len(allItems)-1].UUID,
+                       })

                }
                }

-               // node is type LegacyNodeInfo
-               var newNode LegacyNodeInfo
-               err = json.Unmarshal(encoded, &newNode)
+               var resp arvados.ContainerRequestList
+               err := ac.RequestAndDecode(&resp, "GET", "arvados/v1/container_requests", nil, arvados.ResourceListParams{
+                       Filters: pagefilters,
+                       Limit:   &pagesize,
+                       Order:   "uuid",
+                       Count:   "none",
+               })

                if err != nil {
                if err != nil {

-                       err = fmt.Errorf("Error unmarshalling: %s", err)
-                       return
+                       return nil, fmt.Errorf("error querying container_requests: %w", err)

                }
                }

-               node = newNode
-       } else {
-               // node is type Node
-               var newNode Node
-               err = json.Unmarshal([]byte(contents), &newNode)
-               if err != nil {
-                       err = fmt.Errorf("Error unmarshalling: %s", err)
-                       return
+               if len(resp.Items) == 0 {
+                       // no more pages
+                       return allItems, nil

                }
                }

-               node = newNode
+               allItems = append(allItems, resp.Items...)

        }
        }

-       return

 }
 
 }
 

-func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
-
-       cost = make(map[string]float64)
+func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]consumption, err error) {
+       cost = make(map[string]consumption)

 
 

-       project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache)
+       var project arvados.Group
+       err = loadObject(logger, ac, uuid, uuid, cache, &project)

        if err != nil {
        if err != nil {

-               return nil, fmt.Errorf("Error loading object %s: %s\n", uuid, err.Error())
+               return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())

        }
        }

-
-       // arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
-
-       // Now find all container requests that have the container we found above as requesting_container_uuid
-       var childCrs map[string]interface{}
-       filterset := []arvados.Filter{
+       allItems, err := getContainerRequests(ac, []arvados.Filter{

                {
                        Attr:     "owner_uuid",
                        Operator: "=",
                {
                        Attr:     "owner_uuid",
                        Operator: "=",

-                       Operand:  project["uuid"].(string),
+                       Operand:  project.UUID,

                },
                {
                        Attr:     "requesting_container_uuid",
                        Operator: "=",
                        Operand:  nil,
                },
                },
                {
                        Attr:     "requesting_container_uuid",
                        Operator: "=",
                        Operand:  nil,
                },

-       }
-       err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
-               "filters": filterset,
-               "limit":   10000,

        })
        if err != nil {
        })
        if err != nil {

-               return nil, fmt.Errorf("Error querying container_requests: %s\n", err.Error())
-       }
-       if value, ok := childCrs["items"]; ok {
-               logger.Infof("Collecting top level container requests in project %s\n", uuid)
-               items := value.([]interface{})
-               for _, item := range items {
-                       itemMap := item.(map[string]interface{})
-                       crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
-                       if err != nil {
-                               return nil, fmt.Errorf("Error generating container_request CSV: %s\n", err.Error())
-                       }
-                       for k, v := range crCsv {
-                               cost[k] = v
-                       }
+               return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
+       }
+       if len(allItems) == 0 {
+               logger.Infof("No top level container requests found in project %s", uuid)
+               return
+       }
+       logger.Infof("Collecting top level container requests in project %s", uuid)
+       for _, cr := range allItems {
+               crInfo, err := generateCrInfo(logger, cr.UUID, arv, ac, kc, resultsDir, cache)
+               if err != nil {
+                       return nil, fmt.Errorf("error generating container_request CSV for %s: %s", cr.UUID, err)
+               }
+               for k, v := range crInfo {
+                       cost[k] = v

                }
                }

-       } else {
-               logger.Infof("No top level container requests found in project %s\n", uuid)

        }
        return
 }
 
        }
        return
 }
 

-func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
+func generateCrInfo(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]consumption, err error) {

 
 

-       cost = make(map[string]float64)
+       cost = make(map[string]consumption)

 
 

-       csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+       csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"

        var tmpCsv string
        var tmpCsv string

-       var tmpTotalCost float64
-       var totalCost float64
+       var total, tmpTotal consumption
+       logger.Debugf("Processing %s", uuid)
+
+       var crUUID = uuid
+       if strings.Contains(uuid, "-4zz18-") {
+               // This is a collection, find the associated container request (if any)
+               var c arvados.Collection
+               err = loadObject(logger, ac, uuid, uuid, cache, &c)
+               if err != nil {
+                       return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+               }
+               value, ok := c.Properties["container_request"]
+               if !ok {
+                       return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+               }
+               crUUID, ok = value.(string)
+               if !ok {
+                       return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
+               }
+       }

 
        // This is a container request, find the container
 
        // This is a container request, find the container

-       cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache)
+       var cr arvados.ContainerRequest
+       err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)

        if err != nil {
        if err != nil {

-               return nil, fmt.Errorf("Error loading object %s: %s", uuid, err)
+               return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
+       }
+       if len(cr.ContainerUUID) == 0 {
+               // Nothing to do! E.g. a CR in 'Uncommitted' state.
+               logger.Infof("No container associated with container request %s, skipping", crUUID)
+               return nil, nil

        }
        }

-       container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string), cache)
+       var container arvados.Container
+       err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)

        if err != nil {
        if err != nil {

-               return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+               return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)

        }
 
        topNode, err := getNode(arv, ac, kc, cr)
        if err != nil {
        }
 
        topNode, err := getNode(arv, ac, kc, cr)
        if err != nil {

-               return nil, fmt.Errorf("Error getting node %s: %s\n", cr["uuid"], err)
+               logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+               return nil, nil

        }
        }

-       tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
+       tmpCsv, total = addContainerLine(logger, topNode, cr, container)

        csv += tmpCsv
        csv += tmpCsv

-       totalCost += tmpTotalCost
-       cost[container["uuid"].(string)] = totalCost
-
-       // Now find all container requests that have the container we found above as requesting_container_uuid
-       var childCrs map[string]interface{}
-       filterset := []arvados.Filter{
-               {
-                       Attr:     "requesting_container_uuid",
-                       Operator: "=",
-                       Operand:  container["uuid"].(string),
-               }}
-       err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
-               "filters": filterset,
-               "limit":   10000,
-       })
-       if err != nil {
-               return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
-       }
-       if value, ok := childCrs["items"]; ok {
-               logger.Infof("Collecting child containers for container request %s", uuid)
-               items := value.([]interface{})
-               for _, item := range items {
-                       logger.Info(".")
-                       itemMap := item.(map[string]interface{})
-                       node, err := getNode(arv, ac, kc, itemMap)
-                       if err != nil {
-                               return nil, fmt.Errorf("Error getting node %s: %s\n", itemMap["uuid"], err)
-                       }
-                       logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n")
-                       c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string), cache)
-                       if err != nil {
-                               return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err)
-                       }
-                       tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2)
-                       cost[itemMap["container_uuid"].(string)] = tmpTotalCost
-                       csv += tmpCsv
-                       totalCost += tmpTotalCost
+       cost[container.UUID] = total
+
+       // Find all container requests that have the container we
+       // found above as requesting_container_uuid.
+       allItems, err := getContainerRequests(ac, []arvados.Filter{{
+               Attr:     "requesting_container_uuid",
+               Operator: "=",
+               Operand:  container.UUID,
+       }})
+       logger.Infof("Looking up %d child containers for container %s (%s)", len(allItems), container.UUID, container.FinishedAt)
+       progressTicker := time.NewTicker(5 * time.Second)
+       defer progressTicker.Stop()
+       for i, cr2 := range allItems {
+               select {
+               case <-progressTicker.C:
+                       logger.Infof("... %d of %d", i+1, len(allItems))
+               default:

                }
                }

+               node, err := getNode(arv, ac, kc, cr2)
+               if err != nil {
+                       logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+                       continue
+               }
+               logger.Debug("Child container: " + cr2.ContainerUUID)
+               var c2 arvados.Container
+               err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
+               if err != nil {
+                       return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
+               }
+               tmpCsv, tmpTotal = addContainerLine(logger, node, cr2, c2)
+               cost[cr2.ContainerUUID] = tmpTotal
+               csv += tmpCsv
+               total.Add(tmpTotal)

        }
        }

-       logger.Info(" done\n")
+       logger.Debug("Done collecting child containers")

 
 

-       csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
+       csv += "TOTAL,,,,,," + strconv.FormatFloat(total.duration, 'f', 3, 64) + ",,,," + strconv.FormatFloat(total.cost, 'f', 2, 64) + "\n"

 
 

-       // Write the resulting CSV file
-       fName := resultsDir + "/" + uuid + ".csv"
-       err = ioutil.WriteFile(fName, []byte(csv), 0644)
-       if err != nil {
-               return nil, fmt.Errorf("Error writing file with path %s: %s\n", fName, err.Error())
+       if resultsDir != "" {
+               // Write the resulting CSV file
+               fName := resultsDir + "/" + crUUID + ".csv"
+               err = ioutil.WriteFile(fName, []byte(csv), 0644)
+               if err != nil {
+                       return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+               }
+               logger.Infof("\nUUID report in %s", fName)

        }
 
        return
 }
 
        }
 
        return
 }
 

-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) {
-       exitcode, uuids, resultsDir, cache := parseFlags(prog, args, loader, logger, stderr)
-       if exitcode != 0 {
+func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+       var ok bool
+       ok, exitcode = c.parseFlags(prog, args, logger, stderr)
+       if !ok {

                return
        }
                return
        }

-       err := ensureDirectory(logger, resultsDir)
-       if err != nil {
-               logger.Errorf("%s", err)
-               exitcode = 3
-               return
+       if c.resultsDir != "" {
+               err = ensureDirectory(logger, c.resultsDir)
+               if err != nil {
+                       exitcode = 3
+                       return
+               }

        }
 
        }
 

+       uuidChannel := make(chan string)
+

        // Arvados Client setup
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {
        // Arvados Client setup
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {

-               logger.Errorf("error creating Arvados object: %s", err)
+               err = fmt.Errorf("error creating Arvados object: %s", err)

                exitcode = 1
                return
        }
        kc, err := keepclient.MakeKeepClient(arv)
        if err != nil {
                exitcode = 1
                return
        }
        kc, err := keepclient.MakeKeepClient(arv)
        if err != nil {

-               logger.Errorf("error creating Keep object: %s", err)
+               err = fmt.Errorf("error creating Keep object: %s", err)

                exitcode = 1
                return
        }
 
        ac := arvados.NewClientFromEnv()
 
                exitcode = 1
                return
        }
 
        ac := arvados.NewClientFromEnv()
 

-       cost := make(map[string]float64)
-       for _, uuid := range uuids {
+       // Populate uuidChannel with the requested uuid list
+       go func() {
+               defer close(uuidChannel)
+               for _, uuid := range c.uuids {
+                       uuidChannel <- uuid
+               }
+
+               if !c.begin.IsZero() {
+                       initialParams := arvados.ResourceListParams{
+                               Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}},
+                               Order:   "created_at",
+                       }
+                       params := initialParams
+                       for {
+                               // This list variable must be a new one declared
+                               // inside the loop: otherwise, items in the API
+                               // response would get deep-merged into the items
+                               // loaded in previous iterations.
+                               var list arvados.ContainerRequestList
+
+                               err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+                               if err != nil {
+                                       logger.Errorf("Error getting container request list from Arvados API: %s", err)
+                                       break
+                               }
+                               if len(list.Items) == 0 {
+                                       break
+                               }
+
+                               for _, i := range list.Items {
+                                       uuidChannel <- i.UUID
+                               }
+                               params.Offset += len(list.Items)
+                       }
+
+               }
+       }()
+
+       cost := make(map[string]consumption)
+
+       for uuid := range uuidChannel {
+               logger.Debugf("Considering %s", uuid)

                if strings.Contains(uuid, "-j7d0g-") {
                        // This is a project (group)
                if strings.Contains(uuid, "-j7d0g-") {
                        // This is a project (group)

-                       cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+                       cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)

                        if err != nil {
                        if err != nil {

-                               // FIXME print error
-                               logger.Info(err.Error())

                                exitcode = 1
                                return
                        }
                        for k, v := range cost {
                                cost[k] = v
                        }
                                exitcode = 1
                                return
                        }
                        for k, v := range cost {
                                cost[k] = v
                        }

-               } else if strings.Contains(uuid, "-xvhdp-") {
-                       // This is a container request
-                       crCsv, err := generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+               } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
+                       // This is a container request or collection
+                       var crInfo map[string]consumption
+                       crInfo, err = generateCrInfo(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)

                        if err != nil {
                        if err != nil {

-                               logger.Fatalf("Error generating container_request CSV: %s\n", err.Error())
+                               err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
+                               exitcode = 2
+                               return

                        }
                        }

-                       for k, v := range crCsv {
+                       for k, v := range crInfo {

                                cost[k] = v
                        }
                } else if strings.Contains(uuid, "-tpzed-") {
                        // This is a user. The "Home" project for a user is not a real project.
                        // It is identified by the user uuid. As such, cost analysis for the
                                cost[k] = v
                        }
                } else if strings.Contains(uuid, "-tpzed-") {
                        // This is a user. The "Home" project for a user is not a real project.
                        // It is identified by the user uuid. As such, cost analysis for the

-                       // "Home" project is not supported by this program.
-                       logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+                       // "Home" project is not supported by this program. Skip this uuid, but
+                       // keep going.
+                       logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
+               } else {
+                       logger.Errorf("this argument does not look like a uuid: %s", uuid)
+                       exitcode = 3
+                       return

                }
        }
 
                }
        }
 

-       logger.Info("\n")
-       for k := range cost {
-               logger.Infof("Uuid report in %s/%s.csv\n", resultsDir, k)
-       }
-

        if len(cost) == 0 {
        if len(cost) == 0 {

-               logger.Info("Nothing to do!\n")
+               logger.Info("Nothing to do!")

                return
        }
 
        var csv string
 
                return
        }
 
        var csv string
 

-       csv = "# Aggregate cost accounting for uuids:\n"
-       for _, uuid := range uuids {
+       csv = "# Aggregate cost accounting for uuids:\n# UUID, Duration in seconds, Total cost\n"
+       for _, uuid := range c.uuids {

                csv += "# " + uuid + "\n"
        }
 
                csv += "# " + uuid + "\n"
        }
 

-       var total float64
+       var total consumption

        for k, v := range cost {
        for k, v := range cost {

-               csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n"
-               total += v
+               csv += k + "," + strconv.FormatFloat(v.duration, 'f', 3, 64) + "," + strconv.FormatFloat(v.cost, 'f', 8, 64) + "\n"
+               total.Add(v)

        }
 
        }
 

-       csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
+       csv += "TOTAL," + strconv.FormatFloat(total.duration, 'f', 3, 64) + "," + strconv.FormatFloat(total.cost, 'f', 2, 64) + "\n"

 
 

-       // Write the resulting CSV file
-       aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
-       err = ioutil.WriteFile(aFile, []byte(csv), 0644)
-       if err != nil {
-               logger.Errorf("Error writing file with path %s: %s\n", aFile, err.Error())
-               exitcode = 1
-               return
-       } else {
-               logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile)
+       if c.resultsDir != "" {
+               // Write the resulting CSV file
+               aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+               err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+               if err != nil {
+                       err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
+                       exitcode = 1
+                       return
+               }
+               logger.Infof("Aggregate cost accounting for all supplied uuids in %s", aFile)

        }
        }

+
+       // Output the total dollar amount on stdout
+       fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total.cost, 'f', 2, 64))
+

        return
 }
        return
 }