17609: Merge branch 'master'

[arvados.git] / lib / costanalyzer / costanalyzer.go

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 402e0ec81a160e677a1aefb8df82d589f01c3e20..f9875c6a8e79cf427a378c34a1b0a9d4d9264473 100644 (file)
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -9,10 +9,6 @@ import (
        "errors"
        "flag"
        "fmt"
        "errors"
        "flag"
        "fmt"

-       "git.arvados.org/arvados.git/lib/config"
-       "git.arvados.org/arvados.git/sdk/go/arvados"
-       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
-       "git.arvados.org/arvados.git/sdk/go/keepclient"

        "io"
        "io/ioutil"
        "net/http"
        "io"
        "io/ioutil"
        "net/http"


@@ -21,9 +17,14 @@ import (
        "strings"
        "time"
 
        "strings"
        "time"
 

+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"

        "github.com/sirupsen/logrus"
 )
 
        "github.com/sirupsen/logrus"
 )
 

+const timestampFormat = "2006-01-02T15:04:05"
+

 type nodeInfo struct {
        // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
        Properties struct {
 type nodeInfo struct {
        // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
        Properties struct {


@@ -51,32 +52,42 @@ func (i *arrayFlags) Set(value string) error {
        return nil
 }
 
        return nil
 }
 

-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) {
+       var beginStr, endStr string

        flags := flag.NewFlagSet("", flag.ContinueOnError)
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `
 Usage:
        flags := flag.NewFlagSet("", flag.ContinueOnError)
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `
 Usage:

-  %s [options ...] uuid [uuid ...]
+  %s [options ...] [UUID ...]

 
 

-       This program analyzes the cost of Arvados container requests. For each uuid
-       supplied, it creates a CSV report that lists all the containers used to
-       fulfill the container request, together with the machine type and cost of
-       each container. At least one uuid must be specified.
+       This program analyzes the cost of Arvados container requests and calculates
+       the total cost across all requests. At least one UUID or a timestamp range
+       must be specified.
+
+       When the '-output' option is specified, a set of CSV files with cost details
+       will be written to the provided directory. Each file is a CSV report that lists
+       all the containers used to fulfill the container request, together with the
+       machine type and cost of each container.

 
 

-       When supplied with the uuid of a container request, it will calculate the
+       When supplied with the UUID of a container request, it will calculate the

        cost of that container request and all its children.
 
        cost of that container request and all its children.
 

-       When supplied with the uuid of a collection, it will see if there is a
-       container_request uuid in the properties of the collection, and if so, it
+       When supplied with the UUID of a collection, it will see if there is a
+       container_request UUID in the properties of the collection, and if so, it

        will calculate the cost of that container request and all its children.
 
        will calculate the cost of that container request and all its children.
 

-       When supplied with a project uuid or when supplied with multiple container
-       request or collection uuids, it will create a CSV report for each supplied
-       uuid, as well as a CSV file with aggregate cost accounting for all supplied
-       uuids. The aggregate cost report takes container reuse into account: if a
-       container was reused between several container requests, its cost will only
-       be counted once.
+       When supplied with a project UUID or when supplied with multiple container
+       request or collection UUIDs, it will calculate the total cost for all
+       supplied UUIDs.
+
+       When supplied with a 'begin' and 'end' timestamp (format:
+       %s), it will calculate the cost for all top-level container
+       requests whose containers finished during the specified interval.
+
+       The total cost calculation takes container reuse into account: if a container
+       was reused between several container requests, its cost will only be counted
+       once.

 
        Caveats:
 
 
        Caveats:
 


@@ -97,24 +108,23 @@ Usage:
        permanent cloud nodes that provide the Arvados services, the cost of data
        stored in Arvados, etc.
 
        permanent cloud nodes that provide the Arvados services, the cost of data
        stored in Arvados, etc.
 

-       - When provided with a project uuid, subprojects will not be considered.
+       - When provided with a project UUID, subprojects will not be considered.

 
 

-       In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+       In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and

        ARVADOS_API_TOKEN environment variables must be set.
 
        This program prints the total dollar amount from the aggregate cost
        ARVADOS_API_TOKEN environment variables must be set.
 
        This program prints the total dollar amount from the aggregate cost

-       accounting across all provided uuids on stdout.
-
-       When the '-output' option is specified, a set of CSV files with cost details
-       will be written to the provided directory.
+       accounting across all provided UUIDs on stdout.

 
 Options:
 
 Options:

-`, prog)
+`, prog, timestampFormat)

                flags.PrintDefaults()
        }
        loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
                flags.PrintDefaults()
        }
        loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")

-       flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
-       flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+       flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports")
+       flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+       flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))
+       flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects")

        err = flags.Parse(args)
        if err == flag.ErrHelp {
                err = nil
        err = flags.Parse(args)
        if err == flag.ErrHelp {
                err = nil


@@ -124,9 +134,28 @@ Options:
                exitCode = 2
                return
        }
                exitCode = 2
                return
        }

-       uuids = flags.Args()
+       c.uuids = flags.Args()
+
+       if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {
+               flags.Usage()
+               err = fmt.Errorf("When specifying a date range, both begin and end must be specified")
+               exitCode = 2
+               return
+       }

 
 

-       if len(uuids) < 1 {
+       if len(beginStr) != 0 {
+               var errB, errE error
+               c.begin, errB = time.Parse(timestampFormat, beginStr)
+               c.end, errE = time.Parse(timestampFormat, endStr)
+               if (errB != nil) || (errE != nil) {
+                       flags.Usage()
+                       err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE)
+                       exitCode = 2
+                       return
+               }
+       }
+
+       if (len(c.uuids) < 1) && (len(beginStr) == 0) {

                flags.Usage()
                err = fmt.Errorf("error: no uuid(s) provided")
                exitCode = 2
                flags.Usage()
                err = fmt.Errorf("error: no uuid(s) provided")
                exitCode = 2


@@ -139,8 +168,8 @@ Options:
                return
        }
        logger.SetLevel(lvl)
                return
        }
        logger.SetLevel(lvl)

-       if !cache {
-               logger.Debug("Caching disabled\n")
+       if !c.cache {
+               logger.Debug("Caching disabled")

        }
        return
 }
        }
        return
 }


@@ -220,12 +249,12 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string, object in
        case *arvados.ContainerRequest:
                if v.State == arvados.ContainerRequestStateFinal {
                        reload = false
        case *arvados.ContainerRequest:
                if v.State == arvados.ContainerRequestStateFinal {
                        reload = false

-                       logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+                       logger.Debugf("Loaded object %s from local cache (%s)", uuid, file)

                }
        case *arvados.Container:
                if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
                        reload = false
                }
        case *arvados.Container:
                if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
                        reload = false

-                       logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+                       logger.Debugf("Loaded object %s from local cache (%s)", uuid, file)

                }
        }
        return
                }
        }
        return


@@ -355,7 +384,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
                return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
        }
        if value, ok := childCrs["items"]; ok {
                return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
        }
        if value, ok := childCrs["items"]; ok {

-               logger.Infof("Collecting top level container requests in project %s\n", uuid)
+               logger.Infof("Collecting top level container requests in project %s", uuid)

                items := value.([]interface{})
                for _, item := range items {
                        itemMap := item.(map[string]interface{})
                items := value.([]interface{})
                for _, item := range items {
                        itemMap := item.(map[string]interface{})


@@ -368,7 +397,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
                        }
                }
        } else {
                        }
                }
        } else {

-               logger.Infof("No top level container requests found in project %s\n", uuid)
+               logger.Infof("No top level container requests found in project %s", uuid)

        }
        return
 }
        }
        return
 }


@@ -381,6 +410,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
        var tmpCsv string
        var tmpTotalCost float64
        var totalCost float64
        var tmpCsv string
        var tmpTotalCost float64
        var totalCost float64

+       fmt.Printf("Processing %s\n", uuid)

 
        var crUUID = uuid
        if strings.Contains(uuid, "-4zz18-") {
 
        var crUUID = uuid
        if strings.Contains(uuid, "-4zz18-") {


@@ -406,6 +436,11 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
        if err != nil {
                return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
        }
        if err != nil {
                return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
        }

+       if len(cr.ContainerUUID) == 0 {
+               // Nothing to do! E.g. a CR in 'Uncommitted' state.
+               logger.Infof("No container associated with container request %s, skipping", crUUID)
+               return nil, nil
+       }

        var container arvados.Container
        err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
        if err != nil {
        var container arvados.Container
        err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
        if err != nil {


@@ -414,7 +449,8 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 
        topNode, err := getNode(arv, ac, kc, cr)
        if err != nil {
 
        topNode, err := getNode(arv, ac, kc, cr)
        if err != nil {

-               return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+               logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+               return nil, nil

        }
        tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
        csv += tmpCsv
        }
        tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
        csv += tmpCsv


@@ -436,14 +472,15 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
        if err != nil {
                return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
        }
        if err != nil {
                return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
        }

-       logger.Infof("Collecting child containers for container request %s", crUUID)
+       logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)

        for _, cr2 := range childCrs.Items {
                logger.Info(".")
                node, err := getNode(arv, ac, kc, cr2)
                if err != nil {
        for _, cr2 := range childCrs.Items {
                logger.Info(".")
                node, err := getNode(arv, ac, kc, cr2)
                if err != nil {

-                       return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+                       logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+                       continue

                }
                }

-               logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
+               logger.Debug("\nChild container: " + cr2.ContainerUUID)

                var c2 arvados.Container
                err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
                if err != nil {
                var c2 arvados.Container
                err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
                if err != nil {


@@ -454,7 +491,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
                csv += tmpCsv
                totalCost += tmpTotalCost
        }
                csv += tmpCsv
                totalCost += tmpTotalCost
        }

-       logger.Info(" done\n")
+       logger.Info(" done")

 
        csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
 
 
        csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
 


@@ -465,25 +502,28 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
                if err != nil {
                        return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
                }
                if err != nil {
                        return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
                }

-               logger.Infof("\nUUID report in %s\n\n", fName)
+               logger.Infof("\nUUID report in %s\n", fName)

        }
 
        return
 }
 
        }
 
        return
 }
 

-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
-       exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+       exitcode, err = c.parseFlags(prog, args, logger, stderr)
+

        if exitcode != 0 {
                return
        }
        if exitcode != 0 {
                return
        }

-       if resultsDir != "" {
-               err = ensureDirectory(logger, resultsDir)
+       if c.resultsDir != "" {
+               err = ensureDirectory(logger, c.resultsDir)

                if err != nil {
                        exitcode = 3
                        return
                }
        }
 
                if err != nil {
                        exitcode = 3
                        return
                }
        }
 

+       uuidChannel := make(chan string)
+

        // Arvados Client setup
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {
        // Arvados Client setup
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {


@@ -500,11 +540,51 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
 
        ac := arvados.NewClientFromEnv()
 
 
        ac := arvados.NewClientFromEnv()
 

+       // Populate uuidChannel with the requested uuid list
+       go func() {
+               defer close(uuidChannel)
+               for _, uuid := range c.uuids {
+                       uuidChannel <- uuid
+               }
+
+               if !c.begin.IsZero() {
+                       initialParams := arvados.ResourceListParams{
+                               Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}},
+                               Order:   "created_at",
+                       }
+                       params := initialParams
+                       for {
+                               // This list variable must be a new one declared
+                               // inside the loop: otherwise, items in the API
+                               // response would get deep-merged into the items
+                               // loaded in previous iterations.
+                               var list arvados.ContainerRequestList
+
+                               err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+                               if err != nil {
+                                       logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+                                       break
+                               }
+                               if len(list.Items) == 0 {
+                                       break
+                               }
+
+                               for _, i := range list.Items {
+                                       uuidChannel <- i.UUID
+                               }
+                               params.Offset += len(list.Items)
+                       }
+
+               }
+       }()
+

        cost := make(map[string]float64)
        cost := make(map[string]float64)

-       for _, uuid := range uuids {
+
+       for uuid := range uuidChannel {
+               fmt.Printf("Considering %s\n", uuid)

                if strings.Contains(uuid, "-j7d0g-") {
                        // This is a project (group)
                if strings.Contains(uuid, "-j7d0g-") {
                        // This is a project (group)

-                       cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+                       cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)

                        if err != nil {
                                exitcode = 1
                                return
                        if err != nil {
                                exitcode = 1
                                return


@@ -515,7 +595,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
                        // This is a container request
                        var crCsv map[string]float64
                } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
                        // This is a container request
                        var crCsv map[string]float64

-                       crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+                       crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)

                        if err != nil {
                                err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
                                exitcode = 2
                        if err != nil {
                                err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
                                exitcode = 2


@@ -531,21 +611,21 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                        // keep going.
                        logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
                } else {
                        // keep going.
                        logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
                } else {

-                       logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
+                       logger.Errorf("this argument does not look like a uuid: %s", uuid)

                        exitcode = 3
                        return
                }
        }
 
        if len(cost) == 0 {
                        exitcode = 3
                        return
                }
        }
 
        if len(cost) == 0 {

-               logger.Info("Nothing to do!\n")
+               logger.Info("Nothing to do!")

                return
        }
 
        var csv string
 
        csv = "# Aggregate cost accounting for uuids:\n"
                return
        }
 
        var csv string
 
        csv = "# Aggregate cost accounting for uuids:\n"

-       for _, uuid := range uuids {
+       for _, uuid := range c.uuids {

                csv += "# " + uuid + "\n"
        }
 
                csv += "# " + uuid + "\n"
        }
 


@@ -557,16 +637,16 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
 
        csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
 
 
        csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
 

-       if resultsDir != "" {
+       if c.resultsDir != "" {

                // Write the resulting CSV file
                // Write the resulting CSV file

-               aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+               aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"

                err = ioutil.WriteFile(aFile, []byte(csv), 0644)
                if err != nil {
                        err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
                        exitcode = 1
                        return
                }
                err = ioutil.WriteFile(aFile, []byte(csv), 0644)
                if err != nil {
                        err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
                        exitcode = 1
                        return
                }

-               logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+               logger.Infof("Aggregate cost accounting for all supplied uuids in %s", aFile)

        }
 
        // Output the total dollar amount on stdout
        }
 
        // Output the total dollar amount on stdout