X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e1937e57fe2c0e99b6b636049142cc7598f80231..d8d6bca4b5db4851a29473f08dc600816c977a21:/lib/costanalyzer/costanalyzer.go diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index 8f5395f58d..37e655e53a 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -17,7 +17,6 @@ import ( "io/ioutil" "net/http" "os" - "os/user" "strconv" "strings" "time" @@ -36,6 +35,7 @@ type nodeInfo struct { // Modern ProviderType string Price float64 + Preemptible bool } type arrayFlags []string @@ -45,7 +45,9 @@ func (i *arrayFlags) String() string { } func (i *arrayFlags) Set(value string) error { - *i = append(*i, value) + for _, s := range strings.Split(value, ",") { + *i = append(*i, s) + } return nil } @@ -55,20 +57,26 @@ func parseFlags(prog string, args []string, loader *config.Loader, logger *logru flags.Usage = func() { fmt.Fprintf(flags.Output(), ` Usage: - %s [options ...] + %s [options ...] ... This program analyzes the cost of Arvados container requests. For each uuid supplied, it creates a CSV report that lists all the containers used to fulfill the container request, together with the machine type and cost of - each container. + each container. At least one uuid must be specified. When supplied with the uuid of a container request, it will calculate the - cost of that container request and all its children. When suplied with a - project uuid or when supplied with multiple container request uuids, it will - create a CSV report for each supplied uuid, as well as a CSV file with - aggregate cost accounting for all supplied uuids. The aggregate cost report - takes container reuse into account: if a container was reused between several - container requests, its cost will only be counted once. + cost of that container request and all its children. + + When supplied with the uuid of a collection, it will see if there is a + container_request uuid in the properties of the collection, and if so, it + will calculate the cost of that container request and all its children. + + When supplied with a project uuid or when supplied with multiple container + request or collection uuids, it will create a CSV report for each supplied + uuid, as well as a CSV file with aggregate cost accounting for all supplied + uuids. The aggregate cost report takes container reuse into account: if a + container was reused between several container requests, its cost will only + be counted once. To get the node costs, the progam queries the Arvados API for current cost data for each node type used. This means that the reported cost always @@ -83,17 +91,28 @@ Usage: was fulfilled. This program uses the cost data stored at the time of the execution of the container, stored in the 'node.json' file in its log collection. + - if a container was run on a preemptible ("spot") instance, the cost data + reported by this program may be wildly inaccurate, because it does not have + access to the spot pricing in effect for the node then the container ran. The + UUID report file that is generated when the '-output' option is specified has + a column that indicates the preemptible state of the instance that ran the + container. In order to get the data for the uuids supplied, the ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables must be set. + This program prints the total dollar amount from the aggregate cost + accounting across all provided uuids on stdout. + + When the '-output' option is specified, a set of CSV files with cost details + will be written to the provided directory. + Options: `, prog) flags.PrintDefaults() } loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)") - resultsDir = *flags.String("output", "results", "output `directory` for the CSV reports") - flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once.") + flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports") flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects") err = flags.Parse(args) if err == flag.ErrHelp { @@ -104,10 +123,11 @@ Options: exitCode = 2 return } + uuids = flags.Args() if len(uuids) < 1 { flags.Usage() - err = fmt.Errorf("Error: no uuid(s) provided") + err = fmt.Errorf("error: no uuid(s) provided") exitCode = 2 return } @@ -168,14 +188,14 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container size = node.ProviderType } cost = delta.Seconds() / 3600 * price - csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" + csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" return } func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) { reload = true - if strings.Contains(uuid, "-j7d0g-") { - // We do not cache projects, they have no final state + if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") { + // We do not cache projects or collections, they have no final state return } // See if we have a cached copy of this object @@ -220,12 +240,12 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str if !cache { reload = true } else { - user, err := user.Current() + homeDir, err := os.UserHomeDir() if err != nil { reload = true - logger.Info("Unable to determine current user, not using cache") + logger.Info("Unable to determine current user home directory, not using cache") } else { - cacheDir = user.HomeDir + "/.cache/arvados/costanalyzer/" + cacheDir = homeDir + "/.cache/arvados/costanalyzer/" err = ensureDirectory(logger, cacheDir) if err != nil { reload = true @@ -245,6 +265,8 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil) } else if strings.Contains(uuid, "-dz642-") { err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-4zz18-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil) } else { err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err) return @@ -270,7 +292,7 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) { if cr.LogUUID == "" { - err = errors.New("No log collection") + err = errors.New("no log collection") return } @@ -303,7 +325,6 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien } func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) { - cost = make(map[string]float64) var project arvados.Group @@ -355,19 +376,37 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado cost = make(map[string]float64) - csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n" + csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n" var tmpCsv string var tmpTotalCost float64 var totalCost float64 + var crUUID = uuid + if strings.Contains(uuid, "-4zz18-") { + // This is a collection, find the associated container request (if any) + var c arvados.Collection + err = loadObject(logger, ac, uuid, uuid, cache, &c) + if err != nil { + return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err) + } + value, ok := c.Properties["container_request"] + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid) + } + crUUID, ok = value.(string) + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid) + } + } + // This is a container request, find the container var cr arvados.ContainerRequest - err = loadObject(logger, ac, uuid, uuid, cache, &cr) + err = loadObject(logger, ac, crUUID, crUUID, cache, &cr) if err != nil { return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err) } var container arvados.Container - err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container) + err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container) if err != nil { return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err) } @@ -396,7 +435,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado if err != nil { return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) } - logger.Infof("Collecting child containers for container request %s", uuid) + logger.Infof("Collecting child containers for container request %s", crUUID) for _, cr2 := range childCrs.Items { logger.Info(".") node, err := getNode(arv, ac, kc, cr2) @@ -405,7 +444,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado } logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n") var c2 arvados.Container - err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2) + err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2) if err != nil { return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err) } @@ -418,13 +457,15 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n" - // Write the resulting CSV file - fName := resultsDir + "/" + uuid + ".csv" - err = ioutil.WriteFile(fName, []byte(csv), 0644) - if err != nil { - return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error()) + if resultsDir != "" { + // Write the resulting CSV file + fName := resultsDir + "/" + crUUID + ".csv" + err = ioutil.WriteFile(fName, []byte(csv), 0644) + if err != nil { + return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error()) + } + logger.Infof("\nUUID report in %s\n\n", fName) } - logger.Infof("\nUUID report in %s\n\n", fName) return } @@ -434,10 +475,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log if exitcode != 0 { return } - err = ensureDirectory(logger, resultsDir) - if err != nil { - exitcode = 3 - return + if resultsDir != "" { + err = ensureDirectory(logger, resultsDir) + if err != nil { + exitcode = 3 + return + } } // Arvados Client setup @@ -468,12 +511,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log for k, v := range cost { cost[k] = v } - } else if strings.Contains(uuid, "-xvhdp-") { + } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") { // This is a container request var crCsv map[string]float64 crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache) if err != nil { - err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error()) + err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error()) exitcode = 2 return } @@ -485,7 +528,11 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log // It is identified by the user uuid. As such, cost analysis for the // "Home" project is not supported by this program. Skip this uuid, but // keep going. - logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid) + logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid) + } else { + logger.Errorf("this argument does not look like a uuid: %s\n", uuid) + exitcode = 3 + return } } @@ -509,14 +556,20 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n" - // Write the resulting CSV file - aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv" - err = ioutil.WriteFile(aFile, []byte(csv), 0644) - if err != nil { - err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error()) - exitcode = 1 - return + if resultsDir != "" { + // Write the resulting CSV file + aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv" + err = ioutil.WriteFile(aFile, []byte(csv), 0644) + if err != nil { + err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error()) + exitcode = 1 + return + } + logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile) } - logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile) + + // Output the total dollar amount on stdout + fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64)) + return }