// Modern
ProviderType string
Price float64
+ Preemptible bool
}
type arrayFlags []string
}
func (i *arrayFlags) Set(value string) error {
- *i = append(*i, value)
+ for _, s := range strings.Split(value, ",") {
+ *i = append(*i, s)
+ }
return nil
}
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...]
+ %s [options ...] <uuid> ...
This program analyzes the cost of Arvados container requests. For each uuid
supplied, it creates a CSV report that lists all the containers used to
fulfill the container request, together with the machine type and cost of
- each container.
+ each container. At least one uuid must be specified.
When supplied with the uuid of a container request, it will calculate the
cost of that container request and all its children.
was fulfilled. This program uses the cost data stored at the time of the
execution of the container, stored in the 'node.json' file in its log
collection.
+ - if a container was run on a preemptible ("spot") instance, the cost data
+ reported by this program may be wildly inaccurate, because it does not have
+ access to the spot pricing in effect for the node then the container ran. The
+ UUID report file that is generated when the '-output' option is specified has
+ a column that indicates the preemptible state of the instance that ran the
+ container.
In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
+ This program prints the total dollar amount from the aggregate cost
+ accounting across all provided uuids on stdout.
+
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory.
+
Options:
`, prog)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports (required)")
- flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once. (required)")
+ flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
exitCode = 2
return
}
+ uuids = flags.Args()
if len(uuids) < 1 {
flags.Usage()
- err = fmt.Errorf("Error: no uuid(s) provided")
- exitCode = 2
- return
- }
-
- if resultsDir == "" {
- flags.Usage()
- err = fmt.Errorf("Error: output directory must be specified")
+ err = fmt.Errorf("error: no uuid(s) provided")
exitCode = 2
return
}
size = node.ProviderType
}
cost = delta.Seconds() / 3600 * price
- csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
return
}
func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
if cr.LogUUID == "" {
- err = errors.New("No log collection")
+ err = errors.New("no log collection")
return
}
cost = make(map[string]float64)
- csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
if !ok {
return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
}
- crUUID = value.(string)
+ crUUID, ok = value.(string)
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
+ }
}
// This is a container request, find the container
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
var container arvados.Container
- err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+ err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
if err != nil {
return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", uuid)
+ logger.Infof("Collecting child containers for container request %s", crUUID)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
- err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+ err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
- // Write the resulting CSV file
- fName := resultsDir + "/" + uuid + ".csv"
- err = ioutil.WriteFile(fName, []byte(csv), 0644)
- if err != nil {
- return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ fName := resultsDir + "/" + crUUID + ".csv"
+ err = ioutil.WriteFile(fName, []byte(csv), 0644)
+ if err != nil {
+ return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ }
+ logger.Infof("\nUUID report in %s\n\n", fName)
}
- logger.Infof("\nUUID report in %s\n\n", fName)
return
}
if exitcode != 0 {
return
}
- err = ensureDirectory(logger, resultsDir)
- if err != nil {
- exitcode = 3
- return
+ if resultsDir != "" {
+ err = ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
}
// Arvados Client setup
var crCsv map[string]float64
crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
+ err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
// It is identified by the user uuid. As such, cost analysis for the
// "Home" project is not supported by this program. Skip this uuid, but
// keep going.
- logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
+ } else {
+ logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
+ exitcode = 3
+ return
}
}
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- // Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
- err = ioutil.WriteFile(aFile, []byte(csv), 0644)
- if err != nil {
- err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
- exitcode = 1
- return
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+ if err != nil {
+ err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
+ exitcode = 1
+ return
+ }
+ logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
}
- logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+
+ // Output the total dollar amount on stdout
+ fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
+
return
}