"io/ioutil"
"net/http"
"os"
- "os/user"
"strconv"
"strings"
"time"
// Modern
ProviderType string
Price float64
+ Preemptible bool
}
type arrayFlags []string
}
func (i *arrayFlags) Set(value string) error {
- *i = append(*i, value)
+ for _, s := range strings.Split(value, ",") {
+ *i = append(*i, s)
+ }
return nil
}
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...]
+ %s [options ...] uuid [uuid ...]
This program analyzes the cost of Arvados container requests. For each uuid
supplied, it creates a CSV report that lists all the containers used to
fulfill the container request, together with the machine type and cost of
- each container.
+ each container. At least one uuid must be specified.
When supplied with the uuid of a container request, it will calculate the
- cost of that container request and all its children. When suplied with a
- project uuid or when supplied with multiple container request uuids, it will
- create a CSV report for each supplied uuid, as well as a CSV file with
- aggregate cost accounting for all supplied uuids. The aggregate cost report
- takes container reuse into account: if a container was reused between several
- container requests, its cost will only be counted once.
-
- To get the node costs, the progam queries the Arvados API for current cost
- data for each node type used. This means that the reported cost always
- reflects the cost data as currently defined in the Arvados API configuration
- file.
+ cost of that container request and all its children.
+
+ When supplied with the uuid of a collection, it will see if there is a
+ container_request uuid in the properties of the collection, and if so, it
+ will calculate the cost of that container request and all its children.
+
+ When supplied with a project uuid or when supplied with multiple container
+ request or collection uuids, it will create a CSV report for each supplied
+ uuid, as well as a CSV file with aggregate cost accounting for all supplied
+ uuids. The aggregate cost report takes container reuse into account: if a
+ container was reused between several container requests, its cost will only
+ be counted once.
Caveats:
- - the Arvados API configuration cost data may be out of sync with the cloud
- provider.
- - when generating reports for older container requests, the cost data in the
- Arvados API configuration file may have changed since the container request
- was fulfilled. This program uses the cost data stored at the time of the
+
+ - This program uses the cost data from config.yml at the time of the
execution of the container, stored in the 'node.json' file in its log
- collection.
+ collection. If the cost data was not correctly configured at the time the
+ container was executed, the output from this program will be incorrect.
+
+ - If a container was run on a preemptible ("spot") instance, the cost data
+ reported by this program may be wildly inaccurate, because it does not have
+ access to the spot pricing in effect for the node then the container ran. The
+ UUID report file that is generated when the '-output' option is specified has
+ a column that indicates the preemptible state of the instance that ran the
+ container.
+
+ - This program does not take into account overhead costs like the time spent
+ starting and stopping compute nodes that run containers, the cost of the
+ permanent cloud nodes that provide the Arvados services, the cost of data
+ stored in Arvados, etc.
+
+ - When provided with a project uuid, subprojects will not be considered.
In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
+ This program prints the total dollar amount from the aggregate cost
+ accounting across all provided uuids on stdout.
+
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory.
+
Options:
`, prog)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- resultsDir = *flags.String("output", "results", "output `directory` for the CSV reports")
- flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once.")
+ flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
exitCode = 2
return
}
+ uuids = flags.Args()
if len(uuids) < 1 {
flags.Usage()
- err = fmt.Errorf("Error: no uuid(s) provided")
+ err = fmt.Errorf("error: no uuid(s) provided")
exitCode = 2
return
}
size = node.ProviderType
}
cost = delta.Seconds() / 3600 * price
- csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
return
}
func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
reload = true
- if strings.Contains(uuid, "-j7d0g-") {
- // We do not cache projects, they have no final state
+ if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+ // We do not cache projects or collections, they have no final state
return
}
// See if we have a cached copy of this object
if !cache {
reload = true
} else {
- user, err := user.Current()
+ homeDir, err := os.UserHomeDir()
if err != nil {
reload = true
- logger.Info("Unable to determine current user, not using cache")
+ logger.Info("Unable to determine current user home directory, not using cache")
} else {
- cacheDir = user.HomeDir + "/.cache/arvados/costanalyzer/"
+ cacheDir = homeDir + "/.cache/arvados/costanalyzer/"
err = ensureDirectory(logger, cacheDir)
if err != nil {
reload = true
err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
} else if strings.Contains(uuid, "-dz642-") {
err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-4zz18-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)
} else {
err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
return
func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
if cr.LogUUID == "" {
- err = errors.New("No log collection")
+ err = errors.New("no log collection")
return
}
}
func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
-
cost = make(map[string]float64)
var project arvados.Group
cost = make(map[string]float64)
- csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
+ var crUUID = uuid
+ if strings.Contains(uuid, "-4zz18-") {
+ // This is a collection, find the associated container request (if any)
+ var c arvados.Collection
+ err = loadObject(logger, ac, uuid, uuid, cache, &c)
+ if err != nil {
+ return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+ }
+ value, ok := c.Properties["container_request"]
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+ }
+ crUUID, ok = value.(string)
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
+ }
+ }
+
// This is a container request, find the container
var cr arvados.ContainerRequest
- err = loadObject(logger, ac, uuid, uuid, cache, &cr)
+ err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
+ if len(cr.ContainerUUID) == 0 {
+ // Nothing to do! E.g. a CR in 'Uncommitted' state.
+ logger.Infof("No container associated with container request %s, skipping\n", crUUID)
+ return nil, nil
+ }
var container arvados.Container
- err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+ err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
if err != nil {
return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", uuid)
+ logger.Infof("Collecting child containers for container request %s", crUUID)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
- err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+ err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
- // Write the resulting CSV file
- fName := resultsDir + "/" + uuid + ".csv"
- err = ioutil.WriteFile(fName, []byte(csv), 0644)
- if err != nil {
- return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ fName := resultsDir + "/" + crUUID + ".csv"
+ err = ioutil.WriteFile(fName, []byte(csv), 0644)
+ if err != nil {
+ return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ }
+ logger.Infof("\nUUID report in %s\n\n", fName)
}
- logger.Infof("\nUUID report in %s\n\n", fName)
return
}
if exitcode != 0 {
return
}
- err = ensureDirectory(logger, resultsDir)
- if err != nil {
- exitcode = 3
- return
+ if resultsDir != "" {
+ err = ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
}
// Arvados Client setup
for k, v := range cost {
cost[k] = v
}
- } else if strings.Contains(uuid, "-xvhdp-") {
+ } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+ err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
// It is identified by the user uuid. As such, cost analysis for the
// "Home" project is not supported by this program. Skip this uuid, but
// keep going.
- logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
+ } else {
+ logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
+ exitcode = 3
+ return
}
}
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- // Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
- err = ioutil.WriteFile(aFile, []byte(csv), 0644)
- if err != nil {
- err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
- exitcode = 1
- return
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+ if err != nil {
+ err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
+ exitcode = 1
+ return
+ }
+ logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
}
- logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+
+ // Output the total dollar amount on stdout
+ fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
+
return
}
projects / arvados.git / blobdiff