"errors"
"flag"
"fmt"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
+const timestampFormat = "2006-01-02T15:04:05"
+
type nodeInfo struct {
// Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
Properties struct {
// Modern
ProviderType string
Price float64
+ Preemptible bool
}
type arrayFlags []string
return nil
}
-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) {
+ var beginStr, endStr string
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...] <uuid> ...
+ %s [options ...] [UUID ...]
+
+ This program analyzes the cost of Arvados container requests and calculates
+ the total cost across all requests. At least one UUID or a timestamp range
+ must be specified.
- This program analyzes the cost of Arvados container requests. For each uuid
- supplied, it creates a CSV report that lists all the containers used to
- fulfill the container request, together with the machine type and cost of
- each container. At least one uuid must be specified.
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory. Each file is a CSV report that lists
+ all the containers used to fulfill the container request, together with the
+ machine type and cost of each container.
- When supplied with the uuid of a container request, it will calculate the
+ When supplied with the UUID of a container request, it will calculate the
cost of that container request and all its children.
- When supplied with the uuid of a collection, it will see if there is a
- container_request uuid in the properties of the collection, and if so, it
+ When supplied with the UUID of a collection, it will see if there is a
+ container_request UUID in the properties of the collection, and if so, it
will calculate the cost of that container request and all its children.
- When supplied with a project uuid or when supplied with multiple container
- request or collection uuids, it will create a CSV report for each supplied
- uuid, as well as a CSV file with aggregate cost accounting for all supplied
- uuids. The aggregate cost report takes container reuse into account: if a
- container was reused between several container requests, its cost will only
- be counted once.
+ When supplied with a project UUID or when supplied with multiple container
+ request or collection UUIDs, it will calculate the total cost for all
+ supplied UUIDs.
- To get the node costs, the progam queries the Arvados API for current cost
- data for each node type used. This means that the reported cost always
- reflects the cost data as currently defined in the Arvados API configuration
- file.
+ When supplied with a 'begin' and 'end' timestamp (format:
+ %s), it will calculate the cost for all top-level container
+ requests whose containers finished during the specified interval.
+
+ The total cost calculation takes container reuse into account: if a container
+ was reused between several container requests, its cost will only be counted
+ once.
Caveats:
- - the Arvados API configuration cost data may be out of sync with the cloud
- provider.
- - when generating reports for older container requests, the cost data in the
- Arvados API configuration file may have changed since the container request
- was fulfilled. This program uses the cost data stored at the time of the
+
+ - This program uses the cost data from config.yml at the time of the
execution of the container, stored in the 'node.json' file in its log
- collection.
+ collection. If the cost data was not correctly configured at the time the
+ container was executed, the output from this program will be incorrect.
- In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ - If a container was run on a preemptible ("spot") instance, the cost data
+ reported by this program may be wildly inaccurate, because it does not have
+ access to the spot pricing in effect for the node then the container ran. The
+ UUID report file that is generated when the '-output' option is specified has
+ a column that indicates the preemptible state of the instance that ran the
+ container.
+
+ - This program does not take into account overhead costs like the time spent
+ starting and stopping compute nodes that run containers, the cost of the
+ permanent cloud nodes that provide the Arvados services, the cost of data
+ stored in Arvados, etc.
+
+ - When provided with a project UUID, subprojects will not be considered.
+
+ In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
This program prints the total dollar amount from the aggregate cost
- accounting across all provided uuids on stdout.
-
- When the '-output' option is specified, a set of CSV files with cost details
- will be written to the provided directory.
+ accounting across all provided UUIDs on stdout.
Options:
-`, prog)
+`, prog, timestampFormat)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
- flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+ flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports")
+ flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+ flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))
+ flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
exitCode = 2
return
}
- uuids = flags.Args()
+ c.uuids = flags.Args()
- if len(uuids) < 1 {
+ if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {
flags.Usage()
- err = fmt.Errorf("Error: no uuid(s) provided")
+ err = fmt.Errorf("When specifying a date range, both begin and end must be specified")
exitCode = 2
return
}
+ if len(beginStr) != 0 {
+ var errB, errE error
+ c.begin, errB = time.Parse(timestampFormat, beginStr)
+ c.end, errE = time.Parse(timestampFormat, endStr)
+ if (errB != nil) || (errE != nil) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE)
+ exitCode = 2
+ return
+ }
+ }
+
+ if (len(c.uuids) < 1) && (len(beginStr) == 0) {
+ flags.Usage()
+ err = fmt.Errorf("error: no uuid(s) provided")
+ exitCode = 2
+ return
+ }
+ fmt.Printf("UUIDS: %s\n", c.uuids)
+
lvl, err := logrus.ParseLevel(*loglevel)
if err != nil {
exitCode = 2
return
}
logger.SetLevel(lvl)
- if !cache {
+ if !c.cache {
logger.Debug("Caching disabled\n")
}
return
size = node.ProviderType
}
cost = delta.Seconds() / 3600 * price
- csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
return
}
func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
if cr.LogUUID == "" {
- err = errors.New("No log collection")
+ err = errors.New("no log collection")
return
}
cost = make(map[string]float64)
- csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
+ fmt.Printf("Processing %s\n", uuid)
var crUUID = uuid
if strings.Contains(uuid, "-4zz18-") {
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
+ if len(cr.ContainerUUID) == 0 {
+ // Nothing to do! E.g. a CR in 'Uncommitted' state.
+ logger.Infof("No container associated with container request %s, skipping\n", crUUID)
+ return nil, nil
+ }
var container arvados.Container
- err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+ err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
if err != nil {
return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
topNode, err := getNode(arv, ac, kc, cr)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+ return nil, nil
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", uuid)
+ logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+ continue
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
- err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+ err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
if resultsDir != "" {
// Write the resulting CSV file
- fName := resultsDir + "/" + uuid + ".csv"
+ fName := resultsDir + "/" + crUUID + ".csv"
err = ioutil.WriteFile(fName, []byte(csv), 0644)
if err != nil {
return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
return
}
-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
- exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, err = c.parseFlags(prog, args, logger, stderr)
+
if exitcode != 0 {
return
}
- if resultsDir != "" {
- err = ensureDirectory(logger, resultsDir)
+ if c.resultsDir != "" {
+ err = ensureDirectory(logger, c.resultsDir)
if err != nil {
exitcode = 3
return
}
}
+ uuidChannel := make(chan string)
+
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
ac := arvados.NewClientFromEnv()
+ // Populate uuidChannel with the requested uuid list
+ go func() {
+ defer close(uuidChannel)
+ for _, uuid := range c.uuids {
+ uuidChannel <- uuid
+ }
+
+ if !c.begin.IsZero() {
+ initialParams := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}},
+ Order: "created_at",
+ }
+ params := initialParams
+ for {
+ // This list variable must be a new one declared
+ // inside the loop: otherwise, items in the API
+ // response would get deep-merged into the items
+ // loaded in previous iterations.
+ var list arvados.ContainerRequestList
+
+ err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+ if err != nil {
+ logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+ break
+ }
+ if len(list.Items) == 0 {
+ break
+ }
+
+ for _, i := range list.Items {
+ uuidChannel <- i.UUID
+ }
+ params.Offset += len(list.Items)
+ }
+
+ }
+ }()
+
cost := make(map[string]float64)
- for _, uuid := range uuids {
+
+ for uuid := range uuidChannel {
+ fmt.Printf("Considering %s\n", uuid)
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
- cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
exitcode = 1
return
} else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
- crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
- err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
+ err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
// It is identified by the user uuid. As such, cost analysis for the
// "Home" project is not supported by this program. Skip this uuid, but
// keep going.
- logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
} else {
- logger.Errorf("This argument does not look like a uuid: %s\n", uuid)
+ logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
exitcode = 3
return
}
var csv string
csv = "# Aggregate cost accounting for uuids:\n"
- for _, uuid := range uuids {
+ for _, uuid := range c.uuids {
csv += "# " + uuid + "\n"
}
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- if resultsDir != "" {
+ if c.resultsDir != "" {
// Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
- err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
+ err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
exitcode = 1
return
}
projects / arvados.git / blobdiff