"strings"
"time"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
+const timestampFormat = "2006-01-02T15:04:05"
+
type nodeInfo struct {
// Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
Properties struct {
return nil
}
-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) {
+ var beginStr, endStr string
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...] uuid [uuid ...]
+ %s [options ...] [UUID ...]
+
+ This program analyzes the cost of Arvados container requests and calculates
+ the total cost across all requests. At least one UUID or a timestamp range
+ must be specified.
- This program analyzes the cost of Arvados container requests. For each uuid
- supplied, it creates a CSV report that lists all the containers used to
- fulfill the container request, together with the machine type and cost of
- each container. At least one uuid must be specified.
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory. Each file is a CSV report that lists
+ all the containers used to fulfill the container request, together with the
+ machine type and cost of each container.
- When supplied with the uuid of a container request, it will calculate the
+ When supplied with the UUID of a container request, it will calculate the
cost of that container request and all its children.
- When supplied with the uuid of a collection, it will see if there is a
- container_request uuid in the properties of the collection, and if so, it
+ When supplied with the UUID of a collection, it will see if there is a
+ container_request UUID in the properties of the collection, and if so, it
will calculate the cost of that container request and all its children.
- When supplied with a project uuid or when supplied with multiple container
- request or collection uuids, it will create a CSV report for each supplied
- uuid, as well as a CSV file with aggregate cost accounting for all supplied
- uuids. The aggregate cost report takes container reuse into account: if a
- container was reused between several container requests, its cost will only
- be counted once.
+ When supplied with a project UUID or when supplied with multiple container
+ request or collection UUIDs, it will calculate the total cost for all
+ supplied UUIDs.
+
+ When supplied with a 'begin' and 'end' timestamp (format:
+ %s), it will calculate the cost for all top-level container
+ requests whose containers finished during the specified interval.
+
+ The total cost calculation takes container reuse into account: if a container
+ was reused between several container requests, its cost will only be counted
+ once.
Caveats:
permanent cloud nodes that provide the Arvados services, the cost of data
stored in Arvados, etc.
- - When provided with a project uuid, subprojects will not be considered.
+ - When provided with a project UUID, subprojects will not be considered.
- In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
This program prints the total dollar amount from the aggregate cost
- accounting across all provided uuids on stdout.
-
- When the '-output' option is specified, a set of CSV files with cost details
- will be written to the provided directory.
+ accounting across all provided UUIDs on stdout.
Options:
-`, prog)
+`, prog, timestampFormat)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
- flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+ flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports")
+ flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+ flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))
+ flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
exitCode = 2
return
}
- uuids = flags.Args()
+ c.uuids = flags.Args()
+
+ if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be specified")
+ exitCode = 2
+ return
+ }
+
+ if len(beginStr) != 0 {
+ var errB, errE error
+ c.begin, errB = time.Parse(timestampFormat, beginStr)
+ c.end, errE = time.Parse(timestampFormat, endStr)
+ if (errB != nil) || (errE != nil) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE)
+ exitCode = 2
+ return
+ }
+ }
- if len(uuids) < 1 {
+ if (len(c.uuids) < 1) && (len(beginStr) == 0) {
flags.Usage()
err = fmt.Errorf("error: no uuid(s) provided")
exitCode = 2
return
}
logger.SetLevel(lvl)
- if !cache {
+ if !c.cache {
logger.Debug("Caching disabled")
}
return
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
+ fmt.Printf("Processing %s\n", uuid)
var crUUID = uuid
if strings.Contains(uuid, "-4zz18-") {
topNode, err := getNode(arv, ac, kc, cr)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+ return nil, nil
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", crUUID)
+ logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+ continue
}
logger.Debug("\nChild container: " + cr2.ContainerUUID)
var c2 arvados.Container
return
}
-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
- exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, err = c.parseFlags(prog, args, logger, stderr)
+
if exitcode != 0 {
return
}
- if resultsDir != "" {
- err = ensureDirectory(logger, resultsDir)
+ if c.resultsDir != "" {
+ err = ensureDirectory(logger, c.resultsDir)
if err != nil {
exitcode = 3
return
}
}
+ uuidChannel := make(chan string)
+
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
ac := arvados.NewClientFromEnv()
+ // Populate uuidChannel with the requested uuid list
+ go func() {
+ defer close(uuidChannel)
+ for _, uuid := range c.uuids {
+ uuidChannel <- uuid
+ }
+
+ if !c.begin.IsZero() {
+ initialParams := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}},
+ Order: "created_at",
+ }
+ params := initialParams
+ for {
+ // This list variable must be a new one declared
+ // inside the loop: otherwise, items in the API
+ // response would get deep-merged into the items
+ // loaded in previous iterations.
+ var list arvados.ContainerRequestList
+
+ err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+ if err != nil {
+ logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+ break
+ }
+ if len(list.Items) == 0 {
+ break
+ }
+
+ for _, i := range list.Items {
+ uuidChannel <- i.UUID
+ }
+ params.Offset += len(list.Items)
+ }
+
+ }
+ }()
+
cost := make(map[string]float64)
- for _, uuid := range uuids {
+
+ for uuid := range uuidChannel {
+ fmt.Printf("Considering %s\n", uuid)
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
- cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
exitcode = 1
return
} else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
- crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
var csv string
csv = "# Aggregate cost accounting for uuids:\n"
- for _, uuid := range uuids {
+ for _, uuid := range c.uuids {
csv += "# " + uuid + "\n"
}
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- if resultsDir != "" {
+ if c.resultsDir != "" {
// Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
projects / arvados.git / blobdiff