1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
12 "git.arvados.org/arvados.git/lib/config"
13 "git.arvados.org/arvados.git/sdk/go/arvados"
14 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
15 "git.arvados.org/arvados.git/sdk/go/keepclient"
24 "github.com/sirupsen/logrus"
27 type nodeInfo struct {
28 // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
41 type arrayFlags []string
43 func (i *arrayFlags) String() string {
47 func (i *arrayFlags) Set(value string) error {
48 for _, s := range strings.Split(value, ",") {
54 func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
55 flags := flag.NewFlagSet("", flag.ContinueOnError)
56 flags.SetOutput(stderr)
57 flags.Usage = func() {
58 fmt.Fprintf(flags.Output(), `
60 %s [options ...] uuid [uuid ...]
62 This program analyzes the cost of Arvados container requests. For each uuid
63 supplied, it creates a CSV report that lists all the containers used to
64 fulfill the container request, together with the machine type and cost of
65 each container. At least one uuid must be specified.
67 When supplied with the uuid of a container request, it will calculate the
68 cost of that container request and all its children.
70 When supplied with the uuid of a collection, it will see if there is a
71 container_request uuid in the properties of the collection, and if so, it
72 will calculate the cost of that container request and all its children.
74 When supplied with a project uuid or when supplied with multiple container
75 request or collection uuids, it will create a CSV report for each supplied
76 uuid, as well as a CSV file with aggregate cost accounting for all supplied
77 uuids. The aggregate cost report takes container reuse into account: if a
78 container was reused between several container requests, its cost will only
83 - This program uses the cost data from config.yml at the time of the
84 execution of the container, stored in the 'node.json' file in its log
85 collection. If the cost data was not correctly configured at the time the
86 container was executed, the output from this program will be incorrect.
88 - If a container was run on a preemptible ("spot") instance, the cost data
89 reported by this program may be wildly inaccurate, because it does not have
90 access to the spot pricing in effect for the node then the container ran. The
91 UUID report file that is generated when the '-output' option is specified has
92 a column that indicates the preemptible state of the instance that ran the
95 - This program does not take into account overhead costs like the time spent
96 starting and stopping compute nodes that run containers, the cost of the
97 permanent cloud nodes that provide the Arvados services, the cost of data
98 stored in Arvados, etc.
100 - When provided with a project uuid, subprojects will not be considered.
102 In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
103 ARVADOS_API_TOKEN environment variables must be set.
105 This program prints the total dollar amount from the aggregate cost
106 accounting across all provided uuids on stdout.
108 When the '-output' option is specified, a set of CSV files with cost details
109 will be written to the provided directory.
113 flags.PrintDefaults()
115 loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
116 flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
117 flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
118 err = flags.Parse(args)
119 if err == flag.ErrHelp {
123 } else if err != nil {
131 err = fmt.Errorf("error: no uuid(s) provided")
136 lvl, err := logrus.ParseLevel(*loglevel)
143 logger.Debug("Caching disabled\n")
148 func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
149 statData, err := os.Stat(dir)
150 if os.IsNotExist(err) {
151 err = os.MkdirAll(dir, 0700)
153 return fmt.Errorf("error creating directory %s: %s", dir, err.Error())
156 if !statData.IsDir() {
157 return fmt.Errorf("the path %s is not a directory", dir)
163 func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) {
166 csv += container.UUID + ","
167 csv += string(container.State) + ","
168 if container.StartedAt != nil {
169 csv += container.StartedAt.String() + ","
174 var delta time.Duration
175 if container.FinishedAt != nil {
176 csv += container.FinishedAt.String() + ","
177 delta = container.FinishedAt.Sub(*container.StartedAt)
178 csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
184 if node.Properties.CloudNode.Price != 0 {
185 price = node.Properties.CloudNode.Price
186 size = node.Properties.CloudNode.Size
189 size = node.ProviderType
191 cost = delta.Seconds() / 3600 * price
192 csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
196 func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
198 if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
199 // We do not cache projects or collections, they have no final state
202 // See if we have a cached copy of this object
203 _, err := os.Stat(file)
207 data, err := ioutil.ReadFile(file)
209 logger.Errorf("error reading %q: %s", file, err)
212 err = json.Unmarshal(data, &object)
214 logger.Errorf("failed to unmarshal json: %s: %s", data, err)
218 // See if it is in a final state, if that makes sense
219 switch v := object.(type) {
220 case *arvados.ContainerRequest:
221 if v.State == arvados.ContainerRequestStateFinal {
223 logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
225 case *arvados.Container:
226 if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
228 logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
234 // Load an Arvados object.
235 func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) {
236 file := uuid + ".json"
244 homeDir, err := os.UserHomeDir()
247 logger.Info("Unable to determine current user home directory, not using cache")
249 cacheDir = homeDir + "/.cache/arvados/costanalyzer/"
250 err = ensureDirectory(logger, cacheDir)
253 logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error())
255 reload = loadCachedObject(logger, cacheDir+file, uuid, object)
263 if strings.Contains(uuid, "-j7d0g-") {
264 err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil)
265 } else if strings.Contains(uuid, "-xvhdp-") {
266 err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
267 } else if strings.Contains(uuid, "-dz642-") {
268 err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
269 } else if strings.Contains(uuid, "-4zz18-") {
270 err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)
272 err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
276 err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err)
279 encoded, err := json.MarshalIndent(object, "", " ")
281 err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err)
285 err = ioutil.WriteFile(cacheDir+file, encoded, 0644)
287 err = fmt.Errorf("error writing file %s:\n %s", file, err)
294 func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
295 if cr.LogUUID == "" {
296 err = errors.New("no log collection")
300 var collection arvados.Collection
301 err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
303 err = fmt.Errorf("error getting collection: %s", err)
307 var fs arvados.CollectionFileSystem
308 fs, err = collection.FileSystem(ac, kc)
310 err = fmt.Errorf("error opening collection as filesystem: %s", err)
314 f, err = fs.Open("node.json")
316 err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err)
320 err = json.NewDecoder(f).Decode(&node)
322 err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err)
328 func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
329 cost = make(map[string]float64)
331 var project arvados.Group
332 err = loadObject(logger, ac, uuid, uuid, cache, &project)
334 return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())
337 var childCrs map[string]interface{}
338 filterset := []arvados.Filter{
342 Operand: project.UUID,
345 Attr: "requesting_container_uuid",
350 err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
351 "filters": filterset,
355 return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
357 if value, ok := childCrs["items"]; ok {
358 logger.Infof("Collecting top level container requests in project %s\n", uuid)
359 items := value.([]interface{})
360 for _, item := range items {
361 itemMap := item.(map[string]interface{})
362 crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
364 return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error())
366 for k, v := range crCsv {
371 logger.Infof("No top level container requests found in project %s\n", uuid)
376 func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
378 cost = make(map[string]float64)
380 csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
382 var tmpTotalCost float64
383 var totalCost float64
386 if strings.Contains(uuid, "-4zz18-") {
387 // This is a collection, find the associated container request (if any)
388 var c arvados.Collection
389 err = loadObject(logger, ac, uuid, uuid, cache, &c)
391 return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
393 value, ok := c.Properties["container_request"]
395 return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
397 crUUID, ok = value.(string)
399 return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
403 // This is a container request, find the container
404 var cr arvados.ContainerRequest
405 err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)
407 return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
409 var container arvados.Container
410 err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
412 return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
415 topNode, err := getNode(arv, ac, kc, cr)
417 return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
419 tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
421 totalCost += tmpTotalCost
422 cost[container.UUID] = totalCost
424 // Find all container requests that have the container we found above as requesting_container_uuid
425 var childCrs arvados.ContainerRequestList
426 filterset := []arvados.Filter{
428 Attr: "requesting_container_uuid",
430 Operand: container.UUID,
432 err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
433 "filters": filterset,
437 return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
439 logger.Infof("Collecting child containers for container request %s", crUUID)
440 for _, cr2 := range childCrs.Items {
442 node, err := getNode(arv, ac, kc, cr2)
444 return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
446 logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
447 var c2 arvados.Container
448 err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
450 return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
452 tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2)
453 cost[cr2.ContainerUUID] = tmpTotalCost
455 totalCost += tmpTotalCost
457 logger.Info(" done\n")
459 csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
461 if resultsDir != "" {
462 // Write the resulting CSV file
463 fName := resultsDir + "/" + crUUID + ".csv"
464 err = ioutil.WriteFile(fName, []byte(csv), 0644)
466 return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
468 logger.Infof("\nUUID report in %s\n\n", fName)
474 func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
475 exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
479 if resultsDir != "" {
480 err = ensureDirectory(logger, resultsDir)
487 // Arvados Client setup
488 arv, err := arvadosclient.MakeArvadosClient()
490 err = fmt.Errorf("error creating Arvados object: %s", err)
494 kc, err := keepclient.MakeKeepClient(arv)
496 err = fmt.Errorf("error creating Keep object: %s", err)
501 ac := arvados.NewClientFromEnv()
503 cost := make(map[string]float64)
504 for _, uuid := range uuids {
505 if strings.Contains(uuid, "-j7d0g-") {
506 // This is a project (group)
507 cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
512 for k, v := range cost {
515 } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
516 // This is a container request
517 var crCsv map[string]float64
518 crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
520 err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
524 for k, v := range crCsv {
527 } else if strings.Contains(uuid, "-tpzed-") {
528 // This is a user. The "Home" project for a user is not a real project.
529 // It is identified by the user uuid. As such, cost analysis for the
530 // "Home" project is not supported by this program. Skip this uuid, but
532 logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
534 logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
541 logger.Info("Nothing to do!\n")
547 csv = "# Aggregate cost accounting for uuids:\n"
548 for _, uuid := range uuids {
549 csv += "# " + uuid + "\n"
553 for k, v := range cost {
554 csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n"
558 csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
560 if resultsDir != "" {
561 // Write the resulting CSV file
562 aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
563 err = ioutil.WriteFile(aFile, []byte(csv), 0644)
565 err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
569 logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
572 // Output the total dollar amount on stdout
573 fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))