X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8d62d02834f289b30adafdeeb824ac03da5ff745..5b554bbe0b9104e8a34b87d5570cbf87f0308bce:/lib/costanalyzer/costanalyzer.go diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index d754e88757..edaaa5bd17 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -5,18 +5,15 @@ package costanalyzer import ( - "bytes" "encoding/json" "errors" "flag" "fmt" - "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/keepclient" "io" "io/ioutil" - "log" "net/http" "os" "strconv" @@ -26,54 +23,19 @@ import ( "github.com/sirupsen/logrus" ) -// Dict is a helper type so we don't have to write out 'map[string]interface{}' every time. -type Dict map[string]interface{} - -// LegacyNodeInfo is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3) -// Example: -// { -// "total_cpu_cores":2, -// "total_scratch_mb":33770, -// "cloud_node": -// { -// "price":0.1, -// "size":"m4.large" -// }, -// "total_ram_mb":7986 -// } -type LegacyNodeInfo struct { - CPUCores int64 `json:"total_cpu_cores"` - ScratchMb int64 `json:"total_scratch_mb"` - RAMMb int64 `json:"total_ram_mb"` - CloudNode LegacyCloudNode `json:"cloud_node"` -} - -// LegacyCloudNode is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3) -type LegacyCloudNode struct { - Price float64 `json:"price"` - Size string `json:"size"` -} +const timestampFormat = "2006-01-02T15:04:05" -// Node is a struct for records created by Arvados Dispatch Cloud (Arvados >= 2.0.0) -// Example: -// { -// "Name": "Standard_D1_v2", -// "ProviderType": "Standard_D1_v2", -// "VCPUs": 1, -// "RAM": 3584000000, -// "Scratch": 50000000000, -// "IncludedScratch": 50000000000, -// "AddedScratch": 0, -// "Price": 0.057, -// "Preemptible": false -//} -type Node struct { - VCPUs int64 - Scratch int64 - RAM int64 - Price float64 - Name string +type nodeInfo struct { + // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3) + Properties struct { + CloudNode struct { + Price float64 + Size string + } `json:"cloud_node"` + } + // Modern ProviderType string + Price float64 Preemptible bool } @@ -84,65 +46,118 @@ func (i *arrayFlags) String() string { } func (i *arrayFlags) Set(value string) error { - *i = append(*i, value) + for _, s := range strings.Split(value, ",") { + *i = append(*i, s) + } return nil } -func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string) { +func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) { + var beginStr, endStr string flags := flag.NewFlagSet("", flag.ContinueOnError) flags.SetOutput(stderr) flags.Usage = func() { fmt.Fprintf(flags.Output(), ` Usage: - %s [options ...] - - This program analyzes the cost of Arvados container requests. For each uuid - supplied, it creates a CSV report that lists all the containers used to - fulfill the container request, together with the machine type and cost of - each container. - - When supplied with the uuid of a container request, it will calculate the - cost of that container request and all its children. When suplied with a - project uuid or when supplied with multiple container request uuids, it will - create a CSV report for each supplied uuid, as well as a CSV file with - aggregate cost accounting for all supplied uuids. The aggregate cost report - takes container reuse into account: if a container was reused between several - container requests, its cost will only be counted once. - - To get the node costs, the progam queries the Arvados API for current cost - data for each node type used. This means that the reported cost always - reflects the cost data as currently defined in the Arvados API configuration - file. + %s [options ...] [UUID ...] + + This program analyzes the cost of Arvados container requests and calculates + the total cost across all requests. At least one UUID or a timestamp range + must be specified. + + When the '-output' option is specified, a set of CSV files with cost details + will be written to the provided directory. Each file is a CSV report that lists + all the containers used to fulfill the container request, together with the + machine type and cost of each container. + + When supplied with the UUID of a container request, it will calculate the + cost of that container request and all its children. + + When supplied with the UUID of a collection, it will see if there is a + container_request UUID in the properties of the collection, and if so, it + will calculate the cost of that container request and all its children. + + When supplied with a project UUID or when supplied with multiple container + request or collection UUIDs, it will calculate the total cost for all + supplied UUIDs. + + When supplied with a 'begin' and 'end' timestamp (format: + %s), it will calculate the cost for all top-level container + requests whose containers finished during the specified interval. + + The total cost calculation takes container reuse into account: if a container + was reused between several container requests, its cost will only be counted + once. Caveats: - - the Arvados API configuration cost data may be out of sync with the cloud - provider. - - when generating reports for older container requests, the cost data in the - Arvados API configuration file may have changed since the container request - was fulfilled. - In order to get the data for the uuids supplied, the ARVADOS_API_HOST and + - This program uses the cost data from config.yml at the time of the + execution of the container, stored in the 'node.json' file in its log + collection. If the cost data was not correctly configured at the time the + container was executed, the output from this program will be incorrect. + + - If a container was run on a preemptible ("spot") instance, the cost data + reported by this program may be wildly inaccurate, because it does not have + access to the spot pricing in effect for the node then the container ran. The + UUID report file that is generated when the '-output' option is specified has + a column that indicates the preemptible state of the instance that ran the + container. + + - This program does not take into account overhead costs like the time spent + starting and stopping compute nodes that run containers, the cost of the + permanent cloud nodes that provide the Arvados services, the cost of data + stored in Arvados, etc. + + - When provided with a project UUID, subprojects will not be considered. + + In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables must be set. + This program prints the total dollar amount from the aggregate cost + accounting across all provided UUIDs on stdout. + Options: -`, prog) +`, prog, timestampFormat) flags.PrintDefaults() } - loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)") - resultsDir = *flags.String("output", "results", "output directory for the CSV reports") - flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.") - err := flags.Parse(args) + loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)") + flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports") + flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat)) + flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat)) + flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects") + err = flags.Parse(args) if err == flag.ErrHelp { - exitCode = 0 + err = nil + exitCode = 1 return } else if err != nil { exitCode = 2 return } + c.uuids = flags.Args() + + if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) { + flags.Usage() + err = fmt.Errorf("When specifying a date range, both begin and end must be specified") + exitCode = 2 + return + } + + if len(beginStr) != 0 { + var errB, errE error + c.begin, errB = time.Parse(timestampFormat, beginStr) + c.end, errE = time.Parse(timestampFormat, endStr) + if (errB != nil) || (errE != nil) { + flags.Usage() + err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE) + exitCode = 2 + return + } + } - if len(uuids) < 1 { - logger.Errorf("Error: no uuid(s) provided") + if (len(c.uuids) < 1) && (len(beginStr) == 0) { flags.Usage() + err = fmt.Errorf("error: no uuid(s) provided") exitCode = 2 return } @@ -153,225 +168,207 @@ Options: return } logger.SetLevel(lvl) + if !c.cache { + logger.Debug("Caching disabled\n") + } return } -func ensureDirectory(logger *logrus.Logger, dir string) { +func ensureDirectory(logger *logrus.Logger, dir string) (err error) { statData, err := os.Stat(dir) if os.IsNotExist(err) { err = os.MkdirAll(dir, 0700) if err != nil { - logger.Errorf("Error creating directory %s: %s\n", dir, err.Error()) - os.Exit(1) + return fmt.Errorf("error creating directory %s: %s", dir, err.Error()) } } else { if !statData.IsDir() { - logger.Errorf("The path %s is not a directory\n", dir) - os.Exit(1) + return fmt.Errorf("the path %s is not a directory", dir) } } + return } -func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, container Dict) (csv string, cost float64) { - csv = cr["uuid"].(string) + "," - csv += cr["name"].(string) + "," - csv += container["uuid"].(string) + "," - csv += container["state"].(string) + "," - if container["started_at"] != nil { - csv += container["started_at"].(string) + "," +func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) { + csv = cr.UUID + "," + csv += cr.Name + "," + csv += container.UUID + "," + csv += string(container.State) + "," + if container.StartedAt != nil { + csv += container.StartedAt.String() + "," } else { csv += "," } var delta time.Duration - if container["finished_at"] != nil { - csv += container["finished_at"].(string) + "," - finishedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["finished_at"].(string)) - if err != nil { - fmt.Println(err) - } - startedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["started_at"].(string)) - if err != nil { - fmt.Println(err) - } - delta = finishedTimestamp.Sub(startedTimestamp) + if container.FinishedAt != nil { + csv += container.FinishedAt.String() + "," + delta = container.FinishedAt.Sub(*container.StartedAt) csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + "," } else { csv += ",," } var price float64 var size string - switch n := node.(type) { - case Node: - price = n.Price - size = n.ProviderType - case LegacyNodeInfo: - price = n.CloudNode.Price - size = n.CloudNode.Size - default: - logger.Warn("WARNING: unknown node type found!") + if node.Properties.CloudNode.Price != 0 { + price = node.Properties.CloudNode.Price + size = node.Properties.CloudNode.Size + } else { + price = node.Price + size = node.ProviderType } cost = delta.Seconds() / 3600 * price - csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" + csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" return } -func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object Dict) { +func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) { reload = true + if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") { + // We do not cache projects or collections, they have no final state + return + } // See if we have a cached copy of this object - if _, err := os.Stat(file); err == nil { - data, err := ioutil.ReadFile(file) - if err != nil { - logger.Errorf("error reading %q: %s", file, err) - return - } - err = json.Unmarshal(data, &object) - if err != nil { - logger.Errorf("failed to unmarshal json: %s: %s", data, err) - return - } + _, err := os.Stat(file) + if err != nil { + return + } + data, err := ioutil.ReadFile(file) + if err != nil { + logger.Errorf("error reading %q: %s", file, err) + return + } + err = json.Unmarshal(data, &object) + if err != nil { + logger.Errorf("failed to unmarshal json: %s: %s", data, err) + return + } - // See if it is in a final state, if that makes sense - // Projects (j7d0g) do not have state so they should always be reloaded - if !strings.Contains(uuid, "-j7d0g-") { - if object["state"].(string) == "Complete" || object["state"].(string) == "Failed" { - reload = false - logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) - return - } + // See if it is in a final state, if that makes sense + switch v := object.(type) { + case *arvados.ContainerRequest: + if v.State == arvados.ContainerRequestStateFinal { + reload = false + logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) + } + case *arvados.Container: + if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled { + reload = false + logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) } } return } // Load an Arvados object. -func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict) { - - ensureDirectory(logger, path) - - file := path + "/" + uuid + ".json" +func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) { + file := uuid + ".json" var reload bool - reload, object = loadCachedObject(logger, file, uuid) - - if reload { - var err error - if strings.Contains(uuid, "-d1hrv-") { - err = arv.Get("pipeline_instances", uuid, nil, &object) - } else if strings.Contains(uuid, "-j7d0g-") { - err = arv.Get("groups", uuid, nil, &object) - } else if strings.Contains(uuid, "-xvhdp-") { - err = arv.Get("container_requests", uuid, nil, &object) - } else if strings.Contains(uuid, "-dz642-") { - err = arv.Get("containers", uuid, nil, &object) - } else { - err = arv.Get("jobs", uuid, nil, &object) - } - if err != nil { - logger.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err) - os.Exit(1) - } - encoded, err := json.MarshalIndent(object, "", " ") + var cacheDir string + + if !cache { + reload = true + } else { + homeDir, err := os.UserHomeDir() if err != nil { - logger.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err) - os.Exit(1) + reload = true + logger.Info("Unable to determine current user home directory, not using cache") + } else { + cacheDir = homeDir + "/.cache/arvados/costanalyzer/" + err = ensureDirectory(logger, cacheDir) + if err != nil { + reload = true + logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error()) + } else { + reload = loadCachedObject(logger, cacheDir+file, uuid, object) + } } - err = ioutil.WriteFile(file, encoded, 0644) + } + if !reload { + return + } + + if strings.Contains(uuid, "-j7d0g-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-xvhdp-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-dz642-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-4zz18-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil) + } else { + err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err) + return + } + if err != nil { + err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err) + return + } + encoded, err := json.MarshalIndent(object, "", " ") + if err != nil { + err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err) + return + } + if cacheDir != "" { + err = ioutil.WriteFile(cacheDir+file, encoded, 0644) if err != nil { - logger.Errorf("Error writing file %s:\n %s\n", file, err) - os.Exit(1) + err = fmt.Errorf("error writing file %s:\n %s", file, err) + return } } return } -func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) { - if _, ok := itemMap["log_uuid"]; ok { - if itemMap["log_uuid"] == nil { - err = errors.New("No log collection") - return - } - - var collection arvados.Collection - err = arv.Get("collections", itemMap["log_uuid"].(string), nil, &collection) - if err != nil { - logger.Errorf("error getting collection: %s\n", err) - return - } +func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) { + if cr.LogUUID == "" { + err = errors.New("no log collection") + return + } - var fs arvados.CollectionFileSystem - fs, err = collection.FileSystem(arv2, kc) - if err != nil { - logger.Errorf("error opening collection as filesystem: %s\n", err) - return - } - var f http.File - f, err = fs.Open("node.json") - if err != nil { - logger.Errorf("error opening file in collection: %s\n", err) - return - } + var collection arvados.Collection + err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil) + if err != nil { + err = fmt.Errorf("error getting collection: %s", err) + return + } - var nodeDict Dict - // TODO: checkout io (ioutil?) readall function - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(f) - if err != nil { - logger.Errorf("error reading %q: %s\n", f, err) - return - } - contents := buf.String() - f.Close() + var fs arvados.CollectionFileSystem + fs, err = collection.FileSystem(ac, kc) + if err != nil { + err = fmt.Errorf("error opening collection as filesystem: %s", err) + return + } + var f http.File + f, err = fs.Open("node.json") + if err != nil { + err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err) + return + } - err = json.Unmarshal([]byte(contents), &nodeDict) - if err != nil { - logger.Errorf("error unmarshalling: %s\n", err) - return - } - if val, ok := nodeDict["properties"]; ok { - var encoded []byte - encoded, err = json.MarshalIndent(val, "", " ") - if err != nil { - logger.Errorf("error marshalling: %s\n", err) - return - } - // node is type LegacyNodeInfo - var newNode LegacyNodeInfo - err = json.Unmarshal(encoded, &newNode) - if err != nil { - logger.Errorf("error unmarshalling: %s\n", err) - return - } - node = newNode - } else { - // node is type Node - var newNode Node - err = json.Unmarshal([]byte(contents), &newNode) - if err != nil { - logger.Errorf("error unmarshalling: %s\n", err) - return - } - node = newNode - } + err = json.NewDecoder(f).Decode(&node) + if err != nil { + err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err) + return } return } -func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, resultsDir string) (cost map[string]float64) { - +func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) { cost = make(map[string]float64) - project := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) - - // arv -f uuid container_request list --filters '[["owner_uuid","=",""],["requesting_container_uuid","=",null]]' + var project arvados.Group + err = loadObject(logger, ac, uuid, uuid, cache, &project) + if err != nil { + return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error()) + } - // Now find all container requests that have the container we found above as requesting_container_uuid var childCrs map[string]interface{} filterset := []arvados.Filter{ { Attr: "owner_uuid", Operator: "=", - Operand: project["uuid"].(string), + Operand: project.UUID, }, { Attr: "requesting_container_uuid", @@ -379,16 +376,23 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado Operand: nil, }, } - err := arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs) + err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ + "filters": filterset, + "limit": 10000, + }) if err != nil { - logger.Fatalf("Error querying container_requests: %s\n", err.Error()) + return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) } if value, ok := childCrs["items"]; ok { logger.Infof("Collecting top level container requests in project %s\n", uuid) items := value.([]interface{}) for _, item := range items { itemMap := item.(map[string]interface{}) - for k, v := range generateCrCsv(logger, itemMap["uuid"].(string), arv, arv2, kc, resultsDir) { + crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache) + if err != nil { + return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error()) + } + for k, v := range crCsv { cost[k] = v } } @@ -398,143 +402,230 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado return } -func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, resultsDir string) (cost map[string]float64) { +func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) { cost = make(map[string]float64) - csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n" + csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n" var tmpCsv string var tmpTotalCost float64 var totalCost float64 + fmt.Printf("Processing %s\n", uuid) + + var crUUID = uuid + if strings.Contains(uuid, "-4zz18-") { + // This is a collection, find the associated container request (if any) + var c arvados.Collection + err = loadObject(logger, ac, uuid, uuid, cache, &c) + if err != nil { + return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err) + } + value, ok := c.Properties["container_request"] + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid) + } + crUUID, ok = value.(string) + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid) + } + } // This is a container request, find the container - cr := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) - container := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string)) + var cr arvados.ContainerRequest + err = loadObject(logger, ac, crUUID, crUUID, cache, &cr) + if err != nil { + return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err) + } + if len(cr.ContainerUUID) == 0 { + // Nothing to do! E.g. a CR in 'Uncommitted' state. + logger.Infof("No container associated with container request %s, skipping\n", crUUID) + return nil, nil + } + var container arvados.Container + err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container) + if err != nil { + return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err) + } - topNode, err := getNode(logger, arv, arv2, kc, cr) + topNode, err := getNode(arv, ac, kc, cr) if err != nil { - log.Fatalf("error getting node: %s", err) + logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err) + return nil, nil } tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container) csv += tmpCsv totalCost += tmpTotalCost + cost[container.UUID] = totalCost - cost[container["uuid"].(string)] = totalCost - - // Now find all container requests that have the container we found above as requesting_container_uuid - var childCrs map[string]interface{} + // Find all container requests that have the container we found above as requesting_container_uuid + var childCrs arvados.ContainerRequestList filterset := []arvados.Filter{ { Attr: "requesting_container_uuid", Operator: "=", - Operand: container["uuid"].(string), + Operand: container.UUID, }} - err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs) + err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ + "filters": filterset, + "limit": 10000, + }) if err != nil { - log.Fatal("error querying container_requests", err.Error()) + return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) } - if value, ok := childCrs["items"]; ok { - logger.Infof("Collecting child containers for container request %s", uuid) - items := value.([]interface{}) - for _, item := range items { - logger.Info(".") - itemMap := item.(map[string]interface{}) - node, _ := getNode(logger, arv, arv2, kc, itemMap) - logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n") - c2 := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string)) - tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2) - cost[itemMap["container_uuid"].(string)] = tmpTotalCost - csv += tmpCsv - totalCost += tmpTotalCost + logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt) + for _, cr2 := range childCrs.Items { + logger.Info(".") + node, err := getNode(arv, ac, kc, cr2) + if err != nil { + logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err) + continue } + logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n") + var c2 arvados.Container + err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2) + if err != nil { + return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err) + } + tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2) + cost[cr2.ContainerUUID] = tmpTotalCost + csv += tmpCsv + totalCost += tmpTotalCost } logger.Info(" done\n") csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n" - // Write the resulting CSV file - fName := resultsDir + "/" + uuid + ".csv" - err = ioutil.WriteFile(fName, []byte(csv), 0644) - if err != nil { - logger.Errorf("Error writing file with path %s: %s\n", fName, err.Error()) - os.Exit(1) + if resultsDir != "" { + // Write the resulting CSV file + fName := resultsDir + "/" + crUUID + ".csv" + err = ioutil.WriteFile(fName, []byte(csv), 0644) + if err != nil { + return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error()) + } + logger.Infof("\nUUID report in %s\n\n", fName) } return } -func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) { - exitcode, uuids, resultsDir := parseFlags(prog, args, loader, logger, stderr) +func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) { + exitcode, err = c.parseFlags(prog, args, logger, stderr) + if exitcode != 0 { return } + if c.resultsDir != "" { + err = ensureDirectory(logger, c.resultsDir) + if err != nil { + exitcode = 3 + return + } + } - ensureDirectory(logger, resultsDir) + uuidChannel := make(chan string) // Arvados Client setup arv, err := arvadosclient.MakeArvadosClient() if err != nil { - logger.Errorf("error creating Arvados object: %s", err) - os.Exit(1) + err = fmt.Errorf("error creating Arvados object: %s", err) + exitcode = 1 + return } kc, err := keepclient.MakeKeepClient(arv) if err != nil { - logger.Errorf("error creating Keep object: %s", err) - os.Exit(1) + err = fmt.Errorf("error creating Keep object: %s", err) + exitcode = 1 + return } - arv2 := arvados.NewClientFromEnv() - - cost := make(map[string]float64) + ac := arvados.NewClientFromEnv() - for _, uuid := range uuids { - //csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n" + // Populate uuidChannel with the requested uuid list + go func() { + defer close(uuidChannel) + for _, uuid := range c.uuids { + uuidChannel <- uuid + } - if strings.Contains(uuid, "-d1hrv-") { - // This is a pipeline instance, not a job! Find the cwl-runner job. - pi := loadObject(logger, arv, resultsDir+"/"+uuid, uuid) - for _, v := range pi["components"].(map[string]interface{}) { - x := v.(map[string]interface{}) - y := x["job"].(map[string]interface{}) - uuid = y["uuid"].(string) + if !c.begin.IsZero() { + initialParams := arvados.ResourceListParams{ + Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}}, + Order: "created_at", + } + params := initialParams + for { + // This list variable must be a new one declared + // inside the loop: otherwise, items in the API + // response would get deep-merged into the items + // loaded in previous iterations. + var list arvados.ContainerRequestList + + err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params) + if err != nil { + logger.Errorf("Error getting container request list from Arvados API: %s\n", err) + break + } + if len(list.Items) == 0 { + break + } + + for _, i := range list.Items { + uuidChannel <- i.UUID + } + params.Offset += len(list.Items) } + } + }() - // for projects: - // arv -f uuid container_request list --filters '[["owner_uuid","=",""],["requesting_container_uuid","=",null]]' + cost := make(map[string]float64) + for uuid := range uuidChannel { + fmt.Printf("Considering %s\n", uuid) if strings.Contains(uuid, "-j7d0g-") { // This is a project (group) - for k, v := range handleProject(logger, uuid, arv, arv2, kc, resultsDir) { + cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache) + if err != nil { + exitcode = 1 + return + } + for k, v := range cost { cost[k] = v } - } else if strings.Contains(uuid, "-xvhdp-") { + } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") { // This is a container request - for k, v := range generateCrCsv(logger, uuid, arv, arv2, kc, resultsDir) { + var crCsv map[string]float64 + crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache) + if err != nil { + err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error()) + exitcode = 2 + return + } + for k, v := range crCsv { cost[k] = v } } else if strings.Contains(uuid, "-tpzed-") { // This is a user. The "Home" project for a user is not a real project. // It is identified by the user uuid. As such, cost analysis for the - // "Home" project is not supported by this program. - logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid) + // "Home" project is not supported by this program. Skip this uuid, but + // keep going. + logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid) + } else { + logger.Errorf("this argument does not look like a uuid: %s\n", uuid) + exitcode = 3 + return } } - logger.Info("\n") - for k := range cost { - logger.Infof("Uuid report in %s/%s.csv\n", resultsDir, k) - } - if len(cost) == 0 { logger.Info("Nothing to do!\n") - os.Exit(0) + return } var csv string csv = "# Aggregate cost accounting for uuids:\n" - for _, uuid := range uuids { + for _, uuid := range c.uuids { csv += "# " + uuid + "\n" } @@ -546,14 +637,20 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n" - // Write the resulting CSV file - aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv" - err = ioutil.WriteFile(aFile, []byte(csv), 0644) - if err != nil { - logger.Errorf("Error writing file with path %s: %s\n", aFile, err.Error()) - os.Exit(1) - } else { - logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile) + if c.resultsDir != "" { + // Write the resulting CSV file + aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv" + err = ioutil.WriteFile(aFile, []byte(csv), 0644) + if err != nil { + err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error()) + exitcode = 1 + return + } + logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile) } + + // Output the total dollar amount on stdout + fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64)) + return }