X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c137248849c4b4ad38513f3faebd1782037add09..f3b8d03f7063b162355bccfd71aeb2b8b67bbdbb:/lib/costanalyzer/costanalyzer.go diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index 4a48db1a8b..e68e2cb8c1 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "git.arvados.org/arvados.git/lib/cmd" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/keepclient" @@ -25,6 +26,8 @@ import ( const timestampFormat = "2006-01-02T15:04:05" +var pagesize = 1000 + type nodeInfo struct { // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3) Properties struct { @@ -62,10 +65,9 @@ func (i *arrayFlags) Set(value string) error { return nil } -func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) { +func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (ok bool, exitCode int) { var beginStr, endStr string flags := flag.NewFlagSet("", flag.ContinueOnError) - flags.SetOutput(stderr) flags.Usage = func() { fmt.Fprintf(flags.Output(), ` Usage: @@ -135,22 +137,14 @@ Options: flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat)) flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat)) flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects") - err = flags.Parse(args) - if err == flag.ErrHelp { - err = nil - exitCode = 1 - return - } else if err != nil { - exitCode = 2 - return + if ok, code := cmd.ParseFlags(flags, prog, args, "[uuid ...]", stderr); !ok { + return false, code } c.uuids = flags.Args() if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) { - flags.Usage() - err = fmt.Errorf("When specifying a date range, both begin and end must be specified") - exitCode = 2 - return + fmt.Fprintf(stderr, "When specifying a date range, both begin and end must be specified (try -help)\n") + return false, 2 } if len(beginStr) != 0 { @@ -158,30 +152,26 @@ Options: c.begin, errB = time.Parse(timestampFormat, beginStr) c.end, errE = time.Parse(timestampFormat, endStr) if (errB != nil) || (errE != nil) { - flags.Usage() - err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE) - exitCode = 2 - return + fmt.Fprintf(stderr, "When specifying a date range, both begin and end must be of the format %s %+v, %+v\n", timestampFormat, errB, errE) + return false, 2 } } if (len(c.uuids) < 1) && (len(beginStr) == 0) { - flags.Usage() - err = fmt.Errorf("error: no uuid(s) provided") - exitCode = 2 - return + fmt.Fprintf(stderr, "error: no uuid(s) provided (try -help)\n") + return false, 2 } lvl, err := logrus.ParseLevel(*loglevel) if err != nil { - exitCode = 2 - return + fmt.Fprintf(stderr, "invalid argument to -log-level: %s\n", err) + return false, 2 } logger.SetLevel(lvl) if !c.cache { logger.Debug("Caching disabled") } - return + return true, 0 } func ensureDirectory(logger *logrus.Logger, dir string) (err error) { @@ -367,6 +357,35 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien return } +func getContainerRequests(ac *arvados.Client, filters []arvados.Filter) ([]arvados.ContainerRequest, error) { + var allItems []arvados.ContainerRequest + for { + pagefilters := append([]arvados.Filter(nil), filters...) + if len(allItems) > 0 { + pagefilters = append(pagefilters, arvados.Filter{ + Attr: "uuid", + Operator: ">", + Operand: allItems[len(allItems)-1].UUID, + }) + } + var resp arvados.ContainerRequestList + err := ac.RequestAndDecode(&resp, "GET", "arvados/v1/container_requests", nil, arvados.ResourceListParams{ + Filters: pagefilters, + Limit: &pagesize, + Order: "uuid", + Count: "none", + }) + if err != nil { + return nil, fmt.Errorf("error querying container_requests: %w", err) + } + if len(resp.Items) == 0 { + // no more pages + return allItems, nil + } + allItems = append(allItems, resp.Items...) + } +} + func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]consumption, err error) { cost = make(map[string]consumption) @@ -375,9 +394,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado if err != nil { return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error()) } - - var childCrs map[string]interface{} - filterset := []arvados.Filter{ + allItems, err := getContainerRequests(ac, []arvados.Filter{ { Attr: "owner_uuid", Operator: "=", @@ -388,29 +405,23 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado Operator: "=", Operand: nil, }, - } - err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ - "filters": filterset, - "limit": 10000, }) if err != nil { return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) } - if value, ok := childCrs["items"]; ok { - logger.Infof("Collecting top level container requests in project %s", uuid) - items := value.([]interface{}) - for _, item := range items { - itemMap := item.(map[string]interface{}) - crInfo, err := generateCrInfo(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache) - if err != nil { - return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error()) - } - for k, v := range crInfo { - cost[k] = v - } - } - } else { + if len(allItems) == 0 { logger.Infof("No top level container requests found in project %s", uuid) + return + } + logger.Infof("Collecting top level container requests in project %s", uuid) + for _, cr := range allItems { + crInfo, err := generateCrInfo(logger, cr.UUID, arv, ac, kc, resultsDir, cache) + if err != nil { + return nil, fmt.Errorf("error generating container_request CSV for %s: %s", cr.UUID, err) + } + for k, v := range crInfo { + cost[k] = v + } } return } @@ -468,28 +479,20 @@ func generateCrInfo(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvad csv += tmpCsv cost[container.UUID] = total - // Find all container requests that have the container we found above as requesting_container_uuid - var childCrs arvados.ContainerRequestList - filterset := []arvados.Filter{ - { - Attr: "requesting_container_uuid", - Operator: "=", - Operand: container.UUID, - }} - err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ - "filters": filterset, - "limit": 10000, - }) - if err != nil { - return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) - } - logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt) + // Find all container requests that have the container we + // found above as requesting_container_uuid. + allItems, err := getContainerRequests(ac, []arvados.Filter{{ + Attr: "requesting_container_uuid", + Operator: "=", + Operand: container.UUID, + }}) + logger.Infof("Looking up %d child containers for container %s (%s)", len(allItems), container.UUID, container.FinishedAt) progressTicker := time.NewTicker(5 * time.Second) defer progressTicker.Stop() - for i, cr2 := range childCrs.Items { + for i, cr2 := range allItems { select { case <-progressTicker.C: - logger.Infof("... %d of %d", i+1, len(childCrs.Items)) + logger.Infof("... %d of %d", i+1, len(allItems)) default: } node, err := getNode(arv, ac, kc, cr2) @@ -526,9 +529,9 @@ func generateCrInfo(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvad } func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) { - exitcode, err = c.parseFlags(prog, args, logger, stderr) - - if exitcode != 0 { + var ok bool + ok, exitcode = c.parseFlags(prog, args, logger, stderr) + if !ok { return } if c.resultsDir != "" { @@ -610,7 +613,7 @@ func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger cost[k] = v } } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") { - // This is a container request + // This is a container request or collection var crInfo map[string]consumption crInfo, err = generateCrInfo(logger, uuid, arv, ac, kc, c.resultsDir, c.cache) if err != nil {