From 1f773811a85c564bec4606d7ad05e7805fa41f22 Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Sat, 5 Dec 2020 16:37:00 -0500 Subject: [PATCH] 17187: costanalyzer: add support for collection uuids. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- lib/costanalyzer/costanalyzer.go | 46 ++++++++++++++++++++------- lib/costanalyzer/costanalyzer_test.go | 42 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index 4284542b86..75cbdc0376 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -62,12 +62,18 @@ Usage: each container. When supplied with the uuid of a container request, it will calculate the - cost of that container request and all its children. When suplied with a - project uuid or when supplied with multiple container request uuids, it will - create a CSV report for each supplied uuid, as well as a CSV file with - aggregate cost accounting for all supplied uuids. The aggregate cost report - takes container reuse into account: if a container was reused between several - container requests, its cost will only be counted once. + cost of that container request and all its children. + + When supplied with the uuid of a collection, it will see if there is a + container_request uuid in the properties of the collection, and if so, it + will calculate the cost of that container request and all its children. + + When supplied with a project uuid or when supplied with multiple container + request or collection uuids, it will create a CSV report for each supplied + uuid, as well as a CSV file with aggregate cost accounting for all supplied + uuids. The aggregate cost report takes container reuse into account: if a + container was reused between several container requests, its cost will only + be counted once. To get the node costs, the progam queries the Arvados API for current cost data for each node type used. This means that the reported cost always @@ -180,8 +186,8 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) { reload = true - if strings.Contains(uuid, "-j7d0g-") { - // We do not cache projects, they have no final state + if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") { + // We do not cache projects or collections, they have no final state return } // See if we have a cached copy of this object @@ -251,6 +257,8 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil) } else if strings.Contains(uuid, "-dz642-") { err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-4zz18-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil) } else { err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err) return @@ -309,7 +317,6 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien } func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) { - cost = make(map[string]float64) var project arvados.Group @@ -366,9 +373,24 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado var tmpTotalCost float64 var totalCost float64 + var crUUID = uuid + if strings.Contains(uuid, "-4zz18-") { + // This is a collection, find the associated container request (if any) + var c arvados.Collection + err = loadObject(logger, ac, uuid, uuid, cache, &c) + if err != nil { + return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err) + } + value, ok := c.Properties["container_request"] + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid) + } + crUUID = value.(string) + } + // This is a container request, find the container var cr arvados.ContainerRequest - err = loadObject(logger, ac, uuid, uuid, cache, &cr) + err = loadObject(logger, ac, crUUID, crUUID, cache, &cr) if err != nil { return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err) } @@ -474,12 +496,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log for k, v := range cost { cost[k] = v } - } else if strings.Contains(uuid, "-xvhdp-") { + } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") { // This is a container request var crCsv map[string]float64 crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache) if err != nil { - err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error()) + err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error()) exitcode = 2 return } diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go index 4fab93bf4f..4f0f64dae3 100644 --- a/lib/costanalyzer/costanalyzer_test.go +++ b/lib/costanalyzer/costanalyzer_test.go @@ -177,6 +177,48 @@ func (*Suite) TestContainerRequestUUID(c *check.C) { c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889") } +func (*Suite) TestCollectionUUID(c *check.C) { + var stdout, stderr bytes.Buffer + + // Run costanalyzer with 1 collection uuid, without 'container_request' property + exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 2) + c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*") + + // Update the collection, attach a 'container_request' property + ac := arvados.NewClientFromEnv() + var coll arvados.Collection + + // Update collection record + err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{ + "collection": map[string]interface{}{ + "properties": map[string]interface{}{ + "container_request": arvadostest.CompletedContainerRequestUUID, + }, + }, + }) + c.Assert(err, check.IsNil) + + stdout.Truncate(0) + stderr.Truncate(0) + + // Run costanalyzer with 1 collection uuid + exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889") +} + func (*Suite) TestDoubleContainerRequestUUID(c *check.C) { var stdout, stderr bytes.Buffer // Run costanalyzer with 2 container request uuids -- 2.39.5