each container.
When supplied with the uuid of a container request, it will calculate the
- cost of that container request and all its children. When suplied with a
- project uuid or when supplied with multiple container request uuids, it will
- create a CSV report for each supplied uuid, as well as a CSV file with
- aggregate cost accounting for all supplied uuids. The aggregate cost report
- takes container reuse into account: if a container was reused between several
- container requests, its cost will only be counted once.
+ cost of that container request and all its children.
+
+ When supplied with the uuid of a collection, it will see if there is a
+ container_request uuid in the properties of the collection, and if so, it
+ will calculate the cost of that container request and all its children.
+
+ When supplied with a project uuid or when supplied with multiple container
+ request or collection uuids, it will create a CSV report for each supplied
+ uuid, as well as a CSV file with aggregate cost accounting for all supplied
+ uuids. The aggregate cost report takes container reuse into account: if a
+ container was reused between several container requests, its cost will only
+ be counted once.
To get the node costs, the progam queries the Arvados API for current cost
data for each node type used. This means that the reported cost always
func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
reload = true
- if strings.Contains(uuid, "-j7d0g-") {
- // We do not cache projects, they have no final state
+ if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+ // We do not cache projects or collections, they have no final state
return
}
// See if we have a cached copy of this object
err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
} else if strings.Contains(uuid, "-dz642-") {
err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-4zz18-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)
} else {
err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
return
}
func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
-
cost = make(map[string]float64)
var project arvados.Group
var tmpTotalCost float64
var totalCost float64
+ var crUUID = uuid
+ if strings.Contains(uuid, "-4zz18-") {
+ // This is a collection, find the associated container request (if any)
+ var c arvados.Collection
+ err = loadObject(logger, ac, uuid, uuid, cache, &c)
+ if err != nil {
+ return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+ }
+ value, ok := c.Properties["container_request"]
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+ }
+ crUUID = value.(string)
+ }
+
// This is a container request, find the container
var cr arvados.ContainerRequest
- err = loadObject(logger, ac, uuid, uuid, cache, &cr)
+ err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
for k, v := range cost {
cost[k] = v
}
- } else if strings.Contains(uuid, "-xvhdp-") {
+ } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+ err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
}
+func (*Suite) TestCollectionUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+
+ // Run costanalyzer with 1 collection uuid, without 'container_request' property
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 2)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
+
+ // Update the collection, attach a 'container_request' property
+ ac := arvados.NewClientFromEnv()
+ var coll arvados.Collection
+
+ // Update collection record
+ err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "properties": map[string]interface{}{
+ "container_request": arvadostest.CompletedContainerRequestUUID,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Run costanalyzer with 1 collection uuid
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
projects / arvados.git / commitdiff