import (
"bytes"
- "context"
"io"
"io/ioutil"
"os"
}
func (s *Suite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
// Get the various arvados, arvadosclient, and keep client objects
"IncludedScratch": 64000000000,
"AddedScratch": 0,
"Price": 0.292,
- "Preemptible": false
+ "Preemptible": true
}`
standardD32sV3JSON := `{
"Name": "Standard_D32s_v3",
"Preemptible": false
}`
+ legacyD1V2JSON := `{
+ "properties": {
+ "cloud_node": {
+ "price": 0.073001,
+ "size": "Standard_D1_v2"
+ },
+ "total_cpu_cores": 1,
+ "total_ram_mb": 3418,
+ "total_scratch_mb": 51170
+ }
+}`
+
// Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
- createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON)
}
func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
// Get the CR
var cr arvados.ContainerRequest
- err := ac.RequestAndDecodeContext(context.Background(), &cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
+ err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
c.Assert(err, check.Equals, nil)
c.Assert(cr.LogUUID, check.Equals, logUUID)
// Get the log collection
var coll arvados.Collection
- err = ac.RequestAndDecodeContext(context.Background(), &coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
+ err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
c.Assert(err, check.IsNil)
// Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
c.Assert(mtxt, check.NotNil)
// Update collection record
- err = ac.RequestAndDecodeContext(context.Background(), &coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
"collection": map[string]interface{}{
"manifest_text": mtxt,
},
func (*Suite) TestUsage(c *check.C) {
var stdout, stderr bytes.Buffer
exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
- c.Check(exitcode, check.Equals, 1)
+ c.Check(exitcode, check.Equals, 0)
c.Check(stdout.String(), check.Equals, "")
c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
}
+func (*Suite) TestTimestampRange(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+ // Run costanalyzer with a timestamp range. This should pick up two container requests in "Final" state.
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, "-begin", "2020-11-02T00:00:00", "-end", "2020-11-03T23:59:00"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ uuid2Report, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,763.467,,,,0.01")
+ c.Check(string(uuid2Report), check.Matches, "(?ms).*TOTAL,,,,,,488.775,,,,0.01")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,1245.564,0.01")
+}
+
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 1 container request uuid
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ // Make sure the 'preemptible' flag was picked up
+ c.Check(string(uuidReport), check.Matches, "(?ms).*,Standard_E4s_v3,true,.*")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,86462.000,7.01")
+}
+
+func (*Suite) TestCollectionUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+
+ // Create a collection with no container_request property
+ ac := arvados.NewClientFromEnv()
+ var coll arvados.Collection
+ err := ac.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
+ c.Assert(err, check.IsNil)
+
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, coll.UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 2)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Add a container_request property
+ err = ac.RequestAndDecode(&coll, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "properties": map[string]interface{}{
+ "container_request": arvadostest.CompletedContainerRequestUUID,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ // Re-run costanalyzer on the updated collection
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, coll.UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,86462.000,7.01")
}
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
- uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,42.27")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,172924.000,49.28")
stdout.Truncate(0)
stderr.Truncate(0)
// the analysis with the project uuid. The results should be identical.
ac := arvados.NewClientFromEnv()
var cr arvados.ContainerRequest
- err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
"container_request": map[string]interface{}{
"owner_uuid": arvadostest.AProjectUUID,
},
})
c.Assert(err, check.IsNil)
- err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
"container_request": map[string]interface{}{
"owner_uuid": arvadostest.AProjectUUID,
},
c.Assert(err, check.IsNil)
// Run costanalyzer with the project uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", resultsDir, arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,7.01")
- uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ uuidReport2, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,86462.000,,,,42.27")
re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches = re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err = ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,172924.000,49.28")
+}
+
+func (*Suite) TestUncommittedContainerRequest(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, one of which is in the Uncommitted state, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.UncommittedContainerRequestUUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*No container associated with container request .*")
+
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.01\n")
}
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
+
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.01\n")
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir := c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,763.467,,,,0.01")
- uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,488.775,,,,0.01")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,1245.564,0.01")
}
projects / arvados.git / blobdiff