"IncludedScratch": 64000000000,
"AddedScratch": 0,
"Price": 0.292,
- "Preemptible": false
+ "Preemptible": true
}`
standardD32sV3JSON := `{
"Name": "Standard_D32s_v3",
"Preemptible": false
}`
+ legacyD1V2JSON := `{
+ "properties": {
+ "cloud_node": {
+ "price": 0.073001,
+ "size": "Standard_D1_v2"
+ },
+ "total_cpu_cores": 1,
+ "total_ram_mb": 3418,
+ "total_scratch_mb": 51170
+ }
+}`
+
// Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
- createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON)
}
func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
// Get the CR
var cr arvados.ContainerRequest
- err := arv.Get("container_requests", crUUID, arvadosclient.Dict{}, &cr)
+ err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
c.Assert(err, check.Equals, nil)
c.Assert(cr.LogUUID, check.Equals, logUUID)
// Get the log collection
var coll arvados.Collection
- err = arv.Get("collections", cr.LogUUID, arvadosclient.Dict{}, &coll)
+ err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
c.Assert(err, check.IsNil)
// Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
c.Assert(mtxt, check.NotNil)
// Update collection record
- err = arv.Update("collections", cr.LogUUID, arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": mtxt}}, &coll)
+ err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": mtxt,
+ },
+ })
c.Assert(err, check.IsNil)
}
c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
}
+func (*Suite) TestTimestampRange(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+ // Run costanalyzer with a timestamp range. This should pick up two container requests in "Final" state.
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, "-begin", "2020-11-02T00:00:00", "-end", "2020-11-03T23:59:00"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ uuid2Report, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
+ c.Check(string(uuid2Report), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+}
+
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 1 container request uuid
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ // Make sure the 'preemptible' flag was picked up
+ c.Check(string(uuidReport), check.Matches, "(?ms).*,Standard_E4s_v3,true,.*")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
+func (*Suite) TestCollectionUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+
+ resultsDir := c.MkDir()
+ // Run costanalyzer with 1 collection uuid, without 'container_request' property
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 2)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
+
+ // Update the collection, attach a 'container_request' property
+ ac := arvados.NewClientFromEnv()
+ var coll arvados.Collection
+
+ // Update collection record
+ err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "properties": map[string]interface{}{
+ "container_request": arvadostest.CompletedContainerRequestUUID,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Run costanalyzer with 1 collection uuid
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
- uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
c.Assert(err, check.IsNil)
c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ stdout.Truncate(0)
+ stderr.Truncate(0)
// Now move both container requests into an existing project, and then re-run
// the analysis with the project uuid. The results should be identical.
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, check.Equals, nil)
-
+ ac := arvados.NewClientFromEnv()
var cr arvados.ContainerRequest
- err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
- err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID2, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
// Run costanalyzer with the project uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", resultsDir, arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
- uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ uuidReport2, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
}
+func (*Suite) TestUncommittedContainerRequest(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, one of which is in the Uncommitted state, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.UncommittedContainerRequestUUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*No container associated with container request .*")
+
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.00588088\n")
+}
+
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
+
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.01492030\n")
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir := c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
- uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
}
projects / arvados.git / blobdiff