From dfa749b3eacff1da33fe470a2a4af3e291defdcf Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Wed, 3 Feb 2021 21:43:34 -0500 Subject: [PATCH] 17249: make costanalyzer report preemptible status of the instance in the UUID report file. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- lib/costanalyzer/costanalyzer.go | 11 +++++++++-- lib/costanalyzer/costanalyzer_test.go | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index e8dd186050..37e655e53a 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -35,6 +35,7 @@ type nodeInfo struct { // Modern ProviderType string Price float64 + Preemptible bool } type arrayFlags []string @@ -90,6 +91,12 @@ Usage: was fulfilled. This program uses the cost data stored at the time of the execution of the container, stored in the 'node.json' file in its log collection. + - if a container was run on a preemptible ("spot") instance, the cost data + reported by this program may be wildly inaccurate, because it does not have + access to the spot pricing in effect for the node then the container ran. The + UUID report file that is generated when the '-output' option is specified has + a column that indicates the preemptible state of the instance that ran the + container. In order to get the data for the uuids supplied, the ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables must be set. @@ -181,7 +188,7 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container size = node.ProviderType } cost = delta.Seconds() / 3600 * price - csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" + csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" return } @@ -369,7 +376,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado cost = make(map[string]float64) - csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n" + csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n" var tmpCsv string var tmpTotalCost float64 var totalCost float64 diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go index b1ddf97a36..f4d8d10730 100644 --- a/lib/costanalyzer/costanalyzer_test.go +++ b/lib/costanalyzer/costanalyzer_test.go @@ -53,7 +53,7 @@ func (s *Suite) SetUpSuite(c *check.C) { "IncludedScratch": 64000000000, "AddedScratch": 0, "Price": 0.292, - "Preemptible": false + "Preemptible": true }` standardD32sV3JSON := `{ "Name": "Standard_D32s_v3", @@ -168,6 +168,8 @@ func (*Suite) TestContainerRequestUUID(c *check.C) { uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv") c.Assert(err, check.IsNil) + // Make sure the 'preemptible' flag was picked up + c.Check(string(uuidReport), check.Matches, "(?ms).*,Standard_E4s_v3,true,.*") c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' -- 2.39.5