17249: Merge branch 'master' into 17249-costanalyzer-preemptible

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index e8dd186050fb200ca575a480ff6d15db1d88adf0..37e655e53a3391b8820a3855efeb2ca891dcd5f8 100644 (file)
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -35,6 +35,7 @@ type nodeInfo struct {
        // Modern
        ProviderType string
        Price        float64
+       Preemptible  bool
 }
 
 type arrayFlags []string
@@ -90,6 +91,12 @@ Usage:
        was fulfilled. This program uses the cost data stored at the time of the
        execution of the container, stored in the 'node.json' file in its log
        collection.
+       - if a container was run on a preemptible ("spot") instance, the cost data
+       reported by this program may be wildly inaccurate, because it does not have
+       access to the spot pricing in effect for the node then the container ran. The
+       UUID report file that is generated when the '-output' option is specified has
+       a column that indicates the preemptible state of the instance that ran the
+       container.
 
        In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
        ARVADOS_API_TOKEN environment variables must be set.
@@ -181,7 +188,7 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container
                size = node.ProviderType
        }
        cost = delta.Seconds() / 3600 * price
-       csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+       csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
        return
 }
 
@@ -369,7 +376,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
 
        cost = make(map[string]float64)
 
-       csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+       csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
        var tmpCsv string
        var tmpTotalCost float64
        var totalCost float64

diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index b1ddf97a36af2c3a7a18de7f4ef7e5a865b60199..f4d8d1073068651d8e0f0c0bc72eb91316108763 100644 (file)
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -53,7 +53,7 @@ func (s *Suite) SetUpSuite(c *check.C) {
     "IncludedScratch": 64000000000,
     "AddedScratch": 0,
     "Price": 0.292,
-    "Preemptible": false
+    "Preemptible": true
 }`
        standardD32sV3JSON := `{
     "Name": "Standard_D32s_v3",
@@ -168,6 +168,8 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
 
        uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
        c.Assert(err, check.IsNil)
+       // Make sure the 'preemptible' flag was picked up
+       c.Check(string(uuidReport), check.Matches, "(?ms).*,Standard_E4s_v3,true,.*")
        c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
        re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
        matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'