From 9cdc2f320fe54562711d46ef7a9213697f2013b5 Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Fri, 13 Nov 2020 16:22:57 -0500 Subject: [PATCH] 16950: more changes after review. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- lib/costanalyzer/cmd.go | 4 +- lib/costanalyzer/costanalyzer.go | 338 ++++++++++---------------- lib/costanalyzer/costanalyzer_test.go | 33 ++- 3 files changed, 156 insertions(+), 219 deletions(-) diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go index 6829be7d38..800860ddf4 100644 --- a/lib/costanalyzer/cmd.go +++ b/lib/costanalyzer/cmd.go @@ -28,7 +28,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s logger := ctxlog.New(stderr, "text", "info") defer func() { if err != nil { - logger.WithError(err).Error("fatal") + logger.Error("\n" + err.Error() + "\n") } }() @@ -38,7 +38,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s loader := config.NewLoader(stdin, logger) loader.SkipLegacy = true - exitcode := costanalyzer(prog, args, loader, logger, stdout, stderr) + exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr) return exitcode } diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go index ccd9520bc2..319303b921 100644 --- a/lib/costanalyzer/costanalyzer.go +++ b/lib/costanalyzer/costanalyzer.go @@ -5,8 +5,6 @@ package costanalyzer import ( - "bytes" - "context" "encoding/json" "errors" "flag" @@ -26,52 +24,17 @@ import ( "github.com/sirupsen/logrus" ) -// LegacyNodeInfo is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3) -// Example: -// { -// "total_cpu_cores":2, -// "total_scratch_mb":33770, -// "cloud_node": -// { -// "price":0.1, -// "size":"m4.large" -// }, -// "total_ram_mb":7986 -// } -type LegacyNodeInfo struct { - CPUCores int64 `json:"total_cpu_cores"` - ScratchMb int64 `json:"total_scratch_mb"` - RAMMb int64 `json:"total_ram_mb"` - CloudNode LegacyCloudNode `json:"cloud_node"` -} - -// LegacyCloudNode is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3) -type LegacyCloudNode struct { - Price float64 `json:"price"` - Size string `json:"size"` -} - -// Node is a struct for records created by Arvados Dispatch Cloud (Arvados >= 2.0.0) -// Example: -// { -// "Name": "Standard_D1_v2", -// "ProviderType": "Standard_D1_v2", -// "VCPUs": 1, -// "RAM": 3584000000, -// "Scratch": 50000000000, -// "IncludedScratch": 50000000000, -// "AddedScratch": 0, -// "Price": 0.057, -// "Preemptible": false -//} -type Node struct { - VCPUs int64 - Scratch int64 - RAM int64 - Price float64 - Name string +type nodeInfo struct { + // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3) + Properties struct { + CloudNode struct { + Price float64 + Size string + } `json:"cloud_node"` + } + // Modern ProviderType string - Preemptible bool + Price float64 } type arrayFlags []string @@ -85,7 +48,7 @@ func (i *arrayFlags) Set(value string) error { return nil } -func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool) { +func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) { flags := flag.NewFlagSet("", flag.ContinueOnError) flags.SetOutput(stderr) flags.Usage = func() { @@ -127,12 +90,13 @@ Options: `, prog) flags.PrintDefaults() } - loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)") - resultsDir = *flags.String("output", "results", "output directory for the CSV reports") - flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.") + loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)") + resultsDir = *flags.String("output", "results", "output `directory` for the CSV reports") + flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once.") flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects") - err := flags.Parse(args) + err = flags.Parse(args) if err == flag.ErrHelp { + err = nil exitCode = 1 return } else if err != nil { @@ -141,8 +105,8 @@ Options: } if len(uuids) < 1 { - logger.Errorf("Error: no uuid(s) provided") flags.Usage() + err = fmt.Errorf("Error: no uuid(s) provided") exitCode = 2 return } @@ -161,90 +125,87 @@ func ensureDirectory(logger *logrus.Logger, dir string) (err error) { if os.IsNotExist(err) { err = os.MkdirAll(dir, 0700) if err != nil { - return fmt.Errorf("Error creating directory %s: %s\n", dir, err.Error()) + return fmt.Errorf("error creating directory %s: %s", dir, err.Error()) } } else { if !statData.IsDir() { - return fmt.Errorf("The path %s is not a directory\n", dir) + return fmt.Errorf("the path %s is not a directory", dir) } } return } -func addContainerLine(logger *logrus.Logger, node interface{}, cr, container map[string]interface{}) (csv string, cost float64) { - csv = cr["uuid"].(string) + "," - csv += cr["name"].(string) + "," - csv += container["uuid"].(string) + "," - csv += container["state"].(string) + "," - if container["started_at"] != nil { - csv += container["started_at"].(string) + "," +func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) { + csv = cr.UUID + "," + csv += cr.Name + "," + csv += container.UUID + "," + csv += string(container.State) + "," + if container.StartedAt != nil { + csv += container.StartedAt.String() + "," } else { csv += "," } var delta time.Duration - if container["finished_at"] != nil { - csv += container["finished_at"].(string) + "," - finishedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["finished_at"].(string)) - if err != nil { - fmt.Println(err) - } - startedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["started_at"].(string)) - if err != nil { - fmt.Println(err) - } - delta = finishedTimestamp.Sub(startedTimestamp) + if container.FinishedAt != nil { + csv += container.FinishedAt.String() + "," + delta = container.FinishedAt.Sub(*container.StartedAt) csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + "," } else { csv += ",," } var price float64 var size string - switch n := node.(type) { - case Node: - price = n.Price - size = n.ProviderType - case LegacyNodeInfo: - price = n.CloudNode.Price - size = n.CloudNode.Size - default: - logger.Warn("WARNING: unknown node type found!") + if node.Properties.CloudNode.Price != 0 { + price = node.Properties.CloudNode.Price + size = node.Properties.CloudNode.Size + } else { + price = node.Price + size = node.ProviderType } cost = delta.Seconds() / 3600 * price csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" return } -func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object map[string]interface{}) { +func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) { reload = true // See if we have a cached copy of this object - if _, err := os.Stat(file); err == nil { - data, err := ioutil.ReadFile(file) - if err != nil { - logger.Errorf("error reading %q: %s", file, err) - return - } - err = json.Unmarshal(data, &object) - if err != nil { - logger.Errorf("failed to unmarshal json: %s: %s", data, err) - return - } + _, err := os.Stat(file) + if err != nil { + return + } + data, err := ioutil.ReadFile(file) + if err != nil { + logger.Errorf("error reading %q: %s", file, err) + return + } + err = json.Unmarshal(data, &object) + if err != nil { + logger.Errorf("failed to unmarshal json: %s: %s", data, err) + return + } - // See if it is in a final state, if that makes sense + // See if it is in a final state, if that makes sense + switch v := object.(type) { + case arvados.Group: // Projects (j7d0g) do not have state so they should always be reloaded - if !strings.Contains(uuid, "-j7d0g-") { - if object["state"].(string) == "Complete" || object["state"].(string) == "Failed" { - reload = false - logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) - return - } + case arvados.Container: + if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled { + reload = false + logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) + } + case arvados.ContainerRequest: + if v.State == arvados.ContainerRequestStateFinal { + reload = false + logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) } } return } // Load an Arvados object. -func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string, cache bool) (object map[string]interface{}, err error) { +func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) { err = ensureDirectory(logger, path) if err != nil { return @@ -256,105 +217,70 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st if !cache { reload = true } else { - reload, object = loadCachedObject(logger, file, uuid) + reload = loadCachedObject(logger, file, uuid, &object) } if !reload { return } if strings.Contains(uuid, "-j7d0g-") { - err = arv.Get("groups", uuid, nil, &object) + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil) } else if strings.Contains(uuid, "-xvhdp-") { - err = arv.Get("container_requests", uuid, nil, &object) + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil) } else if strings.Contains(uuid, "-dz642-") { - err = arv.Get("containers", uuid, nil, &object) + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil) } else { - err = arv.Get("jobs", uuid, nil, &object) + err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err) + return } if err != nil { - err = fmt.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err) + err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err) return } encoded, err := json.MarshalIndent(object, "", " ") if err != nil { - err = fmt.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err) + err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err) return } err = ioutil.WriteFile(file, encoded, 0644) if err != nil { - err = fmt.Errorf("Error writing file %s:\n %s\n", file, err) + err = fmt.Errorf("error writing file %s:\n %s", file, err) return } return } -func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, itemMap map[string]interface{}) (node interface{}, err error) { - logUuid, ok := itemMap["log_uuid"] - if !ok { +func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) { + if cr.LogUUID == "" { err = errors.New("No log collection") return } var collection arvados.Collection - err = arv.Get("collections", logUuid.(string), nil, &collection) + err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil) if err != nil { - err = fmt.Errorf("Error getting collection: %s", err) + err = fmt.Errorf("error getting collection: %s", err) return } var fs arvados.CollectionFileSystem fs, err = collection.FileSystem(ac, kc) if err != nil { - err = fmt.Errorf("Error opening collection as filesystem: %s", err) + err = fmt.Errorf("error opening collection as filesystem: %s", err) return } var f http.File f, err = fs.Open("node.json") if err != nil { - err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", logUuid.(string), err) + err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err) return } - var nodeDict map[string]interface{} - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(f) + err = json.NewDecoder(f).Decode(&node) if err != nil { - err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", logUuid.(string), err) + err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err) return } - contents := buf.String() - f.Close() - - err = json.Unmarshal([]byte(contents), &nodeDict) - if err != nil { - err = fmt.Errorf("Error unmarshalling: %s", err) - return - } - if val, ok := nodeDict["properties"]; ok { - var encoded []byte - encoded, err = json.MarshalIndent(val, "", " ") - if err != nil { - err = fmt.Errorf("Error marshalling: %s", err) - return - } - // node is type LegacyNodeInfo - var newNode LegacyNodeInfo - err = json.Unmarshal(encoded, &newNode) - if err != nil { - err = fmt.Errorf("Error unmarshalling: %s", err) - return - } - node = newNode - } else { - // node is type Node - var newNode Node - err = json.Unmarshal([]byte(contents), &newNode) - if err != nil { - err = fmt.Errorf("Error unmarshalling: %s", err) - return - } - node = newNode - } return } @@ -362,20 +288,18 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado cost = make(map[string]float64) - project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache) + var project arvados.Group + err = loadObject(logger, ac, resultsDir+"/"+uuid, uuid, cache, &project) if err != nil { - return nil, fmt.Errorf("Error loading object %s: %s\n", uuid, err.Error()) + return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error()) } - // arv -f uuid container_request list --filters '[["owner_uuid","=",""],["requesting_container_uuid","=",null]]' - - // Now find all container requests that have the container we found above as requesting_container_uuid var childCrs map[string]interface{} filterset := []arvados.Filter{ { Attr: "owner_uuid", Operator: "=", - Operand: project["uuid"].(string), + Operand: project.UUID, }, { Attr: "requesting_container_uuid", @@ -383,12 +307,12 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado Operand: nil, }, } - err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ + err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ "filters": filterset, "limit": 10000, }) if err != nil { - return nil, fmt.Errorf("Error querying container_requests: %s\n", err.Error()) + return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) } if value, ok := childCrs["items"]; ok { logger.Infof("Collecting top level container requests in project %s\n", uuid) @@ -397,7 +321,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado itemMap := item.(map[string]interface{}) crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache) if err != nil { - return nil, fmt.Errorf("Error generating container_request CSV: %s\n", err.Error()) + return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error()) } for k, v := range crCsv { cost[k] = v @@ -419,59 +343,59 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado var totalCost float64 // This is a container request, find the container - cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache) + var cr arvados.ContainerRequest + err = loadObject(logger, ac, resultsDir+"/"+uuid, uuid, cache, &cr) if err != nil { - return nil, fmt.Errorf("Error loading object %s: %s", uuid, err) + return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err) } - container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string), cache) + fmt.Printf("cr: %+v\n", cr) + var container arvados.Container + err = loadObject(logger, ac, resultsDir+"/"+uuid, cr.ContainerUUID, cache, &container) if err != nil { - return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err) + return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err) } topNode, err := getNode(arv, ac, kc, cr) if err != nil { - return nil, fmt.Errorf("Error getting node %s: %s\n", cr["uuid"], err) + return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err) } tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container) csv += tmpCsv totalCost += tmpTotalCost - cost[container["uuid"].(string)] = totalCost + cost[container.UUID] = totalCost - // Now find all container requests that have the container we found above as requesting_container_uuid - var childCrs map[string]interface{} + // Find all container requests that have the container we found above as requesting_container_uuid + var childCrs arvados.ContainerRequestList filterset := []arvados.Filter{ { Attr: "requesting_container_uuid", Operator: "=", - Operand: container["uuid"].(string), + Operand: container.UUID, }} - err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ + err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ "filters": filterset, "limit": 10000, }) if err != nil { return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) } - if value, ok := childCrs["items"]; ok { - logger.Infof("Collecting child containers for container request %s", uuid) - items := value.([]interface{}) - for _, item := range items { - logger.Info(".") - itemMap := item.(map[string]interface{}) - node, err := getNode(arv, ac, kc, itemMap) - if err != nil { - return nil, fmt.Errorf("Error getting node %s: %s\n", itemMap["uuid"], err) - } - logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n") - c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string), cache) - if err != nil { - return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err) - } - tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2) - cost[itemMap["container_uuid"].(string)] = tmpTotalCost - csv += tmpCsv - totalCost += tmpTotalCost + logger.Infof("Collecting child containers for container request %s", uuid) + for _, cr2 := range childCrs.Items { + logger.Info(".") + node, err := getNode(arv, ac, kc, cr2) + if err != nil { + return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err) } + logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n") + var c2 arvados.Container + err = loadObject(logger, ac, resultsDir+"/"+uuid, cr2.ContainerUUID, cache, &c2) + if err != nil { + return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err) + } + tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2) + cost[cr2.ContainerUUID] = tmpTotalCost + csv += tmpCsv + totalCost += tmpTotalCost } logger.Info(" done\n") @@ -481,20 +405,19 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado fName := resultsDir + "/" + uuid + ".csv" err = ioutil.WriteFile(fName, []byte(csv), 0644) if err != nil { - return nil, fmt.Errorf("Error writing file with path %s: %s\n", fName, err.Error()) + return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error()) } return } -func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) { - exitcode, uuids, resultsDir, cache := parseFlags(prog, args, loader, logger, stderr) +func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) { + exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr) if exitcode != 0 { return } - err := ensureDirectory(logger, resultsDir) + err = ensureDirectory(logger, resultsDir) if err != nil { - logger.Errorf("%s", err) exitcode = 3 return } @@ -502,13 +425,13 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log // Arvados Client setup arv, err := arvadosclient.MakeArvadosClient() if err != nil { - logger.Errorf("error creating Arvados object: %s", err) + err = fmt.Errorf("error creating Arvados object: %s", err) exitcode = 1 return } kc, err := keepclient.MakeKeepClient(arv) if err != nil { - logger.Errorf("error creating Keep object: %s", err) + err = fmt.Errorf("error creating Keep object: %s", err) exitcode = 1 return } @@ -521,8 +444,6 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log // This is a project (group) cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache) if err != nil { - // FIXME print error - logger.Info(err.Error()) exitcode = 1 return } @@ -531,9 +452,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log } } else if strings.Contains(uuid, "-xvhdp-") { // This is a container request - crCsv, err := generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache) + var crCsv map[string]float64 + crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache) if err != nil { - logger.Fatalf("Error generating container_request CSV: %s\n", err.Error()) + err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error()) + exitcode = 2 + return } for k, v := range crCsv { cost[k] = v @@ -541,7 +465,8 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log } else if strings.Contains(uuid, "-tpzed-") { // This is a user. The "Home" project for a user is not a real project. // It is identified by the user uuid. As such, cost analysis for the - // "Home" project is not supported by this program. + // "Home" project is not supported by this program. Skip this uuid, but + // keep going. logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid) } } @@ -575,11 +500,10 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv" err = ioutil.WriteFile(aFile, []byte(csv), 0644) if err != nil { - logger.Errorf("Error writing file with path %s: %s\n", aFile, err.Error()) + err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error()) exitcode = 1 return - } else { - logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile) } + logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile) return } diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go index 0a44be8d87..e2fb9e99bd 100644 --- a/lib/costanalyzer/costanalyzer_test.go +++ b/lib/costanalyzer/costanalyzer_test.go @@ -6,7 +6,6 @@ package costanalyzer import ( "bytes" - "context" "io" "io/ioutil" "os" @@ -92,6 +91,18 @@ func (s *Suite) SetUpSuite(c *check.C) { "Preemptible": false }` + legacyD1V2JSON := `{ + "properties": { + "cloud_node": { + "price": 0.073001, + "size": "Standard_D1_v2" + }, + "total_cpu_cores": 1, + "total_ram_mb": 3418, + "total_scratch_mb": 51170 + } +}` + // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON) createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON) @@ -100,19 +111,19 @@ func (s *Suite) SetUpSuite(c *check.C) { createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON) createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON) createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON) - createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON) } func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) { // Get the CR var cr arvados.ContainerRequest - err := ac.RequestAndDecodeContext(context.Background(), &cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil) + err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil) c.Assert(err, check.Equals, nil) c.Assert(cr.LogUUID, check.Equals, logUUID) // Get the log collection var coll arvados.Collection - err = ac.RequestAndDecodeContext(context.Background(), &coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil) + err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil) c.Assert(err, check.IsNil) // Create a node.json file -- the fixture doesn't actually contain the contents of the collection. @@ -131,7 +142,7 @@ func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Cl c.Assert(mtxt, check.NotNil) // Update collection record - err = ac.RequestAndDecodeContext(context.Background(), &coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{ + err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{ "collection": map[string]interface{}{ "manifest_text": mtxt, }, @@ -195,13 +206,13 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) { // the analysis with the project uuid. The results should be identical. ac := arvados.NewClientFromEnv() var cr arvados.ContainerRequest - err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{ + err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{ "container_request": map[string]interface{}{ "owner_uuid": arvadostest.AProjectUUID, }, }) c.Assert(err, check.IsNil) - err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{ + err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{ "container_request": map[string]interface{}{ "owner_uuid": arvadostest.AProjectUUID, }, @@ -234,16 +245,18 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) { var stdout, stderr bytes.Buffer // Run costanalyzer with 2 container request uuids exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Logf("%s", stderr.Bytes()) + c.Logf("%s", stdout.Bytes()) c.Check(exitcode, check.Equals, 0) c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*") uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv") c.Assert(err, check.IsNil) - c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539") + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192") uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv") c.Assert(err, check.IsNil) - c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435") + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088") re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' @@ -251,5 +264,5 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) { aggregateCostReport, err := ioutil.ReadFile(matches[1]) c.Assert(err, check.IsNil) - c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377") + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030") } -- 2.30.2