// Modern
ProviderType string
Price float64
+ Preemptible bool
}
type arrayFlags []string
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...] <uuid> ...
+ %s [options ...] uuid [uuid ...]
This program analyzes the cost of Arvados container requests. For each uuid
supplied, it creates a CSV report that lists all the containers used to
container was reused between several container requests, its cost will only
be counted once.
- To get the node costs, the progam queries the Arvados API for current cost
- data for each node type used. This means that the reported cost always
- reflects the cost data as currently defined in the Arvados API configuration
- file.
-
Caveats:
- - the Arvados API configuration cost data may be out of sync with the cloud
- provider.
- - when generating reports for older container requests, the cost data in the
- Arvados API configuration file may have changed since the container request
- was fulfilled. This program uses the cost data stored at the time of the
+
+ - This program uses the cost data from config.yml at the time of the
execution of the container, stored in the 'node.json' file in its log
- collection.
+ collection. If the cost data was not correctly configured at the time the
+ container was executed, the output from this program will be incorrect.
+
+ - If a container was run on a preemptible ("spot") instance, the cost data
+ reported by this program may be wildly inaccurate, because it does not have
+ access to the spot pricing in effect for the node then the container ran. The
+ UUID report file that is generated when the '-output' option is specified has
+ a column that indicates the preemptible state of the instance that ran the
+ container.
+
+ - This program does not take into account overhead costs like the time spent
+ starting and stopping compute nodes that run containers, the cost of the
+ permanent cloud nodes that provide the Arvados services, the cost of data
+ stored in Arvados, etc.
+
+ - When provided with a project uuid, subprojects will not be considered.
In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
if len(uuids) < 1 {
flags.Usage()
- err = fmt.Errorf("Error: no uuid(s) provided")
+ err = fmt.Errorf("error: no uuid(s) provided")
exitCode = 2
return
}
size = node.ProviderType
}
cost = delta.Seconds() / 3600 * price
- csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ csv += size + "," + fmt.Sprintf("%+v", node.Preemptible) + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
return
}
func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
if cr.LogUUID == "" {
- err = errors.New("No log collection")
+ err = errors.New("no log collection")
return
}
cost = make(map[string]float64)
- csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Preemptible,Hourly node cost,Total cost\n"
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
+ if len(cr.ContainerUUID) == 0 {
+ // Nothing to do! E.g. a CR in 'Uncommitted' state.
+ logger.Infof("No container associated with container request %s, skipping\n", crUUID)
+ return nil, nil
+ }
var container arvados.Container
- err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+ err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
if err != nil {
return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", uuid)
+ logger.Infof("Collecting child containers for container request %s", crUUID)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
- err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+ err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
if resultsDir != "" {
// Write the resulting CSV file
- fName := resultsDir + "/" + uuid + ".csv"
+ fName := resultsDir + "/" + crUUID + ".csv"
err = ioutil.WriteFile(fName, []byte(csv), 0644)
if err != nil {
return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
var crCsv map[string]float64
crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
+ err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
// It is identified by the user uuid. As such, cost analysis for the
// "Home" project is not supported by this program. Skip this uuid, but
// keep going.
- logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
} else {
- logger.Errorf("This argument does not look like a uuid: %s\n", uuid)
+ logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
exitcode = 3
return
}
aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
- err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
+ err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
exitcode = 1
return
}
projects / arvados.git / blobdiff