remote: https://rubygems.org/
specs:
RedCloth (4.3.2)
- actioncable (5.2.4.5)
- actionpack (= 5.2.4.5)
+ actioncable (5.2.6)
+ actionpack (= 5.2.6)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
+ actionmailer (5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.4.5)
- actionview (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionpack (5.2.6)
+ actionview (= 5.2.6)
+ activesupport (= 5.2.6)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionview (5.2.6)
+ activesupport (= 5.2.6)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.4.5)
- activesupport (= 5.2.4.5)
+ activejob (5.2.6)
+ activesupport (= 5.2.6)
globalid (>= 0.3.6)
- activemodel (5.2.4.5)
- activesupport (= 5.2.4.5)
- activerecord (5.2.4.5)
- activemodel (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ activemodel (5.2.6)
+ activesupport (= 5.2.6)
+ activerecord (5.2.6)
+ activemodel (= 5.2.6)
+ activesupport (= 5.2.6)
arel (>= 9.0)
- activestorage (5.2.4.5)
- actionpack (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- marcel (~> 0.3.1)
- activesupport (5.2.4.5)
+ activestorage (5.2.6)
+ actionpack (= 5.2.6)
+ activerecord (= 5.2.6)
+ marcel (~> 1.0.0)
+ activesupport (5.2.6)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
execjs
coffee-script-source (1.12.2)
commonjs (0.2.7)
- concurrent-ruby (1.1.8)
+ concurrent-ruby (1.1.9)
crass (1.0.6)
deep_merge (1.2.1)
docile (1.3.1)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.9.0)
+ loofah (2.10.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (0.3.3)
- mimemagic (~> 0.3.2)
+ marcel (1.0.1)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
mime-types (3.2.2)
mime-types-data (~> 3.2015)
mime-types-data (3.2019.0331)
- mimemagic (0.3.8)
- nokogiri (~> 1)
- mini_mime (1.0.2)
- mini_portile2 (2.5.1)
+ mini_mime (1.1.0)
+ mini_portile2 (2.5.3)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.7)
- nokogiri (1.11.5)
+ nokogiri (1.11.7)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
npm-rails (0.2.1)
rack (>= 1.2.0)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.4.5)
- actioncable (= 5.2.4.5)
- actionmailer (= 5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
- activemodel (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- activestorage (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ rails (5.2.6)
+ actioncable (= 5.2.6)
+ actionmailer (= 5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
+ activemodel (= 5.2.6)
+ activerecord (= 5.2.6)
+ activestorage (= 5.2.6)
+ activesupport (= 5.2.6)
bundler (>= 1.3.0)
- railties (= 5.2.4.5)
+ railties (= 5.2.6)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-html-sanitizer (1.3.0)
loofah (~> 2.3)
rails-perftest (0.0.7)
- railties (5.2.4.5)
- actionpack (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ railties (5.2.6)
+ actionpack (= 5.2.6)
+ activesupport (= 5.2.6)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.7.3)
+ websocket-driver (0.7.4)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
xpath (2.1.0)
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
- golang.org/x/net v0.0.0-20201021035429-f5854403a974
+ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20210603125802-9665404d3644
golang.org/x/tools v0.1.0 // indirect
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007
+ golang.org/x/tools v0.1.2 // indirect
google.golang.org/api v0.13.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
github.com/xanzy/ssh-agent v0.1.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
import (
"io"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
)
-var Command command
+var Command = command{}
-type command struct{}
+type command struct {
+ uuids arrayFlags
+ resultsDir string
+ cache bool
+ begin time.Time
+ end time.Time
+}
// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
-func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (c command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var err error
logger := ctxlog.New(stderr, "text", "info")
logger.SetFormatter(cmd.NoPrefixFormatter{})
}
}()
- loader := config.NewLoader(stdin, logger)
- loader.SkipLegacy = true
-
- exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+ exitcode, err := c.costAnalyzer(prog, args, logger, stdout, stderr)
return exitcode
}
"strings"
"time"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/sirupsen/logrus"
)
+const timestampFormat = "2006-01-02T15:04:05"
+
type nodeInfo struct {
// Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
Properties struct {
return nil
}
-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+func (c *command) parseFlags(prog string, args []string, logger *logrus.Logger, stderr io.Writer) (exitCode int, err error) {
+ var beginStr, endStr string
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...] uuid [uuid ...]
+ %s [options ...] [UUID ...]
+
+ This program analyzes the cost of Arvados container requests and calculates
+ the total cost across all requests. At least one UUID or a timestamp range
+ must be specified.
- This program analyzes the cost of Arvados container requests. For each uuid
- supplied, it creates a CSV report that lists all the containers used to
- fulfill the container request, together with the machine type and cost of
- each container. At least one uuid must be specified.
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory. Each file is a CSV report that lists
+ all the containers used to fulfill the container request, together with the
+ machine type and cost of each container.
- When supplied with the uuid of a container request, it will calculate the
+ When supplied with the UUID of a container request, it will calculate the
cost of that container request and all its children.
- When supplied with the uuid of a collection, it will see if there is a
- container_request uuid in the properties of the collection, and if so, it
+ When supplied with the UUID of a collection, it will see if there is a
+ container_request UUID in the properties of the collection, and if so, it
will calculate the cost of that container request and all its children.
- When supplied with a project uuid or when supplied with multiple container
- request or collection uuids, it will create a CSV report for each supplied
- uuid, as well as a CSV file with aggregate cost accounting for all supplied
- uuids. The aggregate cost report takes container reuse into account: if a
- container was reused between several container requests, its cost will only
- be counted once.
+ When supplied with a project UUID or when supplied with multiple container
+ request or collection UUIDs, it will calculate the total cost for all
+ supplied UUIDs.
+
+ When supplied with a 'begin' and 'end' timestamp (format:
+ %s), it will calculate the cost for all top-level container
+ requests whose containers finished during the specified interval.
+
+ The total cost calculation takes container reuse into account: if a container
+ was reused between several container requests, its cost will only be counted
+ once.
Caveats:
permanent cloud nodes that provide the Arvados services, the cost of data
stored in Arvados, etc.
- - When provided with a project uuid, subprojects will not be considered.
+ - When provided with a project UUID, subprojects will not be considered.
- In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ In order to get the data for the UUIDs supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
This program prints the total dollar amount from the aggregate cost
- accounting across all provided uuids on stdout.
-
- When the '-output' option is specified, a set of CSV files with cost details
- will be written to the provided directory.
+ accounting across all provided UUIDs on stdout.
Options:
-`, prog)
+`, prog, timestampFormat)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
- flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+ flags.StringVar(&c.resultsDir, "output", "", "output `directory` for the CSV reports")
+ flags.StringVar(&beginStr, "begin", "", fmt.Sprintf("timestamp `begin` for date range operation (format: %s)", timestampFormat))
+ flags.StringVar(&endStr, "end", "", fmt.Sprintf("timestamp `end` for date range operation (format: %s)", timestampFormat))
+ flags.BoolVar(&c.cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
exitCode = 2
return
}
- uuids = flags.Args()
+ c.uuids = flags.Args()
+
+ if (len(beginStr) != 0 && len(endStr) == 0) || (len(beginStr) == 0 && len(endStr) != 0) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be specified")
+ exitCode = 2
+ return
+ }
+
+ if len(beginStr) != 0 {
+ var errB, errE error
+ c.begin, errB = time.Parse(timestampFormat, beginStr)
+ c.end, errE = time.Parse(timestampFormat, endStr)
+ if (errB != nil) || (errE != nil) {
+ flags.Usage()
+ err = fmt.Errorf("When specifying a date range, both begin and end must be of the format %s %+v, %+v", timestampFormat, errB, errE)
+ exitCode = 2
+ return
+ }
+ }
- if len(uuids) < 1 {
+ if (len(c.uuids) < 1) && (len(beginStr) == 0) {
flags.Usage()
err = fmt.Errorf("error: no uuid(s) provided")
exitCode = 2
return
}
logger.SetLevel(lvl)
- if !cache {
+ if !c.cache {
logger.Debug("Caching disabled")
}
return
var tmpCsv string
var tmpTotalCost float64
var totalCost float64
+ fmt.Printf("Processing %s\n", uuid)
var crUUID = uuid
if strings.Contains(uuid, "-4zz18-") {
topNode, err := getNode(arv, ac, kc, cr)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr.UUID, cr.UUID, err)
+ return nil, nil
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", crUUID)
+ logger.Infof("Collecting child containers for container request %s (%s)", crUUID, container.FinishedAt)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
if err != nil {
- return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+ continue
}
logger.Debug("\nChild container: " + cr2.ContainerUUID)
var c2 arvados.Container
return
}
-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
- exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+func (c *command) costAnalyzer(prog string, args []string, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, err = c.parseFlags(prog, args, logger, stderr)
+
if exitcode != 0 {
return
}
- if resultsDir != "" {
- err = ensureDirectory(logger, resultsDir)
+ if c.resultsDir != "" {
+ err = ensureDirectory(logger, c.resultsDir)
if err != nil {
exitcode = 3
return
}
}
+ uuidChannel := make(chan string)
+
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
ac := arvados.NewClientFromEnv()
+ // Populate uuidChannel with the requested uuid list
+ go func() {
+ defer close(uuidChannel)
+ for _, uuid := range c.uuids {
+ uuidChannel <- uuid
+ }
+
+ if !c.begin.IsZero() {
+ initialParams := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{"container.finished_at", ">=", c.begin}, {"container.finished_at", "<", c.end}, {"requesting_container_uuid", "=", nil}},
+ Order: "created_at",
+ }
+ params := initialParams
+ for {
+ // This list variable must be a new one declared
+ // inside the loop: otherwise, items in the API
+ // response would get deep-merged into the items
+ // loaded in previous iterations.
+ var list arvados.ContainerRequestList
+
+ err := ac.RequestAndDecode(&list, "GET", "arvados/v1/container_requests", nil, params)
+ if err != nil {
+ logger.Errorf("Error getting container request list from Arvados API: %s\n", err)
+ break
+ }
+ if len(list.Items) == 0 {
+ break
+ }
+
+ for _, i := range list.Items {
+ uuidChannel <- i.UUID
+ }
+ params.Offset += len(list.Items)
+ }
+
+ }
+ }()
+
cost := make(map[string]float64)
- for _, uuid := range uuids {
+
+ for uuid := range uuidChannel {
+ fmt.Printf("Considering %s\n", uuid)
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
- cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ cost, err = handleProject(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
exitcode = 1
return
} else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
- crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, c.resultsDir, c.cache)
if err != nil {
err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
var csv string
csv = "# Aggregate cost accounting for uuids:\n"
- for _, uuid := range uuids {
+ for _, uuid := range c.uuids {
csv += "# " + uuid + "\n"
}
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- if resultsDir != "" {
+ if c.resultsDir != "" {
// Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ aFile := c.resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
}
+func (*Suite) TestTimestampRange(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+ // Run costanalyzer with a timestamp range. This should pick up two container requests in "Final" state.
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, "-begin", "2020-11-02T00:00:00", "-end", "2020-11-03T23:59:00"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ uuid2Report, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
+ c.Check(string(uuid2Report), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+}
+
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
resultsDir := c.MkDir()
else:
# The file already exist on remote collection, skip it.
pass
- self._remote_collection.save(storage_classes=self.storage_classes,
- num_retries=self.num_retries,
+ self._remote_collection.save(num_retries=self.num_retries,
trash_at=self._collection_trash_at())
else:
- if self.storage_classes is None:
- self.storage_classes = ['default']
self._local_collection.save_new(
name=self.name, owner_uuid=self.owner_uuid,
- storage_classes=self.storage_classes,
ensure_unique_name=self.ensure_unique_name,
num_retries=self.num_retries,
trash_at=self._collection_trash_at())
self._remote_collection = arvados.collection.Collection(
update_collection,
api_client=self._api_client,
+ storage_classes_desired=self.storage_classes,
num_retries=self.num_retries)
except arvados.errors.ApiError as error:
raise CollectionUpdateError("Cannot read collection {} ({})".format(update_collection, error))
self._local_collection = arvados.collection.Collection(
self._state['manifest'],
replication_desired=self.replication_desired,
+ storage_classes_desired=(self.storage_classes or ['default']),
put_threads=self.put_threads,
api_client=self._api_client,
num_retries=self.num_retries)
# Split storage-classes argument
storage_classes = None
if args.storage_classes:
- storage_classes = args.storage_classes.strip().split(',')
- if len(storage_classes) > 1:
- logger.error("Multiple storage classes are not supported currently.")
- sys.exit(1)
-
+ storage_classes = args.storage_classes.strip().replace(' ', '').split(',')
# Setup exclude regex from all the --exclude arguments provided
name_patterns = []
self.call_main_with_args,
['--project-uuid', self.Z_UUID, '--stream'])
- def test_error_when_multiple_storage_classes_specified(self):
- self.assertRaises(SystemExit,
- self.call_main_with_args,
- ['--storage-classes', 'hot,cold'])
-
def test_error_when_excluding_absolute_path(self):
tmpdir = self.make_tmpdir()
self.assertRaises(SystemExit,
def test_put_collection_with_storage_classes_specified(self):
collection = self.run_and_find_collection("", ['--storage-classes', 'hot'])
-
self.assertEqual(len(collection['storage_classes_desired']), 1)
self.assertEqual(collection['storage_classes_desired'][0], 'hot')
+ def test_put_collection_with_multiple_storage_classes_specified(self):
+ collection = self.run_and_find_collection("", ['--storage-classes', ' foo, bar ,baz'])
+ self.assertEqual(len(collection['storage_classes_desired']), 3)
+ self.assertEqual(collection['storage_classes_desired'], ['foo', 'bar', 'baz'])
+
def test_put_collection_without_storage_classes_specified(self):
collection = self.run_and_find_collection("")
-
self.assertEqual(len(collection['storage_classes_desired']), 1)
self.assertEqual(collection['storage_classes_desired'][0], 'default')
GEM
remote: https://rubygems.org/
specs:
- actioncable (5.2.4.5)
- actionpack (= 5.2.4.5)
+ actioncable (5.2.6)
+ actionpack (= 5.2.6)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
+ actionmailer (5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.4.5)
- actionview (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionpack (5.2.6)
+ actionview (= 5.2.6)
+ activesupport (= 5.2.6)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.4.5)
- activesupport (= 5.2.4.5)
+ actionview (5.2.6)
+ activesupport (= 5.2.6)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.4.5)
- activesupport (= 5.2.4.5)
+ activejob (5.2.6)
+ activesupport (= 5.2.6)
globalid (>= 0.3.6)
- activemodel (5.2.4.5)
- activesupport (= 5.2.4.5)
- activerecord (5.2.4.5)
- activemodel (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ activemodel (5.2.6)
+ activesupport (= 5.2.6)
+ activerecord (5.2.6)
+ activemodel (= 5.2.6)
+ activesupport (= 5.2.6)
arel (>= 9.0)
- activestorage (5.2.4.5)
- actionpack (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- marcel (~> 0.3.1)
- activesupport (5.2.4.5)
+ activestorage (5.2.6)
+ actionpack (= 5.2.6)
+ activerecord (= 5.2.6)
+ marcel (~> 1.0.0)
+ activesupport (5.2.6)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
net-sftp (>= 2.0.0)
net-ssh (>= 2.0.14)
net-ssh-gateway (>= 1.1.0)
- concurrent-ruby (1.1.8)
+ concurrent-ruby (1.1.9)
crass (1.0.6)
erubi (1.10.0)
execjs (2.7.0)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.9.0)
+ loofah (2.10.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (0.3.3)
- mimemagic (~> 0.3.2)
+ marcel (1.0.1)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
- mimemagic (0.3.8)
- nokogiri (~> 1)
- mini_mime (1.0.2)
- mini_portile2 (2.5.1)
+ mini_mime (1.1.0)
+ mini_portile2 (2.5.3)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nio4r (2.5.7)
- nokogiri (1.11.5)
+ nokogiri (1.11.7)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
oauth2 (1.4.1)
rack (2.2.3)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.4.5)
- actioncable (= 5.2.4.5)
- actionmailer (= 5.2.4.5)
- actionpack (= 5.2.4.5)
- actionview (= 5.2.4.5)
- activejob (= 5.2.4.5)
- activemodel (= 5.2.4.5)
- activerecord (= 5.2.4.5)
- activestorage (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ rails (5.2.6)
+ actioncable (= 5.2.6)
+ actionmailer (= 5.2.6)
+ actionpack (= 5.2.6)
+ actionview (= 5.2.6)
+ activejob (= 5.2.6)
+ activemodel (= 5.2.6)
+ activerecord (= 5.2.6)
+ activestorage (= 5.2.6)
+ activesupport (= 5.2.6)
bundler (>= 1.3.0)
- railties (= 5.2.4.5)
+ railties (= 5.2.6)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-observers (0.1.5)
activemodel (>= 4.0)
rails-perftest (0.0.7)
- railties (5.2.4.5)
- actionpack (= 5.2.4.5)
- activesupport (= 5.2.4.5)
+ railties (5.2.6)
+ actionpack (= 5.2.6)
+ activesupport (= 5.2.6)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.7.3)
+ websocket-driver (0.7.4)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
if self.new_record? || self.state_was == Uncommitted
# Allow create-and-commit in a single operation.
permitted.push(*AttrsPermittedBeforeCommit)
+ elsif mounts_changed? && mounts_was.keys == mounts.keys
+ # Ignore the updated mounts if the only changes are default/zero
+ # values as added by controller, see 17774
+ only_defaults = true
+ mounts.each do |path, mount|
+ (mount.to_a - mounts_was[path].to_a).each do |k, v|
+ if !["", false, nil].index(v)
+ only_defaults = false
+ end
+ end
+ end
+ if only_defaults
+ clear_attribute_change("mounts")
+ end
end
case self.state
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontainr
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
requester_for_running:
uuid: zzzzz-xvhdp-req4runningcntr
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-logscontainer03
requesting_container_uuid: zzzzz-dz642-runningcontainr
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
running_older:
uuid: zzzzz-xvhdp-cr4runningcntn2
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontain2
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
completed:
uuid: zzzzz-xvhdp-cr4completedctr
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontain2
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
cr_for_failed:
uuid: zzzzz-xvhdp-cr4failedcontnr
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-runningcontainr
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
running_to_be_deleted:
uuid: zzzzz-xvhdp-cr5runningcntnr
cwd: test
output_path: test
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
container_uuid: zzzzz-dz642-runnincntrtodel
runtime_constraints:
vcpus: 1
ram: 123
- mounts: {}
completed_with_input_mounts:
uuid: zzzzz-xvhdp-crwithinputmnts
updated_at: <%= 1.minute.ago.to_s(:db) %>
started_at: <%= 1.minute.ago.to_s(:db) %>
container_image: test
- cwd: test
- output_path: test
+ cwd: /tmp
+ output_path: /tmp
command: ["echo", "hello"]
runtime_constraints:
ram: 12000000000
vcpus: 4
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
secret_mounts:
/secret/6x9:
kind: text
updated_at: <%= 2.minute.ago.to_s(:db) %>
started_at: <%= 2.minute.ago.to_s(:db) %>
container_image: test
- cwd: test
- output_path: test
+ cwd: /tmp
+ output_path: /tmp
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
ram: 12000000000
vcpus: 4
cwd: test
output_path: test
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
ram: 12000000000
vcpus: 4
cwd: test
output_path: test
command: ["echo", "hello"]
+ mounts:
+ /tmp:
+ kind: tmp
+ capacity: 24000000000
runtime_constraints:
ram: 12000000000
vcpus: 4
['Committed', false, {container_count: 2}],
['Committed', false, {container_count: 0}],
['Committed', false, {container_count: nil}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"kind" => "tmp", "capacity" => 1000000}}}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp"}}}],
+ # Addition of default values for mounts / runtime_constraints /
+ # scheduling_parameters, as happens in a round-trip through
+ # controller, does not have any real effect and should be
+ # accepted/ignored rather than causing an error when the CR state
+ # dictates those attributes are not allowed to change.
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "exclude_from_output": false}}}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "repository_name": ""}}}],
+ ['Committed', true, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "content": nil}}}],
+ ['Committed', false, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "content": {}}}}],
+ ['Committed', false, {priority: 0, mounts: {"/out" => {"capacity" => 1000000, "kind" => "tmp", "repository_name": "foo"}}}],
+ ['Committed', false, {priority: 0, mounts: {"/out" => {"kind" => "tmp", "capacity" => 1234567}}}],
+ ['Committed', false, {priority: 0, mounts: {}}],
+ ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2}}],
+ ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "keep_cache_ram" => 0}}],
+ ['Committed', true, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "API" => false}}],
+ ['Committed', false, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "keep_cache_ram" => 1}}],
+ ['Committed', false, {priority: 0, runtime_constraints: {"vcpus" => 1, "ram" => 2, "API" => true}}],
+ ['Committed', true, {priority: 0, scheduling_parameters: {"preemptible" => false}}],
+ ['Committed', true, {priority: 0, scheduling_parameters: {"partitions" => []}}],
+ ['Committed', true, {priority: 0, scheduling_parameters: {"max_run_time" => 0}}],
+ ['Committed', false, {priority: 0, scheduling_parameters: {"preemptible" => true}}],
+ ['Committed', false, {priority: 0, scheduling_parameters: {"partitions" => ["foo"]}}],
+ ['Committed', false, {priority: 0, scheduling_parameters: {"max_run_time" => 1}}],
['Final', false, {state: ContainerRequest::Committed, name: "foobar"}],
['Final', false, {name: "foobar", priority: 123}],
['Final', false, {name: "foobar", output_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
projects / arvados.git / commitdiff