17609: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --cc go.mod
index 9062c667d7838f7684f1ac1fd0e98e94e046fe74,0ff679a576e232d318bdc14873069115036967bc..b70f6f3b476789f69f1845bffb6bfd4fcac80885
--- 1/go.mod
--- 2/go.mod
+++ b/go.mod
@@@ -59,10 -59,10 +59,12 @@@ require 
        github.com/src-d/gcfg v1.3.0 // indirect
        github.com/xanzy/ssh-agent v0.1.0 // indirect
        golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-       golang.org/x/net v0.0.0-20201021035429-f5854403a974
+       golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
        golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 +      golang.org/x/sys v0.0.0-20210603125802-9665404d3644
 +      golang.org/x/tools v0.1.0 // indirect
+       golang.org/x/sys v0.0.0-20210510120138-977fb7262007
+       golang.org/x/tools v0.1.2 // indirect
        google.golang.org/api v0.13.0
        gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
        gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405

diff --cc go.sum
index 23731f69a0861ed859456f3895275782fdeb722d,c28ab4624038a143a4d9adf8a9e2593d123183c0..1fd37ed11ce411e625518eb3189c225e80b74616
--- 1/go.sum
--- 2/go.sum
+++ b/go.sum
@@@ -303,12 -309,15 +311,18 @@@ golang.org/x/sys v0.0.0-20190422165155-
  golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
  golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
  golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 +golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
 +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
  golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
  golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

diff --cc lib/costanalyzer/cmd.go
index b08e943c0120e968862c1a35703dbe2d8c90fa01,525ec619b5e4aeeb0b4e271704f13385a41bcefd..6065ad2c0b2cb934607826d8500215bbabf0d868
--- 1/lib/costanalyzer/cmd.go
--- 2/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@@ -6,31 -6,41 +6,34 @@@ package costanalyze
  
  import (
        "io"
+       "time"
  
-       "git.arvados.org/arvados.git/lib/config"
 +      "git.arvados.org/arvados.git/lib/cmd"
        "git.arvados.org/arvados.git/sdk/go/ctxlog"
 -      "github.com/sirupsen/logrus"
  )
  
- var Command command
+ var Command = command{}
  
- type command struct{}
+ type command struct {
+       uuids      arrayFlags
+       resultsDir string
+       cache      bool
+       begin      time.Time
+       end        time.Time
+ }
  
 -type NoPrefixFormatter struct{}
 -
 -func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
 -      return []byte(entry.Message), nil
 -}
 -
  // RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
- func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ func (c command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
        var err error
        logger := ctxlog.New(stderr, "text", "info")
 +      logger.SetFormatter(cmd.NoPrefixFormatter{})
        defer func() {
                if err != nil {
 -                      logger.Error("\n" + err.Error() + "\n")
 +                      logger.Error("\n" + err.Error())
                }
        }()
  
-       loader := config.NewLoader(stdin, logger)
-       loader.SkipLegacy = true
 -      logger.SetFormatter(new(NoPrefixFormatter))
--
-       exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+       exitcode, err := c.costAnalyzer(prog, args, logger, stdout, stderr)
  
        return exitcode
  }

diff --cc lib/costanalyzer/costanalyzer.go
index 23df754bcac8d86ec694153ef2f95980d0977f6c,edaaa5bd178243f2c9044d32398c5ddccc67b5dc..f9875c6a8e79cf427a378c34a1b0a9d4d9264473
--- 1/lib/costanalyzer/costanalyzer.go
--- 2/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@@ -17,10 -20,6 +17,9 @@@ import 
        "strings"
        "time"
  
-       "git.arvados.org/arvados.git/lib/config"
 +      "git.arvados.org/arvados.git/sdk/go/arvados"
 +      "git.arvados.org/arvados.git/sdk/go/arvadosclient"
 +      "git.arvados.org/arvados.git/sdk/go/keepclient"
        "github.com/sirupsen/logrus"
  )
  
@@@ -139,8 -168,8 +168,8 @@@ Options
                return
        }
        logger.SetLevel(lvl)
-       if !cache {
+       if !c.cache {
 -              logger.Debug("Caching disabled\n")
 +              logger.Debug("Caching disabled")
        }
        return
  }
@@@ -446,9 -477,10 +477,10 @@@ func generateCrCsv(logger *logrus.Logge
                logger.Info(".")
                node, err := getNode(arv, ac, kc, cr2)
                if err != nil {
-                       return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+                       logger.Errorf("Skipping container request %s: error getting node %s: %s", cr2.UUID, cr2.UUID, err)
+                       continue
                }
 -              logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
 +              logger.Debug("\nChild container: " + cr2.ContainerUUID)
                var c2 arvados.Container
                err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
                if err != nil {