Merge branch '12913-secondary-mounts' refs #12913
authorPeter Amstutz <pamstutz@veritasgenetics.com>
Tue, 9 Jan 2018 14:50:17 +0000 (09:50 -0500)
committerPeter Amstutz <pamstutz@veritasgenetics.com>
Tue, 9 Jan 2018 14:50:21 +0000 (09:50 -0500)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

25 files changed:
build/run-build-packages-one-target.sh
build/run-build-packages.sh
build/run-tests.sh
cmd/arvados-client/.gitignore [new file with mode: 0644]
cmd/arvados-client/cmd.go [new file with mode: 0644]
cmd/arvados-client/cmd_test.go [new file with mode: 0644]
doc/install/crunch2-slurm/install-slurm.html.textile.liquid
lib/cli/external.go [new file with mode: 0644]
lib/cli/flags.go [new file with mode: 0644]
lib/cli/get.go [new file with mode: 0644]
lib/cli/get_test.go [new file with mode: 0644]
lib/cmd/cmd.go [new file with mode: 0644]
lib/cmd/cmd_test.go [new file with mode: 0644]
lib/cmdtest/leakcheck.go [new file with mode: 0644]
sdk/go/arvadostest/fixtures.go
services/arv-git-httpd/arvados-git-httpd.service
services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
services/dockercleaner/arvados-docker-cleaner.service
services/health/arvados-health.service
services/keep-balance/keep-balance.service
services/keep-web/keep-web.service
services/keepproxy/keepproxy.service
services/keepstore/keepstore.service
services/ws/arvados-ws.service
vendor/vendor.json

index 983c221f3b3b74f7235b04057cb3bd1bee1f9dba..c981b2a9ef679509bb24f7ecd25d005fb9d50920 100755 (executable)
@@ -161,6 +161,7 @@ popd
 
 if test -z "$packages" ; then
     packages="arvados-api-server
+        arvados-client
         arvados-docker-cleaner
         arvados-git-httpd
         arvados-node-manager
index 54f8b0aed17e0f49d49d3ea26a6f3d0e2238355a..c56b74088f070c3c252b7c9f84e92eb5e3fea78c 100755 (executable)
@@ -341,6 +341,8 @@ fi
 cd $WORKSPACE/packages/$TARGET
 export GOPATH=$(mktemp -d)
 go get github.com/kardianos/govendor
+package_go_binary cmd/arvados-client arvados-client \
+    "Arvados command line tool (beta)"
 package_go_binary sdk/go/crunchrunner crunchrunner \
     "Crunchrunner executes a command inside a container and uploads the output"
 package_go_binary services/arv-git-httpd arvados-git-httpd \
index f6c6e8f553fd93b0355cf9ce0224fc75495cc6d7..e831bf3503b8cb006d55b40e64af1c4a22d66dd9 100755 (executable)
@@ -69,7 +69,11 @@ apps/workbench_functionals (*)
 apps/workbench_integration (*)
 apps/workbench_benchmark
 apps/workbench_profile
+cmd/arvados-client
 doc
+lib/cli
+lib/cmd
+lib/crunchstat
 services/api
 services/arv-git-httpd
 services/crunchstat
@@ -474,8 +478,14 @@ export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
-rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-ln -snfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
+for d in \
+    "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+    "$GOPATH/src/git.curoverse.com/arvados.git"; do
+    [[ -d "$d" ]] && rmdir "$d"
+    [[ -h "$d" ]] && rm "$d"
+done
+ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 go get -v github.com/kardianos/govendor \
     || fatal "govendor install failed"
@@ -829,6 +839,10 @@ cd "$GOPATH/src/git.curoverse.com/arvados.git" && \
         fatal "govendor sync failed"
 declare -a gostuff
 gostuff=(
+    cmd/arvados-client
+    lib/cli
+    lib/cmd
+    lib/crunchstat
     sdk/go/arvados
     sdk/go/arvadosclient
     sdk/go/blockdigest
@@ -839,7 +853,6 @@ gostuff=(
     sdk/go/asyncbuf
     sdk/go/crunchrunner
     sdk/go/stats
-    lib/crunchstat
     services/arv-git-httpd
     services/crunchstat
     services/health
diff --git a/cmd/arvados-client/.gitignore b/cmd/arvados-client/.gitignore
new file mode 100644 (file)
index 0000000..21dd863
--- /dev/null
@@ -0,0 +1 @@
+arvados-*
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
new file mode 100644 (file)
index 0000000..b616b54
--- /dev/null
@@ -0,0 +1,79 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+       "fmt"
+       "io"
+       "os"
+       "regexp"
+       "runtime"
+
+       "git.curoverse.com/arvados.git/lib/cli"
+       "git.curoverse.com/arvados.git/lib/cmd"
+)
+
+var (
+       version                = "dev"
+       cmdVersion cmd.Handler = versionCmd{}
+       handler                = cmd.Multi(map[string]cmd.Handler{
+               "-e":        cmdVersion,
+               "version":   cmdVersion,
+               "-version":  cmdVersion,
+               "--version": cmdVersion,
+
+               "copy":     cli.Copy,
+               "create":   cli.Create,
+               "edit":     cli.Edit,
+               "get":      cli.Get,
+               "keep":     cli.Keep,
+               "pipeline": cli.Pipeline,
+               "run":      cli.Run,
+               "tag":      cli.Tag,
+               "ws":       cli.Ws,
+
+               "api_client_authorization": cli.APICall,
+               "api_client":               cli.APICall,
+               "authorized_key":           cli.APICall,
+               "collection":               cli.APICall,
+               "container":                cli.APICall,
+               "container_request":        cli.APICall,
+               "group":                    cli.APICall,
+               "human":                    cli.APICall,
+               "job":                      cli.APICall,
+               "job_task":                 cli.APICall,
+               "keep_disk":                cli.APICall,
+               "keep_service":             cli.APICall,
+               "link":                     cli.APICall,
+               "log":                      cli.APICall,
+               "node":                     cli.APICall,
+               "pipeline_instance":        cli.APICall,
+               "pipeline_template":        cli.APICall,
+               "repository":               cli.APICall,
+               "specimen":                 cli.APICall,
+               "trait":                    cli.APICall,
+               "user_agreement":           cli.APICall,
+               "user":                     cli.APICall,
+               "virtual_machine":          cli.APICall,
+               "workflow":                 cli.APICall,
+       })
+)
+
+type versionCmd struct{}
+
+func (versionCmd) RunCommand(prog string, args []string, _ io.Reader, stdout, _ io.Writer) int {
+       prog = regexp.MustCompile(` -*version$`).ReplaceAllLiteralString(prog, "")
+       fmt.Fprintf(stdout, "%s %s (%s)\n", prog, version, runtime.Version())
+       return 0
+}
+
+func fixLegacyArgs(args []string) []string {
+       flags, _ := cli.LegacyFlagSet()
+       return cmd.SubcommandToFront(args, flags)
+}
+
+func main() {
+       os.Exit(handler.RunCommand(os.Args[0], fixLegacyArgs(os.Args[1:]), os.Stdin, os.Stdout, os.Stderr))
+}
diff --git a/cmd/arvados-client/cmd_test.go b/cmd/arvados-client/cmd_test.go
new file mode 100644 (file)
index 0000000..cbbc7b1
--- /dev/null
@@ -0,0 +1,41 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+       "bytes"
+       "io/ioutil"
+       "testing"
+
+       check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+       check.TestingT(t)
+}
+
+var _ = check.Suite(&ClientSuite{})
+
+type ClientSuite struct{}
+
+func (s *ClientSuite) TestBadCommand(c *check.C) {
+       exited := handler.RunCommand("arvados-client", []string{"no such command"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
+       c.Check(exited, check.Equals, 2)
+}
+
+func (s *ClientSuite) TestBadSubcommandArgs(c *check.C) {
+       exited := handler.RunCommand("arvados-client", []string{"get"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
+       c.Check(exited, check.Equals, 2)
+}
+
+func (s *ClientSuite) TestVersion(c *check.C) {
+       stdout := bytes.NewBuffer(nil)
+       stderr := bytes.NewBuffer(nil)
+       exited := handler.RunCommand("arvados-client", []string{"version"}, bytes.NewReader(nil), stdout, stderr)
+       c.Check(exited, check.Equals, 0)
+       c.Check(stdout.String(), check.Matches, `arvados-client dev \(go[0-9\.]+\)\n`)
+       c.Check(stderr.String(), check.Equals, "")
+}
index 589da07fda1422d6e75047a88f3fd982df1eec52..c69d18b8e4bd2b0b8e3a19802982fdc284eb0e42 100644 (file)
@@ -85,6 +85,8 @@ Whenever you change this file, you will need to update the copy _on every comput
 
 *@ControlMachine@* should be a DNS name that resolves to the SLURM controller (dispatch/API server). This must resolve correctly on all SLURM worker nodes as well as the controller itself. In general SLURM is very sensitive about all of the nodes being able to communicate with the controller _and one another_, all using the same DNS names.
 
+*@SelectType=select/linear@* is needed on cloud-based installations that update node sizes dynamically, but it can only schedule one container at a time on each node. On a static or homogeneous cluster, use @SelectType=select/cons_res@ with @SelectTypeParameters=CR_CPU_Memory@ instead to enable node sharing.
+
 *@NodeName=compute[0-255]@* establishes that the hostnames of the worker nodes will be compute0, compute1, etc. through compute255.
 * There are several ways to compress sequences of names, like @compute[0-9,80,100-110]@. See the "hostlist" discussion in the @slurm.conf(5)@ and @scontrol(1)@ man pages for more information.
 * It is not necessary for all of the nodes listed here to be alive in order for SLURM to work, although you should make sure the DNS entries exist. It is easiest to define lots of hostnames up front, assigning them to real nodes and updating your DNS records as the nodes appear. This minimizes the frequency of @slurm.conf@ updates and use of @scontrol reconfigure@.
diff --git a/lib/cli/external.go b/lib/cli/external.go
new file mode 100644 (file)
index 0000000..35933f9
--- /dev/null
@@ -0,0 +1,124 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+       "fmt"
+       "io"
+       "io/ioutil"
+       "os/exec"
+       "strings"
+       "syscall"
+
+       "git.curoverse.com/arvados.git/lib/cmd"
+)
+
+var (
+       Create = rubyArvCmd{"create"}
+       Edit   = rubyArvCmd{"edit"}
+
+       Copy = externalCmd{"arv-copy"}
+       Tag  = externalCmd{"arv-tag"}
+       Ws   = externalCmd{"arv-ws"}
+       Run  = externalCmd{"arv-run"}
+
+       Keep = cmd.Multi(map[string]cmd.Handler{
+               "get":       externalCmd{"arv-get"},
+               "put":       externalCmd{"arv-put"},
+               "ls":        externalCmd{"arv-ls"},
+               "normalize": externalCmd{"arv-normalize"},
+               "docker":    externalCmd{"arv-keepdocker"},
+       })
+       Pipeline = cmd.Multi(map[string]cmd.Handler{
+               "run": externalCmd{"arv-run-pipeline-instance"},
+       })
+       // user, group, container, specimen, etc.
+       APICall = apiCallCmd{}
+)
+
+// When using the ruby "arv" command, flags must come before the
+// subcommand: "arv --format=yaml get foo" works, but "arv get
+// --format=yaml foo" does not work.
+func legacyFlagsToFront(subcommand string, argsin []string) (argsout []string) {
+       flags, _ := LegacyFlagSet()
+       flags.SetOutput(ioutil.Discard)
+       flags.Parse(argsin)
+       narg := flags.NArg()
+       argsout = append(argsout, argsin[:len(argsin)-narg]...)
+       argsout = append(argsout, subcommand)
+       argsout = append(argsout, argsin[len(argsin)-narg:]...)
+       return
+}
+
+type apiCallCmd struct{}
+
+func (cmd apiCallCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       split := strings.Split(prog, " ")
+       if len(split) < 2 {
+               fmt.Fprintf(stderr, "internal error: no api model in %q\n", prog)
+               return 2
+       }
+       model := split[len(split)-1]
+       return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(model, args), stdin, stdout, stderr)
+}
+
+type rubyArvCmd struct {
+       subcommand string
+}
+
+func (rc rubyArvCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr)
+}
+
+type externalCmd struct {
+       prog string
+}
+
+func (ec externalCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       cmd := exec.Command(ec.prog, args...)
+       cmd.Stdin = stdin
+       cmd.Stdout = stdout
+       cmd.Stderr = stderr
+       err := cmd.Run()
+       switch err := err.(type) {
+       case nil:
+               return 0
+       case *exec.ExitError:
+               status := err.Sys().(syscall.WaitStatus)
+               if status.Exited() {
+                       return status.ExitStatus()
+               }
+               fmt.Fprintf(stderr, "%s failed: %s\n", ec.prog, err)
+               return 1
+       case *exec.Error:
+               fmt.Fprintln(stderr, err)
+               if ec.prog == "arv" || ec.prog == "arv-run-pipeline-instance" {
+                       fmt.Fprint(stderr, rubyInstallHints)
+               } else if strings.HasPrefix(ec.prog, "arv-") {
+                       fmt.Fprint(stderr, pythonInstallHints)
+               }
+               return 1
+       default:
+               fmt.Fprintf(stderr, "error running %s: %s\n", ec.prog, err)
+               return 1
+       }
+}
+
+var (
+       rubyInstallHints = `
+Note: This subcommand uses the arvados-cli Ruby gem. If that is not
+installed, try "gem install arvados-cli", or see
+https://doc.arvados.org/install for more details.
+
+`
+       pythonInstallHints = `
+Note: This subcommand uses the "arvados" Python module. If that is
+not installed, try:
+* "pip install arvados" (either as root or in a virtualenv), or
+* "sudo apt-get install python-arvados-python-client", or
+* see https://doc.arvados.org/install for more details.
+
+`
+)
diff --git a/lib/cli/flags.go b/lib/cli/flags.go
new file mode 100644 (file)
index 0000000..7147e0c
--- /dev/null
@@ -0,0 +1,33 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+       "flag"
+
+       "git.curoverse.com/arvados.git/lib/cmd"
+       "rsc.io/getopt"
+)
+
+type LegacyFlagValues struct {
+       Format  string
+       DryRun  bool
+       Short   bool
+       Verbose bool
+}
+
+func LegacyFlagSet() (cmd.FlagSet, *LegacyFlagValues) {
+       values := &LegacyFlagValues{Format: "json"}
+       flags := getopt.NewFlagSet("", flag.ContinueOnError)
+       flags.BoolVar(&values.DryRun, "dry-run", false, "Don't actually do anything")
+       flags.Alias("n", "dry-run")
+       flags.StringVar(&values.Format, "format", values.Format, "Output format: json, yaml, or uuid")
+       flags.Alias("f", "format")
+       flags.BoolVar(&values.Short, "short", false, "Return only UUIDs (equivalent to --format=uuid)")
+       flags.Alias("s", "short")
+       flags.BoolVar(&values.Verbose, "verbose", false, "Print more debug/progress messages on stderr")
+       flags.Alias("v", "verbose")
+       return flags, values
+}
diff --git a/lib/cli/get.go b/lib/cli/get.go
new file mode 100644 (file)
index 0000000..2c60f43
--- /dev/null
@@ -0,0 +1,75 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+       "encoding/json"
+       "fmt"
+       "io"
+
+       "git.curoverse.com/arvados.git/lib/cmd"
+       "git.curoverse.com/arvados.git/sdk/go/arvados"
+       "github.com/ghodss/yaml"
+)
+
+var Get cmd.Handler = getCmd{}
+
+type getCmd struct{}
+
+func (getCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       var err error
+       defer func() {
+               if err != nil {
+                       fmt.Fprintf(stderr, "%s\n", err)
+               }
+       }()
+
+       flags, opts := LegacyFlagSet()
+       flags.SetOutput(stderr)
+       err = flags.Parse(args)
+       if err != nil {
+               return 2
+       }
+       if len(flags.Args()) != 1 {
+               fmt.Fprintf(stderr, "usage of %s:\n", prog)
+               flags.PrintDefaults()
+               return 2
+       }
+       if opts.Short {
+               opts.Format = "uuid"
+       }
+
+       id := flags.Args()[0]
+       client := arvados.NewClientFromEnv()
+       path, err := client.PathForUUID("show", id)
+       if err != nil {
+               return 1
+       }
+
+       var obj map[string]interface{}
+       err = client.RequestAndDecode(&obj, "GET", path, nil, nil)
+       if err != nil {
+               err = fmt.Errorf("GET %s: %s", path, err)
+               return 1
+       }
+       if opts.Format == "yaml" {
+               var buf []byte
+               buf, err = yaml.Marshal(obj)
+               if err == nil {
+                       _, err = stdout.Write(buf)
+               }
+       } else if opts.Format == "uuid" {
+               fmt.Fprintln(stdout, obj["uuid"])
+       } else {
+               enc := json.NewEncoder(stdout)
+               enc.SetIndent("", "  ")
+               err = enc.Encode(obj)
+       }
+       if err != nil {
+               err = fmt.Errorf("encoding: %s", err)
+               return 1
+       }
+       return 0
+}
diff --git a/lib/cli/get_test.go b/lib/cli/get_test.go
new file mode 100644 (file)
index 0000000..b2128a4
--- /dev/null
@@ -0,0 +1,33 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+       "bytes"
+       "regexp"
+       "testing"
+
+       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+       check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+       check.TestingT(t)
+}
+
+var _ = check.Suite(&GetSuite{})
+
+type GetSuite struct{}
+
+func (s *GetSuite) TestGetCollectionJSON(c *check.C) {
+       stdout := bytes.NewBuffer(nil)
+       stderr := bytes.NewBuffer(nil)
+       exited := Get.RunCommand("arvados-client get", []string{arvadostest.FooCollection}, bytes.NewReader(nil), stdout, stderr)
+       c.Check(stdout.String(), check.Matches, `(?ms){.*"uuid": "`+arvadostest.FooCollection+`".*}\n`)
+       c.Check(stdout.String(), check.Matches, `(?ms){.*"portable_data_hash": "`+regexp.QuoteMeta(arvadostest.FooCollectionPDH)+`".*}\n`)
+       c.Check(stderr.String(), check.Equals, "")
+       c.Check(exited, check.Equals, 0)
+}
diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go
new file mode 100644 (file)
index 0000000..2cc71e6
--- /dev/null
@@ -0,0 +1,116 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// package cmd helps define reusable functions that can be exposed as
+// [subcommands of] command line programs.
+package cmd
+
+import (
+       "flag"
+       "fmt"
+       "io"
+       "io/ioutil"
+       "sort"
+       "strings"
+)
+
+type Handler interface {
+       RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int
+}
+
+type HandlerFunc func(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int
+
+func (f HandlerFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       return f(prog, args, stdin, stdout, stderr)
+}
+
+// Multi is a Handler that looks up its first argument in a map, and
+// invokes the resulting Handler with the remaining args.
+//
+// Example:
+//
+//     os.Exit(Multi(map[string]Handler{
+//             "foobar": HandlerFunc(func(prog string, args []string) int {
+//                     fmt.Println(args[0])
+//                     return 2
+//             }),
+//     })("/usr/bin/multi", []string{"foobar", "baz"}))
+//
+// ...prints "baz" and exits 2.
+type Multi map[string]Handler
+
+func (m Multi) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       if len(args) < 1 {
+               fmt.Fprintf(stderr, "usage: %s command [args]\n", prog)
+               m.Usage(stderr)
+               return 2
+       }
+       if cmd, ok := m[args[0]]; !ok {
+               fmt.Fprintf(stderr, "unrecognized command %q\n", args[0])
+               m.Usage(stderr)
+               return 2
+       } else {
+               return cmd.RunCommand(prog+" "+args[0], args[1:], stdin, stdout, stderr)
+       }
+}
+
+func (m Multi) Usage(stderr io.Writer) {
+       fmt.Fprintf(stderr, "\nAvailable commands:\n")
+       m.listSubcommands(stderr, "")
+}
+
+func (m Multi) listSubcommands(out io.Writer, prefix string) {
+       var subcommands []string
+       for sc := range m {
+               if strings.HasPrefix(sc, "-") {
+                       // Some subcommands have alternate versions
+                       // like "--version" for compatibility. Don't
+                       // clutter the subcommand summary with those.
+                       continue
+               }
+               subcommands = append(subcommands, sc)
+       }
+       sort.Strings(subcommands)
+       for _, sc := range subcommands {
+               switch cmd := m[sc].(type) {
+               case Multi:
+                       cmd.listSubcommands(out, prefix+sc+" ")
+               default:
+                       fmt.Fprintf(out, "    %s%s\n", prefix, sc)
+               }
+       }
+}
+
+type FlagSet interface {
+       Init(string, flag.ErrorHandling)
+       Args() []string
+       NArg() int
+       Parse([]string) error
+       SetOutput(io.Writer)
+       PrintDefaults()
+}
+
+// SubcommandToFront silently parses args using flagset, and returns a
+// copy of args with the first non-flag argument moved to the
+// front. If parsing fails or consumes all of args, args is returned
+// unchanged.
+//
+// SubcommandToFront invokes methods on flagset that have side
+// effects, including Parse. In typical usage, flagset will not used
+// for anything else after being passed to SubcommandToFront.
+func SubcommandToFront(args []string, flagset FlagSet) []string {
+       flagset.Init("", flag.ContinueOnError)
+       flagset.SetOutput(ioutil.Discard)
+       if err := flagset.Parse(args); err != nil || flagset.NArg() == 0 {
+               // No subcommand found.
+               return args
+       }
+       // Move subcommand to the front.
+       flagargs := len(args) - flagset.NArg()
+       newargs := make([]string, len(args))
+       newargs[0] = args[flagargs]
+       copy(newargs[1:flagargs+1], args[:flagargs])
+       copy(newargs[flagargs+1:], args[flagargs+1:])
+       return newargs
+}
diff --git a/lib/cmd/cmd_test.go b/lib/cmd/cmd_test.go
new file mode 100644 (file)
index 0000000..d8a4861
--- /dev/null
@@ -0,0 +1,62 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+       "bytes"
+       "flag"
+       "fmt"
+       "io"
+       "strings"
+       "testing"
+
+       "git.curoverse.com/arvados.git/lib/cmdtest"
+       check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+       check.TestingT(t)
+}
+
+var _ = check.Suite(&CmdSuite{})
+
+type CmdSuite struct{}
+
+var testCmd = Multi(map[string]Handler{
+       "echo": HandlerFunc(func(prog string, args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int {
+               fmt.Fprintln(stdout, strings.Join(args, " "))
+               return 0
+       }),
+})
+
+func (s *CmdSuite) TestHello(c *check.C) {
+       defer cmdtest.LeakCheck(c)()
+       stdout := bytes.NewBuffer(nil)
+       stderr := bytes.NewBuffer(nil)
+       exited := testCmd.RunCommand("prog", []string{"echo", "hello", "world"}, bytes.NewReader(nil), stdout, stderr)
+       c.Check(exited, check.Equals, 0)
+       c.Check(stdout.String(), check.Equals, "hello world\n")
+       c.Check(stderr.String(), check.Equals, "")
+}
+
+func (s *CmdSuite) TestUsage(c *check.C) {
+       defer cmdtest.LeakCheck(c)()
+       stdout := bytes.NewBuffer(nil)
+       stderr := bytes.NewBuffer(nil)
+       exited := testCmd.RunCommand("prog", []string{"nosuchcommand", "hi"}, bytes.NewReader(nil), stdout, stderr)
+       c.Check(exited, check.Equals, 2)
+       c.Check(stdout.String(), check.Equals, "")
+       c.Check(stderr.String(), check.Matches, `(?ms)^unrecognized command "nosuchcommand"\n.*echo.*\n`)
+}
+
+func (s *CmdSuite) TestSubcommandToFront(c *check.C) {
+       defer cmdtest.LeakCheck(c)()
+       flags := flag.NewFlagSet("", flag.ContinueOnError)
+       flags.String("format", "json", "")
+       flags.Bool("n", false, "")
+       args := SubcommandToFront([]string{"--format=yaml", "-n", "-format", "beep", "echo", "hi"}, flags)
+       c.Check(args, check.DeepEquals, []string{"echo", "--format=yaml", "-n", "-format", "beep", "hi"})
+}
diff --git a/lib/cmdtest/leakcheck.go b/lib/cmdtest/leakcheck.go
new file mode 100644 (file)
index 0000000..c132f1b
--- /dev/null
@@ -0,0 +1,54 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// Package cmdtest provides tools for testing command line tools.
+package cmdtest
+
+import (
+       "io"
+       "io/ioutil"
+       "os"
+
+       check "gopkg.in/check.v1"
+)
+
+// LeakCheck tests for output being leaked to os.Stdout and os.Stderr
+// that should be sent elsewhere (e.g., the stdout and stderr streams
+// passed to a cmd.RunFunc).
+//
+// It redirects os.Stdout and os.Stderr to a tempfile, and returns a
+// func, which the caller is expected to defer, that restores os.* and
+// checks that the tempfile is empty.
+//
+// Example:
+//
+//     func (s *Suite) TestSomething(c *check.C) {
+//             defer cmdtest.LeakCheck(c)()
+//             // ... do things that shouldn't print to os.Stderr or os.Stdout
+//     }
+func LeakCheck(c *check.C) func() {
+       tmpfiles := map[string]*os.File{"stdout": nil, "stderr": nil}
+       for i := range tmpfiles {
+               var err error
+               tmpfiles[i], err = ioutil.TempFile("", "")
+               c.Assert(err, check.IsNil)
+               err = os.Remove(tmpfiles[i].Name())
+               c.Assert(err, check.IsNil)
+       }
+
+       stdout, stderr := os.Stdout, os.Stderr
+       os.Stdout, os.Stderr = tmpfiles["stdout"], tmpfiles["stderr"]
+       return func() {
+               os.Stdout, os.Stderr = stdout, stderr
+
+               for i, tmpfile := range tmpfiles {
+                       c.Log("checking %s", i)
+                       _, err := tmpfile.Seek(0, io.SeekStart)
+                       c.Assert(err, check.IsNil)
+                       leaked, err := ioutil.ReadAll(tmpfile)
+                       c.Assert(err, check.IsNil)
+                       c.Check(string(leaked), check.Equals, "")
+               }
+       }
+}
index 3a611a3bfb14359cca0ab890f47da13314d16a9b..5e530658abe96534d9ed3488a975a85a778af16d 100644 (file)
@@ -16,6 +16,7 @@ const (
        SpectatorUserUUID       = "zzzzz-tpzed-l1s2piq4t4mps8r"
        UserAgreementCollection = "zzzzz-4zz18-uukreo9rbgwsujr" // user_agreement_in_anonymously_accessible_project
        FooCollection           = "zzzzz-4zz18-fy296fx3hot09f7"
+       FooCollectionPDH        = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
        NonexistentCollection   = "zzzzz-4zz18-totallynotexist"
        HelloWorldCollection    = "zzzzz-4zz18-4en62shvi99lxd4"
        FooBarDirCollection     = "zzzzz-4zz18-foonbarfilesdir"
index 300b5c1d12287d9e0c771e89f01eb9b4e92c6f5f..6f8cca856b595498b7f6b8ee387e956e838666a2 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados git server
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/git-httpd/git-httpd.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/arvados-git-httpd
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index ac842433d56ea8f379d6e44c4c04fb9d97e1fd7d..508a0dfc1387bafcca41b6b8d851f0960d3fc9b9 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados Crunch Dispatcher for SLURM
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/crunch-dispatch-slurm
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index 6ab3fdfac91ae05e183643b010ca6d093a4b2f3c..b48b46bb9a4c48b52770996fba12821c60e5cb8e 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados Docker Image Cleaner
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/docker-cleaner/docker-cleaner.json
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -23,5 +25,8 @@ RestartPreventExitStatus=2
 # Collection, and if so, invokes it with the "scl" wrapper.
 ExecStart=/bin/sh -c 'if [ -e /opt/rh/python33/root/bin/arvados-docker-cleaner ]; then exec scl enable python33 arvados-docker-cleaner; else exec arvados-docker-cleaner; fi'
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index a3d036549c85027a96b6ffaf7a8b15358d1672d3..dac7c3a1949c9793cf0aaa2cbcf6b297d4a5a241 100644 (file)
@@ -7,16 +7,21 @@ Description=Arvados healthcheck server
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/config.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
 Type=simple
 ExecStart=/usr/bin/arvados-health
 Restart=always
-RestartSec=1s
+RestartSec=1
+
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
 
 [Install]
 WantedBy=multi-user.target
index df08b5be7f2e3f75414d8b2fa8f46bb08bb4c844..325affe5875108a819b3baa07aa964bcd5ef1224 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados Keep Balance
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
 Restart=always
 RestartSec=10s
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index a8f8f1ffa1336329d7ff0f0c6ad76daeb65ddb0d..1931256209eb4f211d031114c8297fb0ee53d01f 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados Keep web gateway
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keep-web/keep-web.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keep-web
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index d7ee97e96325ba278ec75eba6e9f80dedf41f5a3..96dec25ecf77ecc8c3936628829d8dc59aecabc4 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados Keep Proxy
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keepproxy/keepproxy.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keepproxy
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index 14572ae905a1d257b08e404f0dbcde7b3b88715d..8b448e72c3b41687f46dc2c6a6ad8a0c21202258 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados Keep Storage Daemon
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keepstore/keepstore.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keepstore
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index 4385261181922e0edc260e680a9678d7071651dc..9e02d41ca8bb6212684a3fd33b228b7fbffe390f 100644 (file)
@@ -7,9 +7,11 @@ Description=Arvados websocket server
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/ws/ws.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/arvados-ws
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target
index 8551a06fb414573e90d2a802c2806c99e8bf0bd5..9798e86c81a65d5dee61d2a103c925e0e68067e8 100644 (file)
                        "revision": "4c1c3952b7d9d0a061a3fa7b36fd373ba0398ebc",
                        "revisionTime": "2017-04-27T04:12:50Z"
                },
+               {
+                       "checksumSHA1": "iCsyavJDnXC9OY//p52IWJWy7PY=",
+                       "path": "github.com/jbenet/go-context/io",
+                       "revision": "d14ea06fba99483203c19d92cfcd13ebe73135f4",
+                       "revisionTime": "2015-07-11T00:45:18Z"
+               },
                {
                        "checksumSHA1": "uTUsjF7bymOuKvXbW2BpkK/w4Vg=",
                        "path": "github.com/lib/pq",
                        "revision": "2704adc878c21e1329f46f6e56a1c387d788ff94",
                        "revisionTime": "2017-03-24T20:46:54Z"
                },
+               {
+                       "checksumSHA1": "V/quM7+em2ByJbWBLOsEwnY3j/Q=",
+                       "path": "github.com/mitchellh/go-homedir",
+                       "revision": "b8bc1bf767474819792c23f32d8286a45736f1c6",
+                       "revisionTime": "2016-12-03T19:45:07Z"
+               },
                {
                        "checksumSHA1": "OUupkCHnh8e3RHRNcMFqQJqhaQI=",
                        "origin": "github.com/docker/docker/vendor/github.com/opencontainers/go-digest",
                        "revisionTime": "2017-05-17T20:48:28Z"
                },
                {
-                       "checksumSHA1": "9jjO5GjLa0XF/nfWihF02RoH4qc=",
+                       "checksumSHA1": "UwtyqB7CaUWPlw0DVJQvw0IFQZs=",
+                       "path": "github.com/sergi/go-diff/diffmatchpatch",
+                       "revision": "1744e2970ca51c86172c8190fadad617561ed6e7",
+                       "revisionTime": "2017-11-10T11:01:46Z"
+               },
+               {
+                       "checksumSHA1": "8QeSG127zQqbA+YfkO1WkKx/iUI=",
+                       "path": "github.com/src-d/gcfg",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "yf5NBT8BofPfGYCXoLnj7BIA1wo=",
+                       "path": "github.com/src-d/gcfg/scanner",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "C5Z8YVyNTuvupM9AUr9KbPlps4Q=",
+                       "path": "github.com/src-d/gcfg/token",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "mDkN3UpR7auuFbwUuIwExz4DZgY=",
+                       "path": "github.com/src-d/gcfg/types",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "iHiMTBffQvWYlOLu3130JXuQpgQ=",
+                       "path": "github.com/xanzy/ssh-agent",
+                       "revision": "ba9c9e33906f58169366275e3450db66139a31a9",
+                       "revisionTime": "2015-12-15T15:34:51Z"
+               },
+               {
+                       "checksumSHA1": "IQkUIOnvlf0tYloFx9mLaXSvXWQ=",
+                       "path": "golang.org/x/crypto/curve25519",
+                       "revision": "94eea52f7b742c7cbe0b03b22f0c4c8631ece122",
+                       "revisionTime": "2017-11-28T01:43:40Z"
+               },
+               {
+                       "checksumSHA1": "1hwn8cgg4EVXhCpJIqmMbzqnUo0=",
+                       "path": "golang.org/x/crypto/ed25519",
+                       "revision": "94eea52f7b742c7cbe0b03b22f0c4c8631ece122",
+                       "revisionTime": "2017-11-28T01:43:40Z"
+               },
+               {
+                       "checksumSHA1": "LXFcVx8I587SnWmKycSDEq9yvK8=",
+                       "path": "golang.org/x/crypto/ed25519/internal/edwards25519",
+                       "revision": "94eea52f7b742c7cbe0b03b22f0c4c8631ece122",
+                       "revisionTime": "2017-11-28T01:43:40Z"
+               },
+               {
+                       "checksumSHA1": "R9/BEjpj6VT5IZN6EDPs4gjMJz0=",
+                       "path": "golang.org/x/crypto/ssh",
+                       "revision": "94eea52f7b742c7cbe0b03b22f0c4c8631ece122",
+                       "revisionTime": "2017-11-28T01:43:40Z"
+               },
+               {
+                       "checksumSHA1": "NMRX0onGReaL9IfLr0XQ3kl5Id0=",
+                       "path": "golang.org/x/crypto/ssh/agent",
+                       "revision": "94eea52f7b742c7cbe0b03b22f0c4c8631ece122",
+                       "revisionTime": "2017-11-28T01:43:40Z"
+               },
+               {
+                       "checksumSHA1": "zBHtHvMj+MXa1qa4aglBt46uUck=",
+                       "path": "golang.org/x/crypto/ssh/knownhosts",
+                       "revision": "94eea52f7b742c7cbe0b03b22f0c4c8631ece122",
+                       "revisionTime": "2017-11-28T01:43:40Z"
+               },
+               {
+                       "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=",
                        "origin": "github.com/docker/docker/vendor/golang.org/x/net/context",
                        "path": "golang.org/x/net/context",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "d94164867c2df977ceb8d9df3d90364d0a682d05",
+                       "revisionTime": "2017-12-04T17:54:02Z"
                },
                {
                        "checksumSHA1": "WHc3uByvGaMcnSoI21fhzYgbOgg=",
                        "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
                        "revisionTime": "2017-05-17T20:48:28Z"
                },
+               {
+                       "checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=",
+                       "path": "golang.org/x/text/transform",
+                       "revision": "57961680700a5336d15015c8c50686ca5ba362a4",
+                       "revisionTime": "2017-11-28T12:17:19Z"
+               },
+               {
+                       "checksumSHA1": "BwRNKgzIMUxk56OScxyr43BV6IE=",
+                       "path": "golang.org/x/text/unicode/norm",
+                       "revision": "57961680700a5336d15015c8c50686ca5ba362a4",
+                       "revisionTime": "2017-11-28T12:17:19Z"
+               },
                {
                        "checksumSHA1": "CEFTYXtWmgSh+3Ik1NmDaJcz4E0=",
                        "path": "gopkg.in/check.v1",
                        "revision": "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec",
                        "revisionTime": "2016-12-08T18:13:25Z"
                },
+               {
+                       "checksumSHA1": "OMS7bwaAYpmocTLflRC8ctukGo8=",
+                       "path": "gopkg.in/src-d/go-billy.v3",
+                       "revision": "c329b7bc7b9d24905d2bc1b85bfa29f7ae266314",
+                       "revisionTime": "2017-06-27T10:15:46Z"
+               },
+               {
+                       "checksumSHA1": "BfxlZXUAo+4gqICRNkn1fbJu/Jw=",
+                       "path": "gopkg.in/src-d/go-billy.v3/helper/chroot",
+                       "revision": "c329b7bc7b9d24905d2bc1b85bfa29f7ae266314",
+                       "revisionTime": "2017-06-27T10:15:46Z"
+               },
+               {
+                       "checksumSHA1": "PwXEhy280k1rj6obYt+/ZfZ5SzI=",
+                       "path": "gopkg.in/src-d/go-billy.v3/helper/polyfill",
+                       "revision": "c329b7bc7b9d24905d2bc1b85bfa29f7ae266314",
+                       "revisionTime": "2017-06-27T10:15:46Z"
+               },
+               {
+                       "checksumSHA1": "5QGZZIfRAeRt5guZzx6UA52uwEk=",
+                       "path": "gopkg.in/src-d/go-billy.v3/osfs",
+                       "revision": "c329b7bc7b9d24905d2bc1b85bfa29f7ae266314",
+                       "revisionTime": "2017-06-27T10:15:46Z"
+               },
+               {
+                       "checksumSHA1": "gHdA6TMpLCZhKio7ltM7H4eDX2k=",
+                       "path": "gopkg.in/src-d/go-billy.v3/util",
+                       "revision": "c329b7bc7b9d24905d2bc1b85bfa29f7ae266314",
+                       "revisionTime": "2017-06-27T10:15:46Z"
+               },
+               {
+                       "checksumSHA1": "E0ZxsMpf8GRZTfMzens8FK/Odvs=",
+                       "path": "gopkg.in/src-d/go-git.v4",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "1d4PgjU+LhyJ2ROnz/WwrEKVh28=",
+                       "path": "gopkg.in/src-d/go-git.v4/config",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "A3WduoxOIVoBnsDAEZtI798CZtU=",
+                       "path": "gopkg.in/src-d/go-git.v4/internal/revision",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "Re676e7BhdKLVVNfvNhnbiec2JM=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "neTbLTzQ9+vo97D5i+3K9iprn5c=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/cache",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "pHPMiAzXG/TJqTLEKj2SHjxX4zs=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/filemode",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "RzTdA6jLPwD/m6mq+rgS2CB04d4=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/config",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "vOD599V5I9EsWuGT9BIU8ZAyiNY=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/diff",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "mI7Ks4Bumh+OYkxuSBrIeBTAKoA=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/gitignore",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "G0TX3efLdk7noo/n1Dt9Tzempig=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "naVExGq1c3bXZ4+Em3slvB7I8so=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/index",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "RYeQffgqVS4Kbbk2YVcaPadXCiI=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/objfile",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "q22wOnNvgQNTuWAkcYKa1xVk7og=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/packfile",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "rA6jJ2fxdcALXL7EaP8Ja37xXjw=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/pktline",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "6LZ2gIv993WaeqA2QMMvf04BRzk=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/object",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "sBjAjpwQtYwh1xgCC/Np6k1QCxA=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "+iFHG0LBT3gYImczKZy9Gkjogpk=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "wVfbzV5BNhjW/HFFJuTCjkPSJ5M=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "xnSL7xqAg9rUrqzrFy2OtOkfBHY=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/revlist",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "QL14x3W+ObgTwz2wRXyMhxGM9jM=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/storer",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "kKJbFD1KBIE37kZACAzrDdwwzlw=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "aReXFIha6HkU5jPfLWWAEMPhEp4=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/client",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "KLaUkXK0IPfAwIyC9WuzgpKl4Ts=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/file",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "4B79ZyIoeNpT4OWl/CvEQn9RP+g=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/git",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "0H7p/EuPC+LQdRWLoNO775/JIPw=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/http",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "1P0AgwgfasGL7aL5+FuuDan835c=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "WqaZfy8UDC25vPX8DJEUZH9urhw=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/server",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "NY+2qZNBynx/7d7vm20G7nU4LGk=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "FlVLBdu4cjlXj9zjRRNDurRLABU=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "FIbvDZtO3BdDAU+6Ogi1iCqj0cA=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage/filesystem",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "pPcJtH7jHTjiwhlyGDEr6dMOHiM=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "OfzMDAIu253dJ9591gd3w/APq0I=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage/memory",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "AzdUpuGqSNnNK6DgdNjWrn99i3o=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/binary",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "vniUxB6bbDYazl21cOfmhdZZiY8=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/diff",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "UM8j6MDPfIvBJOYrXWYFpN6nwk8=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/ioutil",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "6gGibezR20asX5JgNsGR9AWiF1s=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "EkYWmjvMAaEG9JPYtwE6x7hwxjY=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/filesystem",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "M+6y9mdBFksksEGBceBh9Se3W7Y=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/index",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "7eEw/xsSrFLfSppRf/JIt9u7lbU=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/internal/frame",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "0H0x2urvYdo68NY6fGFJG59lqoI=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder",
+                       "revision": "f9879dd043f84936a1f8acb8a53b74332a7ae135",
+                       "revisionTime": "2017-09-04T17:43:36Z"
+               },
+               {
+                       "checksumSHA1": "I4c3qsEX8KAUTeB9+2pwVX/2ojU=",
+                       "path": "gopkg.in/warnings.v0",
+                       "revision": "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b",
+                       "revisionTime": "2017-11-15T19:30:34Z"
+               },
                {
                        "checksumSHA1": "fALlQNY1fM99NesfLJ50KguWsio=",
                        "path": "gopkg.in/yaml.v2",
                        "revision": "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b",
                        "revisionTime": "2017-04-07T17:21:22Z"
+               },
+               {
+                       "checksumSHA1": "rBIcwbUjE9w1aV0qh7lAL1hcxCQ=",
+                       "path": "rsc.io/getopt",
+                       "revision": "20be20937449f18bb9967c10d732849fb4401e63",
+                       "revisionTime": "2017-08-11T00:05:52Z"
                }
        ],
        "rootPath": "git.curoverse.com/arvados.git"