if test -z "$packages" ; then
packages="arvados-api-server
+ arvados-client
arvados-docker-cleaner
arvados-git-httpd
arvados-node-manager
cd $WORKSPACE/packages/$TARGET
export GOPATH=$(mktemp -d)
go get github.com/kardianos/govendor
+package_go_binary cmd/arvados-client arvados-client \
+ "Arvados command line tool (beta)"
package_go_binary sdk/go/crunchrunner crunchrunner \
"Crunchrunner executes a command inside a container and uploads the output"
package_go_binary services/arv-git-httpd arvados-git-httpd \
apps/workbench_integration (*)
apps/workbench_benchmark
apps/workbench_profile
+cmd/arvados-client
doc
+lib/cli
+lib/cmd
+lib/crunchstat
services/api
services/arv-git-httpd
services/crunchstat
export GOPATH
mkdir -p "$GOPATH/src/git.curoverse.com"
-rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-ln -snfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
+for d in \
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+ "$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -d "$d" ]] && rmdir "$d"
+ [[ -h "$d" ]] && rm "$d"
+done
+ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
|| fatal "symlink failed"
go get -v github.com/kardianos/govendor \
|| fatal "govendor install failed"
fatal "govendor sync failed"
declare -a gostuff
gostuff=(
+ cmd/arvados-client
+ lib/cli
+ lib/cmd
+ lib/crunchstat
sdk/go/arvados
sdk/go/arvadosclient
sdk/go/blockdigest
sdk/go/asyncbuf
sdk/go/crunchrunner
sdk/go/stats
- lib/crunchstat
services/arv-git-httpd
services/crunchstat
services/health
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "regexp"
+ "runtime"
+
+ "git.curoverse.com/arvados.git/lib/cli"
+ "git.curoverse.com/arvados.git/lib/cmd"
+)
+
+var (
+ version = "dev"
+ cmdVersion cmd.Handler = versionCmd{}
+ handler = cmd.Multi(map[string]cmd.Handler{
+ "-e": cmdVersion,
+ "version": cmdVersion,
+ "-version": cmdVersion,
+ "--version": cmdVersion,
+
+ "copy": cli.Copy,
+ "create": cli.Create,
+ "edit": cli.Edit,
+ "get": cli.Get,
+ "keep": cli.Keep,
+ "pipeline": cli.Pipeline,
+ "run": cli.Run,
+ "tag": cli.Tag,
+ "ws": cli.Ws,
+
+ "api_client_authorization": cli.APICall,
+ "api_client": cli.APICall,
+ "authorized_key": cli.APICall,
+ "collection": cli.APICall,
+ "container": cli.APICall,
+ "container_request": cli.APICall,
+ "group": cli.APICall,
+ "human": cli.APICall,
+ "job": cli.APICall,
+ "job_task": cli.APICall,
+ "keep_disk": cli.APICall,
+ "keep_service": cli.APICall,
+ "link": cli.APICall,
+ "log": cli.APICall,
+ "node": cli.APICall,
+ "pipeline_instance": cli.APICall,
+ "pipeline_template": cli.APICall,
+ "repository": cli.APICall,
+ "specimen": cli.APICall,
+ "trait": cli.APICall,
+ "user_agreement": cli.APICall,
+ "user": cli.APICall,
+ "virtual_machine": cli.APICall,
+ "workflow": cli.APICall,
+ })
+)
+
+type versionCmd struct{}
+
+func (versionCmd) RunCommand(prog string, args []string, _ io.Reader, stdout, _ io.Writer) int {
+ prog = regexp.MustCompile(` -*version$`).ReplaceAllLiteralString(prog, "")
+ fmt.Fprintf(stdout, "%s %s (%s)\n", prog, version, runtime.Version())
+ return 0
+}
+
+func fixLegacyArgs(args []string) []string {
+ flags, _ := cli.LegacyFlagSet()
+ return cmd.SubcommandToFront(args, flags)
+}
+
+func main() {
+ os.Exit(handler.RunCommand(os.Args[0], fixLegacyArgs(os.Args[1:]), os.Stdin, os.Stdout, os.Stderr))
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "testing"
+
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&ClientSuite{})
+
+type ClientSuite struct{}
+
+func (s *ClientSuite) TestBadCommand(c *check.C) {
+ exited := handler.RunCommand("arvados-client", []string{"no such command"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
+ c.Check(exited, check.Equals, 2)
+}
+
+func (s *ClientSuite) TestBadSubcommandArgs(c *check.C) {
+ exited := handler.RunCommand("arvados-client", []string{"get"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
+ c.Check(exited, check.Equals, 2)
+}
+
+func (s *ClientSuite) TestVersion(c *check.C) {
+ stdout := bytes.NewBuffer(nil)
+ stderr := bytes.NewBuffer(nil)
+ exited := handler.RunCommand("arvados-client", []string{"version"}, bytes.NewReader(nil), stdout, stderr)
+ c.Check(exited, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, `arvados-client dev \(go[0-9\.]+\)\n`)
+ c.Check(stderr.String(), check.Equals, "")
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os/exec"
+ "strings"
+ "syscall"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+)
+
+var (
+ Create = rubyArvCmd{"create"}
+ Edit = rubyArvCmd{"edit"}
+
+ Copy = externalCmd{"arv-copy"}
+ Tag = externalCmd{"arv-tag"}
+ Ws = externalCmd{"arv-ws"}
+ Run = externalCmd{"arv-run"}
+
+ Keep = cmd.Multi(map[string]cmd.Handler{
+ "get": externalCmd{"arv-get"},
+ "put": externalCmd{"arv-put"},
+ "ls": externalCmd{"arv-ls"},
+ "normalize": externalCmd{"arv-normalize"},
+ "docker": externalCmd{"arv-keepdocker"},
+ })
+ Pipeline = cmd.Multi(map[string]cmd.Handler{
+ "run": externalCmd{"arv-run-pipeline-instance"},
+ })
+ // user, group, container, specimen, etc.
+ APICall = apiCallCmd{}
+)
+
+// When using the ruby "arv" command, flags must come before the
+// subcommand: "arv --format=yaml get foo" works, but "arv get
+// --format=yaml foo" does not work.
+func legacyFlagsToFront(subcommand string, argsin []string) (argsout []string) {
+ flags, _ := LegacyFlagSet()
+ flags.SetOutput(ioutil.Discard)
+ flags.Parse(argsin)
+ narg := flags.NArg()
+ argsout = append(argsout, argsin[:len(argsin)-narg]...)
+ argsout = append(argsout, subcommand)
+ argsout = append(argsout, argsin[len(argsin)-narg:]...)
+ return
+}
+
+type apiCallCmd struct{}
+
+func (cmd apiCallCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ split := strings.Split(prog, " ")
+ if len(split) < 2 {
+ fmt.Fprintf(stderr, "internal error: no api model in %q\n", prog)
+ return 2
+ }
+ model := split[len(split)-1]
+ return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(model, args), stdin, stdout, stderr)
+}
+
+type rubyArvCmd struct {
+ subcommand string
+}
+
+func (rc rubyArvCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr)
+}
+
+type externalCmd struct {
+ prog string
+}
+
+func (ec externalCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ cmd := exec.Command(ec.prog, args...)
+ cmd.Stdin = stdin
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ switch err := err.(type) {
+ case nil:
+ return 0
+ case *exec.ExitError:
+ status := err.Sys().(syscall.WaitStatus)
+ if status.Exited() {
+ return status.ExitStatus()
+ }
+ fmt.Fprintf(stderr, "%s failed: %s\n", ec.prog, err)
+ return 1
+ case *exec.Error:
+ fmt.Fprintln(stderr, err)
+ if ec.prog == "arv" || ec.prog == "arv-run-pipeline-instance" {
+ fmt.Fprint(stderr, rubyInstallHints)
+ } else if strings.HasPrefix(ec.prog, "arv-") {
+ fmt.Fprint(stderr, pythonInstallHints)
+ }
+ return 1
+ default:
+ fmt.Fprintf(stderr, "error running %s: %s\n", ec.prog, err)
+ return 1
+ }
+}
+
+var (
+ rubyInstallHints = `
+Note: This subcommand uses the arvados-cli Ruby gem. If that is not
+installed, try "gem install arvados-cli", or see
+https://doc.arvados.org/install for more details.
+
+`
+ pythonInstallHints = `
+Note: This subcommand uses the "arvados" Python module. If that is
+not installed, try:
+* "pip install arvados" (either as root or in a virtualenv), or
+* "sudo apt-get install python-arvados-python-client", or
+* see https://doc.arvados.org/install for more details.
+
+`
+)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+ "flag"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "rsc.io/getopt"
+)
+
+type LegacyFlagValues struct {
+ Format string
+ DryRun bool
+ Short bool
+ Verbose bool
+}
+
+func LegacyFlagSet() (cmd.FlagSet, *LegacyFlagValues) {
+ values := &LegacyFlagValues{Format: "json"}
+ flags := getopt.NewFlagSet("", flag.ContinueOnError)
+ flags.BoolVar(&values.DryRun, "dry-run", false, "Don't actually do anything")
+ flags.Alias("n", "dry-run")
+ flags.StringVar(&values.Format, "format", values.Format, "Output format: json, yaml, or uuid")
+ flags.Alias("f", "format")
+ flags.BoolVar(&values.Short, "short", false, "Return only UUIDs (equivalent to --format=uuid)")
+ flags.Alias("s", "short")
+ flags.BoolVar(&values.Verbose, "verbose", false, "Print more debug/progress messages on stderr")
+ flags.Alias("v", "verbose")
+ return flags, values
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+
+ "git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "github.com/ghodss/yaml"
+)
+
+var Get cmd.Handler = getCmd{}
+
+type getCmd struct{}
+
+func (getCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ defer func() {
+ if err != nil {
+ fmt.Fprintf(stderr, "%s\n", err)
+ }
+ }()
+
+ flags, opts := LegacyFlagSet()
+ flags.SetOutput(stderr)
+ err = flags.Parse(args)
+ if err != nil {
+ return 2
+ }
+ if len(flags.Args()) != 1 {
+ fmt.Fprintf(stderr, "usage of %s:\n", prog)
+ flags.PrintDefaults()
+ return 2
+ }
+ if opts.Short {
+ opts.Format = "uuid"
+ }
+
+ id := flags.Args()[0]
+ client := arvados.NewClientFromEnv()
+ path, err := client.PathForUUID("show", id)
+ if err != nil {
+ return 1
+ }
+
+ var obj map[string]interface{}
+ err = client.RequestAndDecode(&obj, "GET", path, nil, nil)
+ if err != nil {
+ err = fmt.Errorf("GET %s: %s", path, err)
+ return 1
+ }
+ if opts.Format == "yaml" {
+ var buf []byte
+ buf, err = yaml.Marshal(obj)
+ if err == nil {
+ _, err = stdout.Write(buf)
+ }
+ } else if opts.Format == "uuid" {
+ fmt.Fprintln(stdout, obj["uuid"])
+ } else {
+ enc := json.NewEncoder(stdout)
+ enc.SetIndent("", " ")
+ err = enc.Encode(obj)
+ }
+ if err != nil {
+ err = fmt.Errorf("encoding: %s", err)
+ return 1
+ }
+ return 0
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cli
+
+import (
+ "bytes"
+ "regexp"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&GetSuite{})
+
+type GetSuite struct{}
+
+func (s *GetSuite) TestGetCollectionJSON(c *check.C) {
+ stdout := bytes.NewBuffer(nil)
+ stderr := bytes.NewBuffer(nil)
+ exited := Get.RunCommand("arvados-client get", []string{arvadostest.FooCollection}, bytes.NewReader(nil), stdout, stderr)
+ c.Check(stdout.String(), check.Matches, `(?ms){.*"uuid": "`+arvadostest.FooCollection+`".*}\n`)
+ c.Check(stdout.String(), check.Matches, `(?ms){.*"portable_data_hash": "`+regexp.QuoteMeta(arvadostest.FooCollectionPDH)+`".*}\n`)
+ c.Check(stderr.String(), check.Equals, "")
+ c.Check(exited, check.Equals, 0)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// package cmd helps define reusable functions that can be exposed as
+// [subcommands of] command line programs.
+package cmd
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "sort"
+ "strings"
+)
+
+type Handler interface {
+ RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int
+}
+
+type HandlerFunc func(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int
+
+func (f HandlerFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ return f(prog, args, stdin, stdout, stderr)
+}
+
+// Multi is a Handler that looks up its first argument in a map, and
+// invokes the resulting Handler with the remaining args.
+//
+// Example:
+//
+// os.Exit(Multi(map[string]Handler{
+// "foobar": HandlerFunc(func(prog string, args []string) int {
+// fmt.Println(args[0])
+// return 2
+// }),
+// })("/usr/bin/multi", []string{"foobar", "baz"}))
+//
+// ...prints "baz" and exits 2.
+type Multi map[string]Handler
+
+func (m Multi) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ if len(args) < 1 {
+ fmt.Fprintf(stderr, "usage: %s command [args]\n", prog)
+ m.Usage(stderr)
+ return 2
+ }
+ if cmd, ok := m[args[0]]; !ok {
+ fmt.Fprintf(stderr, "unrecognized command %q\n", args[0])
+ m.Usage(stderr)
+ return 2
+ } else {
+ return cmd.RunCommand(prog+" "+args[0], args[1:], stdin, stdout, stderr)
+ }
+}
+
+func (m Multi) Usage(stderr io.Writer) {
+ fmt.Fprintf(stderr, "\nAvailable commands:\n")
+ m.listSubcommands(stderr, "")
+}
+
+func (m Multi) listSubcommands(out io.Writer, prefix string) {
+ var subcommands []string
+ for sc := range m {
+ if strings.HasPrefix(sc, "-") {
+ // Some subcommands have alternate versions
+ // like "--version" for compatibility. Don't
+ // clutter the subcommand summary with those.
+ continue
+ }
+ subcommands = append(subcommands, sc)
+ }
+ sort.Strings(subcommands)
+ for _, sc := range subcommands {
+ switch cmd := m[sc].(type) {
+ case Multi:
+ cmd.listSubcommands(out, prefix+sc+" ")
+ default:
+ fmt.Fprintf(out, " %s%s\n", prefix, sc)
+ }
+ }
+}
+
+type FlagSet interface {
+ Init(string, flag.ErrorHandling)
+ Args() []string
+ NArg() int
+ Parse([]string) error
+ SetOutput(io.Writer)
+ PrintDefaults()
+}
+
+// SubcommandToFront silently parses args using flagset, and returns a
+// copy of args with the first non-flag argument moved to the
+// front. If parsing fails or consumes all of args, args is returned
+// unchanged.
+//
+// SubcommandToFront invokes methods on flagset that have side
+// effects, including Parse. In typical usage, flagset will not used
+// for anything else after being passed to SubcommandToFront.
+func SubcommandToFront(args []string, flagset FlagSet) []string {
+ flagset.Init("", flag.ContinueOnError)
+ flagset.SetOutput(ioutil.Discard)
+ if err := flagset.Parse(args); err != nil || flagset.NArg() == 0 {
+ // No subcommand found.
+ return args
+ }
+ // Move subcommand to the front.
+ flagargs := len(args) - flagset.NArg()
+ newargs := make([]string, len(args))
+ newargs[0] = args[flagargs]
+ copy(newargs[1:flagargs+1], args[:flagargs])
+ copy(newargs[flagargs+1:], args[flagargs+1:])
+ return newargs
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+
+ "git.curoverse.com/arvados.git/lib/cmdtest"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&CmdSuite{})
+
+type CmdSuite struct{}
+
+var testCmd = Multi(map[string]Handler{
+ "echo": HandlerFunc(func(prog string, args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int {
+ fmt.Fprintln(stdout, strings.Join(args, " "))
+ return 0
+ }),
+})
+
+func (s *CmdSuite) TestHello(c *check.C) {
+ defer cmdtest.LeakCheck(c)()
+ stdout := bytes.NewBuffer(nil)
+ stderr := bytes.NewBuffer(nil)
+ exited := testCmd.RunCommand("prog", []string{"echo", "hello", "world"}, bytes.NewReader(nil), stdout, stderr)
+ c.Check(exited, check.Equals, 0)
+ c.Check(stdout.String(), check.Equals, "hello world\n")
+ c.Check(stderr.String(), check.Equals, "")
+}
+
+func (s *CmdSuite) TestUsage(c *check.C) {
+ defer cmdtest.LeakCheck(c)()
+ stdout := bytes.NewBuffer(nil)
+ stderr := bytes.NewBuffer(nil)
+ exited := testCmd.RunCommand("prog", []string{"nosuchcommand", "hi"}, bytes.NewReader(nil), stdout, stderr)
+ c.Check(exited, check.Equals, 2)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms)^unrecognized command "nosuchcommand"\n.*echo.*\n`)
+}
+
+func (s *CmdSuite) TestSubcommandToFront(c *check.C) {
+ defer cmdtest.LeakCheck(c)()
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.String("format", "json", "")
+ flags.Bool("n", false, "")
+ args := SubcommandToFront([]string{"--format=yaml", "-n", "-format", "beep", "echo", "hi"}, flags)
+ c.Check(args, check.DeepEquals, []string{"echo", "--format=yaml", "-n", "-format", "beep", "hi"})
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// Package cmdtest provides tools for testing command line tools.
+package cmdtest
+
+import (
+ "io"
+ "io/ioutil"
+ "os"
+
+ check "gopkg.in/check.v1"
+)
+
+// LeakCheck tests for output being leaked to os.Stdout and os.Stderr
+// that should be sent elsewhere (e.g., the stdout and stderr streams
+// passed to a cmd.RunFunc).
+//
+// It redirects os.Stdout and os.Stderr to a tempfile, and returns a
+// func, which the caller is expected to defer, that restores os.* and
+// checks that the tempfile is empty.
+//
+// Example:
+//
+// func (s *Suite) TestSomething(c *check.C) {
+// defer cmdtest.LeakCheck(c)()
+// // ... do things that shouldn't print to os.Stderr or os.Stdout
+// }
+func LeakCheck(c *check.C) func() {
+ tmpfiles := map[string]*os.File{"stdout": nil, "stderr": nil}
+ for i := range tmpfiles {
+ var err error
+ tmpfiles[i], err = ioutil.TempFile("", "")
+ c.Assert(err, check.IsNil)
+ err = os.Remove(tmpfiles[i].Name())
+ c.Assert(err, check.IsNil)
+ }
+
+ stdout, stderr := os.Stdout, os.Stderr
+ os.Stdout, os.Stderr = tmpfiles["stdout"], tmpfiles["stderr"]
+ return func() {
+ os.Stdout, os.Stderr = stdout, stderr
+
+ for i, tmpfile := range tmpfiles {
+ c.Log("checking %s", i)
+ _, err := tmpfile.Seek(0, io.SeekStart)
+ c.Assert(err, check.IsNil)
+ leaked, err := ioutil.ReadAll(tmpfile)
+ c.Assert(err, check.IsNil)
+ c.Check(string(leaked), check.Equals, "")
+ }
+ }
+}
SpectatorUserUUID = "zzzzz-tpzed-l1s2piq4t4mps8r"
UserAgreementCollection = "zzzzz-4zz18-uukreo9rbgwsujr" // user_agreement_in_anonymously_accessible_project
FooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
+ FooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
NonexistentCollection = "zzzzz-4zz18-totallynotexist"
HelloWorldCollection = "zzzzz-4zz18-4en62shvi99lxd4"
FooBarDirCollection = "zzzzz-4zz18-foonbarfilesdir"
projects / arvados.git / commitdiff