From a6ee0e990641973aa4d79e550821eb42eab25ca4 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Tue, 2 Jan 2018 00:47:04 -0500 Subject: [PATCH] 12876: Pass commands through to ruby/python programs. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- cmd/arvados-client/cmd.go | 70 ++++++++++++++++-------- cmd/arvados-client/cmd_test.go | 6 +-- lib/cli/external.go | 99 ++++++++++++++++++++++++++++++++++ lib/cli/flags.go | 33 ++++++++++++ lib/cli/get.go | 31 +++++------ lib/cli/get_test.go | 2 +- lib/cmd/cmd.go | 58 +++++++++++--------- lib/cmd/cmd_test.go | 10 ++-- 8 files changed, 236 insertions(+), 73 deletions(-) create mode 100644 lib/cli/external.go create mode 100644 lib/cli/flags.go diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go index 73d772a578..b616b54bd9 100644 --- a/cmd/arvados-client/cmd.go +++ b/cmd/arvados-client/cmd.go @@ -5,7 +5,6 @@ package main import ( - "flag" "fmt" "io" "os" @@ -14,38 +13,67 @@ import ( "git.curoverse.com/arvados.git/lib/cli" "git.curoverse.com/arvados.git/lib/cmd" - "rsc.io/getopt" ) -var version = "dev" +var ( + version = "dev" + cmdVersion cmd.Handler = versionCmd{} + handler = cmd.Multi(map[string]cmd.Handler{ + "-e": cmdVersion, + "version": cmdVersion, + "-version": cmdVersion, + "--version": cmdVersion, -var Run = cmd.Multi(map[string]cmd.RunFunc{ - "get": cli.Get, - "-e": cmdVersion, - "version": cmdVersion, - "-version": cmdVersion, - "--version": cmdVersion, -}) + "copy": cli.Copy, + "create": cli.Create, + "edit": cli.Edit, + "get": cli.Get, + "keep": cli.Keep, + "pipeline": cli.Pipeline, + "run": cli.Run, + "tag": cli.Tag, + "ws": cli.Ws, -func cmdVersion(prog string, args []string, _ io.Reader, stdout, _ io.Writer) int { + "api_client_authorization": cli.APICall, + "api_client": cli.APICall, + "authorized_key": cli.APICall, + "collection": cli.APICall, + "container": cli.APICall, + "container_request": cli.APICall, + "group": cli.APICall, + "human": cli.APICall, + "job": cli.APICall, + "job_task": cli.APICall, + "keep_disk": cli.APICall, + "keep_service": cli.APICall, + "link": cli.APICall, + "log": cli.APICall, + "node": cli.APICall, + "pipeline_instance": cli.APICall, + "pipeline_template": cli.APICall, + "repository": cli.APICall, + "specimen": cli.APICall, + "trait": cli.APICall, + "user_agreement": cli.APICall, + "user": cli.APICall, + "virtual_machine": cli.APICall, + "workflow": cli.APICall, + }) +) + +type versionCmd struct{} + +func (versionCmd) RunCommand(prog string, args []string, _ io.Reader, stdout, _ io.Writer) int { prog = regexp.MustCompile(` -*version$`).ReplaceAllLiteralString(prog, "") fmt.Fprintf(stdout, "%s %s (%s)\n", prog, version, runtime.Version()) return 0 } func fixLegacyArgs(args []string) []string { - flags := getopt.NewFlagSet("", flag.ContinueOnError) - flags.Bool("dry-run", false, "dry run") - flags.Alias("n", "dry-run") - flags.String("format", "json", "output format") - flags.Alias("f", "format") - flags.Bool("short", false, "short") - flags.Alias("s", "short") - flags.Bool("verbose", false, "verbose") - flags.Alias("v", "verbose") + flags, _ := cli.LegacyFlagSet() return cmd.SubcommandToFront(args, flags) } func main() { - os.Exit(Run(os.Args[0], fixLegacyArgs(os.Args[1:]), os.Stdin, os.Stdout, os.Stderr)) + os.Exit(handler.RunCommand(os.Args[0], fixLegacyArgs(os.Args[1:]), os.Stdin, os.Stdout, os.Stderr)) } diff --git a/cmd/arvados-client/cmd_test.go b/cmd/arvados-client/cmd_test.go index b1ab5badfa..cbbc7b1f95 100644 --- a/cmd/arvados-client/cmd_test.go +++ b/cmd/arvados-client/cmd_test.go @@ -22,19 +22,19 @@ var _ = check.Suite(&ClientSuite{}) type ClientSuite struct{} func (s *ClientSuite) TestBadCommand(c *check.C) { - exited := Run("arvados-client", []string{"no such command"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard) + exited := handler.RunCommand("arvados-client", []string{"no such command"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard) c.Check(exited, check.Equals, 2) } func (s *ClientSuite) TestBadSubcommandArgs(c *check.C) { - exited := Run("arvados-client", []string{"get"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard) + exited := handler.RunCommand("arvados-client", []string{"get"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard) c.Check(exited, check.Equals, 2) } func (s *ClientSuite) TestVersion(c *check.C) { stdout := bytes.NewBuffer(nil) stderr := bytes.NewBuffer(nil) - exited := Run("arvados-client", []string{"version"}, bytes.NewReader(nil), stdout, stderr) + exited := handler.RunCommand("arvados-client", []string{"version"}, bytes.NewReader(nil), stdout, stderr) c.Check(exited, check.Equals, 0) c.Check(stdout.String(), check.Matches, `arvados-client dev \(go[0-9\.]+\)\n`) c.Check(stderr.String(), check.Equals, "") diff --git a/lib/cli/external.go b/lib/cli/external.go new file mode 100644 index 0000000000..ba85aae9ea --- /dev/null +++ b/lib/cli/external.go @@ -0,0 +1,99 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "fmt" + "io" + "io/ioutil" + "os/exec" + "strings" + "syscall" + + "git.curoverse.com/arvados.git/lib/cmd" +) + +var ( + Create = rubyArvCmd{"create"} + Edit = rubyArvCmd{"edit"} + + Copy = externalCmd{"arv-copy"} + Tag = externalCmd{"arv-tag"} + Ws = externalCmd{"arv-ws"} + Run = externalCmd{"arv-run"} + + Keep = cmd.Multi(map[string]cmd.Handler{ + "get": externalCmd{"arv-get"}, + "put": externalCmd{"arv-put"}, + "ls": externalCmd{"arv-ls"}, + "normalize": externalCmd{"arv-normalize"}, + "docker": externalCmd{"arv-keepdocker"}, + }) + Pipeline = cmd.Multi(map[string]cmd.Handler{ + "run": externalCmd{"arv-run-pipeline-instance"}, + }) + // user, group, container, specimen, etc. + APICall = apiCallCmd{} +) + +// When using the ruby "arv" command, flags must come before the +// subcommand: "arv --format=yaml get foo" works, but "arv get +// --format=yaml foo" does not work. +func legacyFlagsToFront(subcommand string, argsin []string) (argsout []string) { + flags, _ := LegacyFlagSet() + flags.SetOutput(ioutil.Discard) + flags.Parse(argsin) + narg := flags.NArg() + argsout = append(argsout, argsin[:len(argsin)-narg]...) + argsout = append(argsout, subcommand) + argsout = append(argsout, argsin[len(argsin)-narg:]...) + return +} + +type apiCallCmd struct{} + +func (cmd apiCallCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + split := strings.Split(prog, " ") + if len(split) < 2 { + fmt.Fprintf(stderr, "internal error: no api model in %q\n", prog) + return 2 + } + model := split[len(split)-1] + return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(model, args), stdin, stdout, stderr) +} + +type rubyArvCmd struct { + subcommand string +} + +func (rc rubyArvCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr) +} + +type externalCmd struct { + prog string +} + +func (ec externalCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + cmd := exec.Command(ec.prog, args...) + cmd.Stdin = stdin + cmd.Stdout = stdout + cmd.Stderr = stderr + err := cmd.Run() + switch err := err.(type) { + case nil: + return 0 + case *exec.ExitError: + status := err.Sys().(syscall.WaitStatus) + if status.Exited() { + return status.ExitStatus() + } + fmt.Fprintf(stderr, "%s failed: %s\n", ec.prog, err) + return 1 + default: + fmt.Fprintf(stderr, "error running %s: %s\n", ec.prog, err) + return 1 + } +} diff --git a/lib/cli/flags.go b/lib/cli/flags.go new file mode 100644 index 0000000000..7147e0c8a3 --- /dev/null +++ b/lib/cli/flags.go @@ -0,0 +1,33 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "flag" + + "git.curoverse.com/arvados.git/lib/cmd" + "rsc.io/getopt" +) + +type LegacyFlagValues struct { + Format string + DryRun bool + Short bool + Verbose bool +} + +func LegacyFlagSet() (cmd.FlagSet, *LegacyFlagValues) { + values := &LegacyFlagValues{Format: "json"} + flags := getopt.NewFlagSet("", flag.ContinueOnError) + flags.BoolVar(&values.DryRun, "dry-run", false, "Don't actually do anything") + flags.Alias("n", "dry-run") + flags.StringVar(&values.Format, "format", values.Format, "Output format: json, yaml, or uuid") + flags.Alias("f", "format") + flags.BoolVar(&values.Short, "short", false, "Return only UUIDs (equivalent to --format=uuid)") + flags.Alias("s", "short") + flags.BoolVar(&values.Verbose, "verbose", false, "Print more debug/progress messages on stderr") + flags.Alias("v", "verbose") + return flags, values +} diff --git a/lib/cli/get.go b/lib/cli/get.go index baa1df73e7..2c60f43877 100644 --- a/lib/cli/get.go +++ b/lib/cli/get.go @@ -6,16 +6,19 @@ package cli import ( "encoding/json" - "flag" "fmt" "io" + "git.curoverse.com/arvados.git/lib/cmd" "git.curoverse.com/arvados.git/sdk/go/arvados" "github.com/ghodss/yaml" - "rsc.io/getopt" ) -func Get(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { +var Get cmd.Handler = getCmd{} + +type getCmd struct{} + +func (getCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { var err error defer func() { if err != nil { @@ -23,27 +26,19 @@ func Get(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) } }() - flags := getopt.NewFlagSet(prog, flag.ContinueOnError) + flags, opts := LegacyFlagSet() flags.SetOutput(stderr) - - format := flags.String("format", "json", "output format (json, yaml, or uuid)") - flags.Alias("f", "format") - short := flags.Bool("short", false, "equivalent to --format=uuid") - flags.Alias("s", "short") - flags.Bool("dry-run", false, "dry run (ignored, for compatibility)") - flags.Alias("n", "dry-run") - flags.Bool("verbose", false, "verbose (ignored, for compatibility)") - flags.Alias("v", "verbose") err = flags.Parse(args) if err != nil { return 2 } if len(flags.Args()) != 1 { - flags.Usage() + fmt.Fprintf(stderr, "usage of %s:\n", prog) + flags.PrintDefaults() return 2 } - if *short { - *format = "uuid" + if opts.Short { + opts.Format = "uuid" } id := flags.Args()[0] @@ -59,13 +54,13 @@ func Get(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) err = fmt.Errorf("GET %s: %s", path, err) return 1 } - if *format == "yaml" { + if opts.Format == "yaml" { var buf []byte buf, err = yaml.Marshal(obj) if err == nil { _, err = stdout.Write(buf) } - } else if *format == "uuid" { + } else if opts.Format == "uuid" { fmt.Fprintln(stdout, obj["uuid"]) } else { enc := json.NewEncoder(stdout) diff --git a/lib/cli/get_test.go b/lib/cli/get_test.go index 9b8f1a0089..b2128a42fd 100644 --- a/lib/cli/get_test.go +++ b/lib/cli/get_test.go @@ -25,7 +25,7 @@ type GetSuite struct{} func (s *GetSuite) TestGetCollectionJSON(c *check.C) { stdout := bytes.NewBuffer(nil) stderr := bytes.NewBuffer(nil) - exited := Get("arvados-client get", []string{arvadostest.FooCollection}, bytes.NewReader(nil), stdout, stderr) + exited := Get.RunCommand("arvados-client get", []string{arvadostest.FooCollection}, bytes.NewReader(nil), stdout, stderr) c.Check(stdout.String(), check.Matches, `(?ms){.*"uuid": "`+arvadostest.FooCollection+`".*}\n`) c.Check(stdout.String(), check.Matches, `(?ms){.*"portable_data_hash": "`+regexp.QuoteMeta(arvadostest.FooCollectionPDH)+`".*}\n`) c.Check(stderr.String(), check.Equals, "") diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go index f33354a24b..d04006586f 100644 --- a/lib/cmd/cmd.go +++ b/lib/cmd/cmd.go @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -// package cmd defines a RunFunc type, representing a process that can -// be invoked from a command line. +// package cmd helps define reusable functions that can be exposed as +// [subcommands of] command line programs. package cmd import ( @@ -15,41 +15,47 @@ import ( "strings" ) -// A RunFunc runs a command with the given args, and returns an exit -// code. -type RunFunc func(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int +type Handler interface { + RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int +} + +type HandlerFunc func(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int + +func (f HandlerFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + return f(prog, args, stdin, stdout, stderr) +} -// Multi returns a RunFunc that looks up its first argument in m, and -// invokes the resulting RunFunc with the remaining args. +// Multi is a Handler that looks up its first argument in a map, and +// invokes the resulting Handler with the remaining args. // // Example: // -// os.Exit(Multi(map[string]RunFunc{ -// "foobar": func(prog string, args []string) int { +// os.Exit(Multi(map[string]Handler{ +// "foobar": HandlerFunc(func(prog string, args []string) int { // fmt.Println(args[0]) // return 2 -// }, +// }), // })("/usr/bin/multi", []string{"foobar", "baz"})) // // ...prints "baz" and exits 2. -func Multi(m map[string]RunFunc) RunFunc { - return func(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { - if len(args) < 1 { - fmt.Fprintf(stderr, "usage: %s command [args]\n", prog) - multiUsage(stderr, m) - return 2 - } - if cmd, ok := m[args[0]]; !ok { - fmt.Fprintf(stderr, "unrecognized command %q\n", args[0]) - multiUsage(stderr, m) - return 2 - } else { - return cmd(prog+" "+args[0], args[1:], stdin, stdout, stderr) - } +type Multi map[string]Handler + +func (m Multi) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + if len(args) < 1 { + fmt.Fprintf(stderr, "usage: %s command [args]\n", prog) + m.Usage(stderr) + return 2 + } + if cmd, ok := m[args[0]]; !ok { + fmt.Fprintf(stderr, "unrecognized command %q\n", args[0]) + m.Usage(stderr) + return 2 + } else { + return cmd.RunCommand(prog+" "+args[0], args[1:], stdin, stdout, stderr) } } -func multiUsage(stderr io.Writer, m map[string]RunFunc) { +func (m Multi) Usage(stderr io.Writer) { var subcommands []string for sc := range m { if strings.HasPrefix(sc, "-") { @@ -69,9 +75,11 @@ func multiUsage(stderr io.Writer, m map[string]RunFunc) { type FlagSet interface { Init(string, flag.ErrorHandling) + Args() []string NArg() int Parse([]string) error SetOutput(io.Writer) + PrintDefaults() } // SubcommandToFront silently parses args using flagset, and returns a diff --git a/lib/cmd/cmd_test.go b/lib/cmd/cmd_test.go index 04e98d85c3..d8a4861572 100644 --- a/lib/cmd/cmd_test.go +++ b/lib/cmd/cmd_test.go @@ -25,18 +25,18 @@ var _ = check.Suite(&CmdSuite{}) type CmdSuite struct{} -var testCmd = Multi(map[string]RunFunc{ - "echo": func(prog string, args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int { +var testCmd = Multi(map[string]Handler{ + "echo": HandlerFunc(func(prog string, args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int { fmt.Fprintln(stdout, strings.Join(args, " ")) return 0 - }, + }), }) func (s *CmdSuite) TestHello(c *check.C) { defer cmdtest.LeakCheck(c)() stdout := bytes.NewBuffer(nil) stderr := bytes.NewBuffer(nil) - exited := testCmd("prog", []string{"echo", "hello", "world"}, bytes.NewReader(nil), stdout, stderr) + exited := testCmd.RunCommand("prog", []string{"echo", "hello", "world"}, bytes.NewReader(nil), stdout, stderr) c.Check(exited, check.Equals, 0) c.Check(stdout.String(), check.Equals, "hello world\n") c.Check(stderr.String(), check.Equals, "") @@ -46,7 +46,7 @@ func (s *CmdSuite) TestUsage(c *check.C) { defer cmdtest.LeakCheck(c)() stdout := bytes.NewBuffer(nil) stderr := bytes.NewBuffer(nil) - exited := testCmd("prog", []string{"nosuchcommand", "hi"}, bytes.NewReader(nil), stdout, stderr) + exited := testCmd.RunCommand("prog", []string{"nosuchcommand", "hi"}, bytes.NewReader(nil), stdout, stderr) c.Check(exited, check.Equals, 2) c.Check(stdout.String(), check.Equals, "") c.Check(stderr.String(), check.Matches, `(?ms)^unrecognized command "nosuchcommand"\n.*echo.*\n`) -- 2.30.2