5011: Merge branch 'master' into 5011-arv-put-replication

[arvados.git] / sdk / cli / bin / arv

diff --git a/sdk/cli/bin/arv b/sdk/cli/bin/arv
index 355a947a55b5763239d95f1efae1ac606ea2b49b..36ec037bd80702b27137cb07824ca21cda641d99 100755 (executable)
--- a/sdk/cli/bin/arv
+++ b/sdk/cli/bin/arv
@@ -2,7 +2,7 @@
 
 # Arvados cli client
 #
 
 # Arvados cli client
 #

-# Ward Vandewege <ward@clinicalfuture.com>
+# Ward Vandewege <ward@curoverse.com>

 
 require 'fileutils'
 
 
 require 'fileutils'
 


@@ -15,7 +15,7 @@ end
 begin
   require 'curb'
   require 'rubygems'
 begin
   require 'curb'
   require 'rubygems'

-  require 'google/api_client'
+  require 'arvados/google_api_client'

   require 'json'
   require 'pp'
   require 'trollop'
   require 'json'
   require 'pp'
   require 'trollop'


@@ -23,6 +23,8 @@ begin
   require 'oj'
   require 'active_support/inflector'
   require 'yaml'
   require 'oj'
   require 'active_support/inflector'
   require 'yaml'

+  require 'tempfile'
+  require 'net/http'

 rescue LoadError
   abort <<-EOS
 
 rescue LoadError
   abort <<-EOS
 


@@ -50,28 +52,6 @@ module Kernel
   end
 end
 
   end
 end
 

-class Google::APIClient
- def discovery_document(api, version)
-   api = api.to_s
-   return @discovery_documents["#{api}:#{version}"] ||=
-     begin
-       # fetch new API discovery doc if stale
-       cached_doc = File.expand_path '~/.cache/arvados/discovery_uri.json'
-       if not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400
-         response = self.execute!(:http_method => :get,
-                                  :uri => self.discovery_uri(api, version),
-                                  :authenticated => false)
-         FileUtils.makedirs(File.dirname cached_doc)
-         File.open(cached_doc, 'w') do |f|
-           f.puts response.body
-         end
-       end
-
-       File.open(cached_doc) { |f| JSON.load f }
-     end
- end
-end
-

 class ArvadosClient < Google::APIClient
   def execute(*args)
     if args.last.is_a? Hash
 class ArvadosClient < Google::APIClient
   def execute(*args)
     if args.last.is_a? Hash


@@ -85,8 +65,8 @@ end
 def init_config
   # read authentication data from arvados configuration file if present
   lineno = 0
 def init_config
   # read authentication data from arvados configuration file if present
   lineno = 0

-  config_file = File.expand_path('~/.config/arvados/settings.conf')
-  if File.exist? config_file then
+  config_file = File.expand_path('~/.config/arvados/settings.conf') rescue nil
+  if not config_file.nil? and File.exist? config_file then

     File.open(config_file, 'r').each do |line|
       lineno = lineno + 1
       # skip comments
     File.open(config_file, 'r').each do |line|
       lineno = lineno + 1
       # skip comments


@@ -104,10 +84,17 @@ def init_config
   end
 end
 
   end
 end
 

-subcommands = %w(keep pipeline tag ws edit)
+
+subcommands = %w(copy create edit keep pipeline run tag ws)

 
 def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
   case subcommand
 
 def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
   case subcommand

+  when 'create'
+    arv_create client, arvados, global_opts, remaining_opts
+  when 'edit'
+    arv_edit client, arvados, global_opts, remaining_opts
+  when 'copy', 'tag', 'ws', 'run'
+    exec `which arv-#{subcommand}`.strip, *remaining_opts

   when 'keep'
     @sub = remaining_opts.shift
     if ['get', 'put', 'ls', 'normalize'].index @sub then
   when 'keep'
     @sub = remaining_opts.shift
     if ['get', 'put', 'ls', 'normalize'].index @sub then


@@ -125,24 +112,148 @@ def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
     end
     abort
   when 'pipeline'
     end
     abort
   when 'pipeline'

-    exec `which arv-run-pipeline-instance`.strip, *remaining_opts
-  when 'tag'
-    exec `which arv-tag`.strip, *remaining_opts
-  when 'ws'
-    exec `which arv-ws`.strip, *remaining_opts
-  when 'edit'
-    arv_edit client, arvados, global_opts, remaining_opts
+    sub = remaining_opts.shift
+    if sub == 'run'
+      exec `which arv-run-pipeline-instance`.strip, *remaining_opts
+    else
+      puts "Usage: arv pipeline [method] [--parameters]\n"
+      puts "Use 'arv pipeline [method] --help' to get more information about specific methods.\n\n"
+      puts "Available methods: run"
+    end
+    abort
+  end
+end
+
+def command_exists?(command)
+  File.executable?(command) || ENV['PATH'].split(':').any? {|folder| File.executable?(File.join(folder, command))}
+end
+
+def run_editor path
+  pid = Process::fork
+  if pid.nil?
+    editor = nil
+    [ENV["VISUAL"], ENV["EDITOR"], "nano", "vi"].each do |e|
+      editor ||= e if e and command_exists? e
+    end
+    if editor.nil?
+      abort "Could not find any editor to use, please set $VISUAL or $EDITOR to your desired editor."
+    end
+    exec editor, path
+  else
+    Process.wait pid

   end
   end

+
+  if $?.exitstatus != 0
+    raise "Editor exited with status #{$?.exitstatus}"
+  end
+end
+
+def edit_and_commit_object initial_obj, tmp_stem, global_opts, &block
+
+  content = case global_opts[:format]
+            when 'json'
+              Oj.dump(initial_obj, :indent => 1)
+            when 'yaml'
+              initial_obj.to_yaml
+            else
+              abort "Unrecognized format #{global_opts[:format]}"
+            end
+
+  tmp_file = Tempfile.new([tmp_stem, ".#{global_opts[:format]}"])
+  tmp_file.write(content)
+  tmp_file.close
+
+  begin
+    error_text = ''
+    while true
+      begin
+        run_editor tmp_file.path
+
+        tmp_file.open
+        newcontent = tmp_file.read()
+        tmp_file.close
+
+        # Strip lines starting with '#'
+        newcontent = newcontent.lines.select {|l| !l.start_with? '#'}.join
+
+        # Load the new object
+        newobj = case global_opts[:format]
+                 when 'json'
+                   Oj.load(newcontent)
+                 when 'yaml'
+                   YAML.load(newcontent)
+                 end
+
+        yield newobj
+
+        break
+      rescue => e
+        can_retry = true
+        if e.is_a? Psych::SyntaxError
+          this_error = "YAML error parsing your input: #{e}"
+        elsif e.is_a? JSON::ParserError or e.is_a? Oj::ParseError
+          this_error = "JSON error parsing your input: #{e}"
+        elsif e.is_a? ArvadosAPIError
+          this_error = "API responded with error #{e}"
+        else
+          this_error = "#{e.class}: #{e}"
+          can_retry = false
+        end
+        puts this_error
+
+        tmp_file.open
+        newcontent = tmp_file.read()
+        tmp_file.close
+
+        if newcontent == error_text or not can_retry
+          FileUtils::cp tmp_file.path, tmp_file.path + ".saved"
+          puts "File is unchanged, edit aborted." if can_retry
+          abort "Saved contents to " + tmp_file.path + ".saved"
+        else
+          tmp_file.open
+          tmp_file.truncate 0
+          error_text = this_error.to_s.lines.map {|l| '# ' + l}.join + "\n"
+          error_text += "# Please fix the error and try again.\n"
+          error_text += newcontent.lines.select {|l| !l.start_with? '#'}.join
+          tmp_file.write error_text
+          tmp_file.close
+        end
+      end
+    end
+  ensure
+    tmp_file.close(true)
+  end
+
+  nil
+end
+
+class ArvadosAPIError < RuntimeError
+end
+
+def check_response result
+  begin
+    results = JSON.parse result.body
+  rescue JSON::ParserError, Oj::ParseError => e
+    raise "Failed to parse server response:\n" + e.to_s
+  end
+
+  if result.response.status != 200
+    raise ArvadosAPIError.new("#{result.response.status}: #{
+                              ((results['errors'] && results['errors'].join('\n')) ||
+                                Net::HTTPResponse::CODE_TO_OBJ[status.to_s].to_s.sub(/^Net::HTTP/, '').titleize)}")
+  end
+
+  results

 end
 
 def arv_edit client, arvados, global_opts, remaining_opts
 end
 
 def arv_edit client, arvados, global_opts, remaining_opts

-  n = remaining_opts.shift
-  if n.nil? or n == "-h" or n == "--help"
+  uuid = remaining_opts.shift
+  if uuid.nil? or uuid == "-h" or uuid == "--help"

     puts head_banner
     puts "Usage: arv edit [uuid] [fields...]\n\n"
     puts head_banner
     puts "Usage: arv edit [uuid] [fields...]\n\n"

-    puts "Fetchs the specified Arvados object, select the specified fields, and\n"
+    puts "Fetch the specified Arvados object, select the specified fields, \n"

     puts "open an interactive text editor on a text representation (json or\n"
     puts "open an interactive text editor on a text representation (json or\n"

-    puts "yaml, use --format) and then updates the object.  Will use 'nano'\n"
+    puts "yaml, use --format) and then update the object.  Will use 'nano'\n"

     puts "by default, customize with the EDITOR or VISUAL environment variable.\n"
     exit 255
   end
     puts "by default, customize with the EDITOR or VISUAL environment variable.\n"
     exit 255
   end


@@ -154,9 +265,13 @@ def arv_edit client, arvados, global_opts, remaining_opts
 
   # determine controller
 
 
   # determine controller
 

-  m = /([a-z0-9]{5})-([a-z0-9]{5})-([a-z0-9]{15})/.match n
+  m = /([a-z0-9]{5})-([a-z0-9]{5})-([a-z0-9]{15})/.match uuid

   if !m
   if !m

-    abort puts "#{n} does not appear to be an arvados uuid"
+    if /^[a-f0-9]{32}/.match uuid
+      abort "Arvados collections are not editable."
+    else
+      abort "'#{uuid}' does not appear to be an Arvados uuid"
+    end

   end
 
   rsc = nil
   end
 
   rsc = nil


@@ -172,84 +287,98 @@ def arv_edit client, arvados, global_opts, remaining_opts
     abort "Could not determine resource type #{m[2]}"
   end
 
     abort "Could not determine resource type #{m[2]}"
   end
 

-  api_method = 'arvados.' + rsc + '.get'
-
-  result = client.execute(:api_method => eval(api_method),
-                          :parameters => {"uuid" => n},
-                          :authenticated => false,
-                          :headers => {
-                            authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
-                          })

   begin
   begin

-    results = JSON.parse result.body
-  rescue JSON::ParserError => e
-    abort "Failed to parse server response:\n" + e.to_s
+    result = client.execute(:api_method => eval('arvados.' + rsc + '.get'),
+                            :parameters => {"uuid" => uuid},
+                            :authenticated => false,
+                            :headers => {
+                              authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+                            })
+    oldobj = check_response result
+  rescue => e
+    abort "Server error: #{e}"

   end
 
   if remaining_opts.length > 0
   end
 
   if remaining_opts.length > 0

-    results.select! { |k, v| remaining_opts.include? k }
+    oldobj.select! { |k, v| remaining_opts.include? k }

   end
 
   end
 

-  content = ""
-
-  case global_opts[:format]
-  when 'json'
-    content = Oj.dump(results, :indent => 1)
-  when 'yaml'
-    content = results.to_yaml
+  edit_and_commit_object oldobj, uuid, global_opts do |newobj|
+    newobj.select! {|k| newobj[k] != oldobj[k]}
+    if !newobj.empty?
+      result = client.execute(:api_method => eval('arvados.' + rsc + '.update'),
+                     :parameters => {"uuid" => uuid},
+                     :body_object => { rsc.singularize => newobj },
+                     :authenticated => false,
+                     :headers => {
+                       authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+                     })
+      results = check_response result
+      puts "Updated object #{results['uuid']}"
+    else
+      puts "Object is unchanged, did not update."
+    end

   end
 
   end
 

-  require 'tempfile'
+  exit 0
+end

 
 

-  tmp = Tempfile.new([n, "." + global_opts[:format]])
-  tmp.write(content)
-  tmp.close
+def arv_create client, arvados, global_opts, remaining_opts
+  types = resource_types(arvados.discovery_document)
+  create_opts = Trollop::options do
+    opt :project_uuid, "Project uuid in which to create the object", :type => :string
+    stop_on resource_types(arvados.discovery_document)
+  end

 
 

-  pid = Process::fork
-  if pid.nil?
-    editor ||= ENV["VISUAL"]
-    editor ||= ENV["EDITOR"]
-    editor ||= "nano"
-    exec editor, tmp.path
-  else
-    Process.wait pid
+  object_type = remaining_opts.shift
+  if object_type.nil?
+    abort "Missing resource type, must be one of #{types.join ', '}"

   end
 
   end
 

-  if $?.exitstatus == 0
-    tmp.open
-    newcontent = tmp.read()
+  rsc = arvados.discovery_document["resources"].keys.select { |k| object_type == k.singularize }
+  if rsc.empty?
+    abort "Could not determine resource type #{object_type}"
+  end
+  rsc = rsc.first

 
 

-    newobj = {}
-    case global_opts[:format]
-    when 'json'
-      newobj = Oj.load(newcontent)
-    when 'yaml'
-      newobj = YAML.load(newcontent)
+  discovered_params = arvados.discovery_document["resources"][rsc]["methods"]["create"]["parameters"]
+  method_opts = Trollop::options do
+    banner head_banner
+    banner "Usage: arv create [--project-uuid] #{object_type} [create parameters]"
+    banner ""
+    banner "This method supports the following parameters:"
+    banner ""
+    discovered_params.each do |k,v|
+      opts = Hash.new()
+      opts[:type] = v["type"].to_sym if v.include?("type")
+      if [:datetime, :text, :object, :array].index opts[:type]
+        opts[:type] = :string                       # else trollop bork
+      end
+      opts[:default] = v["default"] if v.include?("default")
+      opts[:default] = v["default"].to_i if opts[:type] == :integer
+      opts[:default] = to_boolean(v["default"]) if opts[:type] == :boolean
+      opts[:required] = true if v.include?("required") and v["required"]
+      description = ''
+      description = '  ' + v["description"] if v.include?("description")
+      opt k.to_sym, description, opts

     end
     end

-    tmp.close
-    tmp.unlink
-
-    if newobj != results
-      api_method = 'arvados.' + rsc + '.update'
-      result = client.execute(:api_method => eval(api_method),
-                              :parameters => {"uuid" => n, rsc.singularize => Oj.dump(newobj)},
-                              :authenticated => false,
-                              :headers => {
-                                authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
-                              })
+  end

 
 

-      begin
-        results = JSON.parse result.body
-      rescue JSON::ParserError => e
-        abort "Failed to parse server response:\n" + e.to_s
-      end
+  initial_obj = {}
+  if create_opts[:project_uuid]
+    initial_obj["owner_uuid"] = create_opts[:project_uuid]
+  end

 
 

-      if result.response.status != 200
-        puts "Update failed.  Server responded #{result.response.status}: #{results['errors']} "
-      end
-    else
-      puts "Object is unchanged, did not update."
-    end
+  edit_and_commit_object initial_obj, "", global_opts do |newobj|
+    result = client.execute(:api_method => eval('arvados.' + rsc + '.create'),
+                   :parameters => method_opts,
+                   :body_object => {object_type => newobj},
+                   :authenticated => false,
+                   :headers => {
+                     authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+                   })
+    results = check_response result
+    puts "Created object #{results['uuid']}"

   end
 
   exit 0
   end
 
   exit 0


@@ -290,20 +419,19 @@ end
 
 def help_resources(option_parser, discovery_document, resource)
   option_parser.educate
 
 def help_resources(option_parser, discovery_document, resource)
   option_parser.educate

-
-  if not resource.nil? and resource != '--help' then
-    Trollop::die "Unknown resource type #{resource.inspect}"
-  end

   exit 255
 end
 
   exit 255
 end
 

-def parse_arguments(discovery_document, subcommands)
+def resource_types discovery_document

   resource_types = Array.new()
   discovery_document["resources"].each do |k,v|
     resource_types << k.singularize
   end
   resource_types = Array.new()
   discovery_document["resources"].each do |k,v|
     resource_types << k.singularize
   end

+  resource_types
+end

 
 

-  resource_types += subcommands
+def parse_arguments(discovery_document, subcommands)
+  resources_and_subcommands = resource_types(discovery_document) + subcommands

 
   option_parser = Trollop::Parser.new do
     version __FILE__
 
   option_parser = Trollop::Parser.new do
     version __FILE__


@@ -332,7 +460,7 @@ def parse_arguments(discovery_document, subcommands)
     banner "Additional options:"
 
     conflicts :short, :format
     banner "Additional options:"
 
     conflicts :short, :format

-    stop_on resource_types
+    stop_on resources_and_subcommands

   end
 
   global_opts = Trollop::with_standard_exception_handling option_parser do
   end
 
   global_opts = Trollop::with_standard_exception_handling option_parser do


@@ -352,7 +480,8 @@ def parse_arguments(discovery_document, subcommands)
   resource = ARGV.shift
 
   if not subcommands.include? resource
   resource = ARGV.shift
 
   if not subcommands.include? resource

-    if global_opts[:resources] or not resource_types.include?(resource)
+    if not resources_and_subcommands.include?(resource)
+      puts "Resource or subcommand '#{resource}' is not recognized.\n\n" if !resource.nil?

       help_resources(option_parser, discovery_document, resource)
     end
 
       help_resources(option_parser, discovery_document, resource)
     end
 


@@ -489,7 +618,7 @@ when
 else
   result = client.execute(:api_method => eval(api_method),
                           :parameters => request_parameters,
 else
   result = client.execute(:api_method => eval(api_method),
                           :parameters => request_parameters,

-                          :body => request_body,
+                          :body_object => request_body,

                           :authenticated => false,
                           :headers => {
                             authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
                           :authenticated => false,
                           :headers => {
                             authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']