X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1a0a58c4f22af82e0a37440af3b0948771bca5e1..df9e166a5ffc4aa79658bec1a5d552a3b413f0d8:/sdk/cli/bin/arv diff --git a/sdk/cli/bin/arv b/sdk/cli/bin/arv index 337d9abdef..36ec037bd8 100755 --- a/sdk/cli/bin/arv +++ b/sdk/cli/bin/arv @@ -2,7 +2,7 @@ # Arvados cli client # -# Ward Vandewege +# Ward Vandewege require 'fileutils' @@ -15,7 +15,7 @@ end begin require 'curb' require 'rubygems' - require 'google/api_client' + require 'arvados/google_api_client' require 'json' require 'pp' require 'trollop' @@ -23,6 +23,8 @@ begin require 'oj' require 'active_support/inflector' require 'yaml' + require 'tempfile' + require 'net/http' rescue LoadError abort <<-EOS @@ -50,36 +52,6 @@ module Kernel end end -class Google::APIClient - def discovery_document(api, version) - api = api.to_s - discovery_uri = self.discovery_uri(api, version) - discovery_uri_hash = Digest::MD5.hexdigest(discovery_uri) - return @discovery_documents[discovery_uri_hash] ||= - begin - # fetch new API discovery doc if stale - cached_doc = File.expand_path "~/.cache/arvados/discovery-#{discovery_uri_hash}.json" rescue nil - - if cached_doc.nil? or not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400 - response = self.execute!(:http_method => :get, - :uri => discovery_uri, - :authenticated => false) - - begin - FileUtils.makedirs(File.dirname cached_doc) - File.open(cached_doc, 'w') do |f| - f.puts response.body - end - rescue - return JSON.load response.body - end - end - - File.open(cached_doc) { |f| JSON.load f } - end - end -end - class ArvadosClient < Google::APIClient def execute(*args) if args.last.is_a? Hash @@ -112,10 +84,17 @@ def init_config end end -subcommands = %w(keep pipeline tag ws edit) + +subcommands = %w(copy create edit keep pipeline run tag ws) def check_subcommands client, arvados, subcommand, global_opts, remaining_opts case subcommand + when 'create' + arv_create client, arvados, global_opts, remaining_opts + when 'edit' + arv_edit client, arvados, global_opts, remaining_opts + when 'copy', 'tag', 'ws', 'run' + exec `which arv-#{subcommand}`.strip, *remaining_opts when 'keep' @sub = remaining_opts.shift if ['get', 'put', 'ls', 'normalize'].index @sub then @@ -142,18 +121,129 @@ def check_subcommands client, arvados, subcommand, global_opts, remaining_opts puts "Available methods: run" end abort - when 'tag' - exec `which arv-tag`.strip, *remaining_opts - when 'ws' - exec `which arv-ws`.strip, *remaining_opts - when 'edit' - arv_edit client, arvados, global_opts, remaining_opts end end -def arv_edit_save_tmp tmp - FileUtils::cp tmp.path, tmp.path + ".saved" - puts "Saved contents to " + tmp.path + ".saved" +def command_exists?(command) + File.executable?(command) || ENV['PATH'].split(':').any? {|folder| File.executable?(File.join(folder, command))} +end + +def run_editor path + pid = Process::fork + if pid.nil? + editor = nil + [ENV["VISUAL"], ENV["EDITOR"], "nano", "vi"].each do |e| + editor ||= e if e and command_exists? e + end + if editor.nil? + abort "Could not find any editor to use, please set $VISUAL or $EDITOR to your desired editor." + end + exec editor, path + else + Process.wait pid + end + + if $?.exitstatus != 0 + raise "Editor exited with status #{$?.exitstatus}" + end +end + +def edit_and_commit_object initial_obj, tmp_stem, global_opts, &block + + content = case global_opts[:format] + when 'json' + Oj.dump(initial_obj, :indent => 1) + when 'yaml' + initial_obj.to_yaml + else + abort "Unrecognized format #{global_opts[:format]}" + end + + tmp_file = Tempfile.new([tmp_stem, ".#{global_opts[:format]}"]) + tmp_file.write(content) + tmp_file.close + + begin + error_text = '' + while true + begin + run_editor tmp_file.path + + tmp_file.open + newcontent = tmp_file.read() + tmp_file.close + + # Strip lines starting with '#' + newcontent = newcontent.lines.select {|l| !l.start_with? '#'}.join + + # Load the new object + newobj = case global_opts[:format] + when 'json' + Oj.load(newcontent) + when 'yaml' + YAML.load(newcontent) + end + + yield newobj + + break + rescue => e + can_retry = true + if e.is_a? Psych::SyntaxError + this_error = "YAML error parsing your input: #{e}" + elsif e.is_a? JSON::ParserError or e.is_a? Oj::ParseError + this_error = "JSON error parsing your input: #{e}" + elsif e.is_a? ArvadosAPIError + this_error = "API responded with error #{e}" + else + this_error = "#{e.class}: #{e}" + can_retry = false + end + puts this_error + + tmp_file.open + newcontent = tmp_file.read() + tmp_file.close + + if newcontent == error_text or not can_retry + FileUtils::cp tmp_file.path, tmp_file.path + ".saved" + puts "File is unchanged, edit aborted." if can_retry + abort "Saved contents to " + tmp_file.path + ".saved" + else + tmp_file.open + tmp_file.truncate 0 + error_text = this_error.to_s.lines.map {|l| '# ' + l}.join + "\n" + error_text += "# Please fix the error and try again.\n" + error_text += newcontent.lines.select {|l| !l.start_with? '#'}.join + tmp_file.write error_text + tmp_file.close + end + end + end + ensure + tmp_file.close(true) + end + + nil +end + +class ArvadosAPIError < RuntimeError +end + +def check_response result + begin + results = JSON.parse result.body + rescue JSON::ParserError, Oj::ParseError => e + raise "Failed to parse server response:\n" + e.to_s + end + + if result.response.status != 200 + raise ArvadosAPIError.new("#{result.response.status}: #{ + ((results['errors'] && results['errors'].join('\n')) || + Net::HTTPResponse::CODE_TO_OBJ[status.to_s].to_s.sub(/^Net::HTTP/, '').titleize)}") + end + + results end def arv_edit client, arvados, global_opts, remaining_opts @@ -180,7 +270,7 @@ def arv_edit client, arvados, global_opts, remaining_opts if /^[a-f0-9]{32}/.match uuid abort "Arvados collections are not editable." else - abort "#{n} does not appear to be an Arvados uuid" + abort "'#{uuid}' does not appear to be an Arvados uuid" end end @@ -197,127 +287,98 @@ def arv_edit client, arvados, global_opts, remaining_opts abort "Could not determine resource type #{m[2]}" end - api_method = 'arvados.' + rsc + '.get' - - result = client.execute(:api_method => eval(api_method), - :parameters => {"uuid" => uuid}, - :authenticated => false, - :headers => { - authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN'] - }) begin - results = JSON.parse result.body - rescue JSON::ParserError => e - abort "Failed to parse server response:\n" + e.to_s + result = client.execute(:api_method => eval('arvados.' + rsc + '.get'), + :parameters => {"uuid" => uuid}, + :authenticated => false, + :headers => { + authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN'] + }) + oldobj = check_response result + rescue => e + abort "Server error: #{e}" end if remaining_opts.length > 0 - results.select! { |k, v| remaining_opts.include? k } + oldobj.select! { |k, v| remaining_opts.include? k } end - content = "" - - case global_opts[:format] - when 'json' - content = Oj.dump(results, :indent => 1) - when 'yaml' - content = results.to_yaml + edit_and_commit_object oldobj, uuid, global_opts do |newobj| + newobj.select! {|k| newobj[k] != oldobj[k]} + if !newobj.empty? + result = client.execute(:api_method => eval('arvados.' + rsc + '.update'), + :parameters => {"uuid" => uuid}, + :body_object => { rsc.singularize => newobj }, + :authenticated => false, + :headers => { + authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN'] + }) + results = check_response result + puts "Updated object #{results['uuid']}" + else + puts "Object is unchanged, did not update." + end end - require 'tempfile' - - tmp = Tempfile.new([uuid, "." + global_opts[:format]]) - tmp.write(content) - tmp.close + exit 0 +end - need_edit = true +def arv_create client, arvados, global_opts, remaining_opts + types = resource_types(arvados.discovery_document) + create_opts = Trollop::options do + opt :project_uuid, "Project uuid in which to create the object", :type => :string + stop_on resource_types(arvados.discovery_document) + end - while need_edit - pid = Process::fork - if pid.nil? - editor ||= ENV["VISUAL"] - editor ||= ENV["EDITOR"] - editor ||= "nano" - exec editor, tmp.path - else - Process.wait pid - end + object_type = remaining_opts.shift + if object_type.nil? + abort "Missing resource type, must be one of #{types.join ', '}" + end - if $?.exitstatus == 0 - tmp.open - newcontent = tmp.read() + rsc = arvados.discovery_document["resources"].keys.select { |k| object_type == k.singularize } + if rsc.empty? + abort "Could not determine resource type #{object_type}" + end + rsc = rsc.first - newobj = {} - begin - case global_opts[:format] - when 'json' - newobj = Oj.load(newcontent) - when 'yaml' - newobj = YAML.load(newcontent) - end - need_edit = false - rescue Exception => e - puts "Parse error! " + e.to_s - n = 1 - newcontent.each_line do |line| - puts "#{n.to_s.rjust 4} #{line}" - n += 1 - end - puts "\nTry again (y/n)? " - yn = "X" - while not ["y", "Y", "n", "N"].include?(yn) - yn = $stdin.read 1 - end - if yn == 'n' or yn == 'N' - arv_edit_save_tmp tmp - abort - end + discovered_params = arvados.discovery_document["resources"][rsc]["methods"]["create"]["parameters"] + method_opts = Trollop::options do + banner head_banner + banner "Usage: arv create [--project-uuid] #{object_type} [create parameters]" + banner "" + banner "This method supports the following parameters:" + banner "" + discovered_params.each do |k,v| + opts = Hash.new() + opts[:type] = v["type"].to_sym if v.include?("type") + if [:datetime, :text, :object, :array].index opts[:type] + opts[:type] = :string # else trollop bork end - else - puts "Editor exited with status #{$?.exitstatus}" - exit $?.exitstatus + opts[:default] = v["default"] if v.include?("default") + opts[:default] = v["default"].to_i if opts[:type] == :integer + opts[:default] = to_boolean(v["default"]) if opts[:type] == :boolean + opts[:required] = true if v.include?("required") and v["required"] + description = '' + description = ' ' + v["description"] if v.include?("description") + opt k.to_sym, description, opts end end - begin - if newobj != results - api_method = 'arvados.' + rsc + '.update' - dumped = Oj.dump(newobj) - - begin - result = client.execute(:api_method => eval(api_method), - :parameters => {"uuid" => uuid}, - :body => { rsc.singularize => dumped }, - :authenticated => false, - :headers => { - authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN'] - }) - rescue Exception => e - puts "Error communicating with server, error was #{e}" - puts "Update body was:" - puts dumped - arv_edit_save_tmp tmp - abort - end - - begin - results = JSON.parse result.body - rescue JSON::ParserError => e - abort "Failed to parse server response:\n" + e.to_s - end + initial_obj = {} + if create_opts[:project_uuid] + initial_obj["owner_uuid"] = create_opts[:project_uuid] + end - if result.response.status != 200 - puts "Update failed. Server responded #{result.response.status}: #{results['errors']} " - puts "Update body was:" - puts dumped - arv_edit_save_tmp tmp - abort - end - else - puts "Object is unchanged, did not update." - end - ensure - tmp.close(true) + edit_and_commit_object initial_obj, "", global_opts do |newobj| + result = client.execute(:api_method => eval('arvados.' + rsc + '.create'), + :parameters => method_opts, + :body_object => {object_type => newobj}, + :authenticated => false, + :headers => { + authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN'] + }) + results = check_response result + puts "Created object #{results['uuid']}" end exit 0 @@ -358,20 +419,19 @@ end def help_resources(option_parser, discovery_document, resource) option_parser.educate - - if not resource.nil? and resource != '--help' then - Trollop::die "Unknown resource type #{resource.inspect}" - end exit 255 end -def parse_arguments(discovery_document, subcommands) +def resource_types discovery_document resource_types = Array.new() discovery_document["resources"].each do |k,v| resource_types << k.singularize end + resource_types +end - resource_types += subcommands +def parse_arguments(discovery_document, subcommands) + resources_and_subcommands = resource_types(discovery_document) + subcommands option_parser = Trollop::Parser.new do version __FILE__ @@ -400,7 +460,7 @@ def parse_arguments(discovery_document, subcommands) banner "Additional options:" conflicts :short, :format - stop_on resource_types + stop_on resources_and_subcommands end global_opts = Trollop::with_standard_exception_handling option_parser do @@ -420,7 +480,7 @@ def parse_arguments(discovery_document, subcommands) resource = ARGV.shift if not subcommands.include? resource - if not resource_types.include?(resource) + if not resources_and_subcommands.include?(resource) puts "Resource or subcommand '#{resource}' is not recognized.\n\n" if !resource.nil? help_resources(option_parser, discovery_document, resource) end @@ -558,7 +618,7 @@ when else result = client.execute(:api_method => eval(api_method), :parameters => request_parameters, - :body => request_body, + :body_object => request_body, :authenticated => false, :headers => { authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']