From: Tom Clegg Date: Tue, 22 Dec 2020 21:48:37 +0000 (-0500) Subject: 16306: Merge branch 'master' X-Git-Tag: 2.2.0~141^2~32 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3aaefcb3c76ff470b475d950398d01255e87712a?hp=c59af50bc2f7a366cd12a8dd6fc7d7e3b1c32480 16306: Merge branch 'master' Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/.gitignore b/.gitignore index 877ccdf4df..beb84b3c20 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ services/api/config/arvados-clients.yml .Rproj.user _version.py *.bak +arvados-snakeoil-ca.pem +.vagrant diff --git a/.licenseignore b/.licenseignore index 81f6b7181d..7ebc82667c 100644 --- a/.licenseignore +++ b/.licenseignore @@ -82,3 +82,7 @@ sdk/java-v2/settings.gradle sdk/cwl/tests/wf/feddemo go.mod go.sum +sdk/python/tests/fed-migrate/CWLFile +sdk/python/tests/fed-migrate/*.cwl +sdk/python/tests/fed-migrate/*.cwlex +doc/install/*.xlsx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 459d7277a5..39483ce62d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ https://github.com/arvados/arvados . Visit [Hacking Arvados](https://dev.arvados.org/projects/arvados/wiki/Hacking) for detailed information about setting up an Arvados development -environment, development process, coding standards, and notes about specific components. +environment, development process, [coding standards](https://dev.arvados.org/projects/arvados/wiki/Coding_Standards), and notes about specific components. If you wish to build the Arvados documentation from a local git clone, see [doc/README.textile](doc/README.textile) for instructions. diff --git a/apps/workbench/Gemfile b/apps/workbench/Gemfile index 24bfba383f..d5b416b539 100644 --- a/apps/workbench/Gemfile +++ b/apps/workbench/Gemfile @@ -4,7 +4,7 @@ source 'https://rubygems.org' -gem 'rails', '~> 5.0.0' +gem 'rails', '~> 5.2.0' gem 'arvados', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/ruby/arvados.gemspec' gem 'activerecord-nulldb-adapter', git: 'https://github.com/arvados/nulldb' @@ -14,6 +14,13 @@ gem 'sass' gem 'mime-types' gem 'responders', '~> 2.0' +# Pin sprockets to < 4.0 to avoid issues when upgrading rails to 5.2 +# See: https://github.com/rails/sprockets-rails/issues/443 +gem 'sprockets', '~> 3.0' + +# Fast app boot times +gem 'bootsnap', require: false + # Note: keeping this out of the "group :assets" section "may" allow us # to use Coffescript for UJS responses. It also prevents a # warning/problem when running tests: "WARN: tilt autoloading @@ -31,8 +38,14 @@ group :assets do gem 'therubyracer', :platforms => :ruby end -group :development do +group :development, :test, :performance do gem 'byebug' + # Pinning launchy because 2.5 requires ruby >= 2.4, which arvbox currently + # doesn't have because of SSO. + gem 'launchy', '~> 2.4.0' +end + +group :development do gem 'ruby-debug-passenger' gem 'rack-mini-profiler', require: false gem 'flamegraph', require: false @@ -48,7 +61,6 @@ group :test, :diagnostics, :performance do end group :test, :performance do - gem 'byebug' gem 'rails-perftest' gem 'ruby-prof' gem 'rvm-capistrano' @@ -70,12 +82,6 @@ gem 'angularjs-rails', '~> 1.3.8' gem 'less' gem 'less-rails' - -# Wiselinks hasn't been updated for many years and it's using deprecated methods -# Use our own Wiselinks fork until this PR is accepted: -# https://github.com/igor-alexandrov/wiselinks/pull/116 -# gem 'wiselinks', git: 'https://github.com/arvados/wiselinks.git', branch: 'rails-5.1-compatibility' - gem 'sshkey' # To use ActiveModel has_secure_password diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock index cb4e7ab9e3..e19172cb2e 100644 --- a/apps/workbench/Gemfile.lock +++ b/apps/workbench/Gemfile.lock @@ -30,39 +30,43 @@ GEM remote: https://rubygems.org/ specs: RedCloth (4.3.2) - actioncable (5.0.7.2) - actionpack (= 5.0.7.2) - nio4r (>= 1.2, < 3.0) - websocket-driver (~> 0.6.1) - actionmailer (5.0.7.2) - actionpack (= 5.0.7.2) - actionview (= 5.0.7.2) - activejob (= 5.0.7.2) + actioncable (5.2.4.3) + actionpack (= 5.2.4.3) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailer (5.2.4.3) + actionpack (= 5.2.4.3) + actionview (= 5.2.4.3) + activejob (= 5.2.4.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.0.7.2) - actionview (= 5.0.7.2) - activesupport (= 5.0.7.2) - rack (~> 2.0) - rack-test (~> 0.6.3) + actionpack (5.2.4.3) + actionview (= 5.2.4.3) + activesupport (= 5.2.4.3) + rack (~> 2.0, >= 2.0.8) + rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.0.7.2) - activesupport (= 5.0.7.2) + actionview (5.2.4.3) + activesupport (= 5.2.4.3) builder (~> 3.1) - erubis (~> 2.7.0) + erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.0.7.2) - activesupport (= 5.0.7.2) + activejob (5.2.4.3) + activesupport (= 5.2.4.3) globalid (>= 0.3.6) - activemodel (5.0.7.2) - activesupport (= 5.0.7.2) - activerecord (5.0.7.2) - activemodel (= 5.0.7.2) - activesupport (= 5.0.7.2) - arel (~> 7.0) - activesupport (5.0.7.2) + activemodel (5.2.4.3) + activesupport (= 5.2.4.3) + activerecord (5.2.4.3) + activemodel (= 5.2.4.3) + activesupport (= 5.2.4.3) + arel (>= 9.0) + activestorage (5.2.4.3) + actionpack (= 5.2.4.3) + activerecord (= 5.2.4.3) + marcel (~> 0.3.1) + activesupport (5.2.4.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -71,9 +75,9 @@ GEM public_suffix (>= 2.0.2, < 5.0) andand (1.3.3) angularjs-rails (1.3.15) - arel (7.1.4) - arvados-google-api-client (0.8.7.3) - activesupport (>= 3.2, < 5.1) + arel (9.0.0) + arvados-google-api-client (0.8.7.4) + activesupport (>= 3.2, < 5.3) addressable (~> 2.3) autoparse (~> 0.3) extlib (~> 0.9) @@ -89,6 +93,8 @@ GEM multi_json (>= 1.0.0) autoprefixer-rails (9.5.1.1) execjs + bootsnap (1.4.7) + msgpack (~> 1.0) bootstrap-sass (3.4.1) autoprefixer-rails (>= 5.2.1) sassc (>= 2.0.0) @@ -96,7 +102,7 @@ GEM railties (>= 3.1) bootstrap-x-editable-rails (1.5.1.1) railties (>= 3.0) - builder (3.2.3) + builder (3.2.4) byebug (11.0.1) capistrano (2.15.9) highline @@ -121,11 +127,11 @@ GEM execjs coffee-script-source (1.12.2) commonjs (0.2.7) - concurrent-ruby (1.1.5) - crass (1.0.5) + concurrent-ruby (1.1.6) + crass (1.0.6) deep_merge (1.2.1) docile (1.3.1) - erubis (2.7.0) + erubi (1.9.0) execjs (2.7.0) extlib (0.9.16) faraday (0.15.4) @@ -167,25 +173,29 @@ GEM railties (>= 4) request_store (~> 1.0) logstash-event (1.2.02) - loofah (2.3.1) + loofah (2.6.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) + marcel (0.3.3) + mimemagic (~> 0.3.2) memoist (0.16.2) metaclass (0.0.4) - method_source (0.9.2) + method_source (1.0.0) mime-types (3.2.2) mime-types-data (~> 3.2015) mime-types-data (3.2019.0331) - mini_mime (1.0.1) + mimemagic (0.3.5) + mini_mime (1.0.2) mini_portile2 (2.4.0) minitest (5.10.3) mocha (1.8.0) metaclass (~> 0.0.1) morrisjs-rails (0.5.1.2) railties (> 3.1, < 6) - multi_json (1.14.1) + msgpack (1.3.3) + multi_json (1.15.0) multipart-post (2.1.1) net-scp (2.0.0) net-ssh (>= 2.6.5, < 6.0.0) @@ -194,13 +204,13 @@ GEM net-ssh (5.2.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) - nio4r (2.3.1) - nokogiri (1.10.8) + nio4r (2.5.2) + nokogiri (1.10.10) mini_portile2 (~> 2.4.0) npm-rails (0.2.1) rails (>= 3.2) oj (3.7.12) - os (1.0.1) + os (1.1.1) passenger (6.0.2) rack rake (>= 0.8.1) @@ -213,23 +223,24 @@ GEM cliver (~> 0.3.1) multi_json (~> 1.0) websocket-driver (>= 0.2.0) - public_suffix (4.0.3) + public_suffix (4.0.5) rack (2.2.3) rack-mini-profiler (1.0.2) rack (>= 1.2.0) - rack-test (0.6.3) - rack (>= 1.0) - rails (5.0.7.2) - actioncable (= 5.0.7.2) - actionmailer (= 5.0.7.2) - actionpack (= 5.0.7.2) - actionview (= 5.0.7.2) - activejob (= 5.0.7.2) - activemodel (= 5.0.7.2) - activerecord (= 5.0.7.2) - activesupport (= 5.0.7.2) + rack-test (1.1.0) + rack (>= 1.0, < 3) + rails (5.2.4.3) + actioncable (= 5.2.4.3) + actionmailer (= 5.2.4.3) + actionpack (= 5.2.4.3) + actionview (= 5.2.4.3) + activejob (= 5.2.4.3) + activemodel (= 5.2.4.3) + activerecord (= 5.2.4.3) + activestorage (= 5.2.4.3) + activesupport (= 5.2.4.3) bundler (>= 1.3.0) - railties (= 5.0.7.2) + railties (= 5.2.4.3) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.4) actionpack (>= 5.0.1.x) @@ -238,15 +249,15 @@ GEM rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) + rails-html-sanitizer (1.3.0) + loofah (~> 2.3) rails-perftest (0.0.7) - railties (5.0.7.2) - actionpack (= 5.0.7.2) - activesupport (= 5.0.7.2) + railties (5.2.4.3) + actionpack (= 5.2.4.3) + activesupport (= 5.2.4.3) method_source rake (>= 0.8.7) - thor (>= 0.18.1, < 2.0) + thor (>= 0.19.0, < 2.0) rake (13.0.1) raphael-rails (2.1.2) rb-fsevent (0.10.3) @@ -305,15 +316,15 @@ GEM therubyracer (0.12.3) libv8 (~> 3.16.14.15) ref - thor (0.20.3) + thor (1.0.1) thread_safe (0.3.6) tilt (2.0.9) - tzinfo (1.2.6) + tzinfo (1.2.7) thread_safe (~> 0.1) uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) - websocket-driver (0.6.5) + websocket-driver (0.7.3) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) xpath (2.1.0) @@ -328,6 +339,7 @@ DEPENDENCIES andand angularjs-rails (~> 1.3.8) arvados! + bootsnap bootstrap-sass (~> 3.4.1) bootstrap-tab-history-rails bootstrap-x-editable-rails @@ -339,6 +351,7 @@ DEPENDENCIES headless (~> 1.0.2) httpclient (~> 2.5) jquery-rails + launchy (~> 2.4.0) less less-rails lograge @@ -354,7 +367,7 @@ DEPENDENCIES piwik_analytics poltergeist (~> 1.5.1) rack-mini-profiler - rails (~> 5.0.0) + rails (~> 5.2.0) rails-controller-testing rails-perftest raphael-rails @@ -369,10 +382,11 @@ DEPENDENCIES signet (< 0.12) simplecov (~> 0.7) simplecov-rcov + sprockets (~> 3.0) sshkey themes_for_rails! therubyracer uglifier (~> 2.0) BUNDLED WITH - 1.16.6 + 1.17.3 diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb index 8d6f897bb6..6d139cd5fd 100644 --- a/apps/workbench/app/controllers/application_controller.rb +++ b/apps/workbench/app/controllers/application_controller.rb @@ -29,7 +29,6 @@ class ApplicationController < ActionController::Base begin rescue_from(ActiveRecord::RecordNotFound, ActionController::RoutingError, - ActionController::UnknownController, AbstractController::ActionNotFound, with: :render_not_found) rescue_from(Exception, @@ -761,7 +760,7 @@ class ApplicationController < ActionController::Base if current_user && !profile_config.empty? current_user_profile = current_user.prefs[:profile] profile_config.each do |k, entry| - if entry['Required'] + if entry[:Required] if !current_user_profile || !current_user_profile[k] || current_user_profile[k].empty? @@ -928,7 +927,7 @@ class ApplicationController < ActionController::Base helper_method :my_starred_projects def my_starred_projects user return if defined?(@starred_projects) && @starred_projects - links = Link.filter([['owner_uuid', 'in', ["#{Rails.configuration.ClusterID}-j7d0g-fffffffffffffff", user.uuid]], + links = Link.filter([['owner_uuid', 'in', ["#{Rails.configuration.ClusterID}-j7d0g-publicfavorites", user.uuid]], ['link_class', '=', 'star'], ['head_uuid', 'is_a', 'arvados#group']]).with_count("none").select(%w(head_uuid)) uuids = links.collect { |x| x.head_uuid } diff --git a/apps/workbench/app/controllers/container_requests_controller.rb b/apps/workbench/app/controllers/container_requests_controller.rb index 8ce068198e..be463b022c 100644 --- a/apps/workbench/app/controllers/container_requests_controller.rb +++ b/apps/workbench/app/controllers/container_requests_controller.rb @@ -121,6 +121,21 @@ class ContainerRequestsController < ApplicationController end end params[:merge] = true + + if !@updates[:reuse_steps].nil? + if @updates[:reuse_steps] == "false" + @updates[:reuse_steps] = false + end + @updates[:command] ||= @object.command + @updates[:command] -= ["--disable-reuse", "--enable-reuse"] + if @updates[:reuse_steps] + @updates[:command].insert(1, "--enable-reuse") + else + @updates[:command].insert(1, "--disable-reuse") + end + @updates.delete(:reuse_steps) + end + begin super rescue => e @@ -134,21 +149,47 @@ class ContainerRequestsController < ApplicationController @object = ContainerRequest.new - # By default the copied CR won't be reusing containers, unless use_existing=true - # param is passed. + # set owner_uuid to that of source, provided it is a project and writable by current user + if params[:work_unit].andand[:owner_uuid] + @object.owner_uuid = src.owner_uuid = params[:work_unit][:owner_uuid] + else + current_project = Group.find(src.owner_uuid) rescue nil + if (current_project && current_project.writable_by.andand.include?(current_user.uuid)) + @object.owner_uuid = src.owner_uuid + end + end + command = src.command - if params[:use_existing] - @object.use_existing = true + if command[0] == 'arvados-cwl-runner' + command.each_with_index do |arg, i| + if arg.start_with? "--project-uuid=" + command[i] = "--project-uuid=#{@object.owner_uuid}" + end + end + command -= ["--disable-reuse", "--enable-reuse"] + command.insert(1, '--enable-reuse') + end + + if params[:use_existing] == "false" + params[:use_existing] = false + elsif params[:use_existing] == "true" + params[:use_existing] = true + end + + if params[:use_existing] || params[:use_existing].nil? + # If nil, reuse workflow steps but not the workflow runner. + @object.use_existing = !!params[:use_existing] + # Pass the correct argument to arvados-cwl-runner command. - if src.command[0] == 'arvados-cwl-runner' - command = src.command - ['--disable-reuse'] + if command[0] == 'arvados-cwl-runner' + command -= ["--disable-reuse", "--enable-reuse"] command.insert(1, '--enable-reuse') end else @object.use_existing = false # Pass the correct argument to arvados-cwl-runner command. - if src.command[0] == 'arvados-cwl-runner' - command = src.command - ['--enable-reuse'] + if command[0] == 'arvados-cwl-runner' + command -= ["--disable-reuse", "--enable-reuse"] command.insert(1, '--disable-reuse') end end @@ -167,12 +208,6 @@ class ContainerRequestsController < ApplicationController @object.scheduling_parameters = src.scheduling_parameters @object.state = 'Uncommitted' - # set owner_uuid to that of source, provided it is a project and writable by current user - current_project = Group.find(src.owner_uuid) rescue nil - if (current_project && current_project.writable_by.andand.include?(current_user.uuid)) - @object.owner_uuid = src.owner_uuid - end - super end diff --git a/apps/workbench/app/controllers/projects_controller.rb b/apps/workbench/app/controllers/projects_controller.rb index 66dc3dcea2..e448e1b453 100644 --- a/apps/workbench/app/controllers/projects_controller.rb +++ b/apps/workbench/app/controllers/projects_controller.rb @@ -133,7 +133,7 @@ class ProjectsController < ApplicationController def remove_items @removed_uuids = [] params[:item_uuids].collect { |uuid| ArvadosBase.find uuid }.each do |item| - if item.class == Collection or item.class == Group + if item.class == Collection or item.class == Group or item.class == Workflow or item.class == ContainerRequest # Use delete API on collections and projects/groups item.destroy @removed_uuids << item.uuid diff --git a/apps/workbench/app/controllers/trash_items_controller.rb b/apps/workbench/app/controllers/trash_items_controller.rb index 12ef20aa66..d8f7ae62c8 100644 --- a/apps/workbench/app/controllers/trash_items_controller.rb +++ b/apps/workbench/app/controllers/trash_items_controller.rb @@ -95,12 +95,12 @@ class TrashItemsController < ApplicationController owner_uuids = @objects.collect(&:owner_uuid).uniq @owners = {} @not_trashed = {} - Group.filter([["uuid", "in", owner_uuids]]).with_count("none").include_trash(true).each do |grp| - @owners[grp.uuid] = grp - end - User.filter([["uuid", "in", owner_uuids]]).with_count("none").include_trash(true).each do |grp| - @owners[grp.uuid] = grp - @not_trashed[grp.uuid] = true + [Group, User].each do |owner_class| + owner_class.filter([["uuid", "in", owner_uuids]]).with_count("none") + .include_trash(true).fetch_multiple_pages(false) + .each do |owner| + @owners[owner.uuid] = owner + end end Group.filter([["uuid", "in", owner_uuids]]).with_count("none").select([:uuid]).each do |grp| @not_trashed[grp.uuid] = true diff --git a/apps/workbench/app/controllers/users_controller.rb b/apps/workbench/app/controllers/users_controller.rb index 27fc12bf4c..21ea7a8e69 100644 --- a/apps/workbench/app/controllers/users_controller.rb +++ b/apps/workbench/app/controllers/users_controller.rb @@ -39,6 +39,18 @@ class UsersController < ApplicationController def profile params[:offer_return_to] ||= params[:return_to] + + # In a federation situation, when you get a user record using + # "current user of token" it can fetch a stale user record from + # the local cluster. So even if profile settings were just written + # to the user record on the login cluster (because the user just + # filled out the profile), those profile settings may not appear + # in the "current user" response because it is returning a cached + # record from the local cluster. + # + # In this case, explicitly fetching user record forces it to get a + # fresh record from the login cluster. + Thread.current[:user] = User.find(current_user.uuid) end def activity diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb index 8c4e5e7d9f..237cf27555 100644 --- a/apps/workbench/app/controllers/work_units_controller.rb +++ b/apps/workbench/app/controllers/work_units_controller.rb @@ -117,12 +117,16 @@ class WorkUnitsController < ApplicationController if hint[:keep_cache] keep_cache = hint[:keep_cache] end + if hint[:acrContainerImage] + attrs['container_image'] = hint[:acrContainerImage] + end end end end end attrs['command'] = ["arvados-cwl-runner", + "--enable-reuse", "--local", "--api=containers", "--project-uuid=#{params['work_unit']['owner_uuid']}", diff --git a/apps/workbench/app/helpers/application_helper.rb b/apps/workbench/app/helpers/application_helper.rb index 330d30976f..786716eb33 100644 --- a/apps/workbench/app/helpers/application_helper.rb +++ b/apps/workbench/app/helpers/application_helper.rb @@ -247,11 +247,15 @@ module ApplicationHelper end input_type = 'text' + opt_selection = nil attrtype = object.class.attribute_info[attr.to_sym].andand[:type] if attrtype == 'text' or attr == 'description' input_type = 'textarea' elsif attrtype == 'datetime' input_type = 'date' + elsif attrtype == 'boolean' + input_type = 'select' + opt_selection = ([{value: "true", text: "true"}, {value: "false", text: "false"}]).to_json else input_type = 'text' end @@ -279,6 +283,7 @@ module ApplicationHelper "data-emptytext" => '(none)', "data-placement" => "bottom", "data-type" => input_type, + "data-source" => opt_selection, "data-title" => "Edit #{attr.to_s.gsub '_', ' '}", "data-name" => htmloptions['selection_name'] || attr, "data-object-uuid" => object.uuid, diff --git a/apps/workbench/app/models/arvados_base.rb b/apps/workbench/app/models/arvados_base.rb index b9162c2aec..c5e1a4ed22 100644 --- a/apps/workbench/app/models/arvados_base.rb +++ b/apps/workbench/app/models/arvados_base.rb @@ -106,6 +106,12 @@ class ArvadosBase end end + # The ActiveModel::Dirty API was changed on Rails 5.2 + # See: https://github.com/rails/rails/commit/c3675f50d2e59b7fc173d7b332860c4b1a24a726#diff-aaddd42c7feb0834b1b5c66af69814d3 + def mutations_from_database + @mutations_from_database ||= ActiveModel::NullMutationTracker.instance + end + def self.columns @discovered_columns = [] if !defined?(@discovered_columns) return @discovered_columns if @discovered_columns.andand.any? diff --git a/apps/workbench/app/models/container_request.rb b/apps/workbench/app/models/container_request.rb index 3c08d94989..be97a6cfb5 100644 --- a/apps/workbench/app/models/container_request.rb +++ b/apps/workbench/app/models/container_request.rb @@ -15,7 +15,31 @@ class ContainerRequest < ArvadosBase true end + def self.copies_to_projects? + false + end + def work_unit(label=nil, child_objects=nil) ContainerWorkUnit.new(self, label, self.uuid, child_objects=child_objects) end + + def editable_attributes + super + ["reuse_steps"] + end + + def reuse_steps + command.each do |arg| + if arg == "--enable-reuse" + return true + end + end + false + end + + def self.attribute_info + self.columns + @attribute_info[:reuse_steps] = {:type => "boolean"} + @attribute_info + end + end diff --git a/apps/workbench/app/models/user.rb b/apps/workbench/app/models/user.rb index 34e8181515..c4b273c6b8 100644 --- a/apps/workbench/app/models/user.rb +++ b/apps/workbench/app/models/user.rb @@ -110,6 +110,6 @@ class User < ArvadosBase end def self.creatable? - current_user and current_user.is_admin + current_user.andand.is_admin end end diff --git a/apps/workbench/app/views/container_requests/_extra_tab_line_buttons.html.erb b/apps/workbench/app/views/container_requests/_extra_tab_line_buttons.html.erb index b698c938a1..7a9d68d983 100644 --- a/apps/workbench/app/views/container_requests/_extra_tab_line_buttons.html.erb +++ b/apps/workbench/app/views/container_requests/_extra_tab_line_buttons.html.erb @@ -9,40 +9,19 @@ SPDX-License-Identifier: AGPL-3.0 %> } - <%= link_to raw(' Re-run...'), - "#", - {class: 'btn btn-sm btn-primary', 'data-toggle' => 'modal', - 'data-target' => '#clone-and-edit-modal-window', - title: 'This will make a copy and take you there. You can then make any needed changes and run it'} %> - - <% end %> diff --git a/apps/workbench/app/views/container_requests/_show_inputs.html.erb b/apps/workbench/app/views/container_requests/_show_inputs.html.erb index fd8e363838..07bf7c4d76 100644 --- a/apps/workbench/app/views/container_requests/_show_inputs.html.erb +++ b/apps/workbench/app/views/container_requests/_show_inputs.html.erb @@ -17,23 +17,23 @@ n_inputs = if @object.mounts[:"/var/lib/cwl/workflow.json"] && @object.mounts[:" <% if workflow %> <% inputs = get_cwl_inputs(workflow) %> <% inputs.each do |input| %> - -
-

- <%= render_cwl_input @object, input, [:mounts, :"/var/lib/cwl/cwl.input.json", :content] %> +

+ + <%= render_cwl_input @object, input, [:mounts, :"/var/lib/cwl/cwl.input.json", :content] %> +

+ <%= input[:doc] %>

-

- <%= input[:doc] %> -

<% end %> <% end %>
<% end %> +

Reuse past workflow steps if available? <%= render_editable_attribute(@object, :reuse_steps) %>

+ <% if n_inputs == 0 %>

This workflow does not need any further inputs specified. Click the "Run" button at the bottom of the page to start the workflow.

<% else %> diff --git a/apps/workbench/app/views/users/_virtual_machines.html.erb b/apps/workbench/app/views/users/_virtual_machines.html.erb index e2ce5b39bc..57b4d6aa38 100644 --- a/apps/workbench/app/views/users/_virtual_machines.html.erb +++ b/apps/workbench/app/views/users/_virtual_machines.html.erb @@ -69,7 +69,7 @@ SPDX-License-Identifier: AGPL-3.0 %> Login name Command line <% if Rails.configuration.Services.WebShell.ExternalURL != URI("") %> - Web shell beta + Web shell <% end %> diff --git a/apps/workbench/app/views/users/profile.html.erb b/apps/workbench/app/views/users/profile.html.erb index 6692196dab..caa22bda11 100644 --- a/apps/workbench/app/views/users/profile.html.erb +++ b/apps/workbench/app/views/users/profile.html.erb @@ -68,29 +68,30 @@ SPDX-License-Identifier: AGPL-3.0 %> <% profile_config.kind_of?(Array) && profile_config.andand.each do |entry| %> - <% if entry['Key'] %> + <% if entry[:Key] %> <% show_save_button = true - label = entry['Required'] ? '* ' : '' - label += entry['FormFieldTitle'] - value = current_user_profile[entry['Key'].to_sym] if current_user_profile + label = entry[:Required] ? '* ' : '' + label += entry[:FormFieldTitle] + value = current_user_profile[entry[:Key].to_sym] if current_user_profile %>
- - <% if entry['Type'] == 'select' %> + <% if entry[:Type] == 'select' %>
- + <% entry[:Options].each do |option, _| %> + <% option = option.to_s %> <% end %>
<% else %>
- +
<% end %>
diff --git a/apps/workbench/app/views/virtual_machines/webshell.html.erb b/apps/workbench/app/views/virtual_machines/webshell.html.erb index 4c63115a16..735583faec 100644 --- a/apps/workbench/app/views/virtual_machines/webshell.html.erb +++ b/apps/workbench/app/views/virtual_machines/webshell.html.erb @@ -30,17 +30,43 @@ SPDX-License-Identifier: AGPL-3.0 %> function login(username, token) { var sh = new ShellInABox("<%= j @webshell_url %>"); - setTimeout(function() { - sh.keysPressed("<%= j params[:login] %>\n"); - setTimeout(function() { - sh.keysPressed("<%= j Thread.current[:arvados_api_token] %>\n"); - sh.vt100('(sent authentication token)\n'); - }, 2000); - }, 2000); + + var findText = function(txt) { + var a = document.querySelectorAll("span.ansi0"); + for (var i = 0; i < a.length; i++) { + if (a[i].textContent.indexOf(txt) > -1) { + return true; + } + } + return false; + } + + var trySendToken = function() { + // change this text when PAM is reconfigured to present a + // password prompt that we can wait for. + if (findText("assword:")) { + sh.keysPressed("<%= j Thread.current[:arvados_api_token] %>\n"); + sh.vt100('(sent authentication token)\n'); + } else { + setTimeout(trySendToken, 200); + } + }; + + var trySendLogin = function() { + if (findText("login:")) { + sh.keysPressed("<%= j params[:login] %>\n"); + // Make this wait shorter when PAM is reconfigured to + // present a password prompt that we can wait for. + setTimeout(trySendToken, 200); + } else { + setTimeout(trySendLogin, 200); + } + }; + + trySendLogin(); } // --> - - - - + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/images/wgs-tutorial/image1.png b/doc/images/wgs-tutorial/image1.png new file mode 100644 index 0000000000..854f441201 Binary files /dev/null and b/doc/images/wgs-tutorial/image1.png differ diff --git a/doc/images/wgs-tutorial/image2.png b/doc/images/wgs-tutorial/image2.png new file mode 100644 index 0000000000..f68d21a8d0 Binary files /dev/null and b/doc/images/wgs-tutorial/image2.png differ diff --git a/doc/images/wgs-tutorial/image3.png b/doc/images/wgs-tutorial/image3.png new file mode 100644 index 0000000000..7651d2f82d Binary files /dev/null and b/doc/images/wgs-tutorial/image3.png differ diff --git a/doc/images/wgs-tutorial/image4.png b/doc/images/wgs-tutorial/image4.png new file mode 100644 index 0000000000..ad805298d7 Binary files /dev/null and b/doc/images/wgs-tutorial/image4.png differ diff --git a/doc/images/wgs-tutorial/image5.png b/doc/images/wgs-tutorial/image5.png new file mode 100644 index 0000000000..8ee9048ee9 Binary files /dev/null and b/doc/images/wgs-tutorial/image5.png differ diff --git a/doc/images/wgs-tutorial/image6.png b/doc/images/wgs-tutorial/image6.png new file mode 100644 index 0000000000..41dc28dedc Binary files /dev/null and b/doc/images/wgs-tutorial/image6.png differ diff --git a/doc/index.html.liquid b/doc/index.html.liquid index c7fd46eb22..d9a2e6bbe2 100644 --- a/doc/index.html.liquid +++ b/doc/index.html.liquid @@ -46,16 +46,13 @@ SPDX-License-Identifier: CC-BY-SA-3.0

Arvados is under active development, see the recent developer activity.

License

-

Most of Arvados is licensed under the GNU AGPL v3. The SDKs are licensed under the Apache License 2.0 so that they can be incorporated into proprietary code. See the COPYING file for more information. +

Most of Arvados is licensed under the GNU AGPL v3. The SDKs are licensed under the Apache License 2.0 and can be incorporated into proprietary code. See Arvados Free Software Licenses for more information.

-

More in-depth guides +

Sections

- - -

User Guide — How to manage data and do analysis with Arvados.

diff --git a/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid b/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid index 0801b7d4e3..06280b467d 100644 --- a/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid +++ b/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid @@ -11,10 +11,6 @@ SPDX-License-Identifier: CC-BY-SA-3.0 This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE). -{% include 'notebox_begin_warning' %} -This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version. -{% include 'notebox_end' %} - h2. Prerequisites h3. Install tooling @@ -142,10 +138,6 @@ $ helm upgrade arvados . h2. Shut down -{% include 'notebox_begin_warning' %} -This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the Arvados cluster will be deleted when it is shut down. This will be fixed in a future version. -{% include 'notebox_end' %} - 
 $ helm del arvados
diff --git a/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid b/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid index 86aaf08f96..9ecb2c8956 100644 --- a/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid +++ b/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid @@ -11,10 +11,6 @@ SPDX-License-Identifier: CC-BY-SA-3.0 This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@. -{% include 'notebox_begin_warning' %} -This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version. -{% include 'notebox_end' %} - h2. Prerequisites h3. Install tooling @@ -128,7 +124,7 @@ $ helm upgrade arvados . h2. Shut down {% include 'notebox_begin_warning' %} -This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the Arvados cluster will be deleted when it is shut down. This will be fixed in a future version. +This Helm chart uses Kubernetes persistent volumes for the Postgresql and Keepstore data volumes. These volumes will be retained after you delete the Arvados helm chart with the command below. Because those volumes are stored in the local Minikube Kubernetes cluster, if you delete that cluster (e.g. with minikube delete) the Kubernetes persistent volumes will also be deleted. {% include 'notebox_end' %} 
diff --git a/doc/install/arvados-on-kubernetes.html.textile.liquid b/doc/install/arvados-on-kubernetes.html.textile.liquid
index ff52aa171f..9169b7810e 100644
--- a/doc/install/arvados-on-kubernetes.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes.html.textile.liquid
@@ -11,10 +11,6 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 Arvados on Kubernetes is implemented as a @Helm 3@ chart.
 
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
 h2(#overview). Overview
 
 This Helm chart provides a basic, small Arvados cluster.
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index 3fbd33928a..3c77ade8da 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -17,8 +17,11 @@ h2. Quick start
 $ git clone https://github.com/arvados/arvados.git
 $ cd arvados/tools/arvbox/bin
 $ ./arvbox start localdemo
+$ ./arvbox adduser demouser demo@example.com
+You can now log in as @demouser@ using the password you selected. + h2. Requirements * Linux 3.x+ and Docker 1.9+ @@ -46,6 +49,9 @@ update stop, pull latest image, run build build arvbox Docker image reboot stop, build arvbox Docker image, run rebuild build arvbox Docker image, no layer cache +checkpoint create database backup +restore restore checkpoint +hotreset reset database and restart API without restarting container reset delete arvbox arvados data (be careful!) destroy delete all arvbox code and data (be careful!) log tail log of specified service @@ -55,6 +61,11 @@ pipe run a bash script piped in from stdin sv change state of service inside arvbox clone clone dev arvbox +adduser + add a user login +removeuser + remove user login +listusers list user logins h2. Install root certificate @@ -69,18 +80,31 @@ Arvbox creates root certificate to authorize Arvbox services. Installing the ro The certificate will be added under the "Arvados testing" organization as "arvbox testing root CA". -To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage (instructions for Debian/Ubuntu): +To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage. + +h3. On Debian/Ubuntu: + + +
cp arvbox-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+
+ -# copy @arvbox-root-cert.pem@ to @/usr/local/share/ca-certificates/@ -# run @/usr/sbin/update-ca-certificates@ +h3. On CentOS: + + +
cp arvbox-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+
+ h2. Configs h3. dev -Development configuration. Boots a complete Arvados environment inside the container. The "arvados", "arvado-dev" and "sso-devise-omniauth-provider" code directories along data directories "postgres", "var", "passenger" and "gems" are bind mounted from the host file system for easy access and persistence across container rebuilds. Services are bound to the Docker container's network IP address and can only be accessed on the local host. +Development configuration. Boots a complete Arvados environment inside the container. The "arvados" and "arvados-dev" code directories along data directories "postgres", "var", "passenger" and "gems" are bind mounted from the host file system for easy access and persistence across container rebuilds. Services are bound to the Docker container's network IP address and can only be accessed on the local host. -In "dev" mode, you can override the default autogenerated settings of Rails projects by adding "application.yml.override" to any Rails project (sso, api, workbench). This can be used to test out API server settings or point Workbench at an alternate API server. +In "dev" mode, you can override the default autogenerated settings of Rails projects by adding "application.yml.override" to any Rails project (api, workbench). This can be used to test out API server settings or point Workbench at an alternate API server. h3. localdemo @@ -134,11 +158,6 @@ h3. ARVADOS_DEV_ROOT The root directory of the Arvados-dev source tree default: $ARVBOX_DATA/arvados-dev -h3. SSO_ROOT - -The root directory of the SSO source tree -default: $ARVBOX_DATA/sso-devise-omniauth-provider - h3. ARVBOX_PUBLISH_IP The IP address on which to publish services when running in public configuration. Overrides default detection of the host's IP address. diff --git a/doc/install/copy_pipeline_from_curoverse.html.textile.liquid b/doc/install/copy_pipeline_from_curoverse.html.textile.liquid deleted file mode 100644 index 2c2b3c466e..0000000000 --- a/doc/install/copy_pipeline_from_curoverse.html.textile.liquid +++ /dev/null @@ -1,68 +0,0 @@ ---- -layout: default -navsection: installguide -title: Copy pipeline from the Arvados Playground -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -This tutorial describes how to find and copy a publicly shared pipeline from the Arvados Playground. Please note that you can use similar steps to copy any template you can access from the Arvados Playground to your cluster. - -h3. Access a public pipeline in the Arvados Playground using Workbench - -the Arvados Playground provides access to some public data, which can be used to experience Arvados in action. Let's access a public pipeline and copy it to your cluster, so that you can run it in your environment. - -Start by visiting the "*Arvados Playground public projects page*":https://playground.arvados.org/projects/public. This page lists all the publicly accessible projects in this arvados installation. Click on one of these projects to open it. We will use "*lobSTR v.3 (Public)*":https://playground.arvados.org/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq as the example in this tutorial. - -Once in the "*lobSTR v.3 (Public)*":https://playground.arvados.org/projects/qr1hi-j7d0g-up6qgpqz5ie2vfq project, click on the *Pipeline templates* tab. In the pipeline templates tab, you will see a template named *lobSTR v.3*. Click on the *Show* button to the left of this name. This will take to you to the "*lobSTR v.3*":https://playground.arvados.org/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu template page. - -Once in this page, you can take the *uuid* of this template from the address bar, which is *qr1hi-p5p6p-9pkaxt6qjnkxhhu*. Next, we will copy this template to your Arvados instance. - -h3. Copying a pipeline template from the Arvados Playground to your cluster - -As described above, navigate to the publicly shared pipeline template "*lobSTR v.3*":https://playground.arvados.org/pipeline_templates/qr1hi-p5p6p-9pkaxt6qjnkxhhu on the Arvados Playground. We will now copy this template with uuid *qr1hi-p5p6p-9pkaxt6qjnkxhhu* to your cluster. - -{% include 'tutorial_expectations' %} - -We will use the Arvados *arv-copy* command to copy this template to your cluster. In order to use arv-copy, first you need to setup the source and destination cluster configuration files. Here, *qr1hi* would be the source cluster and your Arvados instance would be the *dst_cluster*. - -During this setup, if you have an account in the Arvados Playground, you can use "your access token":#using-your-token to create the source configuration file. If you do not have an account in the Arvados Playground, you can use the "anonymous access token":#using-anonymous-token for the source cluster configuration. - -h4(#using-anonymous-token). *Configuring source and destination setup files using anonymous access token* - -Configure the source and destination clusters as described in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html tutorial in user guide, while using *5vqmz9mik2ou2k9objb8pnyce8t97p6vocyaouqo3qalvpmjs5* as the API token for source configuration. - - -
~$ cd ~/.config/arvados
-~$ echo "ARVADOS_API_HOST=qr1hi.arvadosapi.com" >> qr1hi.conf
-~$ echo "ARVADOS_API_TOKEN=5vqmz9mik2ou2k9objb8pnyce8t97p6vocyaouqo3qalvpmjs5" >> qr1hi.conf
-
- - -You can now copy the pipeline template from *qr1hi* to *your cluster*. Replace *dst_cluster* with the *ClusterID* of your cluster. - - -
~$  arv-copy --no-recursive --src qr1hi --dst dst_cluster qr1hi-p5p6p-9pkaxt6qjnkxhhu
-
- - -*Note:* When you are using anonymous access token to copy the template, you will not be able to do a recursive copy since you will not be able to provide the dst-git-repo parameter. In order to perform a recursive copy of the template, you would need to use the Arvados API token from your account as explained in the "using your token":#using-your-token section below. - -h4(#using-your-token). *Configuring source and destination setup files using personal access token* - -If you already have an account in the Arvados Playground, you can follow the instructions in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html user guide to get your *Current token* for source and destination clusters, and use them to create the source *qr1hi.conf* and dst_cluster.conf configuration files. - -You can now copy the pipeline template from *qr1hi* to *your cluster* with or without recursion. Replace *dst_cluster* with the *ClusterID* of your cluster. - -*Non-recursive copy:* - -
~$  arv-copy --no-recursive --src qr1hi --dst dst_cluster qr1hi-p5p6p-9pkaxt6qjnkxhhu
- - -*Recursive copy:* - -
~$ arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial qr1hi-p5p6p-9pkaxt6qjnkxhhu
- diff --git a/doc/install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid b/doc/install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid index 6841778470..151e211653 100644 --- a/doc/install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid +++ b/doc/install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid @@ -100,6 +100,9 @@ Using managed disks: CloudVMs: ImageID: "zzzzz-compute-v1597349873" Driver: azure + # (azure) managed disks: set MaxConcurrentInstanceCreateOps to 20 to avoid timeouts, cf + # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image + MaxConcurrentInstanceCreateOps: 20 DriverParameters: # Credentials. SubscriptionID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX diff --git a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid index a9689e9ac3..3996cc7930 100644 --- a/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid +++ b/doc/install/crunch2-slurm/install-dispatch.html.textile.liquid @@ -22,7 +22,7 @@ crunch-dispatch-slurm is only relevant for on premises clusters that will spool h2(#introduction). Introduction -This assumes you already have a Slurm cluster, and have "set up all of your compute nodes":install-compute-node.html . For information on installing Slurm, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html +This assumes you already have a Slurm cluster, and have "set up all of your compute nodes":install-compute-node.html. Slurm packages are available for CentOS, Debian and Ubuntu. Please see your distribution package repositories. For information on installing Slurm from source, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html The Arvados Slurm dispatcher can run on any node that can submit requests to both the Arvados API server and the Slurm controller (via @sbatch@). It is not resource-intensive, so you can run it on the API server node. diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid index 81d7b21592..f16ae2dad2 100644 --- a/doc/install/index.html.textile.liquid +++ b/doc/install/index.html.textile.liquid @@ -20,11 +20,13 @@ Arvados components can be installed and configured in a number of different ways
table(table table-bordered table-condensed). |||\5=. Appropriate for| -||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation| +||_. Setup difficulty|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation| |"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes| +|"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes| +|"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes| |"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes| |"Automatic single-node install":automatic.html (experimental)|Easy|yes|yes|no|yes|yes| -|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no| +|"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no| |"Cluster Operation Subscription supported by Curii":mailto:info@curii.com|N/A ^3^|yes|yes|yes|yes|yes|
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid index b8442eb060..c7303bbba2 100644 --- a/doc/install/install-api-server.html.textile.liquid +++ b/doc/install/install-api-server.html.textile.liquid @@ -51,22 +51,20 @@ h3. Tokens API: RailsSessionSecretToken: "$rails_secret_token" Collections: - BlobSigningKey: "blob_signing_key" + BlobSigningKey: "$blob_signing_key" -@SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server. +These secret tokens are used to authenticate messages between Arvados components. +* @SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server. +* @ManagementToken@ is used to authenticate access to system metrics. +* @API.RailsSessionSecretToken@ is used to sign session cookies. +* @Collections.BlobSigningKey@ is used to control access to Keep blocks. -@ManagementToken@ is used to authenticate access to system metrics. - -@API.RailsSessionSecretToken@ is required by the API server. - -@Collections.BlobSigningKey@ is used to control access to Keep blocks. - -You can generate a random token for each of these items at the command line like this: +Each token should be a string of at least 50 alphanumeric characters. You can generate a suitable token with the following command: -
~$ tr -dc 0-9a-zA-Z </dev/urandom | head -c50; echo
+
~$ tr -dc 0-9a-zA-Z </dev/urandom | head -c50 ; echo
 

 
 
diff --git a/doc/install/install-compute-ping.html.textile.liquid b/doc/install/install-compute-ping.html.textile.liquid
deleted file mode 100644
index be3f58b61e..0000000000
--- a/doc/install/install-compute-ping.html.textile.liquid
+++ /dev/null
@@ -1,14 +0,0 @@
----
-layout: default
-navsection: installguide
-title: Sample compute node ping script
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-When a new elastic compute node is booted, it needs to contact Arvados to register itself.  Here is an example ping script to run on boot.
-
- {% code 'compute_ping_rb' as ruby %} 
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index 24f37bfb4f..b797c1958e 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -20,7 +20,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 h2(#introduction). Introduction
 
-The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep.  This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols.  This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
 
 h2(#dns). Configure DNS
 
@@ -61,6 +61,8 @@ Collections can be served from their own subdomain:
+This option is preferred if you plan to access Keep using third-party S3 client software, because it accommodates S3 virtual host-style requests and path-style requests without any special client configuration. + h4. Under the main domain Alternately, they can go under the main domain by including @--@: diff --git a/doc/install/install-manual-prerequisites.html.textile.liquid b/doc/install/install-manual-prerequisites.html.textile.liquid index 55095b1f20..e6f1ba8fdc 100644 --- a/doc/install/install-manual-prerequisites.html.textile.liquid +++ b/doc/install/install-manual-prerequisites.html.textile.liquid @@ -11,6 +11,8 @@ SPDX-License-Identifier: CC-BY-SA-3.0 Before attempting installation, you should begin by reviewing supported platforms, choosing backends for identity, storage, and scheduling, and decide how you will distribute Arvados services onto machines. You should also choose an Arvados Cluster ID, choose your hostnames, and aquire TLS certificates. It may be helpful to make notes as you go along using one of these worksheets: "New cluster checklist for AWS":new_cluster_checklist_AWS.xlsx - "New cluster checklist for Azure":new_cluster_checklist_Azure.xlsx - "New cluster checklist for on premises Slurm":new_cluster_checklist_slurm.xlsx +The installation guide describes how to set up a basic standalone Arvados instance. Additional configuration for features including "federation,":{{site.baseurl}}/admin/federation.html "collection versioning,":{{site.baseurl}}/admin/collection-versioning.html "managed properties,":{{site.baseurl}}/admin/collection-managed-properties.html and "storage classes":{{site.baseurl}}/admin/collection-managed-properties.html are described in the "Admin guide.":{{site.baseurl}}/admin + The Arvados storage subsystem is called "keep". The compute subsystem is called "crunch". # "Supported GNU/Linux distributions":#supportedlinux @@ -28,11 +30,11 @@ table(table table-bordered table-condensed). |_. Distribution|_. State|_. Last supported version| |CentOS 7|Supported|Latest| |Debian 10 ("buster")|Supported|Latest| -|Debian 9 ("stretch")|Supported|Latest| |Ubuntu 18.04 ("bionic")|Supported|Latest| |Ubuntu 16.04 ("xenial")|Supported|Latest| -|Ubuntu 14.04 ("trusty")|EOL|1.4.3| +|Debian 9 ("stretch")|EOL|Latest 2.1.X release| |Debian 8 ("jessie")|EOL|1.4.3| +|Ubuntu 14.04 ("trusty")|EOL|1.4.3| |Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)| |Debian 7 ("wheezy")|EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)| |CentOS 6 |EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)| @@ -68,6 +70,7 @@ h2(#identity). Identity provider Choose which backend you will use to authenticate users. * Google login to authenticate users with a Google account. +* OpenID Connect (OIDC) if you have Single-Sign-On (SSO) service that supports the OpenID Connect standard. * LDAP login to authenticate users by username/password using the LDAP protocol, supported by many services such as OpenLDAP and Active Directory. * PAM login to authenticate users by username/password according to the PAM configuration on the controller node. diff --git a/doc/install/install-postgresql.html.textile.liquid b/doc/install/install-postgresql.html.textile.liquid index b25194a9ee..60afa1e24f 100644 --- a/doc/install/install-postgresql.html.textile.liquid +++ b/doc/install/install-postgresql.html.textile.liquid @@ -19,20 +19,18 @@ h3(#centos7). CentOS 7 {% include 'note_python_sc' %} # Install PostgreSQL - 
# yum install rh-postgresql95 rh-postgresql95-postgresql-contrib
-~$ scl enable rh-postgresql95 bash
 + 
# yum install rh-postgresql12 rh-postgresql12-postgresql-contrib
+~$ scl enable rh-postgresql12 bash
 # Initialize the database 
# postgresql-setup initdb
 # Configure the database to accept password connections 
# sed -ri -e 's/^(host +all +all +(127\.0\.0\.1\/32|::1\/128) +)ident$/\1md5/' /var/lib/pgsql/data/pg_hba.conf
 # Configure the database to launch at boot and start now - 
# systemctl enable --now rh-postgresql95-postgresql
 + 
# systemctl enable --now rh-postgresql12-postgresql
 h3(#debian). Debian or Ubuntu -Debian 8 (Jessie) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres. - -Ubuntu 14.04 (Trusty) requires an updated PostgreSQL version, see "the PostgreSQL ubuntu repository":https://www.postgresql.org/download/linux/ubuntu/ +Debian 10 (Buster) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres. # Install PostgreSQL 
# apt-get --no-install-recommends install postgresql postgresql-contrib
 diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid index 5ac5e9e6b8..97854e5240 100644 --- a/doc/install/install-shell-server.html.textile.liquid +++ b/doc/install/install-shell-server.html.textile.liquid @@ -22,9 +22,15 @@ h2(#introduction). Introduction Arvados support for shell nodes allows you to use Arvados permissions to grant Linux shell accounts to users. -A shell node runs the @arvados-login-sync@ service, and has some additional configuration to make it convenient for users to use Arvados utilites and SDKs. Users are allowed to log in and run arbitrary programs. For optimal performance, the Arvados shell server should be on the same LAN as the Arvados cluster. +A shell node runs the @arvados-login-sync@ service to manage user accounts, and typically has Arvados utilities and SDKs pre-installed. Users are allowed to log in and run arbitrary programs. For optimal performance, the Arvados shell server should be on the same LAN as the Arvados cluster. -Because it _contains secrets_ shell nodes should *not* have a copy of the complete @config.yml@. For example, if users have access to the @docker@ daemon, it is trival to gain *root* access to any file on the system. Users sharing a shell node should be implicitly trusted, or not given access to Docker. In more secure environments, the admin should allocate a separate VM for each user. +Because it _contains secrets_ shell nodes should *not* have a copy of the Arvados @config.yml@. + +Shell nodes should be separate virtual machines from the VMs running other Arvados services. You may choose to grant root access to users so that they can customize the node, for example, installing new programs. This has security considerations depending on whether a shell node is single-user or multi-user. + +A single-user shell node should be set up so that it only stores Arvados access tokens that belong to that user. In that case, that user can be safely granted root access without compromising other Arvados users. + +In the multi-user shell node case, a malicious user with @root@ access could access other user's Arvados tokens. Users should only be given @root@ access on a multi-user shell node if you would trust them them to be Arvados administrators. Be aware that with access to the @docker@ daemon, it is trival to gain *root* access to any file on the system, so giving users @docker@ access should be considered equivalent to @root@ access. h2(#dependencies). Install Dependecies and SDKs @@ -52,51 +58,42 @@ Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate h2(#vm-record). Create record for VM -This program makes it possible for Arvados users to log in to the shell server -- subject to permissions assigned by the Arvados administrator -- using the SSH keys they upload to Workbench. It sets up login accounts, updates group membership, and adds users' public keys to the appropriate @authorized_keys@ files. - -Create an Arvados virtual_machine object representing this shell server. This will assign a UUID. +As an admin, create an Arvados virtual_machine object representing this shell server. This will return a uuid. 
-apiserver:~$ arv --format=uuid virtual_machine create --virtual-machine '{"hostname":"your.shell.server.hostname.without.domain"}'
+apiserver:~$ arv --format=uuid virtual_machine create --virtual-machine '{"hostname":"shell.ClusterID.example.com"}'
 zzzzz-2x53u-zzzzzzzzzzzzzzz
-h2(#scoped-token). Create scoped token +h2(#arvados-login-sync). Install arvados-login-sync + +The @arvados-login-sync@ service makes it possible for Arvados users to log in to the shell server. It sets up login accounts, updates group membership, adds each user's SSH public keys to the @~/.ssh/authorized_keys@ file, and adds an Arvados token to @~/.config/arvados/settings.conf@ . -As an Arvados admin user (such as the system root user), create a "scoped token":{{site.baseurl}}/admin/scoped-tokens.html that is permits only reading login information for this VM. Setting a scope on the token means that even though a user with root access on the shell node can access the token, the token is not usable for admin actions on Arvados. +Install the @arvados-login-sync@ program from RubyGems. 
-apiserver:~$ arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/zzzzz-2x53u-zzzzzzzzzzzzzzz/logins"]}'
-{
- ...
- "api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",
- ...
-}
+shellserver:# gem install arvados-login-sync
-Note the UUID and the API token output by the above commands: you will need them in a minute. +h2(#arvados-login-sync). Run arvados-login-sync periodically -h2(#arvados-login-sync). Install arvados-login-sync +Create a cron job to run the @arvados-login-sync@ program every 2 minutes. This will synchronize user accounts. -Install the arvados-login-sync program from RubyGems. +If this is a single-user shell node, then @ARVADOS_API_TOKEN@ should be a token for that user. See "Create a token for a user":{{site.baseurl}}/admin/user-management-cli.html#create-token . - -
-shellserver:# gem install arvados-login-sync
-
- +If this is a multi-user shell node, then @ARVADOS_API_TOKEN@ should be an administrator token such as the @SystemRootToken@. See discussion in the "introduction":#introduction about security on multi-user shell nodes. -Configure cron to run the @arvados-login-sync@ program every 2 minutes. +Set @ARVADOS_VIRTUAL_MACHINE_UUID@ to the UUID from "Create record for VM":#vm-record 
-shellserver:# umask 077; tee /etc/cron.d/arvados-login-sync <<EOF
+shellserver:# umask 0700; tee /etc/cron.d/arvados-login-sync <<EOF
 ARVADOS_API_HOST="ClusterID.example.com"
-ARVADOS_API_TOKEN="the_token_you_created_above"
+ARVADOS_API_TOKEN="xxxxxxxxxxxxxxxxx"
 ARVADOS_VIRTUAL_MACHINE_UUID="zzzzz-2x53u-zzzzzzzzzzzzzzz"
 */2 * * * * root arvados-login-sync
 EOF
@@ -107,8 +104,9 @@ h2(#confirm-working). Confirm working installation
 
 A user should be able to log in to the shell server when the following conditions are satisfied:
 
-# The user has uploaded an SSH public key: Workbench → Account menu → "SSH keys" item → "Add new SSH key" button.
 # As an admin user, you have given the user permission to log in using the Workbench → Admin menu → "Users" item → "Show" button → "Admin" tab → "Setup account" button.
 # The cron job has run.
 
+In order to log in via SSH, the user must also upload an SSH public key.  Alternately, if configured, users can log in using "Webshell":install-webshell.html .
+
 See also "how to add a VM login permission link at the command line":../admin/user-management-cli.html
diff --git a/doc/install/install-webshell.html.textile.liquid b/doc/install/install-webshell.html.textile.liquid
index ae6a8d2109..8275a2a831 100644
--- a/doc/install/install-webshell.html.textile.liquid
+++ b/doc/install/install-webshell.html.textile.liquid
@@ -65,7 +65,7 @@ server {
 
   location /shell.ClusterID {
     if ($request_method = 'OPTIONS') {
-       add_header 'Access-Control-Allow-Origin' '*'; 
+       add_header 'Access-Control-Allow-Origin' '*';
        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
        add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
        add_header 'Access-Control-Max-Age' 1728000;
@@ -146,7 +146,7 @@ SHELLINABOX_ARGS="--disable-ssl --no-beep --service=/she
 
 h2(#config-pam). Configure pam
 
-Use a text editor to create a new file @/etc/pam.d/shellinabox@ with the following configuration. Options that need attention are marked in red.
+Use a text editor to create a new file @/etc/pam.d/shellinabox@ with the following configuration.  Options that need attention are marked in red.
 
 
 # This example is a stock debian "login" file with pam_arvados
@@ -159,7 +159,11 @@ session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux
 session       required   pam_env.so readenv=1
 session       required   pam_env.so readenv=1 envfile=/etc/default/locale
 
+# The first argument is the address of the API server.  The second
+# argument is this shell node's hostname.  The hostname must match the
+# "hostname" field of the virtual_machine record.
 auth [success=1 default=ignore] /usr/lib/pam_arvados.so ClusterID.example.com shell.ClusterID.example.com
+
 auth    requisite            pam_deny.so
 auth    required            pam_permit.so
 
@@ -179,5 +183,4 @@ session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux
 
 h2(#confirm-working). Confirm working installation
 
-A user should be able to log in to the shell server, using webshell via workbench. Please refer to "Accessing an Arvados VM with Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html
-
+A user should now be able to log in to the shell server, using webshell via workbench. Please refer to "Accessing an Arvados VM with Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html
diff --git a/doc/install/new_cluster_checklist_AWS.xlsx b/doc/install/new_cluster_checklist_AWS.xlsx
index 5b98b8aab7..46cd9fdde4 100644
Binary files a/doc/install/new_cluster_checklist_AWS.xlsx and b/doc/install/new_cluster_checklist_AWS.xlsx differ
diff --git a/doc/install/new_cluster_checklist_Azure.xlsx b/doc/install/new_cluster_checklist_Azure.xlsx
index 1092a488ba..ba44c43aa5 100644
Binary files a/doc/install/new_cluster_checklist_Azure.xlsx and b/doc/install/new_cluster_checklist_Azure.xlsx differ
diff --git a/doc/install/new_cluster_checklist_slurm.xlsx b/doc/install/new_cluster_checklist_slurm.xlsx
index 4c9951f0c1..9843f74d17 100644
Binary files a/doc/install/new_cluster_checklist_slurm.xlsx and b/doc/install/new_cluster_checklist_slurm.xlsx differ
diff --git a/doc/install/packages.html.textile.liquid b/doc/install/packages.html.textile.liquid
index ed392b6667..cb7102bb37 100644
--- a/doc/install/packages.html.textile.liquid
+++ b/doc/install/packages.html.textile.liquid
@@ -42,7 +42,6 @@ As root, add the Arvados package repository to your sources.  This command depen
 table(table table-bordered table-condensed).
 |_. OS version|_. Command|
 |Debian 10 ("buster")|echo "deb http://apt.arvados.org/ buster main" | tee /etc/apt/sources.list.d/arvados.list|
-|Debian 9 ("stretch")|echo "deb http://apt.arvados.org/ stretch main" | tee /etc/apt/sources.list.d/arvados.list|
 |Ubuntu 18.04 ("bionic")[1]|echo "deb http://apt.arvados.org/ bionic main" | tee /etc/apt/sources.list.d/arvados.list|
 |Ubuntu 16.04 ("xenial")[1]|echo "deb http://apt.arvados.org/ xenial main" | tee /etc/apt/sources.list.d/arvados.list|
 
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
new file mode 100644
index 0000000000..4ba153faf9
--- /dev/null
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -0,0 +1,110 @@
+---
+layout: default
+navsection: installguide
+title: Multi host Arvados
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Install Saltstack":#saltstack
+# "Install dependencies":#dependencies
+# "Install Arvados using Saltstack":#saltstack
+# "DNS configuration":#final_steps
+# "Initial user and login":#initial_user
+
+h2(#saltstack). Install Saltstack
+
+If you already have a Saltstack environment you can skip this section.
+
+The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
+
+
+
curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+

+
+
+For more information check "Saltstack's documentation":https://docs.saltstack.com/en/latest/topics/installation/index.html
+
+h2(#dependencies). Install dependencies
+
+Arvados depends in a few applications and packages (postgresql, nginx+passenger, ruby) that can also be installed using their respective Saltstack formulas.
+
+The formulas we use are:
+
+* "postgres":https://github.com/saltstack-formulas/postgres-formula.git
+* "nginx":https://github.com/saltstack-formulas/nginx-formula.git
+* "docker":https://github.com/saltstack-formulas/docker-formula.git
+* "locale":https://github.com/saltstack-formulas/locale-formula.git
+
+There are example Salt pillar files for each of those formulas in the "arvados-formula's test/salt/pillar/examples":https://github.com/saltstack-formulas/arvados-formula/tree/master/test/salt/pillar/examples directory. As they are, they allow you to get all the main Arvados components up and running.
+
+h2(#saltstack). Install Arvados using Saltstack
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+The Arvados formula we maintain is located in the Saltstack's community repository of formulas:
+
+* "arvados-formula":https://github.com/saltstack-formulas/arvados-formula.git
+
+The @development@ version lives in our own repository
+
+* "arvados-formula development":https://github.com/arvados/arvados-formula.git
+
+This last one might break from time to time, as we try and add new features. Use with caution.
+
+As much as possible, we try to keep it up to date, with example pillars to help you deploy Arvados.
+
+For those familiar with Saltstack, the process to get it deployed is similar to any other formula:
+
+1. Fork/copy the formula to your Salt master host.
+2. Edit the Arvados, nginx, postgres, locale and docker pillars to match your desired configuration.
+3. Run a @state.apply@ to get it deployed.
+
+h2(#final_steps). DNS configuration
+
+After the setup is done, you need to set up your DNS to be able to access the cluster's nodes.
+
+The simplest way to do this is to add entries in the @/etc/hosts@ file of every host:
+
+
+
export CLUSTER="arva2"
+export DOMAIN="arv.local"
+
+echo A.B.C.a  api ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.b  keep keep.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.c  keep0 keep0.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.d  collections collections.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.e  download download.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.f  ws ws.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.g  workbench workbench.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.h  workbench2 workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+

+
+
+Replacing in each case de @A.B.C.x@ IP with the corresponding IP of the node.
+
+If your infrastructure uses another DNS service setup, add the corresponding entries accordingly.
+
+h2(#initial_user). Initial user and login
+
+At this point you should be able to log into the Arvados cluster.
+
+If you did not change the defaults, the initial URL will be:
+
+* https://workbench.arva2.arv.local
+
+or, in general, the url format will be:
+
+* https://workbench.@.@
+
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
+
+Assuming you didn't change the defaults, the initial credentials are:
+
+* User: 'admin'
+* Password: 'password'
+* Email: 'admin@arva2.arv.local'
diff --git a/doc/install/salt-single-host.html.textile.liquid b/doc/install/salt-single-host.html.textile.liquid
new file mode 100644
index 0000000000..5bed6d05e7
--- /dev/null
+++ b/doc/install/salt-single-host.html.textile.liquid
@@ -0,0 +1,215 @@
+---
+layout: default
+navsection: installguide
+title: Single host Arvados
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Install Saltstack":#saltstack
+# "Single host install using the provision.sh script":#single_host
+# "Final steps":#final_steps
+## "DNS configuration":#dns_configuration
+## "Install root certificate":#ca_root_certificate
+# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
+
+h2(#saltstack). Install Saltstack
+
+If you already have a Saltstack environment you can skip this section.
+
+The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
+
+
+
curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+

+
+
+For more information check "Saltstack's documentation":https://docs.saltstack.com/en/latest/topics/installation/index.html
+
+h2(#single_host). Single host install using the provision.sh script
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+Use the @provision.sh@ script to deploy Arvados, which is implemented with the @arvados-formula@ in a Saltstack master-less setup:
+
+* edit the variables at the very beginning of the file,
+* run the script as root
+* wait for it to finish
+
+This will install all the main Arvados components to get you up and running. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host and your network bandwidth. On a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
+
+If everything goes OK, you'll get some final lines stating something like:
+
+
+
arvados: Succeeded: 109 (changed=9)
+arvados: Failed:      0
+

+
+
+h2(#final_steps). Final configuration steps
+
+h3(#dns_configuration). DNS configuration
+
+After the setup is done, you need to set up your DNS to be able to access the cluster.
+
+The simplest way to do this is to edit your @/etc/hosts@ file (as root):
+
+
+
export CLUSTER="arva2"
+export DOMAIN="arv.local"
+export HOST_IP="127.0.0.2"    # This is valid either if installing in your computer directly
+                              # or in a Vagrant VM. If you're installing it on a remote host
+                              # just change the IP to match that of the host.
+echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+

+
+
+h3(#ca_root_certificate). Install root certificate
+
+Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+
+For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so ypu can add it to your workstation.
+
+Installing the root certificate into your web browser will prevent security errors when accessing Arvados services with your web browser.
+
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button.  Choose @arvados-snakeoil-ca.pem@
+
+The certificate will be added under the "Arvados Formula".
+
+To access your Arvados instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
+
+* On Debian/Ubuntu:
+
+
+
cp arvados-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+

+
+
+* On CentOS:
+
+
+
cp arvados-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+

+
+
+h2(#initial_user). Initial user and login
+
+At this point you should be able to log into the Arvados cluster.
+
+If you changed nothing in the @provision.sh@ script, the initial URL will be:
+
+* https://workbench.arva2.arv.local
+
+or, in general, the url format will be:
+
+* https://workbench.@.@
+
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
+
+Assuming you didn't change these values in the @provision.sh@ script, the initial credentials are:
+
+* User: 'admin'
+* Password: 'password'
+* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+The @provision.sh@ script saves a simple example test workflow in the @/tmp/cluster_tests@. If you want to run it, just change to that directory and run:
+
+
+
cd /tmp/cluster_tests
+./run-test.sh
+

+
+
+It will create a test user, upload a small workflow and run it. If everything goes OK, the output should similar to this (some output was shortened for clarity):
+
+
+
Creating Arvados Standard Docker Images project
+Arvados project uuid is 'arva2-j7d0g-0prd8cjlk6kfl7y'
+{
+ ...
+ "uuid":"arva2-o0j2j-n4zu4cak5iifq2a",
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+}
+Uploading arvados/jobs' docker image to the project
+2.1.1: Pulling from arvados/jobs
+8559a31e96f4: Pulling fs layer
+...
+Status: Downloaded newer image for arvados/jobs:2.1.1
+docker.io/arvados/jobs:2.1.1
+2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
+2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
+arva2-4zz18-1u5pvbld7cvxuy2
+Creating initial user ('admin')
+Setting up user ('admin')
+{
+ "items":[
+  {
+   ...
+   "owner_uuid":"arva2-tpzed-000000000000000",
+   ...
+   "uuid":"arva2-o0j2j-1ownrdne0ok9iox"
+  },
+  {
+   ...
+   "owner_uuid":"arva2-tpzed-000000000000000",
+   ...
+   "uuid":"arva2-o0j2j-1zbeyhcwxc1tvb7"
+  },
+  {
+   ...
+   "email":"admin@arva2.arv.local",
+   ...
+   "owner_uuid":"arva2-tpzed-000000000000000",
+   ...
+   "username":"admin",
+   "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+   ...
+  }
+ ],
+ "kind":"arvados#HashList"
+}
+Activating user 'admin'
+{
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+}
+Running test CWL workflow
+INFO /usr/bin/cwl-runner 2.1.1, arvados-python-client 2.1.1, cwltool 3.0.20200807132242
+INFO Resolved 'hasher-workflow.cwl' to 'file:///tmp/cluster_tests/hasher-workflow.cwl'
+...
+INFO Using cluster arva2 (https://arva2.arv.local:8443/)
+INFO Upload local files: "test.txt"
+INFO Uploaded to ea34d971b71d5536b4f6b7d6c69dc7f6+50 (arva2-4zz18-c8uvwqdry4r8jao)
+INFO Using collection cache size 256 MiB
+INFO [container hasher-workflow.cwl] submitted container_request arva2-xvhdp-v1bkywd58gyocwm
+INFO [container hasher-workflow.cwl] arva2-xvhdp-v1bkywd58gyocwm is Final
+INFO Overall process status is success
+INFO Final output collection d6c69a88147dde9d52a418d50ef788df+123
+{
+    "hasher_out": {
+        "basename": "hasher3.md5sum.txt",
+        "class": "File",
+        "location": "keep:d6c69a88147dde9d52a418d50ef788df+123/hasher3.md5sum.txt",
+        "size": 95
+    }
+}
+INFO Final process status is success
+

+
diff --git a/doc/install/salt-vagrant.html.textile.liquid b/doc/install/salt-vagrant.html.textile.liquid
new file mode 100644
index 0000000000..ed0d5bca62
--- /dev/null
+++ b/doc/install/salt-vagrant.html.textile.liquid
@@ -0,0 +1,127 @@
+---
+layout: default
+navsection: installguide
+title: Arvados in a VM with Vagrant
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Vagrant":#vagrant
+# "Final steps":#final_steps
+## "DNS configuration":#dns_configuration
+## "Install root certificate":#ca_root_certificate
+# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
+
+h2(#vagrant). Vagrant
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+A @Vagrantfile@ is provided to install Arvados in a virtual machine on your computer using "Vagrant":https://www.vagrantup.com/.
+
+To get it running, install Vagrant in your computer, edit the variables at the top of the @provision.sh@ script as needed, and run
+
+
+
vagrant up
+

+
+
+If you want to reconfigure the running box, you can just:
+
+1. edit the pillars to suit your needs
+2. run
+
+
+
vagrant reload --provision
+

+
+
+h2(#final_steps). Final configuration steps
+
+h3(#dns_configuration). DNS configuration
+
+After the setup is done, you need to set up your DNS to be able to access the cluster.
+
+The simplest way to do this is to edit your @/etc/hosts@ file (as root):
+
+
+
export CLUSTER="arva2"
+export DOMAIN="arv.local"
+export HOST_IP="127.0.0.2"    # This is valid either if installing in your computer directly
+                              # or in a Vagrant VM. If you're installing it on a remote host
+                              # just change the IP to match that of the host.
+echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+

+
+
+h3(#ca_root_certificate). Install root certificate
+
+Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+
+For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so ypu can add it to your workstation.
+
+Installing the root certificate into your web browser will prevent security errors when accessing Arvados services with your web browser.
+
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button.  Choose @arvados-snakeoil-ca.pem@
+
+The certificate will be added under the "Arvados Formula".
+
+To access your Arvados instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
+
+* On Debian/Ubuntu:
+
+
+
cp arvados-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+

+
+
+* On CentOS:
+
+
+
cp arvados-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+

+
+
+h2(#initial_user). Initial user and login
+
+At this point you should be able to log into the Arvados cluster.
+
+If you didn't change the defaults, the initial URL will be:
+
+* https://workbench.arva2.arv.local:8443
+
+or, in general, the url format will be:
+
+* https://workbench.@.:8443@
+
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
+
+Assuming you didn't change the defaults, the initial credentials are:
+
+* User: 'admin'
+* Password: 'password'
+* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+As documented in the Single Host installation page, You can run a test workflow to verify the installation finished correctly. To do so, you can follow these steps:
+
+
+
vagrant ssh

+
+
+and once in the instance:
+
+
+
cd /tmp/cluster_tests
+./run-test.sh
+

+
diff --git a/doc/install/salt.html.textile.liquid b/doc/install/salt.html.textile.liquid
new file mode 100644
index 0000000000..8f5ecc8c65
--- /dev/null
+++ b/doc/install/salt.html.textile.liquid
@@ -0,0 +1,29 @@
+---
+layout: default
+navsection: installguide
+title: Salt prerequisites
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Introduction":#introduction
+# "Choose an installation method":#installmethod
+
+h2(#introduction). Introduction
+
+To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://github.com/saltstack-formulas/arvados-formula which can help you get an Arvados cluster up and running.
+
+Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a master/minion setup or master-less.
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+h2(#installmethod). Choose an installation method
+
+The salt formulas can be used in different ways. Choose one of these three options to install Arvados:
+
+* "Use Vagrant to install Arvados in a virtual machine":salt-vagrant.html
+* "Arvados on a single host":salt-single-host.html
+* "Arvados across multiple hosts":salt-multi-host.html
diff --git a/doc/sdk/cli/install.html.textile.liquid b/doc/sdk/cli/install.html.textile.liquid
index 3c60bdfe3a..9657d236ad 100644
--- a/doc/sdk/cli/install.html.textile.liquid
+++ b/doc/sdk/cli/install.html.textile.liquid
@@ -17,7 +17,7 @@ h2. Prerequisites
 # "Install Ruby":../../install/ruby.html
 # "Install the Python SDK":../python/sdk-python.html
 
-The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 9 this is:
+The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 10 this is:
 
 
 $ apt-get install build-essential libcurl4-openssl-dev
diff --git a/doc/sdk/cli/reference.html.textile.liquid b/doc/sdk/cli/reference.html.textile.liquid
index e1d25aaa23..735ba5ca87 100644
--- a/doc/sdk/cli/reference.html.textile.liquid
+++ b/doc/sdk/cli/reference.html.textile.liquid
@@ -42,6 +42,8 @@ Get list of groups
 Delete a group
 @arv group delete --uuid 6dnxa-j7d0g-iw7i6n43d37jtog@
 
+Create an empty collection
+@arv collection create --collection '{"name": "test collection"}'@
 
 h3. Common commands
 
diff --git a/doc/sdk/go/example.html.textile.liquid b/doc/sdk/go/example.html.textile.liquid
index fd62bb67e0..688c45bf34 100644
--- a/doc/sdk/go/example.html.textile.liquid
+++ b/doc/sdk/go/example.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: sdk
-navmenu: Python
+navmenu: Go
 title: Examples
 ...
 {% comment %}
@@ -76,6 +76,6 @@ h2. Example program
 
 You can save this source as a .go file and run it:
 
-{% code 'example_sdk_go' as go %}
+{% code example_sdk_go as go %}
 
 A few more usage examples can be found in the "services/keepproxy":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/services/keepproxy and "sdk/go/keepclient":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/sdk/go/keepclient directories in the arvados source tree.
diff --git a/doc/sdk/java-v2/example.html.textile.liquid b/doc/sdk/java-v2/example.html.textile.liquid
index e73f968c8d..8d2fc2f4af 100644
--- a/doc/sdk/java-v2/example.html.textile.liquid
+++ b/doc/sdk/java-v2/example.html.textile.liquid
@@ -28,7 +28,7 @@ public class CollectionExample {
     public static void main(String[] argv) {
 	ConfigProvider conf = ExternalConfigProvider.builder().
 	    apiProtocol("https").
-	    apiHost("qr1hi.arvadosapi.com").
+	    apiHost("zzzzz.arvadosapi.com").
 	    apiPort(443).
 	    apiToken("...").
 	    build();
diff --git a/doc/sdk/python/arvados-cwl-runner.html.textile.liquid b/doc/sdk/python/arvados-cwl-runner.html.textile.liquid
new file mode 100644
index 0000000000..1cfbd60545
--- /dev/null
+++ b/doc/sdk/python/arvados-cwl-runner.html.textile.liquid
@@ -0,0 +1,71 @@
+---
+layout: default
+navsection: sdk
+navmenu: Python
+title: Arvados CWL Runner
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The Arvados FUSE driver is a Python utility that allows you to see the Keep service as a normal filesystem, so that data can be accessed using standard tools. This driver requires the Python SDK installed in order to access Arvados services.
+
+h2. Installation
+
+If you are logged in to a managed Arvados VM, the @arv-mount@ utility should already be installed.
+
+To use the FUSE driver elsewhere, you can install from a distribution package, or PyPI.
+
+h2. Option 1: Install from distribution packages
+
+First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/packages.html
+
+{% assign arvados_component = 'python3-arvados-cwl-runner' %}
+
+{% include 'install_packages' %}
+
+h2. Option 2: Install with pip
+
+Run @pip install arvados-cwl-runner@ in an appropriate installation environment, such as a virtualenv.
+
+Note:
+
+The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 10 this is:
+
+
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
+

+
+h3. Check Docker access
+
+In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker.  You do not need Docker if the Docker images you intend to use are already available in Arvados.
+
+You can determine if you have access to Docker by running @docker version@:
+
+
+
~$ docker version
+Client:
+ Version:      1.9.1
+ API version:  1.21
+ Go version:   go1.4.2
+ Git commit:   a34a1d5
+ Built:        Fri Nov 20 12:59:02 UTC 2015
+ OS/Arch:      linux/amd64
+
+Server:
+ Version:      1.9.1
+ API version:  1.21
+ Go version:   go1.4.2
+ Git commit:   a34a1d5
+ Built:        Fri Nov 20 12:59:02 UTC 2015
+ OS/Arch:      linux/amd64
+

+
+
+If this returns an error, contact the sysadmin of your cluster for assistance.
+
+h3. Usage
+
+Please refer to the "Accessing Keep from GNU/Linux":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-gnu-linux.html tutorial for more information.
diff --git a/doc/sdk/python/arvados-fuse.html.textile.liquid b/doc/sdk/python/arvados-fuse.html.textile.liquid
index 0ac2d0c7e1..04dca2c849 100644
--- a/doc/sdk/python/arvados-fuse.html.textile.liquid
+++ b/doc/sdk/python/arvados-fuse.html.textile.liquid
@@ -32,16 +32,10 @@ Run @pip install arvados_fuse@ in an appropriate installation environment, such
 
 Note:
 
-The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 9 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 10 this is:
 
 
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev python-llfuse
-

-
-For Python 3 this is:
-
-
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev python3-llfuse
 

 
 h3. Usage
diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index 75c51ee5a8..3aa01bbb56 100644
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -47,7 +47,7 @@ h2. Get input of a CWL workflow
 {% codeblock as python %}
 import arvados
 api = arvados.api()
-container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
 print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])
 {% endcodeblock %}
@@ -58,7 +58,7 @@ h2. Get output of a CWL workflow
 import arvados
 import arvados.collection
 api = arvados.api()
-container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
 collection = arvados.collection.CollectionReader(container_request["output_uuid"])
 print(collection.open("cwl.output.json").read())
@@ -89,7 +89,7 @@ def get_cr_state(cr_uuid):
         elif c['runtime_status'].get('warning', None):
             return 'Warning'
     return c['state']
-container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz'
+container_request_uuid = 'zzzzz-xvhdp-zzzzzzzzzzzzzzz'
 print(get_cr_state(container_request_uuid))
 {% endcodeblock %}
 
@@ -98,7 +98,7 @@ h2. List input of child requests
 {% codeblock as python %}
 import arvados
 api = arvados.api()
-parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
 parent_container_uuid = container_request["container_uuid"]
@@ -117,7 +117,7 @@ h2. List output of child requests
 {% codeblock as python %}
 import arvados
 api = arvados.api()
-parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 namefilter = "bwa%"  # the "like" filter uses SQL pattern match syntax
 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
 parent_container_uuid = container_request["container_uuid"]
@@ -136,7 +136,7 @@ h2. List failed child requests
 {% codeblock as python %}
 import arvados
 api = arvados.api()
-parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
 parent_container_uuid = container_request["container_uuid"]
 child_requests = api.container_requests().list(filters=[
@@ -155,7 +155,7 @@ h2. Get log of a child request
 import arvados
 import arvados.collection
 api = arvados.api()
-container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
 container_request = api.container_requests().get(uuid=container_request_uuid).execute()
 collection = arvados.collection.CollectionReader(container_request["log_uuid"])
 for c in collection:
@@ -169,7 +169,7 @@ h2(#sharing_link). Create a collection sharing link
 import arvados
 api = arvados.api()
 download="https://your.download.server"
-collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz"
+collection_uuid="zzzzz-4zz18-zzzzzzzzzzzzzzz"
 token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
     "GET /arvados/v1/collections/%s" % collection_uuid,
     "GET /arvados/v1/collections/%s/" % collection_uuid,
@@ -185,8 +185,8 @@ Note, if two collections have files of the same name, the contents will be conca
 import arvados
 import arvados.collection
 api = arvados.api()
-project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"
-collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"]
+project_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
+collection_uuids = ["zzzzz-4zz18-aaaaaaaaaaaaaaa", "zzzzz-4zz18-bbbbbbbbbbbbbbb"]
 combined_manifest = ""
 for u in collection_uuids:
     c = api.collections().get(uuid=u).execute()
@@ -201,7 +201,7 @@ h2. Upload a file into a new collection
 import arvados
 import arvados.collection
 
-project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz"
+project_uuid = "zzzzz-j7d0g-zzzzzzzzzzzzzzz"
 collection_name = "My collection"
 filename = "file1.txt"
 
@@ -223,7 +223,7 @@ h2. Download a file from a collection
 import arvados
 import arvados.collection
 
-collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz"
+collection_uuid = "zzzzz-4zz18-zzzzzzzzzzzzzzz"
 filename = "file1.txt"
 
 api = arvados.api()
@@ -257,3 +257,34 @@ for f in files_to_copy:
 target.save_new(name=target_name, owner_uuid=target_project)
 print("Created collection %s" % target.manifest_locator())
 {% endcodeblock %}
+
+h2. Copy files from a collection another collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "x1u39-4zz18-krzg64ufvehgitl"
+target_collection = "x1u39-4zz18-67q94einb8ptznm"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+                 "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection(target_collection)
+
+for f in files_to_copy:
+    target.copy(f, "", source_collection=source)
+
+target.save()
+{% endcodeblock %}
+
+h2. Listing records with paging
+
+Use the @arvados.util.keyset_list_all@ helper method to iterate over all the records matching an optional filter.  This method handles paging internally and returns results incrementally using a Python iterator.  The first parameter of the method takes a @list@ method of an Arvados resource (@collections@, @container_requests@, etc).
+
+{% codeblock as python %}
+import arvados.util
+
+api = arvados.api()
+for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
+    print("got collection " + c["uuid"])
+{% endcodeblock %}
diff --git a/doc/sdk/python/events.html.textile.liquid b/doc/sdk/python/events.html.textile.liquid
index afbec20d95..78fe9272bf 100644
--- a/doc/sdk/python/events.html.textile.liquid
+++ b/doc/sdk/python/events.html.textile.liquid
@@ -2,7 +2,7 @@
 layout: default
 navsection: sdk
 navmenu: Python
-title: Subscribing to events
+title: Subscribing to database events
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -13,7 +13,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 Arvados applications can subscribe to a live event stream from the database.  Events are described in the "Log resource.":{{site.baseurl}}/api/methods/logs.html
 
 {% codeblock as python %}
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import arvados
 import arvados.events
diff --git a/doc/sdk/python/sdk-python.html.textile.liquid b/doc/sdk/python/sdk-python.html.textile.liquid
index fa7c36c24b..e132305f0f 100644
--- a/doc/sdk/python/sdk-python.html.textile.liquid
+++ b/doc/sdk/python/sdk-python.html.textile.liquid
@@ -18,7 +18,7 @@ If you are logged in to an Arvados VM, the Python SDK should be installed.
 
 To use the Python SDK elsewhere, you can install from PyPI or a distribution package.
 
-The Python SDK supports Python 2.7 and 3.4+
+As of Arvados 2.1, the Python SDK requires Python 3.5+.  The last version to support Python 2.7 is Arvados 2.0.4.
 
 h2. Option 1: Install from a distribution package
 
@@ -26,7 +26,7 @@ This installation method is recommended to make the CLI tools available system-w
 
 First, configure the "Arvados package repositories":../../install/packages.html
 
-{% assign arvados_component = 'python-arvados-python-client' %}
+{% assign arvados_component = 'python3-arvados-python-client' %}
 
 {% include 'install_packages' %}
 
@@ -38,16 +38,10 @@ Run @pip install arvados-python-client@ in an appropriate installation environme
 
 Note:
 
-The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 9 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library.  To build the module you may have to first install additional packages.  On Debian 10 this is:
 
 
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev
-

-
-For Python 3 this is
-
-
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev
 

 
 If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip install --pre arvados-python-client@.
@@ -60,8 +54,8 @@ If you installed with pip (option 1, above):
 
 
 
~$ python
-Python 2.7.4 (default, Sep 26 2013, 03:20:26)
-[GCC 4.7.3] on linux2
+Python 3.7.3 (default, Jul 25 2020, 13:03:44)
+[GCC 8.3.0] on linux
 Type "help", "copyright", "credits" or "license" for more information.
 >>> import arvados
 >>> arvados.api('v1')
@@ -74,8 +68,8 @@ If you installed from a distribution package (option 2): the package includes a
 
 
~$ source /usr/share/python2.7/dist/python-arvados-python-client/bin/activate
 (python-arvados-python-client) ~$ python
-Python 2.7.4 (default, Sep 26 2013, 03:20:26)
-[GCC 4.7.3] on linux2
+Python 3.7.3 (default, Jul 25 2020, 13:03:44)
+[GCC 8.3.0] on linux
 Type "help", "copyright", "credits" or "license" for more information.
 >>> import arvados
 >>> arvados.api('v1')
@@ -87,8 +81,8 @@ Or alternatively, by using the Python executable from the virtualenv directly:
 
 
 
~$ /usr/share/python2.7/dist/python-arvados-python-client/bin/python
-Python 2.7.4 (default, Sep 26 2013, 03:20:26)
-[GCC 4.7.3] on linux2
+Python 3.7.3 (default, Jul 25 2020, 13:03:44)
+[GCC 8.3.0] on linux
 Type "help", "copyright", "credits" or "license" for more information.
 >>> import arvados
 >>> arvados.api('v1')
diff --git a/doc/sdk/ruby/example.html.textile.liquid b/doc/sdk/ruby/example.html.textile.liquid
index b8c0dcbb80..f2ea1c09df 100644
--- a/doc/sdk/ruby/example.html.textile.liquid
+++ b/doc/sdk/ruby/example.html.textile.liquid
@@ -55,7 +55,7 @@ first_repo = repos[:items][0]
 puts "UUID of first repo returned is #{first_repo[:uuid]}"
 {% endcodeblock %}
 
-UUID of first repo returned is qr1hi-s0uqq-b1bnybpx3u5temz
+UUID of first repo returned is zzzzz-s0uqq-b1bnybpx3u5temz
 
 h2. update
 
diff --git a/doc/sdk/ruby/index.html.textile.liquid b/doc/sdk/ruby/index.html.textile.liquid
index 6f06722d23..b3b97244ba 100644
--- a/doc/sdk/ruby/index.html.textile.liquid
+++ b/doc/sdk/ruby/index.html.textile.liquid
@@ -22,7 +22,7 @@ h3. Prerequisites
 
 # "Install Ruby":../../install/ruby.html
 
-The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 9 this is:
+The SDK uses @curl@ which depends on the @libcurl@ C library.  To build the module you may have to install additional packages.  On Debian 10 this is:
 
 
 $ apt-get install build-essential libcurl4-openssl-dev
diff --git a/doc/start/getting_started/firstpipeline.html.textile.liquid b/doc/start/getting_started/firstpipeline.html.textile.liquid
deleted file mode 100644
index 43369a3bbf..0000000000
--- a/doc/start/getting_started/firstpipeline.html.textile.liquid
+++ /dev/null
@@ -1,94 +0,0 @@
----
-layout: default
-navsection: start 
-title: Run your first pipeline in minutes
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-h2. LobSTR v3 
-
-In this quickstart guide, we'll run an existing pipeline with pre-existing data. Step-by-step instructions are shown below. You can follow along using your own local install or by using the Arvados Playground (any Google account can be used to log in).
-
-(For more information about this pipeline, see our detailed lobSTR guide).
-
-
-
-Tip: You may need to make your browser window bigger to see full-size images in the gallery above.
diff --git a/doc/start/getting_started/nextsteps.html.textile.liquid b/doc/start/getting_started/nextsteps.html.textile.liquid
deleted file mode 100644
index dd059ea8d4..0000000000
--- a/doc/start/getting_started/nextsteps.html.textile.liquid
+++ /dev/null
@@ -1,12 +0,0 @@
----
-layout: default
-navsection: start 
-title: Check out the User Guide 
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-Now that you've finished the Getting Started guide, check out the "User Guide":{{site.baseurl}}/user/index.html. The User Guide goes into more depth than the Getting Started guide, covers how to develop your own pipelines in addition to using pre-existing pipelines, covers the Arvados command line tools in addition to the Workbench graphical interface to Arvados, and can be referenced in any order.
diff --git a/doc/start/getting_started/publicproject.html.textile.liquid b/doc/start/getting_started/publicproject.html.textile.liquid
deleted file mode 100644
index 0fabad7aa7..0000000000
--- a/doc/start/getting_started/publicproject.html.textile.liquid
+++ /dev/null
@@ -1,133 +0,0 @@
----
-layout: default
-navsection: start
-title: Visit an Arvados Public Project
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-h2. Mason Lab - Pathomap / Ancestry Mapper (Public)
-
-You can see Arvados in action by accessing the Mason Lab - Pathomap / Ancestry Mapper (Public) project. By visiting this project, you can see what an Arvados project is, access data collections in this project, and click through a pipeline instance's contents.
-
-You will be accessing this project in read-only mode and will not be able to make any modifications such as running a new pipeline instance.
-
-
-
-Tip: You may need to make your browser window bigger to see full-size images in the gallery above.
diff --git a/doc/start/getting_started/sharedata.html.textile.liquid b/doc/start/getting_started/sharedata.html.textile.liquid
deleted file mode 100644
index 02e0b70329..0000000000
--- a/doc/start/getting_started/sharedata.html.textile.liquid
+++ /dev/null
@@ -1,102 +0,0 @@
----
-layout: default
-navsection: start 
-title: Sharing Data 
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-You can easily share data entirely through Workbench, the web interface to Arvados.
-
-h2. Upload and share your existing data
-
-Step-by-step instructions are shown below.
-
-
-
-Tip: You may need to make your browser window bigger to see full-size images in the gallery above.
diff --git a/doc/start/index.html.textile.liquid b/doc/start/index.html.textile.liquid
deleted file mode 100644
index cddfb8e441..0000000000
--- a/doc/start/index.html.textile.liquid
+++ /dev/null
@@ -1,133 +0,0 @@
----
-layout: default
-navsection: start 
-title: Welcome to Arvados!
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This guide provides an introduction to using Arvados to solve big data bioinformatics problems.
-
-h2. What is Arvados?
-
-Arvados is a free and open source bioinformatics platform for genomic and biomedical data.
-
-We address the needs of IT directors, lab principals, and bioinformaticians.
-
-h2. Why use Arvados?
-
-Arvados enables you to quickly begin using cloud computing resources in your bioinformatics work. It allows you to track your methods and datasets, share them securely, and easily re-run analyses.
-
-h3. Take a look (Screenshots gallery) 
-
-
-
-Note: Workbench is the web interface to Arvados.
-Tip: You may need to make your browser window bigger to see full-size images in the gallery above.
-
-h3. Key Features
-
-
    
-
  • Track your methods
    
-We log every compute job: software versions, machine images, input and output data hashes. Rely on a computer, not your memory and your note-taking skills.

    • 
-
  • Share your methods
    
-Show other people what you did. Let them use your workflow on their own data. Publish a permalink to your methods and data, so others can reproduce and build on them easily.

    • 
-
  • Track data origin
    
-Did you really only use fully consented public data in this analysis?

    • 
-
  • Get results sooner
    
-Run your compute jobs faster by using multi-nodes and multi-cores, even if your programs are single-threaded.

    • 
-

diff --git a/doc/user/composer/composer.html.textile.liquid b/doc/user/composer/composer.html.textile.liquid
index 400c55b976..b0ff824761 100644
--- a/doc/user/composer/composer.html.textile.liquid
+++ b/doc/user/composer/composer.html.textile.liquid
@@ -48,7 +48,7 @@ h3. 6. Create a new Command Line Tool
 
 h3. 7. Set Docker image, base command, and input port for "sort" tool
 
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:9@)  You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:10@)  You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
 
 !(screenshot)c6.png!
 
diff --git a/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml b/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml
index 73bd9f599c..73dd65c463 100755
--- a/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml
+++ b/doc/user/cwl/bwa-mem/bwa-mem-input-mixed.yml
@@ -15,15 +15,15 @@ cwl:tool: bwa-mem.cwl
 reference:
   class: File
   location: keep:2463fa9efeb75e099685528b3b9071e0+438/19.fasta.bwt
-  arv:collectionUUID: qr1hi-4zz18-pwid4w22a40jp8l
+  arv:collectionUUID: jutro-4zz18-tv416l321i4r01e
 read_p1:
   class: File
   location: keep:ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_1.fastq
-  arv:collectionUUID: qr1hi-4zz18-h615rgfmqt3wje0
+  arv:collectionUUID: jutro-4zz18-8k5hsvee0izv2g3
 read_p2:
   class: File
   location: keep:ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_2.fastq
-  arv:collectionUUID: qr1hi-4zz18-h615rgfmqt3wje0
+  arv:collectionUUID: jutro-4zz18-8k5hsvee0izv2g3
 group_id: arvados_tutorial
 sample_id: HWI-ST1027_129
 PL: illumina
diff --git a/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml b/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml
index 7e71e959eb..e76aa78173 100755
--- a/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml
+++ b/doc/user/cwl/bwa-mem/bwa-mem-input-uuids.yml
@@ -9,13 +9,13 @@
 cwl:tool: bwa-mem.cwl
 reference:
   class: File
-  location: keep:qr1hi-4zz18-pwid4w22a40jp8l/19.fasta.bwt
+  location: keep:jutro-4zz18-tv416l321i4r01e/19.fasta.bwt
 read_p1:
   class: File
-  location: keep:qr1hi-4zz18-h615rgfmqt3wje0/HWI-ST1027_129_D0THKACXX.1_1.fastq
+  location: keep:jutro-4zz18-8k5hsvee0izv2g3/HWI-ST1027_129_D0THKACXX.1_1.fastq
 read_p2:
   class: File
-  location: keep:qr1hi-4zz18-h615rgfmqt3wje0/HWI-ST1027_129_D0THKACXX.1_2.fastq
+  location: keep:jutro-4zz18-8k5hsvee0izv2g3/HWI-ST1027_129_D0THKACXX.1_2.fastq
 group_id: arvados_tutorial
 sample_id: HWI-ST1027_129
 PL: illumina
diff --git a/doc/user/cwl/bwa-mem/bwa-mem.cwl b/doc/user/cwl/bwa-mem/bwa-mem.cwl
index 2001971264..018867c83e 100755
--- a/doc/user/cwl/bwa-mem/bwa-mem.cwl
+++ b/doc/user/cwl/bwa-mem/bwa-mem.cwl
@@ -8,13 +8,13 @@ class: CommandLineTool
 
 hints:
   DockerRequirement:
-    dockerPull: lh3lh3/bwa
+    dockerPull: quay.io/biocontainers/bwa:0.7.17--ha92aebf_3
 
-baseCommand: [mem]
+baseCommand: [bwa, mem]
 
 arguments:
   - {prefix: "-t", valueFrom: $(runtime.cores)}
-  - {prefix: "-R", valueFrom: "@RG\tID:$(inputs.group_id)\tPL:$(inputs.PL)\tSM:$(inputs.sample_id)"}
+  - {prefix: "-R", valueFrom: '@RG\\\tID:$(inputs.group_id)\\\tPL:$(inputs.PL)\\\tSM:$(inputs.sample_id)'}
 
 inputs:
   reference:
diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid
index 505cfc4f59..09a553becf 100644
--- a/doc/user/cwl/cwl-extensions.html.textile.liquid
+++ b/doc/user/cwl/cwl-extensions.html.textile.liquid
@@ -127,7 +127,7 @@ This is an optional extension field appearing on the standard @DockerRequirement
 
 requirements:
   DockerRequirement:
-    dockerPull: "debian:9"
+    dockerPull: "debian:10"
     arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
 

 
diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid
index 725528f44d..761d198ee4 100644
--- a/doc/user/cwl/cwl-run-options.html.textile.liquid
+++ b/doc/user/cwl/cwl-run-options.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Using arvados-cwl-runner"
+title: "arvados-cwl-runner options"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -74,10 +74,10 @@ Use the @--name@ and @--output-name@ options to specify the name of the workflow
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "Example bwa run" --output-name "Example bwa output" bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
 2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
 {
     "aligned_sam": {
@@ -98,9 +98,9 @@ To submit a workflow and exit immediately, use the @--no-wait@ option.  This wil
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --no-wait bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 15:07:52 arvados.arv-run[12480] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 15:07:52 arvados.arv-run[12480] INFO: Uploaded to qr1hi-4zz18-eqnfwrow8aysa9q
-2016-06-30 15:07:52 arvados.cwl-runner[12480] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-qr1hi-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 15:07:52 arvados.arv-run[12480] INFO: Uploaded to zzzzz-4zz18-eqnfwrow8aysa9q
+2016-06-30 15:07:52 arvados.cwl-runner[12480] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+zzzzz-8i9sb-fm2n3b1w0l6bskg
 

 
 
@@ -111,10 +111,10 @@ To run a workflow with local control, use @--local@.  This means that the host w
 
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --local bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 10:05:19 arvados.cwl-runner[16290] INFO: Pipeline instance qr1hi-d1hrv-92wcu6ldtio74r4
-2016-07-01 10:05:28 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-2nzzfbuf9zjrj4g) is Queued
-2016-07-01 10:05:29 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-2nzzfbuf9zjrj4g) is Running
-2016-07-01 10:05:45 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-2nzzfbuf9zjrj4g) is Complete
+2016-07-01 10:05:19 arvados.cwl-runner[16290] INFO: Pipeline instance zzzzz-d1hrv-92wcu6ldtio74r4
+2016-07-01 10:05:28 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-2nzzfbuf9zjrj4g) is Queued
+2016-07-01 10:05:29 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-2nzzfbuf9zjrj4g) is Running
+2016-07-01 10:05:45 arvados.cwl-runner[16290] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-2nzzfbuf9zjrj4g) is Complete
 2016-07-01 10:05:46 arvados.cwl-runner[16290] INFO: Overall process status is success
 {
     "aligned_sam": {
diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid
index 2be803b52a..442a60b04f 100644
--- a/doc/user/cwl/cwl-runner.html.textile.liquid
+++ b/doc/user/cwl/cwl-runner.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Running an Arvados workflow"
+title: "Starting a Workflow at the Command Line"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -13,44 +13,38 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 {% include 'tutorial_expectations' %}
 
-{% include 'notebox_begin' %}
-
-By default, the @arvados-cwl-runner@ is installed on Arvados shell nodes.  If you want to submit jobs from somewhere else, such as your workstation, you may install "arvados-cwl-runner.":#setup
-
-{% include 'notebox_end' %}
-
 This tutorial will demonstrate how to submit a workflow at the command line using @arvados-cwl-runner@.
 
-h2. Running arvados-cwl-runner
+# "Get the tutorial files":#get-files
+# "Submitting a workflow to an Arvados cluster":#submitting
+# "Registering a workflow to use in Workbench":#registering
+# "Make a workflow file directly executable":#executable
 
-h3. Get the example files
+h2(#get-files). Get the tutorial files
 
-The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/arvados/arvados/tree/master/doc/user/cwl/bwa-mem
+The tutorial files are located in the documentation section of the Arvados source repository, which can be found on "git.arvados.org":https://git.arvados.org/arvados.git/tree/HEAD:/doc/user/cwl/bwa-mem or "github":https://github.com/arvados/arvados/tree/master/doc/user/cwl/bwa-mem
 
 
-
~$ git clone https://github.com/arvados/arvados
+
~$ git clone https://git.arvados.org/arvados.git
 ~$ cd arvados/doc/user/cwl/bwa-mem
 

 
 
-The tutorial data is hosted on "https://playground.arvados.org":https://playground.arvados.org (also referred to by the identifier *qr1hi*).  If you are using a different Arvados instance, you may need to copy the data to your own instance.  The easiest way to do this is with "arv-copy":{{site.baseurl}}/user/topics/arv-copy.html (this requires signing up for a free playground.arvados.org account).
+The tutorial data is hosted on "https://playground.arvados.org":https://playground.arvados.org (also referred to by the identifier *pirca*).  If you are using a different Arvados instance, you may need to copy the data to your own instance.  One way to do this is with "arv-copy":{{site.baseurl}}/user/topics/arv-copy.html (this requires signing up for a free playground.arvados.org account).
 
 
-
~$ arv-copy --src qr1hi --dst settings 2463fa9efeb75e099685528b3b9071e0+438
-~$ arv-copy --src qr1hi --dst settings ae480c5099b81e17267b7445e35b4bc7+180
-~$ arv-copy --src qr1hi --dst settings 655c6cd07550151b210961ed1d3852cf+57
+
~$ arv-copy --src pirca --dst settings 2463fa9efeb75e099685528b3b9071e0+438
+~$ arv-copy --src pirca --dst settings ae480c5099b81e17267b7445e35b4bc7+180
 

 
 
 If you do not wish to create an account on "https://playground.arvados.org":https://playground.arvados.org, you may download the files anonymously and upload them to your local Arvados instance:
 
-"https://playground.arvados.org/collections/2463fa9efeb75e099685528b3b9071e0+438":https://playground.arvados.org/collections/2463fa9efeb75e099685528b3b9071e0+438
-
-"https://playground.arvados.org/collections/ae480c5099b81e17267b7445e35b4bc7+180":https://playground.arvados.org/collections/ae480c5099b81e17267b7445e35b4bc7+180
+"https://collections.pirca.arvadosapi.com/c=2463fa9efeb75e099685528b3b9071e0+438/":https://collections.pirca.arvadosapi.com/c=2463fa9efeb75e099685528b3b9071e0+438/
 
-"https://playground.arvados.org/collections/655c6cd07550151b210961ed1d3852cf+57":https://playground.arvados.org/collections/655c6cd07550151b210961ed1d3852cf+57
+"https://collections.pirca.arvadosapi.com/c=ae480c5099b81e17267b7445e35b4bc7+180/":https://collections.pirca.arvadosapi.com/c=ae480c5099b81e17267b7445e35b4bc7+180/
 
-h2. Submitting a workflow to an Arvados cluster
+h2(#submitting). Submitting a workflow to an Arvados cluster
 
 h3. Submit a workflow and wait for results
 
@@ -62,10 +56,10 @@ Use @arvados-cwl-runner@ to submit CWL workflows to Arvados.  After submitting t
 
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
 2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
 2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
 {
     "aligned_sam": {
@@ -88,15 +82,19 @@ If you reference a file in "arv-mount":{{site.baseurl}}/user/tutorials/tutorial-
 
 If you reference a local file which is not in @arv-mount@, then @arvados-cwl-runner@ will upload the file to Keep and use the Keep URI reference from the upload.
 
-You can also execute CWL files directly from Keep:
+You can also execute CWL files that have been uploaded Keep:
 
 
-
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner keep:655c6cd07550151b210961ed1d3852cf+57/bwa-mem.cwl bwa-mem-input.yml
+

+~/arvados/doc/user/cwl/bwa-mem$ arv-put --portable-data-hash --name "bwa-mem.cwl" bwa-mem.cwl
+2020-08-20 13:40:02 arvados.arv_put[12976] INFO: Collection saved as 'bwa-mem.cwl'
+f141fc27e7cfa7f7b6d208df5e0ee01b+59
+~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner keep:f141fc27e7cfa7f7b6d208df5e0ee01b+59/bwa-mem.cwl bwa-mem-input.yml
 arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
 2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
 {
     "aligned_sam": {
@@ -109,50 +107,128 @@ arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107,
 

 
 
+Note: uploading a workflow file to Keep is _not_ the same as registering the workflow for use in Workbench.  See "Registering a workflow to use in Workbench":#registering below.
+
 h3. Work reuse
 
 Workflows submitted with @arvados-cwl-runner@ will take advantage of Arvados job reuse.  If you submit a workflow which is identical to one that has run before, it will short cut the execution and return the result of the previous run.  This also applies to individual workflow steps.  For example, a two step workflow where the first step has run before will reuse results for first step and only execute the new second step.  You can disable this behavior with @--disable-reuse@.
 
 h3. Command line options
 
-See "Using arvados-cwl-runner":{{site.baseurl}}/user/cwl/cwl-run-options.html
+See "arvados-cwl-runner options":{{site.baseurl}}/user/cwl/cwl-run-options.html
 
-h2(#setup). Setting up arvados-cwl-runner
+h2(#registering). Registering a workflow to use in Workbench
 
-By default, the @arvados-cwl-runner@ is installed on Arvados shell nodes.  If you want to submit jobs from somewhere else, such as your workstation, you may install @arvados-cwl-runner@ using @pip@:
+Use @--create-workflow@ to register a CWL workflow with Arvados.  This enables you to share workflows with other Arvados users, and run them by clicking the  Run a process... button on the Workbench Dashboard and on the command line by UUID.
 
 
-
~$ virtualenv ~/venv
-~$ . ~/venv/bin/activate
-~$ pip install -U setuptools
-~$ pip install arvados-cwl-runner
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --create-workflow bwa-mem.cwl
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to zzzzz-4zz18-7e0hedrmkuyoei3
+2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template zzzzz-p5p6p-rjleou1dwr167v5
+zzzzz-p5p6p-rjleou1dwr167v5
 

 
 
-h3. Check Docker access
+You can provide a partial input file to set default values for the workflow input parameters.  You can also use the @--name@ option to set the name of the workflow:
 
-In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker.  You do not need Docker if the Docker images you intend to use are already available in Arvados.
+
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
+2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to zzzzz-4zz18-0f91qkovk4ml18o
+2016-07-01 14:09:50 arvados.cwl-runner[3730] INFO: Created template zzzzz-p5p6p-0deqe6nuuyqns2i
+zzzzz-p5p6p-zuniv58hn8d0qd8
+

+
 
-You can determine if you have access to Docker by running @docker version@:
+h3. Running registered workflows at the command line
+
+You can run a registered workflow at the command line by its UUID:
 
 
-
~$ docker version
-Client:
- Version:      1.9.1
- API version:  1.21
- Go version:   go1.4.2
- Git commit:   a34a1d5
- Built:        Fri Nov 20 12:59:02 UTC 2015
- OS/Arch:      linux/amd64
-
-Server:
- Version:      1.9.1
- API version:  1.21
- Go version:   go1.4.2
- Git commit:   a34a1d5
- Built:        Fri Nov 20 12:59:02 UTC 2015
- OS/Arch:      linux/amd64
+
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner pirca-7fd4e-3nqbw08vtjl8ybz --help
+INFO /home/peter/work/scripts/venv3/bin/arvados-cwl-runner 2.1.0.dev20200814195416, arvados-python-client 2.1.0.dev20200814195416, cwltool 3.0.20200807132242
+INFO Resolved 'pirca-7fd4e-3nqbw08vtjl8ybz' to 'arvwf:pirca-7fd4e-3nqbw08vtjl8ybz#main'
+usage: pirca-7fd4e-3nqbw08vtjl8ybz [-h] [--PL PL] [--group_id GROUP_ID]
+                                   [--read_p1 READ_P1] [--read_p2 READ_P2]
+                                   [--reference REFERENCE]
+                                   [--sample_id SAMPLE_ID]
+                                   [job_order]
+
+positional arguments:
+  job_order             Job input json file
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --PL PL
+  --group_id GROUP_ID
+  --read_p1 READ_P1     The reads, in fastq format.
+  --read_p2 READ_P2     For mate paired reads, the second file (optional).
+  --reference REFERENCE
+                        The index files produced by `bwa index`
+  --sample_id SAMPLE_ID
 

 
 
-If this returns an error, contact the sysadmin of your cluster for assistance.
+h2(#executable). Make a workflow file directly executable
+
+You can make a workflow file directly executable (@cwl-runner@ should be an alias to @arvados-cwl-runner@) by adding the following line to the top of the file:
+
+
+
#!/usr/bin/env cwl-runner
+

+
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem.cwl bwa-mem-input.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
+{
+    "aligned_sam": {
+        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
+        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
+        "class": "File",
+        "size": 30738986
+    }
+}
+

+
+
+You can even make an input file directly executable the same way with the following two lines at the top:
+
+
+
#!/usr/bin/env cwl-runner
+cwl:tool: bwa-mem.cwl
+

+
+
+
+
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem-input.yml
+arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
+2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to zzzzz-4zz18-h7ljh5u76760ww2
+2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job zzzzz-8i9sb-fm2n3b1w0l6bskg
+2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Running
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (zzzzz-8i9sb-fm2n3b1w0l6bskg) is Complete
+2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
+{
+    "aligned_sam": {
+        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
+        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
+        "class": "File",
+        "size": 30738986
+    }
+}
+

+
+
+h2(#setup). Setting up arvados-cwl-runner
+
+See "Arvados CWL Runner":{{site.baseurl}}/sdk/python/arvados-cwl-runner.html
diff --git a/doc/user/cwl/cwl-style.html.textile.liquid b/doc/user/cwl/cwl-style.html.textile.liquid
index ee36014cb5..bd07161ce3 100644
--- a/doc/user/cwl/cwl-style.html.textile.liquid
+++ b/doc/user/cwl/cwl-style.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: Writing Portable High-Performance Workflows
+title: Guidelines for Writing High-Performance Portable Workflows
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
diff --git a/doc/user/cwl/cwl-versions.html.textile.liquid b/doc/user/cwl/cwl-versions.html.textile.liquid
index 5fcfcbe3bc..ac679dc154 100644
--- a/doc/user/cwl/cwl-versions.html.textile.liquid
+++ b/doc/user/cwl/cwl-versions.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: CWL version and API support
+title: CWL version support
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,6 +9,8 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
+Arvados supports CWL v1.0, v1.1 and v1.2.
+
 h2(#v12). Upgrading your workflows to CWL v1.2
 
 If you are starting from a CWL v1.0 document, see "Upgrading your workflows to CWL v1.1":#v11 below.
diff --git a/doc/user/getting_started/check-environment.html.textile.liquid b/doc/user/getting_started/check-environment.html.textile.liquid
index b707891a1e..1097e4e9d8 100644
--- a/doc/user/getting_started/check-environment.html.textile.liquid
+++ b/doc/user/getting_started/check-environment.html.textile.liquid
@@ -16,14 +16,14 @@ Check that you are able to access the Arvados API server using @arv user current
 
 
$ arv user current
 {
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/users/qr1hi-xioed-9z2p3pn12yqdaem",
+ "href":"https://zzzzz.arvadosapi.com/arvados/v1/users/zzzzz-xioed-9z2p3pn12yqdaem",
  "kind":"arvados#user",
  "etag":"8u0xwb9f3otb2xx9hto4wyo03",
- "uuid":"qr1hi-tpzed-92d3kxnimy3d4e8",
- "owner_uuid":"qr1hi-tpqed-23iddeohxta2r59",
+ "uuid":"zzzzz-tpzed-92d3kxnimy3d4e8",
+ "owner_uuid":"zzzzz-tpqed-23iddeohxta2r59",
  "created_at":"2013-12-02T17:05:47Z",
- "modified_by_client_uuid":"qr1hi-xxfg8-owxa2oa2s33jyej",
- "modified_by_user_uuid":"qr1hi-tpqed-23iddeohxta2r59",
+ "modified_by_client_uuid":"zzzzz-xxfg8-owxa2oa2s33jyej",
+ "modified_by_user_uuid":"zzzzz-tpqed-23iddeohxta2r59",
  "modified_at":"2013-12-02T17:07:08Z",
  "updated_at":"2013-12-05T19:51:08Z",
  "email":"you@example.com",
diff --git a/doc/user/getting_started/ssh-access-unix.html.textile.liquid b/doc/user/getting_started/ssh-access-unix.html.textile.liquid
index 284d0a1f04..80cb391314 100644
--- a/doc/user/getting_started/ssh-access-unix.html.textile.liquid
+++ b/doc/user/getting_started/ssh-access-unix.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This document is for accessing an Arvados VM using SSH keys in Unix environments (Linux, OS X, Cygwin). If you would like to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page. If you are using a Windows environment, please visit the "Accessing an Arvados VM with SSH - Windows Environments":ssh-access-windows.html page.
+This document is for accessing an Arvados VM using SSH keys in Unix-like environments (Linux, macOS, Cygwin, Windows Subsystem for Linux). If you would like to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page. If you are using a Windows environment, please visit the "Accessing an Arvados VM with SSH - Windows Environments":ssh-access-windows.html page.
 
 {% include 'ssh_intro' %}
 
@@ -49,7 +49,7 @@ ssh-rsa AAAAB3NzaC1ycEDoNotUseExampleKeyDoNotUseExampleKeyDoNotUseExampleKeyDoNo
 
 Now you can set up @ssh-agent@ (next) or proceed with "adding your key to the Arvados Workbench.":#workbench
 
-h3. Set up ssh-agent (recommended)
+h3. Set up ssh-agent (optional)
 
 If you find you are entering your passphrase frequently, you can use @ssh-agent@ to manage your credentials.  Use @ssh-add -l@ to test if you already have ssh-agent running:
 
@@ -80,11 +80,21 @@ When everything is set up, @ssh-add -l@ should yield output that looks something
 
 {% include 'ssh_addkey' %}
 
-h3. Connecting to the virtual machine
+h3. Connecting directly
 
-Use the following command to connect to the _shell_ VM instance as _you_.  Replace *you@shell* at the end of the following command with your *login* and *hostname* from Workbench:
+If the VM is available on the public Internet (or you are on the same private network as the VM) you can connect directly with @ssh@.  You can probably copy-and-paste the text from *Command line* column directly into a terminal.
 
-notextile. 
$ ssh -o "ProxyCommand ssh -p2222 turnout@switchyard.{{ site.arvados_api_host }} -x -a shell" -x you@shell

+Use the following example command to connect as _you_ to the _shell.ClusterID.example.com_ VM instance.  Replace *you@shell.ClusterID.example.com* at the end of the following command with your *login* and *hostname* from Workbench.
+
+notextile. 
$ ssh you@shell.ClusterID.example.com

+
+h3. Connecting through switchyard
+
+Some Arvados installations use "switchyard" to isolate shell VMs from the public Internet.
+
+Use the following example command to connect to the _shell_ VM instance as _you_.  Replace *you@shell* at the end of the following command with your *login* and *hostname* from Workbench:
+
+notextile. 
$ ssh -o "ProxyCommand ssh -p2222 turnout@switchyard.ClusterID.example.com -x -a shell" -x you@shell

 
 This command does several things at once. You usually cannot log in directly to virtual machines over the public Internet.  Instead, you log into a "switchyard" server and then tell the switchyard which virtual machine you want to connect to.
 
@@ -99,7 +109,7 @@ This command does several things at once. You usually cannot log in directly to
 
 You should now be able to log into the Arvados VM and "check your environment.":check-environment.html
 
-h3. Configuration (recommended)
+h4. Configuration (recommended)
 
 The command line above is cumbersome, but you can configure SSH to remember many of these settings.  Add this text to the file @.ssh/config@ in your home directory (create a new file if @.ssh/config@ doesn't exist):
 
diff --git a/doc/user/getting_started/ssh-access-windows.html.textile.liquid b/doc/user/getting_started/ssh-access-windows.html.textile.liquid
index 0406e7c03b..5cbe2a3285 100644
--- a/doc/user/getting_started/ssh-access-windows.html.textile.liquid
+++ b/doc/user/getting_started/ssh-access-windows.html.textile.liquid
@@ -9,13 +9,13 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This document is for accessing an Arvados VM using SSH keys in Windows environments. If you would like to use to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page. If you are using a Unix environment (Linux, OS X, Cygwin), please visit the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.
+This document is for accessing an Arvados VM using SSH keys in Windows environments using PuTTY.  If you would like to use to access VM through your browser, please visit the "Accessing an Arvados VM with Webshell":vm-login-with-webshell.html page.  If you are using a Unix-like environment (Linux, macOS, Cygwin, or Windows Subsystem for Linux), please visit the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.
 
 {% include 'ssh_intro' %}
 
 h1(#gettingkey). Getting your SSH key
 
-(Note: if you are using the SSH client that comes with "Cygwin":http://cygwin.com, please use instructions found in the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.)
+(Note: If you are using the SSH client that comes with "Cygwin":http://cygwin.com or Windows Subsystem for Linux (WSL) please use instructions found in the "Accessing an Arvados VM with SSH - Unix Environments":ssh-access-unix.html page.)
 
 We will be using PuTTY to connect to Arvados. "PuTTY":http://www.chiark.greenend.org.uk/~sgtatham/putty/ is a free (MIT-licensed) Win32 Telnet and SSH client. PuTTY includes all the tools a Windows user needs to create private keys and make SSH connections to your virtual machines in the Arvados Cloud.
 
@@ -57,6 +57,16 @@ Pageant is a PuTTY utility that manages your private keys so is not necessary to
 
 h3. Initial configuration
 
+h4. Connecting directly
+
+# Open PuTTY from the Start Menu.
+# On the Session screen set the Host Name (or IP address) to “shell.ClusterID.example.com”, which is the hostname listed in the _Virtual Machines_ page.
+# On the Session screen set the Port to “22”.
+# On the Connection %(rarr)→% Data screen set the Auto-login username to the username listed in the *Login name* column on the Arvados Workbench Virtual machines_ page.
+# Return to the Session screen. In the Saved Sessions box, enter a name for this configuration and click Save.
+
+h4. Connecting through switchyard
+
 # Open PuTTY from the Start Menu.
 # On the Session screen set the Host Name (or IP address) to “shell”, which is the hostname listed in the _Virtual Machines_ page.
 # On the Session screen set the Port to “22”.
diff --git a/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid b/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
index 551002e55e..0aeabab11b 100644
--- a/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
+++ b/doc/user/getting_started/vm-login-with-webshell.html.textile.liquid
@@ -15,7 +15,11 @@ h2(#webshell). Access VM using webshell
 
 Webshell gives you access to an arvados virtual machine from your browser with no additional setup.
 
-In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Virtual machines* to see the list of virtual machines you can access.  If you do not have access to any virtual machines, please click on Send request for shell access or send an email to "support@curoverse.com":mailto:support@curoverse.com.
+{% include 'notebox_begin' %}
+Some Arvados clusters may not have webshell set up.  If you do not see a "Log in" button or "web shell" column, you will have to follow the "Unix":ssh-access-unix.html or "Windows":ssh-access-windows.html @ssh@ instructions.
+{% include 'notebox_end' %}
+
+In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Virtual machines* to see the list of virtual machines you can access.  If you do not have access to any virtual machines,  please click on Send request for shell access (if present) or contact your system administrator.  For the Arvados Playground, this is "info@curii.com":mailto:info@curii.com .
 
 Each row in the Virtual Machines panel lists the hostname of the VM, along with a Log in as *you* button under the column "Web shell". Clicking on this button will open up a webshell terminal for you in a new browser tab and log you in.
 
diff --git a/doc/user/getting_started/workbench.html.textile.liquid b/doc/user/getting_started/workbench.html.textile.liquid
index fc704227e0..644cf7d208 100644
--- a/doc/user/getting_started/workbench.html.textile.liquid
+++ b/doc/user/getting_started/workbench.html.textile.liquid
@@ -9,14 +9,26 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-If you are using the default Arvados instance for this guide, you can Access Arvados Workbench using this link:
+{% include 'notebox_begin' %}
+This guide covers the classic Arvados Workbench web application, sometimes referred to as "Workbench 1".  There is also a new Workbench web application under development called "Workbench 2".  Sites which have both Workbench applications installed will have a dropdown menu option "Switch to Workbench 2" to switch between versions.
 
-{{site.arvados_workbench_host}}/
+This guide will be updated to cover "Workbench 2" in the future.
+{% include 'notebox_end' %}
 
-(If you are using a different Arvados instance than the default for this guide, replace *{{ site.arvados_workbench_host }}* with your private instance in all of the examples in this guide.)
+You can access the Arvados Workbench used in this guide using this link:
 
-You may be asked to log in using a Google account.  Arvados uses only your name and email address from Google services for identification, and will never access any personal information.  If you are accessing Arvados for the first time, the Workbench may indicate your account status is *New / inactive*.  If this is the case, contact the administrator of the Arvados instance to request activation of your account.
+{{site.arvados_workbench_host}}
 
-Once your account is active, logging in to the Workbench will present you with the Dashboard. This gives a summary of your projects and recent activity in the Arvados instance.  "You are now ready to run your first pipeline.":{{ site.baseurl }}/user/tutorials/tutorial-workflow-workbench.html
+If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
+
+h2. Playground
+
+Curii operates a public demonstration instance of Arvados called the Arvados Playground, which can be found at https://playground.arvados.org .  Some examples in this guide involve getting data from the Playground instance.
+
+h2. Logging in
+
+You will be asked to log in.  Arvados uses only your name and email address for identification, and will never access any personal information.  If you are accessing Arvados for the first time, the Workbench may indicate your account status is *New / inactive*.  If this is the case, contact the administrator of the Arvados instance to request activation of your account.
+
+Once your account is active, logging in to the Workbench will present you with the Dashboard. This gives a summary of your projects and recent activity in the Arvados instance.  You are now ready to "upload data":{{ site.baseurl }}/user/tutorials/tutorial-keep.html or "run your first workflow.":{{ site.baseurl }}/user/tutorials/tutorial-workflow-workbench.html
 
 !{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/workbench-dashboard.png!
diff --git a/doc/user/index.html.textile.liquid b/doc/user/index.html.textile.liquid
index 909394ef47..e24afc9a44 100644
--- a/doc/user/index.html.textile.liquid
+++ b/doc/user/index.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: Welcome to Arvados™!
+title: Welcome to Arvados™!
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,14 +9,11 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This guide provides a reference for using Arvados to solve scientific big data problems, including:
+Arvados is an "open source":copying/copying.html platform for managing, processing, and sharing genomic and other large scientific and biomedical data.  With Arvados, bioinformaticians run and scale compute-intensive workflows, developers create biomedical applications, and IT administrators manage large compute and storage resources.
 
-* Robust storage of very large files, such as whole genome sequences, using the "Arvados Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html content-addressable cluster file system.
-* Running compute-intensive scientific analysis pipelines, such as genomic alignment and variant calls using the "Arvados Crunch":{{site.baseurl}}/user/tutorials/intro-crunch.html cluster compute engine.
-* Accessing, organizing, and sharing data, workflows and results using the "Arvados Workbench":{{site.baseurl}}/user/getting_started/workbench.html web application.
-* Running an analysis using multiple clusters (HPC, cloud, or hybrid) with "Federated Multi-Cluster Workflows":{{site.baseurl}}/user/cwl/federated-workflows.html .
+This guide provides a reference for using Arvados to solve scientific big data problems.
 
-The examples in this guide use the public Arvados instance located at {{site.arvados_workbench_host}}.  If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
+The examples in this guide use the Arvados instance located at {{site.arvados_workbench_host}}.  If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
 
 h2. Typographic conventions
 
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 0f0e40be9c..d35df4fcec 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -9,103 +9,74 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-{% include 'crunch1only_begin' %}
-On those sites, the "copy a pipeline template" feature described below is not available. However, "copy a workflow" feature is not yet implemented.
-{% include 'crunch1only_end' %}
-
 This tutorial describes how to copy Arvados objects from one cluster to another by using @arv-copy@.
 
 {% include 'tutorial_expectations' %}
 
 h2. arv-copy
 
-@arv-copy@ allows users to copy collections and pipeline templates from one cluster to another. By default, @arv-copy@ will recursively go through a template and copy all dependencies associated with the object.
+@arv-copy@ allows users to copy collections and workflows from one cluster to another. By default, @arv-copy@ will recursively go through the workflow and copy all dependencies associated with the object.
 
-For example, let's copy from the Arvados playground, also known as *qr1hi*, to *dst_cluster*. The names *qr1hi* and *dst_cluster* are interchangable with any cluster name. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *qr1hi*-4zz18-tci4vn4fa95w0zx, the cluster name is qr1hi.
+For example, let's copy from the Arvados playground, also known as *pirca*, to *dstcl*. The names *pirca* and *dstcl* are interchangable with any cluster id. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *zzzzz*-4zz18-tci4vn4fa95w0zx, the cluster name is *zzzzz* .
 
-In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files, one for each cluster. The names of the files must have the format of *ClusterID.conf*. In our example, let's make two files, one for *qr1hi* and one for *dst_cluster*. From your *Current token* page in *qr1hi* and *dst_cluster*, copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
+In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon   in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files in @~/.config/arvados@, one for each cluster. The names of the files must have the format of *ClusterID.conf*. Navigate to the *Current token* page on each of *pirca* and *dstcl* to get the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
 
 !{display: block;margin-left: 25px;margin-right: auto;}{{ site.baseurl }}/images/api-token-host.png!
 
-Copy your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ into the config files as shown below in the shell account from which you are executing the commands. For example, the default shell you may have access to is shell.qr1hi. You can add these files in ~/.config/arvados/ in the qr1hi shell terminal.
+The config file consists of two lines, one for ARVADOS_API_HOST and one for ARVADOS_API_TOKEN:
 
-
-
~$ cd ~/.config/arvados
-~$ echo "ARVADOS_API_HOST=qr1hi.arvadosapi.com" >> qr1hi.conf
-~$ echo "ARVADOS_API_TOKEN=123456789abcdefghijkl" >> qr1hi.conf
-~$ echo "ARVADOS_API_HOST=dst_cluster.arvadosapi.com" >> dst_cluster.conf
-~$ echo "ARVADOS_API_TOKEN=987654321lkjihgfedcba" >> dst_cluster.conf
-

-
+
+ARVADOS_API_HOST=zzzzz.arvadosapi.com
+ARVADOS_API_TOKEN=v2/zzzzz-gj3su-xxxxxxxxxxxxxxx/123456789abcdefghijkl
+

+
+Copy your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ into the config files as shown below in the shell account from which you are executing the commands.  In our example, you need two files, @~/.config/arvados/pirca.conf@ and @~/.config/arvados/dstcl.conf@.
 
-Now you're ready to copy between *qr1hi* and *dst_cluster*!
+Now you're ready to copy between *pirca* and *dstcl*!
 
 h3. How to copy a collection
 
-First, select the uuid of the collection you want to copy from the source cluster. The uuid can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
+First, determine the uuid or portable data hash of the collection you want to copy from the source cluster. The uuid can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
 
-Now copy the collection from *qr1hi* to *dst_cluster*. We will use the uuid @qr1hi-4zz18-tci4vn4fa95w0zx@ as an example. You can find this collection in the lobSTR v.3 project on playground.arvados.org.
+Now copy the collection from *pirca* to *dstcl*. We will use the uuid @jutro-4zz18-tv416l321i4r01e@ as an example. You can find this collection on playground.arvados.org.
 
-
~$ arv-copy --src qr1hi --dst dst_cluster qr1hi-4zz18-tci4vn4fa95w0zx
-qr1hi-4zz18-tci4vn4fa95w0zx: 6.1M / 6.1M 100.0%
-arvados.arv-copy[1234] INFO: Success: created copy with uuid dst_cluster-4zz18-8765943210cdbae
-

-
-
-The output of arv-copy displays the uuid of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in a pre-created project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project uuid.
-
-For example, this will copy the collection to project dst_cluster-j7d0g-a894213ukjhal12 in the destination cluster.
-
- 
~$ arv-copy --src qr1hi --dst dst_cluster --project-uuid dst_cluster-j7d0g-a894213ukjhal12 qr1hi-4zz18-tci4vn4fa95w0zx
+
~$ arv-copy --src pirca --dst dstcl jutro-4zz18-tv416l321i4r01e
+jutro-4zz18-tv416l321i4r01e: 6.1M / 6.1M 100.0%
+arvados.arv-copy[1234] INFO: Success: created copy with uuid dstcl-4zz18-xxxxxxxxxxxxxxx
 

 
 
-h3. How to copy a pipeline template
-
-{% include 'arv_copy_expectations' %}
-
-We will use the uuid @qr1hi-p5p6p-9pkaxt6qjnkxhhu@ as an example pipeline template.
+You can also copy by content address:
 
 
-
~$ arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial qr1hi-p5p6p-9pkaxt6qjnkxhhu
-To git@git.dst_cluster.arvadosapi.com:$USER/tutorial.git
- * [new branch] git_git_qr1hi_arvadosapi_com_arvados_git_ac21f0d45a76294aaca0c0c0fdf06eb72d03368d -> git_git_qr1hi_arvadosapi_com_arvados_git_ac21f0d45a76294aaca0c0c0fdf06eb72d03368d
-arvados.arv-copy[19694] INFO: Success: created copy with uuid dst_cluster-p5p6p-rym2h5ub9m8ofwj
+
~$ arv-copy --src pirca --dst dstcl 2463fa9efeb75e099685528b3b9071e0+438
+2463fa9efeb75e099685528b3b9071e0+438: 6.1M / 6.1M 100.0%
+arvados.arv-copy[1234] INFO: Success: created copy with uuid dstcl-4zz18-xxxxxxxxxxxxxxx
 

 
 
-New branches in the destination git repo will be created for each branch used in the pipeline template. For example, if your source branch was named ac21f0d45a76294aaca0c0c0fdf06eb72d03368d, your new branch will be named @git_git_qr1hi_arvadosapi_com_reponame_git_ac21f0d45a76294aaca0c0c0fdf06eb72d03368d@.
-
-By default, if you copy a pipeline template recursively, you will find that the template as well as all the dependencies are in your home project.
+The output of arv-copy displays the uuid of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in an existing project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project uuid.
 
-If you would like to copy the object without dependencies, you can use the @--no-recursive@ tag.
+For example, this will copy the collection to project dstcl-j7d0g-a894213ukjhal12 in the destination cluster.
 
-For example, we can copy the same object using this tag.
-
-
-
~$ arv-copy --src qr1hi --dst dst_cluster --dst-git-repo $USER/tutorial --no-recursive qr1hi-p5p6p-9pkaxt6qjnkxhhu
+ 
~$ arv-copy --src pirca --dst dstcl --project-uuid dstcl-j7d0g-a894213ukjhal12 jutro-4zz18-tv416l321i4r01e
 

 
 
 h3. How to copy a workflow
 
-We will use the uuid @zzzzz-7fd4e-sampleworkflow1@ as an example workflow.
+We will use the uuid @jutro-7fd4e-mkmmq53m1ze6apx@ as an example workflow.
 
 
-
~$ arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial zzzzz-7fd4e-sampleworkflow1
-zzzzz-4zz18-jidprdejysravcr: 1143M / 1143M 100.0%
-2017-01-04 04:11:58 arvados.arv-copy[5906] INFO:
-2017-01-04 04:11:58 arvados.arv-copy[5906] INFO: Success: created copy with uuid dst_cluster-7fd4e-ojtgpne594ubkt7
+
~$ arv-copy --src jutro --dst pirca --project-uuid pirca-j7d0g-ecak8knpefz8ere jutro-7fd4e-mkmmq53m1ze6apx
+ae480c5099b81e17267b7445e35b4bc7+180: 23M / 23M 100.0%
+2463fa9efeb75e099685528b3b9071e0+438: 156M / 156M 100.0%
+jutro-4zz18-vvvqlops0a0kpdl: 94M / 94M 100.0%
+2020-08-19 17:04:13 arvados.arv-copy[4789] INFO:
+2020-08-19 17:04:13 arvados.arv-copy[4789] INFO: Success: created copy with uuid pirca-7fd4e-s0tw9rfbkpo2fmx
 

 
 
-The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *locations* and *docker images* found in the src workflow definition will also be copied to the destination recursively.
+The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *collections* and *docker images* referenced in the source workflow definition will also be copied to the destination.
 
 If you would like to copy the object without dependencies, you can use the @--no-recursive@ flag.
-
-For example, we can copy the same object non-recursively using the following:
-
-
-
~$ arv-copy --src zzzzz --dst dst_cluster --dst-git-repo $USER/tutorial --no-recursive zzzzz-7fd4e-sampleworkflow1
-

-
diff --git a/doc/user/topics/arv-docker.html.textile.liquid b/doc/user/topics/arv-docker.html.textile.liquid
index e9e8450268..bb1c7dd53e 100644
--- a/doc/user/topics/arv-docker.html.textile.liquid
+++ b/doc/user/topics/arv-docker.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Customizing Crunch environment using Docker"
+title: "Working with Docker images"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,145 +9,80 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This page describes how to customize the runtime environment (e.g., the programs, libraries, and other dependencies needed to run a job) that a crunch script will be run in using "Docker.":https://www.docker.com/  Docker is a tool for building and running containers that isolate applications from other applications running on the same node.  For detailed information about Docker, see the "Docker User Guide.":https://docs.docker.com/userguide/
+This page describes how to set up the runtime environment (e.g., the programs, libraries, and other dependencies needed to run a job) that a workflow step will be run in using "Docker.":https://www.docker.com/  Docker is a tool for building and running containers that isolate applications from other applications running on the same node.  For detailed information about Docker, see the "Docker User Guide.":https://docs.docker.com/userguide/
 
-This page will demonstrate how to:
+This page describes:
 
-# Fetch the arvados/jobs Docker image
-# Manually install additional software into the container
-# Create a new custom image
-# Upload that image to Arvados for use by Crunch jobs
-# Share your image with others
+# "Create a custom image using a Dockerfile":#create
+# "Uploading an image to Arvados":#upload
+# "Sources of pre-built bioinformatics Docker images":#sources
 
 {% include 'tutorial_expectations_workstation' %}
 
 You also need ensure that "Docker is installed,":https://docs.docker.com/installation/ the Docker daemon is running, and you have permission to access Docker.  You can test this by running @docker version@.  If you receive a permission denied error, your user account may need to be added to the @docker@ group.  If you have root access, you can add yourself to the @docker@ group using @$ sudo addgroup $USER docker@ then log out and log back in again; otherwise consult your local sysadmin.
 
-h2. Fetch a starting image
+h2(#create). Create a custom image using a Dockerfile
 
-The easiest way to begin is to start from the "arvados/jobs" image which already has the Arvados SDK installed along with other configuration required for use with Crunch.
+This example shows how to create a Docker image and add the R package.
 
-Download the latest "arvados/jobs" image from the Docker registry:
+First, create new directory called @docker-example@, in that directory create a file called @Dockerfile@.
 
 
-
$ docker pull arvados/jobs:latest
-Pulling repository arvados/jobs
-3132168f2acb: Download complete
-a42b7f2c59b6: Download complete
-e5afdf26a7ae: Download complete
-5cae48636278: Download complete
-7a4f91b70558: Download complete
-a04a275c1fd6: Download complete
-c433ff206a22: Download complete
-b2e539b45f96: Download complete
-073b2581c6be: Download complete
-593915af19dc: Download complete
-32260b35005e: Download complete
-6e5b860c1cde: Download complete
-95f0bfb43d4d: Download complete
-c7fd77eedb96: Download complete
-0d7685aafd00: Download complete
+
$ mkdir docker-example-r-base
+$ cd docker-example-r-base
 

 
 
-h2. Install new packages
-
-Next, enter the container using @docker run@, providing the arvados/jobs image and the program you want to run (in this case the bash shell).
-
 
-
$ docker run --interactive --tty --user root arvados/jobs /bin/bash
-root@fbf1d0f529d5:/#
+
FROM ubuntu:bionic
+RUN apt-get update && apt-get -yq --no-install-recommends install r-base-core
 

 
 
-Next, update the package list using @apt-get update@.
+The "RUN" command is executed inside the container and can be any shell command line.  You are not limited to installing Debian packages.  You may compile programs or libraries from source and install them, edit systemwide configuration files, use other package managers such as @pip@ or @gem@, and perform any other customization necessary to run your program.
 
-
-
root@fbf1d0f529d5:/# apt-get update
-Get:2 http://apt.arvados.org stretch-dev InRelease [3260 B]
-Get:1 http://security-cdn.debian.org/debian-security stretch/updates InRelease [94.3 kB]
-Ign:3 http://cdn-fastly.deb.debian.org/debian stretch InRelease
-Get:4 http://cdn-fastly.deb.debian.org/debian stretch-updates InRelease [91.0 kB]
-Get:5 http://apt.arvados.org stretch-dev/main amd64 Packages [208 kB]
-Get:6 http://cdn-fastly.deb.debian.org/debian stretch Release [118 kB]
-Get:7 http://security-cdn.debian.org/debian-security stretch/updates/main amd64 Packages [499 kB]
-Get:8 http://cdn-fastly.deb.debian.org/debian stretch Release.gpg [2434 B]
-Get:9 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages.diff/Index [10.6 kB]
-Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
-Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
-Fetched 1026 kB in 0s (1384 kB/s)
-Reading package lists... Done
-

-
+You can also visit the "Docker tutorial":https://docs.docker.com/get-started/part2/ for more information and examples.
+
+You should add your Dockerfiles to the same source control repository as the Workflows that use them.
 
-In this example, we will install the "R" statistical language Debian package "r-base-core".  Use @apt-get install@:
+h3. Create a new image
+
+We're now ready to create a new Docker image.  Use @docker build@ to create a new image from the Dockerfile.
 
 
-
root@fbf1d0f529d5:/# apt-get install r-base-core
-Reading package lists... Done
-Building dependency tree
-Reading state information... Done
-The following additional packages will be installed:
-[...]
-done.
+
docker-example-r-base$ docker build -t docker-example-r-base .
 

 
 
+h3. Verify image
+
 Now we can verify that "R" is installed:
 
 
-
root@fbf1d0f529d5:/# R
+
$ docker run -ti docker-example-r-base
+root@57ec8f8b2663:/# R
 
-R version 3.3.3 (2017-03-06) -- "Another Canoe"
-Copyright (C) 2017 The R Foundation for Statistical Computing
+R version 3.4.4 (2018-03-15) -- "Someone to Lean On"
+Copyright (C) 2018 The R Foundation for Statistical Computing
 Platform: x86_64-pc-linux-gnu (64-bit)
-
-R is free software and comes with ABSOLUTELY NO WARRANTY.
-You are welcome to redistribute it under certain conditions.
-Type 'license()' or 'licence()' for distribution details.
-
-R is a collaborative project with many contributors.
-Type 'contributors()' for more information and
-'citation()' on how to cite R or R packages in publications.
-
-Type 'demo()' for some demos, 'help()' for on-line help, or
-'help.start()' for an HTML browser interface to help.
-Type 'q()' to quit R.
-
->
 

 
 
-Note that you are not limited to installing Debian packages.  You may compile programs or libraries from source and install them, edit systemwide configuration files, use other package managers such as @pip@ or @gem@, and perform any other customization necessary to run your program.
+h2(#upload). Upload your image
 
-h2. Create a new image
-
-We're now ready to create a new Docker image.  First, quit the container, then use @docker commit@ to create a new image from the stopped container.  The container id can be found in the default hostname of the container displayed in the prompt, in this case @fbf1d0f529d5@:
+Finally, we are ready to upload the new Docker image to Arvados.  Use @arv-keepdocker@ with the image repository name to upload the image.  Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
 
 
-
root@fbf1d0f529d5:/# exit
-$ docker commit fbf1d0f529d5 arvados/jobs-with-r
-sha256:2818853ff9f9af5d7f77979803baac9c4710790ad2b84c1a754b02728fdff205
-$ docker images
-$ docker images |head
-REPOSITORY            TAG                 IMAGE ID            CREATED             SIZE
-arvados/jobs-with-r   latest              2818853ff9f9        9 seconds ago       703.1 MB
-arvados/jobs          latest              12b9f859d48c        4 days ago          362 MB
-

-
-
-h2. Upload your image
+
$ arv-keepdocker docker-example-r-base
+2020-06-29 13:48:19 arvados.arv_put[769] INFO: Creating new cache file at /home/peter/.cache/arvados/arv-put/39ddb51ebf6c5fcb3d713b5969466967
+206M / 206M 100.0% 2020-06-29 13:48:21 arvados.arv_put[769] INFO:
 
-Finally, we are ready to upload the new Docker image to Arvados.  Use @arv-keepdocker@ with the image repository name to upload the image.  Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
+2020-06-29 13:48:21 arvados.arv_put[769] INFO: Collection saved as 'Docker image docker-example-r-base:latest sha256:edd10'
+zzzzz-4zz18-0tayximqcyb6uf8
 
-
-
$ arv-keepdocker arvados/jobs-with-r
-703M / 703M 100.0%
-Collection saved as 'Docker image arvados/jobs-with-r:latest 2818853ff9f9'
-qr1hi-4zz18-abcdefghijklmno
-$ arv-keepdocker
+$ arv-keepdocker images
 REPOSITORY                      TAG         IMAGE ID      COLLECTION                     CREATED
-arvados/jobs-with-r             latest      2818853ff9f9  qr1hi-4zz18-abcdefghijklmno    Tue Jan 17 20:35:53 2017
+docker-example-r-base           latest      sha256:edd10  zzzzz-4zz18-0tayximqcyb6uf8    Mon Jun 29 17:46:16 2020
 

 
 
@@ -156,14 +91,24 @@ You are now able to specify the runtime environment for your program using @Dock
 
 hints:
   DockerRequirement:
-    dockerPull: arvados/jobs-with-r
+    dockerPull: docker-example-r-base
 

 
-h2. Share Docker images
+h3. Uploading Docker images to a shared project
 
-Docker images are subject to normal Arvados permissions.  If wish to share your Docker image with others (or wish to share a pipeline template that uses your Docker image) you will need to use @arv-keepdocker@ with the @--project-uuid@ option to upload the image to a shared project.
+Docker images are subject to normal Arvados permissions.  If wish to share your Docker image with others you should use @arv-keepdocker@ with the @--project-uuid@ option to add the image to a shared project and ensure that metadata is set correctly.
 
 
-
$ arv-keepdocker arvados/jobs-with-r --project-uuid qr1hi-j7d0g-xxxxxxxxxxxxxxx
+
$ arv-keepdocker docker-example-r-base --project-uuid zzzzz-j7d0g-xxxxxxxxxxxxxxx
 

 
+
+h2(#sources). Sources of pre-built images
+
+In addition to creating your own contianers, there are a number of resources where you can find bioinformatics tools already wrapped in container images:
+
+"BioContainers":https://biocontainers.pro/
+
+"Dockstore":https://dockstore.org/
+
+"Docker Hub":https://hub.docker.com/
diff --git a/doc/user/topics/arv-web.html.textile.liquid b/doc/user/topics/arv-web.html.textile.liquid
deleted file mode 100644
index 9671e97096..0000000000
--- a/doc/user/topics/arv-web.html.textile.liquid
+++ /dev/null
@@ -1,106 +0,0 @@
----
-layout: default
-navsection: userguide
-title: "Using arv-web"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-@arv-web@ enables you to run a custom web service from the contents of an Arvados collection.
-
-{% include 'tutorial_expectations_workstation' %}
-
-h2. Usage
-
-@arv-web@ enables you to set up a web service based on the most recent collection in a project.  An arv-web application is a reproducible, immutable application bundle where the web app is packaged with both the code to run and the data to serve.  Because Arvados Collections can be updated with minimum duplication, it is efficient to produce a new application bundle when the code or data needs to be updated; retaining old application bundles makes it easy to go back and run older versions of your web app.
-
-
-$ cd $HOME/arvados/services/arv-web
-usage: arv-web.py [-h] --project-uuid PROJECT_UUID [--port PORT]
-                  [--image IMAGE]
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --project-uuid PROJECT_UUID
-                        Project uuid to watch
-  --port PORT           Host port to listen on (default 8080)
-  --image IMAGE         Docker image to run
-

-
-At startup, @arv-web@ queries an Arvados project and mounts the most recently modified collection into a temporary directory.  It then runs a Docker image with the collection bound to @/mnt@ inside the container.  When a new collection is added to the project, or an existing project is updated, it will stop the running Docker container, unmount the old collection, mount the new most recently modified collection, and restart the Docker container with the new mount.
-
-h2. Docker container
-
-The @Dockerfile@ in @arvados/docker/arv-web@ builds a Docker image that runs Apache with @/mnt@ as the DocumentRoot.  It is configured to run web applications which use Python WSGI, Ruby Rack, or CGI; to serve static HTML; or browse the contents of the @public@ subdirectory of the collection using default Apache index pages.
-
-To build the Docker image:
-
-
-
~$ cd arvados/docker
-~/arvados/docker$ docker build -t arvados/arv-web arv-web
-

-
-
-h2. Running sample applications
-
-First, in Arvados Workbench, create a new project.  Copy the project UUID from the URL bar (this is the part of the URL after @projects/...@).
-
-Now upload a collection containing a "Python WSGI web app:":http://wsgi.readthedocs.org/en/latest/
-
-
-
~$ cd arvados/services/arv-web
-~/arvados/services/arv-web$ arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-wsgi-app sample-wsgi-app
-0M / 0M 100.0%
-Collection saved as 'sample-wsgi-app'
-zzzzz-4zz18-ebohzfbzh82qmqy
-~/arvados/services/arv-web$ ./arv-web.py --project [zzzzz-j7d0g-yourprojectuuid] --port 8888
-2015-01-30 11:21:00 arvados.arv-web[4897] INFO: Mounting zzzzz-4zz18-ebohzfbzh82qmqy
-2015-01-30 11:21:01 arvados.arv-web[4897] INFO: Starting Docker container arvados/arv-web
-2015-01-30 11:21:02 arvados.arv-web[4897] INFO: Container id e79e70558d585a3e038e4bfbc97e5c511f21b6101443b29a8017bdf3d84689a3
-2015-01-30 11:21:03 arvados.arv-web[4897] INFO: Waiting for events
-

-
-
-The sample application will be available at @http://localhost:8888@.
-
-h3. Updating the application
-
-If you upload a new collection to the same project, arv-web will restart the web service and serve the new collection.  For example, uploading a collection containing a "Ruby Rack web app:":https://github.com/rack/rack/wiki
-
-
-
~$ cd arvados/services/arv-web
-~/arvados/services/arv-web$ arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-rack-app sample-rack-app
-0M / 0M 100.0%
-Collection saved as 'sample-rack-app'
-zzzzz-4zz18-dhhm0ay8k8cqkvg
-

-
-
-@arv-web@ will automatically notice the change, load a new container, and send an update signal (SIGHUP) to the service:
-
-
-2015-01-30 11:21:03 arvados.arv-web[4897] INFO:Waiting for events
-2015-01-30 11:21:04 arvados.arv-web[4897] INFO:create zzzzz-4zz18-dhhm0ay8k8cqkvg
-2015-01-30 11:21:05 arvados.arv-web[4897] INFO:Mounting zzzzz-4zz18-dhhm0ay8k8cqkvg
-2015-01-30 11:21:06 arvados.arv-web[4897] INFO:Sending refresh signal to container
-2015-01-30 11:21:07 arvados.arv-web[4897] INFO:Waiting for events
-

-
-h2. Writing your own applications
-
-The @arvados/arv-web@ image serves Python and Ruby applications using Phusion Passenger and Apache @mod_passenger@.  See "Phusion Passenger users guide for Apache":https://www.phusionpassenger.com/documentation/Users%20guide%20Apache.html for details, and look at the sample apps @arvados/services/arv-web/sample-wsgi-app@ and @arvados/services/arv-web/sample-rack-app@.
-
-You can serve CGI applications using standard Apache CGI support.  See "Apache Tutorial: Dynamic Content with CGI":https://httpd.apache.org/docs/current/howto/cgi.html for details, and look at the sample app @arvados/services/arv-web/sample-cgi-app@.
-
-You can also serve static content from the @public@ directory of the collection.  Look at @arvados/services/arv-web/sample-static-page@ for an example.  If no @index.html@ is found in @public/@, it will render default Apache index pages, permitting simple browsing of the collection contents.
-
-h3. Custom images
-
-You can provide your own Docker image.  The Docker image that will be used create the web application container is specified in the @docker_image@ file in the root of the collection.  You can also specify @--image@ on the command @arv-web@ line to choose the docker image (this will override the contents of @docker_image@).
-
-h3. Reloading the web service
-
-Stopping the Docker container and starting it again can result in a small amount of downtime.  When the collection containing a new or updated web application uses the same Docker image as the currently running web application, it is possible to avoid this downtime by keeping the existing container and only reloading the web server.  This is accomplished by providing a file called @reload@ in the root of the collection, which should contain the commands necessary to reload the web server inside the container.
diff --git a/doc/user/topics/keep.html.textile.liquid b/doc/user/topics/keep.html.textile.liquid
deleted file mode 100644
index 68b6a87d09..0000000000
--- a/doc/user/topics/keep.html.textile.liquid
+++ /dev/null
@@ -1,59 +0,0 @@
----
-layout: default
-navsection: userguide
-title: "How Keep works"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The Arvados distributed file system is called *Keep*.  Keep is a content-addressable file system.  This means that files are managed using special unique identifiers derived from the _contents_ of the file (specifically, the MD5 hash), rather than human-assigned file names.  This has a number of advantages:
-* Files can be stored and replicated across a cluster of servers without requiring a central name server.
-* Both the server and client systematically validate data integrity because the checksum is built into the identifier.
-* Data duplication is minimized—two files with the same contents will have in the same identifier, and will not be stored twice.
-* It avoids data race conditions, since an identifier always points to the same data.
-
-In Keep, information is stored in *data blocks*.  Data blocks are normally between 1 byte and 64 megabytes in size.  If a file exceeds the maximum size of a single data block, the file will be split across multiple data blocks until the entire file can be stored.  These data blocks may be stored and replicated across multiple disks, servers, or clusters.  Each data block has its own identifier for the contents of that specific data block.
-
-In order to reassemble the file, Keep stores a *collection* data block which lists in sequence the data blocks that make up the original file.  A collection data block may store the information for multiple files, including a directory structure.
-
-In this example we will use @c1bad4b39ca5a924e481008009d94e32+210@, which we added to Keep in "how to upload data":{{ site.baseurl }}/user/tutorials/tutorial-keep.html.  First let us examine the contents of this collection using @arv-get@:
-
-
-
~$ arv-get c1bad4b39ca5a924e481008009d94e32+210
-. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
-

-
-
-The command @arv-get@ fetches the contents of the collection @c1bad4b39ca5a924e481008009d94e32+210@.  In this example, this collection includes a single file @var-GS000016015-ASM.tsv.bz2@ which is 227212247 bytes long, and is stored using four sequential data blocks, @204e43b8a1185621ca55a94839582e6f+67108864@, @b9677abbac956bd3e86b1deb28dfac03+67108864@, @fc15aff2a762b13f521baf042140acec+67108864@, and @323d2a3ce20370c4ca1d3462a344f8fd+25885655@.
-
-Let's use @arv-get@ to download the first data block:
-
-notextile. 
~$ cd /scratch/you
-/scratch/you$ arv-get 204e43b8a1185621ca55a94839582e6f+67108864 > block1

-
-{% include 'notebox_begin' %}
-
-When you run this command, you may get this API warning:
-
-notextile. 
WARNING:root:API lookup failed for collection 204e43b8a1185621ca55a94839582e6f+67108864 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://qr1hi.arvadosapi.com/arvados/v1/collections/204e43b8a1185621ca55a94839582e6f%2B67108864?alt=json returned "Not Found">)

-
-This happens because @arv-get@ tries to find a collection with this identifier.  When that fails, it emits this warning, then looks for a datablock instead, which succeeds.
-
-{% include 'notebox_end' %}
-
-Let's look at the size and compute the MD5 hash of @block1@:
-
-
-
/scratch/you$ ls -l block1
--rw-r--r-- 1 you group 67108864 Dec  9 20:14 block1
-/scratch/you$ md5sum block1
-204e43b8a1185621ca55a94839582e6f  block1
-

-
-
-Notice that the block identifer 204e43b8a1185621ca55a94839582e6f+67108864 consists of:
-* the MD5 hash of @block1@, @204e43b8a1185621ca55a94839582e6f@, plus
-* the size of @block1@, @67108864@.
diff --git a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid b/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
deleted file mode 100644
index 544ccbd35e..0000000000
--- a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
+++ /dev/null
@@ -1,173 +0,0 @@
----
-layout: default
-navsection: userguide
-title: "Using GATK with Arvados"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This tutorial demonstrates how to use the Genome Analysis Toolkit (GATK) with Arvados. In this example we will install GATK and then create a VariantFiltration job to assign pass/fail scores to variants in a VCF file.
-
-{% include 'tutorial_expectations' %}
-
-h2. Installing GATK
-
-Download the GATK binary tarball[1] -- e.g., @GenomeAnalysisTK-2.6-4.tar.bz2@ -- and "copy it to your Arvados VM":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
-
-
-
~$ arv-put GenomeAnalysisTK-2.6-4.tar.bz2
-c905c8d8443a9c44274d98b7c6cfaa32+94
-

-
-
-Next, you need the GATK Resource Bundle[2].  This may already be available in Arvados.  If not, you will need to download the files listed below and put them into Keep.
-
-
-
~$ arv keep ls -s d237a90bae3870b3b033aea1e99de4a9+10820
-  50342 1000G_omni2.5.b37.vcf.gz
-      1 1000G_omni2.5.b37.vcf.gz.md5
-    464 1000G_omni2.5.b37.vcf.idx.gz
-      1 1000G_omni2.5.b37.vcf.idx.gz.md5
-  43981 1000G_phase1.indels.b37.vcf.gz
-      1 1000G_phase1.indels.b37.vcf.gz.md5
-    326 1000G_phase1.indels.b37.vcf.idx.gz
-      1 1000G_phase1.indels.b37.vcf.idx.gz.md5
- 537210 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.gz
-      1 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.gz.md5
-   3473 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.idx.gz
-      1 CEUTrio.HiSeq.WGS.b37.bestPractices.phased.b37.vcf.idx.gz.md5
-  19403 Mills_and_1000G_gold_standard.indels.b37.vcf.gz
-      1 Mills_and_1000G_gold_standard.indels.b37.vcf.gz.md5
-    536 Mills_and_1000G_gold_standard.indels.b37.vcf.idx.gz
-      1 Mills_and_1000G_gold_standard.indels.b37.vcf.idx.gz.md5
-  29291 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.gz.md5
-    565 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.idx.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.sites.vcf.idx.gz.md5
-  37930 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.gz.md5
-    592 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.idx.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.raw.subset.b37.vcf.idx.gz.md5
-5898484 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam
-    112 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam.bai.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam.bai.gz.md5
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.bam.md5
-   3837 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.gz.md5
-     65 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.idx.gz
-      1 NA12878.HiSeq.WGS.bwa.cleaned.recal.b37.20.vcf.idx.gz.md5
- 275757 dbsnp_137.b37.excluding_sites_after_129.vcf.gz
-      1 dbsnp_137.b37.excluding_sites_after_129.vcf.gz.md5
-   3735 dbsnp_137.b37.excluding_sites_after_129.vcf.idx.gz
-      1 dbsnp_137.b37.excluding_sites_after_129.vcf.idx.gz.md5
- 998153 dbsnp_137.b37.vcf.gz
-      1 dbsnp_137.b37.vcf.gz.md5
-   3890 dbsnp_137.b37.vcf.idx.gz
-      1 dbsnp_137.b37.vcf.idx.gz.md5
-  58418 hapmap_3.3.b37.vcf.gz
-      1 hapmap_3.3.b37.vcf.gz.md5
-    999 hapmap_3.3.b37.vcf.idx.gz
-      1 hapmap_3.3.b37.vcf.idx.gz.md5
-      3 human_g1k_v37.dict.gz
-      1 human_g1k_v37.dict.gz.md5
-      2 human_g1k_v37.fasta.fai.gz
-      1 human_g1k_v37.fasta.fai.gz.md5
- 849537 human_g1k_v37.fasta.gz
-      1 human_g1k_v37.fasta.gz.md5
-      1 human_g1k_v37.stats.gz
-      1 human_g1k_v37.stats.gz.md5
-      3 human_g1k_v37_decoy.dict.gz
-      1 human_g1k_v37_decoy.dict.gz.md5
-      2 human_g1k_v37_decoy.fasta.fai.gz
-      1 human_g1k_v37_decoy.fasta.fai.gz.md5
- 858592 human_g1k_v37_decoy.fasta.gz
-      1 human_g1k_v37_decoy.fasta.gz.md5
-      1 human_g1k_v37_decoy.stats.gz
-      1 human_g1k_v37_decoy.stats.gz.md5
-

-
-
-h2. Submit a GATK job
-
-The Arvados distribution includes an example crunch script ("crunch_scripts/GATK2-VariantFiltration":https://dev.arvados.org/projects/arvados/repository/revisions/master/entry/crunch_scripts/GATK2-VariantFiltration) that runs the GATK VariantFiltration tool with some default settings.
-
-
-
~$ src_version=76588bfc57f33ea1b36b82ca7187f465b73b4ca4
-~$ vcf_input=5ee633fe2569d2a42dd81b07490d5d13+82
-~$ gatk_binary=c905c8d8443a9c44274d98b7c6cfaa32+94
-~$ gatk_bundle=d237a90bae3870b3b033aea1e99de4a9+10820
-~$ cat >the_job <<EOF
-{
- "script":"GATK2-VariantFiltration",
- "repository":"arvados",
- "script_version":"$src_version",
- "script_parameters":
- {
-  "input":"$vcf_input",
-  "gatk_binary_tarball":"$gatk_binary",
-  "gatk_bundle":"$gatk_bundle"
- }
-}
-EOF
-

-
-
-* @"input"@ is collection containing the source VCF data. Here we are using an exome report from PGP participant hu34D5B9.
-* @"gatk_binary_tarball"@ is a Keep collection containing the GATK 2 binary distribution[1] tar file.
-* @"gatk_bundle"@ is a Keep collection containing the GATK resource bundle[2].
-
-Now start a job:
-
-
-
~$ arv job create --job "$(cat the_job)"
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-n9k7qyp7bs5b9d4",
- "kind":"arvados#job",
- "etag":"9j99n1feoxw3az448f8ises12",
- "uuid":"qr1hi-8i9sb-n9k7qyp7bs5b9d4",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-17T19:02:15Z",
- "modified_by_client_uuid":"qr1hi-ozdt8-obw7foaks3qjyej",
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-17T19:02:15Z",
- "updated_at":"2013-12-17T19:02:15Z",
- "submit_id":null,
- "priority":null,
- "script":"GATK2-VariantFiltration",
- "script_parameters":{
-  "input":"5ee633fe2569d2a42dd81b07490d5d13+82",
-  "gatk_binary_tarball":"c905c8d8443a9c44274d98b7c6cfaa32+94",
-  "gatk_bundle":"d237a90bae3870b3b033aea1e99de4a9+10820"
- },
- "script_version":"76588bfc57f33ea1b36b82ca7187f465b73b4ca4",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":null,
- "finished_at":null,
- "output":null,
- "success":null,
- "running":null,
- "is_locked_by_uuid":null,
- "log":null,
- "runtime_constraints":{},
- "tasks_summary":{}
-}
-

-
-
-Once the job completes, the output can be found in hu34D5B9-exome-filtered.vcf:
-
-
~$ arv keep ls bedd6ff56b3ae9f90d873b1fcb72f9a3+91
-hu34D5B9-exome-filtered.vcf
-

-
-
-h2. Notes
-
-fn1. "Download the GATK tools":http://www.broadinstitute.org/gatk/download
-
-fn2. "Information about the GATK resource bundle":http://gatkforums.broadinstitute.org/discussion/1213/whats-in-the-resource-bundle-and-how-can-i-get-it and "direct download link":ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/2.5/b37/ (if prompted, submit an empty password)
diff --git a/doc/user/topics/tutorial-job1.html.textile.liquid b/doc/user/topics/tutorial-job1.html.textile.liquid
deleted file mode 100644
index f7a2060101..0000000000
--- a/doc/user/topics/tutorial-job1.html.textile.liquid
+++ /dev/null
@@ -1,214 +0,0 @@
----
-layout: default
-navsection: userguide
-title: "Running a Crunch job on the command line"
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This tutorial introduces how to run individual Crunch jobs using the @arv@ command line tool.
-
-{% include 'tutorial_expectations' %}
-
-You will create a job to run the "hash" Crunch script.  The "hash" script computes the MD5 hash of each file in a collection.
-
-h2. Jobs
-
-Crunch pipelines consist of one or more jobs.  A "job" is a single run of a specific version of a Crunch script with a specific input.  You can also run jobs individually.
-
-A request to run a Crunch job are is described using a JSON object.  For example:
-
-
-
~$ cat >~/the_job <<EOF
-{
- "script": "hash",
- "repository": "arvados",
- "script_version": "master",
- "script_parameters": {
-  "input": "c1bad4b39ca5a924e481008009d94e32+210"
- },
- "no_reuse": "true"
-}
-EOF
-

-
-
-* @cat@ is a standard Unix utility that writes a sequence of input to standard output.
-* @<~/the_job@ redirects standard output to a file called @~/the_job@.
-* @"repository"@ is the name of a Git repository to search for the script version.  You can access a list of available git repositories on the Arvados Workbench under "*Code repositories*":{{site.arvados_workbench_host}}/repositories.
-* @"script_version"@ specifies the version of the script that you wish to run.  This can be in the form of an explicit Git revision hash, a tag, or a branch.  Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
-* @"script"@ specifies the name of the script to run.  The script must be given relative to the @crunch_scripts/@ subdirectory of the Git repository.
-* @"script_parameters"@ are provided to the script.  In this case, the input is the PGP data Collection that we "put in Keep earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
-* Setting the @"no_reuse"@ flag tells Crunch not to reuse work from past jobs.  This helps ensure that you can watch a new Job process for the rest of this tutorial, without reusing output from a past run that you made, or somebody else marked as public.  (If you want to experiment, after the first run below finishes, feel free to edit this job to remove the @"no_reuse"@ line and resubmit it.  See what happens!)
-
-Use @arv job create@ to actually submit the job.  It should print out a JSON object which describes the newly created job:
-
-
-
~$ arv job create --job "$(cat ~/the_job)"
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
- "kind":"arvados#job",
- "etag":"ax3cn7w9whq2hdh983yxvq09p",
- "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-16T20:44:32Z",
- "modified_by_client_uuid":"qr1hi-ozdt8-obw7foaks3qjyej",
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-16T20:44:32Z",
- "updated_at":"2013-12-16T20:44:33Z",
- "submit_id":null,
- "priority":null,
- "script":"hash",
- "script_parameters":{
-  "input":"c1bad4b39ca5a924e481008009d94e32+210"
- },
- "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":null,
- "finished_at":null,
- "output":null,
- "success":null,
- "running":null,
- "is_locked_by_uuid":null,
- "log":null,
- "runtime_constraints":{},
- "tasks_summary":{}
-}
-

-
-
-The job is now queued and will start running as soon as it reaches the front of the queue.  Fields to pay attention to include:
-
- * @"uuid"@ is the unique identifier for this specific job.
- * @"script_version"@ is the actual revision of the script used.  This is useful if the version was described using the "repository:branch" format.
-
-h2. Monitor job progress
-
-Go to "*Recent jobs*":{{site.arvados_workbench_host}}/jobs in Workbench.  Your job should be near the top of the table.  This table refreshes automatically.  When the job has completed successfully, it will show finished in the *Status* column.
-
-h2. Inspect the job output
-
-On the "Workbench Dashboard":{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table.  Click on the link under *Output* for your job to go to the files page with the job output.  The files page lists all the files that were output by the job.  Click on the link under the *file* column to view a file, or click on the download button  to download the output file.
-
-On the command line, you can use @arv job get@ to access a JSON object describing the output:
-
-
-
~$ arv job get --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
- "kind":"arvados#job",
- "etag":"1bk98tdj0qipjy0rvrj03ta5r",
- "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-16T20:44:32Z",
- "modified_by_client_uuid":null,
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-16T20:44:55Z",
- "updated_at":"2013-12-16T20:44:55Z",
- "submit_id":null,
- "priority":null,
- "script":"hash",
- "script_parameters":{
-  "input":"c1bad4b39ca5a924e481008009d94e32+210"
- },
- "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":"2013-12-16T20:44:36Z",
- "finished_at":"2013-12-16T20:44:53Z",
- "output":"dd755dbc8d49a67f4fe7dc843e4f10a6+54",
- "success":true,
- "running":false,
- "is_locked_by_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "log":"2afdc6c8b67372ffd22d8ce89d35411f+91",
- "runtime_constraints":{},
- "tasks_summary":{
-  "done":2,
-  "running":0,
-  "failed":0,
-  "todo":0
- }
-}
-

-
-
-* @"output"@ is the unique identifier for this specific job's output.  This is a Keep collection.  Because the output of Arvados jobs should be deterministic, the known expected output is dd755dbc8d49a67f4fe7dc843e4f10a6+54.
-
-Now you can list the files in the collection:
-
-
-
~$ arv keep ls dd755dbc8d49a67f4fe7dc843e4f10a6+54
-./md5sum.txt
-

-
-
-This collection consists of the @md5sum.txt@ file.  Use @arv-get@ to show the contents of the @md5sum.txt@ file:
-
-
-
~$ arv-get dd755dbc8d49a67f4fe7dc843e4f10a6+54/md5sum.txt
-44b8ae3fde7a8a88d2f7ebd237625b4f ./var-GS000016015-ASM.tsv.bz2
-

-
-
-This MD5 hash matches the MD5 hash which we "computed earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
-
-h2. The job log
-
-When the job completes, you can access the job log.  On the Workbench, visit "*Recent jobs*":{{site.arvados_workbench_host}}/jobs %(rarr)→% your job's UUID under the *uuid* column %(rarr)→% the collection link on the *log* row.
-
-On the command line, the Keep identifier listed in the @"log"@ field from @arv job get@ specifies a collection.  You can list the files in the collection:
-
-
-
~$ arv keep ls xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91
-./qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
-

-
-
-The log collection consists of one log file named with the job's UUID.  You can access it using @arv-get@:
-
-
-
~$ arv-get xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91/qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
-2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  check slurm allocation
-2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  node compute13 - 8 slots
-2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  start
-2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Install revision d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Clean-work-dir exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Install exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  script hash
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  script_version d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  runtime_constraints {"max_tasks_per_node":0}
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  start level 0
-2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 0 done, 0 running, 1 todo
-2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 job_task qr1hi-ot0gb-23c1k3kwrf8da62
-2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 started on compute13.1
-2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 0 done, 1 running, 0 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 success in 1 seconds
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 output
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  wait for last 0 children to finish
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  start level 1
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 job_task qr1hi-ot0gb-iwr0o3unqothg28
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 started on compute13.1
-2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 1 done, 1 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 success in 13 seconds
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 output dd755dbc8d49a67f4fe7dc843e4f10a6+54
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  wait for last 0 children to finish
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  status: 2 done, 0 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  release job allocation
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  Freeze not implemented
-2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  collate
-2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  output dd755dbc8d49a67f4fe7dc843e4f10a6+54+K@qr1hi
-2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575  finish
-

-
diff --git a/doc/user/tutorials/add-new-repository.html.textile.liquid b/doc/user/tutorials/add-new-repository.html.textile.liquid
index 9d8e768a78..e28b961238 100644
--- a/doc/user/tutorials/add-new-repository.html.textile.liquid
+++ b/doc/user/tutorials/add-new-repository.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-Arvados repositories are managed through the Git revision control system. You can use these repositories to store your crunch scripts and run them in the arvados cluster.
+Arvados supports managing git repositories. You can access these repositories using your Arvados credentials and share them with other Arvados users.
 
 {% include 'tutorial_expectations' %}
 
diff --git a/doc/user/tutorials/git-arvados-guide.html.textile.liquid b/doc/user/tutorials/git-arvados-guide.html.textile.liquid
index 2e255219d2..a552e4ee00 100644
--- a/doc/user/tutorials/git-arvados-guide.html.textile.liquid
+++ b/doc/user/tutorials/git-arvados-guide.html.textile.liquid
@@ -9,20 +9,13 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This tutorial describes how to work with a new Arvados git repository. Working with an Arvados git repository is analogous to working with other public git repositories. It will show you how to upload custom scripts to a remote Arvados repository, so you can use it in Arvados pipelines.
+This tutorial describes how to work with an Arvados-managed git repository. Working with an Arvados git repository is very similar to working with other public git repositories.
 
 {% include 'tutorial_expectations' %}
 
 {% include 'tutorial_git_repo_expectations' %}
 
-{% include 'notebox_begin' %}
-For more information about using Git, try
-
-
$ man gittutorial

- or *"search Google for Git tutorials":http://google.com/#q=git+tutorial*.
-{% include 'notebox_end' %}
-
-h2. Cloning an Arvados repository
+h2. Cloning a git repository
 
 Before you start using Git, you should do some basic configuration (you only need to do this the first time):
 
@@ -65,33 +58,22 @@ Create a git branch named *tutorial_branch* in the *tutorial* Arvados git reposi
 
 h2. Adding scripts to an Arvados repository
 
-Arvados crunch scripts need to be added in a *crunch_scripts* subdirectory in the repository. If this subdirectory does not exist, first create it in the local repository and change to that directory:
-
-
-
~/tutorial$ mkdir crunch_scripts
-~/tutorial$ cd crunch_scripts

-
-
-Next, using @nano@ or your favorite Unix text editor, create a new file called @hash.py@ in the @crunch_scripts@ directory.
-
-notextile. 
~/tutorial/crunch_scripts$ nano hash.py

-
-Add the following code to compute the MD5 hash of each file in a collection
+A git repository is a good place to store the CWL workflows that you run on Arvados.
 
- {% code 'tutorial_hash_script_py' as python %} 
+First, create a simple CWL CommandLineTool:
 
-Make the file executable:
+notextile. 
~/tutorials$ nano hello.cwl

 
-notextile. 
~/tutorial/crunch_scripts$ chmod +x hash.py

+ {% code tutorial_hello_cwl as yaml %} 
 
 Next, add the file to the git repository.  This tells @git@ that the file should be included on the next commit.
 
-notextile. 
~/tutorial/crunch_scripts$ git add hash.py

+notextile. 
~/tutorial$ git add hello.cwl

 
 Next, commit your changes.  All staged changes are recorded into the local git repository:
 
 
-
~/tutorial/crunch_scripts$ git commit -m "my first script"
+
~/tutorial$ git commit -m "my first script"
 

 
 
@@ -102,4 +84,4 @@ Finally, upload your changes to the remote repository:
 

 
 
-Although this tutorial shows how to add a python script to Arvados, the same steps can be used to add any of your custom bash, R, or python scripts to an Arvados repository.
+The same steps can be used to add any of your custom bash, R, or python scripts to an Arvados repository.
diff --git a/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid b/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
index 2375e8b3d5..9ddec04f5e 100644
--- a/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Keep collection lifecycle"
+title: "Trashing and untrashing data"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,48 +9,40 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-During it's lifetime, a keep collection can be in various states. These states are *persisted*, *expiring*, *trashed*  and *permanently deleted*.
+Collections have a sophisticated data lifecycle, which is documented in the architecture guide at "Collection lifecycle":{{ site.baseurl }}/architecture/keep-data-lifecycle.html#collection_lifecycle.
 
-A collection is *expiring* when it has a *trash_at* time in the future. An expiring collection can be accessed as normal, but is scheduled to be trashed automatically at the *trash_at* time.
+Arvados supports trashing (deletion) of collections. For a period of time after a collection is trashed, it can be "untrashed". After that period, the collection is permanently deleted, though there may still be ways to recover the data, see "Recovering data":{{ site.baseurl }}/admin/keep-recovering-data.html in the admin guide for more details.
 
-A collection is *trashed* when it has a *trash_at* time in the past. The *is_trashed* attribute will also be "true". The delete operation immediately puts the collection in the trash by setting the *trash_at* time to "now". Once trashed, the collection is no longer readable through normal data access APIs. The collection will have *delete_at* set to some time in the future. The trashed collection is recoverable until the delete_at time passes, at which point the collection is permanently deleted.
-
-# "*Collection lifecycle attributes*":#collection_attributes
-# "*Deleting / trashing collections*":#delete-collection
+# "*Trashing (deleting) collections*":#delete-collection
 # "*Recovering trashed collections*":#trash-recovery
 
 {% include 'tutorial_expectations' %}
 
-h2(#collection_attributes). Collection lifecycle attributes
-
-As listed above the attributes that are used to manage a collection lifecycle are it's *is_trashed*, *trash_at*, and *delete_at*. The table below lists the values of these attributes and how they influence the state of a collection and it's accessibility.
+h2(#delete-collection). Trashing (deleting) collections
 
-table(table table-bordered table-condensed).
-|_. collection state|_. is_trashed|_. trash_at|_. delete_at|_. get|_. list|_. list?include_trash=true|_. can be modified|
-|persisted collection|false |null |null |yes |yes |yes |yes |
-|expiring collection|false |future |future |yes  |yes |yes |yes |
-|trashed collection|true |past |future |no |no |yes |only is_trashed, trash_at and delete_at attribtues|
-|deleted collection|true|past |past |no |no |no |no |
+A collection can be trashed using workbench or the arv command line tool.
 
-h2(#delete-collection). Deleting / trashing collections
+h3. Trashing a collection using workbench
 
-A collection can be deleted using either the arv command line tool or the workbench.
+To trash a collection using workbench, go to the Data collections tab in the project, and use the  trash icon for this collection row.
 
 h3. Trashing a collection using arv command line tool
 
 
-arv collection delete --uuid=qr1hi-4zz18-xxxxxxxxxxxxxxx
+arv collection delete --uuid=zzzzz-4zz18-xxxxxxxxxxxxxxx
 

 
-h3. Trashing a collection using workbench
+h2(#trash-recovery). Recovering trashed collections
 
-To trash a collection using workbench, go to the Data collections tab in the project, and use the trash icon for this collection row.
+A collection can be untrashed / recovered using workbench or the arv command line tool.
 
-h2(#trash-recovery). Recovering trashed collections
+h3. Untrashing a collection using workbench
 
-A collection can be un-trashed / recovered using either the arv command line tool or the workbench.
+To untrash a collection using workbench, go to trash page on workbench by clicking on the "Trash" icon in the top navigation in workbench and use the recycle icon or selection dropdown option.
 
-h3. Un-trashing a collection using arv command line tool
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/trash-button-topnav.png!
+
+h3. Untrashing a collection using arv command line tool
 
 You can list the trashed collections using the list command.
 
@@ -61,11 +53,7 @@ arv collection list --include-trash=true --filters '[["is_trashed", "=", "true"]
 You can then untrash a particular collection using arv using it's uuid.
 
 
-arv collection untrash --uuid=qr1hi-4zz18-xxxxxxxxxxxxxxx
+arv collection untrash --uuid=zzzzz-4zz18-xxxxxxxxxxxxxxx
 

 
-h3. Un-trashing a collection using workbench
-
-To untrash a collection using workbench, go to trash page on workbench by clicking on the "Trash" icon in the top navigation in workbench and use the recycle icon or selection dropdown option.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/trash-button-topnav.png!
+The architecture section has a more detailed description of the "data lifecycle":{{ site.baseurl }}/architecture/keep-data-lifecycle.html  in Keep.
diff --git a/doc/user/tutorials/tutorial-keep-get.html.textile.liquid b/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
index f206d302de..05924f8475 100644
--- a/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-get.html.textile.liquid
@@ -11,11 +11,39 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 Arvados Data collections can be downloaded using either the arv commands or using Workbench.
 
-# "*Downloading using arv commands*":#download-using-arv
-# "*Downloading using Workbench*":#download-using-workbench
-# "*Downloading a shared collection using Workbench*":#download-shared-collection
+# "*Download using Workbench*":#download-using-workbench
+# "*Sharing collections*":#download-shared-collection
+# "*Download using command line tools*":#download-using-arv
 
-h2(#download-using-arv). Downloading using arv commands
+h2(#download-using-workbench). Download using Workbench
+
+You can also download Arvados data collections using the Workbench.
+
+Visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu, select your *Home* project. You will see the *Data collections* tab, which lists the collections in this project.
+
+You can access the contents of a collection by clicking on the * Show* button next to the collection. This will take you to the collection's page. Using this page you can see the collection's contents, and download individual files.
+
+You can now download the collection files by clicking on the  button(s).
+
+h2(#download-shared-collection). Sharing collections
+
+h3. Sharing with other Arvados users
+
+Collections can be shared with other users on the Arvados cluster by sharing the parent project.  Navigate to the parent project using the "breadcrumbs" bar, then click on the *Sharing* tab.  From the sharing tab, you can choose which users or groups to share with, and their level of access.
+
+h3. Creating a special download URL
+
+To share a collection with users that do not have an account on your Arvados cluster, visit the collection page using Workbench as described in the above section. Once on this page, click on the Create sharing link button.
+
+This will create a sharing link for the collection as shown below. You can copy the sharing link in this page and share it with other users.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/shared-collection.png!
+
+A user with this url can download this collection by simply accessing this url using browser. It will present a downloadable version of the collection as shown below.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/download-shared-collection.png!
+
+h2(#download-using-arv). Download using command line tools
 
 {% include 'tutorial_expectations' %}
 
@@ -24,38 +52,35 @@ You can download Arvados data collections using the command line tools @arv-ls@
 Use @arv-ls@ to view the contents of a collection:
 
 
-
~$ arv-ls c1bad4b39ca5a924e481008009d94e32+210
-var-GS000016015-ASM.tsv.bz2
+
~$ arv-ls ae480c5099b81e17267b7445e35b4bc7+180
+./HWI-ST1027_129_D0THKACXX.1_1.fastq
+./HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
-
~$ arv-ls 887cd41e9c613463eab2f0d885c6dd96+83
-alice.txt
-bob.txt
-carol.txt
-

-
-
-Use @-s@ to print file sizes rounded up to the nearest kilobyte:
+Use @-s@ to print file sizes, in kilobytes, rounded up:
 
 
-
~$ arv-ls -s c1bad4b39ca5a924e481008009d94e32+210
-221887 var-GS000016015-ASM.tsv.bz2
+
~$ arv-ls -s ae480c5099b81e17267b7445e35b4bc7+180
+     12258 ./HWI-ST1027_129_D0THKACXX.1_1.fastq
+     12258 ./HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
 
 Use @arv-get@ to download the contents of a collection and place it in the directory specified in the second argument (in this example, @.@ for the current directory):
 
 
-
~$ arv-get c1bad4b39ca5a924e481008009d94e32+210/ .
-~$ ls var-GS000016015-ASM.tsv.bz2
-var-GS000016015-ASM.tsv.bz2
+
~$ $ arv-get ae480c5099b81e17267b7445e35b4bc7+180/ .
+23 MiB / 23 MiB 100.0%
+~$ ls
+HWI-ST1027_129_D0THKACXX.1_1.fastq  HWI-ST1027_129_D0THKACXX.1_2.fastq
 

 
 
 You can also download individual files:
 
 
-
~$ arv-get 887cd41e9c613463eab2f0d885c6dd96+83/alice.txt .
+
~$ arv-get ae480c5099b81e17267b7445e35b4bc7+180/HWI-ST1027_129_D0THKACXX.1_1.fastq .
+11 MiB / 11 MiB 100.0%
 

 
 
@@ -65,33 +90,9 @@ If your cluster is "configured to be part of a federation":{{site.baseurl}}/admi
 
 If you request a collection by portable data hash, it will first search the home cluster, then search federated clusters.
 
-You may also request a collection by UUID.  In this case, it will contact the cluster named in the UUID prefix (in this example, @qr1hi@).
+You may also request a collection by UUID.  In this case, it will contact the cluster named in the UUID prefix (in this example, @zzzzz@).
 
 
-
~$ arv-get qr1hi-4zz18-fw6dnjxtkvzdewt/ .
+
~$ arv-get zzzzz-4zz18-fw6dnjxtkvzdewt/ .
 

 
-
-h2(#download-using-workbench). Downloading using Workbench
-
-You can also download Arvados data collections using the Workbench.
-
-Visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu, select your *Home* project. You will see the *Data collections* tab, which lists the collections in this project.
-
-You can access the contents of a collection by clicking on the * Show* button next to the collection. This will take you to the collection's page. Using this page you can see the collection's contents, download individual files, and set sharing options.
-
-You can now download the collection files by clicking on the  button(s).
-
-h2(#download-shared-collection). Downloading a shared collection using Workbench
-
-Collections can be shared to allow downloads by anonymous users.
-
-To share a collection with anonymous users, visit the collection page using Workbench as described in the above section. Once on this page, click on the Create sharing link button.
-
-This will create a sharing link for the collection as shown below. You can copy the sharing link in this page and share it with other users.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/shared-collection.png!
-
-A user with this url can download this collection by simply accessing this url using browser. It will present a downloadable version of the collection as shown below.
-
-!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/download-shared-collection.png!
diff --git a/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
index e176021992..060ae2acbe 100644
--- a/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Accessing Keep from GNU/Linux"
+title: "Access Keep as a GNU/Linux filesystem"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,17 +9,16 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This tutoral describes how to access Arvados collections on GNU/Linux using traditional filesystem tools by mounting Keep as a file system using @arv-mount@.
+GNU/Linux users can use @arv-mount@ or Gnome to mount Keep as a file system in order to access Arvados collections using traditional filesystem tools.
 
 {% include 'tutorial_expectations' %}
 
-h2. Arv-mount
+# "*Mounting at the command line with arv-mount*":#arv-mount
+# "*Mounting in Gnome File manager*":#gnome
 
-@arv-mount@ provides several features:
+h2(#arv-mount). Arv-mount
 
-* You can browse, open and read Keep entries as if they are regular files.
-* It is easy for existing tools to access files in Keep.
-* Data is streamed on demand.  It is not necessary to download an entire file or collection to start processing.
+@arv-mount@ provides a file system view of Arvados Keep using File System in Userspace (FUSE).  You can browse, open and read Keep entries as if they are regular files, and existing tools can access files in Keep.  Data is streamed on demand.  It is not necessary to download an entire file or collection to start processing.
 
 The default mode permits browsing any collection in Arvados as a subdirectory under the mount directory.  To avoid having to fetch a potentially large list of all collections, collection directories only come into existence when explicitly accessed by UUID or portable data hash. For instance, a collection may be found by its content hash in the @keep/by_id@ directory.
 
@@ -59,3 +58,11 @@ Not supported:
 If multiple clients (separate instances of arv-mount or other arvados applications) modify the same file in the same collection within a short time interval, this may result in a conflict.  In this case, the most recent commit wins, and the "loser" will be renamed to a conflict file in the form @name~YYYYMMDD-HHMMSS~conflict~@.
 
 Please note this feature is in beta testing.  In particular, the conflict mechanism is itself currently subject to race conditions with potential for data loss when a collection is being modified simultaneously by multiple clients.  This issue will be resolved in future development.
+
+h2(#gnome). Mounting in Gnome File manager
+
+As an alternative to @arv-mount@ you can also access the WebDAV mount through the Gnome File manager.
+
+# Open "Files"
+# On the left sidebar, click on "Other Locations"
+# At the bottom of the window, enter @davs://collections.ClusterID.example.com/@  When prompted for credentials, enter username "arvados" and a valid Arvados token in the @Password@ field.
diff --git a/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
index 9397d61e05..911b8808eb 100644
--- a/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Accessing Keep from OS X"
+title: "Access Keep from macOS Finder"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,16 +9,16 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-OS X users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
+Users of macOS can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
 
-h3. Browsing Keep (read-only)
+h3. Browsing Keep in Finder (read-only)
 
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, enter username "arvados" and paste a valid Arvados token for the @Password@ field.
 
 This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
 
 h3. Accessing a specific collection in Keep (read-write)
 
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/c=your-collection-uuid@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
 
 This collection is now accessible read/write.
diff --git a/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid b/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
index 29b28fff9c..a40a997ba1 100644
--- a/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: userguide
-title: "Accessing Keep from Windows"
+title: "Access Keep from Windows File Explorer"
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -11,7 +11,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 Windows users can browse Keep read-only via WebDAV. Specific collections can also be accessed read-write via WebDAV.
 
-h3. Browsing Keep (read-only)
+h3. Browsing Keep in File Explorer (read-only)
 
 Use the 'Map network drive' functionality, and enter @https://collections.ClusterID.example.com/@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid Arvados token in the @Password@ field.
 
diff --git a/doc/user/tutorials/tutorial-keep.html.textile.liquid b/doc/user/tutorials/tutorial-keep.html.textile.liquid
index ec7086db96..21efc475c5 100644
--- a/doc/user/tutorials/tutorial-keep.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-keep.html.textile.liquid
@@ -9,13 +9,44 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-Arvados Data collections can be uploaded using either the @arv-put@ command line tool or using Workbench.
+Arvados Data collections can be uploaded using either Workbench or the @arv-put@ command line tool.
 
-# "*Upload using command line tool*":#upload-using-command
 # "*Upload using Workbench*":#upload-using-workbench
+# "*Creating projects*":#creating-projects
+# "*Upload using command line tool*":#upload-using-command
+
+h2(#upload-using-workbench). Upload using Workbench
+
+To upload using Workbench, visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu and select your *Home* project or any other project of your choosing.  You will see the *Data collections* tab for this project, which lists the collections in this project.
+
+To upload files into a new collection, click on *Add data* dropdown menu and select *Upload files from my computer*.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-using-workbench.png!
+
+
This will create a new empty collection in your chosen project and will take you to the *Upload* tab for that collection.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-tab-in-new-collection.png!
+
+Click on the *Browse...* button and select the files you would like to upload. Selected files will be added to a list of files to be uploaded. After you are done selecting files to upload, click on the * Start* button to start upload. This will start uploading files to Arvados and Workbench will show you the progress bar. When upload is completed, you will see an indication to that effect.
+
+!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/files-uploaded.png!
+
+*Note:* If you leave the collection page during the upload, the upload process will be aborted and you will need to upload the files again.
+
+*Note:* You can also use the Upload tab to add additional files to an existing collection.
 
 notextile. 

 
+h2(#creating-projects). Creating projects
+
+Files are organized into Collections, and Collections are organized by Projects.
+
+Click on *Projects* → *Add a new project* to add a top level project.
+
+To create a subproject, navigate to the parent project, and click on *Add a subproject*.
+
+See "Sharing collections":tutorial-keep-get.html#download-shared-collection for information about sharing projects and collections with other users.
+
 h2(#upload-using-command). Upload using command line tool
 
 {% include 'tutorial_expectations' %}
@@ -25,12 +56,12 @@ To upload a file to Keep using @arv-put@:
 
~$ arv-put var-GS000016015-ASM.tsv.bz2
 216M / 216M 100.0%
 Collection saved as ...
-qr1hi-4zz18-xxxxxxxxxxxxxxx
+zzzzz-4zz18-xxxxxxxxxxxxxxx
 

 
 
 
-The output value @qr1hi-4zz18-xxxxxxxxxxxxxxx@ is the uuid of the Arvados collection created.
+The output value @zzzzz-4zz18-xxxxxxxxxxxxxxx@ is the uuid of the Arvados collection created.
 
 Note: The file used in this example is a freely available TSV file containing variant annotations from the "Personal Genome Project (PGP)":http://www.pgp-hms.org participant "hu599905":https://my.pgp-hms.org/profile/hu599905), downloadable "here":https://warehouse.pgp-hms.org/warehouse/f815ec01d5d2f11cb12874ab2ed50daa+234+K@ant/var-GS000016015-ASM.tsv.bz2. Alternatively, you can replace @var-GS000016015-ASM.tsv.bz2@ with the name of any file you have locally, or you could get the TSV file by "downloading it from Keep.":{{site.baseurl}}/user/tutorials/tutorial-keep-get.html
 
@@ -44,7 +75,7 @@ Note: The file used in this example is a freely available TSV file containing va
 ~$ arv-put tmp
 0M / 0M 100.0%
 Collection saved as ...
-qr1hi-4zz18-yyyyyyyyyyyyyyy
+zzzzz-4zz18-yyyyyyyyyyyyyyy
 

 
 
@@ -63,23 +94,3 @@ To move the collection to a different project, check the box at the left of the
 Click on the * Show* button next to the collection's listing on a project page to go to the Workbench page for your collection.  On this page, you can see the collection's contents, download individual files, and set sharing options.
 
 notextile.
- -h2(#upload-using-workbench). Upload using Workbench - -To upload using Workbench, visit the Workbench *Dashboard*. Click on *Projects* dropdown menu in the top navigation menu and select your *Home* project or any other project of your choosing. You will see the *Data collections* tab for this project, which lists the collections in this project. - -To upload files into a new collection, click on *Add data* dropdown menu and select *Upload files from my computer*. - -!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-using-workbench.png! - -
This will create a new empty collection in your chosen project and will take you to the *Upload* tab for that collection. - -!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/upload-tab-in-new-collection.png! - -Click on the *Browse...* button and select the files you would like to upload. Selected files will be added to a list of files to be uploaded. After you are done selecting files to upload, click on the * Start* button to start upload. This will start uploading files to Arvados and Workbench will show you the progress bar. When upload is completed, you will see an indication to that effect. - -!{display: block;margin-left: 25px;margin-right: auto;border:1px solid lightgray;}{{ site.baseurl }}/images/files-uploaded.png! - -*Note:* If you leave the collection page during the upload, the upload process will be aborted and you will need to upload the files again. - -*Note:* You can also use the Upload tab to add additional files to an existing collection. diff --git a/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid b/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid index 8dcb8e674e..8a08225723 100644 --- a/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid +++ b/doc/user/tutorials/tutorial-workflow-workbench.html.textile.liquid @@ -23,13 +23,15 @@ notextile.
# Start from the *Workbench Dashboard*. You can access the Dashboard by clicking on * Dashboard* in the upper left corner of any Workbench page. # Click on the Run a process... button. This will open a dialog box titled *Choose a pipeline or workflow to run*. -# In the search box, type in *Tutorial bwa mem cwl*. -# Select * Tutorial bwa mem cwl* and click the Next: choose inputs button. This will create a new process in your *Home* project and will open it. You can now supply the inputs for the process. Please note that all required inputs are populated with default values and you can change them if you prefer. -# For example, let's see how to change *"reference" parameter* for this workflow. Click the Choose button beneath the *"reference" parameter* header. This will open a dialog box titled *Choose a dataset for "reference" parameter for cwl-runner in bwa-mem.cwl component*. -# Open the *Home * menu and select *All Projects*. Search for and select * Tutorial chromosome 19 reference*. You will then see a list of files. Select * 19-fasta.bwt* and click the OK button. -# Repeat the previous two steps to set the *"read_p1" parameter for cwl-runner script in bwa-mem.cwl component* and *"read_p2" parameter for cwl-runner script in bwa-mem.cwl component* parameters. -# Click on the Run button. The page updates to show you that the process has been submitted to run on the Arvados cluster. -# After the process starts running, you can track the progress by watching log messages from the component(s). This page refreshes automatically. You will see a complete label when the process completes successfully. +# In the search box, type in *bwa-mem.cwl*. +# Select * bwa-mem.cwl* and click the Next: choose inputs button. This will create a new process in your *Home* project and will open it. You can now supply the inputs for the process. Please note that all required inputs are populated with default values and you can change them if you prefer. +# For example, let's see how to set read pair *read_p1* and *read_p2* for this workflow. Click the Choose button beneath the *read_p1* header. This will open a dialog box titled *Choose a file*. +# In the file dialog, click on *Home * menu and then select *All Projects*. +# Enter *HWI-ST1027* into the search box. You will see one or more collections. Click on * HWI-ST1027_129_D0THKACXX for CWL tutorial* +# The right hand panel will list two files. Click on the first one ending in "_1" and click the OK button. +# Repeat the steps 5-8 to set the *read_p2* except selecting the second file ending in "_2" +# Scroll to the bottom of the "Inputs" panel and click on the Run button. The page updates to show you that the process has been submitted to run on the Arvados cluster. +# Once the process starts running, you can track the progress by watching log messages from the component(s). This page refreshes automatically. You will see a complete label when the process completes successfully. # Click on the *Output* link to see the results of the process. This will load a new page listing the output files from this process. You'll see the output SAM file from the alignment tool under the *Files* tab. # Click on the download button to the right of the SAM file to download your results. diff --git a/doc/user/tutorials/wgs-tutorial.html.textile.liquid b/doc/user/tutorials/wgs-tutorial.html.textile.liquid new file mode 100644 index 0000000000..a68d7ca21e --- /dev/null +++ b/doc/user/tutorials/wgs-tutorial.html.textile.liquid @@ -0,0 +1,357 @@ +--- +layout: default +navsection: userguide +title: "Processing Whole Genome Sequences" +... +{% comment %} +Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: CC-BY-SA-3.0 +{% endcomment %} + +
+ +h2. 1. A Brief Introduction to Arvados + +Arvados is an open source platform for managing, processing, and sharing genomic and other large scientific and biomedical data. Arvados helps bioinformaticians run and scale compute-intensive workflows. By running their workflows in Arvados, they can scale their calculations dynamically in the cloud, track methods and datasets, and easily re-run workflow steps or whole workflows when necessary. This tutorial walkthrough shows examples of running a “real-world” workflow and how to navigate and use the Arvados working environment. + +When you log into your account on the Arvados playground ("https://playground.arvados.org":https://playground.arvados.org), you see the Arvados Workbench which is the web application that allows users to interactively access Arvados functionality. For this tutorial, we will largely focus on using the Arvados Workbench since that is an easy way to get started using Arvados. You can also access Arvados via your command line and/or using the available REST API and SDKs. If you are interested, this tutorial walkthrough will have an optional component that will cover using the command line. + +By using the Arvados Workbench or using the command line, you can submit your workflows to run on your Arvados cluster. An Arvados cluster can be hosted in the cloud as well as on premise and on hybrid clusters. The Arvados playground cluster is currently hosted in the cloud. + +You can also use the workbench or command line to access data in the Arvados storage system called Keep which is designed for managing and storing large collections of files on your Arvados cluster. The running of workflows is managed by Crunch. Crunch is designed to maintain data provenance and workflow reproducibility. Crunch automatically tracks data inputs and outputs through Keep and executes workflow processes in Docker containers. In a cloud environment, Crunch optimizes costs by scaling compute on demand. + +_Ways to Learn More About Arvados_ +* To learn more in general about Arvados, please visit the Arvados website here: "https://arvados.org/":https://arvados.org/ +* For a deeper dive into Arvados, the Arvados documentation can be found here: "https://doc.arvados.org/":https://doc.arvados.org/ +* For help on Arvados, visit the Gitter channel here: "https://gitter.im/arvados/community":https://gitter.im/arvados/community + + +h2. 2. A Brief Introduction to the Whole Genome Sequencing (WGS) Processing Tutorial + +The workflow used in this tutorial walkthrough serves as a “real-world” workflow example that takes in WGS data (paired FASTQs) and returns GVCFs and accompanying variant reports. In this walkthrough, we will be processing approximately 10 public genomes made available by the Personal Genome Project. This set of data is from the PGP-UK ("https://www.personalgenomes.org.uk/":https://www.personalgenomes.org.uk/). + +The overall steps in the workflow include: +* Check of FASTQ quality using FastQC ("https://www.bioinformatics.babraham.ac.uk/projects/fastqc/":https://www.bioinformatics.babraham.ac.uk/projects/fastqc/) +* Local alignment using BWA-MEM ("http://bio-bwa.sourceforge.net/bwa.shtml":http://bio-bwa.sourceforge.net/bwa.shtml) +* Variant calling in parallel using GATK Haplotype Caller ("https://gatk.broadinstitute.org/hc/en-us":https://gatk.broadinstitute.org/hc/en-us) +* Generation of an HTML report comparing variants against ClinVar archive ("https://www.ncbi.nlm.nih.gov/clinvar/":https://www.ncbi.nlm.nih.gov/clinvar/) + +The workflow is written in "Common Workflow Language":https://commonwl.org (CWL), the primary way to develop and run workflows for Arvados. + +Below are diagrams of the main workflow which runs the processing across multiple sets of fastq and the main subworkflow (run multiple times in parallel by the main workflow) which processes a single set of FASTQs. This main subworkflow also calls other additional subworkflows including subworkflows that perform variant calling using GATK in parallel by regions and generate the ClinVar HTML variant report. These CWL diagrams (generated using "CWL viewer":https://view.commonwl.org) will give you a basic idea of the flow, input/outputs and workflow steps involved in the tutorial example. However, if you aren’t used to looking at CWL workflow diagrams and/or aren’t particularly interested in this level of detail, do not worry. You will not need to know these particulars to run the workflow. + +
!{width: 100%}{{ site.baseurl }}/images/wgs-tutorial/image2.png! +
_*Figure 1*: Main CWL Workflow for WGS Processing Tutorial. This runs the same WGS subworkflow over multiple pairs FASTQs files._
+ +
!{width: 100%}{{ site.baseurl }}/images/wgs-tutorial/image3.png! +
_*Figure 2*: Main subworkflow for the WGS Processing Tutorial. This subworkflow does alignment, deduplication, variant calling and reporting._
+ +_Ways to Learn More About CWL_ + +* The CWL website has lots of good content including the CWL User Guide: "https://www.commonwl.org/":https://www.commonwl.org/ +* Commonly Asked Questions and Answers can be found in the Discourse Group, here: "https://cwl.discourse.group/":https://cwl.discourse.group/ +* For help on CWL, visit the Gitter channel here: "https://gitter.im/common-workflow-language/common-workflow-language":https://gitter.im/common-workflow-language/common-workflow-language +* Repository of CWL CommandLineTool descriptions for commons tools in bioinformatics: +"https://github.com/common-workflow-library/bio-cwl-tools/":https://github.com/common-workflow-library/bio-cwl-tools/ + + +h2. 3. Setting Up to Run the WGS Processing Workflow + +Let’s get a little familiar with the Arvados Workbench while also setting up to run the WGS processing tutorial workflow. Logging into the workbench will present you with the Dashboard. This gives a summary of your projects and recent activity in your Arvados instance, i.e. the Arvados Playground. The Dashboard will only give you information about projects and activities that you have permissions to view and/or access. Other users' private or restricted projects and activities will not be visible by design. + +h3. 3a. Setting up a New Project + +Projects in Arvados help you organize and track your work - and can contain data, workflow code, details about workflow runs, and results. Let’s begin by setting up a new project for the work you will be doing in this walkthrough. + +To create a new project, go to the Projects dropdown menu and select “Add a New Project”. + +
!{width: 100%}{{ site.baseurl }}/images/wgs-tutorial/image4.png! +
_*Figure 3*: Adding a new project using Arvados Workbench._
+ +Let’s name your project “WGS Processing Tutorial”. You can also add a description of your project using the *Edit* button. The universally unique identifier (UUID) of the project can be found in the URL. + +
!{width: 100%}{{ site.baseurl }}/images/wgs-tutorial/image6.png! +
_*Figure 4*: Renaming new project using Arvados Workbench. The UUID of the project can be found in the URL and is highlighted in yellow in this image for emphasis._
+ +If you choose to use another name for your project, just keep in mind when the project name is referenced in the walkthrough later on. + +h3. 3b. Working with Collections + +Collections in Arvados help organize and manage your data. You can upload your existing data into a collection or reuse data from one or more existing collections. Collections allow us to reorganize our files without duplicating or physically moving the data, making them very efficient to use even when working with terabytes of data. Each collection has a universally unique identifier (collection UUID). This is a constant for this collection, even if we add or remove files -- or rename the collection. You use this if we want to to identify the most recent version of our collection to use in our workflows. + +Arvados uses a content-addressable filesystem (i.e. Keep) where the addresses of files are derived from their contents. A major benefit of this is that Arvados can then verify that when a dataset is retrieved it is the dataset you requested and can track the exact datasets that were used for each of our previous calculations. This is what allows you to be certain that we are always working with the data that you think you are using. You use the content address of a collection when you want to guarantee that you use the same version as input to your workflow. + +
!{width: 100%}{{ site.baseurl }}/images/wgs-tutorial/image1.png! +
_*Figure 5*: A collection in Arvados as viewed via the Arvados Workbench. On the upper left you will find a panel that contains: the name of the collection (editable), a description of the collection (editable), the collection UUID and the content address and content size._
+ +Let’s start working with collections by copying the existing collection that stores the FASTQ data being processed into our new “WGS Processing Tutorial” project. + +First, you must find the collection you are interested in copying over to your project. There are several ways to search for a collection: by collection name, by UUID or by content address. In this case, let’s search for our collection by name. + +In this case it is called “PGP UK FASTQs” and by searching for it in the “search this site” box. It will come up and you can navigate to it. You would do similarly if you would want to search by UUID or content address. + +Now that you have found the collection of FASTQs you want to copy to your project, you can simply use the Copy to project... button and select your new project to copy the collection there. You can rename your collection whatever you wish, or use the default name on copy and add whatever description you would like. + + + +We want to do the same thing for the other inputs to our WGS workflow. Similar to the “PGP UK FASTQs” collection there is a collection of inputs entitled “WGS Processing reference data” and that collection can be copied over in a similar fashion. + +Now that we are a bit more familiar with the Arvados Workbench, projects and collections. Let’s move onto running a workflow. + +h2. 4. Running the WGS Processing Workflow + +In this section, we will be discussing three ways to run the tutorial workflow using Arvados. We will start using the easiest way and then progress to the more involved ways to run a workflow via the command line which will allow you more control over your inputs, workflow parameters and setup. Feel free to end your walkthrough after the first way or to pick and choose the ways that appeal the most to you, fit your experience and/or preferred way of working. + +h3. 4a. Interactively Running a Workflow Using Workbench + +Workflows can be registered in Arvados. Registration allows you to share a workflow with other Arvados users, and let’s them run the workflow by clicking the Run a process… button on the Workbench Dashboard and on the command line by specifying the workflow UUID. Default values can be specified for workflow inputs. + +We have already previously registered the WGS workflow and set default input values for this set of the walkthrough. + +Let’s find the the registered WGS Processing Workflow and run it interactively in our newly created project. + +# To find the registered workflow, you can search for it in the search box located in the top right corner of the Arvados Workbench by looking for the name “WGS Processing Workflow”. +# Once you have found the registered workflow, you can run it your project by using the Run this workflow.. button and selecting your project ("WGS Processing Tutorial") that you set up in Section 3a. +# Default inputs to the registered workflow will be automatically filled in. These inputs will still work. You can verify this by checking the addresses of the collections you copied over to your New Project. +# The input *Directory of paired FASTQ files* will need to be set. Click on Choose button, select "PGP UK FASTQs" in the *Choose a dataset* dialog and then click OK. +# Now, you can submit your workflow by scrolling to the bottom of the page and hitting the Run button. + +Congratulations! You have now submitted your workflow to run. You can move to Section 5 to learn how to check the state of your submitted workflow and Section 6 to learn how to examine the results of and logs from your workflow. + +Let’s now say instead of running a registered workflow you want to run a workflow using the command line. This is a completely optional step in the walkthrough. To do this, you can specify cwl files to define the workflow you want to run and the yml files to specify the inputs to our workflow. In this walkthrough we will give two options (4b) and (4c) for running the workflow on the commandline. Option 4b uses a virtual machine provided by Arvados made accessible via a browser that requires no additional setup. Option 4c allows you to submit from your personal machine but you must install necessary packages and edit configurations to allow you to submit to the Arvados cluster. Please choose whichever works best for you. + +h3. 4b. Optional: Setting up to Run a Workflow Using Command Line and an Arvados Virtual Machine + +Arvados provides a virtual machine which has all the necessary client-side libraries installed to submit to your Arvados cluster using the command line. Webshell gives you access to an Arvados Virtual Machine (VM) from your browser with no additional setup. You can access webshell through the Arvados Workbench. It is the easiest way to try out submitting a workflow to Arvados via the command line. + +New users are playground are automatically given access to a shell account. + +_Note_: the shell accounts are created on an interval and it may take up to two minutes from your initial log in before the shell account is created. + +You can follow the instructions here to access the machine using the browser (also known as using webshell): +* "Accessing an Arvados VM with Webshell":{{ site.baseurl }}/user/getting_started/vm-login-with-webshell.html + +Arvados also allows you to ssh into the shell machine and other hosted VMs instead of using the webshell capabilities. However this tutorial does not cover that option in-depth. If you like to explore it on your own, you can allow the instructions in the documentation here: +* "Accessing an Arvados VM with SSH - Unix Environments":{{ site.baseurl }}/user/getting_started/ssh-access-unix.html +* "Accessing an Arvados VM with SSH - Windows Environments":{{ site.baseurl }}/user/getting_started/ssh-access-windows.html + +Once you can use webshell, you can proceed to section *“4d. Running a Workflow Using the Command Line”* . + +h3. 4c. Optional: Setting up to Run a Workflow Using Command Line and Your Computer + +Instead of using a virtual machine provided by Arvados, you can install the necessary libraries and configure your computer to be able to submit to your Arvados cluster directly. This is more of an advanced option and is for users who are comfortable installing software and libraries and configuring them on their machines. + +To be able to submit workflows to the Arvados cluster, you will need to install the Python SDK on your machine. Additional features can be made available by installing additional libraries, but this is the bare minimum you need to install to do this walkthrough tutorial. You can follow the instructions in the Arvados documentment to install the Python SDK and set the appropriate configurations to access the Arvados Playground. + +* "Installing the Arvados CWL Runner":{{ site.baseurl }}/sdk/python/arvados-cwl-runner.html +* "Setting Configurations to Access the Arvados Playground":{{ site.baseurl }}/user/reference/api-tokens.html + +Once you have your machine set up to submit to the Arvados Playground Cluster, you can proceed to section *“4d. Running a Workflow Using the Command Line”* . + +h3. 4d. Optional: Running a Workflow Using the Command Line + +Now that we have access to a machine that can submit to the Arvados Playground, let’s download the relevant files containing the workflow description and inputs. + +First, we will +* Clone the tutorial repository from GitHub ("https://github.com/arvados/arvados-tutorial":https://github.com/arvados/arvados-tutorial) +* Change directories into the WGS tutorial folder + +
$ git clone https://github.com/arvados/arvados-tutorial.git
+$ cd arvados-tutorial/WGS-processing
+
+ +Recall that CWL is a way to describe command line tools and connect them together to create workflows. YML files can be used to specify input values into these individual command line tools or overarching workflows. + +The tutorial directories are as follows: +* @cwl@ - contains CWL descriptions of workflows and command line tools for the tutorial +* @yml@ - contains YML files for inputs for the main workflow or to test subworkflows command line tools +* @src@ - contains any source code necessary for the tutorial +* @docker@ - contains dockerfiles necessary to re-create any needed docker images used in the tutorial + +Before we run the WGS processing workflow, we want to adjust the inputs to match those in your new project. The workflow that we want to submit is described by the file @/cwl/@ and the inputs are given by the file @/yml/@. Note: while all the cwl files are needed to describe the full workflow only the single yml with the workflow inputs is needed to run the workflow. The additional yml files (in the helper folder) are provided for testing purposes or if one might want to test or run an underlying subworkflow or cwl for a command line tool by itself. + +Several of the inputs in the yml file point to original content addresses of collections that you make copies of in our New Project. These still work because even though we made copies of the collections into our new project we haven’t changed the underlying contents. However, by changing this file is in general how you would alter the inputs in the accompanying yml file for a given workflow. + +The command to submit to the Arvados Playground Cluster is @arvados-cwl-runner@. +To submit the WGS processing workflow , you need to run the following command replacing YOUR_PROJECT_UUID with the UUID of the new project you created for this tutorial. + +
$ arvados-cwl-runner --no-wait --project-uuid YOUR_PROJECT_UUID ./cwl/wgs-processing-wf.cwl ./yml/wgs-processing-wf.yml
+
+ +The @--no-wait@ option will submit the workflow to Arvados, print out the UUID of the job that was submitted to standard output, and exit instead of waiting until the job is finished to return the command prompt. + +The @--project-uuid@ option specifies the project you want the workflow to run in, that means the outputs and log collections as well as the workflow process will be saved in that project + +If the workflow submitted successfully, you should see the following at the end of the output to the screen + +
INFO Final process status is success
+
+ +Now, you are ready to check the state of your submitted workflow. + +h2. 5. Checking the State Of a Submitted Workflow + +Once you have submitted your workflow, you can examine its state interactively using the Arvados Workbench. If you aren’t already viewing your workflow process on the workbench, there several ways to get to your submitted workflow. Here are two of the simplest ways: + +* Via the Dashboard: It should be listed at the top of the list of “Recent Processes”. Just click on the name of your submitted workflow and it will take you to the submitted workflow information. +* Via Your Project: You will want to go back to your new project, using the Projects pulldown menu or searching for the project name. Note: You can mark a Project as a favorite (if/when you have multiple Projects) to make it easier to find on the pulldown menu using the star next to the project name on the project page. + +The process you will be looking for will be titled “WGS processing workflow scattered over samples”(if you submitted via the command line) or NAME OF REGISTERED WORKFLOW container (if you submitted via the Registered Workflow). + +Once you have found your workflow, you can clearly see the state of the overall workflow and underlying steps below by their label. + +Common states you will see are as follows: + +* Queued - Workflow or step is waiting to run +* Running or Active - Workflow is currently running +* Complete - Workflow or step has successfully completed +* Failing - Workflow is running but has steps that have failed +* Failed - Workflow or step did not complete successfully +* Cancelled - Workflow or step was either manually cancelled or was canceled by Arvados due to a system error + +Since Arvados Crunch reuses steps and workflows if possible, this workflow should run relatively quickly since this workflow has been run before and you have access to those previously run steps. You may notice an initial period where the top level job shows the option of canceling while the other steps are filled in with already finished steps. + +h2. 6. Examining a Finished Workflow + +Once your workflow has finished, you can see how long it took the workflow to run, see scaling information, and examine the logs and outputs. Outputs will be only available for steps that have been successfully completed. Outputs will be saved for every step in the workflow and be saved for the workflow itself. Outputs are saved in collections. You can access each collection by clicking on the link corresponding to the output. + +
!{width: 100%}{{ site.baseurl }}/images/wgs-tutorial/image5.png! +
_*Figure 6*: A completed workflow process in Arvados as viewed via the Arvados Workbench. You can click on the outputs link (highlighted in yellow) to view the outputs. Outputs of a workflow are stored in a collection._
+ +If we click on the outputs of the workflow, we will see the output collection. + +Contained in this collection, is the GVCF, tabix index file, and html ClinVar report for each analyzed sample (e.g. set of FASTQs). By clicking on the download button to the right of the file, you can download it to your local machine. You can also use the command line to download single files or whole collections to your machine. You can examine the outputs of a step similarly by using the arrow to expand the panel to see more details. + +Logs for the main process can be found in the Log tab. There several logs available, so here is a basic summary of what some of the more commonly used logs contain. Let's first define a few terms that will help us understand what the logs are tracking. + +As you may recall, Arvados Crunch manages the running of workflows. A _container request_ is an order sent to Arvados Crunch to perform some computational work. Crunch fulfils a request by either choosing a worker node to execute a container, or finding an identical/equivalent container that has already run. You can use _container request_ or _container_ to distinguish between a work order that is submitted to be run and a work order that is actually running or has been run. So our container request in this case is just the submitted workflow we sent to the Arvados cluster. + +A _node_ is a compute resource where Arvardos can schedule work. In our case since the Arvados Playground is running on a cloud, our nodes are virtual machines. @arvados-cwl-runner@ (acr) executes CWL workflows by submitting the individual parts to Arvados as containers and crunch-run is an internal component that runs on nodes and executes containers. + +* @stderr.txt@ +** Captures everything written to standard error by the programs run by the executing container +* @node-info.txt@ and @node.json@ +** Contains information about the nodes that executed this container. For the Arvados Playground, this gives information about the virtual machine instance that ran the container. +node.json gives a high level overview about the instance such as name, price, and RAM while node-info.txt gives more detailed information about the virtual machine (e.g. cpu of each processor) +* @crunch-run.txt@ and @crunchstat.txt@ +** @crunch-run.txt@ has info about how the container's execution environment was set up (e.g., time spent loading the docker image) and timing/results of copying output data to Keep (if applicable) +** @crunchstat.txt@ has info about resource consumption (RAM, cpu, disk, network) by the container while it was running. +* @container.json@ +** Describes the container (unit of work to be done), contains CWL code, runtime constraints (RAM, vcpus) amongst other details +* @arv-mount.txt@ +** Contains information using Arvados Keep on the node executing the container +* @hoststat.txt@ +** Contains about resource consumption (RAM, cpu, disk, network) on the node while it was running +This is different from the log crunchstat.txt because it includes resource consumption of Arvados components that run on the node outside the container such as crunch-run and other processes related to the Keep file system. + +For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “workflow runner”. It tracks which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then creates the necessary container requests. Each step has its own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection. + +Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “workflow runner” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following... + +
2020-06-22T20:30:04.737703197Z INFO /usr/bin/arvados-cwl-runner 2.0.3, arvados-python-client 2.0.3, cwltool 1.0.20190831161204
+2020-06-22T20:30:04.743250012Z INFO Resolved '/var/lib/cwl/workflow.json#main' to 'file:///var/lib/cwl/workflow.json#main'
+2020-06-22T20:30:20.749884298Z INFO Using empty collection d41d8cd98f00b204e9800998ecf8427e+0
+[removing some log contents here for brevity]
+2020-06-22T20:30:35.629783939Z INFO Running inside container su92l-dz642-uaqhoebfh91zsfd
+2020-06-22T20:30:35.741778080Z INFO [workflow WGS processing workflow] start
+2020-06-22T20:30:35.741778080Z INFO [workflow WGS processing workflow] starting step getfastq
+2020-06-22T20:30:35.741778080Z INFO [step getfastq] start
+2020-06-22T20:30:36.085839313Z INFO [step getfastq] completed success
+2020-06-22T20:30:36.212789670Z INFO [workflow WGS processing workflow] starting step bwamem-gatk-report
+2020-06-22T20:30:36.213545871Z INFO [step bwamem-gatk-report] start
+2020-06-22T20:30:36.234224197Z INFO [workflow bwamem-gatk-report] start
+2020-06-22T20:30:36.234892498Z INFO [workflow bwamem-gatk-report] starting step fastqc
+2020-06-22T20:30:36.235154798Z INFO [step fastqc] start
+2020-06-22T20:30:36.237328201Z INFO Using empty collection d41d8cd98f00b204e9800998ecf8427e+0
+
+ +You can see the output of all the work that arvados-cwl-runner does by managing the execution of the CWL workflow and all the underlying steps and subworkflows. + +Now, let’s explore the logs for a step in the workflow. Remember that those logs can be found by expanding the steps and clicking on the link to the log collection. Let’s look at the log for the step that does the alignment. That step is named bwamem-samtools-view. We can see there are 10 of them because we are aligning 10 genomes. Let’s look at *bwamem-samtools-view2.* + +We click the arrow to open up the step, and then can click on the log collection to access the logs. You may notice there are two sets of seemingly identical logs. One listed under a directory named for a container and one up in the main directory. This is done in case your step had to be automatically re-run due to any issues and gives the logs of each re-run. The logs in the main directory are the logs for the successful run. In most cases this does not happen, you will just see one directory and one those logs will match the logs in the main directory. Let’s open the logs labeled node-info.txt and stderr.txt. + +@node-info.txt@ gives us information about detailed information about the virtual machine this step was run on. The tail end of the log should look like the following: + +
Memory Information
+MemTotal:       64465820 kB
+MemFree:        61617620 kB
+MemAvailable:   62590172 kB
+Buffers:           15872 kB
+Cached:          1493300 kB
+SwapCached:            0 kB
+Active:          1070868 kB
+Inactive:        1314248 kB
+Active(anon):     873716 kB
+Inactive(anon):     8444 kB
+Active(file):     197152 kB
+Inactive(file):  1305804 kB
+Unevictable:           0 kB
+Mlocked:               0 kB
+SwapTotal:             0 kB
+SwapFree:              0 kB
+Dirty:               952 kB
+Writeback:             0 kB
+AnonPages:        874968 kB
+Mapped:           115352 kB
+Shmem:              8604 kB
+Slab:             251844 kB
+SReclaimable:     106580 kB
+SUnreclaim:       145264 kB
+KernelStack:        5584 kB
+PageTables:         3832 kB
+NFS_Unstable:          0 kB
+Bounce:                0 kB
+WritebackTmp:          0 kB
+CommitLimit:    32232908 kB
+Committed_AS:    2076668 kB
+VmallocTotal:   34359738367 kB
+VmallocUsed:           0 kB
+VmallocChunk:          0 kB
+Percpu:             5120 kB
+AnonHugePages:    743424 kB
+ShmemHugePages:        0 kB
+ShmemPmdMapped:        0 kB
+HugePages_Total:       0
+HugePages_Free:        0
+HugePages_Rsvd:        0
+HugePages_Surp:        0
+Hugepagesize:       2048 kB
+Hugetlb:               0 kB
+DirectMap4k:      155620 kB
+DirectMap2M:     6703104 kB
+DirectMap1G:    58720256 kB
+
+Disk Space
+Filesystem      1M-blocks  Used Available Use% Mounted on
+/dev/nvme1n1p1       7874  1678      5778  23% /
+/dev/mapper/tmp    381746  1496    380251   1% /tmp
+
+Disk INodes
+Filesystem         Inodes IUsed     IFree IUse% Mounted on
+/dev/nvme1n1p1     516096 42253    473843    9% /
+/dev/mapper/tmp 195549184 44418 195504766    1% /tmp
+
+ +We can see all the details of the virtual machine used for this step, including that it has 16 cores and 64 GIB of RAM. + +@stderr.txt@ gives us everything written to standard error by the programs run in this step. This step ran successfully so we don’t need to use this to debug our step currently. We are just taking a look for practice. + +The tail end of our log should be similar to the following: + +
2020-08-04T04:37:19.674225566Z [main] CMD: /bwa-0.7.17/bwa mem -M -t 16 -R @RG\tID:sample\tSM:sample\tLB:sample\tPL:ILLUMINA\tPU:sample1 -c 250 /keep/18657d75efb4afd31a14bb204d073239+13611/GRCh38_no_alt_plus_hs38d1_analysis_set.fna /keep/a146a06222f9a66b7d141e078fc67660+376237/ERR2122554_1.fastq.gz /keep/a146a06222f9a66b7d141e078fc67660+376237/ERR2122554_2.fastq.gz
+2020-08-04T04:37:19.674225566Z [main] Real time: 35859.344 sec; CPU: 553120.701 sec
+
+ +This is the command we ran to invoke bwa-mem, and the scaling information for running bwa-mem multi-threaded across 16 cores (15.4x). + +We hope that now that you have a bit more familiarity with the logs you can continue to use them to debug and optimize your own workflows as you move forward with using Arvados if your own work in the future. + +h2. 7. Conclusion + +Thank you for working through this walkthrough tutorial. Hopefully this tutorial has helped you get a feel for working with Arvados. This tutorial just covered the basic capabilities of Arvados. There are many more capabilities to explore. Please see the links featured at the end of Section 1 for ways to learn more about Arvados or get help while you are working with Arvados. + +If you would like help setting up your own production instance of Arvados, please contact us at "info@curii.com.":mailto:info@curii.com + +
diff --git a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid index dd537c46ac..0166b8b525 100644 --- a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid +++ b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid @@ -1,7 +1,7 @@ --- layout: default navsection: userguide -title: "Writing a CWL workflow" +title: "Developing workflows with CWL" ... {% comment %} Copyright (C) The Arvados Authors. All rights reserved. @@ -15,7 +15,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0 h2. Developing workflows -For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.1 . +For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.2 . See "Writing Portable High-Performance Workflows":{{site.baseurl}}/user/cwl/cwl-style.html and "Arvados CWL Extensions":{{site.baseurl}}/user/cwl/cwl-extensions.html for additional information about using CWL on Arvados. @@ -23,65 +23,6 @@ See "Repositories of CWL Tools and Workflows":https://www.commonwl.org/#Reposito See "Software for working with CWL":https://www.commonwl.org/#Software_for_working_with_CWL for links to software tools to help create CWL documents. -h2. Using Composer - -You can create new workflows in the browser using "Arvados Composer":{{site.baseurl}}/user/composer/composer.html - -h2. Registering a workflow to use in Workbench - -Use @--create-workflow@ to register a CWL workflow with Arvados. This enables you to share workflows with other Arvados users, and run them by clicking the Run a process... button on the Workbench Dashboard and on the command line by UUID. - - -
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --create-workflow bwa-mem.cwl
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Upload local files: "bwa-mem.cwl"
-2016-07-01 12:21:01 arvados.arv-run[15796] INFO: Uploaded to qr1hi-4zz18-7e0hedrmkuyoei3
-2016-07-01 12:21:01 arvados.cwl-runner[15796] INFO: Created template qr1hi-p5p6p-rjleou1dwr167v5
-qr1hi-p5p6p-rjleou1dwr167v5
-
- - -You can provide a partial input file to set default values for the workflow input parameters. You can also use the @--name@ option to set the name of the workflow: - - -
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner --name "My workflow with defaults" --create-workflow bwa-mem.cwl bwa-mem-template.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Upload local files: "bwa-mem.cwl"
-2016-07-01 14:09:50 arvados.arv-run[3730] INFO: Uploaded to qr1hi-4zz18-0f91qkovk4ml18o
-2016-07-01 14:09:50 arvados.cwl-runner[3730] INFO: Created template qr1hi-p5p6p-0deqe6nuuyqns2i
-qr1hi-p5p6p-zuniv58hn8d0qd8
-
- - -h3. Running registered workflows at the command line - -You can run a registered workflow at the command line by its UUID: - - -
~/arvados/doc/user/cwl/bwa-mem$ arvados-cwl-runner qr1hi-p5p6p-zuniv58hn8d0qd8 --help
-/home/peter/work/scripts/venv/bin/arvados-cwl-runner 0d62edcb9d25bf4dcdb20d8872ea7b438e12fc59 1.0.20161209192028, arvados-python-client 0.1.20161212125425, cwltool 1.0.20161207161158
-Resolved 'qr1hi-p5p6p-zuniv58hn8d0qd8' to 'keep:655c6cd07550151b210961ed1d3852cf+57/bwa-mem.cwl'
-usage: qr1hi-p5p6p-zuniv58hn8d0qd8 [-h] [--PL PL] --group_id GROUP_ID
-                                   --read_p1 READ_P1 [--read_p2 READ_P2]
-                                   [--reference REFERENCE] --sample_id
-                                   SAMPLE_ID
-                                   [job_order]
-
-positional arguments:
-  job_order             Job input json file
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --PL PL
-  --group_id GROUP_ID
-  --read_p1 READ_P1     The reads, in fastq format.
-  --read_p2 READ_P2     For mate paired reads, the second file (optional).
-  --reference REFERENCE
-                        The index files produced by `bwa index`
-  --sample_id SAMPLE_ID
-
- - h2. Using cwltool When developing a workflow, it is often helpful to run it on the local host to avoid the overhead of submitting to the cluster. To execute a workflow only on the local host (without submitting jobs to an Arvados cluster) you can use the @cwltool@ command. Note that when using @cwltool@ you must have the input data accessible on the local file system using either @arv-mount@ or @arv-get@ to fetch the data from Keep. @@ -150,60 +91,3 @@ Final process status is success If you get the error @JavascriptException: Long-running script killed after 20 seconds.@ this may be due to the Dockerized Node.js engine taking too long to start. You may address this by installing Node.js locally (run @apt-get install nodejs@ on Debian or Ubuntu) or by specifying a longer timeout with the @--eval-timeout@ option. For example, run the workflow with @cwltool --eval-timeout=40@ for a 40-second timeout. - -h2. Making workflows directly executable - -You can make a workflow file directly executable (@cwl-runner@ should be an alias to @arvados-cwl-runner@) by adding the following line to the top of the file: - - -
#!/usr/bin/env cwl-runner
-
- - - -
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem.cwl bwa-mem-input.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
-{
-    "aligned_sam": {
-        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
-        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
-        "class": "File",
-        "size": 30738986
-    }
-}
-
- - -You can even make an input file directly executable the same way with the following two lines at the top: - - -
#!/usr/bin/env cwl-runner
-cwl:tool: bwa-mem.cwl
-
- - - -
~/arvados/doc/user/cwl/bwa-mem$ ./bwa-mem-input.yml
-arvados-cwl-runner 1.0.20160628195002, arvados-python-client 0.1.20160616015107, cwltool 1.0.20160629140624
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Upload local files: "bwa-mem.cwl"
-2016-06-30 14:56:36 arvados.arv-run[27002] INFO: Uploaded to qr1hi-4zz18-h7ljh5u76760ww2
-2016-06-30 14:56:40 arvados.cwl-runner[27002] INFO: Submitted job qr1hi-8i9sb-fm2n3b1w0l6bskg
-2016-06-30 14:56:41 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Running
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Job bwa-mem.cwl (qr1hi-8i9sb-fm2n3b1w0l6bskg) is Complete
-2016-06-30 14:57:12 arvados.cwl-runner[27002] INFO: Overall process status is success
-{
-    "aligned_sam": {
-        "path": "keep:54325254b226664960de07b3b9482349+154/HWI-ST1027_129_D0THKACXX.1_1.sam",
-        "checksum": "sha1$0dc46a3126d0b5d4ce213b5f0e86e2d05a54755a",
-        "class": "File",
-        "size": 30738986
-    }
-}
-
- diff --git a/doc/zenweb-liquid.rb b/doc/zenweb-liquid.rb index baa8fe42db..3e8672e021 100644 --- a/doc/zenweb-liquid.rb +++ b/doc/zenweb-liquid.rb @@ -50,7 +50,7 @@ module Zenweb Liquid::Tag.instance_method(:initialize).bind(self).call(tag_name, markup, tokens) if markup =~ Syntax - @template_name = $1 + @template_name_expr = $1 @language = $3 @attributes = {} else @@ -61,9 +61,14 @@ module Zenweb def render(context) require 'coderay' - partial = load_cached_partial(context) + partial = load_cached_partial(@template_name_expr, context) html = '' + # be explicit about errors + context.exception_renderer = lambda do |exc| + exc.is_a?(Liquid::InternalError) ? "Liquid error: #{exc.cause.message}" : exc + end + context.stack do html = CodeRay.scan(partial.root.nodelist.join, @language).div end @@ -98,6 +103,11 @@ module Zenweb partial = partial[1..-1] end + # be explicit about errors + context.exception_renderer = lambda do |exc| + exc.is_a?(Liquid::InternalError) ? "Liquid error: #{exc.cause.message}" : exc + end + context.stack do html = CodeRay.scan(partial, @language).div end diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile index 15993c4bc3..8da58a682d 100644 --- a/docker/jobs/Dockerfile +++ b/docker/jobs/Dockerfile @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -# Based on Debian Stretch +# Based on Debian FROM debian:buster-slim MAINTAINER Arvados Package Maintainers @@ -23,12 +23,10 @@ ARG cwl_runner_version RUN echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version RUN apt-get update -q -RUN apt-get install -yq --no-install-recommends nodejs \ - python-arvados-python-client=$python_sdk_version \ - python3-arvados-cwl-runner=$cwl_runner_version +RUN apt-get install -yq --no-install-recommends python3-arvados-cwl-runner=$cwl_runner_version # use the Python executable from the python-arvados-cwl-runner package -RUN rm -f /usr/bin/python && ln -s /usr/share/python2.7/dist/python-arvados-python-client/bin/python /usr/bin/python +RUN rm -f /usr/bin/python && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3 # Install dependencies and set up system. diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go index fc23eb9132..e45c4e1686 100644 --- a/lib/boot/postgresql.go +++ b/lib/boot/postgresql.go @@ -65,7 +65,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso if err != nil { return fmt.Errorf("user.Lookup(\"postgres\"): %s", err) } - postgresUid, err := strconv.Atoi(postgresUser.Uid) + postgresUID, err := strconv.Atoi(postgresUser.Uid) if err != nil { return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid) } @@ -81,7 +81,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso if err != nil { return err } - err = os.Chown(datadir, postgresUid, 0) + err = os.Chown(datadir, postgresUID, 0) if err != nil { return err } diff --git a/lib/boot/seed.go b/lib/boot/seed.go index 1f07601a09..1f6cb764e0 100644 --- a/lib/boot/seed.go +++ b/lib/boot/seed.go @@ -27,5 +27,9 @@ func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor if err != nil { return err } + err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb") + if err != nil { + return err + } return nil } diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index 138c802e18..20576b6b97 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -470,9 +470,9 @@ func (super *Supervisor) lookPath(prog string) string { return prog } -// Run prog with args, using dir as working directory. If ctx is -// cancelled while the child is running, RunProgram terminates the -// child, waits for it to exit, then returns. +// RunProgram runs prog with args, using dir as working directory. If ctx is +// cancelled while the child is running, RunProgram terminates the child, waits +// for it to exit, then returns. // // Child's environment will have our env vars, plus any given in env. // @@ -650,7 +650,7 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { } if len(svc.InternalURLs) == 0 { svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{ - arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{}, + {Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {}, } } } diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go index c26309aca5..7f949d9bdb 100644 --- a/lib/cloud/azure/azure.go +++ b/lib/cloud/azure/azure.go @@ -419,7 +419,7 @@ func (az *azureInstanceSet) Create( Tags: tags, InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ IPConfigurations: &[]network.InterfaceIPConfiguration{ - network.InterfaceIPConfiguration{ + { Name: to.StringPtr("ip1"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ Subnet: &network.Subnet{ @@ -501,7 +501,7 @@ func (az *azureInstanceSet) Create( StorageProfile: storageProfile, NetworkProfile: &compute.NetworkProfile{ NetworkInterfaces: &[]compute.NetworkInterfaceReference{ - compute.NetworkInterfaceReference{ + { ID: nic.ID, NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ Primary: to.BoolPtr(true), @@ -677,6 +677,10 @@ func (az *azureInstanceSet) manageDisks() { } for ; response.NotDone(); err = response.Next() { + if err != nil { + az.logger.WithError(err).Warn("Error getting next page of disks") + return + } for _, d := range response.Values() { if d.DiskProperties.DiskState == compute.Unattached && d.Name != nil && re.MatchString(*d.Name) && diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go index 7b5a34df59..96d6dca69e 100644 --- a/lib/cloud/azure/azure_test.go +++ b/lib/cloud/azure/azure_test.go @@ -127,7 +127,7 @@ var live = flag.String("live-azure-cfg", "", "Test with real azure API, provide func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) { cluster := arvados.Cluster{ InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{ - "tiny": arvados.InstanceType{ + "tiny": { Name: "tiny", ProviderType: "Standard_D1_v2", VCPUs: 1, @@ -259,7 +259,7 @@ func (*AzureInstanceSetSuite) TestWrapError(c *check.C) { DetailedError: autorest.DetailedError{ Response: &http.Response{ StatusCode: 429, - Header: map[string][]string{"Retry-After": []string{"123"}}, + Header: map[string][]string{"Retry-After": {"123"}}, }, }, ServiceError: &azure.ServiceError{}, diff --git a/lib/cloud/cloudtest/tester.go b/lib/cloud/cloudtest/tester.go index 6093834179..9fd7c9e749 100644 --- a/lib/cloud/cloudtest/tester.go +++ b/lib/cloud/cloudtest/tester.go @@ -12,7 +12,7 @@ import ( "time" "git.arvados.org/arvados.git/lib/cloud" - "git.arvados.org/arvados.git/lib/dispatchcloud/ssh_executor" + "git.arvados.org/arvados.git/lib/dispatchcloud/sshexecutor" "git.arvados.org/arvados.git/lib/dispatchcloud/worker" "git.arvados.org/arvados.git/sdk/go/arvados" "github.com/sirupsen/logrus" @@ -48,7 +48,7 @@ type tester struct { is cloud.InstanceSet testInstance *worker.TagVerifier secret string - executor *ssh_executor.Executor + executor *sshexecutor.Executor showedLoginInfo bool failed bool @@ -127,7 +127,7 @@ func (t *tester) Run() bool { defer t.destroyTestInstance() bootDeadline := time.Now().Add(t.TimeoutBooting) - initCommand := worker.TagVerifier{nil, t.secret}.InitCommand() + initCommand := worker.TagVerifier{Instance: nil, Secret: t.secret, ReportVerified: nil}.InitCommand() t.Logger.WithFields(logrus.Fields{ "InstanceType": t.InstanceType.Name, @@ -150,9 +150,8 @@ func (t *tester) Run() bool { if time.Now().After(bootDeadline) { t.Logger.Error("timed out") return false - } else { - t.sleepSyncInterval() } + t.sleepSyncInterval() } t.Logger.WithField("Instance", t.testInstance.ID()).Info("new instance appeared") t.showLoginInfo() @@ -160,7 +159,7 @@ func (t *tester) Run() bool { // Create() succeeded. Make sure the new instance // appears right away in the Instances() list. lgrC.WithField("Instance", inst.ID()).Info("created instance") - t.testInstance = &worker.TagVerifier{inst, t.secret} + t.testInstance = &worker.TagVerifier{Instance: inst, Secret: t.secret, ReportVerified: nil} t.showLoginInfo() err = t.refreshTestInstance() if err == errTestInstanceNotFound { @@ -236,7 +235,7 @@ func (t *tester) refreshTestInstance() error { "Instance": i.ID(), "Address": i.Address(), }).Info("found our instance in returned list") - t.testInstance = &worker.TagVerifier{i, t.secret} + t.testInstance = &worker.TagVerifier{Instance: i, Secret: t.secret, ReportVerified: nil} if !t.showedLoginInfo { t.showLoginInfo() } @@ -308,7 +307,7 @@ func (t *tester) waitForBoot(deadline time.Time) bool { // current address. func (t *tester) updateExecutor() { if t.executor == nil { - t.executor = ssh_executor.New(t.testInstance) + t.executor = sshexecutor.New(t.testInstance) t.executor.SetTargetPort(t.SSHPort) t.executor.SetSigners(t.SSHKey) } else { diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go index 2de82b1dcf..b20dbfcc98 100644 --- a/lib/cloud/ec2/ec2.go +++ b/lib/cloud/ec2/ec2.go @@ -103,10 +103,10 @@ func awsKeyFingerprint(pk ssh.PublicKey) (md5fp string, sha1fp string, err error sha1pkix := sha1.Sum([]byte(pkix)) md5fp = "" sha1fp = "" - for i := 0; i < len(md5pkix); i += 1 { + for i := 0; i < len(md5pkix); i++ { md5fp += fmt.Sprintf(":%02x", md5pkix[i]) } - for i := 0; i < len(sha1pkix); i += 1 { + for i := 0; i < len(sha1pkix); i++ { sha1fp += fmt.Sprintf(":%02x", sha1pkix[i]) } return md5fp[1:], sha1fp[1:], nil @@ -128,7 +128,7 @@ func (instanceSet *ec2InstanceSet) Create( var ok bool if keyname, ok = instanceSet.keys[md5keyFingerprint]; !ok { keyout, err := instanceSet.client.DescribeKeyPairs(&ec2.DescribeKeyPairsInput{ - Filters: []*ec2.Filter{&ec2.Filter{ + Filters: []*ec2.Filter{{ Name: aws.String("fingerprint"), Values: []*string{&md5keyFingerprint, &sha1keyFingerprint}, }}, @@ -174,7 +174,7 @@ func (instanceSet *ec2InstanceSet) Create( KeyName: &keyname, NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ - &ec2.InstanceNetworkInterfaceSpecification{ + { AssociatePublicIpAddress: aws.Bool(false), DeleteOnTermination: aws.Bool(true), DeviceIndex: aws.Int64(0), @@ -184,7 +184,7 @@ func (instanceSet *ec2InstanceSet) Create( DisableApiTermination: aws.Bool(false), InstanceInitiatedShutdownBehavior: aws.String("terminate"), TagSpecifications: []*ec2.TagSpecification{ - &ec2.TagSpecification{ + { ResourceType: aws.String("instance"), Tags: ec2tags, }}, @@ -192,7 +192,7 @@ func (instanceSet *ec2InstanceSet) Create( } if instanceType.AddedScratch > 0 { - rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{&ec2.BlockDeviceMapping{ + rii.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{ DeviceName: aws.String("/dev/xvdt"), Ebs: &ec2.EbsBlockDevice{ DeleteOnTermination: aws.Bool(true), @@ -251,7 +251,7 @@ func (instanceSet *ec2InstanceSet) Instances(tags cloud.InstanceTags) (instances } } -func (az *ec2InstanceSet) Stop() { +func (instanceSet *ec2InstanceSet) Stop() { } type ec2Instance struct { @@ -308,9 +308,8 @@ func (inst *ec2Instance) Destroy() error { func (inst *ec2Instance) Address() string { if inst.instance.PrivateIpAddress != nil { return *inst.instance.PrivateIpAddress - } else { - return "" } + return "" } func (inst *ec2Instance) RemoteUser() string { diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go index 638f4a77a3..6aa6e857ff 100644 --- a/lib/cloud/ec2/ec2_test.go +++ b/lib/cloud/ec2/ec2_test.go @@ -65,7 +65,7 @@ func (e *ec2stub) DescribeKeyPairs(input *ec2.DescribeKeyPairsInput) (*ec2.Descr } func (e *ec2stub) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) { - return &ec2.Reservation{Instances: []*ec2.Instance{&ec2.Instance{ + return &ec2.Reservation{Instances: []*ec2.Instance{{ InstanceId: aws.String("i-123"), Tags: input.TagSpecifications[0].Tags, }}}, nil @@ -86,7 +86,7 @@ func (e *ec2stub) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.T func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) { cluster := arvados.Cluster{ InstanceTypes: arvados.InstanceTypeMap(map[string]arvados.InstanceType{ - "tiny": arvados.InstanceType{ + "tiny": { Name: "tiny", ProviderType: "t2.micro", VCPUs: 1, @@ -95,7 +95,7 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) Price: .02, Preemptible: false, }, - "tiny-with-extra-scratch": arvados.InstanceType{ + "tiny-with-extra-scratch": { Name: "tiny", ProviderType: "t2.micro", VCPUs: 1, @@ -104,7 +104,7 @@ func GetInstanceSet() (cloud.InstanceSet, cloud.ImageID, arvados.Cluster, error) Preemptible: false, AddedScratch: 20000000000, }, - "tiny-preemptible": arvados.InstanceType{ + "tiny-preemptible": { Name: "tiny", ProviderType: "t2.micro", VCPUs: 1, diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go index 611c95d234..b7d918739b 100644 --- a/lib/cmd/cmd.go +++ b/lib/cmd/cmd.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -// package cmd helps define reusable functions that can be exposed as +// Package cmd helps define reusable functions that can be exposed as // [subcommands of] command line programs. package cmd diff --git a/lib/config/cmd.go b/lib/config/cmd.go index d64106fbce..347e8519a9 100644 --- a/lib/config/cmd.go +++ b/lib/config/cmd.go @@ -91,6 +91,7 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo flags := flag.NewFlagSet("", flag.ContinueOnError) flags.SetOutput(stderr) loader.SetupFlags(flags) + strict := flags.Bool("strict", true, "Strict validation of configuration file (warnings result in non-zero exit code)") err = flags.Parse(args) if err == flag.ErrHelp { @@ -148,22 +149,27 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo fmt.Fprintln(stdout, "Your configuration is relying on deprecated entries. Suggest making the following changes.") stdout.Write(diff) err = nil - return 1 + if *strict { + return 1 + } } else if len(diff) > 0 { fmt.Fprintf(stderr, "Unexpected diff output:\n%s", diff) - return 1 + if *strict { + return 1 + } } else if err != nil { return 1 } if logbuf.Len() > 0 { - return 1 + if *strict { + return 1 + } } if problems { return 1 - } else { - return 0 } + return 0 } func warnAboutProblems(logger logrus.FieldLogger, cfg *arvados.Config) bool { diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index a1b471bd22..7e16688d9d 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -12,6 +12,8 @@ Clusters: xxxxx: + # Token used internally by Arvados components to authenticate to + # one another. Use a string of at least 50 random alphanumerics. SystemRootToken: "" # Token to be included in all healthcheck requests. Disabled by default. @@ -287,6 +289,20 @@ Clusters: # address is used. PreferDomainForUsername: "" + UserSetupMailText: | + <% if not @user.full_name.empty? -%> + <%= @user.full_name %>, + <% else -%> + Hi there, + <% end -%> + + Your Arvados account has been set up. You can log in at + + <%= Rails.configuration.Services.Workbench1.ExternalURL %> + + Thanks, + Your Arvados administrator. + AuditLogs: # Time to keep audit logs, in seconds. (An audit log is a row added # to the "logs" table in the PostgreSQL database each time an @@ -689,6 +705,16 @@ Clusters: ProviderAppID: "" ProviderAppSecret: "" + Test: + # Authenticate users listed here in the config file. This + # feature is intended to be used in test environments, and + # should not be used in production. + Enable: false + Users: + SAMPLE: + Email: alice@example.com + Password: xyzzy + # The cluster ID to delegate the user database. When set, # logins on this cluster will be redirected to the login cluster # (login cluster must appear in RemoteClusters with Proxy: true) @@ -698,6 +724,22 @@ Clusters: # remain valid before it needs to be revalidated. RemoteTokenRefresh: 5m + # How long a client token created from a login flow will be valid without + # asking the user to re-login. Example values: 60m, 8h. + # Default value zero means tokens don't have expiration. + TokenLifetime: 0s + + # When the token is returned to a client, the token itself may + # be restricted from manipulating other tokens based on whether + # the client is "trusted" or not. The local Workbench1 and + # Workbench2 are trusted by default, but if this is a + # LoginCluster, you probably want to include the other Workbench + # instances in the federation in this list. + TrustedClients: + SAMPLE: + "https://workbench.federate1.example": {} + "https://workbench.federate2.example": {} + Git: # Path to git or gitolite-shell executable. Each authenticated # request will execute this program with the single argument "http-backend" @@ -923,6 +965,11 @@ Clusters: # Time before repeating SIGTERM when killing a container. TimeoutSignal: 5s + # Time to give up on a process (most likely arv-mount) that + # still holds a container lockfile after its main supervisor + # process has exited, and declare the instance broken. + TimeoutStaleRunLock: 5s + # Time to give up on SIGTERM and write off the worker. TimeoutTERM: 2m @@ -930,6 +977,12 @@ Clusters: # unlimited). MaxCloudOpsPerSecond: 0 + # Maximum concurrent node creation operations (0 = unlimited). This is + # recommended by Azure in certain scenarios (see + # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image) + # and can be used with other cloud providers too, if desired. + MaxConcurrentInstanceCreateOps: 0 + # Interval between cloud provider syncs/updates ("list all # instances"). SyncInterval: 1m @@ -1299,7 +1352,7 @@ Clusters: # a link to the multi-site search page on a "home" Workbench site. # # Example: - # https://workbench.qr1hi.arvadosapi.com/collections/multisite + # https://workbench.zzzzz.arvadosapi.com/collections/multisite MultiSiteSearch: "" # Should workbench allow management of local git repositories? Set to false if @@ -1317,6 +1370,10 @@ Clusters: VocabularyURL: "" FileViewersConfigURL: "" + # Idle time after which the user's session will be auto closed. + # This feature is disabled when set to zero. + IdleTimeout: 0s + # Workbench welcome screen, this is HTML text that will be # incorporated directly onto the page. WelcomePageHTML: | diff --git a/lib/config/export.go b/lib/config/export.go index f15a299619..0735354b1b 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -170,6 +170,11 @@ var whitelist = map[string]bool{ "Login.SSO.Enable": true, "Login.SSO.ProviderAppID": false, "Login.SSO.ProviderAppSecret": false, + "Login.Test": true, + "Login.Test.Enable": true, + "Login.Test.Users": false, + "Login.TokenLifetime": false, + "Login.TrustedClients": false, "Mail": true, "Mail.EmailFrom": false, "Mail.IssueReporterEmailFrom": false, @@ -210,6 +215,7 @@ var whitelist = map[string]bool{ "Users.PreferDomainForUsername": false, "Users.UserNotifierEmailFrom": false, "Users.UserProfileNotificationAddress": false, + "Users.UserSetupMailText": false, "Volumes": true, "Volumes.*": true, "Volumes.*.*": false, @@ -233,6 +239,7 @@ var whitelist = map[string]bool{ "Workbench.EnableGettingStartedPopup": true, "Workbench.EnablePublicProjectsPage": true, "Workbench.FileViewersConfigURL": true, + "Workbench.IdleTimeout": true, "Workbench.InactivePageHTML": true, "Workbench.LogViewerMaxBytes": true, "Workbench.MultiSiteSearch": true, diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index 8e42eb3505..934131bd8f 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -18,6 +18,8 @@ var DefaultYAML = []byte(`# Copyright (C) The Arvados Authors. All rights reserv Clusters: xxxxx: + # Token used internally by Arvados components to authenticate to + # one another. Use a string of at least 50 random alphanumerics. SystemRootToken: "" # Token to be included in all healthcheck requests. Disabled by default. @@ -293,6 +295,20 @@ Clusters: # address is used. PreferDomainForUsername: "" + UserSetupMailText: | + <% if not @user.full_name.empty? -%> + <%= @user.full_name %>, + <% else -%> + Hi there, + <% end -%> + + Your Arvados account has been set up. You can log in at + + <%= Rails.configuration.Services.Workbench1.ExternalURL %> + + Thanks, + Your Arvados administrator. + AuditLogs: # Time to keep audit logs, in seconds. (An audit log is a row added # to the "logs" table in the PostgreSQL database each time an @@ -695,6 +711,16 @@ Clusters: ProviderAppID: "" ProviderAppSecret: "" + Test: + # Authenticate users listed here in the config file. This + # feature is intended to be used in test environments, and + # should not be used in production. + Enable: false + Users: + SAMPLE: + Email: alice@example.com + Password: xyzzy + # The cluster ID to delegate the user database. When set, # logins on this cluster will be redirected to the login cluster # (login cluster must appear in RemoteClusters with Proxy: true) @@ -704,6 +730,22 @@ Clusters: # remain valid before it needs to be revalidated. RemoteTokenRefresh: 5m + # How long a client token created from a login flow will be valid without + # asking the user to re-login. Example values: 60m, 8h. + # Default value zero means tokens don't have expiration. + TokenLifetime: 0s + + # When the token is returned to a client, the token itself may + # be restricted from manipulating other tokens based on whether + # the client is "trusted" or not. The local Workbench1 and + # Workbench2 are trusted by default, but if this is a + # LoginCluster, you probably want to include the other Workbench + # instances in the federation in this list. + TrustedClients: + SAMPLE: + "https://workbench.federate1.example": {} + "https://workbench.federate2.example": {} + Git: # Path to git or gitolite-shell executable. Each authenticated # request will execute this program with the single argument "http-backend" @@ -929,6 +971,11 @@ Clusters: # Time before repeating SIGTERM when killing a container. TimeoutSignal: 5s + # Time to give up on a process (most likely arv-mount) that + # still holds a container lockfile after its main supervisor + # process has exited, and declare the instance broken. + TimeoutStaleRunLock: 5s + # Time to give up on SIGTERM and write off the worker. TimeoutTERM: 2m @@ -936,6 +983,12 @@ Clusters: # unlimited). MaxCloudOpsPerSecond: 0 + # Maximum concurrent node creation operations (0 = unlimited). This is + # recommended by Azure in certain scenarios (see + # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image) + # and can be used with other cloud providers too, if desired. + MaxConcurrentInstanceCreateOps: 0 + # Interval between cloud provider syncs/updates ("list all # instances"). SyncInterval: 1m @@ -1305,7 +1358,7 @@ Clusters: # a link to the multi-site search page on a "home" Workbench site. # # Example: - # https://workbench.qr1hi.arvadosapi.com/collections/multisite + # https://workbench.zzzzz.arvadosapi.com/collections/multisite MultiSiteSearch: "" # Should workbench allow management of local git repositories? Set to false if @@ -1323,6 +1376,10 @@ Clusters: VocabularyURL: "" FileViewersConfigURL: "" + # Idle time after which the user's session will be auto closed. + # This feature is disabled when set to zero. + IdleTimeout: 0s + # Workbench welcome screen, this is HTML text that will be # incorporated directly onto the page. WelcomePageHTML: | diff --git a/lib/controller/api/routable.go b/lib/controller/api/routable.go index 6049cba8e4..f887448829 100644 --- a/lib/controller/api/routable.go +++ b/lib/controller/api/routable.go @@ -15,3 +15,16 @@ import "context" // it to the router package would cause a circular dependency // router->arvadostest->ctrlctx->router.) type RoutableFunc func(ctx context.Context, opts interface{}) (interface{}, error) + +type RoutableFuncWrapper func(RoutableFunc) RoutableFunc + +// ComposeWrappers (w1, w2, w3, ...) returns a RoutableFuncWrapper that +// composes w1, w2, w3, ... such that w1 is the outermost wrapper. +func ComposeWrappers(wraps ...RoutableFuncWrapper) RoutableFuncWrapper { + return func(f RoutableFunc) RoutableFunc { + for i := len(wraps) - 1; i >= 0; i-- { + f = wraps[i](f) + } + return f + } +} diff --git a/lib/controller/auth_test.go b/lib/controller/auth_test.go new file mode 100644 index 0000000000..ad214b1605 --- /dev/null +++ b/lib/controller/auth_test.go @@ -0,0 +1,126 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package controller + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "time" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/httpserver" + "github.com/sirupsen/logrus" + check "gopkg.in/check.v1" +) + +// Gocheck boilerplate +var _ = check.Suite(&AuthSuite{}) + +type AuthSuite struct { + log logrus.FieldLogger + // testServer and testHandler are the controller being tested, + // "zhome". + testServer *httpserver.Server + testHandler *Handler + // remoteServer ("zzzzz") forwards requests to the Rails API + // provided by the integration test environment. + remoteServer *httpserver.Server + // remoteMock ("zmock") appends each incoming request to + // remoteMockRequests, and returns 200 with an empty JSON + // object. + remoteMock *httpserver.Server + remoteMockRequests []http.Request + + fakeProvider *arvadostest.OIDCProvider +} + +func (s *AuthSuite) SetUpTest(c *check.C) { + s.log = ctxlog.TestLogger(c) + + s.remoteServer = newServerFromIntegrationTestEnv(c) + c.Assert(s.remoteServer.Start(), check.IsNil) + + s.remoteMock = newServerFromIntegrationTestEnv(c) + s.remoteMock.Server.Handler = http.HandlerFunc(http.NotFound) + c.Assert(s.remoteMock.Start(), check.IsNil) + + s.fakeProvider = arvadostest.NewOIDCProvider(c) + s.fakeProvider.AuthEmail = "active-user@arvados.local" + s.fakeProvider.AuthEmailVerified = true + s.fakeProvider.AuthName = "Fake User Name" + s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix()) + s.fakeProvider.PeopleAPIResponse = map[string]interface{}{} + s.fakeProvider.ValidClientID = "test%client$id" + s.fakeProvider.ValidClientSecret = "test#client/secret" + + cluster := &arvados.Cluster{ + ClusterID: "zhome", + PostgreSQL: integrationTestCluster().PostgreSQL, + ForceLegacyAPI14: forceLegacyAPI14, + SystemRootToken: arvadostest.SystemRootToken, + } + cluster.TLS.Insecure = true + cluster.API.MaxItemsPerResponse = 1000 + cluster.API.MaxRequestAmplification = 4 + cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute) + arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST")) + arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost/") + + cluster.RemoteClusters = map[string]arvados.RemoteCluster{ + "zzzzz": { + Host: s.remoteServer.Addr, + Proxy: true, + Scheme: "http", + }, + "zmock": { + Host: s.remoteMock.Addr, + Proxy: true, + Scheme: "http", + }, + "*": { + Scheme: "https", + }, + } + cluster.Login.OpenIDConnect.Enable = true + cluster.Login.OpenIDConnect.Issuer = s.fakeProvider.Issuer.URL + cluster.Login.OpenIDConnect.ClientID = s.fakeProvider.ValidClientID + cluster.Login.OpenIDConnect.ClientSecret = s.fakeProvider.ValidClientSecret + cluster.Login.OpenIDConnect.EmailClaim = "email" + cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified" + + s.testHandler = &Handler{Cluster: cluster} + s.testServer = newServerFromIntegrationTestEnv(c) + s.testServer.Server.Handler = httpserver.HandlerWithContext( + ctxlog.Context(context.Background(), s.log), + httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler))) + c.Assert(s.testServer.Start(), check.IsNil) +} + +func (s *AuthSuite) TestLocalOIDCAccessToken(c *check.C) { + req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil) + req.Header.Set("Authorization", "Bearer "+s.fakeProvider.ValidAccessToken()) + rr := httptest.NewRecorder() + s.testServer.Server.Handler.ServeHTTP(rr, req) + resp := rr.Result() + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + var u arvados.User + c.Check(json.NewDecoder(resp.Body).Decode(&u), check.IsNil) + c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID) + c.Check(u.OwnerUUID, check.Equals, "zzzzz-tpzed-000000000000000") + + // Request again to exercise cache. + req = httptest.NewRequest("GET", "/arvados/v1/users/current", nil) + req.Header.Set("Authorization", "Bearer "+s.fakeProvider.ValidAccessToken()) + rr = httptest.NewRecorder() + s.testServer.Server.Handler.ServeHTTP(rr, req) + resp = rr.Result() + c.Check(resp.StatusCode, check.Equals, http.StatusOK) +} diff --git a/lib/controller/fed_collections.go b/lib/controller/fed_collections.go index c33f5b2894..a0a123129f 100644 --- a/lib/controller/fed_collections.go +++ b/lib/controller/fed_collections.go @@ -157,7 +157,7 @@ type searchRemoteClusterForPDH struct { func fetchRemoteCollectionByUUID( h *genericFederatedRequestHandler, effectiveMethod string, - clusterId *string, + clusterID *string, uuid string, remainder string, w http.ResponseWriter, @@ -170,11 +170,11 @@ func fetchRemoteCollectionByUUID( if uuid != "" { // Collection UUID GET request - *clusterId = uuid[0:5] - if *clusterId != "" && *clusterId != h.handler.Cluster.ClusterID { + *clusterID = uuid[0:5] + if *clusterID != "" && *clusterID != h.handler.Cluster.ClusterID { // request for remote collection by uuid - resp, err := h.handler.remoteClusterRequest(*clusterId, req) - newResponse, err := rewriteSignatures(*clusterId, "", resp, err) + resp, err := h.handler.remoteClusterRequest(*clusterID, req) + newResponse, err := rewriteSignatures(*clusterID, "", resp, err) h.handler.proxy.ForwardResponse(w, newResponse, err) return true } @@ -186,7 +186,7 @@ func fetchRemoteCollectionByUUID( func fetchRemoteCollectionByPDH( h *genericFederatedRequestHandler, effectiveMethod string, - clusterId *string, + clusterID *string, uuid string, remainder string, w http.ResponseWriter, diff --git a/lib/controller/fed_containers.go b/lib/controller/fed_containers.go index c62cea1168..fd4f0521bc 100644 --- a/lib/controller/fed_containers.go +++ b/lib/controller/fed_containers.go @@ -19,7 +19,7 @@ import ( func remoteContainerRequestCreate( h *genericFederatedRequestHandler, effectiveMethod string, - clusterId *string, + clusterID *string, uuid string, remainder string, w http.ResponseWriter, @@ -42,7 +42,7 @@ func remoteContainerRequestCreate( return true } - if *clusterId == "" || *clusterId == h.handler.Cluster.ClusterID { + if *clusterID == "" || *clusterID == h.handler.Cluster.ClusterID { // Submitting container request to local cluster. No // need to set a runtime_token (rails api will create // one when the container runs) or do a remote cluster @@ -66,14 +66,14 @@ func remoteContainerRequestCreate( crString, ok := request["container_request"].(string) if ok { - var crJson map[string]interface{} - err := json.Unmarshal([]byte(crString), &crJson) + var crJSON map[string]interface{} + err := json.Unmarshal([]byte(crString), &crJSON) if err != nil { httpserver.Error(w, err.Error(), http.StatusBadRequest) return true } - request["container_request"] = crJson + request["container_request"] = crJSON } containerRequest, ok := request["container_request"].(map[string]interface{}) @@ -117,7 +117,7 @@ func remoteContainerRequestCreate( req.ContentLength = int64(buf.Len()) req.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len())) - resp, err := h.handler.remoteClusterRequest(*clusterId, req) + resp, err := h.handler.remoteClusterRequest(*clusterID, req) h.handler.proxy.ForwardResponse(w, resp, err) return true } diff --git a/lib/controller/fed_generic.go b/lib/controller/fed_generic.go index 476fd97b05..fc2d96cc55 100644 --- a/lib/controller/fed_generic.go +++ b/lib/controller/fed_generic.go @@ -20,7 +20,7 @@ import ( type federatedRequestDelegate func( h *genericFederatedRequestHandler, effectiveMethod string, - clusterId *string, + clusterID *string, uuid string, remainder string, w http.ResponseWriter, @@ -38,12 +38,12 @@ func (h *genericFederatedRequestHandler) remoteQueryUUIDs(w http.ResponseWriter, clusterID string, uuids []string) (rp []map[string]interface{}, kind string, err error) { found := make(map[string]bool) - prev_len_uuids := len(uuids) + 1 + prevLenUuids := len(uuids) + 1 // Loop while // (1) there are more uuids to query // (2) we're making progress - on each iteration the set of // uuids we are expecting for must shrink. - for len(uuids) > 0 && len(uuids) < prev_len_uuids { + for len(uuids) > 0 && len(uuids) < prevLenUuids { var remoteReq http.Request remoteReq.Header = req.Header remoteReq.Method = "POST" @@ -103,7 +103,7 @@ func (h *genericFederatedRequestHandler) remoteQueryUUIDs(w http.ResponseWriter, l = append(l, u) } } - prev_len_uuids = len(uuids) + prevLenUuids = len(uuids) uuids = l } @@ -111,7 +111,7 @@ func (h *genericFederatedRequestHandler) remoteQueryUUIDs(w http.ResponseWriter, } func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.ResponseWriter, - req *http.Request, clusterId *string) bool { + req *http.Request, clusterID *string) bool { var filters [][]interface{} err := json.Unmarshal([]byte(req.Form.Get("filters")), &filters) @@ -141,17 +141,17 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response if rhs, ok := filter[2].([]interface{}); ok { for _, i := range rhs { if u, ok := i.(string); ok && len(u) == 27 { - *clusterId = u[0:5] + *clusterID = u[0:5] queryClusters[u[0:5]] = append(queryClusters[u[0:5]], u) - expectCount += 1 + expectCount++ } } } } else if op == "=" { if u, ok := filter[2].(string); ok && len(u) == 27 { - *clusterId = u[0:5] + *clusterID = u[0:5] queryClusters[u[0:5]] = append(queryClusters[u[0:5]], u) - expectCount += 1 + expectCount++ } } else { return false @@ -256,10 +256,10 @@ func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.Response func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { m := h.matcher.FindStringSubmatch(req.URL.Path) - clusterId := "" + clusterID := "" if len(m) > 0 && m[2] != "" { - clusterId = m[2] + clusterID = m[2] } // Get form parameters from URL and form body (if POST). @@ -270,7 +270,7 @@ func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *h // Check if the parameters have an explicit cluster_id if req.Form.Get("cluster_id") != "" { - clusterId = req.Form.Get("cluster_id") + clusterID = req.Form.Get("cluster_id") } // Handle the POST-as-GET special case (workaround for large @@ -283,9 +283,9 @@ func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *h } if effectiveMethod == "GET" && - clusterId == "" && + clusterID == "" && req.Form.Get("filters") != "" && - h.handleMultiClusterQuery(w, req, &clusterId) { + h.handleMultiClusterQuery(w, req, &clusterID) { return } @@ -295,15 +295,15 @@ func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *h uuid = m[1][1:] } for _, d := range h.delegates { - if d(h, effectiveMethod, &clusterId, uuid, m[3], w, req) { + if d(h, effectiveMethod, &clusterID, uuid, m[3], w, req) { return } } - if clusterId == "" || clusterId == h.handler.Cluster.ClusterID { + if clusterID == "" || clusterID == h.handler.Cluster.ClusterID { h.next.ServeHTTP(w, req) } else { - resp, err := h.handler.remoteClusterRequest(clusterId, req) + resp, err := h.handler.remoteClusterRequest(clusterID, req) h.handler.proxy.ForwardResponse(w, resp, err) } } diff --git a/lib/controller/federation.go b/lib/controller/federation.go index aceaba8087..cab5e4c4ca 100644 --- a/lib/controller/federation.go +++ b/lib/controller/federation.go @@ -263,17 +263,20 @@ func (h *Handler) saltAuthToken(req *http.Request, remote string) (updatedReq *h return updatedReq, nil } + ctxlog.FromContext(req.Context()).Infof("saltAuthToken: cluster %s token %s remote %s", h.Cluster.ClusterID, creds.Tokens[0], remote) token, err := auth.SaltToken(creds.Tokens[0], remote) if err == auth.ErrObsoleteToken { - // If the token exists in our own database, salt it - // for the remote. Otherwise, assume it was issued by - // the remote, and pass it through unmodified. + // If the token exists in our own database for our own + // user, salt it for the remote. Otherwise, assume it + // was issued by the remote, and pass it through + // unmodified. currentUser, ok, err := h.validateAPItoken(req, creds.Tokens[0]) if err != nil { return nil, err - } else if !ok { - // Not ours; pass through unmodified. + } else if !ok || strings.HasPrefix(currentUser.UUID, remote) { + // Unknown, or cached + belongs to remote; + // pass through unmodified. token = creds.Tokens[0] } else { // Found; make V2 version and salt it. diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go index 418b6811be..130368124c 100644 --- a/lib/controller/federation/conn.go +++ b/lib/controller/federation/conn.go @@ -79,6 +79,14 @@ func saltedTokenProvider(local backend, remoteID string) rpc.TokenProvider { } else if err != nil { return nil, err } + if strings.HasPrefix(aca.UUID, remoteID) { + // We have it cached here, but + // the token belongs to the + // remote target itself, so + // pass it through unmodified. + tokens = append(tokens, token) + continue + } salted, err := auth.SaltToken(aca.TokenV2(), remoteID) if err != nil { return nil, err @@ -111,6 +119,13 @@ func (conn *Conn) chooseBackend(id string) backend { } } +func (conn *Conn) localOrLoginCluster() backend { + if conn.cluster.Login.LoginCluster != "" { + return conn.chooseBackend(conn.cluster.Login.LoginCluster) + } + return conn.local +} + // Call fn with the local backend; then, if fn returned 404, call fn // on the available remote backends (possibly concurrently) until one // succeeds. @@ -196,9 +211,8 @@ func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arva return arvados.LoginResponse{ RedirectLocation: target.String(), }, nil - } else { - return conn.local.Login(ctx, options) } + return conn.local.Login(ctx, options) } func (conn *Conn) Logout(ctx context.Context, options arvados.LogoutOptions) (arvados.LogoutResponse, error) { @@ -235,40 +249,39 @@ func (conn *Conn) CollectionGet(ctx context.Context, options arvados.GetOptions) c.ManifestText = rewriteManifest(c.ManifestText, options.UUID[:5]) } return c, err - } else { - // UUID is a PDH - first := make(chan arvados.Collection, 1) - err := conn.tryLocalThenRemotes(ctx, options.ForwardedFor, func(ctx context.Context, remoteID string, be backend) error { - remoteOpts := options - remoteOpts.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor - c, err := be.CollectionGet(ctx, remoteOpts) - if err != nil { - return err - } - // options.UUID is either hash+size or - // hash+size+hints; only hash+size need to - // match the computed PDH. - if pdh := arvados.PortableDataHash(c.ManifestText); pdh != options.UUID && !strings.HasPrefix(options.UUID, pdh+"+") { - err = httpErrorf(http.StatusBadGateway, "bad portable data hash %q received from remote %q (expected %q)", pdh, remoteID, options.UUID) - ctxlog.FromContext(ctx).Warn(err) - return err - } - if remoteID != "" { - c.ManifestText = rewriteManifest(c.ManifestText, remoteID) - } - select { - case first <- c: - return nil - default: - // lost race, return value doesn't matter - return nil - } - }) + } + // UUID is a PDH + first := make(chan arvados.Collection, 1) + err := conn.tryLocalThenRemotes(ctx, options.ForwardedFor, func(ctx context.Context, remoteID string, be backend) error { + remoteOpts := options + remoteOpts.ForwardedFor = conn.cluster.ClusterID + "-" + options.ForwardedFor + c, err := be.CollectionGet(ctx, remoteOpts) if err != nil { - return arvados.Collection{}, err + return err + } + // options.UUID is either hash+size or + // hash+size+hints; only hash+size need to + // match the computed PDH. + if pdh := arvados.PortableDataHash(c.ManifestText); pdh != options.UUID && !strings.HasPrefix(options.UUID, pdh+"+") { + err = httpErrorf(http.StatusBadGateway, "bad portable data hash %q received from remote %q (expected %q)", pdh, remoteID, options.UUID) + ctxlog.FromContext(ctx).Warn(err) + return err + } + if remoteID != "" { + c.ManifestText = rewriteManifest(c.ManifestText, remoteID) + } + select { + case first <- c: + return nil + default: + // lost race, return value doesn't matter + return nil } - return <-first, nil + }) + if err != nil { + return arvados.Collection{}, err } + return <-first, nil } func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) { @@ -437,9 +450,8 @@ func (conn *Conn) UserList(ctx context.Context, options arvados.ListOptions) (ar return arvados.UserList{}, err } return resp, nil - } else { - return conn.generated_UserList(ctx, options) } + return conn.generated_UserList(ctx, options) } func (conn *Conn) UserCreate(ctx context.Context, options arvados.CreateOptions) (arvados.User, error) { @@ -450,7 +462,18 @@ func (conn *Conn) UserUpdate(ctx context.Context, options arvados.UpdateOptions) if options.BypassFederation { return conn.local.UserUpdate(ctx, options) } - return conn.chooseBackend(options.UUID).UserUpdate(ctx, options) + resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options) + if err != nil { + return resp, err + } + if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) { + // Copy the updated user record to the local cluster + err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp}) + if err != nil { + return arvados.User{}, err + } + } + return resp, err } func (conn *Conn) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) { @@ -462,23 +485,58 @@ func (conn *Conn) UserMerge(ctx context.Context, options arvados.UserMergeOption } func (conn *Conn) UserActivate(ctx context.Context, options arvados.UserActivateOptions) (arvados.User, error) { - return conn.chooseBackend(options.UUID).UserActivate(ctx, options) + return conn.localOrLoginCluster().UserActivate(ctx, options) } func (conn *Conn) UserSetup(ctx context.Context, options arvados.UserSetupOptions) (map[string]interface{}, error) { - return conn.chooseBackend(options.UUID).UserSetup(ctx, options) + upstream := conn.localOrLoginCluster() + if upstream != conn.local { + // When LoginCluster is in effect, and we're setting + // up a remote user, and we want to give that user + // access to a local VM, we can't include the VM in + // the setup call, because the remote cluster won't + // recognize it. + + // Similarly, if we want to create a git repo, + // it should be created on the local cluster, + // not the remote one. + + upstreamOptions := options + upstreamOptions.VMUUID = "" + upstreamOptions.RepoName = "" + + ret, err := upstream.UserSetup(ctx, upstreamOptions) + if err != nil { + return ret, err + } + } + + return conn.local.UserSetup(ctx, options) } func (conn *Conn) UserUnsetup(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - return conn.chooseBackend(options.UUID).UserUnsetup(ctx, options) + return conn.localOrLoginCluster().UserUnsetup(ctx, options) } func (conn *Conn) UserGet(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - return conn.chooseBackend(options.UUID).UserGet(ctx, options) + resp, err := conn.chooseBackend(options.UUID).UserGet(ctx, options) + if err != nil { + return resp, err + } + if options.UUID != resp.UUID { + return arvados.User{}, httpErrorf(http.StatusBadGateway, "Had requested %v but response was for %v", options.UUID, resp.UUID) + } + if options.UUID[:5] != conn.cluster.ClusterID { + err = conn.batchUpdateUsers(ctx, arvados.ListOptions{Select: options.Select}, []arvados.User{resp}) + if err != nil { + return arvados.User{}, err + } + } + return resp, nil } func (conn *Conn) UserGetCurrent(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - return conn.chooseBackend(options.UUID).UserGetCurrent(ctx, options) + return conn.local.UserGetCurrent(ctx, options) } func (conn *Conn) UserGetSystem(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { @@ -514,7 +572,6 @@ func (notFoundError) Error() string { return "not found" } func errStatus(err error) int { if httpErr, ok := err.(interface{ HTTPStatus() int }); ok { return httpErr.HTTPStatus() - } else { - return http.StatusInternalServerError } + return http.StatusInternalServerError } diff --git a/lib/controller/federation/federation_test.go b/lib/controller/federation/federation_test.go index 256afc8e6b..5079b402b7 100644 --- a/lib/controller/federation/federation_test.go +++ b/lib/controller/federation/federation_test.go @@ -38,7 +38,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) { ClusterID: "aaaaa", SystemRootToken: arvadostest.SystemRootToken, RemoteClusters: map[string]arvados.RemoteCluster{ - "aaaaa": arvados.RemoteCluster{ + "aaaaa": { Host: os.Getenv("ARVADOS_API_HOST"), }, }, diff --git a/lib/controller/federation/login_test.go b/lib/controller/federation/login_test.go index ad91bcf802..007f5df8b4 100644 --- a/lib/controller/federation/login_test.go +++ b/lib/controller/federation/login_test.go @@ -43,8 +43,6 @@ func (s *LoginSuite) TestDeferToLoginCluster(c *check.C) { func (s *LoginSuite) TestLogout(c *check.C) { s.cluster.Services.Workbench1.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench1.example.com"} s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench2.example.com"} - s.cluster.Login.Google.Enable = true - s.cluster.Login.Google.ClientID = "zzzzzzzzzzzzzz" s.addHTTPRemote(c, "zhome", &arvadostest.APIStub{}) s.cluster.Login.LoginCluster = "zhome" // s.fed is already set by SetUpTest, but we need to diff --git a/lib/controller/federation/user_test.go b/lib/controller/federation/user_test.go index 09aa5086de..2812c1f41d 100644 --- a/lib/controller/federation/user_test.go +++ b/lib/controller/federation/user_test.go @@ -117,6 +117,60 @@ func (s *UserSuite) TestLoginClusterUserList(c *check.C) { } } +func (s *UserSuite) TestLoginClusterUserGet(c *check.C) { + s.cluster.ClusterID = "local" + s.cluster.Login.LoginCluster = "zzzzz" + s.fed = New(s.cluster) + s.addDirectRemote(c, "zzzzz", rpc.NewConn("zzzzz", &url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}, true, rpc.PassthroughTokenProvider)) + + opts := arvados.GetOptions{UUID: "zzzzz-tpzed-xurymjxw79nv3jz", Select: []string{"uuid", "email"}} + + stub := &arvadostest.APIStub{Error: errors.New("local cluster failure")} + s.fed.local = stub + s.fed.UserGet(s.ctx, opts) + + calls := stub.Calls(stub.UserBatchUpdate) + if c.Check(calls, check.HasLen, 1) { + c.Logf("... stub.UserUpdate called with options: %#v", calls[0].Options) + shouldUpdate := map[string]bool{ + "uuid": false, + "email": true, + "first_name": true, + "last_name": true, + "is_admin": true, + "is_active": true, + "prefs": true, + // can't safely update locally + "owner_uuid": false, + "identity_url": false, + // virtual attrs + "full_name": false, + "is_invited": false, + } + if opts.Select != nil { + // Only the selected + // fields (minus uuid) + // should be updated. + for k := range shouldUpdate { + shouldUpdate[k] = false + } + for _, k := range opts.Select { + if k != "uuid" { + shouldUpdate[k] = true + } + } + } + var uuid string + for uuid = range calls[0].Options.(arvados.UserBatchUpdateOptions).Updates { + } + for k, shouldFind := range shouldUpdate { + _, found := calls[0].Options.(arvados.UserBatchUpdateOptions).Updates[uuid][k] + c.Check(found, check.Equals, shouldFind, check.Commentf("offending attr: %s", k)) + } + } + +} + func (s *UserSuite) TestLoginClusterUserListBypassFederation(c *check.C) { s.cluster.ClusterID = "local" s.cluster.Login.LoginCluster = "zzzzz" diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go index 6a9ad8c15f..031166b291 100644 --- a/lib/controller/federation_test.go +++ b/lib/controller/federation_test.go @@ -820,7 +820,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) { w.WriteHeader(200) w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`)) } - callCount += 1 + callCount++ })).Close() req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s", url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`, @@ -856,7 +856,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) { w.WriteHeader(200) w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`)) } - callCount += 1 + callCount++ })).Close() req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s", url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`, diff --git a/lib/controller/handler.go b/lib/controller/handler.go index 2dd1d816e0..6669e020fd 100644 --- a/lib/controller/handler.go +++ b/lib/controller/handler.go @@ -14,7 +14,9 @@ import ( "sync" "time" + "git.arvados.org/arvados.git/lib/controller/api" "git.arvados.org/arvados.git/lib/controller/federation" + "git.arvados.org/arvados.git/lib/controller/localdb" "git.arvados.org/arvados.git/lib/controller/railsproxy" "git.arvados.org/arvados.git/lib/controller/router" "git.arvados.org/arvados.git/lib/ctrlctx" @@ -23,6 +25,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/health" "git.arvados.org/arvados.git/sdk/go/httpserver" "github.com/jmoiron/sqlx" + // sqlx needs lib/pq to talk to PostgreSQL _ "github.com/lib/pq" ) @@ -87,7 +90,8 @@ func (h *Handler) setup() { Routes: health.Routes{"ping": func() error { _, err := h.db(context.TODO()); return err }}, }) - rtr := router.New(federation.New(h.Cluster), ctrlctx.WrapCallsInTransactions(h.db)) + oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.db) + rtr := router.New(federation.New(h.Cluster), api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls)) mux.Handle("/arvados/v1/config", rtr) mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr) @@ -103,6 +107,7 @@ func (h *Handler) setup() { hs := http.NotFoundHandler() hs = prepend(hs, h.proxyRailsAPI) hs = h.setupProxyRemoteCluster(hs) + hs = prepend(hs, oidcAuthorizer.Middleware) mux.Handle("/", hs) h.handlerStack = mux diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go index ef6b9195f1..7d8266a85c 100644 --- a/lib/controller/handler_test.go +++ b/lib/controller/handler_test.go @@ -334,7 +334,7 @@ func (s *HandlerSuite) TestGetObjects(c *check.C) { "api_clients/" + arvadostest.TrustedWorkbenchAPIClientUUID: nil, "api_client_authorizations/" + arvadostest.AdminTokenUUID: nil, "authorized_keys/" + arvadostest.AdminAuthorizedKeysUUID: nil, - "collections/" + arvadostest.CollectionWithUniqueWordsUUID: map[string]bool{"href": true}, + "collections/" + arvadostest.CollectionWithUniqueWordsUUID: {"href": true}, "containers/" + arvadostest.RunningContainerUUID: nil, "container_requests/" + arvadostest.QueuedContainerRequestUUID: nil, "groups/" + arvadostest.AProjectUUID: nil, @@ -343,7 +343,7 @@ func (s *HandlerSuite) TestGetObjects(c *check.C) { "logs/" + arvadostest.CrunchstatForRunningJobLogUUID: nil, "nodes/" + arvadostest.IdleNodeUUID: nil, "repositories/" + arvadostest.ArvadosRepoUUID: nil, - "users/" + arvadostest.ActiveUserUUID: map[string]bool{"href": true}, + "users/" + arvadostest.ActiveUserUUID: {"href": true}, "virtual_machines/" + arvadostest.TestVMUUID: nil, "workflows/" + arvadostest.WorkflowWithDefinitionYAMLUUID: nil, } diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go index a73f5f9f82..0388f21bee 100644 --- a/lib/controller/integration_test.go +++ b/lib/controller/integration_test.go @@ -8,13 +8,18 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" + "io/ioutil" "math" "net" "net/http" "net/url" "os" + "os/exec" "path/filepath" + "strconv" + "strings" "git.arvados.org/arvados.git/lib/boot" "git.arvados.org/arvados.git/lib/config" @@ -22,6 +27,7 @@ import ( "git.arvados.org/arvados.git/lib/service" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/keepclient" @@ -38,6 +44,7 @@ type testCluster struct { type IntegrationSuite struct { testClusters map[string]*testCluster + oidcprovider *arvadostest.OIDCProvider } func (s *IntegrationSuite) SetUpSuite(c *check.C) { @@ -47,6 +54,14 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { } cwd, _ := os.Getwd() + + s.oidcprovider = arvadostest.NewOIDCProvider(c) + s.oidcprovider.AuthEmail = "user@example.com" + s.oidcprovider.AuthEmailVerified = true + s.oidcprovider.AuthName = "Example User" + s.oidcprovider.ValidClientID = "clientid" + s.oidcprovider.ValidClientSecret = "clientsecret" + s.testClusters = map[string]*testCluster{ "z1111": nil, "z2222": nil, @@ -105,6 +120,24 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { ActivateUsers: true ` } + if id == "z1111" { + yaml += ` + Login: + LoginCluster: z1111 + OpenIDConnect: + Enable: true + Issuer: ` + s.oidcprovider.Issuer.URL + ` + ClientID: ` + s.oidcprovider.ValidClientID + ` + ClientSecret: ` + s.oidcprovider.ValidClientSecret + ` + EmailClaim: email + EmailVerifiedClaim: email_verified +` + } else { + yaml += ` + Login: + LoginCluster: z1111 +` + } loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c)) loader.Path = "-" @@ -139,10 +172,15 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) { } } +// Get rpc connection struct initialized to communicate with the +// specified cluster. func (s *IntegrationSuite) conn(clusterID string) *rpc.Conn { return rpc.NewConn(clusterID, s.testClusters[clusterID].controllerURL, true, rpc.PassthroughTokenProvider) } +// Return Context, Arvados.Client and keepclient structs initialized +// to connect to the specified cluster (by clusterID) using with the supplied +// Arvados token. func (s *IntegrationSuite) clientsWithToken(clusterID string, token string) (context.Context, *arvados.Client, *keepclient.KeepClient) { cl := s.testClusters[clusterID].config.Clusters[clusterID] ctx := auth.NewContext(context.Background(), auth.NewCredentials(token)) @@ -159,7 +197,11 @@ func (s *IntegrationSuite) clientsWithToken(clusterID string, token string) (con return ctx, ac, kc } -func (s *IntegrationSuite) userClients(rootctx context.Context, c *check.C, conn *rpc.Conn, clusterID string, activate bool) (context.Context, *arvados.Client, *keepclient.KeepClient) { +// Log in as a user called "example", get the user's API token, +// initialize clients with the API token, set up the user and +// optionally activate the user. Return client structs for +// communicating with the cluster on behalf of the 'example' user. +func (s *IntegrationSuite) userClients(rootctx context.Context, c *check.C, conn *rpc.Conn, clusterID string, activate bool) (context.Context, *arvados.Client, *keepclient.KeepClient, arvados.User) { login, err := conn.UserSessionCreate(rootctx, rpc.UserSessionCreateOptions{ ReturnTo: ",https://example.com", AuthInfo: rpc.UserSessionAuthInfo{ @@ -189,18 +231,26 @@ func (s *IntegrationSuite) userClients(rootctx context.Context, c *check.C, conn c.Fatalf("failed to activate user -- %#v", user) } } - return ctx, ac, kc + return ctx, ac, kc, user } +// Return Context, arvados.Client and keepclient structs initialized +// to communicate with the cluster as the system root user. func (s *IntegrationSuite) rootClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) { return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].SystemRootToken) } +// Return Context, arvados.Client and keepclient structs initialized +// to communicate with the cluster as the anonymous user. +func (s *IntegrationSuite) anonymousClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) { + return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].Users.AnonymousUserToken) +} + func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { conn1 := s.conn("z1111") rootctx1, _, _ := s.rootClients("z1111") conn3 := s.conn("z3333") - userctx1, ac1, kc1 := s.userClients(rootctx1, c, conn1, "z1111", true) + userctx1, ac1, kc1, _ := s.userClients(rootctx1, c, conn1, "z1111", true) // Create the collection to find its PDH (but don't save it // anywhere yet) @@ -234,12 +284,150 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) { c.Check(coll.PortableDataHash, check.Equals, pdh) } +func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) { + if _, err := exec.LookPath("s3cmd"); err != nil { + c.Skip("s3cmd not in PATH") + return + } + + testText := "IntegrationSuite.TestS3WithFederatedToken" + + conn1 := s.conn("z1111") + rootctx1, _, _ := s.rootClients("z1111") + userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) + conn3 := s.conn("z3333") + + createColl := func(clusterID string) arvados.Collection { + _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken) + var coll arvados.Collection + fs, err := coll.FileSystem(ac, kc) + c.Assert(err, check.IsNil) + f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777) + c.Assert(err, check.IsNil) + _, err = io.WriteString(f, testText) + c.Assert(err, check.IsNil) + err = f.Close() + c.Assert(err, check.IsNil) + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": mtxt, + }}) + c.Assert(err, check.IsNil) + return coll + } + + for _, trial := range []struct { + clusterID string // create the collection on this cluster (then use z3333 to access it) + token string + }{ + // Try the hardest test first: z3333 hasn't seen + // z1111's token yet, and we're just passing the + // opaque secret part, so z3333 has to guess that it + // belongs to z1111. + {"z1111", strings.Split(ac1.AuthToken, "/")[2]}, + {"z3333", strings.Split(ac1.AuthToken, "/")[2]}, + {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)}, + {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)}, + } { + c.Logf("================ %v", trial) + coll := createColl(trial.clusterID) + + cfgjson, err := conn3.ConfigGet(userctx1) + c.Assert(err, check.IsNil) + var cluster arvados.Cluster + err = json.Unmarshal(cfgjson, &cluster) + c.Assert(err, check.IsNil) + + c.Logf("TokenV2 is %s", ac1.AuthToken) + host := cluster.Services.WebDAV.ExternalURL.Host + s3args := []string{ + "--ssl", "--no-check-certificate", + "--host=" + host, "--host-bucket=" + host, + "--access_key=" + trial.token, "--secret_key=" + trial.token, + } + buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput() + c.Check(err, check.IsNil) + c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`) + + buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput() + // Command fails because we don't return Etag header. + flen := strconv.Itoa(len(testText)) + c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`) + } +} + +func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) { + conn1 := s.conn("z1111") + conn3 := s.conn("z3333") + rootctx1, rootac1, rootkc1 := s.rootClients("z1111") + anonctx3, anonac3, _ := s.anonymousClients("z3333") + + // Make sure anonymous token was set + c.Assert(anonac3.AuthToken, check.Not(check.Equals), "") + + // Create the collection to find its PDH (but don't save it + // anywhere yet) + var coll1 arvados.Collection + fs1, err := coll1.FileSystem(rootac1, rootkc1) + c.Assert(err, check.IsNil) + f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777) + c.Assert(err, check.IsNil) + _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionAsAnonymous") + c.Assert(err, check.IsNil) + err = f.Close() + c.Assert(err, check.IsNil) + mtxt, err := fs1.MarshalManifest(".") + c.Assert(err, check.IsNil) + pdh := arvados.PortableDataHash(mtxt) + + // Save the collection on cluster z1111. + coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": mtxt, + }}) + c.Assert(err, check.IsNil) + + // Share it with the anonymous users group. + var outLink arvados.Link + err = rootac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil, + map[string]interface{}{"link": map[string]interface{}{ + "link_class": "permission", + "name": "can_read", + "tail_uuid": "z1111-j7d0g-anonymouspublic", + "head_uuid": coll1.UUID, + }, + }) + c.Check(err, check.IsNil) + + // Current user should be z3 anonymous user + outUser, err := anonac3.CurrentUser() + c.Check(err, check.IsNil) + c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic") + + // Get the token uuid + var outAuth arvados.APIClientAuthorization + err = anonac3.RequestAndDecode(&outAuth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil) + c.Check(err, check.IsNil) + + // Make a v2 token of the z3 anonymous user, and use it on z1 + _, anonac1, _ := s.clientsWithToken("z1111", outAuth.TokenV2()) + outUser2, err := anonac1.CurrentUser() + c.Check(err, check.IsNil) + // z3 anonymous user will be mapped to the z1 anonymous user + c.Check(outUser2.UUID, check.Equals, "z1111-tpzed-anonymouspublic") + + // Retrieve the collection (which is on z1) using anonymous from cluster z3333. + coll, err := conn3.CollectionGet(anonctx3, arvados.GetOptions{UUID: coll1.UUID}) + c.Check(err, check.IsNil) + c.Check(coll.PortableDataHash, check.Equals, pdh) +} + // Get a token from the login cluster (z1111), use it to submit a // container request on z2222. func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) { conn1 := s.conn("z1111") rootctx1, _, _ := s.rootClients("z1111") - _, ac1, _ := s.userClients(rootctx1, c, conn1, "z1111", true) + _, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) // Use ac2 to get the discovery doc with a blank token, so the // SDK doesn't magically pass the z1111 token to z2222 before @@ -310,7 +498,7 @@ func (s *IntegrationSuite) TestListUsers(c *check.C) { rootctx1, _, _ := s.rootClients("z1111") conn1 := s.conn("z1111") conn3 := s.conn("z3333") - userctx1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) + userctx1, _, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true) // Make sure LoginCluster is properly configured for cls := range s.testClusters { @@ -374,3 +562,119 @@ func (s *IntegrationSuite) TestListUsers(c *check.C) { c.Assert(err, check.IsNil) c.Check(user1.IsActive, check.Equals, false) } + +func (s *IntegrationSuite) TestSetupUserWithVM(c *check.C) { + conn1 := s.conn("z1111") + conn3 := s.conn("z3333") + rootctx1, rootac1, _ := s.rootClients("z1111") + + // Create user on LoginCluster z1111 + _, _, _, user := s.userClients(rootctx1, c, conn1, "z1111", false) + + // Make a new root token (because rootClients() uses SystemRootToken) + var outAuth arvados.APIClientAuthorization + err := rootac1.RequestAndDecode(&outAuth, "POST", "/arvados/v1/api_client_authorizations", nil, nil) + c.Check(err, check.IsNil) + + // Make a v2 root token to communicate with z3333 + rootctx3, rootac3, _ := s.clientsWithToken("z3333", outAuth.TokenV2()) + + // Create VM on z3333 + var outVM arvados.VirtualMachine + err = rootac3.RequestAndDecode(&outVM, "POST", "/arvados/v1/virtual_machines", nil, + map[string]interface{}{"virtual_machine": map[string]interface{}{ + "hostname": "example", + }, + }) + c.Check(outVM.UUID[0:5], check.Equals, "z3333") + c.Check(err, check.IsNil) + + // Make sure z3333 user list is up to date + _, err = conn3.UserList(rootctx3, arvados.ListOptions{Limit: 1000}) + c.Check(err, check.IsNil) + + // Try to set up user on z3333 with the VM + _, err = conn3.UserSetup(rootctx3, arvados.UserSetupOptions{UUID: user.UUID, VMUUID: outVM.UUID}) + c.Check(err, check.IsNil) + + var outLinks arvados.LinkList + err = rootac3.RequestAndDecode(&outLinks, "GET", "/arvados/v1/links", nil, + arvados.ListOptions{ + Limit: 1000, + Filters: []arvados.Filter{ + { + Attr: "tail_uuid", + Operator: "=", + Operand: user.UUID, + }, + { + Attr: "head_uuid", + Operator: "=", + Operand: outVM.UUID, + }, + { + Attr: "name", + Operator: "=", + Operand: "can_login", + }, + { + Attr: "link_class", + Operator: "=", + Operand: "permission", + }}}) + c.Check(err, check.IsNil) + + c.Check(len(outLinks.Items), check.Equals, 1) +} + +func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) { + conn1 := s.conn("z1111") + rootctx1, _, _ := s.rootClients("z1111") + s.userClients(rootctx1, c, conn1, "z1111", true) + + accesstoken := s.oidcprovider.ValidAccessToken() + + for _, clusterid := range []string{"z1111", "z2222"} { + c.Logf("trying clusterid %s", clusterid) + + conn := s.conn(clusterid) + ctx, ac, kc := s.clientsWithToken(clusterid, accesstoken) + + var coll arvados.Collection + + // Write some file data and create a collection + { + fs, err := coll.FileSystem(ac, kc) + c.Assert(err, check.IsNil) + f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777) + c.Assert(err, check.IsNil) + _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth") + c.Assert(err, check.IsNil) + err = f.Close() + c.Assert(err, check.IsNil) + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{ + "manifest_text": mtxt, + }}) + c.Assert(err, check.IsNil) + } + + // Read the collection & file data + { + user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{}) + c.Assert(err, check.IsNil) + c.Check(user.FullName, check.Equals, "Example User") + coll, err = conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID}) + c.Assert(err, check.IsNil) + c.Check(coll.ManifestText, check.Not(check.Equals), "") + fs, err := coll.FileSystem(ac, kc) + c.Assert(err, check.IsNil) + f, err := fs.Open("test.txt") + c.Assert(err, check.IsNil) + buf, err := ioutil.ReadAll(f) + c.Assert(err, check.IsNil) + c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth")) + } + } +} diff --git a/lib/controller/localdb/login.go b/lib/controller/localdb/login.go index ee1ea56924..f4632751e3 100644 --- a/lib/controller/localdb/login.go +++ b/lib/controller/localdb/login.go @@ -33,8 +33,14 @@ func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) log wantSSO := cluster.Login.SSO.Enable wantPAM := cluster.Login.PAM.Enable wantLDAP := cluster.Login.LDAP.Enable + wantTest := cluster.Login.Test.Enable + wantLoginCluster := cluster.Login.LoginCluster != "" && cluster.Login.LoginCluster != cluster.ClusterID switch { - case wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP: + case 1 != countTrue(wantGoogle, wantOpenIDConnect, wantSSO, wantPAM, wantLDAP, wantTest, wantLoginCluster): + return errorLoginController{ + error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, Login.LDAP, Login.Test, or Login.LoginCluster must be set"), + } + case wantGoogle: return &oidcLoginController{ Cluster: cluster, RailsProxy: railsProxy, @@ -45,7 +51,7 @@ func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) log EmailClaim: "email", EmailVerifiedClaim: "email_verified", } - case !wantGoogle && wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP: + case wantOpenIDConnect: return &oidcLoginController{ Cluster: cluster, RailsProxy: railsProxy, @@ -56,19 +62,33 @@ func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) log EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim, UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim, } - case !wantGoogle && !wantOpenIDConnect && wantSSO && !wantPAM && !wantLDAP: + case wantSSO: return &ssoLoginController{railsProxy} - case !wantGoogle && !wantOpenIDConnect && !wantSSO && wantPAM && !wantLDAP: + case wantPAM: return &pamLoginController{Cluster: cluster, RailsProxy: railsProxy} - case !wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && wantLDAP: + case wantLDAP: return &ldapLoginController{Cluster: cluster, RailsProxy: railsProxy} + case wantTest: + return &testLoginController{Cluster: cluster, RailsProxy: railsProxy} + case wantLoginCluster: + return &federatedLoginController{Cluster: cluster} default: return errorLoginController{ - error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, and Login.LDAP must be enabled"), + error: errors.New("BUG: missing case in login controller setup switch"), } } } +func countTrue(vals ...bool) int { + n := 0 + for _, val := range vals { + if val { + n++ + } + } + return n +} + // Login and Logout are passed through to the wrapped railsProxy; // UserAuthenticate is rejected. type ssoLoginController struct{ *railsProxy } @@ -89,6 +109,20 @@ func (ctrl errorLoginController) UserAuthenticate(context.Context, arvados.UserA return arvados.APIClientAuthorization{}, ctrl.error } +type federatedLoginController struct { + Cluster *arvados.Cluster +} + +func (ctrl federatedLoginController) Login(context.Context, arvados.LoginOptions) (arvados.LoginResponse, error) { + return arvados.LoginResponse{}, httpserver.ErrorWithStatus(errors.New("Should have been redirected to login cluster"), http.StatusBadRequest) +} +func (ctrl federatedLoginController) Logout(_ context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) { + return noopLogout(ctrl.Cluster, opts) +} +func (ctrl federatedLoginController) UserAuthenticate(context.Context, arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) { + return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest) +} + func noopLogout(cluster *arvados.Cluster, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) { target := opts.ReturnTo if target == "" { @@ -107,7 +141,7 @@ func createAPIClientAuthorization(ctx context.Context, conn *rpc.Conn, rootToken // Send a fake ReturnTo value instead of the caller's // opts.ReturnTo. We won't follow the resulting // redirect target anyway. - ReturnTo: ",https://none.invalid", + ReturnTo: ",https://controller.api.client.invalid", AuthInfo: authinfo, }) if err != nil { diff --git a/lib/controller/localdb/login_ldap_test.go b/lib/controller/localdb/login_ldap_test.go index 700d757c27..bce1ecfcf2 100644 --- a/lib/controller/localdb/login_ldap_test.go +++ b/lib/controller/localdb/login_ldap_test.go @@ -64,7 +64,7 @@ func (s *LDAPSuite) SetUpSuite(c *check.C) { return []*godap.LDAPSimpleSearchResultEntry{} } return []*godap.LDAPSimpleSearchResultEntry{ - &godap.LDAPSimpleSearchResultEntry{ + { DN: "cn=" + req.FilterValue + "," + req.BaseDN, Attrs: map[string]interface{}{ "SN": req.FilterValue, diff --git a/lib/controller/localdb/login_oidc.go b/lib/controller/localdb/login_oidc.go index 9274d75d7c..5f96da5624 100644 --- a/lib/controller/localdb/login_oidc.go +++ b/lib/controller/localdb/login_oidc.go @@ -9,9 +9,11 @@ import ( "context" "crypto/hmac" "crypto/sha256" + "database/sql" "encoding/base64" "errors" "fmt" + "io" "net/http" "net/url" "strings" @@ -19,17 +21,29 @@ import ( "text/template" "time" + "git.arvados.org/arvados.git/lib/controller/api" + "git.arvados.org/arvados.git/lib/controller/railsproxy" "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/lib/ctrlctx" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/httpserver" "github.com/coreos/go-oidc" + lru "github.com/hashicorp/golang-lru" + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" "golang.org/x/oauth2" "google.golang.org/api/option" "google.golang.org/api/people/v1" ) +const ( + tokenCacheSize = 1000 + tokenCacheNegativeTTL = time.Minute * 5 + tokenCacheTTL = time.Minute * 10 +) + type oidcLoginController struct { Cluster *arvados.Cluster RailsProxy *railsProxy @@ -106,51 +120,56 @@ func (ctrl *oidcLoginController) Login(ctx context.Context, opts arvados.LoginOp // one Google account. oauth2.SetAuthURLParam("prompt", "select_account")), }, nil - } else { - // Callback after OIDC sign-in. - state := ctrl.parseOAuth2State(opts.State) - if !state.verify([]byte(ctrl.Cluster.SystemRootToken)) { - return loginError(errors.New("invalid OAuth2 state")) - } - oauth2Token, err := ctrl.oauth2conf.Exchange(ctx, opts.Code) - if err != nil { - return loginError(fmt.Errorf("error in OAuth2 exchange: %s", err)) - } - rawIDToken, ok := oauth2Token.Extra("id_token").(string) - if !ok { - return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token")) - } - idToken, err := ctrl.verifier.Verify(ctx, rawIDToken) - if err != nil { - return loginError(fmt.Errorf("error verifying ID token: %s", err)) - } - authinfo, err := ctrl.getAuthInfo(ctx, oauth2Token, idToken) - if err != nil { - return loginError(err) - } - ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{ctrl.Cluster.SystemRootToken}}) - return ctrl.RailsProxy.UserSessionCreate(ctxRoot, rpc.UserSessionCreateOptions{ - ReturnTo: state.Remote + "," + state.ReturnTo, - AuthInfo: *authinfo, - }) } + // Callback after OIDC sign-in. + state := ctrl.parseOAuth2State(opts.State) + if !state.verify([]byte(ctrl.Cluster.SystemRootToken)) { + return loginError(errors.New("invalid OAuth2 state")) + } + oauth2Token, err := ctrl.oauth2conf.Exchange(ctx, opts.Code) + if err != nil { + return loginError(fmt.Errorf("error in OAuth2 exchange: %s", err)) + } + rawIDToken, ok := oauth2Token.Extra("id_token").(string) + if !ok { + return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token")) + } + idToken, err := ctrl.verifier.Verify(ctx, rawIDToken) + if err != nil { + return loginError(fmt.Errorf("error verifying ID token: %s", err)) + } + authinfo, err := ctrl.getAuthInfo(ctx, oauth2Token, idToken) + if err != nil { + return loginError(err) + } + ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{ctrl.Cluster.SystemRootToken}}) + return ctrl.RailsProxy.UserSessionCreate(ctxRoot, rpc.UserSessionCreateOptions{ + ReturnTo: state.Remote + "," + state.ReturnTo, + AuthInfo: *authinfo, + }) } func (ctrl *oidcLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) { return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest) } +// claimser can decode arbitrary claims into a map. Implemented by +// *oauth2.IDToken and *oauth2.UserInfo. +type claimser interface { + Claims(interface{}) error +} + // Use a person's token to get all of their email addresses, with the // primary address at index 0. The provided defaultAddr is always // included in the returned slice, and is used as the primary if the // Google API does not indicate one. -func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) { +func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, claimser claimser) (*rpc.UserSessionAuthInfo, error) { var ret rpc.UserSessionAuthInfo defer ctxlog.FromContext(ctx).WithField("ret", &ret).Debug("getAuthInfo returned") var claims map[string]interface{} - if err := idToken.Claims(&claims); err != nil { - return nil, fmt.Errorf("error extracting claims from ID token: %s", err) + if err := claimser.Claims(&claims); err != nil { + return nil, fmt.Errorf("error extracting claims from token: %s", err) } else if verified, _ := claims[ctrl.EmailVerifiedClaim].(bool); verified || ctrl.EmailVerifiedClaim == "" { // Fall back to this info if the People API call // (below) doesn't return a primary && verified email. @@ -190,9 +209,8 @@ func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2. // only the "fix config" advice to the user. ctxlog.FromContext(ctx).WithError(err).WithField("email", ret.Email).Error("People API is not enabled") return nil, errors.New("configuration error: Login.GoogleAlternateEmailAddresses is true, but Google People API is not enabled") - } else { - return nil, fmt.Errorf("error getting profile info from People API: %s", err) } + return nil, fmt.Errorf("error getting profile info from People API: %s", err) } // The given/family names returned by the People API and @@ -299,3 +317,178 @@ func (s oauth2State) computeHMAC(key []byte) []byte { fmt.Fprintf(mac, "%x %s %s", s.Time, s.Remote, s.ReturnTo) return mac.Sum(nil) } + +func OIDCAccessTokenAuthorizer(cluster *arvados.Cluster, getdb func(context.Context) (*sqlx.DB, error)) *oidcTokenAuthorizer { + // We want ctrl to be nil if the chosen controller is not a + // *oidcLoginController, so we can ignore the 2nd return value + // of this type cast. + ctrl, _ := chooseLoginController(cluster, railsproxy.NewConn(cluster)).(*oidcLoginController) + cache, err := lru.New2Q(tokenCacheSize) + if err != nil { + panic(err) + } + return &oidcTokenAuthorizer{ + ctrl: ctrl, + getdb: getdb, + cache: cache, + } +} + +type oidcTokenAuthorizer struct { + ctrl *oidcLoginController + getdb func(context.Context) (*sqlx.DB, error) + cache *lru.TwoQueueCache +} + +func (ta *oidcTokenAuthorizer) Middleware(w http.ResponseWriter, r *http.Request, next http.Handler) { + if ta.ctrl == nil { + // Not using a compatible (OIDC) login controller. + } else if authhdr := strings.Split(r.Header.Get("Authorization"), " "); len(authhdr) > 1 && (authhdr[0] == "OAuth2" || authhdr[0] == "Bearer") { + err := ta.registerToken(r.Context(), authhdr[1]) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + next.ServeHTTP(w, r) +} + +func (ta *oidcTokenAuthorizer) WrapCalls(origFunc api.RoutableFunc) api.RoutableFunc { + if ta.ctrl == nil { + // Not using a compatible (OIDC) login controller. + return origFunc + } + return func(ctx context.Context, opts interface{}) (_ interface{}, err error) { + creds, ok := auth.FromContext(ctx) + if !ok { + return origFunc(ctx, opts) + } + // Check each token in the incoming request. If any + // are OAuth2 access tokens, swap them out for Arvados + // tokens. + for _, tok := range creds.Tokens { + err = ta.registerToken(ctx, tok) + if err != nil { + return nil, err + } + } + return origFunc(ctx, opts) + } +} + +// registerToken checks whether tok is a valid OIDC Access Token and, +// if so, ensures that an api_client_authorizations row exists so that +// RailsAPI will accept it as an Arvados token. +func (ta *oidcTokenAuthorizer) registerToken(ctx context.Context, tok string) error { + if tok == ta.ctrl.Cluster.SystemRootToken || strings.HasPrefix(tok, "v2/") { + return nil + } + if cached, hit := ta.cache.Get(tok); !hit { + // Fall through to database and OIDC provider checks + // below + } else if exp, ok := cached.(time.Time); ok { + // cached negative result (value is expiry time) + if time.Now().Before(exp) { + return nil + } + ta.cache.Remove(tok) + } else { + // cached positive result + aca := cached.(arvados.APIClientAuthorization) + var expiring bool + if aca.ExpiresAt != "" { + t, err := time.Parse(time.RFC3339Nano, aca.ExpiresAt) + if err != nil { + return fmt.Errorf("error parsing expires_at value: %w", err) + } + expiring = t.Before(time.Now().Add(time.Minute)) + } + if !expiring { + return nil + } + } + + db, err := ta.getdb(ctx) + if err != nil { + return err + } + tx, err := db.Beginx() + if err != nil { + return err + } + defer tx.Rollback() + ctx = ctrlctx.NewWithTransaction(ctx, tx) + + // We use hmac-sha256(accesstoken,systemroottoken) as the + // secret part of our own token, and avoid storing the auth + // provider's real secret in our database. + mac := hmac.New(sha256.New, []byte(ta.ctrl.Cluster.SystemRootToken)) + io.WriteString(mac, tok) + hmac := fmt.Sprintf("%x", mac.Sum(nil)) + + var expiring bool + err = tx.QueryRowContext(ctx, `select (expires_at is not null and expires_at - interval '1 minute' <= current_timestamp at time zone 'UTC') from api_client_authorizations where api_token=$1`, hmac).Scan(&expiring) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("database error while checking token: %w", err) + } else if err == nil && !expiring { + // Token is already in the database as an Arvados + // token, and isn't about to expire, so we can pass it + // through to RailsAPI etc. regardless of whether it's + // an OIDC access token. + return nil + } + updating := err == nil + + // Check whether the token is a valid OIDC access token. If + // so, swap it out for an Arvados token (creating/updating an + // api_client_authorizations row if needed) which downstream + // server components will accept. + err = ta.ctrl.setup() + if err != nil { + return fmt.Errorf("error setting up OpenID Connect provider: %s", err) + } + oauth2Token := &oauth2.Token{ + AccessToken: tok, + } + userinfo, err := ta.ctrl.provider.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token)) + if err != nil { + ta.cache.Add(tok, time.Now().Add(tokenCacheNegativeTTL)) + return nil + } + ctxlog.FromContext(ctx).WithField("userinfo", userinfo).Debug("(*oidcTokenAuthorizer)registerToken: got userinfo") + authinfo, err := ta.ctrl.getAuthInfo(ctx, oauth2Token, userinfo) + if err != nil { + return err + } + + // Expiry time for our token is one minute longer than our + // cache TTL, so we don't pass it through to RailsAPI just as + // it's expiring. + exp := time.Now().UTC().Add(tokenCacheTTL + time.Minute) + + var aca arvados.APIClientAuthorization + if updating { + _, err = tx.ExecContext(ctx, `update api_client_authorizations set expires_at=$1 where api_token=$2`, exp, hmac) + if err != nil { + return fmt.Errorf("error updating token expiry time: %w", err) + } + ctxlog.FromContext(ctx).WithField("HMAC", hmac).Debug("(*oidcTokenAuthorizer)registerToken: updated api_client_authorizations row") + } else { + aca, err = createAPIClientAuthorization(ctx, ta.ctrl.RailsProxy, ta.ctrl.Cluster.SystemRootToken, *authinfo) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `update api_client_authorizations set api_token=$1, expires_at=$2 where uuid=$3`, hmac, exp, aca.UUID) + if err != nil { + return fmt.Errorf("error adding OIDC access token to database: %w", err) + } + aca.APIToken = hmac + ctxlog.FromContext(ctx).WithFields(logrus.Fields{"UUID": aca.UUID, "HMAC": hmac}).Debug("(*oidcTokenAuthorizer)registerToken: inserted api_client_authorizations row") + } + err = tx.Commit() + if err != nil { + return err + } + ta.cache.Add(tok, aca) + return nil +} diff --git a/lib/controller/localdb/login_oidc_test.go b/lib/controller/localdb/login_oidc_test.go index 2ccb1fce2a..9bc6f90ea9 100644 --- a/lib/controller/localdb/login_oidc_test.go +++ b/lib/controller/localdb/login_oidc_test.go @@ -7,9 +7,6 @@ package localdb import ( "bytes" "context" - "crypto/rand" - "crypto/rsa" - "encoding/base64" "encoding/json" "fmt" "net/http" @@ -27,7 +24,6 @@ import ( "git.arvados.org/arvados.git/sdk/go/auth" "git.arvados.org/arvados.git/sdk/go/ctxlog" check "gopkg.in/check.v1" - jose "gopkg.in/square/go-jose.v2" ) // Gocheck boilerplate @@ -38,22 +34,10 @@ func Test(t *testing.T) { var _ = check.Suite(&OIDCLoginSuite{}) type OIDCLoginSuite struct { - cluster *arvados.Cluster - localdb *Conn - railsSpy *arvadostest.Proxy - fakeIssuer *httptest.Server - fakePeopleAPI *httptest.Server - fakePeopleAPIResponse map[string]interface{} - issuerKey *rsa.PrivateKey - - // expected token request - validCode string - validClientID string - validClientSecret string - // desired response from token endpoint - authEmail string - authEmailVerified bool - authName string + cluster *arvados.Cluster + localdb *Conn + railsSpy *arvadostest.Proxy + fakeProvider *arvadostest.OIDCProvider } func (s *OIDCLoginSuite) TearDownSuite(c *check.C) { @@ -64,103 +48,12 @@ func (s *OIDCLoginSuite) TearDownSuite(c *check.C) { } func (s *OIDCLoginSuite) SetUpTest(c *check.C) { - var err error - s.issuerKey, err = rsa.GenerateKey(rand.Reader, 2048) - c.Assert(err, check.IsNil) - - s.authEmail = "active-user@arvados.local" - s.authEmailVerified = true - s.authName = "Fake User Name" - s.fakeIssuer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req.ParseForm() - c.Logf("fakeIssuer: got req: %s %s %s", req.Method, req.URL, req.Form) - w.Header().Set("Content-Type", "application/json") - switch req.URL.Path { - case "/.well-known/openid-configuration": - json.NewEncoder(w).Encode(map[string]interface{}{ - "issuer": s.fakeIssuer.URL, - "authorization_endpoint": s.fakeIssuer.URL + "/auth", - "token_endpoint": s.fakeIssuer.URL + "/token", - "jwks_uri": s.fakeIssuer.URL + "/jwks", - "userinfo_endpoint": s.fakeIssuer.URL + "/userinfo", - }) - case "/token": - var clientID, clientSecret string - auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic ")) - authsplit := strings.Split(string(auth), ":") - if len(authsplit) == 2 { - clientID, _ = url.QueryUnescape(authsplit[0]) - clientSecret, _ = url.QueryUnescape(authsplit[1]) - } - if clientID != s.validClientID || clientSecret != s.validClientSecret { - c.Logf("fakeIssuer: expected (%q, %q) got (%q, %q)", s.validClientID, s.validClientSecret, clientID, clientSecret) - w.WriteHeader(http.StatusUnauthorized) - return - } - - if req.Form.Get("code") != s.validCode || s.validCode == "" { - w.WriteHeader(http.StatusUnauthorized) - return - } - idToken, _ := json.Marshal(map[string]interface{}{ - "iss": s.fakeIssuer.URL, - "aud": []string{clientID}, - "sub": "fake-user-id", - "exp": time.Now().UTC().Add(time.Minute).Unix(), - "iat": time.Now().UTC().Unix(), - "nonce": "fake-nonce", - "email": s.authEmail, - "email_verified": s.authEmailVerified, - "name": s.authName, - "alt_verified": true, // for custom claim tests - "alt_email": "alt_email@example.com", // for custom claim tests - "alt_username": "desired-username", // for custom claim tests - }) - json.NewEncoder(w).Encode(struct { - AccessToken string `json:"access_token"` - TokenType string `json:"token_type"` - RefreshToken string `json:"refresh_token"` - ExpiresIn int32 `json:"expires_in"` - IDToken string `json:"id_token"` - }{ - AccessToken: s.fakeToken(c, []byte("fake access token")), - TokenType: "Bearer", - RefreshToken: "test-refresh-token", - ExpiresIn: 30, - IDToken: s.fakeToken(c, idToken), - }) - case "/jwks": - json.NewEncoder(w).Encode(jose.JSONWebKeySet{ - Keys: []jose.JSONWebKey{ - {Key: s.issuerKey.Public(), Algorithm: string(jose.RS256), KeyID: ""}, - }, - }) - case "/auth": - w.WriteHeader(http.StatusInternalServerError) - case "/userinfo": - w.WriteHeader(http.StatusInternalServerError) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - s.validCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix()) - - s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req.ParseForm() - c.Logf("fakePeopleAPI: got req: %s %s %s", req.Method, req.URL, req.Form) - w.Header().Set("Content-Type", "application/json") - switch req.URL.Path { - case "/v1/people/me": - if f := req.Form.Get("personFields"); f != "emailAddresses,names" { - w.WriteHeader(http.StatusBadRequest) - break - } - json.NewEncoder(w).Encode(s.fakePeopleAPIResponse) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - s.fakePeopleAPIResponse = map[string]interface{}{} + s.fakeProvider = arvadostest.NewOIDCProvider(c) + s.fakeProvider.AuthEmail = "active-user@arvados.local" + s.fakeProvider.AuthEmailVerified = true + s.fakeProvider.AuthName = "Fake User Name" + s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix()) + s.fakeProvider.PeopleAPIResponse = map[string]interface{}{} cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() c.Assert(err, check.IsNil) @@ -171,13 +64,13 @@ func (s *OIDCLoginSuite) SetUpTest(c *check.C) { s.cluster.Login.Google.ClientID = "test%client$id" s.cluster.Login.Google.ClientSecret = "test#client/secret" s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com" - s.validClientID = "test%client$id" - s.validClientSecret = "test#client/secret" + s.fakeProvider.ValidClientID = "test%client$id" + s.fakeProvider.ValidClientSecret = "test#client/secret" s.localdb = NewConn(s.cluster) c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil)) - s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeIssuer.URL - s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL + s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeProvider.Issuer.URL + s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI) *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider) @@ -206,7 +99,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_Start(c *check.C) { c.Check(err, check.IsNil) target, err := url.Parse(resp.RedirectLocation) c.Check(err, check.IsNil) - issuerURL, _ := url.Parse(s.fakeIssuer.URL) + issuerURL, _ := url.Parse(s.fakeProvider.Issuer.URL) c.Check(target.Host, check.Equals, issuerURL.Host) q := target.Query() c.Check(q.Get("client_id"), check.Equals, "test%client$id") @@ -232,7 +125,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_InvalidCode(c *check.C) { func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) { s.startLogin(c) resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: "bogus-state", }) c.Check(err, check.IsNil) @@ -241,20 +134,20 @@ func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) { } func (s *OIDCLoginSuite) setupPeopleAPIError(c *check.C) { - s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + s.fakeProvider.PeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusForbidden) fmt.Fprintln(w, `Error 403: accessNotConfigured`) })) - s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL + s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL } func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) { s.localdb.loginController.(*oidcLoginController).UseGooglePeopleAPI = false - s.authEmail = "joe.smith@primary.example.com" + s.fakeProvider.AuthEmail = "joe.smith@primary.example.com" s.setupPeopleAPIError(c) state := s.startLogin(c) _, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) c.Check(err, check.IsNil) @@ -294,7 +187,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) { s.setupPeopleAPIError(c) state := s.startLogin(c) resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) c.Check(err, check.IsNil) @@ -304,11 +197,11 @@ func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) { func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { s.cluster.Login.Google.Enable = false s.cluster.Login.OpenIDConnect.Enable = true - json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeIssuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer) + json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer) s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id" s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret" - s.validClientID = "oidc#client#id" - s.validClientSecret = "oidc#client#secret" + s.fakeProvider.ValidClientID = "oidc#client#id" + s.fakeProvider.ValidClientSecret = "oidc#client#secret" for _, trial := range []struct { expectEmail string // "" if failure expected setup func() @@ -317,8 +210,8 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { expectEmail: "user@oidc.example.com", setup: func() { c.Log("=== succeed because email_verified is false but not required") - s.authEmail = "user@oidc.example.com" - s.authEmailVerified = false + s.fakeProvider.AuthEmail = "user@oidc.example.com" + s.fakeProvider.AuthEmailVerified = false s.cluster.Login.OpenIDConnect.EmailClaim = "email" s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "" s.cluster.Login.OpenIDConnect.UsernameClaim = "" @@ -328,8 +221,8 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { expectEmail: "", setup: func() { c.Log("=== fail because email_verified is false and required") - s.authEmail = "user@oidc.example.com" - s.authEmailVerified = false + s.fakeProvider.AuthEmail = "user@oidc.example.com" + s.fakeProvider.AuthEmailVerified = false s.cluster.Login.OpenIDConnect.EmailClaim = "email" s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified" s.cluster.Login.OpenIDConnect.UsernameClaim = "" @@ -339,8 +232,8 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { expectEmail: "user@oidc.example.com", setup: func() { c.Log("=== succeed because email_verified is false but config uses custom 'verified' claim") - s.authEmail = "user@oidc.example.com" - s.authEmailVerified = false + s.fakeProvider.AuthEmail = "user@oidc.example.com" + s.fakeProvider.AuthEmailVerified = false s.cluster.Login.OpenIDConnect.EmailClaim = "email" s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified" s.cluster.Login.OpenIDConnect.UsernameClaim = "" @@ -350,8 +243,8 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { expectEmail: "alt_email@example.com", setup: func() { c.Log("=== succeed with custom 'email' and 'email_verified' claims") - s.authEmail = "bad@wrong.example.com" - s.authEmailVerified = false + s.fakeProvider.AuthEmail = "bad@wrong.example.com" + s.fakeProvider.AuthEmailVerified = false s.cluster.Login.OpenIDConnect.EmailClaim = "alt_email" s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified" s.cluster.Login.OpenIDConnect.UsernameClaim = "alt_username" @@ -368,7 +261,7 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { state := s.startLogin(c) resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) c.Assert(err, check.IsNil) @@ -399,7 +292,7 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) { func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) { state := s.startLogin(c) resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) c.Check(err, check.IsNil) @@ -436,8 +329,8 @@ func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) { } func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) { - s.authEmail = "joe.smith@primary.example.com" - s.fakePeopleAPIResponse = map[string]interface{}{ + s.fakeProvider.AuthEmail = "joe.smith@primary.example.com" + s.fakeProvider.PeopleAPIResponse = map[string]interface{}{ "names": []map[string]interface{}{ { "metadata": map[string]interface{}{"primary": false}, @@ -453,7 +346,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) { } state := s.startLogin(c) s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) @@ -463,11 +356,11 @@ func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) { } func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) { - s.authName = "Joe P. Smith" - s.authEmail = "joe.smith@primary.example.com" + s.fakeProvider.AuthName = "Joe P. Smith" + s.fakeProvider.AuthEmail = "joe.smith@primary.example.com" state := s.startLogin(c) s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) @@ -478,8 +371,8 @@ func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) { // People API returns some additional email addresses. func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) { - s.authEmail = "joe.smith@primary.example.com" - s.fakePeopleAPIResponse = map[string]interface{}{ + s.fakeProvider.AuthEmail = "joe.smith@primary.example.com" + s.fakeProvider.PeopleAPIResponse = map[string]interface{}{ "emailAddresses": []map[string]interface{}{ { "metadata": map[string]interface{}{"verified": true}, @@ -496,7 +389,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) { } state := s.startLogin(c) s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) @@ -507,8 +400,8 @@ func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) { // Primary address is not the one initially returned by oidc. func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) { - s.authEmail = "joe.smith@alternate.example.com" - s.fakePeopleAPIResponse = map[string]interface{}{ + s.fakeProvider.AuthEmail = "joe.smith@alternate.example.com" + s.fakeProvider.PeopleAPIResponse = map[string]interface{}{ "emailAddresses": []map[string]interface{}{ { "metadata": map[string]interface{}{"verified": true, "primary": true}, @@ -526,7 +419,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *chec } state := s.startLogin(c) s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) authinfo := getCallbackAuthInfo(c, s.railsSpy) @@ -536,9 +429,9 @@ func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *chec } func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) { - s.authEmail = "joe.smith@unverified.example.com" - s.authEmailVerified = false - s.fakePeopleAPIResponse = map[string]interface{}{ + s.fakeProvider.AuthEmail = "joe.smith@unverified.example.com" + s.fakeProvider.AuthEmailVerified = false + s.fakeProvider.PeopleAPIResponse = map[string]interface{}{ "emailAddresses": []map[string]interface{}{ { "metadata": map[string]interface{}{"verified": true}, @@ -552,7 +445,7 @@ func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) { } state := s.startLogin(c) s.localdb.Login(context.Background(), arvados.LoginOptions{ - Code: s.validCode, + Code: s.fakeProvider.ValidCode, State: state, }) @@ -574,23 +467,6 @@ func (s *OIDCLoginSuite) startLogin(c *check.C) (state string) { return } -func (s *OIDCLoginSuite) fakeToken(c *check.C, payload []byte) string { - signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: s.issuerKey}, nil) - if err != nil { - c.Error(err) - } - object, err := signer.Sign(payload) - if err != nil { - c.Error(err) - } - t, err := object.CompactSerialize() - if err != nil { - c.Error(err) - } - c.Logf("fakeToken(%q) == %q", payload, t) - return t -} - func getCallbackAuthInfo(c *check.C, railsSpy *arvadostest.Proxy) (authinfo rpc.UserSessionAuthInfo) { for _, dump := range railsSpy.RequestDumps { c.Logf("spied request: %q", dump) diff --git a/lib/controller/localdb/login_testuser.go b/lib/controller/localdb/login_testuser.go new file mode 100644 index 0000000000..5852273529 --- /dev/null +++ b/lib/controller/localdb/login_testuser.go @@ -0,0 +1,104 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "bytes" + "context" + "fmt" + "html/template" + + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/sirupsen/logrus" +) + +type testLoginController struct { + Cluster *arvados.Cluster + RailsProxy *railsProxy +} + +func (ctrl *testLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) { + return noopLogout(ctrl.Cluster, opts) +} + +func (ctrl *testLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) { + tmpl, err := template.New("form").Parse(loginform) + if err != nil { + return arvados.LoginResponse{}, err + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, opts) + if err != nil { + return arvados.LoginResponse{}, err + } + return arvados.LoginResponse{HTML: buf}, nil +} + +func (ctrl *testLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) { + for username, user := range ctrl.Cluster.Login.Test.Users { + if (opts.Username == username || opts.Username == user.Email) && opts.Password == user.Password { + ctxlog.FromContext(ctx).WithFields(logrus.Fields{ + "username": username, + "email": user.Email, + }).Debug("test authentication succeeded") + return createAPIClientAuthorization(ctx, ctrl.RailsProxy, ctrl.Cluster.SystemRootToken, rpc.UserSessionAuthInfo{ + Username: username, + Email: user.Email, + }) + } + } + return arvados.APIClientAuthorization{}, fmt.Errorf("authentication failed for user %q with password len=%d", opts.Username, len(opts.Password)) +} + +const loginform = ` + + + Arvados test login + + + +

Arvados test login

+
+ + username + password + +
+

+
+ + + +` diff --git a/lib/controller/localdb/login_testuser_test.go b/lib/controller/localdb/login_testuser_test.go new file mode 100644 index 0000000000..7589088899 --- /dev/null +++ b/lib/controller/localdb/login_testuser_test.go @@ -0,0 +1,103 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/lib/ctrlctx" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/jmoiron/sqlx" + check "gopkg.in/check.v1" +) + +var _ = check.Suite(&TestUserSuite{}) + +type TestUserSuite struct { + cluster *arvados.Cluster + ctrl *testLoginController + railsSpy *arvadostest.Proxy + db *sqlx.DB + + // transaction context + ctx context.Context + rollback func() error +} + +func (s *TestUserSuite) SetUpSuite(c *check.C) { + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() + c.Assert(err, check.IsNil) + s.cluster, err = cfg.GetCluster("") + c.Assert(err, check.IsNil) + s.cluster.Login.Test.Enable = true + s.cluster.Login.Test.Users = map[string]arvados.TestUser{ + "valid": {Email: "valid@example.com", Password: "v@l1d"}, + } + s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI) + s.ctrl = &testLoginController{ + Cluster: s.cluster, + RailsProxy: rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider), + } + s.db = arvadostest.DB(c, s.cluster) +} + +func (s *TestUserSuite) SetUpTest(c *check.C) { + tx, err := s.db.Beginx() + c.Assert(err, check.IsNil) + s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx) + s.rollback = tx.Rollback +} + +func (s *TestUserSuite) TearDownTest(c *check.C) { + if s.rollback != nil { + s.rollback() + } +} + +func (s *TestUserSuite) TestLogin(c *check.C) { + for _, trial := range []struct { + success bool + username string + password string + }{ + {false, "foo", "bar"}, + {false, "", ""}, + {false, "valid", ""}, + {false, "", "v@l1d"}, + {true, "valid", "v@l1d"}, + {true, "valid@example.com", "v@l1d"}, + } { + c.Logf("=== %#v", trial) + resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{ + Username: trial.username, + Password: trial.password, + }) + if trial.success { + c.Check(err, check.IsNil) + c.Check(resp.APIToken, check.Not(check.Equals), "") + c.Check(resp.UUID, check.Matches, `zzzzz-gj3su-.*`) + c.Check(resp.Scopes, check.DeepEquals, []string{"all"}) + + authinfo := getCallbackAuthInfo(c, s.railsSpy) + c.Check(authinfo.Email, check.Equals, "valid@example.com") + c.Check(authinfo.AlternateEmails, check.DeepEquals, []string(nil)) + } else { + c.Check(err, check.ErrorMatches, `authentication failed.*`) + } + } +} + +func (s *TestUserSuite) TestLoginForm(c *check.C) { + resp, err := s.ctrl.Login(s.ctx, arvados.LoginOptions{ + ReturnTo: "https://localhost:12345/example", + }) + c.Check(err, check.IsNil) + c.Check(resp.HTML.String(), check.Matches, `(?ms).*
.*`) +} diff --git a/lib/controller/railsproxy/railsproxy.go b/lib/controller/railsproxy/railsproxy.go index ff9de36b75..515dd5df0f 100644 --- a/lib/controller/railsproxy/railsproxy.go +++ b/lib/controller/railsproxy/railsproxy.go @@ -15,8 +15,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/arvados" ) -// For now, FindRailsAPI always uses the rails API running on this -// node. +// FindRailsAPI always uses the rails API running on this node, for now. func FindRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) { var best *url.URL for target := range cluster.Services.RailsAPI.InternalURLs { diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go index 729d8bdde0..cd98b64718 100644 --- a/lib/controller/rpc/conn.go +++ b/lib/controller/rpc/conn.go @@ -26,11 +26,11 @@ import ( type TokenProvider func(context.Context) ([]string, error) func PassthroughTokenProvider(ctx context.Context) ([]string, error) { - if incoming, ok := auth.FromContext(ctx); !ok { + incoming, ok := auth.FromContext(ctx) + if !ok { return nil, errors.New("no token provided") - } else { - return incoming.Tokens, nil } + return incoming.Tokens, nil } type Conn struct { @@ -170,9 +170,8 @@ func (conn *Conn) relativeToBaseURL(location string) string { u.User = nil u.Host = "" return u.String() - } else { - return location } + return location } func (conn *Conn) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) { diff --git a/lib/controller/rpc/conn_test.go b/lib/controller/rpc/conn_test.go index f43cc1ddee..cf4dbc4767 100644 --- a/lib/controller/rpc/conn_test.go +++ b/lib/controller/rpc/conn_test.go @@ -24,7 +24,11 @@ func Test(t *testing.T) { var _ = check.Suite(&RPCSuite{}) -const contextKeyTestTokens = "testTokens" +type key int + +const ( + contextKeyTestTokens key = iota +) type RPCSuite struct { log logrus.FieldLogger diff --git a/lib/controller/semaphore.go b/lib/controller/semaphore.go index ff607bbb57..e1cda33f93 100644 --- a/lib/controller/semaphore.go +++ b/lib/controller/semaphore.go @@ -8,7 +8,6 @@ func semaphore(max int) (acquire, release func()) { if max > 0 { ch := make(chan bool, max) return func() { ch <- true }, func() { <-ch } - } else { - return func() {}, func() {} } + return func() {}, func() {} } diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go new file mode 100644 index 0000000000..9b0685225b --- /dev/null +++ b/lib/costanalyzer/cmd.go @@ -0,0 +1,43 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package costanalyzer + +import ( + "io" + + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/sirupsen/logrus" +) + +var Command command + +type command struct{} + +type NoPrefixFormatter struct{} + +func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) { + return []byte(entry.Message), nil +} + +// RunCommand implements the subcommand "costanalyzer ..." +func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + var err error + logger := ctxlog.New(stderr, "text", "info") + defer func() { + if err != nil { + logger.Error("\n" + err.Error() + "\n") + } + }() + + logger.SetFormatter(new(NoPrefixFormatter)) + + loader := config.NewLoader(stdin, logger) + loader.SkipLegacy = true + + exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr) + + return exitcode +} diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go new file mode 100644 index 0000000000..e8dd186050 --- /dev/null +++ b/lib/costanalyzer/costanalyzer.go @@ -0,0 +1,568 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package costanalyzer + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/keepclient" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +type nodeInfo struct { + // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3) + Properties struct { + CloudNode struct { + Price float64 + Size string + } `json:"cloud_node"` + } + // Modern + ProviderType string + Price float64 +} + +type arrayFlags []string + +func (i *arrayFlags) String() string { + return "" +} + +func (i *arrayFlags) Set(value string) error { + for _, s := range strings.Split(value, ",") { + *i = append(*i, s) + } + return nil +} + +func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) { + flags := flag.NewFlagSet("", flag.ContinueOnError) + flags.SetOutput(stderr) + flags.Usage = func() { + fmt.Fprintf(flags.Output(), ` +Usage: + %s [options ...] ... + + This program analyzes the cost of Arvados container requests. For each uuid + supplied, it creates a CSV report that lists all the containers used to + fulfill the container request, together with the machine type and cost of + each container. At least one uuid must be specified. + + When supplied with the uuid of a container request, it will calculate the + cost of that container request and all its children. + + When supplied with the uuid of a collection, it will see if there is a + container_request uuid in the properties of the collection, and if so, it + will calculate the cost of that container request and all its children. + + When supplied with a project uuid or when supplied with multiple container + request or collection uuids, it will create a CSV report for each supplied + uuid, as well as a CSV file with aggregate cost accounting for all supplied + uuids. The aggregate cost report takes container reuse into account: if a + container was reused between several container requests, its cost will only + be counted once. + + To get the node costs, the progam queries the Arvados API for current cost + data for each node type used. This means that the reported cost always + reflects the cost data as currently defined in the Arvados API configuration + file. + + Caveats: + - the Arvados API configuration cost data may be out of sync with the cloud + provider. + - when generating reports for older container requests, the cost data in the + Arvados API configuration file may have changed since the container request + was fulfilled. This program uses the cost data stored at the time of the + execution of the container, stored in the 'node.json' file in its log + collection. + + In order to get the data for the uuids supplied, the ARVADOS_API_HOST and + ARVADOS_API_TOKEN environment variables must be set. + + This program prints the total dollar amount from the aggregate cost + accounting across all provided uuids on stdout. + + When the '-output' option is specified, a set of CSV files with cost details + will be written to the provided directory. + +Options: +`, prog) + flags.PrintDefaults() + } + loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)") + flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports") + flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects") + err = flags.Parse(args) + if err == flag.ErrHelp { + err = nil + exitCode = 1 + return + } else if err != nil { + exitCode = 2 + return + } + uuids = flags.Args() + + if len(uuids) < 1 { + flags.Usage() + err = fmt.Errorf("error: no uuid(s) provided") + exitCode = 2 + return + } + + lvl, err := logrus.ParseLevel(*loglevel) + if err != nil { + exitCode = 2 + return + } + logger.SetLevel(lvl) + if !cache { + logger.Debug("Caching disabled\n") + } + return +} + +func ensureDirectory(logger *logrus.Logger, dir string) (err error) { + statData, err := os.Stat(dir) + if os.IsNotExist(err) { + err = os.MkdirAll(dir, 0700) + if err != nil { + return fmt.Errorf("error creating directory %s: %s", dir, err.Error()) + } + } else { + if !statData.IsDir() { + return fmt.Errorf("the path %s is not a directory", dir) + } + } + return +} + +func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) { + csv = cr.UUID + "," + csv += cr.Name + "," + csv += container.UUID + "," + csv += string(container.State) + "," + if container.StartedAt != nil { + csv += container.StartedAt.String() + "," + } else { + csv += "," + } + + var delta time.Duration + if container.FinishedAt != nil { + csv += container.FinishedAt.String() + "," + delta = container.FinishedAt.Sub(*container.StartedAt) + csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + "," + } else { + csv += ",," + } + var price float64 + var size string + if node.Properties.CloudNode.Price != 0 { + price = node.Properties.CloudNode.Price + size = node.Properties.CloudNode.Size + } else { + price = node.Price + size = node.ProviderType + } + cost = delta.Seconds() / 3600 * price + csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n" + return +} + +func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) { + reload = true + if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") { + // We do not cache projects or collections, they have no final state + return + } + // See if we have a cached copy of this object + _, err := os.Stat(file) + if err != nil { + return + } + data, err := ioutil.ReadFile(file) + if err != nil { + logger.Errorf("error reading %q: %s", file, err) + return + } + err = json.Unmarshal(data, &object) + if err != nil { + logger.Errorf("failed to unmarshal json: %s: %s", data, err) + return + } + + // See if it is in a final state, if that makes sense + switch v := object.(type) { + case *arvados.ContainerRequest: + if v.State == arvados.ContainerRequestStateFinal { + reload = false + logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) + } + case *arvados.Container: + if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled { + reload = false + logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file) + } + } + return +} + +// Load an Arvados object. +func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) { + file := uuid + ".json" + + var reload bool + var cacheDir string + + if !cache { + reload = true + } else { + homeDir, err := os.UserHomeDir() + if err != nil { + reload = true + logger.Info("Unable to determine current user home directory, not using cache") + } else { + cacheDir = homeDir + "/.cache/arvados/costanalyzer/" + err = ensureDirectory(logger, cacheDir) + if err != nil { + reload = true + logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error()) + } else { + reload = loadCachedObject(logger, cacheDir+file, uuid, object) + } + } + } + if !reload { + return + } + + if strings.Contains(uuid, "-j7d0g-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-xvhdp-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-dz642-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil) + } else if strings.Contains(uuid, "-4zz18-") { + err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil) + } else { + err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err) + return + } + if err != nil { + err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err) + return + } + encoded, err := json.MarshalIndent(object, "", " ") + if err != nil { + err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err) + return + } + if cacheDir != "" { + err = ioutil.WriteFile(cacheDir+file, encoded, 0644) + if err != nil { + err = fmt.Errorf("error writing file %s:\n %s", file, err) + return + } + } + return +} + +func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) { + if cr.LogUUID == "" { + err = errors.New("no log collection") + return + } + + var collection arvados.Collection + err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil) + if err != nil { + err = fmt.Errorf("error getting collection: %s", err) + return + } + + var fs arvados.CollectionFileSystem + fs, err = collection.FileSystem(ac, kc) + if err != nil { + err = fmt.Errorf("error opening collection as filesystem: %s", err) + return + } + var f http.File + f, err = fs.Open("node.json") + if err != nil { + err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err) + return + } + + err = json.NewDecoder(f).Decode(&node) + if err != nil { + err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err) + return + } + return +} + +func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) { + cost = make(map[string]float64) + + var project arvados.Group + err = loadObject(logger, ac, uuid, uuid, cache, &project) + if err != nil { + return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error()) + } + + var childCrs map[string]interface{} + filterset := []arvados.Filter{ + { + Attr: "owner_uuid", + Operator: "=", + Operand: project.UUID, + }, + { + Attr: "requesting_container_uuid", + Operator: "=", + Operand: nil, + }, + } + err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ + "filters": filterset, + "limit": 10000, + }) + if err != nil { + return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) + } + if value, ok := childCrs["items"]; ok { + logger.Infof("Collecting top level container requests in project %s\n", uuid) + items := value.([]interface{}) + for _, item := range items { + itemMap := item.(map[string]interface{}) + crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache) + if err != nil { + return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error()) + } + for k, v := range crCsv { + cost[k] = v + } + } + } else { + logger.Infof("No top level container requests found in project %s\n", uuid) + } + return +} + +func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) { + + cost = make(map[string]float64) + + csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n" + var tmpCsv string + var tmpTotalCost float64 + var totalCost float64 + + var crUUID = uuid + if strings.Contains(uuid, "-4zz18-") { + // This is a collection, find the associated container request (if any) + var c arvados.Collection + err = loadObject(logger, ac, uuid, uuid, cache, &c) + if err != nil { + return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err) + } + value, ok := c.Properties["container_request"] + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid) + } + crUUID, ok = value.(string) + if !ok { + return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid) + } + } + + // This is a container request, find the container + var cr arvados.ContainerRequest + err = loadObject(logger, ac, crUUID, crUUID, cache, &cr) + if err != nil { + return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err) + } + var container arvados.Container + err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container) + if err != nil { + return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err) + } + + topNode, err := getNode(arv, ac, kc, cr) + if err != nil { + return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err) + } + tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container) + csv += tmpCsv + totalCost += tmpTotalCost + cost[container.UUID] = totalCost + + // Find all container requests that have the container we found above as requesting_container_uuid + var childCrs arvados.ContainerRequestList + filterset := []arvados.Filter{ + { + Attr: "requesting_container_uuid", + Operator: "=", + Operand: container.UUID, + }} + err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{ + "filters": filterset, + "limit": 10000, + }) + if err != nil { + return nil, fmt.Errorf("error querying container_requests: %s", err.Error()) + } + logger.Infof("Collecting child containers for container request %s", crUUID) + for _, cr2 := range childCrs.Items { + logger.Info(".") + node, err := getNode(arv, ac, kc, cr2) + if err != nil { + return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err) + } + logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n") + var c2 arvados.Container + err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2) + if err != nil { + return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err) + } + tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2) + cost[cr2.ContainerUUID] = tmpTotalCost + csv += tmpCsv + totalCost += tmpTotalCost + } + logger.Info(" done\n") + + csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n" + + if resultsDir != "" { + // Write the resulting CSV file + fName := resultsDir + "/" + crUUID + ".csv" + err = ioutil.WriteFile(fName, []byte(csv), 0644) + if err != nil { + return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error()) + } + logger.Infof("\nUUID report in %s\n\n", fName) + } + + return +} + +func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) { + exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr) + if exitcode != 0 { + return + } + if resultsDir != "" { + err = ensureDirectory(logger, resultsDir) + if err != nil { + exitcode = 3 + return + } + } + + // Arvados Client setup + arv, err := arvadosclient.MakeArvadosClient() + if err != nil { + err = fmt.Errorf("error creating Arvados object: %s", err) + exitcode = 1 + return + } + kc, err := keepclient.MakeKeepClient(arv) + if err != nil { + err = fmt.Errorf("error creating Keep object: %s", err) + exitcode = 1 + return + } + + ac := arvados.NewClientFromEnv() + + cost := make(map[string]float64) + for _, uuid := range uuids { + if strings.Contains(uuid, "-j7d0g-") { + // This is a project (group) + cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache) + if err != nil { + exitcode = 1 + return + } + for k, v := range cost { + cost[k] = v + } + } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") { + // This is a container request + var crCsv map[string]float64 + crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache) + if err != nil { + err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error()) + exitcode = 2 + return + } + for k, v := range crCsv { + cost[k] = v + } + } else if strings.Contains(uuid, "-tpzed-") { + // This is a user. The "Home" project for a user is not a real project. + // It is identified by the user uuid. As such, cost analysis for the + // "Home" project is not supported by this program. Skip this uuid, but + // keep going. + logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid) + } else { + logger.Errorf("this argument does not look like a uuid: %s\n", uuid) + exitcode = 3 + return + } + } + + if len(cost) == 0 { + logger.Info("Nothing to do!\n") + return + } + + var csv string + + csv = "# Aggregate cost accounting for uuids:\n" + for _, uuid := range uuids { + csv += "# " + uuid + "\n" + } + + var total float64 + for k, v := range cost { + csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n" + total += v + } + + csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n" + + if resultsDir != "" { + // Write the resulting CSV file + aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv" + err = ioutil.WriteFile(aFile, []byte(csv), 0644) + if err != nil { + err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error()) + exitcode = 1 + return + } + logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile) + } + + // Output the total dollar amount on stdout + fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64)) + + return +} diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go new file mode 100644 index 0000000000..b1ddf97a36 --- /dev/null +++ b/lib/costanalyzer/costanalyzer_test.go @@ -0,0 +1,325 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package costanalyzer + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "regexp" + "testing" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" + "gopkg.in/check.v1" +) + +func Test(t *testing.T) { + check.TestingT(t) +} + +var _ = check.Suite(&Suite{}) + +type Suite struct{} + +func (s *Suite) TearDownSuite(c *check.C) { + // Undo any changes/additions to the database so they don't affect subsequent tests. + arvadostest.ResetEnv() +} + +func (s *Suite) SetUpSuite(c *check.C) { + arvadostest.StartAPI() + arvadostest.StartKeep(2, true) + + // Get the various arvados, arvadosclient, and keep client objects + ac := arvados.NewClientFromEnv() + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, check.Equals, nil) + arv.ApiToken = arvadostest.ActiveToken + kc, err := keepclient.MakeKeepClient(arv) + c.Assert(err, check.Equals, nil) + + standardE4sV3JSON := `{ + "Name": "Standard_E4s_v3", + "ProviderType": "Standard_E4s_v3", + "VCPUs": 4, + "RAM": 34359738368, + "Scratch": 64000000000, + "IncludedScratch": 64000000000, + "AddedScratch": 0, + "Price": 0.292, + "Preemptible": false +}` + standardD32sV3JSON := `{ + "Name": "Standard_D32s_v3", + "ProviderType": "Standard_D32s_v3", + "VCPUs": 32, + "RAM": 137438953472, + "Scratch": 256000000000, + "IncludedScratch": 256000000000, + "AddedScratch": 0, + "Price": 1.76, + "Preemptible": false +}` + + standardA1V2JSON := `{ + "Name": "a1v2", + "ProviderType": "Standard_A1_v2", + "VCPUs": 1, + "RAM": 2147483648, + "Scratch": 10000000000, + "IncludedScratch": 10000000000, + "AddedScratch": 0, + "Price": 0.043, + "Preemptible": false +}` + + standardA2V2JSON := `{ + "Name": "a2v2", + "ProviderType": "Standard_A2_v2", + "VCPUs": 2, + "RAM": 4294967296, + "Scratch": 20000000000, + "IncludedScratch": 20000000000, + "AddedScratch": 0, + "Price": 0.091, + "Preemptible": false +}` + + legacyD1V2JSON := `{ + "properties": { + "cloud_node": { + "price": 0.073001, + "size": "Standard_D1_v2" + }, + "total_cpu_cores": 1, + "total_ram_mb": 3418, + "total_scratch_mb": 51170 + } +}` + + // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON) + + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.DiagnosticsContainerRequest1LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON) + createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON) +} + +func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) { + // Get the CR + var cr arvados.ContainerRequest + err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil) + c.Assert(err, check.Equals, nil) + c.Assert(cr.LogUUID, check.Equals, logUUID) + + // Get the log collection + var coll arvados.Collection + err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil) + c.Assert(err, check.IsNil) + + // Create a node.json file -- the fixture doesn't actually contain the contents of the collection. + fs, err := coll.FileSystem(ac, kc) + c.Assert(err, check.IsNil) + f, err := fs.OpenFile("node.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777) + c.Assert(err, check.IsNil) + _, err = io.WriteString(f, nodeJSON) + c.Assert(err, check.IsNil) + err = f.Close() + c.Assert(err, check.IsNil) + + // Flush the data to Keep + mtxt, err := fs.MarshalManifest(".") + c.Assert(err, check.IsNil) + c.Assert(mtxt, check.NotNil) + + // Update collection record + err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{ + "collection": map[string]interface{}{ + "manifest_text": mtxt, + }, + }) + c.Assert(err, check.IsNil) +} + +func (*Suite) TestUsage(c *check.C) { + var stdout, stderr bytes.Buffer + exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 1) + c.Check(stdout.String(), check.Equals, "") + c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`) +} + +func (*Suite) TestContainerRequestUUID(c *check.C) { + var stdout, stderr bytes.Buffer + resultsDir := c.MkDir() + // Run costanalyzer with 1 container request uuid + exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889") +} + +func (*Suite) TestCollectionUUID(c *check.C) { + var stdout, stderr bytes.Buffer + + resultsDir := c.MkDir() + // Run costanalyzer with 1 collection uuid, without 'container_request' property + exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 2) + c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*") + + // Update the collection, attach a 'container_request' property + ac := arvados.NewClientFromEnv() + var coll arvados.Collection + + // Update collection record + err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{ + "collection": map[string]interface{}{ + "properties": map[string]interface{}{ + "container_request": arvadostest.CompletedContainerRequestUUID, + }, + }, + }) + c.Assert(err, check.IsNil) + + stdout.Truncate(0) + stderr.Truncate(0) + + // Run costanalyzer with 1 collection uuid + resultsDir = c.MkDir() + exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889") +} + +func (*Suite) TestDoubleContainerRequestUUID(c *check.C) { + var stdout, stderr bytes.Buffer + resultsDir := c.MkDir() + // Run costanalyzer with 2 container request uuids + exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + + uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111") + + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000") + stdout.Truncate(0) + stderr.Truncate(0) + + // Now move both container requests into an existing project, and then re-run + // the analysis with the project uuid. The results should be identical. + ac := arvados.NewClientFromEnv() + var cr arvados.ContainerRequest + err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{ + "container_request": map[string]interface{}{ + "owner_uuid": arvadostest.AProjectUUID, + }, + }) + c.Assert(err, check.IsNil) + err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{ + "container_request": map[string]interface{}{ + "owner_uuid": arvadostest.AProjectUUID, + }, + }) + c.Assert(err, check.IsNil) + + // Run costanalyzer with the project uuid + resultsDir = c.MkDir() + exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", resultsDir, arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889") + + uuidReport2, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111") + + re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err = ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000") +} + +func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) { + var stdout, stderr bytes.Buffer + // Run costanalyzer with 2 container request uuids, without output directory specified + exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*") + + // Check that the total amount was printed to stdout + c.Check(stdout.String(), check.Matches, "0.01492030\n") + + stdout.Truncate(0) + stderr.Truncate(0) + + // Run costanalyzer with 2 container request uuids + resultsDir := c.MkDir() + exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr) + c.Check(exitcode, check.Equals, 0) + c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*") + + uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192") + + uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv") + c.Assert(err, check.IsNil) + c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088") + + re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`) + matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv' + + aggregateCostReport, err := ioutil.ReadFile(matches[1]) + c.Assert(err, check.IsNil) + + c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030") +} diff --git a/lib/crunchrun/background.go b/lib/crunchrun/background.go index bf039afa0a..4bb249380f 100644 --- a/lib/crunchrun/background.go +++ b/lib/crunchrun/background.go @@ -132,7 +132,7 @@ func kill(uuid string, signal syscall.Signal, stdout, stderr io.Writer) error { var pi procinfo err = json.NewDecoder(f).Decode(&pi) if err != nil { - return fmt.Errorf("decode %s: %s\n", path, err) + return fmt.Errorf("decode %s: %s", path, err) } if pi.UUID != uuid || pi.PID == 0 { @@ -162,7 +162,7 @@ func kill(uuid string, signal syscall.Signal, stdout, stderr io.Writer) error { return nil } -// List UUIDs of active crunch-run processes. +// ListProcesses lists UUIDs of active crunch-run processes. func ListProcesses(stdout, stderr io.Writer) int { // filepath.Walk does not follow symlinks, so we must walk // lockdir+"/." in case lockdir itself is a symlink. @@ -218,6 +218,24 @@ func ListProcesses(stdout, stderr io.Writer) int { return nil } + proc, err := os.FindProcess(pi.PID) + if err != nil { + // FindProcess should have succeeded, even if the + // process does not exist. + fmt.Fprintf(stderr, "%s: find process %d: %s", path, pi.PID, err) + return nil + } + err = proc.Signal(syscall.Signal(0)) + if err != nil { + // Process is dead, even though lockfile was + // still locked. Most likely a stuck arv-mount + // process that inherited the lock from + // crunch-run. Report container UUID as + // "stale". + fmt.Fprintln(stdout, pi.UUID, "stale") + return nil + } + fmt.Fprintln(stdout, pi.UUID) return nil })) diff --git a/lib/crunchrun/copier.go b/lib/crunchrun/copier.go index b1497277f2..1b0f168b88 100644 --- a/lib/crunchrun/copier.go +++ b/lib/crunchrun/copier.go @@ -195,9 +195,8 @@ func (cp *copier) walkMount(dest, src string, maxSymlinks int, walkMountsBelow b } if walkMountsBelow { return cp.walkMountsBelow(dest, src) - } else { - return nil } + return nil } func (cp *copier) walkMountsBelow(dest, src string) error { diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go index c8f171ca9b..341938354c 100644 --- a/lib/crunchrun/crunchrun.go +++ b/lib/crunchrun/crunchrun.go @@ -455,11 +455,11 @@ func (runner *ContainerRunner) SetupMounts() (err error) { } for bind := range runner.SecretMounts { if _, ok := runner.Container.Mounts[bind]; ok { - return fmt.Errorf("Secret mount %q conflicts with regular mount", bind) + return fmt.Errorf("secret mount %q conflicts with regular mount", bind) } if runner.SecretMounts[bind].Kind != "json" && runner.SecretMounts[bind].Kind != "text" { - return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.", + return fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted", bind, runner.SecretMounts[bind].Kind) } binds = append(binds, bind) @@ -474,7 +474,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { if bind == "stdout" || bind == "stderr" { // Is it a "file" mount kind? if mnt.Kind != "file" { - return fmt.Errorf("Unsupported mount kind '%s' for %s. Only 'file' is supported.", mnt.Kind, bind) + return fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind) } // Does path start with OutputPath? @@ -490,7 +490,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { if bind == "stdin" { // Is it a "collection" mount kind? if mnt.Kind != "collection" && mnt.Kind != "json" { - return fmt.Errorf("Unsupported mount kind '%s' for stdin. Only 'collection' or 'json' are supported.", mnt.Kind) + return fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind) } } @@ -500,7 +500,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" { if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" { - return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind) + return fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind) } } @@ -508,17 +508,17 @@ func (runner *ContainerRunner) SetupMounts() (err error) { case mnt.Kind == "collection" && bind != "stdin": var src string if mnt.UUID != "" && mnt.PortableDataHash != "" { - return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount") + return fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount") } if mnt.UUID != "" { if mnt.Writable { - return fmt.Errorf("Writing to existing collections currently not permitted.") + return fmt.Errorf("writing to existing collections currently not permitted") } pdhOnly = false src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID) } else if mnt.PortableDataHash != "" { if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") { - return fmt.Errorf("Can never write to a collection specified by portable data hash") + return fmt.Errorf("can never write to a collection specified by portable data hash") } idx := strings.Index(mnt.PortableDataHash, "/") if idx > 0 { @@ -539,7 +539,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount) arvMountCmd = append(arvMountCmd, "--mount-tmp") arvMountCmd = append(arvMountCmd, fmt.Sprintf("tmp%d", tmpcount)) - tmpcount += 1 + tmpcount++ } if mnt.Writable { if bind == runner.Container.OutputPath { @@ -559,15 +559,15 @@ func (runner *ContainerRunner) SetupMounts() (err error) { var tmpdir string tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp") if err != nil { - return fmt.Errorf("While creating mount temp dir: %v", err) + return fmt.Errorf("while creating mount temp dir: %v", err) } st, staterr := os.Stat(tmpdir) if staterr != nil { - return fmt.Errorf("While Stat on temp dir: %v", staterr) + return fmt.Errorf("while Stat on temp dir: %v", staterr) } err = os.Chmod(tmpdir, st.Mode()|os.ModeSetgid|0777) if staterr != nil { - return fmt.Errorf("While Chmod temp dir: %v", err) + return fmt.Errorf("while Chmod temp dir: %v", err) } runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind)) if bind == runner.Container.OutputPath { @@ -618,7 +618,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { } if runner.HostOutputDir == "" { - return fmt.Errorf("Output path does not correspond to a writable mount point") + return fmt.Errorf("output path does not correspond to a writable mount point") } if wantAPI := runner.Container.RuntimeConstraints.API; needCertMount && wantAPI != nil && *wantAPI { @@ -640,20 +640,20 @@ func (runner *ContainerRunner) SetupMounts() (err error) { runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token) if err != nil { - return fmt.Errorf("While trying to start arv-mount: %v", err) + return fmt.Errorf("while trying to start arv-mount: %v", err) } for _, p := range collectionPaths { _, err = os.Stat(p) if err != nil { - return fmt.Errorf("While checking that input files exist: %v", err) + return fmt.Errorf("while checking that input files exist: %v", err) } } for _, cp := range copyFiles { st, err := os.Stat(cp.src) if err != nil { - return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err) + return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err) } if st.IsDir() { err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error { @@ -674,7 +674,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { } return os.Chmod(target, walkinfo.Mode()|os.ModeSetgid|0777) } else { - return fmt.Errorf("Source %q is not a regular file or directory", cp.src) + return fmt.Errorf("source %q is not a regular file or directory", cp.src) } }) } else if st.Mode().IsRegular() { @@ -684,7 +684,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) { } } if err != nil { - return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err) + return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err) } } @@ -870,25 +870,24 @@ func (runner *ContainerRunner) LogNodeRecord() error { return err } return w.Close() - } else { - // Dispatched via crunch-dispatch-slurm. Look up - // apiserver's node record corresponding to - // $SLURMD_NODENAME. - hostname := os.Getenv("SLURMD_NODENAME") - if hostname == "" { - hostname, _ = os.Hostname() - } - _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) { - // The "info" field has admin-only info when - // obtained with a privileged token, and - // should not be logged. - node, ok := resp.(map[string]interface{}) - if ok { - delete(node, "info") - } - }) - return err } + // Dispatched via crunch-dispatch-slurm. Look up + // apiserver's node record corresponding to + // $SLURMD_NODENAME. + hostname := os.Getenv("SLURMD_NODENAME") + if hostname == "" { + hostname, _ = os.Hostname() + } + _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) { + // The "info" field has admin-only info when + // obtained with a privileged token, and + // should not be logged. + node, ok := resp.(map[string]interface{}) + if ok { + delete(node, "info") + } + }) + return err } func (runner *ContainerRunner) logAPIResponse(label, path string, params map[string]interface{}, munge func(interface{})) (logged bool, err error) { @@ -945,15 +944,15 @@ func (runner *ContainerRunner) AttachStreams() (err error) { // If stdin mount is provided, attach it to the docker container var stdinRdr arvados.File - var stdinJson []byte + var stdinJSON []byte if stdinMnt, ok := runner.Container.Mounts["stdin"]; ok { if stdinMnt.Kind == "collection" { var stdinColl arvados.Collection - collId := stdinMnt.UUID - if collId == "" { - collId = stdinMnt.PortableDataHash + collID := stdinMnt.UUID + if collID == "" { + collID = stdinMnt.PortableDataHash } - err = runner.ContainerArvClient.Get("collections", collId, nil, &stdinColl) + err = runner.ContainerArvClient.Get("collections", collID, nil, &stdinColl) if err != nil { return fmt.Errorf("While getting stdin collection: %v", err) } @@ -967,14 +966,14 @@ func (runner *ContainerRunner) AttachStreams() (err error) { return fmt.Errorf("While getting stdin collection path %v: %v", stdinMnt.Path, err) } } else if stdinMnt.Kind == "json" { - stdinJson, err = json.Marshal(stdinMnt.Content) + stdinJSON, err = json.Marshal(stdinMnt.Content) if err != nil { return fmt.Errorf("While encoding stdin json data: %v", err) } } } - stdinUsed := stdinRdr != nil || len(stdinJson) != 0 + stdinUsed := stdinRdr != nil || len(stdinJSON) != 0 response, err := runner.Docker.ContainerAttach(context.TODO(), runner.ContainerID, dockertypes.ContainerAttachOptions{Stream: true, Stdin: stdinUsed, Stdout: true, Stderr: true}) if err != nil { @@ -1017,9 +1016,9 @@ func (runner *ContainerRunner) AttachStreams() (err error) { stdinRdr.Close() response.CloseWrite() }() - } else if len(stdinJson) != 0 { + } else if len(stdinJSON) != 0 { go func() { - _, err := io.Copy(response.Conn, bytes.NewReader(stdinJson)) + _, err := io.Copy(response.Conn, bytes.NewReader(stdinJSON)) if err != nil { runner.CrunchLog.Printf("While writing stdin json to docker container: %v", err) runner.stop(nil) @@ -1489,7 +1488,7 @@ func (runner *ContainerRunner) ContainerToken() (string, error) { return runner.token, nil } -// UpdateContainerComplete updates the container record state on API +// UpdateContainerFinal updates the container record state on API // server to "Complete" or "Cancelled" func (runner *ContainerRunner) UpdateContainerFinal() error { update := arvadosclient.Dict{} @@ -1815,18 +1814,18 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s } } - containerId := flags.Arg(0) + containerID := flags.Arg(0) switch { case *detach && !ignoreDetachFlag: - return Detach(containerId, prog, args, os.Stdout, os.Stderr) + return Detach(containerID, prog, args, os.Stdout, os.Stderr) case *kill >= 0: - return KillProcess(containerId, syscall.Signal(*kill), os.Stdout, os.Stderr) + return KillProcess(containerID, syscall.Signal(*kill), os.Stdout, os.Stderr) case *list: return ListProcesses(os.Stdout, os.Stderr) } - if containerId == "" { + if containerID == "" { log.Printf("usage: %s [options] UUID", prog) return 1 } @@ -1840,14 +1839,14 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s api, err := arvadosclient.MakeArvadosClient() if err != nil { - log.Printf("%s: %v", containerId, err) + log.Printf("%s: %v", containerID, err) return 1 } api.Retries = 8 kc, kcerr := keepclient.MakeKeepClient(api) if kcerr != nil { - log.Printf("%s: %v", containerId, kcerr) + log.Printf("%s: %v", containerID, kcerr) return 1 } kc.BlockCache = &keepclient.BlockCache{MaxBlocks: 2} @@ -1857,21 +1856,21 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s // minimum version we want to support. docker, dockererr := dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil) - cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, docker, containerId) + cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, docker, containerID) if err != nil { log.Print(err) return 1 } if dockererr != nil { - cr.CrunchLog.Printf("%s: %v", containerId, dockererr) + cr.CrunchLog.Printf("%s: %v", containerID, dockererr) cr.checkBrokenNode(dockererr) cr.CrunchLog.Close() return 1 } - parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerId+".") + parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".") if tmperr != nil { - log.Printf("%s: %v", containerId, tmperr) + log.Printf("%s: %v", containerID, tmperr) return 1 } @@ -1905,7 +1904,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s } if runerr != nil { - log.Printf("%s: %v", containerId, runerr) + log.Printf("%s: %v", containerID, runerr) return 1 } return 0 diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go index e8c7660d1a..eb83bbd410 100644 --- a/lib/crunchrun/crunchrun_test.go +++ b/lib/crunchrun/crunchrun_test.go @@ -74,7 +74,7 @@ type KeepTestClient struct { var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n" var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120" -var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7" +var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7" var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n" var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54" @@ -157,9 +157,8 @@ func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, if container == "abcde" { // t.fn gets executed in ContainerWait return nil - } else { - return errors.New("Invalid container id") } + return errors.New("Invalid container id") } func (t *TestDockerClient) ContainerRemove(ctx context.Context, container string, options dockertypes.ContainerRemoveOptions) error { @@ -196,9 +195,8 @@ func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string if t.imageLoaded == image { return dockertypes.ImageInspect{}, nil, nil - } else { - return dockertypes.ImageInspect{}, nil, errors.New("") } + return dockertypes.ImageInspect{}, nil, errors.New("") } func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) { @@ -208,10 +206,9 @@ func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet _, err := io.Copy(ioutil.Discard, input) if err != nil { return dockertypes.ImageLoadResponse{}, err - } else { - t.imageLoaded = hwImageId - return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil } + t.imageLoaded = hwImageID + return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil } func (*TestDockerClient) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) { @@ -260,9 +257,8 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par case method == "GET" && resourceType == "containers" && action == "secret_mounts": if client.secretMounts != nil { return json.Unmarshal(client.secretMounts, output) - } else { - return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output) } + return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output) default: return fmt.Errorf("Not found") } @@ -429,7 +425,7 @@ func (fw FileWrapper) Sync() error { } func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) { - if filename == hwImageId+".tar" { + if filename == hwImageID+".tar" { rdr := ioutil.NopCloser(&bytes.Buffer{}) client.Called = true return FileWrapper{rdr, 1321984}, nil @@ -451,10 +447,10 @@ func (s *TestSuite) TestLoadImage(c *C) { cr.ContainerArvClient = &ArvTestClient{} cr.ContainerKeepClient = kc - _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + _, err = cr.Docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{}) c.Check(err, IsNil) - _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId) + _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageID) c.Check(err, NotNil) cr.Container.ContainerImage = hwPDH @@ -467,13 +463,13 @@ func (s *TestSuite) TestLoadImage(c *C) { c.Check(err, IsNil) defer func() { - cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + cr.Docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{}) }() c.Check(kc.Called, Equals, true) - c.Check(cr.ContainerConfig.Image, Equals, hwImageId) + c.Check(cr.ContainerConfig.Image, Equals, hwImageID) - _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId) + _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageID) c.Check(err, IsNil) // (2) Test using image that's already loaded @@ -483,7 +479,7 @@ func (s *TestSuite) TestLoadImage(c *C) { err = cr.LoadImage() c.Check(err, IsNil) c.Check(kc.Called, Equals, false) - c.Check(cr.ContainerConfig.Image, Equals, hwImageId) + c.Check(cr.ContainerConfig.Image, Equals, hwImageID) } @@ -775,7 +771,7 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi s.docker.exitCode = exitCode s.docker.fn = fn - s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{}) api = &ArvTestClient{Container: rec} s.docker.api = api @@ -1135,7 +1131,7 @@ func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) { t.logWriter.Write(dockerLog(1, "foo\n")) t.logWriter.Close() } - s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{}) api := &ArvTestClient{Container: rec} kc := &KeepTestClient{} @@ -1510,7 +1506,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, NotNil) - c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`) + c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1527,7 +1523,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, NotNil) - c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`) + c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`) os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1622,7 +1618,7 @@ func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func(t *TestDoc c.Check(err, IsNil) s.docker.fn = fn - s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{}) + s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{}) api = &ArvTestClient{Container: rec} kc := &KeepTestClient{} @@ -1658,7 +1654,7 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) { }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) - c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true) + c.Check(strings.Contains(err.Error(), "unsupported mount kind 'tmp' for stdout"), Equals, true) } func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) { @@ -1669,7 +1665,7 @@ func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) { }`, func(t *TestDockerClient) {}) c.Check(err, NotNil) - c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true) + c.Check(strings.Contains(err.Error(), "unsupported mount kind 'collection' for stdout"), Equals, true) } func (s *TestSuite) TestFullRunWithAPI(c *C) { diff --git a/lib/crunchrun/logging.go b/lib/crunchrun/logging.go index d5de184e5c..050894383d 100644 --- a/lib/crunchrun/logging.go +++ b/lib/crunchrun/logging.go @@ -335,7 +335,7 @@ func (arvlog *ArvLogWriter) rateLimit(line []byte, now time.Time) (bool, []byte) arvlog.bytesLogged += lineSize arvlog.logThrottleBytesSoFar += lineSize - arvlog.logThrottleLinesSoFar += 1 + arvlog.logThrottleLinesSoFar++ if arvlog.bytesLogged > crunchLimitLogBytesPerJob { message = fmt.Sprintf("%s Exceeded log limit %d bytes (crunch_limit_log_bytes_per_job). Log will be truncated.", @@ -368,9 +368,8 @@ func (arvlog *ArvLogWriter) rateLimit(line []byte, now time.Time) (bool, []byte) // instead of the log message that exceeded the limit. message += " A complete log is still being written to Keep, and will be available when the job finishes." return true, []byte(message) - } else { - return arvlog.logThrottleIsOpen, line } + return arvlog.logThrottleIsOpen, line } // load the rate limit discovery config parameters diff --git a/lib/crunchrun/logging_test.go b/lib/crunchrun/logging_test.go index fab333b433..e3fa3af0bb 100644 --- a/lib/crunchrun/logging_test.go +++ b/lib/crunchrun/logging_test.go @@ -23,9 +23,9 @@ type TestTimestamper struct { count int } -func (this *TestTimestamper) Timestamp(t time.Time) string { - this.count += 1 - t, err := time.ParseInLocation(time.RFC3339Nano, fmt.Sprintf("2015-12-29T15:51:45.%09dZ", this.count), t.Location()) +func (stamper *TestTimestamper) Timestamp(t time.Time) string { + stamper.count++ + t, err := time.ParseInLocation(time.RFC3339Nano, fmt.Sprintf("2015-12-29T15:51:45.%09dZ", stamper.count), t.Location()) if err != nil { panic(err) } diff --git a/lib/ctrlctx/db.go b/lib/ctrlctx/db.go index 127be489df..36d79d3d2e 100644 --- a/lib/ctrlctx/db.go +++ b/lib/ctrlctx/db.go @@ -12,6 +12,7 @@ import ( "git.arvados.org/arvados.git/lib/controller/api" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/jmoiron/sqlx" + // sqlx needs lib/pq to talk to PostgreSQL _ "github.com/lib/pq" ) diff --git a/lib/dispatchcloud/container/queue.go b/lib/dispatchcloud/container/queue.go index 45b346383f..7a2727c1e9 100644 --- a/lib/dispatchcloud/container/queue.go +++ b/lib/dispatchcloud/container/queue.go @@ -145,11 +145,11 @@ func (cq *Queue) Forget(uuid string) { func (cq *Queue) Get(uuid string) (arvados.Container, bool) { cq.mtx.Lock() defer cq.mtx.Unlock() - if ctr, ok := cq.current[uuid]; !ok { + ctr, ok := cq.current[uuid] + if !ok { return arvados.Container{}, false - } else { - return ctr.Container, true } + return ctr.Container, true } // Entries returns all cache entries, keyed by container UUID. @@ -382,7 +382,7 @@ func (cq *Queue) poll() (map[string]*arvados.Container, error) { *next[upd.UUID] = upd } } - selectParam := []string{"uuid", "state", "priority", "runtime_constraints", "container_image", "mounts", "scheduling_parameters"} + selectParam := []string{"uuid", "state", "priority", "runtime_constraints", "container_image", "mounts", "scheduling_parameters", "created_at"} limitParam := 1000 mine, err := cq.fetchAll(arvados.ResourceListParams{ diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go index 02b6c976ae..7614a143ab 100644 --- a/lib/dispatchcloud/dispatcher.go +++ b/lib/dispatchcloud/dispatcher.go @@ -17,7 +17,7 @@ import ( "git.arvados.org/arvados.git/lib/cloud" "git.arvados.org/arvados.git/lib/dispatchcloud/container" "git.arvados.org/arvados.git/lib/dispatchcloud/scheduler" - "git.arvados.org/arvados.git/lib/dispatchcloud/ssh_executor" + "git.arvados.org/arvados.git/lib/dispatchcloud/sshexecutor" "git.arvados.org/arvados.git/lib/dispatchcloud/worker" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/auth" @@ -100,7 +100,7 @@ func (disp *dispatcher) Close() { // Make a worker.Executor for the given instance. func (disp *dispatcher) newExecutor(inst cloud.Instance) worker.Executor { - exr := ssh_executor.New(inst) + exr := sshexecutor.New(inst) exr.SetTargetPort(disp.Cluster.Containers.CloudVMs.SSHPort) exr.SetSigners(disp.sshKey) return exr @@ -181,7 +181,7 @@ func (disp *dispatcher) run() { if pollInterval <= 0 { pollInterval = defaultPollInterval } - sched := scheduler.New(disp.Context, disp.queue, disp.pool, staleLockTimeout, pollInterval) + sched := scheduler.New(disp.Context, disp.queue, disp.pool, disp.Registry, staleLockTimeout, pollInterval) sched.Start() defer sched.Stop() diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go index aa5f22a501..d5d90bf351 100644 --- a/lib/dispatchcloud/dispatcher_test.go +++ b/lib/dispatchcloud/dispatcher_test.go @@ -66,6 +66,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) { ProbeInterval: arvados.Duration(5 * time.Millisecond), MaxProbesPerSecond: 1000, TimeoutSignal: arvados.Duration(3 * time.Millisecond), + TimeoutStaleRunLock: arvados.Duration(3 * time.Millisecond), TimeoutTERM: arvados.Duration(20 * time.Millisecond), ResourceTags: map[string]string{"testtag": "test value"}, TagKeyPrefix: "test:", @@ -115,6 +116,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) { ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) { return ChooseInstanceType(s.cluster, ctr) }, + Logger: ctxlog.TestLogger(c), } for i := 0; i < 200; i++ { queue.Containers = append(queue.Containers, arvados.Container{ @@ -168,8 +170,10 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) { stubvm.ReportBroken = time.Now().Add(time.Duration(rand.Int63n(200)) * time.Millisecond) default: stubvm.CrunchRunCrashRate = 0.1 + stubvm.ArvMountDeadlockRate = 0.1 } } + s.stubDriver.Bugf = c.Errorf start := time.Now() go s.disp.run() @@ -213,6 +217,20 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) { c.Check(resp.Body.String(), check.Matches, `(?ms).*boot_outcomes{outcome="success"} [^0].*`) c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="shutdown"} [^0].*`) c.Check(resp.Body.String(), check.Matches, `(?ms).*instances_disappeared{state="unknown"} 0\n.*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds{quantile="0.95"} [0-9.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_count [0-9]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ssh_seconds_sum [0-9.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds{quantile="0.95"} [0-9.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_count [0-9]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_to_ready_for_container_seconds_sum [0-9.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_count [0-9]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_shutdown_request_to_disappearance_seconds_sum [0-9.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_count [0-9]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*time_from_queue_to_crunch_run_seconds_sum [0-9e+.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="success"} [0-9]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="success"} [0-9e+.]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_count{outcome="fail"} [0-9]*`) + c.Check(resp.Body.String(), check.Matches, `(?ms).*run_probe_duration_seconds_sum{outcome="fail"} [0-9e+.]*`) } func (s *DispatcherSuite) TestAPIPermissions(c *check.C) { @@ -303,7 +321,7 @@ func (s *DispatcherSuite) TestInstancesAPI(c *check.C) { time.Sleep(time.Millisecond) } c.Assert(len(sr.Items), check.Equals, 1) - c.Check(sr.Items[0].Instance, check.Matches, "stub.*") + c.Check(sr.Items[0].Instance, check.Matches, "inst.*") c.Check(sr.Items[0].WorkerState, check.Equals, "booting") c.Check(sr.Items[0].Price, check.Equals, 0.123) c.Check(sr.Items[0].LastContainerUUID, check.Equals, "") diff --git a/lib/dispatchcloud/driver.go b/lib/dispatchcloud/driver.go index f2a6c92630..fe498d0484 100644 --- a/lib/dispatchcloud/driver.go +++ b/lib/dispatchcloud/driver.go @@ -17,7 +17,7 @@ import ( "golang.org/x/crypto/ssh" ) -// Map of available cloud drivers. +// Drivers is a map of available cloud drivers. // Clusters.*.Containers.CloudVMs.Driver configuration values // correspond to keys in this map. var Drivers = map[string]cloud.Driver{ @@ -180,7 +180,6 @@ func (inst instrumentedInstance) SetTags(tags cloud.InstanceTags) error { func boolLabelValue(v bool) string { if v { return "1" - } else { - return "0" } + return "0" } diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go index 4447f084a9..b9d653a821 100644 --- a/lib/dispatchcloud/scheduler/run_queue.go +++ b/lib/dispatchcloud/scheduler/run_queue.go @@ -33,6 +33,7 @@ func (sch *Scheduler) runQueue() { dontstart := map[arvados.InstanceType]bool{} var overquota []container.QueueEnt // entries that are unmappable because of worker pool quota + var containerAllocatedWorkerBootingCount int tryrun: for i, ctr := range sorted { @@ -51,36 +52,35 @@ tryrun: overquota = sorted[i:] break tryrun } + if sch.pool.KillContainer(ctr.UUID, "about to lock") { + logger.Info("not locking: crunch-run process from previous attempt has not exited") + continue + } go sch.lockContainer(logger, ctr.UUID) unalloc[it]-- case arvados.ContainerStateLocked: if unalloc[it] > 0 { unalloc[it]-- } else if sch.pool.AtQuota() { - logger.Debug("not starting: AtQuota and no unalloc workers") + // Don't let lower-priority containers + // starve this one by using keeping + // idle workers alive on different + // instance types. + logger.Debug("unlocking: AtQuota and no unalloc workers") + sch.queue.Unlock(ctr.UUID) overquota = sorted[i:] break tryrun + } else if logger.Info("creating new instance"); sch.pool.Create(it) { + // Success. (Note pool.Create works + // asynchronously and does its own + // logging, so we don't need to.) } else { - logger.Info("creating new instance") - if !sch.pool.Create(it) { - // (Note pool.Create works - // asynchronously and logs its - // own failures, so we don't - // need to log this as a - // failure.) - - sch.queue.Unlock(ctr.UUID) - // Don't let lower-priority - // containers starve this one - // by using keeping idle - // workers alive on different - // instance types. TODO: - // avoid getting starved here - // if instances of a specific - // type always fail. - overquota = sorted[i:] - break tryrun - } + // Failed despite not being at quota, + // e.g., cloud ops throttled. TODO: + // avoid getting starved here if + // instances of a specific type always + // fail. + continue } if dontstart[it] { @@ -88,14 +88,20 @@ tryrun: // a higher-priority container on the // same instance type. Don't let this // one sneak in ahead of it. + } else if sch.pool.KillContainer(ctr.UUID, "about to start") { + logger.Info("not restarting yet: crunch-run process from previous attempt has not exited") } else if sch.pool.StartContainer(it, ctr) { // Success. } else { + containerAllocatedWorkerBootingCount += 1 dontstart[it] = true } } } + sch.mContainersAllocatedNotStarted.Set(float64(containerAllocatedWorkerBootingCount)) + sch.mContainersNotAllocatedOverQuota.Set(float64(len(overquota))) + if len(overquota) > 0 { // Unlock any containers that are unmappable while // we're at quota. diff --git a/lib/dispatchcloud/scheduler/run_queue_test.go b/lib/dispatchcloud/scheduler/run_queue_test.go index 32c6b3b24d..fd1d0a870b 100644 --- a/lib/dispatchcloud/scheduler/run_queue_test.go +++ b/lib/dispatchcloud/scheduler/run_queue_test.go @@ -13,6 +13,9 @@ import ( "git.arvados.org/arvados.git/lib/dispatchcloud/worker" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" + + "github.com/prometheus/client_golang/prometheus/testutil" + check "gopkg.in/check.v1" ) @@ -38,7 +41,7 @@ type stubPool struct { idle map[arvados.InstanceType]int unknown map[arvados.InstanceType]int running map[string]time.Time - atQuota bool + quota int canCreate int creates []arvados.InstanceType starts []string @@ -46,7 +49,11 @@ type stubPool struct { sync.Mutex } -func (p *stubPool) AtQuota() bool { return p.atQuota } +func (p *stubPool) AtQuota() bool { + p.Lock() + defer p.Unlock() + return len(p.unalloc)+len(p.running)+len(p.unknown) >= p.quota +} func (p *stubPool) Subscribe() <-chan struct{} { return p.notify } func (p *stubPool) Unsubscribe(<-chan struct{}) {} func (p *stubPool) Running() map[string]time.Time { @@ -83,8 +90,9 @@ func (p *stubPool) ForgetContainer(uuid string) { func (p *stubPool) KillContainer(uuid, reason string) bool { p.Lock() defer p.Unlock() - delete(p.running, uuid) - return true + defer delete(p.running, uuid) + t, ok := p.running[uuid] + return ok && t.IsZero() } func (p *stubPool) Shutdown(arvados.InstanceType) bool { p.shutdowns++ @@ -121,11 +129,8 @@ var _ = check.Suite(&SchedulerSuite{}) type SchedulerSuite struct{} -// Assign priority=4 container to idle node. Create a new instance for -// the priority=3 container. Don't try to start any priority<3 -// containers because priority=3 container didn't start -// immediately. Don't try to create any other nodes after the failed -// create. +// Assign priority=4 container to idle node. Create new instances for +// the priority=3, 2, 1 containers. func (*SchedulerSuite) TestUseIdleWorkers(c *check.C) { ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c)) queue := test.Queue{ @@ -171,6 +176,7 @@ func (*SchedulerSuite) TestUseIdleWorkers(c *check.C) { } queue.Update() pool := stubPool{ + quota: 1000, unalloc: map[arvados.InstanceType]int{ test.InstanceType(1): 1, test.InstanceType(2): 2, @@ -182,8 +188,8 @@ func (*SchedulerSuite) TestUseIdleWorkers(c *check.C) { running: map[string]time.Time{}, canCreate: 0, } - New(ctx, &queue, &pool, time.Millisecond, time.Millisecond).runQueue() - c.Check(pool.creates, check.DeepEquals, []arvados.InstanceType{test.InstanceType(1)}) + New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond).runQueue() + c.Check(pool.creates, check.DeepEquals, []arvados.InstanceType{test.InstanceType(1), test.InstanceType(1), test.InstanceType(1)}) c.Check(pool.starts, check.DeepEquals, []string{test.ContainerUUID(4)}) c.Check(pool.running, check.HasLen, 1) for uuid := range pool.running { @@ -191,14 +197,14 @@ func (*SchedulerSuite) TestUseIdleWorkers(c *check.C) { } } -// If Create() fails, shutdown some nodes, and don't call Create() -// again. Don't call Create() at all if AtQuota() is true. +// If pool.AtQuota() is true, shutdown some unalloc nodes, and don't +// call Create(). func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) { ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c)) - for quota := 0; quota < 2; quota++ { + for quota := 1; quota < 3; quota++ { c.Logf("quota=%d", quota) shouldCreate := []arvados.InstanceType{} - for i := 0; i < quota; i++ { + for i := 1; i < quota; i++ { shouldCreate = append(shouldCreate, test.InstanceType(3)) } queue := test.Queue{ @@ -226,7 +232,7 @@ func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) { } queue.Update() pool := stubPool{ - atQuota: quota == 0, + quota: quota, unalloc: map[arvados.InstanceType]int{ test.InstanceType(2): 2, }, @@ -238,10 +244,15 @@ func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) { starts: []string{}, canCreate: 0, } - New(ctx, &queue, &pool, time.Millisecond, time.Millisecond).runQueue() + New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond).runQueue() c.Check(pool.creates, check.DeepEquals, shouldCreate) - c.Check(pool.starts, check.DeepEquals, []string{}) - c.Check(pool.shutdowns, check.Not(check.Equals), 0) + if len(shouldCreate) == 0 { + c.Check(pool.starts, check.DeepEquals, []string{}) + c.Check(pool.shutdowns, check.Not(check.Equals), 0) + } else { + c.Check(pool.starts, check.DeepEquals, []string{test.ContainerUUID(2)}) + c.Check(pool.shutdowns, check.Equals, 0) + } } } @@ -250,6 +261,7 @@ func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) { func (*SchedulerSuite) TestStartWhileCreating(c *check.C) { ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c)) pool := stubPool{ + quota: 1000, unalloc: map[arvados.InstanceType]int{ test.InstanceType(1): 2, test.InstanceType(2): 2, @@ -327,7 +339,7 @@ func (*SchedulerSuite) TestStartWhileCreating(c *check.C) { }, } queue.Update() - New(ctx, &queue, &pool, time.Millisecond, time.Millisecond).runQueue() + New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond).runQueue() c.Check(pool.creates, check.DeepEquals, []arvados.InstanceType{test.InstanceType(2), test.InstanceType(1)}) c.Check(pool.starts, check.DeepEquals, []string{uuids[6], uuids[5], uuids[3], uuids[2]}) running := map[string]bool{} @@ -344,6 +356,7 @@ func (*SchedulerSuite) TestStartWhileCreating(c *check.C) { func (*SchedulerSuite) TestKillNonexistentContainer(c *check.C) { ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c)) pool := stubPool{ + quota: 1000, unalloc: map[arvados.InstanceType]int{ test.InstanceType(2): 0, }, @@ -351,7 +364,7 @@ func (*SchedulerSuite) TestKillNonexistentContainer(c *check.C) { test.InstanceType(2): 0, }, running: map[string]time.Time{ - test.ContainerUUID(2): time.Time{}, + test.ContainerUUID(2): {}, }, } queue := test.Queue{ @@ -370,10 +383,87 @@ func (*SchedulerSuite) TestKillNonexistentContainer(c *check.C) { }, } queue.Update() - sch := New(ctx, &queue, &pool, time.Millisecond, time.Millisecond) + sch := New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond) c.Check(pool.running, check.HasLen, 1) sch.sync() for deadline := time.Now().Add(time.Second); len(pool.Running()) > 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) { } c.Check(pool.Running(), check.HasLen, 0) } + +func (*SchedulerSuite) TestContainersMetrics(c *check.C) { + ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c)) + queue := test.Queue{ + ChooseType: chooseType, + Containers: []arvados.Container{ + { + UUID: test.ContainerUUID(1), + Priority: 1, + State: arvados.ContainerStateLocked, + CreatedAt: time.Now().Add(-10 * time.Second), + RuntimeConstraints: arvados.RuntimeConstraints{ + VCPUs: 1, + RAM: 1 << 30, + }, + }, + }, + } + queue.Update() + + // Create a pool with one unallocated (idle/booting/unknown) worker, + // and `idle` and `unknown` not set (empty). Iow this worker is in the booting + // state, and the container will be allocated but not started yet. + pool := stubPool{ + unalloc: map[arvados.InstanceType]int{test.InstanceType(1): 1}, + } + sch := New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond) + sch.runQueue() + sch.updateMetrics() + + c.Check(int(testutil.ToFloat64(sch.mContainersAllocatedNotStarted)), check.Equals, 1) + c.Check(int(testutil.ToFloat64(sch.mContainersNotAllocatedOverQuota)), check.Equals, 0) + c.Check(int(testutil.ToFloat64(sch.mLongestWaitTimeSinceQueue)), check.Equals, 10) + + // Create a pool without workers. The queued container will not be started, and the + // 'over quota' metric will be 1 because no workers are available and canCreate defaults + // to zero. + pool = stubPool{} + sch = New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond) + sch.runQueue() + sch.updateMetrics() + + c.Check(int(testutil.ToFloat64(sch.mContainersAllocatedNotStarted)), check.Equals, 0) + c.Check(int(testutil.ToFloat64(sch.mContainersNotAllocatedOverQuota)), check.Equals, 1) + c.Check(int(testutil.ToFloat64(sch.mLongestWaitTimeSinceQueue)), check.Equals, 10) + + // Reset the queue, and create a pool with an idle worker. The queued + // container will be started immediately and mLongestWaitTimeSinceQueue + // should be zero. + queue = test.Queue{ + ChooseType: chooseType, + Containers: []arvados.Container{ + { + UUID: test.ContainerUUID(1), + Priority: 1, + State: arvados.ContainerStateLocked, + CreatedAt: time.Now().Add(-10 * time.Second), + RuntimeConstraints: arvados.RuntimeConstraints{ + VCPUs: 1, + RAM: 1 << 30, + }, + }, + }, + } + queue.Update() + + pool = stubPool{ + idle: map[arvados.InstanceType]int{test.InstanceType(1): 1}, + unalloc: map[arvados.InstanceType]int{test.InstanceType(1): 1}, + running: map[string]time.Time{}, + } + sch = New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond) + sch.runQueue() + sch.updateMetrics() + + c.Check(int(testutil.ToFloat64(sch.mLongestWaitTimeSinceQueue)), check.Equals, 0) +} diff --git a/lib/dispatchcloud/scheduler/scheduler.go b/lib/dispatchcloud/scheduler/scheduler.go index 6409ea031a..c3e67dd11f 100644 --- a/lib/dispatchcloud/scheduler/scheduler.go +++ b/lib/dispatchcloud/scheduler/scheduler.go @@ -11,7 +11,9 @@ import ( "sync" "time" + "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -31,6 +33,7 @@ type Scheduler struct { logger logrus.FieldLogger queue ContainerQueue pool WorkerPool + reg *prometheus.Registry staleLockTimeout time.Duration queueUpdateInterval time.Duration @@ -41,17 +44,22 @@ type Scheduler struct { runOnce sync.Once stop chan struct{} stopped chan struct{} + + mContainersAllocatedNotStarted prometheus.Gauge + mContainersNotAllocatedOverQuota prometheus.Gauge + mLongestWaitTimeSinceQueue prometheus.Gauge } // New returns a new unstarted Scheduler. // // Any given queue and pool should not be used by more than one // scheduler at a time. -func New(ctx context.Context, queue ContainerQueue, pool WorkerPool, staleLockTimeout, queueUpdateInterval time.Duration) *Scheduler { - return &Scheduler{ +func New(ctx context.Context, queue ContainerQueue, pool WorkerPool, reg *prometheus.Registry, staleLockTimeout, queueUpdateInterval time.Duration) *Scheduler { + sch := &Scheduler{ logger: ctxlog.FromContext(ctx), queue: queue, pool: pool, + reg: reg, staleLockTimeout: staleLockTimeout, queueUpdateInterval: queueUpdateInterval, wakeup: time.NewTimer(time.Second), @@ -59,6 +67,59 @@ func New(ctx context.Context, queue ContainerQueue, pool WorkerPool, staleLockTi stopped: make(chan struct{}), uuidOp: map[string]string{}, } + sch.registerMetrics(reg) + return sch +} + +func (sch *Scheduler) registerMetrics(reg *prometheus.Registry) { + if reg == nil { + reg = prometheus.NewRegistry() + } + sch.mContainersAllocatedNotStarted = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "containers_allocated_not_started", + Help: "Number of containers allocated to a worker but not started yet (worker is booting).", + }) + reg.MustRegister(sch.mContainersAllocatedNotStarted) + sch.mContainersNotAllocatedOverQuota = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "containers_not_allocated_over_quota", + Help: "Number of containers not allocated to a worker because the system has hit a quota.", + }) + reg.MustRegister(sch.mContainersNotAllocatedOverQuota) + sch.mLongestWaitTimeSinceQueue = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "containers_longest_wait_time_seconds", + Help: "Current longest wait time of any container since queuing, and before the start of crunch-run.", + }) + reg.MustRegister(sch.mLongestWaitTimeSinceQueue) +} + +func (sch *Scheduler) updateMetrics() { + earliest := time.Time{} + entries, _ := sch.queue.Entries() + running := sch.pool.Running() + for _, ent := range entries { + if ent.Container.Priority > 0 && + (ent.Container.State == arvados.ContainerStateQueued || ent.Container.State == arvados.ContainerStateLocked) { + // Exclude containers that are preparing to run the payload (i.e. + // ContainerStateLocked and running on a worker, most likely loading the + // payload image + if _, ok := running[ent.Container.UUID]; !ok { + if ent.Container.CreatedAt.Before(earliest) || earliest.IsZero() { + earliest = ent.Container.CreatedAt + } + } + } + } + if !earliest.IsZero() { + sch.mLongestWaitTimeSinceQueue.Set(time.Since(earliest).Seconds()) + } else { + sch.mLongestWaitTimeSinceQueue.Set(0) + } } // Start starts the scheduler. @@ -113,6 +174,7 @@ func (sch *Scheduler) run() { for { sch.runQueue() sch.sync() + sch.updateMetrics() select { case <-sch.stop: return diff --git a/lib/dispatchcloud/scheduler/sync.go b/lib/dispatchcloud/scheduler/sync.go index 116ca76431..fc683505f9 100644 --- a/lib/dispatchcloud/scheduler/sync.go +++ b/lib/dispatchcloud/scheduler/sync.go @@ -109,13 +109,17 @@ func (sch *Scheduler) cancel(uuid string, reason string) { } func (sch *Scheduler) kill(uuid string, reason string) { + if !sch.uuidLock(uuid, "kill") { + return + } + defer sch.uuidUnlock(uuid) sch.pool.KillContainer(uuid, reason) sch.pool.ForgetContainer(uuid) } func (sch *Scheduler) requeue(ent container.QueueEnt, reason string) { uuid := ent.Container.UUID - if !sch.uuidLock(uuid, "cancel") { + if !sch.uuidLock(uuid, "requeue") { return } defer sch.uuidUnlock(uuid) diff --git a/lib/dispatchcloud/scheduler/sync_test.go b/lib/dispatchcloud/scheduler/sync_test.go index 538f5ea8cf..a3ff0636e1 100644 --- a/lib/dispatchcloud/scheduler/sync_test.go +++ b/lib/dispatchcloud/scheduler/sync_test.go @@ -48,7 +48,7 @@ func (*SchedulerSuite) TestForgetIrrelevantContainers(c *check.C) { ents, _ := queue.Entries() c.Check(ents, check.HasLen, 1) - sch := New(ctx, &queue, &pool, time.Millisecond, time.Millisecond) + sch := New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond) sch.sync() ents, _ = queue.Entries() @@ -80,7 +80,7 @@ func (*SchedulerSuite) TestCancelOrphanedContainers(c *check.C) { ents, _ := queue.Entries() c.Check(ents, check.HasLen, 1) - sch := New(ctx, &queue, &pool, time.Millisecond, time.Millisecond) + sch := New(ctx, &queue, &pool, nil, time.Millisecond, time.Millisecond) // Sync shouldn't cancel the container because it might be // running on the VM with state=="unknown". diff --git a/lib/dispatchcloud/ssh_executor/executor.go b/lib/dispatchcloud/sshexecutor/executor.go similarity index 98% rename from lib/dispatchcloud/ssh_executor/executor.go rename to lib/dispatchcloud/sshexecutor/executor.go index 79b82e6c37..c37169921c 100644 --- a/lib/dispatchcloud/ssh_executor/executor.go +++ b/lib/dispatchcloud/sshexecutor/executor.go @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: AGPL-3.0 -// Package ssh_executor provides an implementation of pool.Executor +// Package sshexecutor provides an implementation of pool.Executor // using a long-lived multiplexed SSH session. -package ssh_executor +package sshexecutor import ( "bytes" diff --git a/lib/dispatchcloud/ssh_executor/executor_test.go b/lib/dispatchcloud/sshexecutor/executor_test.go similarity index 99% rename from lib/dispatchcloud/ssh_executor/executor_test.go rename to lib/dispatchcloud/sshexecutor/executor_test.go index b7f3aadd8a..b4afeafa82 100644 --- a/lib/dispatchcloud/ssh_executor/executor_test.go +++ b/lib/dispatchcloud/sshexecutor/executor_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -package ssh_executor +package sshexecutor import ( "bytes" diff --git a/lib/dispatchcloud/test/queue.go b/lib/dispatchcloud/test/queue.go index 11d410fb1b..3598ec6da0 100644 --- a/lib/dispatchcloud/test/queue.go +++ b/lib/dispatchcloud/test/queue.go @@ -11,6 +11,7 @@ import ( "git.arvados.org/arvados.git/lib/dispatchcloud/container" "git.arvados.org/arvados.git/sdk/go/arvados" + "github.com/sirupsen/logrus" ) // Queue is a test stub for container.Queue. The caller specifies the @@ -23,6 +24,8 @@ type Queue struct { // must not be nil. ChooseType func(*arvados.Container) (arvados.InstanceType, error) + Logger logrus.FieldLogger + entries map[string]container.QueueEnt updTime time.Time subscribers map[<-chan struct{}]chan struct{} @@ -166,13 +169,35 @@ func (q *Queue) Notify(upd arvados.Container) bool { defer q.mtx.Unlock() for i, ctr := range q.Containers { if ctr.UUID == upd.UUID { - if ctr.State != arvados.ContainerStateComplete && ctr.State != arvados.ContainerStateCancelled { + if allowContainerUpdate[ctr.State][upd.State] { q.Containers[i] = upd return true } + if q.Logger != nil { + q.Logger.WithField("ContainerUUID", ctr.UUID).Infof("test.Queue rejected update from %s to %s", ctr.State, upd.State) + } return false } } q.Containers = append(q.Containers, upd) return true } + +var allowContainerUpdate = map[arvados.ContainerState]map[arvados.ContainerState]bool{ + arvados.ContainerStateQueued: { + arvados.ContainerStateQueued: true, + arvados.ContainerStateLocked: true, + arvados.ContainerStateCancelled: true, + }, + arvados.ContainerStateLocked: { + arvados.ContainerStateQueued: true, + arvados.ContainerStateLocked: true, + arvados.ContainerStateRunning: true, + arvados.ContainerStateCancelled: true, + }, + arvados.ContainerStateRunning: { + arvados.ContainerStateRunning: true, + arvados.ContainerStateCancelled: true, + arvados.ContainerStateComplete: true, + }, +} diff --git a/lib/dispatchcloud/test/ssh_service.go b/lib/dispatchcloud/test/ssh_service.go index f1fde4f422..31919b566d 100644 --- a/lib/dispatchcloud/test/ssh_service.go +++ b/lib/dispatchcloud/test/ssh_service.go @@ -18,6 +18,8 @@ import ( check "gopkg.in/check.v1" ) +// LoadTestKey returns a public/private ssh keypair, read from the files +// identified by the path of the private key. func LoadTestKey(c *check.C, fnm string) (ssh.PublicKey, ssh.Signer) { rawpubkey, err := ioutil.ReadFile(fnm + ".pub") c.Assert(err, check.IsNil) diff --git a/lib/dispatchcloud/test/stub_driver.go b/lib/dispatchcloud/test/stub_driver.go index 7a1f423016..4d32cf221c 100644 --- a/lib/dispatchcloud/test/stub_driver.go +++ b/lib/dispatchcloud/test/stub_driver.go @@ -34,6 +34,11 @@ type StubDriver struct { // VM's error rate and other behaviors. SetupVM func(*StubVM) + // Bugf, if set, is called if a bug is detected in the caller + // or stub. Typically set to (*check.C)Errorf. If unset, + // logger.Warnf is called instead. + Bugf func(string, ...interface{}) + // StubVM's fake crunch-run uses this Queue to read and update // container state. Queue *Queue @@ -99,6 +104,7 @@ type StubInstanceSet struct { allowCreateCall time.Time allowInstancesCall time.Time + lastInstanceID int } func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID, tags cloud.InstanceTags, cmd cloud.InitCommand, authKey ssh.PublicKey) (cloud.Instance, error) { @@ -112,21 +118,20 @@ func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID, } if sis.allowCreateCall.After(time.Now()) { return nil, RateLimitError{sis.allowCreateCall} - } else { - sis.allowCreateCall = time.Now().Add(sis.driver.MinTimeBetweenCreateCalls) } - + sis.allowCreateCall = time.Now().Add(sis.driver.MinTimeBetweenCreateCalls) ak := sis.driver.AuthorizedKeys if authKey != nil { ak = append([]ssh.PublicKey{authKey}, ak...) } + sis.lastInstanceID++ svm := &StubVM{ sis: sis, - id: cloud.InstanceID(fmt.Sprintf("stub-%s-%x", it.ProviderType, math_rand.Int63())), + id: cloud.InstanceID(fmt.Sprintf("inst%d,%s", sis.lastInstanceID, it.ProviderType)), tags: copyTags(tags), providerType: it.ProviderType, initCommand: cmd, - running: map[string]int64{}, + running: map[string]stubProcess{}, killing: map[string]bool{}, } svm.SSHService = SSHService{ @@ -147,9 +152,8 @@ func (sis *StubInstanceSet) Instances(cloud.InstanceTags) ([]cloud.Instance, err defer sis.mtx.RUnlock() if sis.allowInstancesCall.After(time.Now()) { return nil, RateLimitError{sis.allowInstancesCall} - } else { - sis.allowInstancesCall = time.Now().Add(sis.driver.MinTimeBetweenInstancesCalls) } + sis.allowInstancesCall = time.Now().Add(sis.driver.MinTimeBetweenInstancesCalls) var r []cloud.Instance for _, ss := range sis.servers { r = append(r, ss.Instance()) @@ -185,6 +189,8 @@ type StubVM struct { CrunchRunMissing bool CrunchRunCrashRate float64 CrunchRunDetachDelay time.Duration + ArvMountMaxExitLag time.Duration + ArvMountDeadlockRate float64 ExecuteContainer func(arvados.Container) int CrashRunningContainer func(arvados.Container) @@ -194,12 +200,21 @@ type StubVM struct { initCommand cloud.InitCommand providerType string SSHService SSHService - running map[string]int64 + running map[string]stubProcess killing map[string]bool lastPID int64 + deadlocked string sync.Mutex } +type stubProcess struct { + pid int64 + + // crunch-run has exited, but arv-mount process (or something) + // still holds lock in /var/run/ + exited bool +} + func (svm *StubVM) Instance() stubInstance { svm.Lock() defer svm.Unlock() @@ -252,7 +267,7 @@ func (svm *StubVM) Exec(env map[string]string, command string, stdin io.Reader, svm.Lock() svm.lastPID++ pid := svm.lastPID - svm.running[uuid] = pid + svm.running[uuid] = stubProcess{pid: pid} svm.Unlock() time.Sleep(svm.CrunchRunDetachDelay) fmt.Fprintf(stderr, "starting %s\n", uuid) @@ -263,93 +278,110 @@ func (svm *StubVM) Exec(env map[string]string, command string, stdin io.Reader, }) logger.Printf("[test] starting crunch-run stub") go func() { + var ctr arvados.Container + var started, completed bool + defer func() { + logger.Print("[test] exiting crunch-run stub") + svm.Lock() + defer svm.Unlock() + if svm.running[uuid].pid != pid { + bugf := svm.sis.driver.Bugf + if bugf == nil { + bugf = logger.Warnf + } + bugf("[test] StubDriver bug or caller bug: pid %d exiting, running[%s].pid==%d", pid, uuid, svm.running[uuid].pid) + return + } + if !completed { + logger.WithField("State", ctr.State).Print("[test] crashing crunch-run stub") + if started && svm.CrashRunningContainer != nil { + svm.CrashRunningContainer(ctr) + } + } + sproc := svm.running[uuid] + sproc.exited = true + svm.running[uuid] = sproc + svm.Unlock() + time.Sleep(svm.ArvMountMaxExitLag * time.Duration(math_rand.Float64())) + svm.Lock() + if math_rand.Float64() >= svm.ArvMountDeadlockRate { + delete(svm.running, uuid) + } + }() + crashluck := math_rand.Float64() + wantCrash := crashluck < svm.CrunchRunCrashRate + wantCrashEarly := crashluck < svm.CrunchRunCrashRate/2 + ctr, ok := queue.Get(uuid) if !ok { logger.Print("[test] container not in queue") return } - defer func() { - if ctr.State == arvados.ContainerStateRunning && svm.CrashRunningContainer != nil { - svm.CrashRunningContainer(ctr) - } - }() - - if crashluck > svm.CrunchRunCrashRate/2 { - time.Sleep(time.Duration(math_rand.Float64()*20) * time.Millisecond) - ctr.State = arvados.ContainerStateRunning - if !queue.Notify(ctr) { - ctr, _ = queue.Get(uuid) - logger.Print("[test] erroring out because state=Running update was rejected") - return - } - } - time.Sleep(time.Duration(math_rand.Float64()*20) * time.Millisecond) svm.Lock() - defer svm.Unlock() - if svm.running[uuid] != pid { - logger.Print("[test] container was killed") + killed := svm.killing[uuid] + svm.Unlock() + if killed || wantCrashEarly { return } - delete(svm.running, uuid) - if crashluck < svm.CrunchRunCrashRate { + ctr.State = arvados.ContainerStateRunning + started = queue.Notify(ctr) + if !started { + ctr, _ = queue.Get(uuid) + logger.Print("[test] erroring out because state=Running update was rejected") + return + } + + if wantCrash { logger.WithField("State", ctr.State).Print("[test] crashing crunch-run stub") - } else { - if svm.ExecuteContainer != nil { - ctr.ExitCode = svm.ExecuteContainer(ctr) - } - logger.WithField("ExitCode", ctr.ExitCode).Print("[test] exiting crunch-run stub") - ctr.State = arvados.ContainerStateComplete - go queue.Notify(ctr) + return + } + if svm.ExecuteContainer != nil { + ctr.ExitCode = svm.ExecuteContainer(ctr) } + logger.WithField("ExitCode", ctr.ExitCode).Print("[test] completing container") + ctr.State = arvados.ContainerStateComplete + completed = queue.Notify(ctr) }() return 0 } if command == "crunch-run --list" { svm.Lock() defer svm.Unlock() - for uuid := range svm.running { - fmt.Fprintf(stdout, "%s\n", uuid) + for uuid, sproc := range svm.running { + if sproc.exited { + fmt.Fprintf(stdout, "%s stale\n", uuid) + } else { + fmt.Fprintf(stdout, "%s\n", uuid) + } } if !svm.ReportBroken.IsZero() && svm.ReportBroken.Before(time.Now()) { fmt.Fprintln(stdout, "broken") } + fmt.Fprintln(stdout, svm.deadlocked) return 0 } if strings.HasPrefix(command, "crunch-run --kill ") { svm.Lock() - pid, running := svm.running[uuid] - if running && !svm.killing[uuid] { + sproc, running := svm.running[uuid] + if running && !sproc.exited { svm.killing[uuid] = true - go func() { - time.Sleep(time.Duration(math_rand.Float64()*30) * time.Millisecond) - svm.Lock() - defer svm.Unlock() - if svm.running[uuid] == pid { - // Kill only if the running entry - // hasn't since been killed and - // replaced with a different one. - delete(svm.running, uuid) - } - delete(svm.killing, uuid) - }() svm.Unlock() time.Sleep(time.Duration(math_rand.Float64()*2) * time.Millisecond) svm.Lock() - _, running = svm.running[uuid] + sproc, running = svm.running[uuid] } svm.Unlock() - if running { + if running && !sproc.exited { fmt.Fprintf(stderr, "%s: container is running\n", uuid) return 1 - } else { - fmt.Fprintf(stderr, "%s: container is not running\n", uuid) - return 0 } + fmt.Fprintf(stderr, "%s: container is not running\n", uuid) + return 0 } if command == "true" { return 0 diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go index 12bc1cdd71..a25ed60150 100644 --- a/lib/dispatchcloud/worker/pool.go +++ b/lib/dispatchcloud/worker/pool.go @@ -64,15 +64,16 @@ type Executor interface { } const ( - defaultSyncInterval = time.Minute - defaultProbeInterval = time.Second * 10 - defaultMaxProbesPerSecond = 10 - defaultTimeoutIdle = time.Minute - defaultTimeoutBooting = time.Minute * 10 - defaultTimeoutProbe = time.Minute * 10 - defaultTimeoutShutdown = time.Second * 10 - defaultTimeoutTERM = time.Minute * 2 - defaultTimeoutSignal = time.Second * 5 + defaultSyncInterval = time.Minute + defaultProbeInterval = time.Second * 10 + defaultMaxProbesPerSecond = 10 + defaultTimeoutIdle = time.Minute + defaultTimeoutBooting = time.Minute * 10 + defaultTimeoutProbe = time.Minute * 10 + defaultTimeoutShutdown = time.Second * 10 + defaultTimeoutTERM = time.Minute * 2 + defaultTimeoutSignal = time.Second * 5 + defaultTimeoutStaleRunLock = time.Second * 5 // Time after a quota error to try again anyway, even if no // instances have been shutdown. @@ -85,9 +86,8 @@ const ( func duration(conf arvados.Duration, def time.Duration) time.Duration { if conf > 0 { return time.Duration(conf) - } else { - return def } + return def } // NewPool creates a Pool of workers backed by instanceSet. @@ -96,27 +96,29 @@ func duration(conf arvados.Duration, def time.Duration) time.Duration { // cluster configuration. func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *prometheus.Registry, instanceSetID cloud.InstanceSetID, instanceSet cloud.InstanceSet, newExecutor func(cloud.Instance) Executor, installPublicKey ssh.PublicKey, cluster *arvados.Cluster) *Pool { wp := &Pool{ - logger: logger, - arvClient: arvClient, - instanceSetID: instanceSetID, - instanceSet: &throttledInstanceSet{InstanceSet: instanceSet}, - newExecutor: newExecutor, - bootProbeCommand: cluster.Containers.CloudVMs.BootProbeCommand, - runnerSource: cluster.Containers.CloudVMs.DeployRunnerBinary, - imageID: cloud.ImageID(cluster.Containers.CloudVMs.ImageID), - instanceTypes: cluster.InstanceTypes, - maxProbesPerSecond: cluster.Containers.CloudVMs.MaxProbesPerSecond, - probeInterval: duration(cluster.Containers.CloudVMs.ProbeInterval, defaultProbeInterval), - syncInterval: duration(cluster.Containers.CloudVMs.SyncInterval, defaultSyncInterval), - timeoutIdle: duration(cluster.Containers.CloudVMs.TimeoutIdle, defaultTimeoutIdle), - timeoutBooting: duration(cluster.Containers.CloudVMs.TimeoutBooting, defaultTimeoutBooting), - timeoutProbe: duration(cluster.Containers.CloudVMs.TimeoutProbe, defaultTimeoutProbe), - timeoutShutdown: duration(cluster.Containers.CloudVMs.TimeoutShutdown, defaultTimeoutShutdown), - timeoutTERM: duration(cluster.Containers.CloudVMs.TimeoutTERM, defaultTimeoutTERM), - timeoutSignal: duration(cluster.Containers.CloudVMs.TimeoutSignal, defaultTimeoutSignal), - installPublicKey: installPublicKey, - tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix, - stop: make(chan bool), + logger: logger, + arvClient: arvClient, + instanceSetID: instanceSetID, + instanceSet: &throttledInstanceSet{InstanceSet: instanceSet}, + newExecutor: newExecutor, + bootProbeCommand: cluster.Containers.CloudVMs.BootProbeCommand, + runnerSource: cluster.Containers.CloudVMs.DeployRunnerBinary, + imageID: cloud.ImageID(cluster.Containers.CloudVMs.ImageID), + instanceTypes: cluster.InstanceTypes, + maxProbesPerSecond: cluster.Containers.CloudVMs.MaxProbesPerSecond, + maxConcurrentInstanceCreateOps: cluster.Containers.CloudVMs.MaxConcurrentInstanceCreateOps, + probeInterval: duration(cluster.Containers.CloudVMs.ProbeInterval, defaultProbeInterval), + syncInterval: duration(cluster.Containers.CloudVMs.SyncInterval, defaultSyncInterval), + timeoutIdle: duration(cluster.Containers.CloudVMs.TimeoutIdle, defaultTimeoutIdle), + timeoutBooting: duration(cluster.Containers.CloudVMs.TimeoutBooting, defaultTimeoutBooting), + timeoutProbe: duration(cluster.Containers.CloudVMs.TimeoutProbe, defaultTimeoutProbe), + timeoutShutdown: duration(cluster.Containers.CloudVMs.TimeoutShutdown, defaultTimeoutShutdown), + timeoutTERM: duration(cluster.Containers.CloudVMs.TimeoutTERM, defaultTimeoutTERM), + timeoutSignal: duration(cluster.Containers.CloudVMs.TimeoutSignal, defaultTimeoutSignal), + timeoutStaleRunLock: duration(cluster.Containers.CloudVMs.TimeoutStaleRunLock, defaultTimeoutStaleRunLock), + installPublicKey: installPublicKey, + tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix, + stop: make(chan bool), } wp.registerMetrics(reg) go func() { @@ -132,26 +134,28 @@ func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *promethe // zero Pool should not be used. Call NewPool to create a new Pool. type Pool struct { // configuration - logger logrus.FieldLogger - arvClient *arvados.Client - instanceSetID cloud.InstanceSetID - instanceSet *throttledInstanceSet - newExecutor func(cloud.Instance) Executor - bootProbeCommand string - runnerSource string - imageID cloud.ImageID - instanceTypes map[string]arvados.InstanceType - syncInterval time.Duration - probeInterval time.Duration - maxProbesPerSecond int - timeoutIdle time.Duration - timeoutBooting time.Duration - timeoutProbe time.Duration - timeoutShutdown time.Duration - timeoutTERM time.Duration - timeoutSignal time.Duration - installPublicKey ssh.PublicKey - tagKeyPrefix string + logger logrus.FieldLogger + arvClient *arvados.Client + instanceSetID cloud.InstanceSetID + instanceSet *throttledInstanceSet + newExecutor func(cloud.Instance) Executor + bootProbeCommand string + runnerSource string + imageID cloud.ImageID + instanceTypes map[string]arvados.InstanceType + syncInterval time.Duration + probeInterval time.Duration + maxProbesPerSecond int + maxConcurrentInstanceCreateOps int + timeoutIdle time.Duration + timeoutBooting time.Duration + timeoutProbe time.Duration + timeoutShutdown time.Duration + timeoutTERM time.Duration + timeoutSignal time.Duration + timeoutStaleRunLock time.Duration + installPublicKey ssh.PublicKey + tagKeyPrefix string // private state subscribers map[<-chan struct{}]chan<- struct{} @@ -168,16 +172,18 @@ type Pool struct { runnerMD5 [md5.Size]byte runnerCmd string - throttleCreate throttle - throttleInstances throttle - - mContainersRunning prometheus.Gauge - mInstances *prometheus.GaugeVec - mInstancesPrice *prometheus.GaugeVec - mVCPUs *prometheus.GaugeVec - mMemory *prometheus.GaugeVec - mBootOutcomes *prometheus.CounterVec - mDisappearances *prometheus.CounterVec + mContainersRunning prometheus.Gauge + mInstances *prometheus.GaugeVec + mInstancesPrice *prometheus.GaugeVec + mVCPUs *prometheus.GaugeVec + mMemory *prometheus.GaugeVec + mBootOutcomes *prometheus.CounterVec + mDisappearances *prometheus.CounterVec + mTimeToSSH prometheus.Summary + mTimeToReadyForContainer prometheus.Summary + mTimeFromShutdownToGone prometheus.Summary + mTimeFromQueueToCrunchRun prometheus.Summary + mRunProbeDuration *prometheus.SummaryVec } type createCall struct { @@ -298,7 +304,19 @@ func (wp *Pool) Create(it arvados.InstanceType) bool { } wp.mtx.Lock() defer wp.mtx.Unlock() - if time.Now().Before(wp.atQuotaUntil) || wp.throttleCreate.Error() != nil { + if time.Now().Before(wp.atQuotaUntil) || wp.instanceSet.throttleCreate.Error() != nil { + return false + } + // The maxConcurrentInstanceCreateOps knob throttles the number of node create + // requests in flight. It was added to work around a limitation in Azure's + // managed disks, which support no more than 20 concurrent node creation + // requests from a single disk image (cf. + // https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image). + // The code assumes that node creation, from Azure's perspective, means the + // period until the instance appears in the "get all instances" list. + if wp.maxConcurrentInstanceCreateOps > 0 && len(wp.creating) >= wp.maxConcurrentInstanceCreateOps { + logger.Info("reached MaxConcurrentInstanceCreateOps") + wp.instanceSet.throttleCreate.ErrorUntil(errors.New("reached MaxConcurrentInstanceCreateOps"), time.Now().Add(5*time.Second), wp.notify) return false } now := time.Now() @@ -312,7 +330,7 @@ func (wp *Pool) Create(it arvados.InstanceType) bool { wp.tagKeyPrefix + tagKeyIdleBehavior: string(IdleBehaviorRun), wp.tagKeyPrefix + tagKeyInstanceSecret: secret, } - initCmd := TagVerifier{nil, secret}.InitCommand() + initCmd := TagVerifier{nil, secret, nil}.InitCommand() inst, err := wp.instanceSet.Create(it, wp.imageID, tags, initCmd, wp.installPublicKey) wp.mtx.Lock() defer wp.mtx.Unlock() @@ -356,6 +374,23 @@ func (wp *Pool) SetIdleBehavior(id cloud.InstanceID, idleBehavior IdleBehavior) return nil } +// Successful connection to the SSH daemon, update the mTimeToSSH metric +func (wp *Pool) reportSSHConnected(inst cloud.Instance) { + wp.mtx.Lock() + defer wp.mtx.Unlock() + wkr := wp.workers[inst.ID()] + if wkr.state != StateBooting || !wkr.firstSSHConnection.IsZero() { + // the node is not in booting state (can happen if a-d-c is restarted) OR + // this is not the first SSH connection + return + } + + wkr.firstSSHConnection = time.Now() + if wp.mTimeToSSH != nil { + wp.mTimeToSSH.Observe(wkr.firstSSHConnection.Sub(wkr.appeared).Seconds()) + } +} + // Add or update worker attached to the given instance. // // The second return value is true if a new worker is created. @@ -366,7 +401,7 @@ func (wp *Pool) SetIdleBehavior(id cloud.InstanceID, idleBehavior IdleBehavior) // Caller must have lock. func (wp *Pool) updateWorker(inst cloud.Instance, it arvados.InstanceType) (*worker, bool) { secret := inst.Tags()[wp.tagKeyPrefix+tagKeyInstanceSecret] - inst = TagVerifier{inst, secret} + inst = TagVerifier{Instance: inst, Secret: secret, ReportVerified: wp.reportSSHConnected} id := inst.ID() if wkr := wp.workers[id]; wkr != nil { wkr.executor.SetTarget(inst) @@ -615,6 +650,46 @@ func (wp *Pool) registerMetrics(reg *prometheus.Registry) { wp.mDisappearances.WithLabelValues(v).Add(0) } reg.MustRegister(wp.mDisappearances) + wp.mTimeToSSH = prometheus.NewSummary(prometheus.SummaryOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "instances_time_to_ssh_seconds", + Help: "Number of seconds between instance creation and the first successful SSH connection.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, + }) + reg.MustRegister(wp.mTimeToSSH) + wp.mTimeToReadyForContainer = prometheus.NewSummary(prometheus.SummaryOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "instances_time_to_ready_for_container_seconds", + Help: "Number of seconds between the first successful SSH connection and ready to run a container.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, + }) + reg.MustRegister(wp.mTimeToReadyForContainer) + wp.mTimeFromShutdownToGone = prometheus.NewSummary(prometheus.SummaryOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "instances_time_from_shutdown_request_to_disappearance_seconds", + Help: "Number of seconds between the first shutdown attempt and the disappearance of the worker.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, + }) + reg.MustRegister(wp.mTimeFromShutdownToGone) + wp.mTimeFromQueueToCrunchRun = prometheus.NewSummary(prometheus.SummaryOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "containers_time_from_queue_to_crunch_run_seconds", + Help: "Number of seconds between the queuing of a container and the start of crunch-run.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, + }) + reg.MustRegister(wp.mTimeFromQueueToCrunchRun) + wp.mRunProbeDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "arvados", + Subsystem: "dispatchcloud", + Name: "instances_run_probe_duration_seconds", + Help: "Number of seconds per runProbe call.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, + }, []string{"outcome"}) + reg.MustRegister(wp.mRunProbeDuration) } func (wp *Pool) runMetrics() { @@ -884,6 +959,10 @@ func (wp *Pool) sync(threshold time.Time, instances []cloud.Instance) { if wp.mDisappearances != nil { wp.mDisappearances.WithLabelValues(stateString[wkr.state]).Inc() } + // wkr.destroyed.IsZero() can happen if instance disappeared but we weren't trying to shut it down + if wp.mTimeFromShutdownToGone != nil && !wkr.destroyed.IsZero() { + wp.mTimeFromShutdownToGone.Observe(time.Now().Sub(wkr.destroyed).Seconds()) + } delete(wp.workers, id) go wkr.Close() notify = true diff --git a/lib/dispatchcloud/worker/pool_test.go b/lib/dispatchcloud/worker/pool_test.go index 0c173c107d..a85f7383ab 100644 --- a/lib/dispatchcloud/worker/pool_test.go +++ b/lib/dispatchcloud/worker/pool_test.go @@ -199,6 +199,46 @@ func (suite *PoolSuite) TestDrain(c *check.C) { } } +func (suite *PoolSuite) TestNodeCreateThrottle(c *check.C) { + logger := ctxlog.TestLogger(c) + driver := test.StubDriver{HoldCloudOps: true} + instanceSet, err := driver.InstanceSet(nil, "test-instance-set-id", nil, logger) + c.Assert(err, check.IsNil) + + type1 := test.InstanceType(1) + pool := &Pool{ + logger: logger, + instanceSet: &throttledInstanceSet{InstanceSet: instanceSet}, + maxConcurrentInstanceCreateOps: 1, + instanceTypes: arvados.InstanceTypeMap{ + type1.Name: type1, + }, + } + + c.Check(pool.Unallocated()[type1], check.Equals, 0) + res := pool.Create(type1) + c.Check(pool.Unallocated()[type1], check.Equals, 1) + c.Check(res, check.Equals, true) + + res = pool.Create(type1) + c.Check(pool.Unallocated()[type1], check.Equals, 1) + c.Check(res, check.Equals, false) + + pool.instanceSet.throttleCreate.err = nil + pool.maxConcurrentInstanceCreateOps = 2 + + res = pool.Create(type1) + c.Check(pool.Unallocated()[type1], check.Equals, 2) + c.Check(res, check.Equals, true) + + pool.instanceSet.throttleCreate.err = nil + pool.maxConcurrentInstanceCreateOps = 0 + + res = pool.Create(type1) + c.Check(pool.Unallocated()[type1], check.Equals, 3) + c.Check(res, check.Equals, true) +} + func (suite *PoolSuite) TestCreateUnallocShutdown(c *check.C) { logger := ctxlog.TestLogger(c) driver := test.StubDriver{HoldCloudOps: true} diff --git a/lib/dispatchcloud/worker/verify.go b/lib/dispatchcloud/worker/verify.go index 597950fca6..559bb28973 100644 --- a/lib/dispatchcloud/worker/verify.go +++ b/lib/dispatchcloud/worker/verify.go @@ -23,7 +23,8 @@ var ( type TagVerifier struct { cloud.Instance - Secret string + Secret string + ReportVerified func(cloud.Instance) } func (tv TagVerifier) InitCommand() cloud.InitCommand { @@ -31,6 +32,9 @@ func (tv TagVerifier) InitCommand() cloud.InitCommand { } func (tv TagVerifier) VerifyHostKey(pubKey ssh.PublicKey, client *ssh.Client) error { + if tv.ReportVerified != nil { + tv.ReportVerified(tv.Instance) + } if err := tv.Instance.VerifyHostKey(pubKey, client); err != cloud.ErrNotImplemented || tv.Secret == "" { // If the wrapped instance indicates it has a way to // verify the key, return that decision. diff --git a/lib/dispatchcloud/worker/worker.go b/lib/dispatchcloud/worker/worker.go index 5d2360f3cc..9e89d7daaf 100644 --- a/lib/dispatchcloud/worker/worker.go +++ b/lib/dispatchcloud/worker/worker.go @@ -103,11 +103,14 @@ type worker struct { updated time.Time busy time.Time destroyed time.Time + firstSSHConnection time.Time lastUUID string running map[string]*remoteRunner // remember to update state idle<->running when this changes starting map[string]*remoteRunner // remember to update state idle<->running when this changes probing chan struct{} bootOutcomeReported bool + timeToReadyReported bool + staleRunLockSince time.Time } func (wkr *worker) onUnkillable(uuid string) { @@ -140,6 +143,17 @@ func (wkr *worker) reportBootOutcome(outcome BootOutcome) { wkr.bootOutcomeReported = true } +// caller must have lock. +func (wkr *worker) reportTimeBetweenFirstSSHAndReadyForContainer() { + if wkr.timeToReadyReported { + return + } + if wkr.wp.mTimeToSSH != nil { + wkr.wp.mTimeToReadyForContainer.Observe(time.Since(wkr.firstSSHConnection).Seconds()) + } + wkr.timeToReadyReported = true +} + // caller must have lock. func (wkr *worker) setIdleBehavior(idleBehavior IdleBehavior) { wkr.logger.WithField("IdleBehavior", idleBehavior).Info("set idle behavior") @@ -163,6 +177,9 @@ func (wkr *worker) startContainer(ctr arvados.Container) { } go func() { rr.Start() + if wkr.wp.mTimeFromQueueToCrunchRun != nil { + wkr.wp.mTimeFromQueueToCrunchRun.Observe(time.Since(ctr.CreatedAt).Seconds()) + } wkr.mtx.Lock() defer wkr.mtx.Unlock() now := time.Now() @@ -175,7 +192,7 @@ func (wkr *worker) startContainer(ctr arvados.Container) { } // ProbeAndUpdate conducts appropriate boot/running probes (if any) -// for the worker's curent state. If a previous probe is still +// for the worker's current state. If a previous probe is still // running, it does nothing. // // It should be called in a new goroutine. @@ -313,6 +330,9 @@ func (wkr *worker) probeAndUpdate() { // Update state if this was the first successful boot-probe. if booted && (wkr.state == StateUnknown || wkr.state == StateBooting) { + if wkr.state == StateBooting { + wkr.reportTimeBetweenFirstSSHAndReadyForContainer() + } // Note: this will change again below if // len(wkr.starting)+len(wkr.running) > 0. wkr.state = StateIdle @@ -356,6 +376,7 @@ func (wkr *worker) probeRunning() (running []string, reportsBroken, ok bool) { if u := wkr.instance.RemoteUser(); u != "root" { cmd = "sudo " + cmd } + before := time.Now() stdout, stderr, err := wkr.executor.Execute(nil, cmd, nil) if err != nil { wkr.logger.WithFields(logrus.Fields{ @@ -363,16 +384,48 @@ func (wkr *worker) probeRunning() (running []string, reportsBroken, ok bool) { "stdout": string(stdout), "stderr": string(stderr), }).WithError(err).Warn("probe failed") + wkr.wp.mRunProbeDuration.WithLabelValues("fail").Observe(time.Now().Sub(before).Seconds()) return } + wkr.wp.mRunProbeDuration.WithLabelValues("success").Observe(time.Now().Sub(before).Seconds()) ok = true + + staleRunLock := false for _, s := range strings.Split(string(stdout), "\n") { - if s == "broken" { + // Each line of the "crunch-run --list" output is one + // of the following: + // + // * a container UUID, indicating that processes + // related to that container are currently running. + // Optionally followed by " stale", indicating that + // the crunch-run process itself has exited (the + // remaining process is probably arv-mount). + // + // * the string "broken", indicating that the instance + // appears incapable of starting containers. + // + // See ListProcesses() in lib/crunchrun/background.go. + if s == "" { + // empty string following final newline + } else if s == "broken" { reportsBroken = true - } else if s != "" { + } else if toks := strings.Split(s, " "); len(toks) == 1 { running = append(running, s) + } else if toks[1] == "stale" { + wkr.logger.WithField("ContainerUUID", toks[0]).Info("probe reported stale run lock") + staleRunLock = true } } + wkr.mtx.Lock() + defer wkr.mtx.Unlock() + if !staleRunLock { + wkr.staleRunLockSince = time.Time{} + } else if wkr.staleRunLockSince.IsZero() { + wkr.staleRunLockSince = time.Now() + } else if dur := time.Now().Sub(wkr.staleRunLockSince); dur > wkr.wp.timeoutStaleRunLock { + wkr.logger.WithField("Duration", dur).Warn("reporting broken after reporting stale run lock for too long") + reportsBroken = true + } return } diff --git a/lib/dispatchcloud/worker/worker_test.go b/lib/dispatchcloud/worker/worker_test.go index a4c2a6370f..cfb7a1bfb7 100644 --- a/lib/dispatchcloud/worker/worker_test.go +++ b/lib/dispatchcloud/worker/worker_test.go @@ -17,6 +17,7 @@ import ( "git.arvados.org/arvados.git/lib/dispatchcloud/test" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/prometheus/client_golang/prometheus" check "gopkg.in/check.v1" ) @@ -239,6 +240,7 @@ func (suite *WorkerSuite) TestProbeAndUpdate(c *check.C) { runnerData: trial.deployRunner, runnerMD5: md5.Sum(trial.deployRunner), } + wp.registerMetrics(prometheus.NewRegistry()) if trial.deployRunner != nil { svHash := md5.Sum(trial.deployRunner) wp.runnerCmd = fmt.Sprintf("/var/run/arvados/crunch-run~%x", svHash) diff --git a/lib/install/deps.go b/lib/install/deps.go index da45b393bf..cc9595db64 100644 --- a/lib/install/deps.go +++ b/lib/install/deps.go @@ -125,7 +125,8 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "bsdmainutils", "build-essential", "cadaver", - "cython", + "curl", + "cython3", "daemontools", // lib/boot uses setuidgid to drop privileges when running as root "default-jdk-headless", "default-jre-headless", @@ -138,7 +139,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "libjson-perl", "libpam-dev", "libpcre3-dev", - "libpython2.7-dev", + "libpq-dev", "libreadline-dev", "libssl-dev", "libwww-perl", @@ -154,11 +155,16 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read "postgresql", "postgresql-contrib", "python3-dev", - "python-epydoc", + "python3-venv", + "python3-virtualenv", "r-base", "r-cran-testthat", + "r-cran-devtools", + "r-cran-knitr", + "r-cran-markdown", + "r-cran-roxygen2", + "r-cran-xml", "sudo", - "virtualenv", "wget", "xvfb", ) @@ -316,10 +322,10 @@ rm ${zip} DataDirectory string LogFile string } - if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil { + if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil { err = fmt.Errorf("pg_lsclusters: %s", err2) return 1 - } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 { + } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 { logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters)) } else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil { err = fmt.Errorf("error parsing pg_lsclusters output: %s", err) diff --git a/lib/mount/command.go b/lib/mount/command.go index 86a9085bda..e92af24075 100644 --- a/lib/mount/command.go +++ b/lib/mount/command.go @@ -9,6 +9,7 @@ import ( "io" "log" "net/http" + // pprof is only imported to register its HTTP handlers _ "net/http/pprof" "os" diff --git a/lib/pam/fpm-info.sh b/lib/pam/fpm-info.sh index 3366b8e79a..43c04a67e2 100644 --- a/lib/pam/fpm-info.sh +++ b/lib/pam/fpm-info.sh @@ -3,3 +3,5 @@ # SPDX-License-Identifier: Apache-2.0 fpm_depends+=(ca-certificates) + +fpm_args+=(--conflicts=libpam-arvados) diff --git a/lib/pam/pam-configs-arvados b/lib/pam/pam-configs-arvados index 37ed4b86ab..3d695bf96b 100644 --- a/lib/pam/pam-configs-arvados +++ b/lib/pam/pam-configs-arvados @@ -2,11 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -# This file is packaged as /usr/share/pam-configs/arvados-go; see build/run-library.sh - -# 1. Run `pam-auth-update` and choose Arvados authentication -# 2. In /etc/pam.d/common-auth, change "api.example" to your ARVADOS_API_HOST -# 3. In /etc/pam.d/common-auth, change "shell.example" to this host's hostname +# 1. Copy the contents of this file *minus all comment lines* to /usr/share/pam-configs/arvados-go +# 2. Run `pam-auth-update` and choose Arvados authentication +# 3. In /etc/pam.d/common-auth, change "api.example" to your ARVADOS_API_HOST +# 4. In /etc/pam.d/common-auth, change "shell.example" to this host's hostname # (as it appears in the Arvados virtual_machines list) Name: Arvados authentication diff --git a/lib/service/cmd.go b/lib/service/cmd.go index 901fda2289..9ca2431258 100644 --- a/lib/service/cmd.go +++ b/lib/service/cmd.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -// package service provides a cmd.Handler that brings up a system service. +// Package service provides a cmd.Handler that brings up a system service. package service import ( @@ -165,8 +165,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout return 0 } -const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" - func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) { svc, ok := svcs.Map()[prog] if !ok { diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go index 4a984c9e78..10591d9b55 100644 --- a/lib/service/cmd_test.go +++ b/lib/service/cmd_test.go @@ -29,6 +29,11 @@ func Test(t *testing.T) { var _ = check.Suite(&Suite{}) type Suite struct{} +type key int + +const ( + contextKey key = iota +) func (*Suite) TestCommand(c *check.C) { cf, err := ioutil.TempFile("", "cmd_test.") @@ -42,11 +47,11 @@ func (*Suite) TestCommand(c *check.C) { defer cancel() cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler { - c.Check(ctx.Value("foo"), check.Equals, "bar") + c.Check(ctx.Value(contextKey), check.Equals, "bar") c.Check(token, check.Equals, "abcde") return &testHandler{ctx: ctx, healthCheck: healthCheck} }) - cmd.(*command).ctx = context.WithValue(ctx, "foo", "bar") + cmd.(*command).ctx = context.WithValue(ctx, contextKey, "bar") done := make(chan bool) var stdin, stdout, stderr bytes.Buffer diff --git a/lib/service/tls.go b/lib/service/tls.go index db3c567eec..c6307b76ab 100644 --- a/lib/service/tls.go +++ b/lib/service/tls.go @@ -23,7 +23,7 @@ func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogge key, cert := cluster.TLS.Key, cluster.TLS.Certificate if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") { - return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...") + return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified with a 'file://' prefix") } key, cert = key[7:], cert[7:] diff --git a/sdk/R/DESCRIPTION b/sdk/R/DESCRIPTION index 878a709014..75ac892b4b 100644 --- a/sdk/R/DESCRIPTION +++ b/sdk/R/DESCRIPTION @@ -1,9 +1,9 @@ Package: ArvadosR Type: Package Title: Arvados R SDK -Version: 0.0.5 -Authors@R: person("Fuad", "Muhic", role = c("aut", "cre"), email = "fmuhic@capeannenterprises.com") -Maintainer: Ward Vandewege +Version: 0.0.6 +Authors@R: c(person("Fuad", "Muhic", role = c("aut", "ctr"), email = "fmuhic@capeannenterprises.com"), + person("Peter", "Amstutz", role = c("cre"), email = "peter.amstutz@curii.com")) Description: This is the Arvados R SDK URL: http://doc.arvados.org License: Apache-2.0 diff --git a/sdk/R/R/Arvados.R b/sdk/R/R/Arvados.R index 744cb3c296..528a606650 100644 --- a/sdk/R/R/Arvados.R +++ b/sdk/R/R/Arvados.R @@ -1,130 +1,60 @@ -#' users.get -#' -#' users.get is a method defined in Arvados class. -#' -#' @usage arv$users.get(uuid) -#' @param uuid The UUID of the User in question. -#' @return User object. -#' @name users.get -NULL - -#' users.create -#' -#' users.create is a method defined in Arvados class. -#' -#' @usage arv$users.create(user, ensure_unique_name = "false") -#' @param user User object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return User object. -#' @name users.create -NULL - -#' users.update -#' -#' users.update is a method defined in Arvados class. -#' -#' @usage arv$users.update(user, uuid) -#' @param user User object. -#' @param uuid The UUID of the User in question. -#' @return User object. -#' @name users.update -NULL - -#' users.delete -#' -#' users.delete is a method defined in Arvados class. -#' -#' @usage arv$users.delete(uuid) -#' @param uuid The UUID of the User in question. -#' @return User object. -#' @name users.delete -NULL - -#' users.current -#' -#' users.current is a method defined in Arvados class. -#' -#' @usage arv$users.current(NULL) -#' @return User object. -#' @name users.current -NULL - -#' users.system -#' -#' users.system is a method defined in Arvados class. -#' -#' @usage arv$users.system(NULL) -#' @return User object. -#' @name users.system -NULL - -#' users.activate -#' -#' users.activate is a method defined in Arvados class. -#' -#' @usage arv$users.activate(uuid) -#' @param uuid -#' @return User object. -#' @name users.activate -NULL +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 -#' users.setup +#' api_clients.get #' -#' users.setup is a method defined in Arvados class. +#' api_clients.get is a method defined in Arvados class. #' -#' @usage arv$users.setup(user = NULL, openid_prefix = NULL, -#' repo_name = NULL, vm_uuid = NULL, send_notification_email = "false") -#' @param user -#' @param openid_prefix -#' @param repo_name -#' @param vm_uuid -#' @param send_notification_email -#' @return User object. -#' @name users.setup +#' @usage arv$api_clients.get(uuid) +#' @param uuid The UUID of the ApiClient in question. +#' @return ApiClient object. +#' @name api_clients.get NULL -#' users.unsetup +#' api_clients.create #' -#' users.unsetup is a method defined in Arvados class. +#' api_clients.create is a method defined in Arvados class. #' -#' @usage arv$users.unsetup(uuid) -#' @param uuid -#' @return User object. -#' @name users.unsetup +#' @usage arv$api_clients.create(apiclient, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param apiClient ApiClient object. +#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return ApiClient object. +#' @name api_clients.create NULL -#' users.update_uuid +#' api_clients.update #' -#' users.update_uuid is a method defined in Arvados class. +#' api_clients.update is a method defined in Arvados class. #' -#' @usage arv$users.update_uuid(uuid, new_uuid) -#' @param uuid -#' @param new_uuid -#' @return User object. -#' @name users.update_uuid +#' @usage arv$api_clients.update(apiclient, +#' uuid) +#' @param apiClient ApiClient object. +#' @param uuid The UUID of the ApiClient in question. +#' @return ApiClient object. +#' @name api_clients.update NULL -#' users.merge +#' api_clients.delete #' -#' users.merge is a method defined in Arvados class. +#' api_clients.delete is a method defined in Arvados class. #' -#' @usage arv$users.merge(new_owner_uuid, -#' new_user_token, redirect_to_new_user = NULL) -#' @param new_owner_uuid -#' @param new_user_token -#' @param redirect_to_new_user -#' @return User object. -#' @name users.merge +#' @usage arv$api_clients.delete(uuid) +#' @param uuid The UUID of the ApiClient in question. +#' @return ApiClient object. +#' @name api_clients.delete NULL -#' users.list +#' api_clients.list #' -#' users.list is a method defined in Arvados class. +#' api_clients.list is a method defined in Arvados class. #' -#' @usage arv$users.list(filters = NULL, +#' @usage arv$api_clients.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -133,8 +63,10 @@ NULL #' @param limit #' @param offset #' @param count -#' @return UserList object. -#' @name users.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return ApiClientList object. +#' @name api_clients.list NULL #' api_client_authorizations.get @@ -152,9 +84,10 @@ NULL #' api_client_authorizations.create is a method defined in Arvados class. #' #' @usage arv$api_client_authorizations.create(apiclientauthorization, -#' ensure_unique_name = "false") +#' ensure_unique_name = "false", cluster_id = NULL) #' @param apiClientAuthorization ApiClientAuthorization object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. +#' @param cluster_id Create object on a remote federated cluster instead of the current one. #' @return ApiClientAuthorization object. #' @name api_client_authorizations.create NULL @@ -209,7 +142,7 @@ NULL #' @usage arv$api_client_authorizations.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -218,111 +151,173 @@ NULL #' @param limit #' @param offset #' @param count +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only #' @return ApiClientAuthorizationList object. #' @name api_client_authorizations.list NULL -#' containers.get +#' authorized_keys.get #' -#' containers.get is a method defined in Arvados class. +#' authorized_keys.get is a method defined in Arvados class. #' -#' @usage arv$containers.get(uuid) -#' @param uuid The UUID of the Container in question. -#' @return Container object. -#' @name containers.get +#' @usage arv$authorized_keys.get(uuid) +#' @param uuid The UUID of the AuthorizedKey in question. +#' @return AuthorizedKey object. +#' @name authorized_keys.get NULL -#' containers.create +#' authorized_keys.create #' -#' containers.create is a method defined in Arvados class. +#' authorized_keys.create is a method defined in Arvados class. #' -#' @usage arv$containers.create(container, -#' ensure_unique_name = "false") -#' @param container Container object. +#' @usage arv$authorized_keys.create(authorizedkey, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param authorizedKey AuthorizedKey object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Container object. -#' @name containers.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return AuthorizedKey object. +#' @name authorized_keys.create NULL -#' containers.update +#' authorized_keys.update #' -#' containers.update is a method defined in Arvados class. +#' authorized_keys.update is a method defined in Arvados class. #' -#' @usage arv$containers.update(container, +#' @usage arv$authorized_keys.update(authorizedkey, #' uuid) -#' @param container Container object. -#' @param uuid The UUID of the Container in question. -#' @return Container object. -#' @name containers.update +#' @param authorizedKey AuthorizedKey object. +#' @param uuid The UUID of the AuthorizedKey in question. +#' @return AuthorizedKey object. +#' @name authorized_keys.update NULL -#' containers.delete +#' authorized_keys.delete #' -#' containers.delete is a method defined in Arvados class. +#' authorized_keys.delete is a method defined in Arvados class. #' -#' @usage arv$containers.delete(uuid) -#' @param uuid The UUID of the Container in question. -#' @return Container object. -#' @name containers.delete +#' @usage arv$authorized_keys.delete(uuid) +#' @param uuid The UUID of the AuthorizedKey in question. +#' @return AuthorizedKey object. +#' @name authorized_keys.delete NULL -#' containers.auth +#' authorized_keys.list #' -#' containers.auth is a method defined in Arvados class. +#' authorized_keys.list is a method defined in Arvados class. #' -#' @usage arv$containers.auth(uuid) -#' @param uuid -#' @return Container object. -#' @name containers.auth +#' @usage arv$authorized_keys.list(filters = NULL, +#' where = NULL, order = NULL, select = NULL, +#' distinct = NULL, limit = "100", offset = "0", +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) +#' @param filters +#' @param where +#' @param order +#' @param select +#' @param distinct +#' @param limit +#' @param offset +#' @param count +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return AuthorizedKeyList object. +#' @name authorized_keys.list NULL -#' containers.lock +#' collections.get #' -#' containers.lock is a method defined in Arvados class. +#' collections.get is a method defined in Arvados class. #' -#' @usage arv$containers.lock(uuid) -#' @param uuid -#' @return Container object. -#' @name containers.lock +#' @usage arv$collections.get(uuid) +#' @param uuid The UUID of the Collection in question. +#' @return Collection object. +#' @name collections.get NULL -#' containers.unlock +#' collections.create #' -#' containers.unlock is a method defined in Arvados class. +#' collections.create is a method defined in Arvados class. #' -#' @usage arv$containers.unlock(uuid) -#' @param uuid -#' @return Container object. -#' @name containers.unlock +#' @usage arv$collections.create(collection, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param collection Collection object. +#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return Collection object. +#' @name collections.create NULL -#' containers.secret_mounts +#' collections.update #' -#' containers.secret_mounts is a method defined in Arvados class. +#' collections.update is a method defined in Arvados class. #' -#' @usage arv$containers.secret_mounts(uuid) -#' @param uuid -#' @return Container object. -#' @name containers.secret_mounts +#' @usage arv$collections.update(collection, +#' uuid) +#' @param collection Collection object. +#' @param uuid The UUID of the Collection in question. +#' @return Collection object. +#' @name collections.update NULL -#' containers.current +#' collections.delete #' -#' containers.current is a method defined in Arvados class. +#' collections.delete is a method defined in Arvados class. #' -#' @usage arv$containers.current(NULL) -#' @return Container object. -#' @name containers.current +#' @usage arv$collections.delete(uuid) +#' @param uuid The UUID of the Collection in question. +#' @return Collection object. +#' @name collections.delete NULL -#' containers.list +#' collections.provenance #' -#' containers.list is a method defined in Arvados class. +#' collections.provenance is a method defined in Arvados class. #' -#' @usage arv$containers.list(filters = NULL, +#' @usage arv$collections.provenance(uuid) +#' @param uuid +#' @return Collection object. +#' @name collections.provenance +NULL + +#' collections.used_by +#' +#' collections.used_by is a method defined in Arvados class. +#' +#' @usage arv$collections.used_by(uuid) +#' @param uuid +#' @return Collection object. +#' @name collections.used_by +NULL + +#' collections.trash +#' +#' collections.trash is a method defined in Arvados class. +#' +#' @usage arv$collections.trash(uuid) +#' @param uuid +#' @return Collection object. +#' @name collections.trash +NULL + +#' collections.untrash +#' +#' collections.untrash is a method defined in Arvados class. +#' +#' @usage arv$collections.untrash(uuid) +#' @param uuid +#' @return Collection object. +#' @name collections.untrash +NULL + +#' collections.list +#' +#' collections.list is a method defined in Arvados class. +#' +#' @usage arv$collections.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL, +#' include_trash = NULL, include_old_versions = NULL) #' @param filters #' @param where #' @param order @@ -331,62 +326,116 @@ NULL #' @param limit #' @param offset #' @param count -#' @return ContainerList object. -#' @name containers.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @param include_trash Include collections whose is_trashed attribute is true. +#' @param include_old_versions Include past collection versions. +#' @return CollectionList object. +#' @name collections.list NULL -#' api_clients.get +#' containers.get #' -#' api_clients.get is a method defined in Arvados class. +#' containers.get is a method defined in Arvados class. #' -#' @usage arv$api_clients.get(uuid) -#' @param uuid The UUID of the ApiClient in question. -#' @return ApiClient object. -#' @name api_clients.get +#' @usage arv$containers.get(uuid) +#' @param uuid The UUID of the Container in question. +#' @return Container object. +#' @name containers.get NULL -#' api_clients.create +#' containers.create #' -#' api_clients.create is a method defined in Arvados class. +#' containers.create is a method defined in Arvados class. #' -#' @usage arv$api_clients.create(apiclient, -#' ensure_unique_name = "false") -#' @param apiClient ApiClient object. +#' @usage arv$containers.create(container, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param container Container object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return ApiClient object. -#' @name api_clients.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return Container object. +#' @name containers.create NULL -#' api_clients.update +#' containers.update #' -#' api_clients.update is a method defined in Arvados class. +#' containers.update is a method defined in Arvados class. #' -#' @usage arv$api_clients.update(apiclient, +#' @usage arv$containers.update(container, #' uuid) -#' @param apiClient ApiClient object. -#' @param uuid The UUID of the ApiClient in question. -#' @return ApiClient object. -#' @name api_clients.update +#' @param container Container object. +#' @param uuid The UUID of the Container in question. +#' @return Container object. +#' @name containers.update NULL -#' api_clients.delete +#' containers.delete #' -#' api_clients.delete is a method defined in Arvados class. +#' containers.delete is a method defined in Arvados class. #' -#' @usage arv$api_clients.delete(uuid) -#' @param uuid The UUID of the ApiClient in question. -#' @return ApiClient object. -#' @name api_clients.delete +#' @usage arv$containers.delete(uuid) +#' @param uuid The UUID of the Container in question. +#' @return Container object. +#' @name containers.delete NULL -#' api_clients.list +#' containers.auth #' -#' api_clients.list is a method defined in Arvados class. +#' containers.auth is a method defined in Arvados class. #' -#' @usage arv$api_clients.list(filters = NULL, +#' @usage arv$containers.auth(uuid) +#' @param uuid +#' @return Container object. +#' @name containers.auth +NULL + +#' containers.lock +#' +#' containers.lock is a method defined in Arvados class. +#' +#' @usage arv$containers.lock(uuid) +#' @param uuid +#' @return Container object. +#' @name containers.lock +NULL + +#' containers.unlock +#' +#' containers.unlock is a method defined in Arvados class. +#' +#' @usage arv$containers.unlock(uuid) +#' @param uuid +#' @return Container object. +#' @name containers.unlock +NULL + +#' containers.secret_mounts +#' +#' containers.secret_mounts is a method defined in Arvados class. +#' +#' @usage arv$containers.secret_mounts(uuid) +#' @param uuid +#' @return Container object. +#' @name containers.secret_mounts +NULL + +#' containers.current +#' +#' containers.current is a method defined in Arvados class. +#' +#' @usage arv$containers.current(NULL) +#' @return Container object. +#' @name containers.current +NULL + +#' containers.list +#' +#' containers.list is a method defined in Arvados class. +#' +#' @usage arv$containers.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -395,8 +444,10 @@ NULL #' @param limit #' @param offset #' @param count -#' @return ApiClientList object. -#' @name api_clients.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return ContainerList object. +#' @name containers.list NULL #' container_requests.get @@ -414,9 +465,10 @@ NULL #' container_requests.create is a method defined in Arvados class. #' #' @usage arv$container_requests.create(containerrequest, -#' ensure_unique_name = "false") +#' ensure_unique_name = "false", cluster_id = NULL) #' @param containerRequest ContainerRequest object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. +#' @param cluster_id Create object on a remote federated cluster instead of the current one. #' @return ContainerRequest object. #' @name container_requests.create NULL @@ -450,7 +502,8 @@ NULL #' @usage arv$container_requests.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL, +#' include_trash = NULL) #' @param filters #' @param where #' @param order @@ -459,62 +512,96 @@ NULL #' @param limit #' @param offset #' @param count +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @param include_trash Include container requests whose owner project is trashed. #' @return ContainerRequestList object. #' @name container_requests.list NULL -#' authorized_keys.get +#' groups.get #' -#' authorized_keys.get is a method defined in Arvados class. +#' groups.get is a method defined in Arvados class. #' -#' @usage arv$authorized_keys.get(uuid) -#' @param uuid The UUID of the AuthorizedKey in question. -#' @return AuthorizedKey object. -#' @name authorized_keys.get +#' @usage arv$groups.get(uuid) +#' @param uuid The UUID of the Group in question. +#' @return Group object. +#' @name groups.get NULL -#' authorized_keys.create +#' groups.create #' -#' authorized_keys.create is a method defined in Arvados class. +#' groups.create is a method defined in Arvados class. #' -#' @usage arv$authorized_keys.create(authorizedkey, -#' ensure_unique_name = "false") -#' @param authorizedKey AuthorizedKey object. +#' @usage arv$groups.create(group, ensure_unique_name = "false", +#' cluster_id = NULL, async = "false") +#' @param group Group object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return AuthorizedKey object. -#' @name authorized_keys.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @param async defer permissions update +#' @return Group object. +#' @name groups.create NULL -#' authorized_keys.update +#' groups.update #' -#' authorized_keys.update is a method defined in Arvados class. +#' groups.update is a method defined in Arvados class. #' -#' @usage arv$authorized_keys.update(authorizedkey, -#' uuid) -#' @param authorizedKey AuthorizedKey object. -#' @param uuid The UUID of the AuthorizedKey in question. -#' @return AuthorizedKey object. -#' @name authorized_keys.update +#' @usage arv$groups.update(group, uuid, +#' async = "false") +#' @param group Group object. +#' @param uuid The UUID of the Group in question. +#' @param async defer permissions update +#' @return Group object. +#' @name groups.update NULL -#' authorized_keys.delete +#' groups.delete #' -#' authorized_keys.delete is a method defined in Arvados class. +#' groups.delete is a method defined in Arvados class. #' -#' @usage arv$authorized_keys.delete(uuid) -#' @param uuid The UUID of the AuthorizedKey in question. -#' @return AuthorizedKey object. -#' @name authorized_keys.delete +#' @usage arv$groups.delete(uuid) +#' @param uuid The UUID of the Group in question. +#' @return Group object. +#' @name groups.delete NULL -#' authorized_keys.list +#' groups.contents #' -#' authorized_keys.list is a method defined in Arvados class. +#' groups.contents is a method defined in Arvados class. #' -#' @usage arv$authorized_keys.list(filters = NULL, +#' @usage arv$groups.contents(filters = NULL, +#' where = NULL, order = NULL, distinct = NULL, +#' limit = "100", offset = "0", count = "exact", +#' cluster_id = NULL, bypass_federation = NULL, +#' include_trash = NULL, uuid = NULL, recursive = NULL, +#' include = NULL) +#' @param filters +#' @param where +#' @param order +#' @param distinct +#' @param limit +#' @param offset +#' @param count +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @param include_trash Include items whose is_trashed attribute is true. +#' @param uuid +#' @param recursive Include contents from child groups recursively. +#' @param include Include objects referred to by listed field in "included" (only owner_uuid) +#' @return Group object. +#' @name groups.contents +NULL + +#' groups.shared +#' +#' groups.shared is a method defined in Arvados class. +#' +#' @usage arv$groups.shared(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL, +#' include_trash = NULL, include = NULL) #' @param filters #' @param where #' @param order @@ -523,102 +610,120 @@ NULL #' @param limit #' @param offset #' @param count -#' @return AuthorizedKeyList object. -#' @name authorized_keys.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @param include_trash Include items whose is_trashed attribute is true. +#' @param include +#' @return Group object. +#' @name groups.shared NULL -#' collections.get +#' groups.trash #' -#' collections.get is a method defined in Arvados class. +#' groups.trash is a method defined in Arvados class. #' -#' @usage arv$collections.get(uuid) -#' @param uuid The UUID of the Collection in question. -#' @return Collection object. -#' @name collections.get +#' @usage arv$groups.trash(uuid) +#' @param uuid +#' @return Group object. +#' @name groups.trash NULL -#' collections.create +#' groups.untrash #' -#' collections.create is a method defined in Arvados class. +#' groups.untrash is a method defined in Arvados class. #' -#' @usage arv$collections.create(collection, -#' ensure_unique_name = "false") -#' @param collection Collection object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Collection object. -#' @name collections.create +#' @usage arv$groups.untrash(uuid) +#' @param uuid +#' @return Group object. +#' @name groups.untrash NULL -#' collections.update +#' groups.list #' -#' collections.update is a method defined in Arvados class. +#' groups.list is a method defined in Arvados class. #' -#' @usage arv$collections.update(collection, -#' uuid) -#' @param collection Collection object. -#' @param uuid The UUID of the Collection in question. -#' @return Collection object. -#' @name collections.update +#' @usage arv$groups.list(filters = NULL, +#' where = NULL, order = NULL, select = NULL, +#' distinct = NULL, limit = "100", offset = "0", +#' count = "exact", cluster_id = NULL, bypass_federation = NULL, +#' include_trash = NULL) +#' @param filters +#' @param where +#' @param order +#' @param select +#' @param distinct +#' @param limit +#' @param offset +#' @param count +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @param include_trash Include items whose is_trashed attribute is true. +#' @return GroupList object. +#' @name groups.list NULL -#' collections.delete +#' keep_services.get #' -#' collections.delete is a method defined in Arvados class. +#' keep_services.get is a method defined in Arvados class. #' -#' @usage arv$collections.delete(uuid) -#' @param uuid The UUID of the Collection in question. -#' @return Collection object. -#' @name collections.delete +#' @usage arv$keep_services.get(uuid) +#' @param uuid The UUID of the KeepService in question. +#' @return KeepService object. +#' @name keep_services.get NULL -#' collections.provenance +#' keep_services.create #' -#' collections.provenance is a method defined in Arvados class. +#' keep_services.create is a method defined in Arvados class. #' -#' @usage arv$collections.provenance(uuid) -#' @param uuid -#' @return Collection object. -#' @name collections.provenance +#' @usage arv$keep_services.create(keepservice, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param keepService KeepService object. +#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return KeepService object. +#' @name keep_services.create NULL -#' collections.used_by +#' keep_services.update #' -#' collections.used_by is a method defined in Arvados class. +#' keep_services.update is a method defined in Arvados class. #' -#' @usage arv$collections.used_by(uuid) -#' @param uuid -#' @return Collection object. -#' @name collections.used_by +#' @usage arv$keep_services.update(keepservice, +#' uuid) +#' @param keepService KeepService object. +#' @param uuid The UUID of the KeepService in question. +#' @return KeepService object. +#' @name keep_services.update NULL -#' collections.trash +#' keep_services.delete #' -#' collections.trash is a method defined in Arvados class. +#' keep_services.delete is a method defined in Arvados class. #' -#' @usage arv$collections.trash(uuid) -#' @param uuid -#' @return Collection object. -#' @name collections.trash +#' @usage arv$keep_services.delete(uuid) +#' @param uuid The UUID of the KeepService in question. +#' @return KeepService object. +#' @name keep_services.delete NULL -#' collections.untrash +#' keep_services.accessible #' -#' collections.untrash is a method defined in Arvados class. +#' keep_services.accessible is a method defined in Arvados class. #' -#' @usage arv$collections.untrash(uuid) -#' @param uuid -#' @return Collection object. -#' @name collections.untrash +#' @usage arv$keep_services.accessible(NULL) +#' @return KeepService object. +#' @name keep_services.accessible NULL -#' collections.list +#' keep_services.list #' -#' collections.list is a method defined in Arvados class. +#' keep_services.list is a method defined in Arvados class. #' -#' @usage arv$collections.list(filters = NULL, +#' @usage arv$keep_services.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact", include_trash = NULL) +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -627,61 +732,64 @@ NULL #' @param limit #' @param offset #' @param count -#' @param include_trash Include collections whose is_trashed attribute is true. -#' @return CollectionList object. -#' @name collections.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return KeepServiceList object. +#' @name keep_services.list NULL -#' humans.get +#' links.get #' -#' humans.get is a method defined in Arvados class. +#' links.get is a method defined in Arvados class. #' -#' @usage arv$humans.get(uuid) -#' @param uuid The UUID of the Human in question. -#' @return Human object. -#' @name humans.get +#' @usage arv$links.get(uuid) +#' @param uuid The UUID of the Link in question. +#' @return Link object. +#' @name links.get NULL -#' humans.create +#' links.create #' -#' humans.create is a method defined in Arvados class. +#' links.create is a method defined in Arvados class. #' -#' @usage arv$humans.create(human, ensure_unique_name = "false") -#' @param human Human object. +#' @usage arv$links.create(link, ensure_unique_name = "false", +#' cluster_id = NULL) +#' @param link Link object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Human object. -#' @name humans.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return Link object. +#' @name links.create NULL -#' humans.update +#' links.update #' -#' humans.update is a method defined in Arvados class. +#' links.update is a method defined in Arvados class. #' -#' @usage arv$humans.update(human, uuid) -#' @param human Human object. -#' @param uuid The UUID of the Human in question. -#' @return Human object. -#' @name humans.update +#' @usage arv$links.update(link, uuid) +#' @param link Link object. +#' @param uuid The UUID of the Link in question. +#' @return Link object. +#' @name links.update NULL -#' humans.delete +#' links.delete #' -#' humans.delete is a method defined in Arvados class. +#' links.delete is a method defined in Arvados class. #' -#' @usage arv$humans.delete(uuid) -#' @param uuid The UUID of the Human in question. -#' @return Human object. -#' @name humans.delete +#' @usage arv$links.delete(uuid) +#' @param uuid The UUID of the Link in question. +#' @return Link object. +#' @name links.delete NULL -#' humans.list +#' links.list #' -#' humans.list is a method defined in Arvados class. +#' links.list is a method defined in Arvados class. #' -#' @usage arv$humans.list(filters = NULL, +#' @usage arv$links.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -690,60 +798,74 @@ NULL #' @param limit #' @param offset #' @param count -#' @return HumanList object. -#' @name humans.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return LinkList object. +#' @name links.list +NULL + +#' links.get_permissions +#' +#' links.get_permissions is a method defined in Arvados class. +#' +#' @usage arv$links.get_permissions(uuid) +#' @param uuid +#' @return Link object. +#' @name links.get_permissions NULL -#' job_tasks.get +#' logs.get #' -#' job_tasks.get is a method defined in Arvados class. +#' logs.get is a method defined in Arvados class. #' -#' @usage arv$job_tasks.get(uuid) -#' @param uuid The UUID of the JobTask in question. -#' @return JobTask object. -#' @name job_tasks.get +#' @usage arv$logs.get(uuid) +#' @param uuid The UUID of the Log in question. +#' @return Log object. +#' @name logs.get NULL -#' job_tasks.create +#' logs.create #' -#' job_tasks.create is a method defined in Arvados class. +#' logs.create is a method defined in Arvados class. #' -#' @usage arv$job_tasks.create(jobtask, ensure_unique_name = "false") -#' @param jobTask JobTask object. +#' @usage arv$logs.create(log, ensure_unique_name = "false", +#' cluster_id = NULL) +#' @param log Log object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return JobTask object. -#' @name job_tasks.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return Log object. +#' @name logs.create NULL -#' job_tasks.update +#' logs.update #' -#' job_tasks.update is a method defined in Arvados class. +#' logs.update is a method defined in Arvados class. #' -#' @usage arv$job_tasks.update(jobtask, uuid) -#' @param jobTask JobTask object. -#' @param uuid The UUID of the JobTask in question. -#' @return JobTask object. -#' @name job_tasks.update +#' @usage arv$logs.update(log, uuid) +#' @param log Log object. +#' @param uuid The UUID of the Log in question. +#' @return Log object. +#' @name logs.update NULL -#' job_tasks.delete +#' logs.delete #' -#' job_tasks.delete is a method defined in Arvados class. +#' logs.delete is a method defined in Arvados class. #' -#' @usage arv$job_tasks.delete(uuid) -#' @param uuid The UUID of the JobTask in question. -#' @return JobTask object. -#' @name job_tasks.delete +#' @usage arv$logs.delete(uuid) +#' @param uuid The UUID of the Log in question. +#' @return Log object. +#' @name logs.delete NULL -#' job_tasks.list +#' logs.list #' -#' job_tasks.list is a method defined in Arvados class. +#' logs.list is a method defined in Arvados class. #' -#' @usage arv$job_tasks.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' @usage arv$logs.list(filters = NULL, where = NULL, +#' order = NULL, select = NULL, distinct = NULL, +#' limit = "100", offset = "0", count = "exact", +#' cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -752,196 +874,145 @@ NULL #' @param limit #' @param offset #' @param count -#' @return JobTaskList object. -#' @name job_tasks.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return LogList object. +#' @name logs.list NULL -#' jobs.get +#' users.get #' -#' jobs.get is a method defined in Arvados class. +#' users.get is a method defined in Arvados class. #' -#' @usage arv$jobs.get(uuid) -#' @param uuid The UUID of the Job in question. -#' @return Job object. -#' @name jobs.get +#' @usage arv$users.get(uuid) +#' @param uuid The UUID of the User in question. +#' @return User object. +#' @name users.get NULL -#' jobs.create +#' users.create #' -#' jobs.create is a method defined in Arvados class. +#' users.create is a method defined in Arvados class. #' -#' @usage arv$jobs.create(job, ensure_unique_name = "false", -#' find_or_create = "false", filters = NULL, -#' minimum_script_version = NULL, exclude_script_versions = NULL) -#' @param job Job object. +#' @usage arv$users.create(user, ensure_unique_name = "false", +#' cluster_id = NULL) +#' @param user User object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @param find_or_create -#' @param filters -#' @param minimum_script_version -#' @param exclude_script_versions -#' @return Job object. -#' @name jobs.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return User object. +#' @name users.create NULL -#' jobs.update +#' users.update #' -#' jobs.update is a method defined in Arvados class. +#' users.update is a method defined in Arvados class. #' -#' @usage arv$jobs.update(job, uuid) -#' @param job Job object. -#' @param uuid The UUID of the Job in question. -#' @return Job object. -#' @name jobs.update +#' @usage arv$users.update(user, uuid, bypass_federation = NULL) +#' @param user User object. +#' @param uuid The UUID of the User in question. +#' @param bypass_federation +#' @return User object. +#' @name users.update NULL -#' jobs.delete +#' users.delete #' -#' jobs.delete is a method defined in Arvados class. +#' users.delete is a method defined in Arvados class. #' -#' @usage arv$jobs.delete(uuid) -#' @param uuid The UUID of the Job in question. -#' @return Job object. -#' @name jobs.delete +#' @usage arv$users.delete(uuid) +#' @param uuid The UUID of the User in question. +#' @return User object. +#' @name users.delete NULL -#' jobs.queue +#' users.current #' -#' jobs.queue is a method defined in Arvados class. +#' users.current is a method defined in Arvados class. #' -#' @usage arv$jobs.queue(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return Job object. -#' @name jobs.queue +#' @usage arv$users.current(NULL) +#' @return User object. +#' @name users.current NULL -#' jobs.queue_size +#' users.system #' -#' jobs.queue_size is a method defined in Arvados class. +#' users.system is a method defined in Arvados class. #' -#' @usage arv$jobs.queue_size(NULL) -#' @return Job object. -#' @name jobs.queue_size +#' @usage arv$users.system(NULL) +#' @return User object. +#' @name users.system NULL -#' jobs.cancel +#' users.activate #' -#' jobs.cancel is a method defined in Arvados class. +#' users.activate is a method defined in Arvados class. #' -#' @usage arv$jobs.cancel(uuid) +#' @usage arv$users.activate(uuid) #' @param uuid -#' @return Job object. -#' @name jobs.cancel +#' @return User object. +#' @name users.activate NULL -#' jobs.lock +#' users.setup #' -#' jobs.lock is a method defined in Arvados class. +#' users.setup is a method defined in Arvados class. #' -#' @usage arv$jobs.lock(uuid) +#' @usage arv$users.setup(uuid = NULL, user = NULL, +#' repo_name = NULL, vm_uuid = NULL, send_notification_email = "false") #' @param uuid -#' @return Job object. -#' @name jobs.lock +#' @param user +#' @param repo_name +#' @param vm_uuid +#' @param send_notification_email +#' @return User object. +#' @name users.setup NULL -#' jobs.list +#' users.unsetup #' -#' jobs.list is a method defined in Arvados class. +#' users.unsetup is a method defined in Arvados class. #' -#' @usage arv$jobs.list(filters = NULL, where = NULL, -#' order = NULL, select = NULL, distinct = NULL, -#' limit = "100", offset = "0", count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return JobList object. -#' @name jobs.list +#' @usage arv$users.unsetup(uuid) +#' @param uuid +#' @return User object. +#' @name users.unsetup NULL -#' keep_disks.get +#' users.update_uuid #' -#' keep_disks.get is a method defined in Arvados class. +#' users.update_uuid is a method defined in Arvados class. #' -#' @usage arv$keep_disks.get(uuid) -#' @param uuid The UUID of the KeepDisk in question. -#' @return KeepDisk object. -#' @name keep_disks.get +#' @usage arv$users.update_uuid(uuid, new_uuid) +#' @param uuid +#' @param new_uuid +#' @return User object. +#' @name users.update_uuid NULL -#' keep_disks.create +#' users.merge #' -#' keep_disks.create is a method defined in Arvados class. +#' users.merge is a method defined in Arvados class. #' -#' @usage arv$keep_disks.create(keepdisk, -#' ensure_unique_name = "false") -#' @param keepDisk KeepDisk object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return KeepDisk object. -#' @name keep_disks.create -NULL - -#' keep_disks.update -#' -#' keep_disks.update is a method defined in Arvados class. -#' -#' @usage arv$keep_disks.update(keepdisk, -#' uuid) -#' @param keepDisk KeepDisk object. -#' @param uuid The UUID of the KeepDisk in question. -#' @return KeepDisk object. -#' @name keep_disks.update -NULL - -#' keep_disks.delete -#' -#' keep_disks.delete is a method defined in Arvados class. -#' -#' @usage arv$keep_disks.delete(uuid) -#' @param uuid The UUID of the KeepDisk in question. -#' @return KeepDisk object. -#' @name keep_disks.delete -NULL - -#' keep_disks.ping -#' -#' keep_disks.ping is a method defined in Arvados class. -#' -#' @usage arv$keep_disks.ping(uuid = NULL, -#' ping_secret, node_uuid = NULL, filesystem_uuid = NULL, -#' service_host = NULL, service_port, service_ssl_flag) -#' @param uuid -#' @param ping_secret -#' @param node_uuid -#' @param filesystem_uuid -#' @param service_host -#' @param service_port -#' @param service_ssl_flag -#' @return KeepDisk object. -#' @name keep_disks.ping +#' @usage arv$users.merge(new_owner_uuid, +#' new_user_token = NULL, redirect_to_new_user = NULL, +#' old_user_uuid = NULL, new_user_uuid = NULL) +#' @param new_owner_uuid +#' @param new_user_token +#' @param redirect_to_new_user +#' @param old_user_uuid +#' @param new_user_uuid +#' @return User object. +#' @name users.merge NULL -#' keep_disks.list +#' users.list #' -#' keep_disks.list is a method defined in Arvados class. +#' users.list is a method defined in Arvados class. #' -#' @usage arv$keep_disks.list(filters = NULL, +#' @usage arv$users.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -950,74 +1021,74 @@ NULL #' @param limit #' @param offset #' @param count -#' @return KeepDiskList object. -#' @name keep_disks.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return UserList object. +#' @name users.list NULL -#' nodes.get +#' repositories.get #' -#' nodes.get is a method defined in Arvados class. +#' repositories.get is a method defined in Arvados class. #' -#' @usage arv$nodes.get(uuid) -#' @param uuid The UUID of the Node in question. -#' @return Node object. -#' @name nodes.get +#' @usage arv$repositories.get(uuid) +#' @param uuid The UUID of the Repository in question. +#' @return Repository object. +#' @name repositories.get NULL -#' nodes.create +#' repositories.create #' -#' nodes.create is a method defined in Arvados class. +#' repositories.create is a method defined in Arvados class. #' -#' @usage arv$nodes.create(node, ensure_unique_name = "false", -#' assign_slot = NULL) -#' @param node Node object. +#' @usage arv$repositories.create(repository, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param repository Repository object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @param assign_slot assign slot and hostname -#' @return Node object. -#' @name nodes.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return Repository object. +#' @name repositories.create NULL -#' nodes.update +#' repositories.update #' -#' nodes.update is a method defined in Arvados class. +#' repositories.update is a method defined in Arvados class. #' -#' @usage arv$nodes.update(node, uuid, assign_slot = NULL) -#' @param node Node object. -#' @param uuid The UUID of the Node in question. -#' @param assign_slot assign slot and hostname -#' @return Node object. -#' @name nodes.update +#' @usage arv$repositories.update(repository, +#' uuid) +#' @param repository Repository object. +#' @param uuid The UUID of the Repository in question. +#' @return Repository object. +#' @name repositories.update NULL -#' nodes.delete +#' repositories.delete #' -#' nodes.delete is a method defined in Arvados class. +#' repositories.delete is a method defined in Arvados class. #' -#' @usage arv$nodes.delete(uuid) -#' @param uuid The UUID of the Node in question. -#' @return Node object. -#' @name nodes.delete +#' @usage arv$repositories.delete(uuid) +#' @param uuid The UUID of the Repository in question. +#' @return Repository object. +#' @name repositories.delete NULL -#' nodes.ping +#' repositories.get_all_permissions #' -#' nodes.ping is a method defined in Arvados class. +#' repositories.get_all_permissions is a method defined in Arvados class. #' -#' @usage arv$nodes.ping(uuid, ping_secret) -#' @param uuid -#' @param ping_secret -#' @return Node object. -#' @name nodes.ping +#' @usage arv$repositories.get_all_permissions(NULL) +#' @return Repository object. +#' @name repositories.get_all_permissions NULL -#' nodes.list +#' repositories.list #' -#' nodes.list is a method defined in Arvados class. +#' repositories.list is a method defined in Arvados class. #' -#' @usage arv$nodes.list(filters = NULL, +#' @usage arv$repositories.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -1026,143 +1097,84 @@ NULL #' @param limit #' @param offset #' @param count -#' @return NodeList object. -#' @name nodes.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return RepositoryList object. +#' @name repositories.list NULL -#' links.get +#' virtual_machines.get #' -#' links.get is a method defined in Arvados class. +#' virtual_machines.get is a method defined in Arvados class. #' -#' @usage arv$links.get(uuid) -#' @param uuid The UUID of the Link in question. -#' @return Link object. -#' @name links.get +#' @usage arv$virtual_machines.get(uuid) +#' @param uuid The UUID of the VirtualMachine in question. +#' @return VirtualMachine object. +#' @name virtual_machines.get NULL -#' links.create +#' virtual_machines.create #' -#' links.create is a method defined in Arvados class. +#' virtual_machines.create is a method defined in Arvados class. #' -#' @usage arv$links.create(link, ensure_unique_name = "false") -#' @param link Link object. +#' @usage arv$virtual_machines.create(virtualmachine, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param virtualMachine VirtualMachine object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Link object. -#' @name links.create -NULL - -#' links.update -#' -#' links.update is a method defined in Arvados class. -#' -#' @usage arv$links.update(link, uuid) -#' @param link Link object. -#' @param uuid The UUID of the Link in question. -#' @return Link object. -#' @name links.update +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return VirtualMachine object. +#' @name virtual_machines.create NULL -#' links.delete +#' virtual_machines.update #' -#' links.delete is a method defined in Arvados class. +#' virtual_machines.update is a method defined in Arvados class. #' -#' @usage arv$links.delete(uuid) -#' @param uuid The UUID of the Link in question. -#' @return Link object. -#' @name links.delete +#' @usage arv$virtual_machines.update(virtualmachine, +#' uuid) +#' @param virtualMachine VirtualMachine object. +#' @param uuid The UUID of the VirtualMachine in question. +#' @return VirtualMachine object. +#' @name virtual_machines.update NULL -#' links.list +#' virtual_machines.delete #' -#' links.list is a method defined in Arvados class. +#' virtual_machines.delete is a method defined in Arvados class. #' -#' @usage arv$links.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return LinkList object. -#' @name links.list +#' @usage arv$virtual_machines.delete(uuid) +#' @param uuid The UUID of the VirtualMachine in question. +#' @return VirtualMachine object. +#' @name virtual_machines.delete NULL -#' links.get_permissions +#' virtual_machines.logins #' -#' links.get_permissions is a method defined in Arvados class. +#' virtual_machines.logins is a method defined in Arvados class. #' -#' @usage arv$links.get_permissions(uuid) +#' @usage arv$virtual_machines.logins(uuid) #' @param uuid -#' @return Link object. -#' @name links.get_permissions -NULL - -#' keep_services.get -#' -#' keep_services.get is a method defined in Arvados class. -#' -#' @usage arv$keep_services.get(uuid) -#' @param uuid The UUID of the KeepService in question. -#' @return KeepService object. -#' @name keep_services.get -NULL - -#' keep_services.create -#' -#' keep_services.create is a method defined in Arvados class. -#' -#' @usage arv$keep_services.create(keepservice, -#' ensure_unique_name = "false") -#' @param keepService KeepService object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return KeepService object. -#' @name keep_services.create -NULL - -#' keep_services.update -#' -#' keep_services.update is a method defined in Arvados class. -#' -#' @usage arv$keep_services.update(keepservice, -#' uuid) -#' @param keepService KeepService object. -#' @param uuid The UUID of the KeepService in question. -#' @return KeepService object. -#' @name keep_services.update -NULL - -#' keep_services.delete -#' -#' keep_services.delete is a method defined in Arvados class. -#' -#' @usage arv$keep_services.delete(uuid) -#' @param uuid The UUID of the KeepService in question. -#' @return KeepService object. -#' @name keep_services.delete +#' @return VirtualMachine object. +#' @name virtual_machines.logins NULL -#' keep_services.accessible +#' virtual_machines.get_all_logins #' -#' keep_services.accessible is a method defined in Arvados class. +#' virtual_machines.get_all_logins is a method defined in Arvados class. #' -#' @usage arv$keep_services.accessible(NULL) -#' @return KeepService object. -#' @name keep_services.accessible +#' @usage arv$virtual_machines.get_all_logins(NULL) +#' @return VirtualMachine object. +#' @name virtual_machines.get_all_logins NULL -#' keep_services.list +#' virtual_machines.list #' -#' keep_services.list is a method defined in Arvados class. +#' virtual_machines.list is a method defined in Arvados class. #' -#' @usage arv$keep_services.list(filters = NULL, +#' @usage arv$virtual_machines.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -1171,62 +1183,65 @@ NULL #' @param limit #' @param offset #' @param count -#' @return KeepServiceList object. -#' @name keep_services.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return VirtualMachineList object. +#' @name virtual_machines.list NULL -#' pipeline_templates.get +#' workflows.get #' -#' pipeline_templates.get is a method defined in Arvados class. +#' workflows.get is a method defined in Arvados class. #' -#' @usage arv$pipeline_templates.get(uuid) -#' @param uuid The UUID of the PipelineTemplate in question. -#' @return PipelineTemplate object. -#' @name pipeline_templates.get +#' @usage arv$workflows.get(uuid) +#' @param uuid The UUID of the Workflow in question. +#' @return Workflow object. +#' @name workflows.get NULL -#' pipeline_templates.create +#' workflows.create #' -#' pipeline_templates.create is a method defined in Arvados class. +#' workflows.create is a method defined in Arvados class. #' -#' @usage arv$pipeline_templates.create(pipelinetemplate, -#' ensure_unique_name = "false") -#' @param pipelineTemplate PipelineTemplate object. +#' @usage arv$workflows.create(workflow, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param workflow Workflow object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return PipelineTemplate object. -#' @name pipeline_templates.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return Workflow object. +#' @name workflows.create NULL -#' pipeline_templates.update +#' workflows.update #' -#' pipeline_templates.update is a method defined in Arvados class. +#' workflows.update is a method defined in Arvados class. #' -#' @usage arv$pipeline_templates.update(pipelinetemplate, +#' @usage arv$workflows.update(workflow, #' uuid) -#' @param pipelineTemplate PipelineTemplate object. -#' @param uuid The UUID of the PipelineTemplate in question. -#' @return PipelineTemplate object. -#' @name pipeline_templates.update +#' @param workflow Workflow object. +#' @param uuid The UUID of the Workflow in question. +#' @return Workflow object. +#' @name workflows.update NULL -#' pipeline_templates.delete +#' workflows.delete #' -#' pipeline_templates.delete is a method defined in Arvados class. +#' workflows.delete is a method defined in Arvados class. #' -#' @usage arv$pipeline_templates.delete(uuid) -#' @param uuid The UUID of the PipelineTemplate in question. -#' @return PipelineTemplate object. -#' @name pipeline_templates.delete +#' @usage arv$workflows.delete(uuid) +#' @param uuid The UUID of the Workflow in question. +#' @return Workflow object. +#' @name workflows.delete NULL -#' pipeline_templates.list +#' workflows.list #' -#' pipeline_templates.list is a method defined in Arvados class. +#' workflows.list is a method defined in Arvados class. #' -#' @usage arv$pipeline_templates.list(filters = NULL, +#' @usage arv$workflows.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -1235,209 +1250,83 @@ NULL #' @param limit #' @param offset #' @param count -#' @return PipelineTemplateList object. -#' @name pipeline_templates.list +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return WorkflowList object. +#' @name workflows.list NULL -#' pipeline_instances.get +#' user_agreements.get #' -#' pipeline_instances.get is a method defined in Arvados class. +#' user_agreements.get is a method defined in Arvados class. #' -#' @usage arv$pipeline_instances.get(uuid) -#' @param uuid The UUID of the PipelineInstance in question. -#' @return PipelineInstance object. -#' @name pipeline_instances.get +#' @usage arv$user_agreements.get(uuid) +#' @param uuid The UUID of the UserAgreement in question. +#' @return UserAgreement object. +#' @name user_agreements.get NULL -#' pipeline_instances.create +#' user_agreements.create #' -#' pipeline_instances.create is a method defined in Arvados class. +#' user_agreements.create is a method defined in Arvados class. #' -#' @usage arv$pipeline_instances.create(pipelineinstance, -#' ensure_unique_name = "false") -#' @param pipelineInstance PipelineInstance object. +#' @usage arv$user_agreements.create(useragreement, +#' ensure_unique_name = "false", cluster_id = NULL) +#' @param userAgreement UserAgreement object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return PipelineInstance object. -#' @name pipeline_instances.create +#' @param cluster_id Create object on a remote federated cluster instead of the current one. +#' @return UserAgreement object. +#' @name user_agreements.create NULL -#' pipeline_instances.update +#' user_agreements.update #' -#' pipeline_instances.update is a method defined in Arvados class. +#' user_agreements.update is a method defined in Arvados class. #' -#' @usage arv$pipeline_instances.update(pipelineinstance, +#' @usage arv$user_agreements.update(useragreement, #' uuid) -#' @param pipelineInstance PipelineInstance object. -#' @param uuid The UUID of the PipelineInstance in question. -#' @return PipelineInstance object. -#' @name pipeline_instances.update +#' @param userAgreement UserAgreement object. +#' @param uuid The UUID of the UserAgreement in question. +#' @return UserAgreement object. +#' @name user_agreements.update NULL -#' pipeline_instances.delete +#' user_agreements.delete #' -#' pipeline_instances.delete is a method defined in Arvados class. +#' user_agreements.delete is a method defined in Arvados class. #' -#' @usage arv$pipeline_instances.delete(uuid) -#' @param uuid The UUID of the PipelineInstance in question. -#' @return PipelineInstance object. -#' @name pipeline_instances.delete -NULL - -#' pipeline_instances.cancel -#' -#' pipeline_instances.cancel is a method defined in Arvados class. -#' -#' @usage arv$pipeline_instances.cancel(uuid) -#' @param uuid -#' @return PipelineInstance object. -#' @name pipeline_instances.cancel -NULL - -#' pipeline_instances.list -#' -#' pipeline_instances.list is a method defined in Arvados class. -#' -#' @usage arv$pipeline_instances.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return PipelineInstanceList object. -#' @name pipeline_instances.list -NULL - -#' repositories.get -#' -#' repositories.get is a method defined in Arvados class. -#' -#' @usage arv$repositories.get(uuid) -#' @param uuid The UUID of the Repository in question. -#' @return Repository object. -#' @name repositories.get -NULL - -#' repositories.create -#' -#' repositories.create is a method defined in Arvados class. -#' -#' @usage arv$repositories.create(repository, -#' ensure_unique_name = "false") -#' @param repository Repository object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Repository object. -#' @name repositories.create -NULL - -#' repositories.update -#' -#' repositories.update is a method defined in Arvados class. -#' -#' @usage arv$repositories.update(repository, -#' uuid) -#' @param repository Repository object. -#' @param uuid The UUID of the Repository in question. -#' @return Repository object. -#' @name repositories.update -NULL - -#' repositories.delete -#' -#' repositories.delete is a method defined in Arvados class. -#' -#' @usage arv$repositories.delete(uuid) -#' @param uuid The UUID of the Repository in question. -#' @return Repository object. -#' @name repositories.delete -NULL - -#' repositories.get_all_permissions -#' -#' repositories.get_all_permissions is a method defined in Arvados class. -#' -#' @usage arv$repositories.get_all_permissions(NULL) -#' @return Repository object. -#' @name repositories.get_all_permissions -NULL - -#' repositories.list -#' -#' repositories.list is a method defined in Arvados class. -#' -#' @usage arv$repositories.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return RepositoryList object. -#' @name repositories.list -NULL - -#' specimens.get -#' -#' specimens.get is a method defined in Arvados class. -#' -#' @usage arv$specimens.get(uuid) -#' @param uuid The UUID of the Specimen in question. -#' @return Specimen object. -#' @name specimens.get -NULL - -#' specimens.create -#' -#' specimens.create is a method defined in Arvados class. -#' -#' @usage arv$specimens.create(specimen, -#' ensure_unique_name = "false") -#' @param specimen Specimen object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Specimen object. -#' @name specimens.create +#' @usage arv$user_agreements.delete(uuid) +#' @param uuid The UUID of the UserAgreement in question. +#' @return UserAgreement object. +#' @name user_agreements.delete NULL -#' specimens.update +#' user_agreements.signatures #' -#' specimens.update is a method defined in Arvados class. +#' user_agreements.signatures is a method defined in Arvados class. #' -#' @usage arv$specimens.update(specimen, -#' uuid) -#' @param specimen Specimen object. -#' @param uuid The UUID of the Specimen in question. -#' @return Specimen object. -#' @name specimens.update +#' @usage arv$user_agreements.signatures(NULL) +#' @return UserAgreement object. +#' @name user_agreements.signatures NULL -#' specimens.delete +#' user_agreements.sign #' -#' specimens.delete is a method defined in Arvados class. +#' user_agreements.sign is a method defined in Arvados class. #' -#' @usage arv$specimens.delete(uuid) -#' @param uuid The UUID of the Specimen in question. -#' @return Specimen object. -#' @name specimens.delete +#' @usage arv$user_agreements.sign(NULL) +#' @return UserAgreement object. +#' @name user_agreements.sign NULL -#' specimens.list +#' user_agreements.list #' -#' specimens.list is a method defined in Arvados class. +#' user_agreements.list is a method defined in Arvados class. #' -#' @usage arv$specimens.list(filters = NULL, +#' @usage arv$user_agreements.list(filters = NULL, #' where = NULL, order = NULL, select = NULL, #' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' count = "exact", cluster_id = NULL, bypass_federation = NULL) #' @param filters #' @param where #' @param order @@ -1446,1774 +1335,312 @@ NULL #' @param limit #' @param offset #' @param count -#' @return SpecimenList object. -#' @name specimens.list -NULL - -#' logs.get -#' -#' logs.get is a method defined in Arvados class. -#' -#' @usage arv$logs.get(uuid) -#' @param uuid The UUID of the Log in question. -#' @return Log object. -#' @name logs.get -NULL - -#' logs.create -#' -#' logs.create is a method defined in Arvados class. -#' -#' @usage arv$logs.create(log, ensure_unique_name = "false") -#' @param log Log object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Log object. -#' @name logs.create -NULL - -#' logs.update -#' -#' logs.update is a method defined in Arvados class. -#' -#' @usage arv$logs.update(log, uuid) -#' @param log Log object. -#' @param uuid The UUID of the Log in question. -#' @return Log object. -#' @name logs.update +#' @param cluster_id List objects on a remote federated cluster instead of the current one. +#' @param bypass_federation bypass federation behavior, list items from local instance database only +#' @return UserAgreementList object. +#' @name user_agreements.list NULL -#' logs.delete +#' user_agreements.new #' -#' logs.delete is a method defined in Arvados class. +#' user_agreements.new is a method defined in Arvados class. #' -#' @usage arv$logs.delete(uuid) -#' @param uuid The UUID of the Log in question. -#' @return Log object. -#' @name logs.delete +#' @usage arv$user_agreements.new(NULL) +#' @return UserAgreement object. +#' @name user_agreements.new NULL -#' logs.list +#' configs.get #' -#' logs.list is a method defined in Arvados class. +#' configs.get is a method defined in Arvados class. #' -#' @usage arv$logs.list(filters = NULL, where = NULL, -#' order = NULL, select = NULL, distinct = NULL, -#' limit = "100", offset = "0", count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return LogList object. -#' @name logs.list +#' @usage arv$configs.get(NULL) +#' @return object. +#' @name configs.get NULL -#' traits.get +#' project.get #' -#' traits.get is a method defined in Arvados class. +#' projects.get is equivalent to groups.get method. #' -#' @usage arv$traits.get(uuid) -#' @param uuid The UUID of the Trait in question. -#' @return Trait object. -#' @name traits.get +#' @usage arv$projects.get(uuid) +#' @param uuid The UUID of the Group in question. +#' @return Group object. +#' @name projects.get NULL -#' traits.create +#' project.create #' -#' traits.create is a method defined in Arvados class. +#' projects.create wrapps groups.create method by setting group_class attribute to "project". #' -#' @usage arv$traits.create(trait, ensure_unique_name = "false") -#' @param trait Trait object. +#' @usage arv$projects.create(group, ensure_unique_name = "false") +#' @param group Group object. #' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Trait object. -#' @name traits.create +#' @return Group object. +#' @name projects.create NULL -#' traits.update +#' project.update #' -#' traits.update is a method defined in Arvados class. +#' projects.update wrapps groups.update method by setting group_class attribute to "project". #' -#' @usage arv$traits.update(trait, uuid) -#' @param trait Trait object. -#' @param uuid The UUID of the Trait in question. -#' @return Trait object. -#' @name traits.update +#' @usage arv$projects.update(group, uuid) +#' @param group Group object. +#' @param uuid The UUID of the Group in question. +#' @return Group object. +#' @name projects.update NULL -#' traits.delete +#' project.delete #' -#' traits.delete is a method defined in Arvados class. +#' projects.delete is equivalent to groups.delete method. #' -#' @usage arv$traits.delete(uuid) -#' @param uuid The UUID of the Trait in question. -#' @return Trait object. -#' @name traits.delete +#' @usage arv$project.delete(uuid) +#' @param uuid The UUID of the Group in question. +#' @return Group object. +#' @name projects.delete NULL -#' traits.list +#' project.list #' -#' traits.list is a method defined in Arvados class. +#' projects.list wrapps groups.list method by setting group_class attribute to "project". #' -#' @usage arv$traits.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") +#' @usage arv$projects.list(filters = NULL, +#' where = NULL, order = NULL, distinct = NULL, +#' limit = "100", offset = "0", count = "exact", +#' include_trash = NULL, uuid = NULL, recursive = NULL) #' @param filters #' @param where #' @param order -#' @param select #' @param distinct #' @param limit #' @param offset #' @param count -#' @return TraitList object. -#' @name traits.list -NULL - -#' virtual_machines.get -#' -#' virtual_machines.get is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.get(uuid) -#' @param uuid The UUID of the VirtualMachine in question. -#' @return VirtualMachine object. -#' @name virtual_machines.get -NULL - -#' virtual_machines.create -#' -#' virtual_machines.create is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.create(virtualmachine, -#' ensure_unique_name = "false") -#' @param virtualMachine VirtualMachine object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return VirtualMachine object. -#' @name virtual_machines.create -NULL - -#' virtual_machines.update -#' -#' virtual_machines.update is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.update(virtualmachine, -#' uuid) -#' @param virtualMachine VirtualMachine object. -#' @param uuid The UUID of the VirtualMachine in question. -#' @return VirtualMachine object. -#' @name virtual_machines.update +#' @param include_trash Include items whose is_trashed attribute is true. +#' @param uuid +#' @param recursive Include contents from child groups recursively. +#' @return Group object. +#' @name projects.list NULL -#' virtual_machines.delete -#' -#' virtual_machines.delete is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.delete(uuid) -#' @param uuid The UUID of the VirtualMachine in question. -#' @return VirtualMachine object. -#' @name virtual_machines.delete -NULL - -#' virtual_machines.logins -#' -#' virtual_machines.logins is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.logins(uuid) -#' @param uuid -#' @return VirtualMachine object. -#' @name virtual_machines.logins -NULL - -#' virtual_machines.get_all_logins -#' -#' virtual_machines.get_all_logins is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.get_all_logins(NULL) -#' @return VirtualMachine object. -#' @name virtual_machines.get_all_logins -NULL - -#' virtual_machines.list -#' -#' virtual_machines.list is a method defined in Arvados class. -#' -#' @usage arv$virtual_machines.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return VirtualMachineList object. -#' @name virtual_machines.list -NULL - -#' workflows.get -#' -#' workflows.get is a method defined in Arvados class. -#' -#' @usage arv$workflows.get(uuid) -#' @param uuid The UUID of the Workflow in question. -#' @return Workflow object. -#' @name workflows.get -NULL - -#' workflows.create -#' -#' workflows.create is a method defined in Arvados class. -#' -#' @usage arv$workflows.create(workflow, -#' ensure_unique_name = "false") -#' @param workflow Workflow object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Workflow object. -#' @name workflows.create -NULL - -#' workflows.update -#' -#' workflows.update is a method defined in Arvados class. -#' -#' @usage arv$workflows.update(workflow, -#' uuid) -#' @param workflow Workflow object. -#' @param uuid The UUID of the Workflow in question. -#' @return Workflow object. -#' @name workflows.update -NULL - -#' workflows.delete -#' -#' workflows.delete is a method defined in Arvados class. -#' -#' @usage arv$workflows.delete(uuid) -#' @param uuid The UUID of the Workflow in question. -#' @return Workflow object. -#' @name workflows.delete -NULL - -#' workflows.list -#' -#' workflows.list is a method defined in Arvados class. -#' -#' @usage arv$workflows.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return WorkflowList object. -#' @name workflows.list -NULL - -#' groups.get -#' -#' groups.get is a method defined in Arvados class. -#' -#' @usage arv$groups.get(uuid) -#' @param uuid The UUID of the Group in question. -#' @return Group object. -#' @name groups.get -NULL - -#' groups.create -#' -#' groups.create is a method defined in Arvados class. -#' -#' @usage arv$groups.create(group, ensure_unique_name = "false") -#' @param group Group object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Group object. -#' @name groups.create -NULL - -#' groups.update -#' -#' groups.update is a method defined in Arvados class. -#' -#' @usage arv$groups.update(group, uuid) -#' @param group Group object. -#' @param uuid The UUID of the Group in question. -#' @return Group object. -#' @name groups.update -NULL - -#' groups.delete -#' -#' groups.delete is a method defined in Arvados class. -#' -#' @usage arv$groups.delete(uuid) -#' @param uuid The UUID of the Group in question. -#' @return Group object. -#' @name groups.delete -NULL - -#' groups.contents -#' -#' groups.contents is a method defined in Arvados class. -#' -#' @usage arv$groups.contents(filters = NULL, -#' where = NULL, order = NULL, distinct = NULL, -#' limit = "100", offset = "0", count = "exact", -#' include_trash = NULL, uuid = NULL, recursive = NULL) -#' @param filters -#' @param where -#' @param order -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @param include_trash Include items whose is_trashed attribute is true. -#' @param uuid -#' @param recursive Include contents from child groups recursively. -#' @return Group object. -#' @name groups.contents -NULL - -#' groups.trash -#' -#' groups.trash is a method defined in Arvados class. -#' -#' @usage arv$groups.trash(uuid) -#' @param uuid -#' @return Group object. -#' @name groups.trash -NULL - -#' groups.untrash -#' -#' groups.untrash is a method defined in Arvados class. -#' -#' @usage arv$groups.untrash(uuid) -#' @param uuid -#' @return Group object. -#' @name groups.untrash -NULL - -#' groups.list -#' -#' groups.list is a method defined in Arvados class. -#' -#' @usage arv$groups.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact", include_trash = NULL) -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @param include_trash Include items whose is_trashed attribute is true. -#' @return GroupList object. -#' @name groups.list -NULL - -#' user_agreements.get -#' -#' user_agreements.get is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.get(uuid) -#' @param uuid The UUID of the UserAgreement in question. -#' @return UserAgreement object. -#' @name user_agreements.get -NULL - -#' user_agreements.create -#' -#' user_agreements.create is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.create(useragreement, -#' ensure_unique_name = "false") -#' @param userAgreement UserAgreement object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return UserAgreement object. -#' @name user_agreements.create -NULL - -#' user_agreements.update -#' -#' user_agreements.update is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.update(useragreement, -#' uuid) -#' @param userAgreement UserAgreement object. -#' @param uuid The UUID of the UserAgreement in question. -#' @return UserAgreement object. -#' @name user_agreements.update -NULL - -#' user_agreements.delete -#' -#' user_agreements.delete is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.delete(uuid) -#' @param uuid The UUID of the UserAgreement in question. -#' @return UserAgreement object. -#' @name user_agreements.delete -NULL - -#' user_agreements.signatures -#' -#' user_agreements.signatures is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.signatures(NULL) -#' @return UserAgreement object. -#' @name user_agreements.signatures -NULL - -#' user_agreements.sign -#' -#' user_agreements.sign is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.sign(NULL) -#' @return UserAgreement object. -#' @name user_agreements.sign -NULL - -#' user_agreements.list -#' -#' user_agreements.list is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.list(filters = NULL, -#' where = NULL, order = NULL, select = NULL, -#' distinct = NULL, limit = "100", offset = "0", -#' count = "exact") -#' @param filters -#' @param where -#' @param order -#' @param select -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @return UserAgreementList object. -#' @name user_agreements.list -NULL - -#' user_agreements.new -#' -#' user_agreements.new is a method defined in Arvados class. -#' -#' @usage arv$user_agreements.new(NULL) -#' @return UserAgreement object. -#' @name user_agreements.new -NULL - -#' project.get -#' -#' projects.get is equivalent to groups.get method. -#' -#' @usage arv$projects.get(uuid) -#' @param uuid The UUID of the Group in question. -#' @return Group object. -#' @name projects.get -NULL - -#' project.create -#' -#' projects.create wrapps groups.create method by setting group_class attribute to "project". -#' -#' @usage arv$projects.create(group, ensure_unique_name = "false") -#' @param group Group object. -#' @param ensure_unique_name Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision. -#' @return Group object. -#' @name projects.create -NULL - -#' project.update -#' -#' projects.update wrapps groups.update method by setting group_class attribute to "project". -#' -#' @usage arv$projects.update(group, uuid) -#' @param group Group object. -#' @param uuid The UUID of the Group in question. -#' @return Group object. -#' @name projects.update -NULL - -#' project.delete -#' -#' projects.delete is equivalent to groups.delete method. -#' -#' @usage arv$project.delete(uuid) -#' @param uuid The UUID of the Group in question. -#' @return Group object. -#' @name projects.delete -NULL - -#' project.list -#' -#' projects.list wrapps groups.list method by setting group_class attribute to "project". -#' -#' @usage arv$projects.list(filters = NULL, -#' where = NULL, order = NULL, distinct = NULL, -#' limit = "100", offset = "0", count = "exact", -#' include_trash = NULL, uuid = NULL, recursive = NULL) -#' @param filters -#' @param where -#' @param order -#' @param distinct -#' @param limit -#' @param offset -#' @param count -#' @param include_trash Include items whose is_trashed attribute is true. -#' @param uuid -#' @param recursive Include contents from child groups recursively. -#' @return Group object. -#' @name projects.list -NULL - -#' Arvados -#' -#' Arvados class gives users ability to access Arvados REST API. -#' -#' @section Usage: -#' \preformatted{arv = Arvados$new(authToken = NULL, hostName = NULL, numRetries = 0)} -#' -#' @section Arguments: -#' \describe{ -#' \item{authToken}{Authentification token. If not specified ARVADOS_API_TOKEN environment variable will be used.} -#' \item{hostName}{Host name. If not specified ARVADOS_API_HOST environment variable will be used.} -#' \item{numRetries}{Number which specifies how many times to retry failed service requests.} -#' } -#' -#' @section Methods: -#' \describe{ -#' \item{}{\code{\link{api_client_authorizations.create}}} -#' \item{}{\code{\link{api_client_authorizations.create_system_auth}}} -#' \item{}{\code{\link{api_client_authorizations.current}}} -#' \item{}{\code{\link{api_client_authorizations.delete}}} -#' \item{}{\code{\link{api_client_authorizations.get}}} -#' \item{}{\code{\link{api_client_authorizations.list}}} -#' \item{}{\code{\link{api_client_authorizations.update}}} -#' \item{}{\code{\link{api_clients.create}}} -#' \item{}{\code{\link{api_clients.delete}}} -#' \item{}{\code{\link{api_clients.get}}} -#' \item{}{\code{\link{api_clients.list}}} -#' \item{}{\code{\link{api_clients.update}}} -#' \item{}{\code{\link{authorized_keys.create}}} -#' \item{}{\code{\link{authorized_keys.delete}}} -#' \item{}{\code{\link{authorized_keys.get}}} -#' \item{}{\code{\link{authorized_keys.list}}} -#' \item{}{\code{\link{authorized_keys.update}}} -#' \item{}{\code{\link{collections.create}}} -#' \item{}{\code{\link{collections.delete}}} -#' \item{}{\code{\link{collections.get}}} -#' \item{}{\code{\link{collections.list}}} -#' \item{}{\code{\link{collections.provenance}}} -#' \item{}{\code{\link{collections.trash}}} -#' \item{}{\code{\link{collections.untrash}}} -#' \item{}{\code{\link{collections.update}}} -#' \item{}{\code{\link{collections.used_by}}} -#' \item{}{\code{\link{container_requests.create}}} -#' \item{}{\code{\link{container_requests.delete}}} -#' \item{}{\code{\link{container_requests.get}}} -#' \item{}{\code{\link{container_requests.list}}} -#' \item{}{\code{\link{container_requests.update}}} -#' \item{}{\code{\link{containers.auth}}} -#' \item{}{\code{\link{containers.create}}} -#' \item{}{\code{\link{containers.current}}} -#' \item{}{\code{\link{containers.delete}}} -#' \item{}{\code{\link{containers.get}}} -#' \item{}{\code{\link{containers.list}}} -#' \item{}{\code{\link{containers.lock}}} -#' \item{}{\code{\link{containers.secret_mounts}}} -#' \item{}{\code{\link{containers.unlock}}} -#' \item{}{\code{\link{containers.update}}} -#' \item{}{\code{\link{groups.contents}}} -#' \item{}{\code{\link{groups.create}}} -#' \item{}{\code{\link{groups.delete}}} -#' \item{}{\code{\link{groups.get}}} -#' \item{}{\code{\link{groups.list}}} -#' \item{}{\code{\link{groups.trash}}} -#' \item{}{\code{\link{groups.untrash}}} -#' \item{}{\code{\link{groups.update}}} -#' \item{}{\code{\link{humans.create}}} -#' \item{}{\code{\link{humans.delete}}} -#' \item{}{\code{\link{humans.get}}} -#' \item{}{\code{\link{humans.list}}} -#' \item{}{\code{\link{humans.update}}} -#' \item{}{\code{\link{jobs.cancel}}} -#' \item{}{\code{\link{jobs.create}}} -#' \item{}{\code{\link{jobs.delete}}} -#' \item{}{\code{\link{jobs.get}}} -#' \item{}{\code{\link{jobs.list}}} -#' \item{}{\code{\link{jobs.lock}}} -#' \item{}{\code{\link{jobs.queue}}} -#' \item{}{\code{\link{jobs.queue_size}}} -#' \item{}{\code{\link{jobs.update}}} -#' \item{}{\code{\link{job_tasks.create}}} -#' \item{}{\code{\link{job_tasks.delete}}} -#' \item{}{\code{\link{job_tasks.get}}} -#' \item{}{\code{\link{job_tasks.list}}} -#' \item{}{\code{\link{job_tasks.update}}} -#' \item{}{\code{\link{keep_disks.create}}} -#' \item{}{\code{\link{keep_disks.delete}}} -#' \item{}{\code{\link{keep_disks.get}}} -#' \item{}{\code{\link{keep_disks.list}}} -#' \item{}{\code{\link{keep_disks.ping}}} -#' \item{}{\code{\link{keep_disks.update}}} -#' \item{}{\code{\link{keep_services.accessible}}} -#' \item{}{\code{\link{keep_services.create}}} -#' \item{}{\code{\link{keep_services.delete}}} -#' \item{}{\code{\link{keep_services.get}}} -#' \item{}{\code{\link{keep_services.list}}} -#' \item{}{\code{\link{keep_services.update}}} -#' \item{}{\code{\link{links.create}}} -#' \item{}{\code{\link{links.delete}}} -#' \item{}{\code{\link{links.get}}} -#' \item{}{\code{\link{links.get_permissions}}} -#' \item{}{\code{\link{links.list}}} -#' \item{}{\code{\link{links.update}}} -#' \item{}{\code{\link{logs.create}}} -#' \item{}{\code{\link{logs.delete}}} -#' \item{}{\code{\link{logs.get}}} -#' \item{}{\code{\link{logs.list}}} -#' \item{}{\code{\link{logs.update}}} -#' \item{}{\code{\link{nodes.create}}} -#' \item{}{\code{\link{nodes.delete}}} -#' \item{}{\code{\link{nodes.get}}} -#' \item{}{\code{\link{nodes.list}}} -#' \item{}{\code{\link{nodes.ping}}} -#' \item{}{\code{\link{nodes.update}}} -#' \item{}{\code{\link{pipeline_instances.cancel}}} -#' \item{}{\code{\link{pipeline_instances.create}}} -#' \item{}{\code{\link{pipeline_instances.delete}}} -#' \item{}{\code{\link{pipeline_instances.get}}} -#' \item{}{\code{\link{pipeline_instances.list}}} -#' \item{}{\code{\link{pipeline_instances.update}}} -#' \item{}{\code{\link{pipeline_templates.create}}} -#' \item{}{\code{\link{pipeline_templates.delete}}} -#' \item{}{\code{\link{pipeline_templates.get}}} -#' \item{}{\code{\link{pipeline_templates.list}}} -#' \item{}{\code{\link{pipeline_templates.update}}} -#' \item{}{\code{\link{projects.create}}} -#' \item{}{\code{\link{projects.delete}}} -#' \item{}{\code{\link{projects.get}}} -#' \item{}{\code{\link{projects.list}}} -#' \item{}{\code{\link{projects.update}}} -#' \item{}{\code{\link{repositories.create}}} -#' \item{}{\code{\link{repositories.delete}}} -#' \item{}{\code{\link{repositories.get}}} -#' \item{}{\code{\link{repositories.get_all_permissions}}} -#' \item{}{\code{\link{repositories.list}}} -#' \item{}{\code{\link{repositories.update}}} -#' \item{}{\code{\link{specimens.create}}} -#' \item{}{\code{\link{specimens.delete}}} -#' \item{}{\code{\link{specimens.get}}} -#' \item{}{\code{\link{specimens.list}}} -#' \item{}{\code{\link{specimens.update}}} -#' \item{}{\code{\link{traits.create}}} -#' \item{}{\code{\link{traits.delete}}} -#' \item{}{\code{\link{traits.get}}} -#' \item{}{\code{\link{traits.list}}} -#' \item{}{\code{\link{traits.update}}} -#' \item{}{\code{\link{user_agreements.create}}} -#' \item{}{\code{\link{user_agreements.delete}}} -#' \item{}{\code{\link{user_agreements.get}}} -#' \item{}{\code{\link{user_agreements.list}}} -#' \item{}{\code{\link{user_agreements.new}}} -#' \item{}{\code{\link{user_agreements.sign}}} -#' \item{}{\code{\link{user_agreements.signatures}}} -#' \item{}{\code{\link{user_agreements.update}}} -#' \item{}{\code{\link{users.activate}}} -#' \item{}{\code{\link{users.create}}} -#' \item{}{\code{\link{users.current}}} -#' \item{}{\code{\link{users.delete}}} -#' \item{}{\code{\link{users.get}}} -#' \item{}{\code{\link{users.list}}} -#' \item{}{\code{\link{users.merge}}} -#' \item{}{\code{\link{users.setup}}} -#' \item{}{\code{\link{users.system}}} -#' \item{}{\code{\link{users.unsetup}}} -#' \item{}{\code{\link{users.update}}} -#' \item{}{\code{\link{users.update_uuid}}} -#' \item{}{\code{\link{virtual_machines.create}}} -#' \item{}{\code{\link{virtual_machines.delete}}} -#' \item{}{\code{\link{virtual_machines.get}}} -#' \item{}{\code{\link{virtual_machines.get_all_logins}}} -#' \item{}{\code{\link{virtual_machines.list}}} -#' \item{}{\code{\link{virtual_machines.logins}}} -#' \item{}{\code{\link{virtual_machines.update}}} -#' \item{}{\code{\link{workflows.create}}} -#' \item{}{\code{\link{workflows.delete}}} -#' \item{}{\code{\link{workflows.get}}} -#' \item{}{\code{\link{workflows.list}}} -#' \item{}{\code{\link{workflows.update}}} -#' } -#' -#' @name Arvados -#' @examples -#' \dontrun{ -#' arv <- Arvados$new("your Arvados token", "example.arvadosapi.com") -#' -#' collection <- arv$collections.get("uuid") -#' -#' collectionList <- arv$collections.list(list(list("name", "like", "Test%"))) -#' collectionList <- listAll(arv$collections.list, list(list("name", "like", "Test%"))) -#' -#' deletedCollection <- arv$collections.delete("uuid") -#' -#' updatedCollection <- arv$collections.update(list(name = "New name", description = "New description"), -#' "uuid") -#' -#' createdCollection <- arv$collections.create(list(name = "Example", -#' description = "This is a test collection")) -#' } -NULL - -#' @export -Arvados <- R6::R6Class( - - "Arvados", - - public = list( - - initialize = function(authToken = NULL, hostName = NULL, numRetries = 0) - { - if(!is.null(hostName)) - Sys.setenv(ARVADOS_API_HOST = hostName) - - if(!is.null(authToken)) - Sys.setenv(ARVADOS_API_TOKEN = authToken) - - hostName <- Sys.getenv("ARVADOS_API_HOST") - token <- Sys.getenv("ARVADOS_API_TOKEN") - - if(hostName == "" | token == "") - stop(paste("Please provide host name and authentification token", - "or set ARVADOS_API_HOST and ARVADOS_API_TOKEN", - "environment variables.")) - - private$token <- token - private$host <- paste0("https://", hostName, "/arvados/v1/") - private$numRetries <- numRetries - private$REST <- RESTService$new(token, hostName, - HttpRequest$new(), HttpParser$new(), - numRetries) - - }, - - projects.get = function(uuid) - { - self$groups.get(uuid) - }, - - projects.create = function(group, ensure_unique_name = "false") - { - group <- c("group_class" = "project", group) - self$groups.create(group, ensure_unique_name) - }, - - projects.update = function(group, uuid) - { - group <- c("group_class" = "project", group) - self$groups.update(group, uuid) - }, - - projects.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact", - include_trash = NULL) - { - filters[[length(filters) + 1]] <- list("group_class", "=", "project") - self$groups.list(filters, where, order, select, distinct, - limit, offset, count, include_trash) - }, - - projects.delete = function(uuid) - { - self$groups.delete(uuid) - }, - - users.get = function(uuid) - { - endPoint <- stringr::str_interp("users/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.create = function(user, ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("users") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(user) > 0) - body <- jsonlite::toJSON(list(user = user), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.update = function(user, uuid) - { - endPoint <- stringr::str_interp("users/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(user) > 0) - body <- jsonlite::toJSON(list(user = user), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.delete = function(uuid) - { - endPoint <- stringr::str_interp("users/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("DELETE", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.current = function() - { - endPoint <- stringr::str_interp("users/current") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.system = function() - { - endPoint <- stringr::str_interp("users/system") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.activate = function(uuid) - { - endPoint <- stringr::str_interp("users/${uuid}/activate") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.setup = function(user = NULL, openid_prefix = NULL, - repo_name = NULL, vm_uuid = NULL, send_notification_email = "false") - { - endPoint <- stringr::str_interp("users/setup") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(user = user, openid_prefix = openid_prefix, - repo_name = repo_name, vm_uuid = vm_uuid, - send_notification_email = send_notification_email) - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.unsetup = function(uuid) - { - endPoint <- stringr::str_interp("users/${uuid}/unsetup") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.update_uuid = function(uuid, new_uuid) - { - endPoint <- stringr::str_interp("users/${uuid}/update_uuid") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(new_uuid = new_uuid) - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.merge = function(new_owner_uuid, new_user_token, - redirect_to_new_user = NULL) - { - endPoint <- stringr::str_interp("users/merge") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(new_owner_uuid = new_owner_uuid, - new_user_token = new_user_token, redirect_to_new_user = redirect_to_new_user) - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - users.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") - { - endPoint <- stringr::str_interp("users") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.get = function(uuid) - { - endPoint <- stringr::str_interp("api_client_authorizations/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.create = function(apiclientauthorization, - ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("api_client_authorizations") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(apiclientauthorization) > 0) - body <- jsonlite::toJSON(list(apiclientauthorization = apiclientauthorization), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.update = function(apiclientauthorization, uuid) - { - endPoint <- stringr::str_interp("api_client_authorizations/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(apiclientauthorization) > 0) - body <- jsonlite::toJSON(list(apiclientauthorization = apiclientauthorization), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.delete = function(uuid) - { - endPoint <- stringr::str_interp("api_client_authorizations/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("DELETE", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.create_system_auth = function(api_client_id = NULL, scopes = NULL) - { - endPoint <- stringr::str_interp("api_client_authorizations/create_system_auth") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(api_client_id = api_client_id, - scopes = scopes) - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.current = function() - { - endPoint <- stringr::str_interp("api_client_authorizations/current") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_client_authorizations.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") - { - endPoint <- stringr::str_interp("api_client_authorizations") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.get = function(uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.create = function(container, ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("containers") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(container) > 0) - body <- jsonlite::toJSON(list(container = container), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.update = function(container, uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(container) > 0) - body <- jsonlite::toJSON(list(container = container), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.delete = function(uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("DELETE", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.auth = function(uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}/auth") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.lock = function(uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}/lock") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.unlock = function(uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}/unlock") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.secret_mounts = function(uuid) - { - endPoint <- stringr::str_interp("containers/${uuid}/secret_mounts") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.current = function() - { - endPoint <- stringr::str_interp("containers/current") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - containers.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") - { - endPoint <- stringr::str_interp("containers") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_clients.get = function(uuid) - { - endPoint <- stringr::str_interp("api_clients/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_clients.create = function(apiclient, ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("api_clients") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(apiclient) > 0) - body <- jsonlite::toJSON(list(apiclient = apiclient), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_clients.update = function(apiclient, uuid) - { - endPoint <- stringr::str_interp("api_clients/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(apiclient) > 0) - body <- jsonlite::toJSON(list(apiclient = apiclient), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - api_clients.delete = function(uuid) - { - endPoint <- stringr::str_interp("api_clients/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("DELETE", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, +#' Arvados +#' +#' Arvados class gives users ability to access Arvados REST API. +#' +#' @section Usage: +#' \preformatted{arv = Arvados$new(authToken = NULL, hostName = NULL, numRetries = 0)} +#' +#' @section Arguments: +#' \describe{ +#' \item{authToken}{Authentification token. If not specified ARVADOS_API_TOKEN environment variable will be used.} +#' \item{hostName}{Host name. If not specified ARVADOS_API_HOST environment variable will be used.} +#' \item{numRetries}{Number which specifies how many times to retry failed service requests.} +#' } +#' +#' @section Methods: +#' \describe{ +#' \item{}{\code{\link{api_client_authorizations.create}}} +#' \item{}{\code{\link{api_client_authorizations.create_system_auth}}} +#' \item{}{\code{\link{api_client_authorizations.current}}} +#' \item{}{\code{\link{api_client_authorizations.delete}}} +#' \item{}{\code{\link{api_client_authorizations.get}}} +#' \item{}{\code{\link{api_client_authorizations.list}}} +#' \item{}{\code{\link{api_client_authorizations.update}}} +#' \item{}{\code{\link{api_clients.create}}} +#' \item{}{\code{\link{api_clients.delete}}} +#' \item{}{\code{\link{api_clients.get}}} +#' \item{}{\code{\link{api_clients.list}}} +#' \item{}{\code{\link{api_clients.update}}} +#' \item{}{\code{\link{authorized_keys.create}}} +#' \item{}{\code{\link{authorized_keys.delete}}} +#' \item{}{\code{\link{authorized_keys.get}}} +#' \item{}{\code{\link{authorized_keys.list}}} +#' \item{}{\code{\link{authorized_keys.update}}} +#' \item{}{\code{\link{collections.create}}} +#' \item{}{\code{\link{collections.delete}}} +#' \item{}{\code{\link{collections.get}}} +#' \item{}{\code{\link{collections.list}}} +#' \item{}{\code{\link{collections.provenance}}} +#' \item{}{\code{\link{collections.trash}}} +#' \item{}{\code{\link{collections.untrash}}} +#' \item{}{\code{\link{collections.update}}} +#' \item{}{\code{\link{collections.used_by}}} +#' \item{}{\code{\link{configs.get}}} +#' \item{}{\code{\link{container_requests.create}}} +#' \item{}{\code{\link{container_requests.delete}}} +#' \item{}{\code{\link{container_requests.get}}} +#' \item{}{\code{\link{container_requests.list}}} +#' \item{}{\code{\link{container_requests.update}}} +#' \item{}{\code{\link{containers.auth}}} +#' \item{}{\code{\link{containers.create}}} +#' \item{}{\code{\link{containers.current}}} +#' \item{}{\code{\link{containers.delete}}} +#' \item{}{\code{\link{containers.get}}} +#' \item{}{\code{\link{containers.list}}} +#' \item{}{\code{\link{containers.lock}}} +#' \item{}{\code{\link{containers.secret_mounts}}} +#' \item{}{\code{\link{containers.unlock}}} +#' \item{}{\code{\link{containers.update}}} +#' \item{}{\code{\link{groups.contents}}} +#' \item{}{\code{\link{groups.create}}} +#' \item{}{\code{\link{groups.delete}}} +#' \item{}{\code{\link{groups.get}}} +#' \item{}{\code{\link{groups.list}}} +#' \item{}{\code{\link{groups.shared}}} +#' \item{}{\code{\link{groups.trash}}} +#' \item{}{\code{\link{groups.untrash}}} +#' \item{}{\code{\link{groups.update}}} +#' \item{}{\code{\link{keep_services.accessible}}} +#' \item{}{\code{\link{keep_services.create}}} +#' \item{}{\code{\link{keep_services.delete}}} +#' \item{}{\code{\link{keep_services.get}}} +#' \item{}{\code{\link{keep_services.list}}} +#' \item{}{\code{\link{keep_services.update}}} +#' \item{}{\code{\link{links.create}}} +#' \item{}{\code{\link{links.delete}}} +#' \item{}{\code{\link{links.get}}} +#' \item{}{\code{\link{links.get_permissions}}} +#' \item{}{\code{\link{links.list}}} +#' \item{}{\code{\link{links.update}}} +#' \item{}{\code{\link{logs.create}}} +#' \item{}{\code{\link{logs.delete}}} +#' \item{}{\code{\link{logs.get}}} +#' \item{}{\code{\link{logs.list}}} +#' \item{}{\code{\link{logs.update}}} +#' \item{}{\code{\link{projects.create}}} +#' \item{}{\code{\link{projects.delete}}} +#' \item{}{\code{\link{projects.get}}} +#' \item{}{\code{\link{projects.list}}} +#' \item{}{\code{\link{projects.update}}} +#' \item{}{\code{\link{repositories.create}}} +#' \item{}{\code{\link{repositories.delete}}} +#' \item{}{\code{\link{repositories.get}}} +#' \item{}{\code{\link{repositories.get_all_permissions}}} +#' \item{}{\code{\link{repositories.list}}} +#' \item{}{\code{\link{repositories.update}}} +#' \item{}{\code{\link{user_agreements.create}}} +#' \item{}{\code{\link{user_agreements.delete}}} +#' \item{}{\code{\link{user_agreements.get}}} +#' \item{}{\code{\link{user_agreements.list}}} +#' \item{}{\code{\link{user_agreements.new}}} +#' \item{}{\code{\link{user_agreements.sign}}} +#' \item{}{\code{\link{user_agreements.signatures}}} +#' \item{}{\code{\link{user_agreements.update}}} +#' \item{}{\code{\link{users.activate}}} +#' \item{}{\code{\link{users.create}}} +#' \item{}{\code{\link{users.current}}} +#' \item{}{\code{\link{users.delete}}} +#' \item{}{\code{\link{users.get}}} +#' \item{}{\code{\link{users.list}}} +#' \item{}{\code{\link{users.merge}}} +#' \item{}{\code{\link{users.setup}}} +#' \item{}{\code{\link{users.system}}} +#' \item{}{\code{\link{users.unsetup}}} +#' \item{}{\code{\link{users.update}}} +#' \item{}{\code{\link{users.update_uuid}}} +#' \item{}{\code{\link{virtual_machines.create}}} +#' \item{}{\code{\link{virtual_machines.delete}}} +#' \item{}{\code{\link{virtual_machines.get}}} +#' \item{}{\code{\link{virtual_machines.get_all_logins}}} +#' \item{}{\code{\link{virtual_machines.list}}} +#' \item{}{\code{\link{virtual_machines.logins}}} +#' \item{}{\code{\link{virtual_machines.update}}} +#' \item{}{\code{\link{workflows.create}}} +#' \item{}{\code{\link{workflows.delete}}} +#' \item{}{\code{\link{workflows.get}}} +#' \item{}{\code{\link{workflows.list}}} +#' \item{}{\code{\link{workflows.update}}} +#' } +#' +#' @name Arvados +#' @examples +#' \dontrun{ +#' arv <- Arvados$new("your Arvados token", "example.arvadosapi.com") +#' +#' collection <- arv$collections.get("uuid") +#' +#' collectionList <- arv$collections.list(list(list("name", "like", "Test%"))) +#' collectionList <- listAll(arv$collections.list, list(list("name", "like", "Test%"))) +#' +#' deletedCollection <- arv$collections.delete("uuid") +#' +#' updatedCollection <- arv$collections.update(list(name = "New name", description = "New description"), +#' "uuid") +#' +#' createdCollection <- arv$collections.create(list(name = "Example", +#' description = "This is a test collection")) +#' } +NULL - api_clients.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") - { - endPoint <- stringr::str_interp("api_clients") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, +#' @export +Arvados <- R6::R6Class( - container_requests.get = function(uuid) - { - endPoint <- stringr::str_interp("container_requests/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, + "Arvados", - container_requests.create = function(containerrequest, - ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("container_requests") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(containerrequest) > 0) - body <- jsonlite::toJSON(list(containerrequest = containerrequest), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, + public = list( - container_requests.update = function(containerrequest, uuid) + initialize = function(authToken = NULL, hostName = NULL, numRetries = 0) { - endPoint <- stringr::str_interp("container_requests/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(containerrequest) > 0) - body <- jsonlite::toJSON(list(containerrequest = containerrequest), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, + if(!is.null(hostName)) + Sys.setenv(ARVADOS_API_HOST = hostName) - container_requests.delete = function(uuid) - { - endPoint <- stringr::str_interp("container_requests/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("DELETE", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, + if(!is.null(authToken)) + Sys.setenv(ARVADOS_API_TOKEN = authToken) + + hostName <- Sys.getenv("ARVADOS_API_HOST") + token <- Sys.getenv("ARVADOS_API_TOKEN") + + if(hostName == "" | token == "") + stop(paste("Please provide host name and authentification token", + "or set ARVADOS_API_HOST and ARVADOS_API_TOKEN", + "environment variables.")) + + private$token <- token + private$host <- paste0("https://", hostName, "/arvados/v1/") + private$numRetries <- numRetries + private$REST <- RESTService$new(token, hostName, + HttpRequest$new(), HttpParser$new(), + numRetries) - container_requests.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") - { - endPoint <- stringr::str_interp("container_requests") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource }, - authorized_keys.get = function(uuid) + projects.get = function(uuid) { - endPoint <- stringr::str_interp("authorized_keys/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource + self$groups.get(uuid) }, - authorized_keys.create = function(authorizedkey, - ensure_unique_name = "false") + projects.create = function(group, ensure_unique_name = "false") { - endPoint <- stringr::str_interp("authorized_keys") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(authorizedkey) > 0) - body <- jsonlite::toJSON(list(authorizedkey = authorizedkey), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource + group <- c("group_class" = "project", group) + self$groups.create(group, ensure_unique_name) }, - authorized_keys.update = function(authorizedkey, uuid) + projects.update = function(group, uuid) { - endPoint <- stringr::str_interp("authorized_keys/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(authorizedkey) > 0) - body <- jsonlite::toJSON(list(authorizedkey = authorizedkey), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource + group <- c("group_class" = "project", group) + self$groups.update(group, uuid) }, - authorized_keys.delete = function(uuid) + projects.list = function(filters = NULL, where = NULL, + order = NULL, select = NULL, distinct = NULL, + limit = "100", offset = "0", count = "exact", + include_trash = NULL) { - endPoint <- stringr::str_interp("authorized_keys/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - body <- NULL - - response <- private$REST$http$exec("DELETE", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource + filters[[length(filters) + 1]] <- list("group_class", "=", "project") + self$groups.list(filters, where, order, select, distinct, + limit, offset, count, include_trash) }, - authorized_keys.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") - { - endPoint <- stringr::str_interp("authorized_keys") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource + projects.delete = function(uuid) + { + self$groups.delete(uuid) }, - collections.get = function(uuid) + api_clients.get = function(uuid) { - endPoint <- stringr::str_interp("collections/${uuid}") + endPoint <- stringr::str_interp("api_clients/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3229,16 +1656,18 @@ Arvados <- R6::R6Class( resource }, - collections.create = function(collection, ensure_unique_name = "false") + api_clients.create = function(apiclient, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("collections") + endPoint <- stringr::str_interp("api_clients") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(collection) > 0) - body <- jsonlite::toJSON(list(collection = collection), + if(length(apiclient) > 0) + body <- jsonlite::toJSON(list(apiclient = apiclient), auto_unbox = TRUE) else body <- NULL @@ -3253,16 +1682,16 @@ Arvados <- R6::R6Class( resource }, - collections.update = function(collection, uuid) + api_clients.update = function(apiclient, uuid) { - endPoint <- stringr::str_interp("collections/${uuid}") + endPoint <- stringr::str_interp("api_clients/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(collection) > 0) - body <- jsonlite::toJSON(list(collection = collection), + if(length(apiclient) > 0) + body <- jsonlite::toJSON(list(apiclient = apiclient), auto_unbox = TRUE) else body <- NULL @@ -3277,11 +1706,11 @@ Arvados <- R6::R6Class( resource }, - collections.delete = function(uuid) + api_clients.delete = function(uuid) { - endPoint <- stringr::str_interp("collections/${uuid}") + endPoint <- stringr::str_interp("api_clients/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3297,13 +1726,19 @@ Arvados <- R6::R6Class( resource }, - collections.provenance = function(uuid) + api_clients.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("collections/${uuid}/provenance") + endPoint <- stringr::str_interp("api_clients") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(filters = filters, where = where, + order = order, select = select, distinct = distinct, + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -3317,11 +1752,11 @@ Arvados <- R6::R6Class( resource }, - collections.used_by = function(uuid) + api_client_authorizations.get = function(uuid) { - endPoint <- stringr::str_interp("collections/${uuid}/used_by") + endPoint <- stringr::str_interp("api_client_authorizations/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3337,15 +1772,21 @@ Arvados <- R6::R6Class( resource }, - collections.trash = function(uuid) + api_client_authorizations.create = function(apiclientauthorization, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("collections/${uuid}/trash") + endPoint <- stringr::str_interp("api_client_authorizations") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - body <- NULL + if(length(apiclientauthorization) > 0) + body <- jsonlite::toJSON(list(apiclientauthorization = apiclientauthorization), + auto_unbox = TRUE) + else + body <- NULL response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) @@ -3357,43 +1798,21 @@ Arvados <- R6::R6Class( resource }, - collections.untrash = function(uuid) + api_client_authorizations.update = function(apiclientauthorization, uuid) { - endPoint <- stringr::str_interp("collections/${uuid}/untrash") + endPoint <- stringr::str_interp("api_client_authorizations/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - collections.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact", include_trash = NULL) - { - endPoint <- stringr::str_interp("collections") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count, - include_trash = include_trash) - - body <- NULL + if(length(apiclientauthorization) > 0) + body <- jsonlite::toJSON(list(apiclientauthorization = apiclientauthorization), + auto_unbox = TRUE) + else + body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, + response <- private$REST$http$exec("PUT", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -3403,17 +1822,17 @@ Arvados <- R6::R6Class( resource }, - humans.get = function(uuid) + api_client_authorizations.delete = function(uuid) { - endPoint <- stringr::str_interp("humans/${uuid}") + endPoint <- stringr::str_interp("api_client_authorizations/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, + response <- private$REST$http$exec("DELETE", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -3423,19 +1842,16 @@ Arvados <- R6::R6Class( resource }, - humans.create = function(human, ensure_unique_name = "false") + api_client_authorizations.create_system_auth = function(api_client_id = NULL, scopes = NULL) { - endPoint <- stringr::str_interp("humans") + endPoint <- stringr::str_interp("api_client_authorizations/create_system_auth") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(api_client_id = api_client_id, + scopes = scopes) - if(length(human) > 0) - body <- jsonlite::toJSON(list(human = human), - auto_unbox = TRUE) - else - body <- NULL + body <- NULL response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) @@ -3447,41 +1863,17 @@ Arvados <- R6::R6Class( resource }, - humans.update = function(human, uuid) - { - endPoint <- stringr::str_interp("humans/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(human) > 0) - body <- jsonlite::toJSON(list(human = human), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - humans.delete = function(uuid) + api_client_authorizations.current = function() { - endPoint <- stringr::str_interp("humans/${uuid}") + endPoint <- stringr::str_interp("api_client_authorizations/current") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -3491,17 +1883,19 @@ Arvados <- R6::R6Class( resource }, - humans.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + api_client_authorizations.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("humans") + endPoint <- stringr::str_interp("api_client_authorizations") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -3515,11 +1909,11 @@ Arvados <- R6::R6Class( resource }, - job_tasks.get = function(uuid) + authorized_keys.get = function(uuid) { - endPoint <- stringr::str_interp("job_tasks/${uuid}") + endPoint <- stringr::str_interp("authorized_keys/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3535,16 +1929,18 @@ Arvados <- R6::R6Class( resource }, - job_tasks.create = function(jobtask, ensure_unique_name = "false") + authorized_keys.create = function(authorizedkey, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("job_tasks") + endPoint <- stringr::str_interp("authorized_keys") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(jobtask) > 0) - body <- jsonlite::toJSON(list(jobtask = jobtask), + if(length(authorizedkey) > 0) + body <- jsonlite::toJSON(list(authorizedkey = authorizedkey), auto_unbox = TRUE) else body <- NULL @@ -3559,16 +1955,16 @@ Arvados <- R6::R6Class( resource }, - job_tasks.update = function(jobtask, uuid) + authorized_keys.update = function(authorizedkey, uuid) { - endPoint <- stringr::str_interp("job_tasks/${uuid}") + endPoint <- stringr::str_interp("authorized_keys/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(jobtask) > 0) - body <- jsonlite::toJSON(list(jobtask = jobtask), + if(length(authorizedkey) > 0) + body <- jsonlite::toJSON(list(authorizedkey = authorizedkey), auto_unbox = TRUE) else body <- NULL @@ -3583,11 +1979,11 @@ Arvados <- R6::R6Class( resource }, - job_tasks.delete = function(uuid) + authorized_keys.delete = function(uuid) { - endPoint <- stringr::str_interp("job_tasks/${uuid}") + endPoint <- stringr::str_interp("authorized_keys/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3603,18 +1999,19 @@ Arvados <- R6::R6Class( resource }, - job_tasks.list = function(filters = NULL, + authorized_keys.list = function(filters = NULL, where = NULL, order = NULL, select = NULL, distinct = NULL, limit = "100", offset = "0", - count = "exact") + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("job_tasks") + endPoint <- stringr::str_interp("authorized_keys") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -3628,11 +2025,11 @@ Arvados <- R6::R6Class( resource }, - jobs.get = function(uuid) + collections.get = function(uuid) { - endPoint <- stringr::str_interp("jobs/${uuid}") + endPoint <- stringr::str_interp("collections/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3648,21 +2045,18 @@ Arvados <- R6::R6Class( resource }, - jobs.create = function(job, ensure_unique_name = "false", - find_or_create = "false", filters = NULL, - minimum_script_version = NULL, exclude_script_versions = NULL) + collections.create = function(collection, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("jobs") + endPoint <- stringr::str_interp("collections") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(ensure_unique_name = ensure_unique_name, - find_or_create = find_or_create, filters = filters, - minimum_script_version = minimum_script_version, - exclude_script_versions = exclude_script_versions) + cluster_id = cluster_id) - if(length(job) > 0) - body <- jsonlite::toJSON(list(job = job), + if(length(collection) > 0) + body <- jsonlite::toJSON(list(collection = collection), auto_unbox = TRUE) else body <- NULL @@ -3677,16 +2071,16 @@ Arvados <- R6::R6Class( resource }, - jobs.update = function(job, uuid) + collections.update = function(collection, uuid) { - endPoint <- stringr::str_interp("jobs/${uuid}") + endPoint <- stringr::str_interp("collections/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(job) > 0) - body <- jsonlite::toJSON(list(job = job), + if(length(collection) > 0) + body <- jsonlite::toJSON(list(collection = collection), auto_unbox = TRUE) else body <- NULL @@ -3701,11 +2095,11 @@ Arvados <- R6::R6Class( resource }, - jobs.delete = function(uuid) + collections.delete = function(uuid) { - endPoint <- stringr::str_interp("jobs/${uuid}") + endPoint <- stringr::str_interp("collections/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3721,17 +2115,13 @@ Arvados <- R6::R6Class( resource }, - jobs.queue = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + collections.provenance = function(uuid) { - endPoint <- stringr::str_interp("jobs/queue") + endPoint <- stringr::str_interp("collections/${uuid}/provenance") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + queryArgs <- NULL body <- NULL @@ -3745,11 +2135,11 @@ Arvados <- R6::R6Class( resource }, - jobs.queue_size = function() + collections.used_by = function(uuid) { - endPoint <- stringr::str_interp("jobs/queue_size") + endPoint <- stringr::str_interp("collections/${uuid}/used_by") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3765,11 +2155,11 @@ Arvados <- R6::R6Class( resource }, - jobs.cancel = function(uuid) + collections.trash = function(uuid) { - endPoint <- stringr::str_interp("jobs/${uuid}/cancel") + endPoint <- stringr::str_interp("collections/${uuid}/trash") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3785,11 +2175,11 @@ Arvados <- R6::R6Class( resource }, - jobs.lock = function(uuid) + collections.untrash = function(uuid) { - endPoint <- stringr::str_interp("jobs/${uuid}/lock") + endPoint <- stringr::str_interp("collections/${uuid}/untrash") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3805,17 +2195,21 @@ Arvados <- R6::R6Class( resource }, - jobs.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + collections.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL, + include_trash = NULL, include_old_versions = NULL) { - endPoint <- stringr::str_interp("jobs") + endPoint <- stringr::str_interp("collections") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation, + include_trash = include_trash, include_old_versions = include_old_versions) body <- NULL @@ -3829,11 +2223,11 @@ Arvados <- R6::R6Class( resource }, - keep_disks.get = function(uuid) + containers.get = function(uuid) { - endPoint <- stringr::str_interp("keep_disks/${uuid}") + endPoint <- stringr::str_interp("containers/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3849,16 +2243,18 @@ Arvados <- R6::R6Class( resource }, - keep_disks.create = function(keepdisk, ensure_unique_name = "false") + containers.create = function(container, ensure_unique_name = "false", + cluster_id = NULL) { - endPoint <- stringr::str_interp("keep_disks") + endPoint <- stringr::str_interp("containers") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(keepdisk) > 0) - body <- jsonlite::toJSON(list(keepdisk = keepdisk), + if(length(container) > 0) + body <- jsonlite::toJSON(list(container = container), auto_unbox = TRUE) else body <- NULL @@ -3873,16 +2269,16 @@ Arvados <- R6::R6Class( resource }, - keep_disks.update = function(keepdisk, uuid) + containers.update = function(container, uuid) { - endPoint <- stringr::str_interp("keep_disks/${uuid}") + endPoint <- stringr::str_interp("containers/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(keepdisk) > 0) - body <- jsonlite::toJSON(list(keepdisk = keepdisk), + if(length(container) > 0) + body <- jsonlite::toJSON(list(container = container), auto_unbox = TRUE) else body <- NULL @@ -3897,11 +2293,11 @@ Arvados <- R6::R6Class( resource }, - keep_disks.delete = function(uuid) + containers.delete = function(uuid) { - endPoint <- stringr::str_interp("keep_disks/${uuid}") + endPoint <- stringr::str_interp("containers/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -3917,43 +2313,13 @@ Arvados <- R6::R6Class( resource }, - keep_disks.ping = function(uuid = NULL, ping_secret, - node_uuid = NULL, filesystem_uuid = NULL, - service_host = NULL, service_port, service_ssl_flag) - { - endPoint <- stringr::str_interp("keep_disks/ping") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(uuid = uuid, ping_secret = ping_secret, - node_uuid = node_uuid, filesystem_uuid = filesystem_uuid, - service_host = service_host, service_port = service_port, - service_ssl_flag = service_ssl_flag) - - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - keep_disks.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") + containers.auth = function(uuid) { - endPoint <- stringr::str_interp("keep_disks") + endPoint <- stringr::str_interp("containers/${uuid}/auth") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + queryArgs <- NULL body <- NULL @@ -3967,42 +2333,16 @@ Arvados <- R6::R6Class( resource }, - nodes.get = function(uuid) + containers.lock = function(uuid) { - endPoint <- stringr::str_interp("nodes/${uuid}") + endPoint <- stringr::str_interp("containers/${uuid}/lock") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - nodes.create = function(node, ensure_unique_name = "false", - assign_slot = NULL) - { - endPoint <- stringr::str_interp("nodes") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name, - assign_slot = assign_slot) - - if(length(node) > 0) - body <- jsonlite::toJSON(list(node = node), - auto_unbox = TRUE) - else - body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4013,21 +2353,17 @@ Arvados <- R6::R6Class( resource }, - nodes.update = function(node, uuid, assign_slot = NULL) + containers.unlock = function(uuid) { - endPoint <- stringr::str_interp("nodes/${uuid}") + endPoint <- stringr::str_interp("containers/${uuid}/unlock") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(assign_slot = assign_slot) + queryArgs <- NULL - if(length(node) > 0) - body <- jsonlite::toJSON(list(node = node), - auto_unbox = TRUE) - else - body <- NULL + body <- NULL - response <- private$REST$http$exec("PUT", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4037,17 +2373,17 @@ Arvados <- R6::R6Class( resource }, - nodes.delete = function(uuid) + containers.secret_mounts = function(uuid) { - endPoint <- stringr::str_interp("nodes/${uuid}") + endPoint <- stringr::str_interp("containers/${uuid}/secret_mounts") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4057,17 +2393,17 @@ Arvados <- R6::R6Class( resource }, - nodes.ping = function(uuid, ping_secret) + containers.current = function() { - endPoint <- stringr::str_interp("nodes/${uuid}/ping") + endPoint <- stringr::str_interp("containers/current") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ping_secret = ping_secret) + queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4077,17 +2413,19 @@ Arvados <- R6::R6Class( resource }, - nodes.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + containers.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("nodes") + endPoint <- stringr::str_interp("containers") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -4101,11 +2439,11 @@ Arvados <- R6::R6Class( resource }, - links.get = function(uuid) + container_requests.get = function(uuid) { - endPoint <- stringr::str_interp("links/${uuid}") + endPoint <- stringr::str_interp("container_requests/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4121,16 +2459,18 @@ Arvados <- R6::R6Class( resource }, - links.create = function(link, ensure_unique_name = "false") + container_requests.create = function(containerrequest, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("links") + endPoint <- stringr::str_interp("container_requests") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(link) > 0) - body <- jsonlite::toJSON(list(link = link), + if(length(containerrequest) > 0) + body <- jsonlite::toJSON(list(containerrequest = containerrequest), auto_unbox = TRUE) else body <- NULL @@ -4145,16 +2485,16 @@ Arvados <- R6::R6Class( resource }, - links.update = function(link, uuid) + container_requests.update = function(containerrequest, uuid) { - endPoint <- stringr::str_interp("links/${uuid}") + endPoint <- stringr::str_interp("container_requests/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(link) > 0) - body <- jsonlite::toJSON(list(link = link), + if(length(containerrequest) > 0) + body <- jsonlite::toJSON(list(containerrequest = containerrequest), auto_unbox = TRUE) else body <- NULL @@ -4169,11 +2509,11 @@ Arvados <- R6::R6Class( resource }, - links.delete = function(uuid) + container_requests.delete = function(uuid) { - endPoint <- stringr::str_interp("links/${uuid}") + endPoint <- stringr::str_interp("container_requests/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4189,37 +2529,21 @@ Arvados <- R6::R6Class( resource }, - links.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + container_requests.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL, + include_trash = NULL) { - endPoint <- stringr::str_interp("links") + endPoint <- stringr::str_interp("container_requests") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) - - body <- NULL - - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - links.get_permissions = function(uuid) - { - endPoint <- stringr::str_interp("permissions/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation, + include_trash = include_trash) body <- NULL @@ -4233,11 +2557,11 @@ Arvados <- R6::R6Class( resource }, - keep_services.get = function(uuid) + groups.get = function(uuid) { - endPoint <- stringr::str_interp("keep_services/${uuid}") + endPoint <- stringr::str_interp("groups/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4253,17 +2577,18 @@ Arvados <- R6::R6Class( resource }, - keep_services.create = function(keepservice, - ensure_unique_name = "false") + groups.create = function(group, ensure_unique_name = "false", + cluster_id = NULL, async = "false") { - endPoint <- stringr::str_interp("keep_services") + endPoint <- stringr::str_interp("groups") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id, async = async) - if(length(keepservice) > 0) - body <- jsonlite::toJSON(list(keepservice = keepservice), + if(length(group) > 0) + body <- jsonlite::toJSON(list(group = group), auto_unbox = TRUE) else body <- NULL @@ -4278,16 +2603,16 @@ Arvados <- R6::R6Class( resource }, - keep_services.update = function(keepservice, uuid) + groups.update = function(group, uuid, async = "false") { - endPoint <- stringr::str_interp("keep_services/${uuid}") + endPoint <- stringr::str_interp("groups/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(async = async) - if(length(keepservice) > 0) - body <- jsonlite::toJSON(list(keepservice = keepservice), + if(length(group) > 0) + body <- jsonlite::toJSON(list(group = group), auto_unbox = TRUE) else body <- NULL @@ -4302,11 +2627,11 @@ Arvados <- R6::R6Class( resource }, - keep_services.delete = function(uuid) + groups.delete = function(uuid) { - endPoint <- stringr::str_interp("keep_services/${uuid}") + endPoint <- stringr::str_interp("groups/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4322,13 +2647,22 @@ Arvados <- R6::R6Class( resource }, - keep_services.accessible = function() + groups.contents = function(filters = NULL, + where = NULL, order = NULL, distinct = NULL, + limit = "100", offset = "0", count = "exact", + cluster_id = NULL, bypass_federation = NULL, + include_trash = NULL, uuid = NULL, recursive = NULL, + include = NULL) { - endPoint <- stringr::str_interp("keep_services/accessible") + endPoint <- stringr::str_interp("groups/contents") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(filters = filters, where = where, + order = order, distinct = distinct, limit = limit, + offset = offset, count = count, cluster_id = cluster_id, + bypass_federation = bypass_federation, include_trash = include_trash, + uuid = uuid, recursive = recursive, include = include) body <- NULL @@ -4342,18 +2676,21 @@ Arvados <- R6::R6Class( resource }, - keep_services.list = function(filters = NULL, + groups.shared = function(filters = NULL, where = NULL, order = NULL, select = NULL, distinct = NULL, limit = "100", offset = "0", - count = "exact") + count = "exact", cluster_id = NULL, bypass_federation = NULL, + include_trash = NULL, include = NULL) { - endPoint <- stringr::str_interp("keep_services") + endPoint <- stringr::str_interp("groups/shared") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation, + include_trash = include_trash, include = include) body <- NULL @@ -4367,41 +2704,16 @@ Arvados <- R6::R6Class( resource }, - pipeline_templates.get = function(uuid) + groups.trash = function(uuid) { - endPoint <- stringr::str_interp("pipeline_templates/${uuid}") + endPoint <- stringr::str_interp("groups/${uuid}/trash") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - pipeline_templates.create = function(pipelinetemplate, - ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("pipeline_templates") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(pipelinetemplate) > 0) - body <- jsonlite::toJSON(list(pipelinetemplate = pipelinetemplate), - auto_unbox = TRUE) - else - body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4412,41 +2724,17 @@ Arvados <- R6::R6Class( resource }, - pipeline_templates.update = function(pipelinetemplate, uuid) - { - endPoint <- stringr::str_interp("pipeline_templates/${uuid}") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- NULL - - if(length(pipelinetemplate) > 0) - body <- jsonlite::toJSON(list(pipelinetemplate = pipelinetemplate), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("PUT", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - pipeline_templates.delete = function(uuid) + groups.untrash = function(uuid) { - endPoint <- stringr::str_interp("pipeline_templates/${uuid}") + endPoint <- stringr::str_interp("groups/${uuid}/untrash") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4456,18 +2744,21 @@ Arvados <- R6::R6Class( resource }, - pipeline_templates.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") + groups.list = function(filters = NULL, where = NULL, + order = NULL, select = NULL, distinct = NULL, + limit = "100", offset = "0", count = "exact", + cluster_id = NULL, bypass_federation = NULL, + include_trash = NULL) { - endPoint <- stringr::str_interp("pipeline_templates") + endPoint <- stringr::str_interp("groups") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation, + include_trash = include_trash) body <- NULL @@ -4481,11 +2772,11 @@ Arvados <- R6::R6Class( resource }, - pipeline_instances.get = function(uuid) + keep_services.get = function(uuid) { - endPoint <- stringr::str_interp("pipeline_instances/${uuid}") + endPoint <- stringr::str_interp("keep_services/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4501,17 +2792,18 @@ Arvados <- R6::R6Class( resource }, - pipeline_instances.create = function(pipelineinstance, - ensure_unique_name = "false") + keep_services.create = function(keepservice, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("pipeline_instances") + endPoint <- stringr::str_interp("keep_services") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(pipelineinstance) > 0) - body <- jsonlite::toJSON(list(pipelineinstance = pipelineinstance), + if(length(keepservice) > 0) + body <- jsonlite::toJSON(list(keepservice = keepservice), auto_unbox = TRUE) else body <- NULL @@ -4526,16 +2818,16 @@ Arvados <- R6::R6Class( resource }, - pipeline_instances.update = function(pipelineinstance, uuid) + keep_services.update = function(keepservice, uuid) { - endPoint <- stringr::str_interp("pipeline_instances/${uuid}") + endPoint <- stringr::str_interp("keep_services/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(pipelineinstance) > 0) - body <- jsonlite::toJSON(list(pipelineinstance = pipelineinstance), + if(length(keepservice) > 0) + body <- jsonlite::toJSON(list(keepservice = keepservice), auto_unbox = TRUE) else body <- NULL @@ -4550,11 +2842,11 @@ Arvados <- R6::R6Class( resource }, - pipeline_instances.delete = function(uuid) + keep_services.delete = function(uuid) { - endPoint <- stringr::str_interp("pipeline_instances/${uuid}") + endPoint <- stringr::str_interp("keep_services/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4570,17 +2862,17 @@ Arvados <- R6::R6Class( resource }, - pipeline_instances.cancel = function(uuid) + keep_services.accessible = function() { - endPoint <- stringr::str_interp("pipeline_instances/${uuid}/cancel") + endPoint <- stringr::str_interp("keep_services/accessible") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4590,18 +2882,19 @@ Arvados <- R6::R6Class( resource }, - pipeline_instances.list = function(filters = NULL, + keep_services.list = function(filters = NULL, where = NULL, order = NULL, select = NULL, distinct = NULL, limit = "100", offset = "0", - count = "exact") + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("pipeline_instances") + endPoint <- stringr::str_interp("keep_services") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -4615,11 +2908,11 @@ Arvados <- R6::R6Class( resource }, - repositories.get = function(uuid) + links.get = function(uuid) { - endPoint <- stringr::str_interp("repositories/${uuid}") + endPoint <- stringr::str_interp("links/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4635,16 +2928,18 @@ Arvados <- R6::R6Class( resource }, - repositories.create = function(repository, ensure_unique_name = "false") + links.create = function(link, ensure_unique_name = "false", + cluster_id = NULL) { - endPoint <- stringr::str_interp("repositories") + endPoint <- stringr::str_interp("links") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(repository) > 0) - body <- jsonlite::toJSON(list(repository = repository), + if(length(link) > 0) + body <- jsonlite::toJSON(list(link = link), auto_unbox = TRUE) else body <- NULL @@ -4659,16 +2954,16 @@ Arvados <- R6::R6Class( resource }, - repositories.update = function(repository, uuid) + links.update = function(link, uuid) { - endPoint <- stringr::str_interp("repositories/${uuid}") + endPoint <- stringr::str_interp("links/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(repository) > 0) - body <- jsonlite::toJSON(list(repository = repository), + if(length(link) > 0) + body <- jsonlite::toJSON(list(link = link), auto_unbox = TRUE) else body <- NULL @@ -4683,11 +2978,11 @@ Arvados <- R6::R6Class( resource }, - repositories.delete = function(uuid) + links.delete = function(uuid) { - endPoint <- stringr::str_interp("repositories/${uuid}") + endPoint <- stringr::str_interp("links/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4703,13 +2998,19 @@ Arvados <- R6::R6Class( resource }, - repositories.get_all_permissions = function() + links.list = function(filters = NULL, where = NULL, + order = NULL, select = NULL, distinct = NULL, + limit = "100", offset = "0", count = "exact", + cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("repositories/get_all_permissions") + endPoint <- stringr::str_interp("links") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(filters = filters, where = where, + order = order, select = select, distinct = distinct, + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -4723,18 +3024,13 @@ Arvados <- R6::R6Class( resource }, - repositories.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") + links.get_permissions = function(uuid) { - endPoint <- stringr::str_interp("repositories") + endPoint <- stringr::str_interp("permissions/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + queryArgs <- NULL body <- NULL @@ -4748,11 +3044,11 @@ Arvados <- R6::R6Class( resource }, - specimens.get = function(uuid) + logs.get = function(uuid) { - endPoint <- stringr::str_interp("specimens/${uuid}") + endPoint <- stringr::str_interp("logs/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4768,16 +3064,18 @@ Arvados <- R6::R6Class( resource }, - specimens.create = function(specimen, ensure_unique_name = "false") + logs.create = function(log, ensure_unique_name = "false", + cluster_id = NULL) { - endPoint <- stringr::str_interp("specimens") + endPoint <- stringr::str_interp("logs") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(specimen) > 0) - body <- jsonlite::toJSON(list(specimen = specimen), + if(length(log) > 0) + body <- jsonlite::toJSON(list(log = log), auto_unbox = TRUE) else body <- NULL @@ -4792,16 +3090,16 @@ Arvados <- R6::R6Class( resource }, - specimens.update = function(specimen, uuid) + logs.update = function(log, uuid) { - endPoint <- stringr::str_interp("specimens/${uuid}") + endPoint <- stringr::str_interp("logs/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(specimen) > 0) - body <- jsonlite::toJSON(list(specimen = specimen), + if(length(log) > 0) + body <- jsonlite::toJSON(list(log = log), auto_unbox = TRUE) else body <- NULL @@ -4816,11 +3114,11 @@ Arvados <- R6::R6Class( resource }, - specimens.delete = function(uuid) + logs.delete = function(uuid) { - endPoint <- stringr::str_interp("specimens/${uuid}") + endPoint <- stringr::str_interp("logs/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4836,18 +3134,19 @@ Arvados <- R6::R6Class( resource }, - specimens.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") + logs.list = function(filters = NULL, where = NULL, + order = NULL, select = NULL, distinct = NULL, + limit = "100", offset = "0", count = "exact", + cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("specimens") + endPoint <- stringr::str_interp("logs") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -4861,11 +3160,11 @@ Arvados <- R6::R6Class( resource }, - logs.get = function(uuid) + users.get = function(uuid) { - endPoint <- stringr::str_interp("logs/${uuid}") + endPoint <- stringr::str_interp("users/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -4881,16 +3180,18 @@ Arvados <- R6::R6Class( resource }, - logs.create = function(log, ensure_unique_name = "false") + users.create = function(user, ensure_unique_name = "false", + cluster_id = NULL) { - endPoint <- stringr::str_interp("logs") + endPoint <- stringr::str_interp("users") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(log) > 0) - body <- jsonlite::toJSON(list(log = log), + if(length(user) > 0) + body <- jsonlite::toJSON(list(user = user), auto_unbox = TRUE) else body <- NULL @@ -4905,21 +3206,41 @@ Arvados <- R6::R6Class( resource }, - logs.update = function(log, uuid) + users.update = function(user, uuid, bypass_federation = NULL) { - endPoint <- stringr::str_interp("logs/${uuid}") + endPoint <- stringr::str_interp("users/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(bypass_federation = bypass_federation) - if(length(log) > 0) - body <- jsonlite::toJSON(list(log = log), + if(length(user) > 0) + body <- jsonlite::toJSON(list(user = user), auto_unbox = TRUE) else body <- NULL - response <- private$REST$http$exec("PUT", url, headers, body, + response <- private$REST$http$exec("PUT", url, headers, body, + queryArgs, private$numRetries) + resource <- private$REST$httpParser$parseJSONResponse(response) + + if(!is.null(resource$errors)) + stop(resource$errors) + + resource + }, + + users.delete = function(uuid) + { + endPoint <- stringr::str_interp("users/${uuid}") + url <- paste0(private$host, endPoint) + headers <- list(Authorization = paste("Bearer", private$token), + "Content-Type" = "application/json") + queryArgs <- NULL + + body <- NULL + + response <- private$REST$http$exec("DELETE", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4929,17 +3250,17 @@ Arvados <- R6::R6Class( resource }, - logs.delete = function(uuid) + users.current = function() { - endPoint <- stringr::str_interp("logs/${uuid}") + endPoint <- stringr::str_interp("users/current") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4949,17 +3270,13 @@ Arvados <- R6::R6Class( resource }, - logs.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + users.system = function() { - endPoint <- stringr::str_interp("logs") + endPoint <- stringr::str_interp("users/system") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + queryArgs <- NULL body <- NULL @@ -4973,17 +3290,17 @@ Arvados <- R6::R6Class( resource }, - traits.get = function(uuid) + users.activate = function(uuid) { - endPoint <- stringr::str_interp("traits/${uuid}") + endPoint <- stringr::str_interp("users/${uuid}/activate") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -4993,19 +3310,18 @@ Arvados <- R6::R6Class( resource }, - traits.create = function(trait, ensure_unique_name = "false") + users.setup = function(uuid = NULL, user = NULL, + repo_name = NULL, vm_uuid = NULL, send_notification_email = "false") { - endPoint <- stringr::str_interp("traits") + endPoint <- stringr::str_interp("users/setup") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(uuid = uuid, user = user, + repo_name = repo_name, vm_uuid = vm_uuid, + send_notification_email = send_notification_email) - if(length(trait) > 0) - body <- jsonlite::toJSON(list(trait = trait), - auto_unbox = TRUE) - else - body <- NULL + body <- NULL response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) @@ -5017,21 +3333,17 @@ Arvados <- R6::R6Class( resource }, - traits.update = function(trait, uuid) + users.unsetup = function(uuid) { - endPoint <- stringr::str_interp("traits/${uuid}") + endPoint <- stringr::str_interp("users/${uuid}/unsetup") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(trait) > 0) - body <- jsonlite::toJSON(list(trait = trait), - auto_unbox = TRUE) - else - body <- NULL + body <- NULL - response <- private$REST$http$exec("PUT", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5041,17 +3353,17 @@ Arvados <- R6::R6Class( resource }, - traits.delete = function(uuid) + users.update_uuid = function(uuid, new_uuid) { - endPoint <- stringr::str_interp("traits/${uuid}") + endPoint <- stringr::str_interp("users/${uuid}/update_uuid") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(new_uuid = new_uuid) body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5061,21 +3373,21 @@ Arvados <- R6::R6Class( resource }, - traits.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact") + users.merge = function(new_owner_uuid, new_user_token = NULL, + redirect_to_new_user = NULL, old_user_uuid = NULL, + new_user_uuid = NULL) { - endPoint <- stringr::str_interp("traits") + endPoint <- stringr::str_interp("users/merge") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + queryArgs <- list(new_owner_uuid = new_owner_uuid, + new_user_token = new_user_token, redirect_to_new_user = redirect_to_new_user, + old_user_uuid = old_user_uuid, new_user_uuid = new_user_uuid) body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5085,13 +3397,19 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.get = function(uuid) + users.list = function(filters = NULL, where = NULL, + order = NULL, select = NULL, distinct = NULL, + limit = "100", offset = "0", count = "exact", + cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("virtual_machines/${uuid}") + endPoint <- stringr::str_interp("users") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(filters = filters, where = where, + order = order, select = select, distinct = distinct, + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -5105,22 +3423,17 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.create = function(virtualmachine, - ensure_unique_name = "false") + repositories.get = function(uuid) { - endPoint <- stringr::str_interp("virtual_machines") + endPoint <- stringr::str_interp("repositories/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- NULL - if(length(virtualmachine) > 0) - body <- jsonlite::toJSON(list(virtualmachine = virtualmachine), - auto_unbox = TRUE) - else - body <- NULL + body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5130,21 +3443,23 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.update = function(virtualmachine, uuid) + repositories.create = function(repository, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("virtual_machines/${uuid}") + endPoint <- stringr::str_interp("repositories") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(virtualmachine) > 0) - body <- jsonlite::toJSON(list(virtualmachine = virtualmachine), + if(length(repository) > 0) + body <- jsonlite::toJSON(list(repository = repository), auto_unbox = TRUE) else body <- NULL - response <- private$REST$http$exec("PUT", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5154,17 +3469,21 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.delete = function(uuid) + repositories.update = function(repository, uuid) { - endPoint <- stringr::str_interp("virtual_machines/${uuid}") + endPoint <- stringr::str_interp("repositories/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - body <- NULL + if(length(repository) > 0) + body <- jsonlite::toJSON(list(repository = repository), + auto_unbox = TRUE) + else + body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("PUT", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5174,17 +3493,17 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.logins = function(uuid) + repositories.delete = function(uuid) { - endPoint <- stringr::str_interp("virtual_machines/${uuid}/logins") + endPoint <- stringr::str_interp("repositories/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, + response <- private$REST$http$exec("DELETE", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5194,11 +3513,11 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.get_all_logins = function() + repositories.get_all_permissions = function() { - endPoint <- stringr::str_interp("virtual_machines/get_all_logins") + endPoint <- stringr::str_interp("repositories/get_all_permissions") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5214,18 +3533,19 @@ Arvados <- R6::R6Class( resource }, - virtual_machines.list = function(filters = NULL, + repositories.list = function(filters = NULL, where = NULL, order = NULL, select = NULL, distinct = NULL, limit = "100", offset = "0", - count = "exact") + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("virtual_machines") + endPoint <- stringr::str_interp("repositories") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -5239,11 +3559,11 @@ Arvados <- R6::R6Class( resource }, - workflows.get = function(uuid) + virtual_machines.get = function(uuid) { - endPoint <- stringr::str_interp("workflows/${uuid}") + endPoint <- stringr::str_interp("virtual_machines/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5259,16 +3579,18 @@ Arvados <- R6::R6Class( resource }, - workflows.create = function(workflow, ensure_unique_name = "false") + virtual_machines.create = function(virtualmachine, + ensure_unique_name = "false", cluster_id = NULL) { - endPoint <- stringr::str_interp("workflows") + endPoint <- stringr::str_interp("virtual_machines") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - if(length(workflow) > 0) - body <- jsonlite::toJSON(list(workflow = workflow), + if(length(virtualmachine) > 0) + body <- jsonlite::toJSON(list(virtualmachine = virtualmachine), auto_unbox = TRUE) else body <- NULL @@ -5283,16 +3605,16 @@ Arvados <- R6::R6Class( resource }, - workflows.update = function(workflow, uuid) + virtual_machines.update = function(virtualmachine, uuid) { - endPoint <- stringr::str_interp("workflows/${uuid}") + endPoint <- stringr::str_interp("virtual_machines/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - if(length(workflow) > 0) - body <- jsonlite::toJSON(list(workflow = workflow), + if(length(virtualmachine) > 0) + body <- jsonlite::toJSON(list(virtualmachine = virtualmachine), auto_unbox = TRUE) else body <- NULL @@ -5307,11 +3629,11 @@ Arvados <- R6::R6Class( resource }, - workflows.delete = function(uuid) + virtual_machines.delete = function(uuid) { - endPoint <- stringr::str_interp("workflows/${uuid}") + endPoint <- stringr::str_interp("virtual_machines/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5327,18 +3649,13 @@ Arvados <- R6::R6Class( resource }, - workflows.list = function(filters = NULL, - where = NULL, order = NULL, select = NULL, - distinct = NULL, limit = "100", offset = "0", - count = "exact") + virtual_machines.logins = function(uuid) { - endPoint <- stringr::str_interp("workflows") + endPoint <- stringr::str_interp("virtual_machines/${uuid}/logins") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + queryArgs <- NULL body <- NULL @@ -5352,11 +3669,11 @@ Arvados <- R6::R6Class( resource }, - groups.get = function(uuid) + virtual_machines.get_all_logins = function() { - endPoint <- stringr::str_interp("groups/${uuid}") + endPoint <- stringr::str_interp("virtual_machines/get_all_logins") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5372,45 +3689,23 @@ Arvados <- R6::R6Class( resource }, - groups.create = function(group, ensure_unique_name = "false") - { - endPoint <- stringr::str_interp("groups") - url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), - "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) - - if(length(group) > 0) - body <- jsonlite::toJSON(list(group = group), - auto_unbox = TRUE) - else - body <- NULL - - response <- private$REST$http$exec("POST", url, headers, body, - queryArgs, private$numRetries) - resource <- private$REST$httpParser$parseJSONResponse(response) - - if(!is.null(resource$errors)) - stop(resource$errors) - - resource - }, - - groups.update = function(group, uuid) + virtual_machines.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("groups/${uuid}") + endPoint <- stringr::str_interp("virtual_machines") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- NULL + queryArgs <- list(filters = filters, where = where, + order = order, select = select, distinct = distinct, + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) - if(length(group) > 0) - body <- jsonlite::toJSON(list(group = group), - auto_unbox = TRUE) - else - body <- NULL + body <- NULL - response <- private$REST$http$exec("PUT", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5420,17 +3715,17 @@ Arvados <- R6::R6Class( resource }, - groups.delete = function(uuid) + workflows.get = function(uuid) { - endPoint <- stringr::str_interp("groups/${uuid}") + endPoint <- stringr::str_interp("workflows/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("DELETE", url, headers, body, + response <- private$REST$http$exec("GET", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5440,23 +3735,23 @@ Arvados <- R6::R6Class( resource }, - groups.contents = function(filters = NULL, - where = NULL, order = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact", - include_trash = NULL, uuid = NULL, recursive = NULL) + workflows.create = function(workflow, ensure_unique_name = "false", + cluster_id = NULL) { - endPoint <- stringr::str_interp("groups/contents") + endPoint <- stringr::str_interp("workflows") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(filters = filters, where = where, - order = order, distinct = distinct, limit = limit, - offset = offset, count = count, include_trash = include_trash, - uuid = uuid, recursive = recursive) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) - body <- NULL + if(length(workflow) > 0) + body <- jsonlite::toJSON(list(workflow = workflow), + auto_unbox = TRUE) + else + body <- NULL - response <- private$REST$http$exec("GET", url, headers, body, + response <- private$REST$http$exec("POST", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5466,17 +3761,21 @@ Arvados <- R6::R6Class( resource }, - groups.trash = function(uuid) + workflows.update = function(workflow, uuid) { - endPoint <- stringr::str_interp("groups/${uuid}/trash") + endPoint <- stringr::str_interp("workflows/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL - body <- NULL + if(length(workflow) > 0) + body <- jsonlite::toJSON(list(workflow = workflow), + auto_unbox = TRUE) + else + body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, + response <- private$REST$http$exec("PUT", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5486,17 +3785,17 @@ Arvados <- R6::R6Class( resource }, - groups.untrash = function(uuid) + workflows.delete = function(uuid) { - endPoint <- stringr::str_interp("groups/${uuid}/untrash") + endPoint <- stringr::str_interp("workflows/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL body <- NULL - response <- private$REST$http$exec("POST", url, headers, body, + response <- private$REST$http$exec("DELETE", url, headers, body, queryArgs, private$numRetries) resource <- private$REST$httpParser$parseJSONResponse(response) @@ -5506,19 +3805,19 @@ Arvados <- R6::R6Class( resource }, - groups.list = function(filters = NULL, where = NULL, - order = NULL, select = NULL, distinct = NULL, - limit = "100", offset = "0", count = "exact", - include_trash = NULL) + workflows.list = function(filters = NULL, + where = NULL, order = NULL, select = NULL, + distinct = NULL, limit = "100", offset = "0", + count = "exact", cluster_id = NULL, bypass_federation = NULL) { - endPoint <- stringr::str_interp("groups") + endPoint <- stringr::str_interp("workflows") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, limit = limit, offset = offset, count = count, - include_trash = include_trash) + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -5536,7 +3835,7 @@ Arvados <- R6::R6Class( { endPoint <- stringr::str_interp("user_agreements/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5553,13 +3852,14 @@ Arvados <- R6::R6Class( }, user_agreements.create = function(useragreement, - ensure_unique_name = "false") + ensure_unique_name = "false", cluster_id = NULL) { endPoint <- stringr::str_interp("user_agreements") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") - queryArgs <- list(ensure_unique_name = ensure_unique_name) + queryArgs <- list(ensure_unique_name = ensure_unique_name, + cluster_id = cluster_id) if(length(useragreement) > 0) body <- jsonlite::toJSON(list(useragreement = useragreement), @@ -5581,7 +3881,7 @@ Arvados <- R6::R6Class( { endPoint <- stringr::str_interp("user_agreements/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5605,7 +3905,7 @@ Arvados <- R6::R6Class( { endPoint <- stringr::str_interp("user_agreements/${uuid}") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5625,7 +3925,7 @@ Arvados <- R6::R6Class( { endPoint <- stringr::str_interp("user_agreements/signatures") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5645,7 +3945,7 @@ Arvados <- R6::R6Class( { endPoint <- stringr::str_interp("user_agreements/sign") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL @@ -5664,15 +3964,16 @@ Arvados <- R6::R6Class( user_agreements.list = function(filters = NULL, where = NULL, order = NULL, select = NULL, distinct = NULL, limit = "100", offset = "0", - count = "exact") + count = "exact", cluster_id = NULL, bypass_federation = NULL) { endPoint <- stringr::str_interp("user_agreements") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- list(filters = filters, where = where, order = order, select = select, distinct = distinct, - limit = limit, offset = offset, count = count) + limit = limit, offset = offset, count = count, + cluster_id = cluster_id, bypass_federation = bypass_federation) body <- NULL @@ -5690,7 +3991,27 @@ Arvados <- R6::R6Class( { endPoint <- stringr::str_interp("user_agreements/new") url <- paste0(private$host, endPoint) - headers <- list(Authorization = paste("OAuth2", private$token), + headers <- list(Authorization = paste("Bearer", private$token), + "Content-Type" = "application/json") + queryArgs <- NULL + + body <- NULL + + response <- private$REST$http$exec("GET", url, headers, body, + queryArgs, private$numRetries) + resource <- private$REST$httpParser$parseJSONResponse(response) + + if(!is.null(resource$errors)) + stop(resource$errors) + + resource + }, + + configs.get = function() + { + endPoint <- stringr::str_interp("config") + url <- paste0(private$host, endPoint) + headers <- list(Authorization = paste("Bearer", private$token), "Content-Type" = "application/json") queryArgs <- NULL diff --git a/sdk/R/R/ArvadosFile.R b/sdk/R/R/ArvadosFile.R index 70bb4450ec..fb1d3b335c 100644 --- a/sdk/R/R/ArvadosFile.R +++ b/sdk/R/R/ArvadosFile.R @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -source("./R/util.R") - #' ArvadosFile #' #' ArvadosFile class represents a file inside Arvados collection. diff --git a/sdk/R/R/Collection.R b/sdk/R/R/Collection.R index 8869d7be67..9ed758c0a4 100644 --- a/sdk/R/R/Collection.R +++ b/sdk/R/R/Collection.R @@ -2,11 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -source("./R/Subcollection.R") -source("./R/ArvadosFile.R") -source("./R/RESTService.R") -source("./R/util.R") - #' Collection #' #' Collection class provides interface for working with Arvados collections. @@ -121,9 +116,8 @@ Collection <- R6::R6Class( private$REST$create(file, self$uuid) newTreeBranch$setCollection(self) + newTreeBranch }) - - "Created" } else { diff --git a/sdk/R/R/CollectionTree.R b/sdk/R/R/CollectionTree.R index 5f7a29455a..e01e7e8de9 100644 --- a/sdk/R/R/CollectionTree.R +++ b/sdk/R/R/CollectionTree.R @@ -2,10 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -source("./R/Subcollection.R") -source("./R/ArvadosFile.R") -source("./R/util.R") - CollectionTree <- R6::R6Class( "CollectionTree", public = list( diff --git a/sdk/R/R/HttpParser.R b/sdk/R/R/HttpParser.R index cd492166a1..60bf782827 100644 --- a/sdk/R/R/HttpParser.R +++ b/sdk/R/R/HttpParser.R @@ -31,14 +31,13 @@ HttpParser <- R6::R6Class( { text <- rawToChar(response$content) doc <- XML::xmlParse(text, asText=TRUE) - base <- paste(paste("/", strsplit(uri, "/")[[1]][-1:-3], sep="", collapse=""), "/", sep="") + base <- paste("/", strsplit(uri, "/")[[1]][4], "/", sep="") result <- unlist( XML::xpathApply(doc, "//D:response/D:href", function(node) { sub(base, "", URLdecode(XML::xmlValue(node)), fixed=TRUE) }) ) - result <- result[result != ""] - result[-1] + result[result != ""] }, getFileSizesFromResponse = function(response, uri) diff --git a/sdk/R/R/HttpRequest.R b/sdk/R/R/HttpRequest.R index 07defca90f..18b36f9689 100644 --- a/sdk/R/R/HttpRequest.R +++ b/sdk/R/R/HttpRequest.R @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -source("./R/util.R") - HttpRequest <- R6::R6Class( "HttrRequest", @@ -54,7 +52,7 @@ HttpRequest <- R6::R6Class( { query <- paste0(names(query), "=", query, collapse = "&") - return(paste0("/?", query)) + return(paste0("?", query)) } return("") diff --git a/sdk/R/R/RESTService.R b/sdk/R/R/RESTService.R index 78b2c35e32..9c65e72861 100644 --- a/sdk/R/R/RESTService.R +++ b/sdk/R/R/RESTService.R @@ -36,16 +36,13 @@ RESTService <- R6::R6Class( { if(is.null(private$webDavHostName)) { - discoveryDocumentURL <- paste0("https://", private$rawHostName, - "/discovery/v1/apis/arvados/v1/rest") + publicConfigURL <- paste0("https://", private$rawHostName, + "/arvados/v1/config") - headers <- list(Authorization = paste("OAuth2", self$token)) - - serverResponse <- self$http$exec("GET", discoveryDocumentURL, headers, - retryTimes = self$numRetries) + serverResponse <- self$http$exec("GET", publicConfigURL, retryTimes = self$numRetries) - discoveryDocument <- self$httpParser$parseJSONResponse(serverResponse) - private$webDavHostName <- discoveryDocument$keepWebServiceUrl + configDocument <- self$httpParser$parseJSONResponse(serverResponse) + private$webDavHostName <- configDocument$Services$WebDAVDownload$ExternalURL if(is.null(private$webDavHostName)) stop("Unable to find WebDAV server.") @@ -118,7 +115,7 @@ RESTService <- R6::R6Class( collectionURL <- URLencode(paste0(self$getWebDavHostName(), "c=", uuid)) - headers <- list("Authorization" = paste("OAuth2", self$token)) + headers <- list("Authorization" = paste("Bearer", self$token)) response <- self$http$exec("PROPFIND", collectionURL, headers, retryTimes = self$numRetries) diff --git a/sdk/R/R/Subcollection.R b/sdk/R/R/Subcollection.R index 17a9ef3ee3..981bd687a2 100644 --- a/sdk/R/R/Subcollection.R +++ b/sdk/R/R/Subcollection.R @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -source("./R/util.R") - #' Subcollection #' #' Subcollection class represents a folder inside Arvados collection. diff --git a/sdk/R/R/autoGenAPI.R b/sdk/R/R/autoGenAPI.R index 1aef20b6cb..c86684f8b0 100644 --- a/sdk/R/R/autoGenAPI.R +++ b/sdk/R/R/autoGenAPI.R @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 getAPIDocument <- function(){ - url <- "https://4xphq.arvadosapi.com/discovery/v1/apis/arvados/v1/rest" + url <- "https://jutro.arvadosapi.com/discovery/v1/apis/arvados/v1/rest" serverResponse <- httr::RETRY("GET", url = url) httr::content(serverResponse, as = "parsed", type = "application/json") @@ -17,6 +17,10 @@ generateAPI <- function() discoveryDocument <- getAPIDocument() methodResources <- discoveryDocument$resources + + # Don't emit deprecated APIs + methodResources <- methodResources[!(names(methodResources) %in% c("jobs", "job_tasks", "pipeline_templates", "pipeline_instances", + "keep_disks", "nodes", "humans", "traits", "specimens"))] resourceNames <- names(methodResources) methodDoc <- genMethodsDoc(methodResources, resourceNames) @@ -34,6 +38,10 @@ generateAPI <- function() arvadosAPIFooter) fileConn <- file("./R/Arvados.R", "w") + writeLines(c( + "# Copyright (C) The Arvados Authors. All rights reserved.", + "#", + "# SPDX-License-Identifier: Apache-2.0", ""), fileConn) writeLines(unlist(arvadosClass), fileConn) close(fileConn) NULL @@ -252,7 +260,7 @@ getRequestURL <- function(methodMetaData) getRequestHeaders <- function() { - c("headers <- list(Authorization = paste(\"OAuth2\", private$token), ", + c("headers <- list(Authorization = paste(\"Bearer\", private$token), ", " \"Content-Type\" = \"application/json\")") } diff --git a/sdk/R/README.Rmd b/sdk/R/README.Rmd index c1d6c7cf4f..8cc89d9020 100644 --- a/sdk/R/README.Rmd +++ b/sdk/R/README.Rmd @@ -14,7 +14,7 @@ knitr::opts_chunk$set(eval=FALSE) ``` ```{r} -install.packages("ArvadosR", repos=c("http://r.arvados.org", getOption("repos")["CRAN"]), dependencies=TRUE) +install.packages("ArvadosR", repos=c("https://r.arvados.org", getOption("repos")["CRAN"]), dependencies=TRUE) ``` Note: on Linux, you may have to install supporting packages. @@ -71,6 +71,12 @@ arv$setNumRetries(5) collection <- arv$collections.get("uuid") ``` +Be aware that the result from `collections.get` is _not_ a +`Collection` class. The object returned from this method lets you +access collection fields like "name" and "description". The +`Collection` class lets you access the files in the collection for +reading and writing, and is described in the next section. + * List collections: ```{r} @@ -78,9 +84,7 @@ collection <- arv$collections.get("uuid") collectionList <- arv$collections.list(list(list("name", "like", "Test%"))) collectionList <- arv$collections.list(list(list("name", "like", "Test%")), limit = 10, offset = 2) -``` -```{r} # count of total number of items (may be more than returned due to paging) collectionList$items_available @@ -106,7 +110,7 @@ deletedCollection <- arv$collections.delete("uuid") updatedCollection <- arv$collections.update(list(name = "New name", description = "New description"), "uuid") ``` -* Create collection: +* Create a new collection: ```{r} newCollection <- arv$collections.create(list(name = "Example", description = "This is a test collection")) @@ -115,7 +119,7 @@ newCollection <- arv$collections.create(list(name = "Example", description = "Th #### Manipulating collection content -* Create collection object: +* Initialize a collection object: ```{r} collection <- Collection$new(arv, "uuid") @@ -150,13 +154,13 @@ mytable <- read.table(arvConnection) * Write a table: ```{r} -arvadosFile <- collection$create("myoutput.txt") +arvadosFile <- collection$create("myoutput.txt")[[1]] arvConnection <- arvadosFile$connection("w") write.table(mytable, arvConnection) arvadosFile$flush() ``` -* Write to existing file (override current content of the file): +* Write to existing file (overwrites current content of the file): ```{r} arvadosFile <- collection$get("location/to/my/file.cpp") @@ -183,7 +187,7 @@ or size <- arvadosSubcollection$getSizeInBytes() ``` -* Create new file in a collection: +* Create new file in a collection (returns a vector of one or more ArvadosFile objects): ```{r} collection$create(files) @@ -192,7 +196,7 @@ collection$create(files) Example: ```{r} -mainFile <- collection$create("cpp/src/main.cpp") +mainFile <- collection$create("cpp/src/main.cpp")[[1]] fileList <- collection$create(c("cpp/src/main.cpp", "cpp/src/util.h")) ``` diff --git a/sdk/R/tests/testthat/test-ArvadosFile.R b/sdk/R/tests/testthat/test-ArvadosFile.R index e3457c993f..da7d52c67d 100644 --- a/sdk/R/tests/testthat/test-ArvadosFile.R +++ b/sdk/R/tests/testthat/test-ArvadosFile.R @@ -23,7 +23,7 @@ test_that("get always returns NULL", { dog <- ArvadosFile$new("dog") responseIsNull <- is.null(dog$get("something")) - expect_that(responseIsNull, is_true()) + expect_true(responseIsNull) }) test_that("getFirst always returns NULL", { @@ -31,7 +31,7 @@ test_that("getFirst always returns NULL", { dog <- ArvadosFile$new("dog") responseIsNull <- is.null(dog$getFirst()) - expect_that(responseIsNull, is_true()) + expect_true(responseIsNull) }) test_that(paste("getSizeInBytes returns zero if arvadosFile", @@ -266,8 +266,8 @@ test_that("move moves arvados file inside collection tree", { dogIsNullOnOldLocation <- is.null(collection$get("animal/dog")) dogExistsOnNewLocation <- !is.null(collection$get("dog")) - expect_that(dogIsNullOnOldLocation, is_true()) - expect_that(dogExistsOnNewLocation, is_true()) + expect_true(dogIsNullOnOldLocation) + expect_true(dogExistsOnNewLocation) }) test_that(paste("copy raises exception if arvados file", @@ -339,8 +339,8 @@ test_that("copy copies arvados file inside collection tree", { dogExistsOnOldLocation <- !is.null(collection$get("animal/dog")) dogExistsOnNewLocation <- !is.null(collection$get("dog")) - expect_that(dogExistsOnOldLocation, is_true()) - expect_that(dogExistsOnNewLocation, is_true()) + expect_true(dogExistsOnOldLocation) + expect_true(dogExistsOnNewLocation) }) test_that("duplicate performs deep cloning of Arvados file", { diff --git a/sdk/R/tests/testthat/test-Collection.R b/sdk/R/tests/testthat/test-Collection.R index 636359ae21..20a2ecf05b 100644 --- a/sdk/R/tests/testthat/test-Collection.R +++ b/sdk/R/tests/testthat/test-Collection.R @@ -86,7 +86,7 @@ test_that(paste("add adds ArvadosFile or Subcollection", dog <- collection$get("animal/dog") dogExistsInCollection <- !is.null(dog) && dog$getName() == "dog" - expect_that(dogExistsInCollection, is_true()) + expect_true(dogExistsInCollection) expect_that(fakeREST$createCallCount, equals(1)) }) @@ -119,8 +119,8 @@ test_that(paste("create adds files specified by fileNames", dogExistsInCollection <- !is.null(dog) && dog$getName() == "dog" catExistsInCollection <- !is.null(cat) && cat$getName() == "cat" - expect_that(dogExistsInCollection, is_true()) - expect_that(catExistsInCollection, is_true()) + expect_true(dogExistsInCollection) + expect_true(catExistsInCollection) expect_that(fakeREST$createCallCount, equals(2)) }) @@ -168,8 +168,8 @@ test_that(paste("remove removes files specified by paths", dogExistsInCollection <- !is.null(dog) && dog$getName() == "dog" catExistsInCollection <- !is.null(cat) && cat$getName() == "cat" - expect_that(dogExistsInCollection, is_false()) - expect_that(catExistsInCollection, is_false()) + expect_false(dogExistsInCollection) + expect_false(catExistsInCollection) expect_that(fakeREST$deleteCallCount, equals(2)) }) @@ -188,8 +188,8 @@ test_that(paste("move moves content to a new location inside file tree", dogIsNullOnOldLocation <- is.null(collection$get("animal/dog")) dogExistsOnNewLocation <- !is.null(collection$get("dog")) - expect_that(dogIsNullOnOldLocation, is_true()) - expect_that(dogExistsOnNewLocation, is_true()) + expect_true(dogIsNullOnOldLocation) + expect_true(dogExistsOnNewLocation) expect_that(fakeREST$moveCallCount, equals(1)) }) @@ -219,7 +219,7 @@ test_that("getFileListing returns sorted collection content received from REST s contentMatchExpected <- all(collection$getFileListing() == c("animal", "animal/fish", "ball")) - expect_that(contentMatchExpected, is_true()) + expect_true(contentMatchExpected) #2 calls because Collection$new calls getFileListing once expect_that(fakeREST$getCollectionContentCallCount, equals(2)) @@ -237,7 +237,7 @@ test_that("get returns arvados file or subcollection from internal tree structur fish <- collection$get("animal/fish") fishIsNotNull <- !is.null(fish) - expect_that(fishIsNotNull, is_true()) + expect_true(fishIsNotNull) expect_that(fish$getName(), equals("fish")) }) @@ -256,8 +256,8 @@ test_that(paste("copy copies content to a new location inside file tree", dogExistsOnOldLocation <- !is.null(collection$get("animal/dog")) dogExistsOnNewLocation <- !is.null(collection$get("dog")) - expect_that(dogExistsOnOldLocation, is_true()) - expect_that(dogExistsOnNewLocation, is_true()) + expect_true(dogExistsOnOldLocation) + expect_true(dogExistsOnNewLocation) expect_that(fakeREST$copyCallCount, equals(1)) }) diff --git a/sdk/R/tests/testthat/test-CollectionTree.R b/sdk/R/tests/testthat/test-CollectionTree.R index 1a3aefecd0..c4bf9a1da7 100644 --- a/sdk/R/tests/testthat/test-CollectionTree.R +++ b/sdk/R/tests/testthat/test-CollectionTree.R @@ -34,16 +34,16 @@ test_that("constructor creates file tree from character array properly", { boat$getCollection() == "myCollection" expect_that(root$getName(), equals("")) - expect_that(rootIsOfTypeSubcollection, is_true()) - expect_that(rootHasNoParent, is_true()) - expect_that(animalIsOfTypeSubcollection, is_true()) - expect_that(animalsParentIsRoot, is_true()) - expect_that(animalContainsDog, is_true()) - expect_that(dogIsOfTypeArvadosFile, is_true()) - expect_that(dogsParentIsAnimal, is_true()) - expect_that(boatIsOfTypeArvadosFile, is_true()) - expect_that(boatsParentIsRoot, is_true()) - expect_that(allElementsBelongToSameCollection, is_true()) + expect_true(rootIsOfTypeSubcollection) + expect_true(rootHasNoParent) + expect_true(animalIsOfTypeSubcollection) + expect_true(animalsParentIsRoot) + expect_true(animalContainsDog) + expect_true(dogIsOfTypeArvadosFile) + expect_true(dogsParentIsAnimal) + expect_true(boatIsOfTypeArvadosFile) + expect_true(boatsParentIsRoot) + expect_true(allElementsBelongToSameCollection) }) test_that("getElement returns element from tree if element exists on specified path", { @@ -72,7 +72,7 @@ test_that("getElement returns NULL from tree if element doesn't exists on specif fish <- collectionTree$getElement("animal/fish") fishIsNULL <- is.null(fish) - expect_that(fishIsNULL, is_true()) + expect_true(fishIsNULL) }) test_that("getElement trims ./ from start of relativePath", { diff --git a/sdk/R/tests/testthat/test-HttpParser.R b/sdk/R/tests/testthat/test-HttpParser.R index 82c0fb0dd2..fb9f379b36 100644 --- a/sdk/R/tests/testthat/test-HttpParser.R +++ b/sdk/R/tests/testthat/test-HttpParser.R @@ -18,7 +18,7 @@ test_that("parseJSONResponse generates and returns JSON object from server respo result <- parser$parseJSONResponse(serverResponse) barExists <- !is.null(result$bar) - expect_that(barExists, is_true()) + expect_true(barExists) expect_that(unlist(result$bar$foo), equals(10)) }) @@ -40,7 +40,7 @@ test_that(paste("parseResponse generates and returns character vector", webDAVResponseSample = paste0("/c=aaaaa-bbbbb-ccccccccccccccc/c=aaaaa-bbbbb-ccccccccccccccc/Fri, 11 Jan 2018 1", "1:11:11 GMT argparse.ArgumentParser action="store_false", default=True, help=argparse.SUPPRESS) + parser.add_argument("--disable-color", dest="enable_color", + action="store_false", default=True, + help=argparse.SUPPRESS) + parser.add_argument("--disable-js-validation", action="store_true", default=False, help=argparse.SUPPRESS) parser.add_argument("--thread-count", type=int, - default=1, help="Number of threads to use for job submit and output collection.") + default=4, help="Number of threads to use for job submit and output collection.") parser.add_argument("--http-timeout", type=int, default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).") diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml index dce1bd4d02..8a3fa3173a 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml @@ -240,6 +240,12 @@ $graph: MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request. jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache" + acrContainerImage: + type: string? + doc: | + The container image containing the correct version of + arvados-cwl-runner to use when invoking the workflow on + Arvados. - name: ClusterTarget type: record diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml index b9b9e61651..95ed0a75bc 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml @@ -184,6 +184,12 @@ $graph: MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request. jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache" + acrContainerImage: + type: string? + doc: | + The container image containing the correct version of + arvados-cwl-runner to use when invoking the workflow on + Arvados. - name: ClusterTarget type: record diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml index b9b9e61651..95ed0a75bc 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml @@ -184,6 +184,12 @@ $graph: MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request. jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache" + acrContainerImage: + type: string? + doc: | + The container image containing the correct version of + arvados-cwl-runner to use when invoking the workflow on + Arvados. - name: ClusterTarget type: record diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index fb23c2ccf7..7b81bfb447 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -21,8 +21,7 @@ import ruamel.yaml as yaml from cwltool.errors import WorkflowException from cwltool.process import UnsupportedRequirement, shortname -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class -from cwltool.utils import aslist +from cwltool.utils import aslist, adjustFileObjs, adjustDirObjs, visit_class from cwltool.job import JobBase import arvados.collection @@ -235,7 +234,9 @@ class ArvadosContainer(JobBase): container_request["container_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, runtimeContext.pull_image, - runtimeContext.project_uuid) + runtimeContext.project_uuid, + runtimeContext.force_docker_pull, + runtimeContext.tmp_outdir_prefix) network_req, _ = self.get_requirement("NetworkAccess") if network_req: @@ -325,8 +326,8 @@ class ArvadosContainer(JobBase): logger.info("%s reused container %s", self.arvrunner.label(self), response["container_uuid"]) else: logger.info("%s %s state is %s", self.arvrunner.label(self), response["uuid"], response["state"]) - except Exception: - logger.exception("%s got an error", self.arvrunner.label(self)) + except Exception as e: + logger.exception("%s error submitting container\n%s", self.arvrunner.label(self), e) logger.debug("Container request was %s", container_request) self.output_callback({}, "permanentFail") @@ -475,6 +476,7 @@ class RunnerContainer(Runner): "--api=containers", "--no-log-timestamps", "--disable-validate", + "--disable-color", "--eval-timeout=%s" % self.arvrunner.eval_timeout, "--thread-count=%s" % self.arvrunner.thread_count, "--enable-reuse" if self.enable_reuse else "--disable-reuse", diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py index a8f56ad1d4..3c82082713 100644 --- a/sdk/cwl/arvados_cwl/arvdocker.py +++ b/sdk/cwl/arvados_cwl/arvdocker.py @@ -18,7 +18,8 @@ logger = logging.getLogger('arvados.cwl-runner') cached_lookups = {} cached_lookups_lock = threading.Lock() -def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid): +def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid, + force_pull, tmp_outdir_prefix): """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker.""" if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement: @@ -48,7 +49,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid if not images: # Fetch Docker image if necessary. try: - cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image) + cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image, + force_pull, tmp_outdir_prefix) except OSError as e: raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e)) diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 97c5fafe79..6067ae9f44 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -17,16 +17,17 @@ from cwltool.pack import pack from cwltool.load_tool import fetch_document, resolve_and_validate_document from cwltool.process import shortname from cwltool.workflow import Workflow, WorkflowException, WorkflowStep -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class +from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class from cwltool.context import LoadingContext import ruamel.yaml as yaml from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields, discover_secondary_files, - make_builder) + make_builder, arvados_jobs_image) from .pathmapper import ArvPathMapper, trim_listing from .arvtool import ArvadosCommandTool, set_cluster_target +from ._version import __version__ from .perf import Perf @@ -37,7 +38,8 @@ max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdir sum_res_pars = ("outdirMin", "outdirMax") def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, - submit_runner_ram=0, name=None, merged_map=None): + submit_runner_ram=0, name=None, merged_map=None, + submit_runner_image=None): packed = packed_workflow(arvRunner, tool, merged_map) @@ -57,18 +59,25 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, upload_dependencies(arvRunner, name, tool.doc_loader, packed, tool.tool["id"], False) + wf_runner_resources = None + + hints = main.get("hints", []) + found = False + for h in hints: + if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources": + wf_runner_resources = h + found = True + break + if not found: + wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"} + hints.append(wf_runner_resources) + + wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__) + if submit_runner_ram: - hints = main.get("hints", []) - found = False - for h in hints: - if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources": - h["ramMin"] = submit_runner_ram - found = True - break - if not found: - hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources", - "ramMin": submit_runner_ram}) - main["hints"] = hints + wf_runner_resources["ramMin"] = submit_runner_ram + + main["hints"] = hints body = { "workflow": { diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index e8d1347ddf..947b630bab 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -37,7 +37,7 @@ from .arvworkflow import ArvadosWorkflow, upload_workflow from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size from .perf import Perf from .pathmapper import NoFollowPathMapper -from .task_queue import TaskQueue +from cwltool.task_queue import TaskQueue from .context import ArvLoadingContext, ArvRuntimeContext from ._version import __version__ @@ -524,6 +524,8 @@ The 'jobs' API is no longer supported. def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None): self.debug = runtimeContext.debug + logger.info("Using cluster %s (%s)", self.api.config()["ClusterID"], self.api.config()["Services"]["Controller"]["ExternalURL"]) + updated_tool.visit(self.check_features) self.project_uuid = runtimeContext.project_uuid @@ -596,7 +598,8 @@ The 'jobs' API is no longer supported. uuid=existing_uuid, submit_runner_ram=runtimeContext.submit_runner_ram, name=runtimeContext.name, - merged_map=merged_map), + merged_map=merged_map, + submit_runner_image=runtimeContext.submit_runner_image), "success") self.apply_reqs(job_order, tool) diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py index 5bad290773..e0b2d25bc5 100644 --- a/sdk/cwl/arvados_cwl/pathmapper.py +++ b/sdk/cwl/arvados_cwl/pathmapper.py @@ -21,7 +21,9 @@ import arvados.collection from schema_salad.sourceline import SourceLine from arvados.errors import ApiError -from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs +from cwltool.pathmapper import PathMapper, MapperEnt +from cwltool.utils import adjustFileObjs, adjustDirObjs +from cwltool.stdfsaccess import abspath from cwltool.workflow import WorkflowException from .http import http_to_keep diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index b10f02d140..42d4b552ac 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -31,8 +31,7 @@ import cwltool.workflow from cwltool.process import (scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname, Process, fill_in_defaults) from cwltool.load_tool import fetch_document -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class -from cwltool.utils import aslist +from cwltool.utils import aslist, adjustFileObjs, adjustDirObjs, visit_class from cwltool.builder import substitute from cwltool.pack import pack from cwltool.update import INTERNAL_VERSION @@ -444,9 +443,14 @@ def upload_docker(arvrunner, tool): # TODO: can be supported by containers API, but not jobs API. raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError( "Option 'dockerOutputDirectory' of DockerRequirement not supported.") - arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid) + arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) else: - arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs"}, True, arvrunner.project_uuid) + arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__}, + True, arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) elif isinstance(tool, cwltool.workflow.Workflow): for s in tool.steps: upload_docker(arvrunner, s.embedded_tool) @@ -479,7 +483,10 @@ def packed_workflow(arvrunner, tool, merged_map): if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles: v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]] if v.get("class") == "DockerRequirement": - v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, arvrunner.project_uuid) + v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, + arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) for l in v: visit(v[l], cur_id) if isinstance(v, list): @@ -584,7 +591,9 @@ def arvados_jobs_image(arvrunner, img): """Determine if the right arvados/jobs image version is available. If not, try to pull and upload it.""" try: - return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid) + return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) except Exception as e: raise Exception("Docker image %s is not available\n%s" % (img, e) ) diff --git a/sdk/cwl/arvados_cwl/task_queue.py b/sdk/cwl/arvados_cwl/task_queue.py deleted file mode 100644 index d75fec6c63..0000000000 --- a/sdk/cwl/arvados_cwl/task_queue.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -from future import standard_library -standard_library.install_aliases() -from builtins import range -from builtins import object - -import queue -import threading -import logging - -logger = logging.getLogger('arvados.cwl-runner') - -class TaskQueue(object): - def __init__(self, lock, thread_count): - self.thread_count = thread_count - self.task_queue = queue.Queue(maxsize=self.thread_count) - self.task_queue_threads = [] - self.lock = lock - self.in_flight = 0 - self.error = None - - for r in range(0, self.thread_count): - t = threading.Thread(target=self.task_queue_func) - self.task_queue_threads.append(t) - t.start() - - def task_queue_func(self): - while True: - task = self.task_queue.get() - if task is None: - return - try: - task() - except Exception as e: - logger.exception("Unhandled exception running task") - self.error = e - - with self.lock: - self.in_flight -= 1 - - def add(self, task, unlock, check_done): - if self.thread_count > 1: - with self.lock: - self.in_flight += 1 - else: - task() - return - - while True: - try: - unlock.release() - if check_done.is_set(): - return - self.task_queue.put(task, block=True, timeout=3) - return - except queue.Full: - pass - finally: - unlock.acquire() - - - def drain(self): - try: - # Drain queue - while not self.task_queue.empty(): - self.task_queue.get(True, .1) - except queue.Empty: - pass - - def join(self): - for t in self.task_queue_threads: - self.task_queue.put(None) - for t in self.task_queue_threads: - t.join() diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py index d9ce12487a..c3936617f0 100644 --- a/sdk/cwl/arvados_version.py +++ b/sdk/cwl/arvados_version.py @@ -6,36 +6,42 @@ import subprocess import time import os import re +import sys SETUP_DIR = os.path.dirname(os.path.abspath(__file__)) +VERSION_PATHS = { + SETUP_DIR, + os.path.abspath(os.path.join(SETUP_DIR, "../python")), + os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh")) + } def choose_version_from(): - sdk_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip() - cwl_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', SETUP_DIR]).strip() - if int(sdk_ts) > int(cwl_ts): - getver = os.path.join(SETUP_DIR, "../python") - else: - getver = SETUP_DIR + ts = {} + for path in VERSION_PATHS: + ts[subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', path]).strip()] = path + + sorted_ts = sorted(ts.items()) + getver = sorted_ts[-1][1] + print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr) return getver def git_version_at_commit(): curdir = choose_version_from() myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', '--format=%H', curdir]).strip() - myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode() return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: - return fp.write("__version__ = '%s'\n" % v) + v = v.replace("~dev", ".dev").replace("~rc", "rc") + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: + return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: - return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: + return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") @@ -45,7 +51,12 @@ def get_version(setup_dir, module): else: try: save_version(setup_dir, module, git_version_at_commit()) - except (subprocess.CalledProcessError, OSError): + except (subprocess.CalledProcessError, OSError) as err: + print("ERROR: {0}".format(err), file=sys.stderr) pass return read_version(setup_dir, module) + +# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh +if __name__ == '__main__': + print(get_version(SETUP_DIR, "arvados_cwl")) diff --git a/sdk/cwl/bin/arvados-cwl-runner b/sdk/cwl/bin/arvados-cwl-runner index 55ce31e666..8ea3f3d3e2 100755 --- a/sdk/cwl/bin/arvados-cwl-runner +++ b/sdk/cwl/bin/arvados-cwl-runner @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/bin/cwl-runner b/sdk/cwl/bin/cwl-runner index 55ce31e666..8ea3f3d3e2 100755 --- a/sdk/cwl/bin/cwl-runner +++ b/sdk/cwl/bin/cwl-runner @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/fpm-info.sh b/sdk/cwl/fpm-info.sh index 66176b940b..9a52ee7021 100644 --- a/sdk/cwl/fpm-info.sh +++ b/sdk/cwl/fpm-info.sh @@ -2,11 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 +fpm_depends+=(nodejs) + case "$TARGET" in - debian8) - fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls) - ;; - debian9 | ubuntu1604) + ubuntu1604) fpm_depends+=(libcurl3-gnutls) ;; debian* | ubuntu*) diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py deleted file mode 100644 index d6a4c24a78..0000000000 --- a/sdk/cwl/gittaggers.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -from builtins import str -from builtins import next - -from setuptools.command.egg_info import egg_info -import subprocess -import time -import os - -SETUP_DIR = os.path.dirname(__file__) or '.' - -def choose_version_from(): - sdk_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip() - cwl_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', SETUP_DIR]).strip() - if int(sdk_ts) > int(cwl_ts): - getver = os.path.join(SETUP_DIR, "../python") - else: - getver = SETUP_DIR - return getver - -class EggInfoFromGit(egg_info): - """Tag the build with git commit timestamp. - - If a build tag has already been set (e.g., "egg_info -b", building - from source package), leave it alone. - """ - def git_latest_tag(self): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - - def git_timestamp_tag(self): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', choose_version_from()]).strip() - return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))) - - def tags(self): - if self.tag_build is None: - self.tag_build = self.git_latest_tag() + self.git_timestamp_tag() - return egg_info.tags(self) diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py index c8ab71e50b..a2fba730c5 100644 --- a/sdk/cwl/setup.py +++ b/sdk/cwl/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -39,7 +39,7 @@ setup(name='arvados-cwl-runner', # file to determine what version of cwltool and schema-salad to # build. install_requires=[ - 'cwltool==3.0.20200807132242', + 'cwltool==3.0.20201121085451', 'schema-salad==7.0.20200612160654', 'arvados-python-client{}'.format(pysdk_dep), 'setuptools', diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh index 6de404f448..0021bc8d90 100755 --- a/sdk/cwl/test_with_arvbox.sh +++ b/sdk/cwl/test_with_arvbox.sh @@ -129,7 +129,7 @@ fi export ARVADOS_API_HOST=localhost:8000 export ARVADOS_API_HOST_INSECURE=1 -export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token) +export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados-arvbox/superuser_token) if test -n "$build" ; then /usr/src/arvados/build/build-dev-docker-jobs-image.sh @@ -141,7 +141,11 @@ else . /usr/src/arvados/build/run-library.sh TMPHERE=\$(pwd) cd /usr/src/arvados + + # This defines python_sdk_version and cwl_runner_version with python-style + # package suffixes (.dev/rc) calculate_python_sdk_cwl_package_versions + cd \$TMPHERE set -u diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl index 9bf1c20aab..1054d8f29b 100644 --- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl +++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl @@ -6,6 +6,12 @@ "$graph": [ { "class": "Workflow", + "hints": [ + { + "acrContainerImage": "999999999999999999999999999999d3+99", + "class": "http://arvados.org/cwl#WorkflowRunnerResources" + } + ], "id": "#main", "inputs": [], "outputs": [], @@ -82,4 +88,4 @@ } ], "cwlVersion": "v1.0" -} \ No newline at end of file +} diff --git a/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl b/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl index e1cacdcaf7..0005b36572 100644 --- a/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl +++ b/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl @@ -44,6 +44,7 @@ requirements: r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters}; if (r["Clusters"][inputs.this_cluster_id]) { r["Clusters"][inputs.this_cluster_id]["Login"] = {"LoginCluster": inputs.cluster_ids[0]}; + r["Clusters"][inputs.this_cluster_id]["Users"] = {"AutoAdminFirstUser": false}; } return JSON.stringify(r); } @@ -65,7 +66,7 @@ requirements: arguments: - shellQuote: false valueFrom: | - docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados + docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados-arvbox docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config $(inputs.arvbox_bin.path) sv restart api $(inputs.arvbox_bin.path) sv restart controller diff --git a/sdk/cwl/tests/federation/arvboxcwl/start.cwl b/sdk/cwl/tests/federation/arvboxcwl/start.cwl index c933de254a..2c453f768c 100644 --- a/sdk/cwl/tests/federation/arvboxcwl/start.cwl +++ b/sdk/cwl/tests/federation/arvboxcwl/start.cwl @@ -98,4 +98,4 @@ arguments: $(inputs.arvbox_bin.path) restart $(inputs.arvbox_mode) fi $(inputs.arvbox_bin.path) status > status.txt - $(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt + $(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/superuser_token > superuser_token.txt diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 0698db70ff..7aaa27fd63 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -68,7 +68,7 @@ def stubs(func): stubs.keep_client = keep_client2 stubs.docker_images = { "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")], - "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")], + "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")], "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")], "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")], } @@ -302,8 +302,8 @@ def stubs(func): 'secret_mounts': {}, 'state': 'Committed', 'command': ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'], 'name': 'submit_wf.cwl', @@ -412,8 +412,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = [ 'arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -436,8 +436,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = [ 'arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] expect_container["use_existing"] = False @@ -468,8 +468,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=stop', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -491,8 +491,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", "--output-name="+output_name, '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -513,8 +513,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", "--debug", "--storage-classes=foo", '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -525,7 +525,7 @@ class TestSubmit(unittest.TestCase): stubs.expect_container_request_uuid + '\n') self.assertEqual(exited, 0) - @mock.patch("arvados_cwl.task_queue.TaskQueue") + @mock.patch("cwltool.task_queue.TaskQueue") @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job") @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection") @stubs @@ -546,7 +546,7 @@ class TestSubmit(unittest.TestCase): make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz') self.assertEqual(exited, 0) - @mock.patch("arvados_cwl.task_queue.TaskQueue") + @mock.patch("cwltool.task_queue.TaskQueue") @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job") @mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection") @stubs @@ -576,8 +576,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', "--intermediate-output-ttl=3600", @@ -599,8 +599,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', "--trash-intermediate", @@ -623,8 +623,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", "--output-tags="+output_tags, '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -700,8 +700,8 @@ class TestSubmit(unittest.TestCase): 'name': 'expect_arvworkflow.cwl#main', 'container_image': '999999999999999999999999999999d3+99', 'command': ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'], 'cwd': '/var/spool/cwl', @@ -771,7 +771,7 @@ class TestSubmit(unittest.TestCase): ], 'requirements': [ { - 'dockerPull': 'debian:8', + 'dockerPull': 'debian:buster-slim', 'class': 'DockerRequirement', "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99" } @@ -795,8 +795,8 @@ class TestSubmit(unittest.TestCase): 'name': 'a test workflow', 'container_image': "999999999999999999999999999999d3+99", 'command': ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'], 'cwd': '/var/spool/cwl', @@ -859,8 +859,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["owner_uuid"] = project_uuid expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - "--eval-timeout=20", "--thread-count=1", + '--no-log-timestamps', '--disable-validate', '--disable-color', + "--eval-timeout=20", "--thread-count=4", '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '--project-uuid='+project_uuid, @@ -881,8 +881,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=60.0', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=60.0', '--thread-count=4', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -902,8 +902,8 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=500", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -924,7 +924,7 @@ class TestSubmit(unittest.TestCase): expect_container = copy.deepcopy(stubs.expect_container_spec) expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', + '--no-log-timestamps', '--disable-validate', '--disable-color', '--eval-timeout=20', '--thread-count=20', '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue', @@ -994,8 +994,8 @@ class TestSubmit(unittest.TestCase): "arv": "http://arvados.org/cwl#", } expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers', - '--no-log-timestamps', '--disable-validate', - '--eval-timeout=20', '--thread-count=1', + '--no-log-timestamps', '--disable-validate', '--disable-color', + '--eval-timeout=20', '--thread-count=4', '--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'] @@ -1059,8 +1059,9 @@ class TestSubmit(unittest.TestCase): "--api=containers", "--no-log-timestamps", "--disable-validate", + "--disable-color", "--eval-timeout=20", - '--thread-count=1', + '--thread-count=4', "--enable-reuse", "--collection-cache-size=256", '--debug', @@ -1133,7 +1134,7 @@ class TestSubmit(unittest.TestCase): "hints": [ { "class": "DockerRequirement", - "dockerPull": "debian:8", + "dockerPull": "debian:buster-slim", "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99" }, { @@ -1354,7 +1355,7 @@ class TestSubmit(unittest.TestCase): class TestCreateWorkflow(unittest.TestCase): existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml" expect_workflow = StripYAMLComments( - open("tests/wf/expect_packed.cwl").read()) + open("tests/wf/expect_upload_packed.cwl").read().rstrip()) @stubs def test_create(self, stubs): @@ -1472,7 +1473,7 @@ class TestCreateWorkflow(unittest.TestCase): stubs.capture_stdout, sys.stderr, api_client=stubs.api) toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl" - expect_workflow = StripYAMLComments(open(toolfile).read()) + expect_workflow = StripYAMLComments(open(toolfile).read().rstrip()) body = { "workflow": { diff --git a/sdk/cwl/tests/test_tq.py b/sdk/cwl/tests/test_tq.py index a094890650..05e5116d72 100644 --- a/sdk/cwl/tests/test_tq.py +++ b/sdk/cwl/tests/test_tq.py @@ -11,7 +11,7 @@ import logging import os import threading -from arvados_cwl.task_queue import TaskQueue +from cwltool.task_queue import TaskQueue def success_task(): pass diff --git a/sdk/cwl/tests/tool/submit_tool.cwl b/sdk/cwl/tests/tool/submit_tool.cwl index aadbd56351..f8193d9f63 100644 --- a/sdk/cwl/tests/tool/submit_tool.cwl +++ b/sdk/cwl/tests/tool/submit_tool.cwl @@ -11,7 +11,7 @@ class: CommandLineTool cwlVersion: v1.0 requirements: - class: DockerRequirement - dockerPull: debian:8 + dockerPull: debian:buster-slim inputs: - id: x type: File diff --git a/sdk/cwl/tests/tool/tool_with_sf.cwl b/sdk/cwl/tests/tool/tool_with_sf.cwl index 0beb7ad78f..c0c3c7a6b7 100644 --- a/sdk/cwl/tests/tool/tool_with_sf.cwl +++ b/sdk/cwl/tests/tool/tool_with_sf.cwl @@ -11,7 +11,7 @@ class: CommandLineTool cwlVersion: v1.0 requirements: - class: DockerRequirement - dockerPull: debian:8 + dockerPull: debian:buster-slim inputs: - id: x type: File diff --git a/sdk/cwl/tests/wf/16169-step.cwl b/sdk/cwl/tests/wf/16169-step.cwl index ce6f2c0c93..69054f569d 100644 --- a/sdk/cwl/tests/wf/16169-step.cwl +++ b/sdk/cwl/tests/wf/16169-step.cwl @@ -7,7 +7,7 @@ cwlVersion: v1.0 requirements: InlineJavascriptRequirement: {} DockerRequirement: - dockerPull: debian:stretch-slim + dockerPull: debian:buster-slim inputs: d: Directory outputs: diff --git a/sdk/cwl/tests/wf/expect_arvworkflow.cwl b/sdk/cwl/tests/wf/expect_arvworkflow.cwl index 5739ddc7b4..116adcbf66 100644 --- a/sdk/cwl/tests/wf/expect_arvworkflow.cwl +++ b/sdk/cwl/tests/wf/expect_arvworkflow.cwl @@ -25,4 +25,4 @@ $graph: type: string outputs: [] requirements: - - {class: DockerRequirement, dockerPull: 'debian:8'} + - {class: DockerRequirement, dockerPull: 'debian:buster-slim'} diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl index cb2e5ff56e..4715c10a5e 100644 --- a/sdk/cwl/tests/wf/expect_packed.cwl +++ b/sdk/cwl/tests/wf/expect_packed.cwl @@ -25,7 +25,7 @@ "requirements": [ { "class": "DockerRequirement", - "dockerPull": "debian:8", + "dockerPull": "debian:buster-slim", "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99" } ] diff --git a/sdk/cwl/tests/wf/expect_upload_packed.cwl b/sdk/cwl/tests/wf/expect_upload_packed.cwl new file mode 100644 index 0000000000..0b13e3a819 --- /dev/null +++ b/sdk/cwl/tests/wf/expect_upload_packed.cwl @@ -0,0 +1,100 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +{ + "$graph": [ + { + "baseCommand": "cat", + "class": "CommandLineTool", + "id": "#submit_tool.cwl", + "inputs": [ + { + "default": { + "class": "File", + "location": "keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt" + }, + "id": "#submit_tool.cwl/x", + "inputBinding": { + "position": 1 + }, + "type": "File" + } + ], + "outputs": [], + "requirements": [ + { + "class": "DockerRequirement", + "dockerPull": "debian:buster-slim", + "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99" + } + ] + }, + { + "class": "Workflow", + "hints": [ + { + "acrContainerImage": "999999999999999999999999999999d3+99", + "class": "http://arvados.org/cwl#WorkflowRunnerResources" + } + ], + "id": "#main", + "inputs": [ + { + "default": { + "basename": "blorp.txt", + "class": "File", + "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt", + "nameext": ".txt", + "nameroot": "blorp", + "size": 16 + }, + "id": "#main/x", + "type": "File" + }, + { + "default": { + "basename": "99999999999999999999999999999998+99", + "class": "Directory", + "location": "keep:99999999999999999999999999999998+99" + }, + "id": "#main/y", + "type": "Directory" + }, + { + "default": { + "basename": "anonymous", + "class": "Directory", + "listing": [ + { + "basename": "renamed.txt", + "class": "File", + "location": "keep:99999999999999999999999999999998+99/file1.txt", + "nameext": ".txt", + "nameroot": "renamed", + "size": 0 + } + ] + }, + "id": "#main/z", + "type": "Directory" + } + ], + "outputs": [], + "steps": [ + { + "id": "#main/step1", + "in": [ + { + "id": "#main/step1/x", + "source": "#main/x" + } + ], + "out": [], + "run": "#submit_tool.cwl" + } + ] + } + ], + "cwlVersion": "v1.0" +} diff --git a/sdk/cwl/tests/wf/secret_wf.cwl b/sdk/cwl/tests/wf/secret_wf.cwl index 05d950d18c..5d2e231ec8 100644 --- a/sdk/cwl/tests/wf/secret_wf.cwl +++ b/sdk/cwl/tests/wf/secret_wf.cwl @@ -10,7 +10,7 @@ hints: "cwltool:Secrets": secrets: [pw] DockerRequirement: - dockerPull: debian:8 + dockerPull: debian:buster-slim inputs: pw: string outputs: diff --git a/sdk/cwl/tests/wf/submit_wf_packed.cwl b/sdk/cwl/tests/wf/submit_wf_packed.cwl index 83ba584b20..cd00170313 100644 --- a/sdk/cwl/tests/wf/submit_wf_packed.cwl +++ b/sdk/cwl/tests/wf/submit_wf_packed.cwl @@ -7,7 +7,7 @@ $graph: - class: CommandLineTool requirements: - class: DockerRequirement - dockerPull: debian:8 + dockerPull: debian:buster-slim 'http://arvados.org/cwl#dockerCollectionPDH': 999999999999999999999999999999d4+99 inputs: - id: '#submit_tool.cwl/x' diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile index dd067e9778..1e0068ffd4 100644 --- a/sdk/dev-jobs.dockerfile +++ b/sdk/dev-jobs.dockerfile @@ -13,13 +13,13 @@ # (This dockerfile file must be located in the arvados/sdk/ directory because # of the docker build root.) -FROM debian:9 -MAINTAINER Ward Vandewege +FROM debian:buster-slim +MAINTAINER Arvados Package Maintainers ENV DEBIAN_FRONTEND noninteractive -ARG pythoncmd=python -ARG pipcmd=pip +ARG pythoncmd=python3 +ARG pipcmd=pip3 RUN apt-get update -q && apt-get install -qy --no-install-recommends \ git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \ diff --git a/sdk/go/arvados/blob_signature.go b/sdk/go/arvados/blob_signature.go index 132939547a..2202016bcc 100644 --- a/sdk/go/arvados/blob_signature.go +++ b/sdk/go/arvados/blob_signature.go @@ -57,9 +57,8 @@ func SignManifest(manifest string, apiToken string, expiry time.Time, ttl time.D return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string { if mBlkRe.MatchString(tok) { return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret) - } else { - return tok } + return tok }) } diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go index 562c8c1e7d..52c75d5113 100644 --- a/sdk/go/arvados/client.go +++ b/sdk/go/arvados/client.go @@ -69,14 +69,14 @@ type Client struct { defaultRequestID string } -// The default http.Client used by a Client with Insecure==true and -// Client==nil. +// InsecureHTTPClient is the default http.Client used by a Client with +// Insecure==true and Client==nil. var InsecureHTTPClient = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: true}}} -// The default http.Client used by a Client otherwise. +// DefaultSecureClient is the default http.Client used by a Client otherwise. var DefaultSecureClient = &http.Client{} // NewClientFromConfig creates a new Client that uses the endpoints in @@ -306,6 +306,7 @@ func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io. return c.RequestAndDecodeContext(context.Background(), dst, method, path, body, params) } +// RequestAndDecodeContext does the same as RequestAndDecode, but with a context func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, method, path string, body io.Reader, params interface{}) error { if body, ok := body.(io.Closer); ok { // Ensure body is closed even if we error out early diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 41c20c8db2..a8d601d5f6 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -17,9 +17,8 @@ import ( var DefaultConfigFile = func() string { if path := os.Getenv("ARVADOS_CONFIG"); path != "" { return path - } else { - return "/etc/arvados/config.yml" } + return "/etc/arvados/config.yml" }() type Config struct { @@ -50,12 +49,12 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) { } } } - if cc, ok := sc.Clusters[clusterID]; !ok { + cc, ok := sc.Clusters[clusterID] + if !ok { return nil, fmt.Errorf("cluster %q is not configured", clusterID) - } else { - cc.ClusterID = clusterID - return &cc, nil } + cc.ClusterID = clusterID + return &cc, nil } type WebDAVCacheConfig struct { @@ -177,8 +176,14 @@ type Cluster struct { ProviderAppID string ProviderAppSecret string } + Test struct { + Enable bool + Users map[string]TestUser + } LoginCluster string RemoteTokenRefresh Duration + TokenLifetime Duration + TrustedClients map[string]struct{} } Mail struct { MailchimpAPIKey string @@ -215,6 +220,7 @@ type Cluster struct { UserNotifierEmailFrom string UserProfileNotificationAddress string PreferDomainForUsername string + UserSetupMailText string } Volumes map[string]Volume Workbench struct { @@ -255,6 +261,7 @@ type Cluster struct { InactivePageHTML string SSHHelpPageHTML string SSHHelpHostSuffix string + IdleTimeout Duration } ForceLegacyAPI14 bool @@ -330,6 +337,11 @@ type Service struct { ExternalURL URL } +type TestUser struct { + Email string + Password string +} + // URL is a url.URL that is also usable as a JSON key/value. type URL url.URL @@ -437,23 +449,25 @@ type ContainersConfig struct { type CloudVMsConfig struct { Enable bool - BootProbeCommand string - DeployRunnerBinary string - ImageID string - MaxCloudOpsPerSecond int - MaxProbesPerSecond int - PollInterval Duration - ProbeInterval Duration - SSHPort string - SyncInterval Duration - TimeoutBooting Duration - TimeoutIdle Duration - TimeoutProbe Duration - TimeoutShutdown Duration - TimeoutSignal Duration - TimeoutTERM Duration - ResourceTags map[string]string - TagKeyPrefix string + BootProbeCommand string + DeployRunnerBinary string + ImageID string + MaxCloudOpsPerSecond int + MaxProbesPerSecond int + MaxConcurrentInstanceCreateOps int + PollInterval Duration + ProbeInterval Duration + SSHPort string + SyncInterval Duration + TimeoutBooting Duration + TimeoutIdle Duration + TimeoutProbe Duration + TimeoutShutdown Duration + TimeoutSignal Duration + TimeoutStaleRunLock Duration + TimeoutTERM Duration + ResourceTags map[string]string + TagKeyPrefix string Driver string DriverParameters json.RawMessage diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go index 3d08f2235a..265944e81d 100644 --- a/sdk/go/arvados/container.go +++ b/sdk/go/arvados/container.go @@ -32,7 +32,7 @@ type Container struct { FinishedAt *time.Time `json:"finished_at"` // nil if not yet finished } -// Container is an arvados#container resource. +// ContainerRequest is an arvados#container_request resource. type ContainerRequest struct { UUID string `json:"uuid"` OwnerUUID string `json:"owner_uuid"` @@ -127,7 +127,7 @@ const ( ContainerStateCancelled = ContainerState("Cancelled") ) -// ContainerState is a string corresponding to a valid Container state. +// ContainerRequestState is a string corresponding to a valid Container Request state. type ContainerRequestState string const ( diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go index 5e57fed3be..aa75fee7c4 100644 --- a/sdk/go/arvados/fs_base.go +++ b/sdk/go/arvados/fs_base.go @@ -598,9 +598,8 @@ func (fs *fileSystem) remove(name string, recursive bool) error { func (fs *fileSystem) Sync() error { if syncer, ok := fs.root.(syncer); ok { return syncer.Sync() - } else { - return ErrInvalidOperation } + return ErrInvalidOperation } func (fs *fileSystem) Flush(string, bool) error { diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go index 0edc48162b..1de558a1bd 100644 --- a/sdk/go/arvados/fs_collection.go +++ b/sdk/go/arvados/fs_collection.go @@ -109,16 +109,15 @@ func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime t inodes: make(map[string]inode), }, }, nil - } else { - return &filenode{ - fs: fs, - fileinfo: fileinfo{ - name: name, - mode: perm & ^os.ModeDir, - modTime: modTime, - }, - }, nil } + return &filenode{ + fs: fs, + fileinfo: fileinfo{ + name: name, + mode: perm & ^os.ModeDir, + modTime: modTime, + }, + }, nil } func (fs *collectionFileSystem) Child(name string, replace func(inode) (inode, error)) (inode, error) { @@ -568,8 +567,6 @@ func (fn *filenode) Write(p []byte, startPtr filenodePtr) (n int, ptr filenodePt seg.Truncate(len(cando)) fn.memsize += int64(len(cando)) fn.segments[cur] = seg - cur++ - prev++ } } @@ -731,12 +728,11 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize // it fails, we'll try again next time. close(done) return nil - } else { - // In sync mode, we proceed regardless of - // whether another flush is in progress: It - // can't finish before we do, because we hold - // fn's lock until we finish our own writes. } + // In sync mode, we proceed regardless of + // whether another flush is in progress: It + // can't finish before we do, because we hold + // fn's lock until we finish our own writes. seg.flushing = done offsets = append(offsets, len(block)) if len(refs) == 1 { @@ -804,9 +800,8 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize }() if sync { return <-errs - } else { - return nil } + return nil } type flushOpts struct { @@ -1109,9 +1104,9 @@ func (dn *dirnode) loadManifest(txt string) error { // situation might be rare anyway) segIdx, pos = 0, 0 } - for next := int64(0); segIdx < len(segments); segIdx++ { + for ; segIdx < len(segments); segIdx++ { seg := segments[segIdx] - next = pos + int64(seg.Len()) + next := pos + int64(seg.Len()) if next <= offset || seg.Len() == 0 { pos = next continue diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go index cb2e54bda2..86facd681e 100644 --- a/sdk/go/arvados/fs_project_test.go +++ b/sdk/go/arvados/fs_project_test.go @@ -214,6 +214,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) { // Ensure collection was flushed by Sync var latest Collection err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil) + c.Check(err, check.IsNil) c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`) // Delete test.txt behind s.fs's back by updating the diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go index da1710374e..eb7988422d 100644 --- a/sdk/go/arvados/keep_service.go +++ b/sdk/go/arvados/keep_service.go @@ -140,7 +140,7 @@ func (s *KeepService) Untrash(ctx context.Context, c *Client, blk string) error return nil } -// Index returns an unsorted list of blocks at the given mount point. +// IndexMount returns an unsorted list of blocks at the given mount point. func (s *KeepService) IndexMount(ctx context.Context, c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) { return s.index(ctx, c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix)) } diff --git a/sdk/go/arvados/link.go b/sdk/go/arvados/link.go index fdddfc537d..f7d1f35a3c 100644 --- a/sdk/go/arvados/link.go +++ b/sdk/go/arvados/link.go @@ -17,7 +17,7 @@ type Link struct { Properties map[string]interface{} `json:"properties"` } -// UserList is an arvados#userList resource. +// LinkList is an arvados#linkList resource. type LinkList struct { Items []Link `json:"items"` ItemsAvailable int `json:"items_available"` diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go index e2c0466627..d90c618f7a 100644 --- a/sdk/go/arvadosclient/arvadosclient.go +++ b/sdk/go/arvadosclient/arvadosclient.go @@ -50,7 +50,7 @@ var ( defaultHTTPClientMtx sync.Mutex ) -// Indicates an error that was returned by the API server. +// APIServerError contains an error that was returned by the API server. type APIServerError struct { // Address of server returning error, of the form "host:port". ServerAddress string @@ -70,12 +70,11 @@ func (e APIServerError) Error() string { e.HttpStatusCode, e.HttpStatusMessage, e.ServerAddress) - } else { - return fmt.Sprintf("arvados API server error: %d: %s returned by %s", - e.HttpStatusCode, - e.HttpStatusMessage, - e.ServerAddress) } + return fmt.Sprintf("arvados API server error: %d: %s returned by %s", + e.HttpStatusCode, + e.HttpStatusMessage, + e.ServerAddress) } // StringBool tests whether s is suggestive of true. It returns true @@ -85,10 +84,10 @@ func StringBool(s string) bool { return s == "1" || s == "yes" || s == "true" } -// Helper type so we don't have to write out 'map[string]interface{}' every time. +// Dict is a helper type so we don't have to write out 'map[string]interface{}' every time. type Dict map[string]interface{} -// Information about how to contact the Arvados server +// ArvadosClient contains information about how to contact the Arvados server type ArvadosClient struct { // https Scheme string @@ -211,7 +210,7 @@ func (c *ArvadosClient) CallRaw(method string, resourceType string, uuid string, Scheme: scheme, Host: c.ApiServer} - if resourceType != API_DISCOVERY_RESOURCE { + if resourceType != ApiDiscoveryResource { u.Path = "/arvados/v1" } @@ -379,7 +378,7 @@ func (c *ArvadosClient) Delete(resource string, uuid string, parameters Dict, ou return c.Call("DELETE", resource, uuid, "", parameters, output) } -// Modify attributes of a resource. See Call for argument descriptions. +// Update attributes of a resource. See Call for argument descriptions. func (c *ArvadosClient) Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error) { return c.Call("PUT", resourceType, uuid, "", parameters, output) } @@ -401,7 +400,7 @@ func (c *ArvadosClient) List(resource string, parameters Dict, output interface{ return c.Call("GET", resource, "", "", parameters, output) } -const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest" +const ApiDiscoveryResource = "discovery/v1/apis/arvados/v1/rest" // Discovery returns the value of the given parameter in the discovery // document. Returns a non-nil error if the discovery document cannot @@ -410,7 +409,7 @@ const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest" func (c *ArvadosClient) Discovery(parameter string) (value interface{}, err error) { if len(c.DiscoveryDoc) == 0 { c.DiscoveryDoc = make(Dict) - err = c.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &c.DiscoveryDoc) + err = c.Call("GET", ApiDiscoveryResource, "", "", nil, &c.DiscoveryDoc) if err != nil { return nil, err } @@ -420,24 +419,23 @@ func (c *ArvadosClient) Discovery(parameter string) (value interface{}, err erro value, found = c.DiscoveryDoc[parameter] if found { return value, nil - } else { - return value, ErrInvalidArgument } + return value, ErrInvalidArgument } -func (ac *ArvadosClient) httpClient() *http.Client { - if ac.Client != nil { - return ac.Client +func (c *ArvadosClient) httpClient() *http.Client { + if c.Client != nil { + return c.Client } - c := &defaultSecureHTTPClient - if ac.ApiInsecure { - c = &defaultInsecureHTTPClient + cl := &defaultSecureHTTPClient + if c.ApiInsecure { + cl = &defaultInsecureHTTPClient } - if *c == nil { + if *cl == nil { defaultHTTPClientMtx.Lock() defer defaultHTTPClientMtx.Unlock() - *c = &http.Client{Transport: &http.Transport{ - TLSClientConfig: MakeTLSConfig(ac.ApiInsecure)}} + *cl = &http.Client{Transport: &http.Transport{ + TLSClientConfig: MakeTLSConfig(c.ApiInsecure)}} } - return *c + return *cl } diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go index fa5f539360..039d7ae116 100644 --- a/sdk/go/arvadostest/api.go +++ b/sdk/go/arvadostest/api.go @@ -30,163 +30,163 @@ func (as *APIStub) BaseURL() url.URL { return url.URL{Scheme: "https", Host: "apistub.example.com"} } func (as *APIStub) ConfigGet(ctx context.Context) (json.RawMessage, error) { - as.appendCall(as.ConfigGet, ctx, nil) + as.appendCall(ctx, as.ConfigGet, nil) return nil, as.Error } func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) { - as.appendCall(as.Login, ctx, options) + as.appendCall(ctx, as.Login, options) return arvados.LoginResponse{}, as.Error } func (as *APIStub) Logout(ctx context.Context, options arvados.LogoutOptions) (arvados.LogoutResponse, error) { - as.appendCall(as.Logout, ctx, options) + as.appendCall(ctx, as.Logout, options) return arvados.LogoutResponse{}, as.Error } func (as *APIStub) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) { - as.appendCall(as.CollectionCreate, ctx, options) + as.appendCall(ctx, as.CollectionCreate, options) return arvados.Collection{}, as.Error } func (as *APIStub) CollectionUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Collection, error) { - as.appendCall(as.CollectionUpdate, ctx, options) + as.appendCall(ctx, as.CollectionUpdate, options) return arvados.Collection{}, as.Error } func (as *APIStub) CollectionGet(ctx context.Context, options arvados.GetOptions) (arvados.Collection, error) { - as.appendCall(as.CollectionGet, ctx, options) + as.appendCall(ctx, as.CollectionGet, options) return arvados.Collection{}, as.Error } func (as *APIStub) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) { - as.appendCall(as.CollectionList, ctx, options) + as.appendCall(ctx, as.CollectionList, options) return arvados.CollectionList{}, as.Error } func (as *APIStub) CollectionProvenance(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) { - as.appendCall(as.CollectionProvenance, ctx, options) + as.appendCall(ctx, as.CollectionProvenance, options) return nil, as.Error } func (as *APIStub) CollectionUsedBy(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) { - as.appendCall(as.CollectionUsedBy, ctx, options) + as.appendCall(ctx, as.CollectionUsedBy, options) return nil, as.Error } func (as *APIStub) CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) { - as.appendCall(as.CollectionDelete, ctx, options) + as.appendCall(ctx, as.CollectionDelete, options) return arvados.Collection{}, as.Error } func (as *APIStub) CollectionTrash(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) { - as.appendCall(as.CollectionTrash, ctx, options) + as.appendCall(ctx, as.CollectionTrash, options) return arvados.Collection{}, as.Error } func (as *APIStub) CollectionUntrash(ctx context.Context, options arvados.UntrashOptions) (arvados.Collection, error) { - as.appendCall(as.CollectionUntrash, ctx, options) + as.appendCall(ctx, as.CollectionUntrash, options) return arvados.Collection{}, as.Error } func (as *APIStub) ContainerCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Container, error) { - as.appendCall(as.ContainerCreate, ctx, options) + as.appendCall(ctx, as.ContainerCreate, options) return arvados.Container{}, as.Error } func (as *APIStub) ContainerUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) { - as.appendCall(as.ContainerUpdate, ctx, options) + as.appendCall(ctx, as.ContainerUpdate, options) return arvados.Container{}, as.Error } func (as *APIStub) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) { - as.appendCall(as.ContainerGet, ctx, options) + as.appendCall(ctx, as.ContainerGet, options) return arvados.Container{}, as.Error } func (as *APIStub) ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error) { - as.appendCall(as.ContainerList, ctx, options) + as.appendCall(ctx, as.ContainerList, options) return arvados.ContainerList{}, as.Error } func (as *APIStub) ContainerDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Container, error) { - as.appendCall(as.ContainerDelete, ctx, options) + as.appendCall(ctx, as.ContainerDelete, options) return arvados.Container{}, as.Error } func (as *APIStub) ContainerLock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) { - as.appendCall(as.ContainerLock, ctx, options) + as.appendCall(ctx, as.ContainerLock, options) return arvados.Container{}, as.Error } func (as *APIStub) ContainerUnlock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) { - as.appendCall(as.ContainerUnlock, ctx, options) + as.appendCall(ctx, as.ContainerUnlock, options) return arvados.Container{}, as.Error } func (as *APIStub) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) { - as.appendCall(as.SpecimenCreate, ctx, options) + as.appendCall(ctx, as.SpecimenCreate, options) return arvados.Specimen{}, as.Error } func (as *APIStub) SpecimenUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Specimen, error) { - as.appendCall(as.SpecimenUpdate, ctx, options) + as.appendCall(ctx, as.SpecimenUpdate, options) return arvados.Specimen{}, as.Error } func (as *APIStub) SpecimenGet(ctx context.Context, options arvados.GetOptions) (arvados.Specimen, error) { - as.appendCall(as.SpecimenGet, ctx, options) + as.appendCall(ctx, as.SpecimenGet, options) return arvados.Specimen{}, as.Error } func (as *APIStub) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) { - as.appendCall(as.SpecimenList, ctx, options) + as.appendCall(ctx, as.SpecimenList, options) return arvados.SpecimenList{}, as.Error } func (as *APIStub) SpecimenDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Specimen, error) { - as.appendCall(as.SpecimenDelete, ctx, options) + as.appendCall(ctx, as.SpecimenDelete, options) return arvados.Specimen{}, as.Error } func (as *APIStub) UserCreate(ctx context.Context, options arvados.CreateOptions) (arvados.User, error) { - as.appendCall(as.UserCreate, ctx, options) + as.appendCall(ctx, as.UserCreate, options) return arvados.User{}, as.Error } func (as *APIStub) UserUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.User, error) { - as.appendCall(as.UserUpdate, ctx, options) + as.appendCall(ctx, as.UserUpdate, options) return arvados.User{}, as.Error } func (as *APIStub) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) { - as.appendCall(as.UserUpdateUUID, ctx, options) + as.appendCall(ctx, as.UserUpdateUUID, options) return arvados.User{}, as.Error } func (as *APIStub) UserActivate(ctx context.Context, options arvados.UserActivateOptions) (arvados.User, error) { - as.appendCall(as.UserActivate, ctx, options) + as.appendCall(ctx, as.UserActivate, options) return arvados.User{}, as.Error } func (as *APIStub) UserSetup(ctx context.Context, options arvados.UserSetupOptions) (map[string]interface{}, error) { - as.appendCall(as.UserSetup, ctx, options) + as.appendCall(ctx, as.UserSetup, options) return nil, as.Error } func (as *APIStub) UserUnsetup(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - as.appendCall(as.UserUnsetup, ctx, options) + as.appendCall(ctx, as.UserUnsetup, options) return arvados.User{}, as.Error } func (as *APIStub) UserGet(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - as.appendCall(as.UserGet, ctx, options) + as.appendCall(ctx, as.UserGet, options) return arvados.User{}, as.Error } func (as *APIStub) UserGetCurrent(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - as.appendCall(as.UserGetCurrent, ctx, options) + as.appendCall(ctx, as.UserGetCurrent, options) return arvados.User{}, as.Error } func (as *APIStub) UserGetSystem(ctx context.Context, options arvados.GetOptions) (arvados.User, error) { - as.appendCall(as.UserGetSystem, ctx, options) + as.appendCall(ctx, as.UserGetSystem, options) return arvados.User{}, as.Error } func (as *APIStub) UserList(ctx context.Context, options arvados.ListOptions) (arvados.UserList, error) { - as.appendCall(as.UserList, ctx, options) + as.appendCall(ctx, as.UserList, options) return arvados.UserList{}, as.Error } func (as *APIStub) UserDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.User, error) { - as.appendCall(as.UserDelete, ctx, options) + as.appendCall(ctx, as.UserDelete, options) return arvados.User{}, as.Error } func (as *APIStub) UserMerge(ctx context.Context, options arvados.UserMergeOptions) (arvados.User, error) { - as.appendCall(as.UserMerge, ctx, options) + as.appendCall(ctx, as.UserMerge, options) return arvados.User{}, as.Error } func (as *APIStub) UserBatchUpdate(ctx context.Context, options arvados.UserBatchUpdateOptions) (arvados.UserList, error) { - as.appendCall(as.UserBatchUpdate, ctx, options) + as.appendCall(ctx, as.UserBatchUpdate, options) return arvados.UserList{}, as.Error } func (as *APIStub) UserAuthenticate(ctx context.Context, options arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) { - as.appendCall(as.UserAuthenticate, ctx, options) + as.appendCall(ctx, as.UserAuthenticate, options) return arvados.APIClientAuthorization{}, as.Error } func (as *APIStub) APIClientAuthorizationCurrent(ctx context.Context, options arvados.GetOptions) (arvados.APIClientAuthorization, error) { - as.appendCall(as.APIClientAuthorizationCurrent, ctx, options) + as.appendCall(ctx, as.APIClientAuthorizationCurrent, options) return arvados.APIClientAuthorization{}, as.Error } -func (as *APIStub) appendCall(method interface{}, ctx context.Context, options interface{}) { +func (as *APIStub) appendCall(ctx context.Context, method interface{}, options interface{}) { as.mtx.Lock() defer as.mtx.Unlock() as.calls = append(as.calls, APIStubCall{method, ctx, options}) diff --git a/sdk/go/arvadostest/db.go b/sdk/go/arvadostest/db.go index 41ecfacc48..c20f61db26 100644 --- a/sdk/go/arvadostest/db.go +++ b/sdk/go/arvadostest/db.go @@ -10,6 +10,7 @@ import ( "git.arvados.org/arvados.git/lib/ctrlctx" "git.arvados.org/arvados.git/sdk/go/arvados" "github.com/jmoiron/sqlx" + // sqlx needs lib/pq to talk to PostgreSQL _ "github.com/lib/pq" "gopkg.in/check.v1" ) diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go index 5677f4deca..aeb5a47e6d 100644 --- a/sdk/go/arvadostest/fixtures.go +++ b/sdk/go/arvadostest/fixtures.go @@ -44,7 +44,27 @@ const ( RunningContainerUUID = "zzzzz-dz642-runningcontainr" - CompletedContainerUUID = "zzzzz-dz642-compltcontainer" + CompletedContainerUUID = "zzzzz-dz642-compltcontainer" + CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr" + CompletedContainerRequestUUID2 = "zzzzz-xvhdp-cr4completedcr2" + + CompletedDiagnosticsContainerRequest1UUID = "zzzzz-xvhdp-diagnostics0001" + CompletedDiagnosticsContainerRequest2UUID = "zzzzz-xvhdp-diagnostics0002" + CompletedDiagnosticsContainer1UUID = "zzzzz-dz642-diagcompreq0001" + CompletedDiagnosticsContainer2UUID = "zzzzz-dz642-diagcompreq0002" + DiagnosticsContainerRequest1LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog1" + DiagnosticsContainerRequest2LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog2" + + CompletedDiagnosticsHasher1ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0001" + CompletedDiagnosticsHasher2ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0002" + CompletedDiagnosticsHasher3ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0003" + CompletedDiagnosticsHasher1ContainerUUID = "zzzzz-dz642-diagcomphasher1" + CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2" + CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3" + + Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001" + Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002" + Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003" ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123" ArvadosRepoName = "arvados" @@ -73,6 +93,9 @@ const ( TestVMUUID = "zzzzz-2x53u-382brsig8rp3064" CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud" + + LogCollectionUUID = "zzzzz-4zz18-logcollection01" + LogCollectionUUID2 = "zzzzz-4zz18-logcollection02" ) // PathologicalManifest : A valid manifest designed to test diff --git a/sdk/go/arvadostest/oidc_provider.go b/sdk/go/arvadostest/oidc_provider.go new file mode 100644 index 0000000000..96205f919f --- /dev/null +++ b/sdk/go/arvadostest/oidc_provider.go @@ -0,0 +1,174 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package arvadostest + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "time" + + "gopkg.in/check.v1" + "gopkg.in/square/go-jose.v2" +) + +type OIDCProvider struct { + // expected token request + ValidCode string + ValidClientID string + ValidClientSecret string + // desired response from token endpoint + AuthEmail string + AuthEmailVerified bool + AuthName string + + PeopleAPIResponse map[string]interface{} + + key *rsa.PrivateKey + Issuer *httptest.Server + PeopleAPI *httptest.Server + c *check.C +} + +func NewOIDCProvider(c *check.C) *OIDCProvider { + p := &OIDCProvider{c: c} + var err error + p.key, err = rsa.GenerateKey(rand.Reader, 2048) + c.Assert(err, check.IsNil) + p.Issuer = httptest.NewServer(http.HandlerFunc(p.serveOIDC)) + p.PeopleAPI = httptest.NewServer(http.HandlerFunc(p.servePeopleAPI)) + return p +} + +func (p *OIDCProvider) ValidAccessToken() string { + return p.fakeToken([]byte("fake access token")) +} + +func (p *OIDCProvider) serveOIDC(w http.ResponseWriter, req *http.Request) { + req.ParseForm() + p.c.Logf("serveOIDC: got req: %s %s %s", req.Method, req.URL, req.Form) + w.Header().Set("Content-Type", "application/json") + switch req.URL.Path { + case "/.well-known/openid-configuration": + json.NewEncoder(w).Encode(map[string]interface{}{ + "issuer": p.Issuer.URL, + "authorization_endpoint": p.Issuer.URL + "/auth", + "token_endpoint": p.Issuer.URL + "/token", + "jwks_uri": p.Issuer.URL + "/jwks", + "userinfo_endpoint": p.Issuer.URL + "/userinfo", + }) + case "/token": + var clientID, clientSecret string + auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic ")) + authsplit := strings.Split(string(auth), ":") + if len(authsplit) == 2 { + clientID, _ = url.QueryUnescape(authsplit[0]) + clientSecret, _ = url.QueryUnescape(authsplit[1]) + } + if clientID != p.ValidClientID || clientSecret != p.ValidClientSecret { + p.c.Logf("OIDCProvider: expected (%q, %q) got (%q, %q)", p.ValidClientID, p.ValidClientSecret, clientID, clientSecret) + w.WriteHeader(http.StatusUnauthorized) + return + } + + if req.Form.Get("code") != p.ValidCode || p.ValidCode == "" { + w.WriteHeader(http.StatusUnauthorized) + return + } + idToken, _ := json.Marshal(map[string]interface{}{ + "iss": p.Issuer.URL, + "aud": []string{clientID}, + "sub": "fake-user-id", + "exp": time.Now().UTC().Add(time.Minute).Unix(), + "iat": time.Now().UTC().Unix(), + "nonce": "fake-nonce", + "email": p.AuthEmail, + "email_verified": p.AuthEmailVerified, + "name": p.AuthName, + "alt_verified": true, // for custom claim tests + "alt_email": "alt_email@example.com", // for custom claim tests + "alt_username": "desired-username", // for custom claim tests + }) + json.NewEncoder(w).Encode(struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int32 `json:"expires_in"` + IDToken string `json:"id_token"` + }{ + AccessToken: p.ValidAccessToken(), + TokenType: "Bearer", + RefreshToken: "test-refresh-token", + ExpiresIn: 30, + IDToken: p.fakeToken(idToken), + }) + case "/jwks": + json.NewEncoder(w).Encode(jose.JSONWebKeySet{ + Keys: []jose.JSONWebKey{ + {Key: p.key.Public(), Algorithm: string(jose.RS256), KeyID: ""}, + }, + }) + case "/auth": + w.WriteHeader(http.StatusInternalServerError) + case "/userinfo": + if authhdr := req.Header.Get("Authorization"); strings.TrimPrefix(authhdr, "Bearer ") != p.ValidAccessToken() { + p.c.Logf("OIDCProvider: bad auth %q", authhdr) + w.WriteHeader(http.StatusUnauthorized) + return + } + json.NewEncoder(w).Encode(map[string]interface{}{ + "sub": "fake-user-id", + "name": p.AuthName, + "given_name": p.AuthName, + "family_name": "", + "alt_username": "desired-username", + "email": p.AuthEmail, + "email_verified": p.AuthEmailVerified, + }) + default: + w.WriteHeader(http.StatusNotFound) + } +} + +func (p *OIDCProvider) servePeopleAPI(w http.ResponseWriter, req *http.Request) { + req.ParseForm() + p.c.Logf("servePeopleAPI: got req: %s %s %s", req.Method, req.URL, req.Form) + w.Header().Set("Content-Type", "application/json") + switch req.URL.Path { + case "/v1/people/me": + if f := req.Form.Get("personFields"); f != "emailAddresses,names" { + w.WriteHeader(http.StatusBadRequest) + break + } + json.NewEncoder(w).Encode(p.PeopleAPIResponse) + default: + w.WriteHeader(http.StatusNotFound) + } +} + +func (p *OIDCProvider) fakeToken(payload []byte) string { + signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: p.key}, nil) + if err != nil { + p.c.Error(err) + return "" + } + object, err := signer.Sign(payload) + if err != nil { + p.c.Error(err) + return "" + } + t, err := object.CompactSerialize() + if err != nil { + p.c.Error(err) + return "" + } + p.c.Logf("fakeToken(%q) == %q", payload, t) + return t +} diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go index b6a85e05e7..f1c2e243b5 100644 --- a/sdk/go/auth/auth.go +++ b/sdk/go/auth/auth.go @@ -97,7 +97,7 @@ func (a *Credentials) loadTokenFromCookie(r *http.Request) { a.Tokens = append(a.Tokens, string(token)) } -// LoadTokensFromHTTPRequestBody() loads credentials from the request +// LoadTokensFromHTTPRequestBody loads credentials from the request // body. // // This is separate from LoadTokensFromHTTPRequest() because it's not diff --git a/sdk/go/auth/salt.go b/sdk/go/auth/salt.go index 667a30f5ef..2140215986 100644 --- a/sdk/go/auth/salt.go +++ b/sdk/go/auth/salt.go @@ -26,9 +26,8 @@ func SaltToken(token, remote string) (string, error) { if len(parts) < 3 || parts[0] != "v2" { if reObsoleteToken.MatchString(token) { return "", ErrObsoleteToken - } else { - return "", ErrTokenFormat } + return "", ErrTokenFormat } uuid := parts[1] secret := parts[2] diff --git a/sdk/go/blockdigest/blockdigest.go b/sdk/go/blockdigest/blockdigest.go index b9ecc45abc..ecb09964ec 100644 --- a/sdk/go/blockdigest/blockdigest.go +++ b/sdk/go/blockdigest/blockdigest.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -// Stores a Block Locator Digest compactly. Can be used as a map key. +// Package blockdigest stores a Block Locator Digest compactly. Can be used as a map key. package blockdigest import ( @@ -15,8 +15,8 @@ import ( var LocatorPattern = regexp.MustCompile( "^[0-9a-fA-F]{32}\\+[0-9]+(\\+[A-Z][A-Za-z0-9@_-]*)*$") -// Stores a Block Locator Digest compactly, up to 128 bits. -// Can be used as a map key. +// BlockDigest stores a Block Locator Digest compactly, up to 128 bits. Can be +// used as a map key. type BlockDigest struct { H uint64 L uint64 @@ -41,7 +41,7 @@ func (w DigestWithSize) String() string { return fmt.Sprintf("%s+%d", w.Digest.String(), w.Size) } -// Will create a new BlockDigest unless an error is encountered. +// FromString creates a new BlockDigest unless an error is encountered. func FromString(s string) (dig BlockDigest, err error) { if len(s) != 32 { err = fmt.Errorf("Block digest should be exactly 32 characters but this one is %d: %s", len(s), s) diff --git a/sdk/go/blockdigest/blockdigest_test.go b/sdk/go/blockdigest/blockdigest_test.go index a9994f7047..9e8f9a4a0f 100644 --- a/sdk/go/blockdigest/blockdigest_test.go +++ b/sdk/go/blockdigest/blockdigest_test.go @@ -13,8 +13,8 @@ import ( func getStackTrace() string { buf := make([]byte, 1000) - bytes_written := runtime.Stack(buf, false) - return "Stack Trace:\n" + string(buf[:bytes_written]) + bytesWritten := runtime.Stack(buf, false) + return "Stack Trace:\n" + string(buf[:bytesWritten]) } func expectEqual(t *testing.T, actual interface{}, expected interface{}) { diff --git a/sdk/go/blockdigest/testing.go b/sdk/go/blockdigest/testing.go index 7716a71b20..6c7d3bf1e2 100644 --- a/sdk/go/blockdigest/testing.go +++ b/sdk/go/blockdigest/testing.go @@ -6,7 +6,7 @@ package blockdigest -// Just used for testing when we need some distinct BlockDigests +// MakeTestBlockDigest is used for testing with distinct BlockDigests func MakeTestBlockDigest(i int) BlockDigest { return BlockDigest{L: uint64(i)} } diff --git a/sdk/go/health/handler_test.go b/sdk/go/health/handler_test.go index c9f6a0b675..097e292d38 100644 --- a/sdk/go/health/handler_test.go +++ b/sdk/go/health/handler_test.go @@ -81,9 +81,8 @@ func (s *Suite) TestPingOverride(c *check.C) { ok = !ok if ok { return nil - } else { - return errors.New("good error") } + return errors.New("good error") }, }, } diff --git a/sdk/go/httpserver/logger.go b/sdk/go/httpserver/logger.go index 59981e3e55..5336488df0 100644 --- a/sdk/go/httpserver/logger.go +++ b/sdk/go/httpserver/logger.go @@ -64,9 +64,8 @@ func rewrapResponseWriter(w http.ResponseWriter, wrapped http.ResponseWriter) ht http.ResponseWriter http.Hijacker }{w, hijacker} - } else { - return w } + return w } func Logger(req *http.Request) logrus.FieldLogger { diff --git a/sdk/go/keepclient/hashcheck.go b/sdk/go/keepclient/hashcheck.go index 9295c14cc2..0966e072ea 100644 --- a/sdk/go/keepclient/hashcheck.go +++ b/sdk/go/keepclient/hashcheck.go @@ -29,36 +29,36 @@ type HashCheckingReader struct { // Reads from the underlying reader, update the hashing function, and // pass the results through. Returns BadChecksum (instead of EOF) on // the last read if the checksum doesn't match. -func (this HashCheckingReader) Read(p []byte) (n int, err error) { - n, err = this.Reader.Read(p) +func (hcr HashCheckingReader) Read(p []byte) (n int, err error) { + n, err = hcr.Reader.Read(p) if n > 0 { - this.Hash.Write(p[:n]) + hcr.Hash.Write(p[:n]) } if err == io.EOF { - sum := this.Hash.Sum(nil) - if fmt.Sprintf("%x", sum) != this.Check { + sum := hcr.Hash.Sum(nil) + if fmt.Sprintf("%x", sum) != hcr.Check { err = BadChecksum } } return n, err } -// WriteTo writes the entire contents of this.Reader to dest. Returns +// WriteTo writes the entire contents of hcr.Reader to dest. Returns // BadChecksum if writing is successful but the checksum doesn't // match. -func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) { - if writeto, ok := this.Reader.(io.WriterTo); ok { - written, err = writeto.WriteTo(io.MultiWriter(dest, this.Hash)) +func (hcr HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) { + if writeto, ok := hcr.Reader.(io.WriterTo); ok { + written, err = writeto.WriteTo(io.MultiWriter(dest, hcr.Hash)) } else { - written, err = io.Copy(io.MultiWriter(dest, this.Hash), this.Reader) + written, err = io.Copy(io.MultiWriter(dest, hcr.Hash), hcr.Reader) } if err != nil { return written, err } - sum := this.Hash.Sum(nil) - if fmt.Sprintf("%x", sum) != this.Check { + sum := hcr.Hash.Sum(nil) + if fmt.Sprintf("%x", sum) != hcr.Check { return written, BadChecksum } @@ -68,10 +68,10 @@ func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error // Close reads all remaining data from the underlying Reader and // returns BadChecksum if the checksum doesn't match. It also closes // the underlying Reader if it implements io.ReadCloser. -func (this HashCheckingReader) Close() (err error) { - _, err = io.Copy(this.Hash, this.Reader) +func (hcr HashCheckingReader) Close() (err error) { + _, err = io.Copy(hcr.Hash, hcr.Reader) - if closer, ok := this.Reader.(io.Closer); ok { + if closer, ok := hcr.Reader.(io.Closer); ok { closeErr := closer.Close() if err == nil { err = closeErr @@ -80,7 +80,7 @@ func (this HashCheckingReader) Close() (err error) { if err != nil { return err } - if fmt.Sprintf("%x", this.Hash.Sum(nil)) != this.Check { + if fmt.Sprintf("%x", hcr.Hash.Sum(nil)) != hcr.Check { return BadChecksum } return nil diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go index b18d7e0464..21913ff967 100644 --- a/sdk/go/keepclient/keepclient.go +++ b/sdk/go/keepclient/keepclient.go @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -/* Provides low-level Get/Put primitives for accessing Arvados Keep blocks. */ +// Package keepclient provides low-level Get/Put primitives for accessing +// Arvados Keep blocks. package keepclient import ( @@ -25,7 +26,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/httpserver" ) -// A Keep "block" is 64MB. +// BLOCKSIZE defines the length of a Keep "block", which is 64MB. const BLOCKSIZE = 64 * 1024 * 1024 var ( @@ -82,14 +83,14 @@ var ErrNoSuchKeepServer = errors.New("No keep server matching the given UUID is // ErrIncompleteIndex is returned when the Index response does not end with a new empty line var ErrIncompleteIndex = errors.New("Got incomplete index") -const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas" -const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored" +const XKeepDesiredReplicas = "X-Keep-Desired-Replicas" +const XKeepReplicasStored = "X-Keep-Replicas-Stored" type HTTPClient interface { Do(*http.Request) (*http.Response, error) } -// Information about Arvados and Keep servers. +// KeepClient holds information about Arvados and Keep servers. type KeepClient struct { Arvados *arvadosclient.ArvadosClient Want_replicas int @@ -139,7 +140,7 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient { } } -// Put a block given the block hash, a reader, and the number of bytes +// PutHR puts a block given the block hash, a reader, and the number of bytes // to read from the reader (which must be between 0 and BLOCKSIZE). // // Returns the locator for the written block, the number of replicas @@ -191,11 +192,11 @@ func (kc *KeepClient) PutB(buffer []byte) (string, int, error) { // // If the block hash and data size are known, PutHR is more efficient. func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error) { - if buffer, err := ioutil.ReadAll(r); err != nil { + buffer, err := ioutil.ReadAll(r) + if err != nil { return "", 0, err - } else { - return kc.PutB(buffer) } + return kc.PutB(buffer) } func (kc *KeepClient) getOrHead(method string, locator string, header http.Header) (io.ReadCloser, int64, string, http.Header, error) { @@ -216,7 +217,7 @@ func (kc *KeepClient) getOrHead(method string, locator string, header http.Heade var errs []string - tries_remaining := 1 + kc.Retries + triesRemaining := 1 + kc.Retries serversToTry := kc.getSortedRoots(locator) @@ -225,8 +226,8 @@ func (kc *KeepClient) getOrHead(method string, locator string, header http.Heade var retryList []string - for tries_remaining > 0 { - tries_remaining -= 1 + for triesRemaining > 0 { + triesRemaining-- retryList = nil for _, host := range serversToTry { @@ -290,10 +291,9 @@ func (kc *KeepClient) getOrHead(method string, locator string, header http.Heade Hash: md5.New(), Check: locator[0:32], }, expectLength, url, resp.Header, nil - } else { - resp.Body.Close() - return nil, expectLength, url, resp.Header, nil } + resp.Body.Close() + return nil, expectLength, url, resp.Header, nil } serversToTry = retryList } @@ -333,7 +333,7 @@ func (kc *KeepClient) LocalLocator(locator string) (string, error) { return loc, nil } -// Get() retrieves a block, given a locator. Returns a reader, the +// Get retrieves a block, given a locator. Returns a reader, the // expected data length, the URL the block is being fetched from, and // an error. // @@ -345,13 +345,13 @@ func (kc *KeepClient) Get(locator string) (io.ReadCloser, int64, string, error) return rdr, size, url, err } -// ReadAt() retrieves a portion of block from the cache if it's +// ReadAt retrieves a portion of block from the cache if it's // present, otherwise from the network. func (kc *KeepClient) ReadAt(locator string, p []byte, off int) (int, error) { return kc.cache().ReadAt(kc, locator, p, off) } -// Ask() verifies that a block with the given hash is available and +// Ask verifies that a block with the given hash is available and // readable, according to at least one Keep service. Unlike Get, it // does not retrieve the data or verify that the data content matches // the hash specified by the locator. @@ -416,7 +416,7 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error return bytes.NewReader(respBody[0 : len(respBody)-1]), nil } -// LocalRoots() returns the map of local (i.e., disk and proxy) Keep +// LocalRoots returns the map of local (i.e., disk and proxy) Keep // services: uuid -> baseURI. func (kc *KeepClient) LocalRoots() map[string]string { kc.discoverServices() @@ -425,7 +425,7 @@ func (kc *KeepClient) LocalRoots() map[string]string { return kc.localRoots } -// GatewayRoots() returns the map of Keep remote gateway services: +// GatewayRoots returns the map of Keep remote gateway services: // uuid -> baseURI. func (kc *KeepClient) GatewayRoots() map[string]string { kc.discoverServices() @@ -434,7 +434,7 @@ func (kc *KeepClient) GatewayRoots() map[string]string { return kc.gatewayRoots } -// WritableLocalRoots() returns the map of writable local Keep services: +// WritableLocalRoots returns the map of writable local Keep services: // uuid -> baseURI. func (kc *KeepClient) WritableLocalRoots() map[string]string { kc.discoverServices() @@ -493,9 +493,8 @@ func (kc *KeepClient) getSortedRoots(locator string) []string { func (kc *KeepClient) cache() *BlockCache { if kc.BlockCache != nil { return kc.BlockCache - } else { - return DefaultBlockCache } + return DefaultBlockCache } func (kc *KeepClient) ClearBlockCache() { @@ -576,9 +575,8 @@ var reqIDGen = httpserver.IDGenerator{Prefix: "req-"} func (kc *KeepClient) getRequestID() string { if kc.RequestID != "" { return kc.RequestID - } else { - return reqIDGen.Next() } + return reqIDGen.Next() } type Locator struct { diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index a1801b2145..57a89b50aa 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -97,7 +97,7 @@ func (s *ServerRequiredSuite) TestDefaultReplications(c *C) { type StubPutHandler struct { c *C expectPath string - expectApiToken string + expectAPIToken string expectBody string expectStorageClass string handled chan string @@ -105,7 +105,7 @@ type StubPutHandler struct { func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath) - sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken)) + sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken)) sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass) body, err := ioutil.ReadAll(req.Body) sph.c.Check(err, Equals, nil) @@ -139,9 +139,9 @@ func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string, kc, _ := MakeKeepClient(arv) reader, writer := io.Pipe() - upload_status := make(chan uploadStatus) + uploadStatusChan := make(chan uploadStatus) - f(kc, ks.url, reader, writer, upload_status) + f(kc, ks.url, reader, writer, uploadStatusChan) } func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { @@ -156,15 +156,15 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { make(chan string)} UploadToStubHelper(c, st, - func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) { kc.StorageClasses = []string{"hot"} - go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID()) + go kc.uploadToKeepServer(url, st.expectPath, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled - status := <-upload_status + status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) } @@ -179,12 +179,12 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { make(chan string)} UploadToStubHelper(c, st, - func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID()) + func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) { + go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID()) <-st.handled - status := <-upload_status + status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) } @@ -209,7 +209,7 @@ func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http. fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id")) if fh.count == 0 { resp.WriteHeader(500) - fh.count += 1 + fh.count++ fh.handled <- fmt.Sprintf("http://%s", req.Host) } else { fh.successhandler.ServeHTTP(resp, req) @@ -233,16 +233,16 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { UploadToStubHelper(c, st, func(kc *KeepClient, url string, reader io.ReadCloser, - writer io.WriteCloser, upload_status chan uploadStatus) { + writer io.WriteCloser, uploadStatusChan chan uploadStatus) { - go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID()) + go kc.uploadToKeepServer(url, hash, reader, uploadStatusChan, 3, kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled - status := <-upload_status + status := <-uploadStatusChan c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash)) c.Check(status.statusCode, Equals, 500) }) @@ -256,7 +256,7 @@ type KeepServer struct { func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) { ks = make([]KeepServer, n) - for i := 0; i < n; i += 1 { + for i := 0; i < n; i++ { ks[i] = RunFakeKeepServer(st) } @@ -464,14 +464,14 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { type StubGetHandler struct { c *C expectPath string - expectApiToken string + expectAPIToken string httpStatus int body []byte } func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath) - sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken)) + sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken)) resp.WriteHeader(sgh.httpStatus) resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body))) resp.Write(sgh.body) @@ -535,6 +535,7 @@ func (s *StandaloneSuite) TestGetEmptyBlock(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -769,9 +770,9 @@ type BarHandler struct { handled chan string } -func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { +func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Write([]byte("bar")) - this.handled <- fmt.Sprintf("http://%s", req.Host) + h.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestChecksum(c *C) { @@ -859,9 +860,9 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { c.Check(n, Equals, int64(3)) c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash)) - read_content, err2 := ioutil.ReadAll(r) + readContent, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) - c.Check(read_content, DeepEquals, content) + c.Check(readContent, DeepEquals, content) } func (s *ServerRequiredSuite) TestPutGetHead(c *C) { @@ -891,9 +892,9 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { c.Check(n, Equals, int64(len(content))) c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash)) - read_content, err2 := ioutil.ReadAll(r) + readContent, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) - c.Check(read_content, DeepEquals, content) + c.Check(readContent, DeepEquals, content) } { n, url2, err := kc.Ask(hash) @@ -920,9 +921,9 @@ type StubProxyHandler struct { handled chan string } -func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { +func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Header().Set("X-Keep-Replicas-Stored", "2") - this.handled <- fmt.Sprintf("http://%s", req.Host) + h.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestPutProxy(c *C) { diff --git a/sdk/go/keepclient/root_sorter.go b/sdk/go/keepclient/root_sorter.go index afeb802849..c46b7185e6 100644 --- a/sdk/go/keepclient/root_sorter.go +++ b/sdk/go/keepclient/root_sorter.go @@ -33,10 +33,9 @@ func NewRootSorter(serviceRoots map[string]string, hash string) *RootSorter { func (rs RootSorter) getWeight(hash string, uuid string) string { if len(uuid) == 27 { return Md5String(hash + uuid[12:]) - } else { - // Only useful for testing, a set of one service root, etc. - return Md5String(hash + uuid) } + // Only useful for testing, a set of one service root, etc. + return Md5String(hash + uuid) } func (rs RootSorter) GetSortedRoots() []string { diff --git a/sdk/go/keepclient/root_sorter_test.go b/sdk/go/keepclient/root_sorter_test.go index bd3bb0ba8e..a6fbaeded3 100644 --- a/sdk/go/keepclient/root_sorter_test.go +++ b/sdk/go/keepclient/root_sorter_test.go @@ -19,14 +19,14 @@ func FakeSvcRoot(i uint64) string { return fmt.Sprintf("https://%x.svc/", i) } -func FakeSvcUuid(i uint64) string { +func FakeSvcUUID(i uint64) string { return fmt.Sprintf("zzzzz-bi6l4-%015x", i) } func FakeServiceRoots(n uint64) map[string]string { sr := map[string]string{} for i := uint64(0); i < n; i++ { - sr[FakeSvcUuid(i)] = FakeSvcRoot(i) + sr[FakeSvcUUID(i)] = FakeSvcRoot(i) } return sr } @@ -45,19 +45,19 @@ func (*RootSorterSuite) ReferenceSet(c *C) { fakeroots := FakeServiceRoots(16) // These reference probe orders are explained further in // ../../python/tests/test_keep_client.py: - expected_orders := []string{ + expectedOrders := []string{ "3eab2d5fc9681074", "097dba52e648f1c3", "c5b4e023f8a7d691", "9d81c02e76a3bf54", } - for h, expected_order := range expected_orders { + for h, expectedOrder := range expectedOrders { hash := Md5String(fmt.Sprintf("%064x", h)) roots := NewRootSorter(fakeroots, hash).GetSortedRoots() - for i, svc_id_s := range strings.Split(expected_order, "") { - svc_id, err := strconv.ParseUint(svc_id_s, 16, 64) + for i, svcIDs := range strings.Split(expectedOrder, "") { + svcID, err := strconv.ParseUint(svcIDs, 16, 64) c.Assert(err, Equals, nil) - c.Check(roots[i], Equals, FakeSvcRoot(svc_id)) + c.Check(roots[i], Equals, FakeSvcRoot(svcID)) } } } diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go index 71b4b5ed26..3b1afe1e28 100644 --- a/sdk/go/keepclient/support.go +++ b/sdk/go/keepclient/support.go @@ -18,7 +18,7 @@ import ( "git.arvados.org/arvados.git/sdk/go/arvadosclient" ) -// Function used to emit debug messages. The easiest way to enable +// DebugPrintf emits debug messages. The easiest way to enable // keepclient debug messages in your application is to assign // log.Printf to DebugPrintf. var DebugPrintf = func(string, ...interface{}) {} @@ -48,22 +48,22 @@ type svcList struct { } type uploadStatus struct { - err error - url string - statusCode int - replicas_stored int - response string + err error + url string + statusCode int + replicasStored int + response string } -func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader, - upload_status chan<- uploadStatus, expectedLength int64, reqid string) { +func (kc *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader, + uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) { var req *http.Request var err error var url = fmt.Sprintf("%s/%s", host, hash) if req, err = http.NewRequest("PUT", url, nil); err != nil { DebugPrintf("DEBUG: [%s] Error creating request PUT %v error: %v", reqid, url, err.Error()) - upload_status <- uploadStatus{err, url, 0, 0, ""} + uploadStatusChan <- uploadStatus{err, url, 0, 0, ""} return } @@ -77,22 +77,22 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea } req.Header.Add("X-Request-Id", reqid) - req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken) + req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken) req.Header.Add("Content-Type", "application/octet-stream") - req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas)) - if len(this.StorageClasses) > 0 { - req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", ")) + req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(kc.Want_replicas)) + if len(kc.StorageClasses) > 0 { + req.Header.Add("X-Keep-Storage-Classes", strings.Join(kc.StorageClasses, ", ")) } var resp *http.Response - if resp, err = this.httpClient().Do(req); err != nil { + if resp, err = kc.httpClient().Do(req); err != nil { DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error()) - upload_status <- uploadStatus{err, url, 0, 0, err.Error()} + uploadStatusChan <- uploadStatus{err, url, 0, 0, err.Error()} return } rep := 1 - if xr := resp.Header.Get(X_Keep_Replicas_Stored); xr != "" { + if xr := resp.Header.Get(XKeepReplicasStored); xr != "" { fmt.Sscanf(xr, "%d", &rep) } @@ -103,75 +103,75 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea response := strings.TrimSpace(string(respbody)) if err2 != nil && err2 != io.EOF { DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, err2.Error(), response) - upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response} + uploadStatusChan <- uploadStatus{err2, url, resp.StatusCode, rep, response} } else if resp.StatusCode == http.StatusOK { DebugPrintf("DEBUG: [%s] Upload %v success", reqid, url) - upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response} + uploadStatusChan <- uploadStatus{nil, url, resp.StatusCode, rep, response} } else { if resp.StatusCode >= 300 && response == "" { response = resp.Status } DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, resp.StatusCode, response) - upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response} + uploadStatusChan <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response} } } -func (this *KeepClient) putReplicas( +func (kc *KeepClient) putReplicas( hash string, getReader func() io.Reader, expectedLength int64) (locator string, replicas int, err error) { - reqid := this.getRequestID() + reqid := kc.getRequestID() // Calculate the ordering for uploading to servers - sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots() + sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots() // The next server to try contacting - next_server := 0 + nextServer := 0 // The number of active writers active := 0 // Used to communicate status from the upload goroutines - upload_status := make(chan uploadStatus) + uploadStatusChan := make(chan uploadStatus) defer func() { // Wait for any abandoned uploads (e.g., we started // two uploads and the first replied with replicas=2) // to finish before closing the status channel. go func() { for active > 0 { - <-upload_status + <-uploadStatusChan } - close(upload_status) + close(uploadStatusChan) }() }() replicasDone := 0 - replicasTodo := this.Want_replicas + replicasTodo := kc.Want_replicas - replicasPerThread := this.replicasPerService + replicasPerThread := kc.replicasPerService if replicasPerThread < 1 { // unlimited or unknown replicasPerThread = replicasTodo } - retriesRemaining := 1 + this.Retries + retriesRemaining := 1 + kc.Retries var retryServers []string lastError := make(map[string]string) for retriesRemaining > 0 { - retriesRemaining -= 1 - next_server = 0 + retriesRemaining-- + nextServer = 0 retryServers = []string{} for replicasTodo > 0 { for active*replicasPerThread < replicasTodo { // Start some upload requests - if next_server < len(sv) { - DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[next_server]) - go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, reqid) - next_server += 1 - active += 1 + if nextServer < len(sv) { + DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer]) + go kc.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid) + nextServer++ + active++ } else { if active == 0 && retriesRemaining == 0 { msg := "Could not write sufficient replicas: " @@ -180,9 +180,8 @@ func (this *KeepClient) putReplicas( } msg = msg[:len(msg)-2] return locator, replicasDone, InsufficientReplicasError(errors.New(msg)) - } else { - break } + break } } DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v", @@ -190,13 +189,13 @@ func (this *KeepClient) putReplicas( // Now wait for something to happen. if active > 0 { - status := <-upload_status - active -= 1 + status := <-uploadStatusChan + active-- if status.statusCode == 200 { // good news! - replicasDone += status.replicas_stored - replicasTodo -= status.replicas_stored + replicasDone += status.replicasStored + replicasTodo -= status.replicasStored locator = status.response delete(lastError, status.url) } else { diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go index ec9ae6ece0..954fb710c0 100644 --- a/sdk/go/manifest/manifest.go +++ b/sdk/go/manifest/manifest.go @@ -48,7 +48,7 @@ type FileStreamSegment struct { Name string } -// Represents a single line from a manifest. +// ManifestStream represents a single line from a manifest. type ManifestStream struct { StreamName string Blocks []string @@ -152,32 +152,32 @@ func (s *ManifestStream) FileSegmentIterByName(filepath string) <-chan *FileSegm return ch } -func firstBlock(offsets []uint64, range_start uint64) int { - // range_start/block_start is the inclusive lower bound - // range_end/block_end is the exclusive upper bound +func firstBlock(offsets []uint64, rangeStart uint64) int { + // rangeStart/blockStart is the inclusive lower bound + // rangeEnd/blockEnd is the exclusive upper bound hi := len(offsets) - 1 var lo int i := ((hi + lo) / 2) - block_start := offsets[i] - block_end := offsets[i+1] + blockStart := offsets[i] + blockEnd := offsets[i+1] // perform a binary search for the first block - // assumes that all of the blocks are contiguous, so range_start is guaranteed + // assumes that all of the blocks are contiguous, so rangeStart is guaranteed // to either fall into the range of a block or be outside the block range entirely - for !(range_start >= block_start && range_start < block_end) { + for !(rangeStart >= blockStart && rangeStart < blockEnd) { if lo == i { // must be out of range, fail return -1 } - if range_start > block_start { + if rangeStart > blockStart { lo = i } else { hi = i } i = ((hi + lo) / 2) - block_start = offsets[i] - block_end = offsets[i+1] + blockStart = offsets[i] + blockEnd = offsets[i+1] } return i } @@ -357,7 +357,7 @@ func (stream segmentedStream) normalizedText(name string) string { } sort.Strings(sortedfiles) - stream_tokens := []string{EscapeName(name)} + streamTokens := []string{EscapeName(name)} blocks := make(map[blockdigest.BlockDigest]int64) var streamoffset int64 @@ -367,50 +367,50 @@ func (stream segmentedStream) normalizedText(name string) string { for _, segment := range stream[streamfile] { b, _ := ParseBlockLocator(segment.Locator) if _, ok := blocks[b.Digest]; !ok { - stream_tokens = append(stream_tokens, segment.Locator) + streamTokens = append(streamTokens, segment.Locator) blocks[b.Digest] = streamoffset streamoffset += int64(b.Size) } } } - if len(stream_tokens) == 1 { - stream_tokens = append(stream_tokens, "d41d8cd98f00b204e9800998ecf8427e+0") + if len(streamTokens) == 1 { + streamTokens = append(streamTokens, "d41d8cd98f00b204e9800998ecf8427e+0") } for _, streamfile := range sortedfiles { // Add in file segments - span_start := int64(-1) - span_end := int64(0) + spanStart := int64(-1) + spanEnd := int64(0) fout := EscapeName(streamfile) for _, segment := range stream[streamfile] { // Collapse adjacent segments b, _ := ParseBlockLocator(segment.Locator) streamoffset = blocks[b.Digest] + int64(segment.Offset) - if span_start == -1 { - span_start = streamoffset - span_end = streamoffset + int64(segment.Len) + if spanStart == -1 { + spanStart = streamoffset + spanEnd = streamoffset + int64(segment.Len) } else { - if streamoffset == span_end { - span_end += int64(segment.Len) + if streamoffset == spanEnd { + spanEnd += int64(segment.Len) } else { - stream_tokens = append(stream_tokens, fmt.Sprintf("%d:%d:%s", span_start, span_end-span_start, fout)) - span_start = streamoffset - span_end = streamoffset + int64(segment.Len) + streamTokens = append(streamTokens, fmt.Sprintf("%d:%d:%s", spanStart, spanEnd-spanStart, fout)) + spanStart = streamoffset + spanEnd = streamoffset + int64(segment.Len) } } } - if span_start != -1 { - stream_tokens = append(stream_tokens, fmt.Sprintf("%d:%d:%s", span_start, span_end-span_start, fout)) + if spanStart != -1 { + streamTokens = append(streamTokens, fmt.Sprintf("%d:%d:%s", spanStart, spanEnd-spanStart, fout)) } if len(stream[streamfile]) == 0 { - stream_tokens = append(stream_tokens, fmt.Sprintf("0:0:%s", fout)) + streamTokens = append(streamTokens, fmt.Sprintf("0:0:%s", fout)) } } - return strings.Join(stream_tokens, " ") + "\n" + return strings.Join(streamTokens, " ") + "\n" } func (m segmentedManifest) manifestTextForPath(srcpath, relocate string) string { @@ -429,12 +429,12 @@ func (m segmentedManifest) manifestTextForPath(srcpath, relocate string) string filesegs, okfile := stream[filename] if okfile { newstream := make(segmentedStream) - relocate_stream, relocate_filename := splitPath(relocate) - if relocate_filename == "" { - relocate_filename = filename + relocateStream, relocateFilename := splitPath(relocate) + if relocateFilename == "" { + relocateFilename = filename } - newstream[relocate_filename] = filesegs - return newstream.normalizedText(relocate_stream) + newstream[relocateFilename] = filesegs + return newstream.normalizedText(relocateStream) } } @@ -529,6 +529,8 @@ func (m *Manifest) FileSegmentIterByName(filepath string) <-chan *FileSegment { return ch } +// BlockIterWithDuplicates iterates over the block locators of a manifest. +// // Blocks may appear multiple times within the same manifest if they // are used by multiple files. In that case this Iterator will output // the same block multiple times. diff --git a/sdk/go/stats/duration.go b/sdk/go/stats/duration.go index cf91726334..facb71d212 100644 --- a/sdk/go/stats/duration.go +++ b/sdk/go/stats/duration.go @@ -29,7 +29,7 @@ func (d *Duration) UnmarshalJSON(data []byte) error { return d.Set(string(data)) } -// Value implements flag.Value +// Set implements flag.Value func (d *Duration) Set(s string) error { sec, err := strconv.ParseFloat(s, 64) if err == nil { diff --git a/sdk/python/README.rst b/sdk/python/README.rst index a03d6afe6a..570e398a28 100644 --- a/sdk/python/README.rst +++ b/sdk/python/README.rst @@ -39,11 +39,11 @@ Installing on Debian systems 1. Add this Arvados repository to your sources list:: - deb http://apt.arvados.org/ stretch main + deb http://apt.arvados.org/ buster main 2. Update your package list. -3. Install the ``python-arvados-python-client`` package. +3. Install the ``python3-arvados-python-client`` package. Configuration ------------- diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index ae687c50bd..315fc74a71 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -14,6 +14,7 @@ import logging import os import re import socket +import sys import time import types @@ -32,6 +33,9 @@ RETRY_DELAY_INITIAL = 2 RETRY_DELAY_BACKOFF = 2 RETRY_COUNT = 2 +if sys.version_info >= (3,): + httplib2.SSLHandshakeError = None + class OrderedJsonModel(apiclient.model.JsonModel): """Model class for JSON that preserves the contents' order. diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py index 5f12b62eeb..93fd6b598a 100755 --- a/sdk/python/arvados/commands/arv_copy.py +++ b/sdk/python/arvados/commands/arv_copy.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -# arv-copy [--recursive] [--no-recursive] object-uuid src dst +# arv-copy [--recursive] [--no-recursive] object-uuid # # Copies an object from Arvados instance src to instance dst. # @@ -34,6 +34,7 @@ import sys import logging import tempfile import urllib.parse +import io import arvados import arvados.config @@ -87,17 +88,17 @@ def main(): '-f', '--force', dest='force', action='store_true', help='Perform copy even if the object appears to exist at the remote destination.') copy_opts.add_argument( - '--src', dest='source_arvados', required=True, + '--src', dest='source_arvados', help='The name of the source Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.') copy_opts.add_argument( - '--dst', dest='destination_arvados', required=True, + '--dst', dest='destination_arvados', help='The name of the destination Arvados instance (required) - points at an Arvados config file. May be either a pathname to a config file, or (for example) "foo" as shorthand for $HOME/.config/arvados/foo.conf.') copy_opts.add_argument( '--recursive', dest='recursive', action='store_true', - help='Recursively copy any dependencies for this object. (default)') + help='Recursively copy any dependencies for this object, and subprojects. (default)') copy_opts.add_argument( '--no-recursive', dest='recursive', action='store_false', - help='Do not copy any dependencies. NOTE: if this option is given, the copied object will need to be updated manually in order to be functional.') + help='Do not copy any dependencies or subprojects.') copy_opts.add_argument( '--project-uuid', dest='project_uuid', help='The UUID of the project at the destination to which the collection or workflow should be copied.') @@ -118,6 +119,9 @@ def main(): else: logger.setLevel(logging.INFO) + if not args.source_arvados: + args.source_arvados = args.object_uuid[:5] + # Create API clients for the source and destination instances src_arv = api_for_instance(args.source_arvados) dst_arv = api_for_instance(args.destination_arvados) @@ -135,6 +139,9 @@ def main(): elif t == 'Workflow': set_src_owner_uuid(src_arv.workflows(), args.object_uuid, args) result = copy_workflow(args.object_uuid, src_arv, dst_arv, args) + elif t == 'Group': + set_src_owner_uuid(src_arv.groups(), args.object_uuid, args) + result = copy_project(args.object_uuid, src_arv, dst_arv, args.project_uuid, args) else: abort("cannot copy object {} of type {}".format(args.object_uuid, t)) @@ -170,6 +177,10 @@ def set_src_owner_uuid(resource, uuid, args): # $HOME/.config/arvados/instance_name.conf # def api_for_instance(instance_name): + if not instance_name: + # Use environment + return arvados.api('v1', model=OrderedJsonModel()) + if '/' in instance_name: config_file = instance_name else: @@ -296,7 +307,14 @@ def copy_workflow(wf_uuid, src, dst, args): # copy the workflow itself del wf['uuid'] wf['owner_uuid'] = args.project_uuid - return dst.workflows().create(body=wf).execute(num_retries=args.retries) + + existing = dst.workflows().list(filters=[["owner_uuid", "=", args.project_uuid], + ["name", "=", wf["name"]]]).execute() + if len(existing["items"]) == 0: + return dst.workflows().create(body=wf).execute(num_retries=args.retries) + else: + return dst.workflows().update(uuid=existing["items"][0]["uuid"], body=wf).execute(num_retries=args.retries) + def workflow_collections(obj, locations, docker_images): if isinstance(obj, dict): @@ -305,7 +323,7 @@ def workflow_collections(obj, locations, docker_images): if loc.startswith("keep:"): locations.append(loc[5:]) - docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None) + docker_image = obj.get('dockerImageId', None) or obj.get('dockerPull', None) or obj.get('acrContainerImage', None) if docker_image is not None: ds = docker_image.split(":", 1) tag = ds[1] if len(ds)==2 else 'latest' @@ -516,7 +534,7 @@ def copy_collection(obj_uuid, src, dst, args): # a new manifest as we go. src_keep = arvados.keep.KeepClient(api_client=src, num_retries=args.retries) dst_keep = arvados.keep.KeepClient(api_client=dst, num_retries=args.retries) - dst_manifest = "" + dst_manifest = io.StringIO() dst_locators = {} bytes_written = 0 bytes_expected = total_collection_size(manifest) @@ -527,14 +545,15 @@ def copy_collection(obj_uuid, src, dst, args): for line in manifest.splitlines(): words = line.split() - dst_manifest += words[0] + dst_manifest.write(words[0]) for word in words[1:]: try: loc = arvados.KeepLocator(word) except ValueError: # If 'word' can't be parsed as a locator, # presume it's a filename. - dst_manifest += ' ' + word + dst_manifest.write(' ') + dst_manifest.write(word) continue blockhash = loc.md5sum # copy this block if we haven't seen it before @@ -547,17 +566,18 @@ def copy_collection(obj_uuid, src, dst, args): dst_locator = dst_keep.put(data) dst_locators[blockhash] = dst_locator bytes_written += loc.size - dst_manifest += ' ' + dst_locators[blockhash] - dst_manifest += "\n" + dst_manifest.write(' ') + dst_manifest.write(dst_locators[blockhash]) + dst_manifest.write("\n") if progress_writer: progress_writer.report(obj_uuid, bytes_written, bytes_expected) progress_writer.finish() # Copy the manifest and save the collection. - logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest) + logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest.getvalue()) - c['manifest_text'] = dst_manifest + c['manifest_text'] = dst_manifest.getvalue() return create_collection_from(c, src, dst, args) def select_git_url(api, repo_name, retries, allow_insecure_http, allow_insecure_http_opt): @@ -632,6 +652,42 @@ def copy_docker_image(docker_image, docker_image_tag, src, dst, args): else: logger.warning('Could not find docker image {}:{}'.format(docker_image, docker_image_tag)) +def copy_project(obj_uuid, src, dst, owner_uuid, args): + + src_project_record = src.groups().get(uuid=obj_uuid).execute(num_retries=args.retries) + + # Create/update the destination project + existing = dst.groups().list(filters=[["owner_uuid", "=", owner_uuid], + ["name", "=", src_project_record["name"]]]).execute(num_retries=args.retries) + if len(existing["items"]) == 0: + project_record = dst.groups().create(body={"group": {"group_class": "project", + "owner_uuid": owner_uuid, + "name": src_project_record["name"]}}).execute(num_retries=args.retries) + else: + project_record = existing["items"][0] + + dst.groups().update(uuid=project_record["uuid"], + body={"group": { + "description": src_project_record["description"]}}).execute(num_retries=args.retries) + + args.project_uuid = project_record["uuid"] + + logger.debug('Copying %s to %s', obj_uuid, project_record["uuid"]) + + # Copy collections + copy_collections([col["uuid"] for col in arvados.util.list_all(src.collections().list, filters=[["owner_uuid", "=", obj_uuid]])], + src, dst, args) + + # Copy workflows + for w in arvados.util.list_all(src.workflows().list, filters=[["owner_uuid", "=", obj_uuid]]): + copy_workflow(w["uuid"], src, dst, args) + + if args.recursive: + for g in arvados.util.list_all(src.groups().list, filters=[["owner_uuid", "=", obj_uuid]]): + copy_project(g["uuid"], src, dst, project_record["uuid"], args) + + return project_record + # git_rev_parse(rev, repo) # # Returns the 40-character commit hash corresponding to 'rev' in @@ -654,7 +710,7 @@ def git_rev_parse(rev, repo): # Special case: if handed a Keep locator hash, return 'Collection'. # def uuid_type(api, object_uuid): - if re.match(r'^[a-f0-9]{32}\+[0-9]+(\+[A-Za-z0-9+-]+)?$', object_uuid): + if re.match(arvados.util.keep_locator_pattern, object_uuid): return 'Collection' p = object_uuid.split('-') if len(p) == 3: diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py index 1e52714916..eb68297625 100755 --- a/sdk/python/arvados/commands/get.py +++ b/sdk/python/arvados/commands/get.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py index a45775470a..1e64eeb1da 100644 --- a/sdk/python/arvados/commands/run.py +++ b/sdk/python/arvados/commands/run.py @@ -236,7 +236,7 @@ def uploadfiles(files, api, dry_run=False, num_retries=0, # empty collection pdh = collection.portable_data_hash() assert (pdh == config.EMPTY_BLOCK_LOCATOR), "Empty collection portable_data_hash did not have expected locator, was %s" % pdh - logger.info("Using empty collection %s", pdh) + logger.debug("Using empty collection %s", pdh) for c in files: c.keepref = "%s/%s" % (pdh, c.fn) diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py index 6c9822e9f0..2380e48b73 100644 --- a/sdk/python/arvados/util.py +++ b/sdk/python/arvados/util.py @@ -388,6 +388,67 @@ def list_all(fn, num_retries=0, **kwargs): offset = c['offset'] + len(c['items']) return items +def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs): + pagesize = 1000 + kwargs["limit"] = pagesize + kwargs["count"] = 'none' + kwargs["order"] = ["%s %s" % (order_key, "asc" if ascending else "desc"), "uuid asc"] + other_filters = kwargs.get("filters", []) + + if "select" in kwargs and "uuid" not in kwargs["select"]: + kwargs["select"].append("uuid") + + nextpage = [] + tot = 0 + expect_full_page = True + seen_prevpage = set() + seen_thispage = set() + lastitem = None + prev_page_all_same_order_key = False + + while True: + kwargs["filters"] = nextpage+other_filters + items = fn(**kwargs).execute(num_retries=num_retries) + + if len(items["items"]) == 0: + if prev_page_all_same_order_key: + nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]] + prev_page_all_same_order_key = False + continue + else: + return + + seen_prevpage = seen_thispage + seen_thispage = set() + + for i in items["items"]: + # In cases where there's more than one record with the + # same order key, the result could include records we + # already saw in the last page. Skip them. + if i["uuid"] in seen_prevpage: + continue + seen_thispage.add(i["uuid"]) + yield i + + firstitem = items["items"][0] + lastitem = items["items"][-1] + + if firstitem[order_key] == lastitem[order_key]: + # Got a page where every item has the same order key. + # Switch to using uuid for paging. + nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">", lastitem["uuid"]]] + prev_page_all_same_order_key = True + else: + # Start from the last order key seen, but skip the last + # known uuid to avoid retrieving the same row twice. If + # there are multiple rows with the same order key it is + # still likely we'll end up retrieving duplicate rows. + # That's handled by tracking the "seen" rows for each page + # so they can be skipped if they show up on the next page. + nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]] + prev_page_all_same_order_key = False + + def ca_certs_path(fallback=httplib2.CA_CERTS): """Return the path of the best available CA certs source. diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py index 9aabff4292..092131d930 100644 --- a/sdk/python/arvados_version.py +++ b/sdk/python/arvados_version.py @@ -6,21 +6,41 @@ import subprocess import time import os import re +import sys + +SETUP_DIR = os.path.dirname(os.path.abspath(__file__)) +VERSION_PATHS = { + SETUP_DIR, + os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh")) + } + +def choose_version_from(): + ts = {} + for path in VERSION_PATHS: + ts[subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', path]).strip()] = path + + sorted_ts = sorted(ts.items()) + getver = sorted_ts[-1][1] + print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr) + return getver def git_version_at_commit(): - curdir = os.path.dirname(os.path.abspath(__file__)) + curdir = choose_version_from() myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', '--format=%H', curdir]).strip() - myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode() return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: - return fp.write("__version__ = '%s'\n" % v) + v = v.replace("~dev", ".dev").replace("~rc", "rc") + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: + return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: - return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: + return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") @@ -30,7 +50,12 @@ def get_version(setup_dir, module): else: try: save_version(setup_dir, module, git_version_at_commit()) - except (subprocess.CalledProcessError, OSError): + except (subprocess.CalledProcessError, OSError) as err: + print("ERROR: {0}".format(err), file=sys.stderr) pass return read_version(setup_dir, module) + +# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh +if __name__ == '__main__': + print(get_version(SETUP_DIR, "arvados")) diff --git a/sdk/python/bin/arv-copy b/sdk/python/bin/arv-copy index ad020d7068..289c7db843 100755 --- a/sdk/python/bin/arv-copy +++ b/sdk/python/bin/arv-copy @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-federation-migrate b/sdk/python/bin/arv-federation-migrate index a4c097473b..8a47966321 100755 --- a/sdk/python/bin/arv-federation-migrate +++ b/sdk/python/bin/arv-federation-migrate @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-get b/sdk/python/bin/arv-get index 3216374bf9..a36914192f 100755 --- a/sdk/python/bin/arv-get +++ b/sdk/python/bin/arv-get @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-keepdocker b/sdk/python/bin/arv-keepdocker index c90bb03ce3..5c16016af2 100755 --- a/sdk/python/bin/arv-keepdocker +++ b/sdk/python/bin/arv-keepdocker @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-ls b/sdk/python/bin/arv-ls index b612fda9d5..7703148743 100755 --- a/sdk/python/bin/arv-ls +++ b/sdk/python/bin/arv-ls @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-migrate-docker19 b/sdk/python/bin/arv-migrate-docker19 index 6995c01c14..6aee15254a 100755 --- a/sdk/python/bin/arv-migrate-docker19 +++ b/sdk/python/bin/arv-migrate-docker19 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-normalize b/sdk/python/bin/arv-normalize index eab21f1179..effcd7edae 100755 --- a/sdk/python/bin/arv-normalize +++ b/sdk/python/bin/arv-normalize @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-put b/sdk/python/bin/arv-put index eaeecfb2f1..e41437a863 100755 --- a/sdk/python/bin/arv-put +++ b/sdk/python/bin/arv-put @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/bin/arv-ws b/sdk/python/bin/arv-ws index 4e84918068..2b601296b4 100755 --- a/sdk/python/bin/arv-ws +++ b/sdk/python/bin/arv-ws @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/gittaggers.py b/sdk/python/gittaggers.py deleted file mode 100644 index f3278fcc1d..0000000000 --- a/sdk/python/gittaggers.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -from setuptools.command.egg_info import egg_info -import subprocess -import time - -class EggInfoFromGit(egg_info): - """Tag the build with git commit timestamp. - - If a build tag has already been set (e.g., "egg_info -b", building - from source package), leave it alone. - """ - def git_latest_tag(self): - gittags = subprocess.check_output(['git', 'tag', '-l']).split() - gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True) - return str(next(iter(gittags)).decode('utf-8')) - - def git_timestamp_tag(self): - gitinfo = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', '.']).strip() - return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))) - - def tags(self): - if self.tag_build is None: - self.tag_build = self.git_latest_tag()+self.git_timestamp_tag() - return egg_info.tags(self) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 589533177a..8bd43f5960 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/python/tests/fed-migrate/README b/sdk/python/tests/fed-migrate/README index 1591b7e17e..7b7c473e4b 100644 --- a/sdk/python/tests/fed-migrate/README +++ b/sdk/python/tests/fed-migrate/README @@ -6,10 +6,10 @@ arv-federation-migrate should be in the path or the full path supplied in the 'fed_migrate' input parameter. # Create arvbox containers fedbox(1,2,3) for the federation -$ cwltool arvbox-make-federation.cwl --arvbox_base ~/.arvbox > fed.json +$ cwltool --preserve-environment=SSH_AUTH_SOCK arvbox-make-federation.cwl --arvbox_base ~/.arvbox > fed.json # Configure containers and run tests -$ cwltool fed-migrate.cwl fed.json +$ cwltool --preserve-environment=SSH_AUTH_SOCK fed-migrate.cwl fed.json CWL for running the test is generated using cwl-ex: diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwl b/sdk/python/tests/fed-migrate/fed-migrate.cwl index 19c2b58ef7..bb11f0a6e6 100644 --- a/sdk/python/tests/fed-migrate/fed-migrate.cwl +++ b/sdk/python/tests/fed-migrate/fed-migrate.cwl @@ -293,7 +293,7 @@ $graph: - arguments: - arvbox - cat - - /var/lib/arvados/superuser_token + - /var/lib/arvados-arvbox/superuser_token class: CommandLineTool cwlVersion: v1.0 id: '#superuser_tok' @@ -476,10 +476,10 @@ $graph: ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) - cat /var/lib/arvados/vm-uuid) + cat /var/lib/arvados-arvbox/vm-uuid) ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat - /var/lib/arvados/superuser_token) + /var/lib/arvados-arvbox/superuser_token) while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_TOKEN" diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwlex b/sdk/python/tests/fed-migrate/fed-migrate.cwlex index e0beaa91d6..4c1db0f43b 100644 --- a/sdk/python/tests/fed-migrate/fed-migrate.cwlex +++ b/sdk/python/tests/fed-migrate/fed-migrate.cwlex @@ -34,8 +34,8 @@ $(inputs.arvbox_bin.path) hotreset while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done -ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/vm-uuid) -ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token) +ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/vm-uuid) +ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/superuser_token) while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_TOKEN" https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID >/dev/null ; do sleep 3 ; done >>> @@ -47,4 +47,4 @@ while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_T report = run_test(arvados_api_hosts, superuser_tokens=supertok, fed_migrate) return supertok, report -} \ No newline at end of file +} diff --git a/sdk/python/tests/fed-migrate/superuser-tok.cwl b/sdk/python/tests/fed-migrate/superuser-tok.cwl index d2ce253a93..e2ad5db5d6 100755 --- a/sdk/python/tests/fed-migrate/superuser-tok.cwl +++ b/sdk/python/tests/fed-migrate/superuser-tok.cwl @@ -16,4 +16,4 @@ requirements: envDef: ARVBOX_CONTAINER: "$(inputs.container)" InlineJavascriptRequirement: {} -arguments: [arvbox, cat, /var/lib/arvados/superuser_token] +arguments: [arvbox, cat, /var/lib/arvados-arvbox/superuser_token] diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index fe32547fcb..c79aa4e945 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -43,6 +43,14 @@ import arvados.config ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..')) SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services') + +# Work around https://bugs.python.org/issue27805, should be no longer +# necessary from sometime in Python 3.8.x +if not os.environ.get('ARVADOS_DEBUG', ''): + WRITE_MODE = 'a' +else: + WRITE_MODE = 'w' + if 'GOPATH' in os.environ: # Add all GOPATH bin dirs to PATH -- but insert them after the # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler @@ -65,6 +73,7 @@ if not os.path.exists(TEST_TMPDIR): my_api_host = None _cached_config = {} _cached_db_config = {} +_already_used_port = {} def find_server_pid(PID_PATH, wait=10): now = time.time() @@ -173,11 +182,15 @@ def find_available_port(): would take care of the races, and this wouldn't be needed at all. """ - sock = socket.socket() - sock.bind(('0.0.0.0', 0)) - port = sock.getsockname()[1] - sock.close() - return port + global _already_used_port + while True: + sock = socket.socket() + sock.bind(('0.0.0.0', 0)) + port = sock.getsockname()[1] + sock.close() + if port not in _already_used_port: + _already_used_port[port] = True + return port def _wait_until_port_listens(port, timeout=10, warn=True): """Wait for a process to start listening on the given port. @@ -327,7 +340,7 @@ def run(leave_running_atexit=False): env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) - logf = open(_logfilename('railsapi'), 'a') + logf = open(_logfilename('railsapi'), WRITE_MODE) railsapi = subprocess.Popen( ['bundle', 'exec', 'passenger', 'start', '-p{}'.format(port), @@ -409,7 +422,7 @@ def run_controller(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: return stop_controller() - logf = open(_logfilename('controller'), 'a') + logf = open(_logfilename('controller'), WRITE_MODE) port = internal_port_from_config("Controller") controller = subprocess.Popen( ["arvados-server", "controller"], @@ -429,7 +442,7 @@ def run_ws(): return stop_ws() port = internal_port_from_config("Websocket") - logf = open(_logfilename('ws'), 'a') + logf = open(_logfilename('ws'), WRITE_MODE) ws = subprocess.Popen( ["arvados-server", "ws"], stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) @@ -462,7 +475,7 @@ def _start_keep(n, blob_signing=False): yaml.safe_dump(confdata, f) keep_cmd = ["keepstore", "-config", conf] - with open(_logfilename('keep{}'.format(n)), 'a') as logf: + with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf: with open('/dev/null') as _stdin: child = subprocess.Popen( keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True) @@ -529,7 +542,7 @@ def run_keep_proxy(): port = internal_port_from_config("Keepproxy") env = os.environ.copy() env['ARVADOS_API_TOKEN'] = auth_token('anonymous') - logf = open(_logfilename('keepproxy'), 'a') + logf = open(_logfilename('keepproxy'), WRITE_MODE) kp = subprocess.Popen( ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True) @@ -568,7 +581,7 @@ def run_arv_git_httpd(): gitport = internal_port_from_config("GitHTTP") env = os.environ.copy() env.pop('ARVADOS_API_TOKEN', None) - logf = open(_logfilename('arv-git-httpd'), 'a') + logf = open(_logfilename('arv-git-httpd'), WRITE_MODE) agh = subprocess.Popen(['arv-git-httpd'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) with open(_pidfile('arv-git-httpd'), 'w') as f: @@ -587,7 +600,7 @@ def run_keep_web(): keepwebport = internal_port_from_config("WebDAV") env = os.environ.copy() - logf = open(_logfilename('keep-web'), 'a') + logf = open(_logfilename('keep-web'), WRITE_MODE) keepweb = subprocess.Popen( ['keep-web'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) @@ -660,7 +673,7 @@ def setup_config(): health_httpd_external_port = find_available_port() keepproxy_port = find_available_port() keepproxy_external_port = find_available_port() - keepstore_ports = sorted([str(find_available_port()) for _ in xrange(0,4)]) + keepstore_ports = sorted([str(find_available_port()) for _ in range(0,4)]) keep_web_port = find_available_port() keep_web_external_port = find_available_port() keep_web_dl_port = find_available_port() diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py index 324d6e05d7..452c2beba2 100644 --- a/sdk/python/tests/test_arv_copy.py +++ b/sdk/python/tests/test_arv_copy.py @@ -7,11 +7,18 @@ import os import sys import tempfile import unittest +import shutil +import arvados.api +from arvados.collection import Collection, CollectionReader import arvados.commands.arv_copy as arv_copy from . import arvados_testutil as tutil +from . import run_test_server + +class ArvCopyVersionTestCase(run_test_server.TestCaseWithServers, tutil.VersionChecker): + MAIN_SERVER = {} + KEEP_SERVER = {} -class ArvCopyTestCase(unittest.TestCase, tutil.VersionChecker): def run_copy(self, args): sys.argv = ['arv-copy'] + args return arv_copy.main() @@ -26,3 +33,50 @@ class ArvCopyTestCase(unittest.TestCase, tutil.VersionChecker): with self.assertRaises(SystemExit): self.run_copy(['--version']) self.assertVersionOutput(out, err) + + def test_copy_project(self): + api = arvados.api() + src_proj = api.groups().create(body={"group": {"name": "arv-copy project", "group_class": "project"}}).execute()["uuid"] + + c = Collection() + with c.open('foo', 'wt') as f: + f.write('foo') + c.save_new("arv-copy foo collection", owner_uuid=src_proj) + + dest_proj = api.groups().create(body={"group": {"name": "arv-copy dest project", "group_class": "project"}}).execute()["uuid"] + + tmphome = tempfile.mkdtemp() + home_was = os.environ['HOME'] + os.environ['HOME'] = tmphome + try: + cfgdir = os.path.join(tmphome, ".config", "arvados") + os.makedirs(cfgdir) + with open(os.path.join(cfgdir, "zzzzz.conf"), "wt") as f: + f.write("ARVADOS_API_HOST=%s\n" % os.environ["ARVADOS_API_HOST"]) + f.write("ARVADOS_API_TOKEN=%s\n" % os.environ["ARVADOS_API_TOKEN"]) + f.write("ARVADOS_API_HOST_INSECURE=1\n") + + contents = api.groups().list(filters=[["owner_uuid", "=", dest_proj]]).execute() + assert len(contents["items"]) == 0 + + try: + self.run_copy(["--project-uuid", dest_proj, src_proj]) + except SystemExit as e: + assert e.code == 0 + + contents = api.groups().list(filters=[["owner_uuid", "=", dest_proj]]).execute() + assert len(contents["items"]) == 1 + + assert contents["items"][0]["name"] == "arv-copy project" + copied_project = contents["items"][0]["uuid"] + + contents = api.collections().list(filters=[["owner_uuid", "=", copied_project]]).execute() + assert len(contents["items"]) == 1 + + assert contents["items"][0]["uuid"] != c.manifest_locator() + assert contents["items"][0]["name"] == "arv-copy foo collection" + assert contents["items"][0]["portable_data_hash"] == c.portable_data_hash() + + finally: + os.environ['HOME'] = home_was + shutil.rmtree(tmphome) diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py index 87074dbdfb..1c0e437b41 100644 --- a/sdk/python/tests/test_util.py +++ b/sdk/python/tests/test_util.py @@ -7,6 +7,7 @@ import subprocess import unittest import arvados +import arvados.util class MkdirDashPTest(unittest.TestCase): def setUp(self): @@ -38,3 +39,139 @@ class RunCommandTestCase(unittest.TestCase): def test_failure(self): with self.assertRaises(arvados.errors.CommandFailedError): arvados.util.run_command(['false']) + +class KeysetTestHelper: + def __init__(self, expect): + self.n = 0 + self.expect = expect + + def fn(self, **kwargs): + if self.expect[self.n][0] != kwargs: + raise Exception("Didn't match %s != %s" % (self.expect[self.n][0], kwargs)) + return self + + def execute(self, num_retries): + self.n += 1 + return self.expect[self.n-1][1] + +class KeysetListAllTestCase(unittest.TestCase): + def test_empty(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn)) + self.assertEqual(ls, []) + + def test_oneitem(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []}, + {"items": [{"created_at": "1", "uuid": "1"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "1"], ["uuid", ">", "1"]]}, + {"items": []} + ],[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "1"]]}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn)) + self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}]) + + def test_onepage2(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []}, + {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn)) + self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]) + + def test_onepage3(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []}, + {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "3"], ["uuid", "!=", "3"]]}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn)) + self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}]) + + + def test_twopage(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []}, + {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]}, + {"items": [{"created_at": "3", "uuid": "3"}, {"created_at": "4", "uuid": "4"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "4"]]}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn)) + self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, + {"created_at": "2", "uuid": "2"}, + {"created_at": "3", "uuid": "3"}, + {"created_at": "4", "uuid": "4"} + ]) + + def test_repeated_key(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []}, + {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "3"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "3"]]}, + {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "4"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "2"], ["uuid", ">", "4"]]}, + {"items": []} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "2"]]}, + {"items": [{"created_at": "3", "uuid": "5"}, {"created_at": "4", "uuid": "6"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "6"]]}, + {"items": []} + ], + ]) + + ls = list(arvados.util.keyset_list_all(ks.fn)) + self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, + {"created_at": "2", "uuid": "2"}, + {"created_at": "2", "uuid": "3"}, + {"created_at": "2", "uuid": "4"}, + {"created_at": "3", "uuid": "5"}, + {"created_at": "4", "uuid": "6"} + ]) + + def test_onepage_withfilter(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["foo", ">", "bar"]]}, + {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"], ["foo", ">", "bar"]]}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn, filters=[["foo", ">", "bar"]])) + self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]) + + + def test_onepage_desc(self): + ks = KeysetTestHelper([[ + {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": []}, + {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}]} + ], [ + {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": [["created_at", "<=", "1"], ["uuid", "!=", "1"]]}, + {"items": []} + ]]) + + ls = list(arvados.util.keyset_list_all(ks.fn, ascending=False)) + self.assertEqual(ls, [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}]) diff --git a/sdk/ruby/arvados.gemspec b/sdk/ruby/arvados.gemspec index 019e156a56..7cc2fd931c 100644 --- a/sdk/ruby/arvados.gemspec +++ b/sdk/ruby/arvados.gemspec @@ -18,6 +18,7 @@ begin else version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip end + version = version.sub("~dev", ".dev").sub("~rc", ".rc") git_timestamp = Time.at(git_timestamp.to_i).utc ensure ENV["GIT_DIR"] = git_dir @@ -31,7 +32,7 @@ Gem::Specification.new do |s| s.summary = "Arvados client library" s.description = "Arvados client library, git commit #{git_hash}" s.authors = ["Arvados Authors"] - s.email = 'gem-dev@curoverse.com' + s.email = 'packaging@arvados.org' s.licenses = ['Apache-2.0'] s.files = ["lib/arvados.rb", "lib/arvados/google_api_client.rb", "lib/arvados/collection.rb", "lib/arvados/keep.rb", diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb index 2644a06579..e1ae76ed29 100644 --- a/services/api/app/controllers/application_controller.rb +++ b/services/api/app/controllers/application_controller.rb @@ -182,7 +182,7 @@ class ApplicationController < ActionController::Base if params[pname].is_a?(Boolean) return params[pname] else - logger.warn "Warning: received non-boolean parameter '#{pname}' on #{self.class.inspect}." + logger.warn "Warning: received non-boolean value #{params[pname].inspect} for boolean parameter #{pname} on #{self.class.inspect}, treating as false." end end false @@ -578,7 +578,7 @@ class ApplicationController < ActionController::Base if @objects.respond_to? :except list[:items_available] = @objects. except(:limit).except(:offset). - distinct.count(:id) + count(@distinct ? :id : '*') end when 'none' else @@ -611,7 +611,7 @@ class ApplicationController < ActionController::Base # Make sure params[key] is either true or false -- not a # string, not nil, etc. if not params.include?(key) - params[key] = info[:default] + params[key] = info[:default] || false elsif [false, 'false', '0', 0].include? params[key] params[key] = false elsif [true, 'true', '1', 1].include? params[key] diff --git a/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb b/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb index cd466cf1fb..59e359232e 100644 --- a/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb +++ b/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb @@ -81,7 +81,9 @@ class Arvados::V1::ApiClientAuthorizationsController < ApplicationController val.is_a?(String) && (attr == 'uuid' || attr == 'api_token') } end - @objects = model_class.where('user_id=?', current_user.id) + if current_api_client_authorization.andand.api_token != Rails.configuration.SystemRootToken + @objects = model_class.where('user_id=?', current_user.id) + end if wanted_scopes.compact.any? # We can't filter on scopes effectively using AR/postgres. # Instead we get the entire result set, do our own filtering on @@ -122,8 +124,8 @@ class Arvados::V1::ApiClientAuthorizationsController < ApplicationController def find_object_by_uuid uuid_param = params[:uuid] || params[:id] - if (uuid_param != current_api_client_authorization.andand.uuid and - not Thread.current[:api_client].andand.is_trusted) + if (uuid_param != current_api_client_authorization.andand.uuid && + !Thread.current[:api_client].andand.is_trusted) return forbidden end @limit = 1 diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb index 81b9ca9e5b..440ac64016 100644 --- a/services/api/app/controllers/arvados/v1/collections_controller.rb +++ b/services/api/app/controllers/arvados/v1/collections_controller.rb @@ -13,10 +13,10 @@ class Arvados::V1::CollectionsController < ApplicationController (super rescue {}). merge({ include_trash: { - type: 'boolean', required: false, description: "Include collections whose is_trashed attribute is true." + type: 'boolean', required: false, default: false, description: "Include collections whose is_trashed attribute is true.", }, include_old_versions: { - type: 'boolean', required: false, description: "Include past collection versions." + type: 'boolean', required: false, default: false, description: "Include past collection versions.", }, }) end @@ -25,10 +25,10 @@ class Arvados::V1::CollectionsController < ApplicationController (super rescue {}). merge({ include_trash: { - type: 'boolean', required: false, description: "Show collection even if its is_trashed attribute is true." + type: 'boolean', required: false, default: false, description: "Show collection even if its is_trashed attribute is true.", }, include_old_versions: { - type: 'boolean', required: false, description: "Include past collection versions." + type: 'boolean', required: false, default: true, description: "Include past collection versions.", }, }) end @@ -43,24 +43,32 @@ class Arvados::V1::CollectionsController < ApplicationController super end - def find_objects_for_index - opts = {} - if params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name) - opts.update({include_trash: true}) - end - if params[:include_old_versions] || @include_old_versions - opts.update({include_old_versions: true}) + def update + # preserve_version should be disabled unless explicitly asked otherwise. + if !resource_attrs[:preserve_version] + resource_attrs[:preserve_version] = false end + super + end + + def find_objects_for_index + opts = { + include_trash: params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name), + include_old_versions: params[:include_old_versions] || false, + } @objects = Collection.readable_by(*@read_users, opts) if !opts.empty? super end def find_object_by_uuid - @include_old_versions = true - if loc = Keep::Locator.parse(params[:id]) loc.strip_hints! + opts = { + include_trash: params[:include_trash], + include_old_versions: params[:include_old_versions], + } + # It matters which Collection object we pick because we use it to get signed_manifest_text, # the value of which is affected by the value of trash_at. # @@ -72,14 +80,13 @@ class Arvados::V1::CollectionsController < ApplicationController # it will select the Collection object with the longest # available lifetime. - if c = Collection.readable_by(*@read_users).where({ portable_data_hash: loc.to_s }).order("trash_at desc").limit(1).first + if c = Collection.readable_by(*@read_users, opts).where({ portable_data_hash: loc.to_s }).order("trash_at desc").limit(1).first @object = { uuid: c.portable_data_hash, portable_data_hash: c.portable_data_hash, manifest_text: c.signed_manifest_text, } end - true else super end diff --git a/services/api/app/controllers/arvados/v1/container_requests_controller.rb b/services/api/app/controllers/arvados/v1/container_requests_controller.rb index 3d5d4616ef..07b8098f5b 100644 --- a/services/api/app/controllers/arvados/v1/container_requests_controller.rb +++ b/services/api/app/controllers/arvados/v1/container_requests_controller.rb @@ -15,7 +15,7 @@ class Arvados::V1::ContainerRequestsController < ApplicationController (super rescue {}). merge({ include_trash: { - type: 'boolean', required: false, description: "Include container requests whose owner project is trashed." + type: 'boolean', required: false, default: false, description: "Include container requests whose owner project is trashed.", }, }) end @@ -24,7 +24,7 @@ class Arvados::V1::ContainerRequestsController < ApplicationController (super rescue {}). merge({ include_trash: { - type: 'boolean', required: false, description: "Show container request even if its owner project is trashed." + type: 'boolean', required: false, default: false, description: "Show container request even if its owner project is trashed.", }, }) end diff --git a/services/api/app/controllers/arvados/v1/groups_controller.rb b/services/api/app/controllers/arvados/v1/groups_controller.rb index 46d3a75a3a..394b5603b7 100644 --- a/services/api/app/controllers/arvados/v1/groups_controller.rb +++ b/services/api/app/controllers/arvados/v1/groups_controller.rb @@ -14,7 +14,7 @@ class Arvados::V1::GroupsController < ApplicationController (super rescue {}). merge({ include_trash: { - type: 'boolean', required: false, description: "Include items whose is_trashed attribute is true." + type: 'boolean', required: false, default: false, description: "Include items whose is_trashed attribute is true.", }, }) end @@ -23,7 +23,7 @@ class Arvados::V1::GroupsController < ApplicationController (super rescue {}). merge({ include_trash: { - type: 'boolean', required: false, description: "Show group/project even if its is_trashed attribute is true." + type: 'boolean', required: false, default: false, description: "Show group/project even if its is_trashed attribute is true.", }, }) end @@ -32,13 +32,16 @@ class Arvados::V1::GroupsController < ApplicationController params = _index_requires_parameters. merge({ uuid: { - type: 'string', required: false, default: nil + type: 'string', required: false, default: nil, }, recursive: { - type: 'boolean', required: false, description: 'Include contents from child groups recursively.' + type: 'boolean', required: false, default: false, description: 'Include contents from child groups recursively.', }, include: { - type: 'string', required: false, description: 'Include objects referred to by listed field in "included" (only owner_uuid)' + type: 'string', required: false, description: 'Include objects referred to by listed field in "included" (only owner_uuid).', + }, + include_old_versions: { + type: 'boolean', required: false, default: false, description: 'Include past collection versions.', } }) params.delete(:select) @@ -53,7 +56,7 @@ class Arvados::V1::GroupsController < ApplicationController type: 'boolean', location: 'query', default: false, - description: 'defer permissions update' + description: 'defer permissions update', } } ) @@ -67,7 +70,7 @@ class Arvados::V1::GroupsController < ApplicationController type: 'boolean', location: 'query', default: false, - description: 'defer permissions update' + description: 'defer permissions update', } } ) @@ -268,7 +271,7 @@ class Arvados::V1::GroupsController < ApplicationController @select = nil where_conds = filter_by_owner if klass == Collection - @select = klass.selectable_attributes - ["manifest_text"] + @select = klass.selectable_attributes - ["manifest_text", "unsigned_manifest_text"] elsif klass == Group where_conds = where_conds.merge(group_class: "project") end @@ -283,8 +286,10 @@ class Arvados::V1::GroupsController < ApplicationController end end.compact - @objects = klass.readable_by(*@read_users, {:include_trash => params[:include_trash]}). - order(request_order).where(where_conds) + @objects = klass.readable_by(*@read_users, { + :include_trash => params[:include_trash], + :include_old_versions => params[:include_old_versions] + }).order(request_order).where(where_conds) if params['exclude_home_project'] @objects = exclude_home @objects, klass diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb index 58a3fd168d..2d6b05269d 100644 --- a/services/api/app/controllers/arvados/v1/jobs_controller.rb +++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb @@ -40,16 +40,16 @@ class Arvados::V1::JobsController < ApplicationController (super rescue {}). merge({ find_or_create: { - type: 'boolean', required: false, default: false + type: 'boolean', required: false, default: false, }, filters: { - type: 'array', required: false + type: 'array', required: false, }, minimum_script_version: { - type: 'string', required: false + type: 'string', required: false, }, exclude_script_versions: { - type: 'array', required: false + type: 'array', required: false, }, }) end diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb index b9aba2726f..9e19397994 100644 --- a/services/api/app/controllers/arvados/v1/schema_controller.rb +++ b/services/api/app/controllers/arvados/v1/schema_controller.rb @@ -36,7 +36,7 @@ class Arvados::V1::SchemaController < ApplicationController # format is YYYYMMDD, must be fixed width (needs to be lexically # sortable), updated manually, may be used by clients to # determine availability of API server features. - revision: "20200331", + revision: "20201210", source_version: AppVersion.hash, sourceVersion: AppVersion.hash, # source_version should be deprecated in the future packageVersion: AppVersion.package_version, diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb index 867b9a6e6a..f4d42edf6c 100644 --- a/services/api/app/controllers/arvados/v1/users_controller.rb +++ b/services/api/app/controllers/arvados/v1/users_controller.rb @@ -22,7 +22,15 @@ class Arvados::V1::UsersController < ApplicationController rescue ActiveRecord::RecordNotUnique retry end - u.update_attributes!(nullify_attrs(attrs)) + needupdate = {} + nullify_attrs(attrs).each do |k,v| + if !v.nil? && u.send(k) != v + needupdate[k] = v + end + end + if needupdate.length > 0 + u.update_attributes!(needupdate) + end @objects << u end @offset = 0 @@ -124,16 +132,8 @@ class Arvados::V1::UsersController < ApplicationController end @response = @object.setup(repo_name: full_repo_name, - vm_uuid: params[:vm_uuid]) - - # setup succeeded. send email to user - if params[:send_notification_email] - begin - UserNotifier.account_is_setup(@object).deliver_now - rescue => e - logger.warn "Failed to send email to #{@object.email}: #{e}" - end - end + vm_uuid: params[:vm_uuid], + send_notification_email: params[:send_notification_email]) send_json kind: "arvados#HashList", items: @response.as_api_response(nil) end @@ -222,7 +222,7 @@ class Arvados::V1::UsersController < ApplicationController type: 'string', required: false, }, redirect_to_new_user: { - type: 'boolean', required: false, + type: 'boolean', required: false, default: false, }, old_user_uuid: { type: 'string', required: false, @@ -236,19 +236,19 @@ class Arvados::V1::UsersController < ApplicationController def self._setup_requires_parameters { uuid: { - type: 'string', required: false + type: 'string', required: false, }, user: { - type: 'object', required: false + type: 'object', required: false, }, repo_name: { - type: 'string', required: false + type: 'string', required: false, }, vm_uuid: { - type: 'string', required: false + type: 'string', required: false, }, send_notification_email: { - type: 'boolean', required: false, default: false + type: 'boolean', required: false, default: false, }, } end @@ -256,7 +256,7 @@ class Arvados::V1::UsersController < ApplicationController def self._update_requires_parameters super.merge({ bypass_federation: { - type: 'boolean', required: false, + type: 'boolean', required: false, default: false, }, }) end diff --git a/services/api/app/controllers/user_sessions_controller.rb b/services/api/app/controllers/user_sessions_controller.rb index 582b98cf2d..8e3c3ac5e3 100644 --- a/services/api/app/controllers/user_sessions_controller.rb +++ b/services/api/app/controllers/user_sessions_controller.rb @@ -147,10 +147,15 @@ class UserSessionsController < ApplicationController find_or_create_by(url_prefix: api_client_url_prefix) end + token_expiration = nil + if Rails.configuration.Login.TokenLifetime > 0 + token_expiration = Time.now + Rails.configuration.Login.TokenLifetime + end @api_client_auth = ApiClientAuthorization. new(user: user, api_client: @api_client, created_by_ip_address: remote_ip, + expires_at: token_expiration, scopes: ["all"]) @api_client_auth.save! diff --git a/services/api/app/middlewares/arvados_api_token.rb b/services/api/app/middlewares/arvados_api_token.rb index acdc485811..be4e8bb0b6 100644 --- a/services/api/app/middlewares/arvados_api_token.rb +++ b/services/api/app/middlewares/arvados_api_token.rb @@ -43,7 +43,7 @@ class ArvadosApiToken auth = nil [params["api_token"], params["oauth_token"], - env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([-\/a-zA-Z0-9]+)/).andand[2], + env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([!-~]+)/).andand[2], *reader_tokens, ].each do |supplied| next if !supplied diff --git a/services/api/app/models/api_client.rb b/services/api/app/models/api_client.rb index 8ed693f820..015b61dc49 100644 --- a/services/api/app/models/api_client.rb +++ b/services/api/app/models/api_client.rb @@ -15,16 +15,34 @@ class ApiClient < ArvadosModel end def is_trusted - norm(self.url_prefix) == norm(Rails.configuration.Services.Workbench1.ExternalURL) || - norm(self.url_prefix) == norm(Rails.configuration.Services.Workbench2.ExternalURL) || - super + (from_trusted_url && Rails.configuration.Login.TokenLifetime == 0) || super end protected + def from_trusted_url + norm_url_prefix = norm(self.url_prefix) + + [Rails.configuration.Services.Workbench1.ExternalURL, + Rails.configuration.Services.Workbench2.ExternalURL, + "https://controller.api.client.invalid"].each do |url| + if norm_url_prefix == norm(url) + return true + end + end + + Rails.configuration.Login.TrustedClients.keys.each do |url| + if norm_url_prefix == norm(url) + return true + end + end + + false + end + def norm url # normalize URL for comparison - url = URI(url) + url = URI(url.to_s) if url.scheme == "https" url.port == "443" end diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb index a4d49c35c1..9290e01a1a 100644 --- a/services/api/app/models/api_client_authorization.rb +++ b/services/api/app/models/api_client_authorization.rb @@ -113,7 +113,7 @@ class ApiClientAuthorization < ArvadosModel return ApiClientAuthorization.new(user: User.find_by_uuid(system_user_uuid), uuid: Rails.configuration.ClusterID+"-gj3su-000000000000000", api_token: token, - api_client: ApiClient.new(is_trusted: true, url_prefix: "")) + api_client: system_root_token_api_client) else return nil end @@ -128,6 +128,11 @@ class ApiClientAuthorization < ArvadosModel return auth end + token_uuid = '' + secret = token + stored_secret = nil # ...if different from secret + optional = nil + case token[0..2] when 'v2/' _, token_uuid, secret, optional = token.split('/') @@ -170,125 +175,189 @@ class ApiClientAuthorization < ArvadosModel return auth end - token_uuid_prefix = token_uuid[0..4] - if token_uuid_prefix == Rails.configuration.ClusterID + upstream_cluster_id = token_uuid[0..4] + if upstream_cluster_id == Rails.configuration.ClusterID # Token is supposedly issued by local cluster, but if the # token were valid, we would have been found in the database # in the above query. return nil - elsif token_uuid_prefix.length != 5 + elsif upstream_cluster_id.length != 5 # malformed return nil end - # Invariant: token_uuid_prefix != Rails.configuration.ClusterID - # - # In other words the remaing code in this method below is the - # case that determines whether to accept a token that was issued - # by a remote cluster when the token absent or expired in our - # database. To begin, we need to ask the cluster that issued - # the token to [re]validate it. - clnt = ApiClientAuthorization.make_http_client(uuid_prefix: token_uuid_prefix) - - host = remote_host(uuid_prefix: token_uuid_prefix) - if !host - Rails.logger.warn "remote authentication rejected: no host for #{token_uuid_prefix.inspect}" + else + # token is not a 'v2' token. It could be just the secret part + # ("v1 token") -- or it could be an OpenIDConnect access token, + # in which case either (a) the controller will have inserted a + # row with api_token = hmac(systemroottoken,oidctoken) before + # forwarding it, or (b) we'll have done that ourselves, or (c) + # we'll need to ask LoginCluster to validate it for us below, + # and then insert a local row for a faster lookup next time. + hmac = OpenSSL::HMAC.hexdigest('sha256', Rails.configuration.SystemRootToken, token) + auth = ApiClientAuthorization. + includes(:user, :api_client). + where('api_token in (?, ?) and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token, hmac). + first + if auth && auth.user + return auth + elsif !Rails.configuration.Login.LoginCluster.blank? && Rails.configuration.Login.LoginCluster != Rails.configuration.ClusterID + # An unrecognized non-v2 token might be an OIDC Access Token + # that can be verified by our login cluster in the code + # below. If so, we'll stuff the database with hmac instead of + # the real OIDC token. + upstream_cluster_id = Rails.configuration.Login.LoginCluster + stored_secret = hmac + else return nil end + end + + # Invariant: upstream_cluster_id != Rails.configuration.ClusterID + # + # In other words the remaining code in this method decides + # whether to accept a token that was issued by a remote cluster + # when the token is absent or expired in our database. To + # begin, we need to ask the cluster that issued the token to + # [re]validate it. + clnt = ApiClientAuthorization.make_http_client(uuid_prefix: upstream_cluster_id) + + host = remote_host(uuid_prefix: upstream_cluster_id) + if !host + Rails.logger.warn "remote authentication rejected: no host for #{upstream_cluster_id.inspect}" + return nil + end + begin + remote_user = SafeJSON.load( + clnt.get_content('https://' + host + '/arvados/v1/users/current', + {'remote' => Rails.configuration.ClusterID}, + {'Authorization' => 'Bearer ' + token})) + rescue => e + Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}" + return nil + end + + # Check the response is well formed. + if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String) + Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}" + return nil + end + + remote_user_prefix = remote_user['uuid'][0..4] + + if token_uuid == '' + # Use the same UUID as the remote when caching the token. begin - remote_user = SafeJSON.load( - clnt.get_content('https://' + host + '/arvados/v1/users/current', + remote_token = SafeJSON.load( + clnt.get_content('https://' + host + '/arvados/v1/api_client_authorizations/current', {'remote' => Rails.configuration.ClusterID}, {'Authorization' => 'Bearer ' + token})) + token_uuid = remote_token['uuid'] + if !token_uuid.match(HasUuid::UUID_REGEX) || token_uuid[0..4] != upstream_cluster_id + raise "remote cluster #{upstream_cluster_id} returned invalid token uuid #{token_uuid.inspect}" + end rescue => e - Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}" + Rails.logger.warn "error getting remote token details for #{token.inspect}: #{e}" return nil end + end - # Check the response is well formed. - if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String) - Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}" - return nil - end + # Clusters can only authenticate for their own users. + if remote_user_prefix != upstream_cluster_id + Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{upstream_cluster_id}" + return nil + end - remote_user_prefix = remote_user['uuid'][0..4] + # Invariant: remote_user_prefix == upstream_cluster_id + # therefore: remote_user_prefix != Rails.configuration.ClusterID - # Clusters can only authenticate for their own users. - if remote_user_prefix != token_uuid_prefix - Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{token_uuid_prefix}" - return nil - end + # Add or update user and token in local database so we can + # validate subsequent requests faster. - # Invariant: remote_user_prefix == token_uuid_prefix - # therefore: remote_user_prefix != Rails.configuration.ClusterID + if remote_user['uuid'][-22..-1] == '-tpzed-anonymouspublic' + # Special case: map the remote anonymous user to local anonymous user + remote_user['uuid'] = anonymous_user_uuid + end - # Add or update user and token in local database so we can - # validate subsequent requests faster. + user = User.find_by_uuid(remote_user['uuid']) - user = User.find_by_uuid(remote_user['uuid']) + if !user + # Create a new record for this user. + user = User.new(uuid: remote_user['uuid'], + is_active: false, + is_admin: false, + email: remote_user['email'], + owner_uuid: system_user_uuid) + user.set_initial_username(requested: remote_user['username']) + end - if !user - # Create a new record for this user. - user = User.new(uuid: remote_user['uuid'], - is_active: false, - is_admin: false, - email: remote_user['email'], - owner_uuid: system_user_uuid) - user.set_initial_username(requested: remote_user['username']) + # Sync user record. + act_as_system_user do + %w[first_name last_name email prefs].each do |attr| + user.send(attr+'=', remote_user[attr]) end - # Sync user record. - if remote_user_prefix == Rails.configuration.Login.LoginCluster - # Remote cluster controls our user database, set is_active if - # remote is active. If remote is not active, user will be - # unsetup (see below). - user.is_active = true if remote_user['is_active'] - user.is_admin = remote_user['is_admin'] - else - if Rails.configuration.Users.NewUsersAreActive || - Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"] - # Default policy is to activate users - user.is_active = true if remote_user['is_active'] - end + if remote_user['uuid'][-22..-1] == '-tpzed-000000000000000' + user.first_name = "root" + user.last_name = "from cluster #{remote_user_prefix}" end - %w[first_name last_name email prefs].each do |attr| - user.send(attr+'=', remote_user[attr]) - end + user.save! - act_as_system_user do - if user.is_active && !remote_user['is_active'] - user.unsetup + if user.is_invited && !remote_user['is_invited'] + # Remote user is not "invited" state, they should be unsetup, which + # also makes them inactive. + user.unsetup + else + if !user.is_invited && remote_user['is_invited'] and + (remote_user_prefix == Rails.configuration.Login.LoginCluster or + Rails.configuration.Users.AutoSetupNewUsers or + Rails.configuration.Users.NewUsersAreActive or + Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"]) + user.setup end - user.save! + if !user.is_active && remote_user['is_active'] && user.is_invited and + (remote_user_prefix == Rails.configuration.Login.LoginCluster or + Rails.configuration.Users.NewUsersAreActive or + Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"]) + user.update_attributes!(is_active: true) + elsif user.is_active && !remote_user['is_active'] + user.update_attributes!(is_active: false) + end - # We will accept this token (and avoid reloading the user - # record) for 'RemoteTokenRefresh' (default 5 minutes). - # Possible todo: - # Request the actual api_client_auth record from the remote - # server in case it wants the token to expire sooner. - auth = ApiClientAuthorization.find_or_create_by(uuid: token_uuid) do |auth| - auth.user = user - auth.api_client_id = 0 + if remote_user_prefix == Rails.configuration.Login.LoginCluster and + user.is_active and + user.is_admin != remote_user['is_admin'] + # Remote cluster controls our user database, including the + # admin flag. + user.update_attributes!(is_admin: remote_user['is_admin']) end - auth.update_attributes!(user: user, - api_token: secret, - api_client_id: 0, - expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh) - Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db" end - return auth - else - # token is not a 'v2' token - auth = ApiClientAuthorization. - includes(:user, :api_client). - where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token). - first - if auth && auth.user - return auth + + # We will accept this token (and avoid reloading the user + # record) for 'RemoteTokenRefresh' (default 5 minutes). + # Possible todo: + # Request the actual api_client_auth record from the remote + # server in case it wants the token to expire sooner. + auth = ApiClientAuthorization.find_or_create_by(uuid: token_uuid) do |auth| + auth.user = user + auth.api_client_id = 0 end + # If stored_secret is set, we save stored_secret in the database + # but return the real secret to the caller. This way, if we end + # up returning the auth record to the client, they see the same + # secret they supplied, instead of the HMAC we saved in the + # database. + stored_secret = stored_secret || secret + auth.update_attributes!(user: user, + api_token: stored_secret, + api_client_id: 0, + expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh) + Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db" + auth.api_token = secret + return auth end return nil @@ -325,7 +394,6 @@ class ApiClientAuthorization < ArvadosModel end def log_update - super unless (saved_changes.keys - UNLOGGED_CHANGES).empty? end end diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb index 6fb8ff2b33..6a0a58f08d 100644 --- a/services/api/app/models/arvados_model.rb +++ b/services/api/app/models/arvados_model.rb @@ -286,6 +286,7 @@ class ArvadosModel < ApplicationRecord sql_conds = nil user_uuids = users_list.map { |u| u.uuid } + all_user_uuids = [] # For details on how the trashed_groups table is constructed, see # see db/migrate/20200501150153_permission_table.rb @@ -296,21 +297,19 @@ class ArvadosModel < ApplicationRecord exclude_trashed_records = "AND (#{sql_table}.trash_at is NULL or #{sql_table}.trash_at > statement_timestamp())" end + trashed_check = "" + if !include_trash && sql_table != "api_client_authorizations" + trashed_check = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} " + + "where trash_at <= statement_timestamp()) #{exclude_trashed_records}" + end + if users_list.select { |u| u.is_admin }.any? # Admin skips most permission checks, but still want to filter on trashed items. - if !include_trash - if sql_table != "api_client_authorizations" - # Only include records where the owner is not trashed - sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} "+ - "where trash_at <= statement_timestamp()) #{exclude_trashed_records}" - end + if !include_trash && sql_table != "api_client_authorizations" + # Only include records where the owner is not trashed + sql_conds = trashed_check end else - trashed_check = "" - if !include_trash then - trashed_check = "AND target_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} where trash_at <= statement_timestamp())" - end - # The core of the permission check is a join against the # materialized_permissions table to determine if the user has at # least read permission to either the object itself or its @@ -321,19 +320,38 @@ class ArvadosModel < ApplicationRecord # A user can have can_manage access to another user, this grants # full access to all that user's stuff. To implement that we # need to include those other users in the permission query. - user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: ":user_uuids", perm_level: 1} + + # This was previously implemented by embedding the subquery + # directly into the query, but it was discovered later that this + # causes the Postgres query planner to do silly things because + # the query heuristics assumed the subquery would have a lot + # more rows that it does, and choose a bad merge strategy. By + # doing the query here and embedding the result as a constant, + # Postgres also knows exactly how many items there are and can + # choose the right query strategy. + # + # (note: you could also do this with a temporary table, but that + # would require all every request be wrapped in a transaction, + # which is not currently the case). + + all_user_uuids = ActiveRecord::Base.connection.exec_query %{ +#{USER_UUIDS_SUBQUERY_TEMPLATE % {user: "'#{user_uuids.join "', '"}'", perm_level: 1}} +}, + 'readable_by.user_uuids' + + user_uuids_subquery = ":user_uuids" # Note: it is possible to combine the direct_check and - # owner_check into a single EXISTS() clause, however it turns + # owner_check into a single IN (SELECT) clause, however it turns # out query optimizer doesn't like it and forces a sequential - # table scan. Constructing the query with separate EXISTS() + # table scan. Constructing the query with separate IN (SELECT) # clauses enables it to use the index. # # see issue 13208 for details. # Match a direct read permission link from the user to the record uuid direct_check = "#{sql_table}.uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+ - "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check})" + "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1)" # Match a read permission for the user to the record's # owner_uuid. This is so we can have a permissions table that @@ -353,8 +371,17 @@ class ArvadosModel < ApplicationRecord # other user owns. owner_check = "" if sql_table != "api_client_authorizations" and sql_table != "groups" then - owner_check = "OR #{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+ - "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check} AND traverse_owned) " + owner_check = "#{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+ + "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 AND traverse_owned) " + + # We want to do owner_check before direct_check in the OR + # clause. The order of the OR clause isn't supposed to + # matter, but in practice, it does -- apparently in the + # absence of other hints, it uses the ordering from the query. + # For certain types of queries (like filtering on owner_uuid), + # every item will match the owner_check clause, so then + # Postgres will optimize out the direct_check entirely. + direct_check = " OR " + direct_check end links_cond = "" @@ -366,7 +393,7 @@ class ArvadosModel < ApplicationRecord "(#{sql_table}.head_uuid IN (#{user_uuids_subquery}) OR #{sql_table}.tail_uuid IN (#{user_uuids_subquery})))" end - sql_conds = "(#{direct_check} #{owner_check} #{links_cond}) #{exclude_trashed_records}" + sql_conds = "(#{owner_check} #{direct_check} #{links_cond}) #{trashed_check.empty? ? "" : "AND"} #{trashed_check}" end @@ -380,7 +407,7 @@ class ArvadosModel < ApplicationRecord end self.where(sql_conds, - user_uuids: user_uuids, + user_uuids: all_user_uuids.collect{|c| c["target_uuid"]}, permission_link_classes: ['permission', 'resources']) end @@ -454,7 +481,7 @@ class ArvadosModel < ApplicationRecord end def logged_attributes - attributes.except(*Rails.configuration.AuditLogs.UnloggedAttributes.keys) + attributes.except(*Rails.configuration.AuditLogs.UnloggedAttributes.stringify_keys.keys) end def self.full_text_searchable_columns @@ -627,7 +654,12 @@ class ArvadosModel < ApplicationRecord end def permission_to_destroy - permission_to_update + if [system_user_uuid, system_group_uuid, anonymous_group_uuid, + anonymous_user_uuid, public_project_uuid].include? uuid + false + else + permission_to_update + end end def maybe_update_modified_by_fields diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb index 8b549a71ab..4e7b64cf53 100644 --- a/services/api/app/models/collection.rb +++ b/services/api/app/models/collection.rb @@ -62,6 +62,8 @@ class Collection < ArvadosModel t.add :file_size_total end + UNLOGGED_CHANGES = ['preserve_version', 'updated_at'] + after_initialize do @signatures_checked = false @computed_pdh_for_manifest_text = false @@ -269,11 +271,12 @@ class Collection < ArvadosModel snapshot = self.dup snapshot.uuid = nil # Reset UUID so it's created as a new record snapshot.created_at = self.created_at + snapshot.modified_at = self.modified_at_was end # Restore requested changes on the current version changes.keys.each do |attr| - if attr == 'preserve_version' && changes[attr].last == false + if attr == 'preserve_version' && changes[attr].last == false && !should_preserve_version next # Ignore false assignment, once true it'll be true until next version end self.attributes = {attr => changes[attr].last} @@ -285,7 +288,6 @@ class Collection < ArvadosModel if should_preserve_version self.version += 1 - self.preserve_version = false end yield @@ -294,14 +296,22 @@ class Collection < ArvadosModel if snapshot snapshot.attributes = self.syncable_updates leave_modified_by_user_alone do - act_as_system_user do - snapshot.save + leave_modified_at_alone do + act_as_system_user do + snapshot.save + end end end end end end + def maybe_update_modified_by_fields + if !(self.changes.keys - ['updated_at', 'preserve_version']).empty? + super + end + end + def syncable_updates updates = {} if self.changes.any? @@ -356,6 +366,7 @@ class Collection < ArvadosModel idle_threshold = Rails.configuration.Collections.PreserveVersionIfIdle if !self.preserve_version_was && + !self.preserve_version && (idle_threshold < 0 || (idle_threshold > 0 && self.modified_at_was > db_current_time-idle_threshold.seconds)) return false @@ -739,4 +750,8 @@ class Collection < ArvadosModel self.current_version_uuid ||= self.uuid true end + + def log_update + super unless (saved_changes.keys - UNLOGGED_CHANGES).empty? + end end diff --git a/services/api/app/models/database_seeds.rb b/services/api/app/models/database_seeds.rb index 39f491503e..67bd3d10d7 100644 --- a/services/api/app/models/database_seeds.rb +++ b/services/api/app/models/database_seeds.rb @@ -7,14 +7,18 @@ require 'update_permissions' class DatabaseSeeds extend CurrentApiClient def self.install - system_user - system_group - all_users_group - anonymous_group - anonymous_group_read_permission - anonymous_user - empty_collection - refresh_permissions + batch_update_permissions do + system_user + system_group + all_users_group + anonymous_group + anonymous_group_read_permission + anonymous_user + system_root_token_api_client + public_project_group + public_project_read_permission + empty_collection + end refresh_trashed end end diff --git a/services/api/app/models/link.rb b/services/api/app/models/link.rb index 0d7334e44e..83043a56d1 100644 --- a/services/api/app/models/link.rb +++ b/services/api/app/models/link.rb @@ -136,12 +136,14 @@ class Link < ArvadosModel def call_update_permissions if self.link_class == 'permission' update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid + current_user.forget_cached_group_perms end end def clear_permissions if self.link_class == 'permission' update_permissions tail_uuid, head_uuid, REVOKE_PERM, self.uuid + current_user.forget_cached_group_perms end end diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb index 778ad7d0bb..da7e7b310d 100644 --- a/services/api/app/models/user.rb +++ b/services/api/app/models/user.rb @@ -26,7 +26,7 @@ class User < ArvadosModel before_update :verify_repositories_empty, :if => Proc.new { username.nil? and username_changed? } - before_update :setup_on_activate + after_update :setup_on_activate before_create :check_auto_admin before_create :set_initial_username, :if => Proc.new { @@ -38,7 +38,8 @@ class User < ArvadosModel after_create :auto_setup_new_user, :if => Proc.new { Rails.configuration.Users.AutoSetupNewUsers and (uuid != system_user_uuid) and - (uuid != anonymous_user_uuid) + (uuid != anonymous_user_uuid) and + (uuid[0..4] == Rails.configuration.ClusterID) } after_create :send_admin_notifications @@ -160,6 +161,10 @@ SELECT 1 FROM #{PERMISSION_VIEW} MaterializedPermission.where("user_uuid = ? and target_uuid != ?", uuid, uuid).delete_all end + def forget_cached_group_perms + @group_perms = nil + end + def remove_self_from_permissions MaterializedPermission.where("target_uuid = ?", uuid).delete_all check_permissions_against_full_refresh @@ -190,33 +195,78 @@ SELECT user_uuid, target_uuid, perm_level # and perm_hash[:write] are true if this user can read and write # objects owned by group_uuid. def group_permissions(level=1) - group_perms = {} - - user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: "$2"} + @group_perms ||= {} + if @group_perms.empty? + user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: 1} - ActiveRecord::Base.connection. - exec_query(%{ + ActiveRecord::Base.connection. + exec_query(%{ SELECT target_uuid, perm_level FROM #{PERMISSION_VIEW} - WHERE user_uuid in (#{user_uuids_subquery}) and perm_level >= $2 + WHERE user_uuid in (#{user_uuids_subquery}) and perm_level >= 1 }, - # "name" arg is a query label that appears in logs: - "User.group_permissions", - # "binds" arg is an array of [col_id, value] for '$1' vars: - [[nil, uuid], - [nil, level]]). - rows.each do |group_uuid, max_p_val| - group_perms[group_uuid] = PERMS_FOR_VAL[max_p_val.to_i] + # "name" arg is a query label that appears in logs: + "User.group_permissions", + # "binds" arg is an array of [col_id, value] for '$1' vars: + [[nil, uuid]]). + rows.each do |group_uuid, max_p_val| + @group_perms[group_uuid] = PERMS_FOR_VAL[max_p_val.to_i] + end + end + + case level + when 1 + @group_perms + when 2 + @group_perms.select {|k,v| v[:write] } + when 3 + @group_perms.select {|k,v| v[:manage] } + else + raise "level must be 1, 2 or 3" end - group_perms end # create links - def setup(repo_name: nil, vm_uuid: nil) - repo_perm = create_user_repo_link repo_name - vm_login_perm = create_vm_login_permission_link(vm_uuid, username) if vm_uuid + def setup(repo_name: nil, vm_uuid: nil, send_notification_email: nil) + newly_invited = Link.where(tail_uuid: self.uuid, + head_uuid: all_users_group_uuid, + link_class: 'permission', + name: 'can_read').empty? + + # Add can_read link from this user to "all users" which makes this + # user "invited" group_perm = create_user_group_link + # Add git repo + repo_perm = if (!repo_name.nil? || Rails.configuration.Users.AutoSetupNewUsersWithRepository) and !username.nil? + repo_name ||= "#{username}/#{username}" + create_user_repo_link repo_name + end + + # Add virtual machine + if vm_uuid.nil? and !Rails.configuration.Users.AutoSetupNewUsersWithVmUUID.empty? + vm_uuid = Rails.configuration.Users.AutoSetupNewUsersWithVmUUID + end + + vm_login_perm = if vm_uuid && username + create_vm_login_permission_link(vm_uuid, username) + end + + # Send welcome email + if send_notification_email.nil? + send_notification_email = Rails.configuration.Mail.SendUserSetupNotificationEmail + end + + if newly_invited and send_notification_email and !Rails.configuration.Users.UserSetupMailText.empty? + begin + UserNotifier.account_is_setup(self).deliver_now + rescue => e + logger.warn "Failed to send email to #{self.email}: #{e}" + end + end + + forget_cached_group_perms + return [repo_perm, vm_login_perm, group_perm, self].compact end @@ -254,7 +304,9 @@ SELECT target_uuid, perm_level self.prefs = {} # mark the user as inactive + self.is_admin = false # can't be admin and inactive self.is_active = false + forget_cached_group_perms self.save! end @@ -745,17 +797,6 @@ update #{PERMISSION_VIEW} set target_uuid=$1 where target_uuid = $2 # Automatically setup new user during creation def auto_setup_new_user setup - if username - create_vm_login_permission_link(Rails.configuration.Users.AutoSetupNewUsersWithVmUUID, - username) - repo_name = "#{username}/#{username}" - if Rails.configuration.Users.AutoSetupNewUsersWithRepository and - Repository.where(name: repo_name).first.nil? - repo = Repository.create!(name: repo_name, owner_uuid: uuid) - Link.create!(tail_uuid: uuid, head_uuid: repo.uuid, - link_class: "permission", name: "can_manage") - end - end end # Send notification if the user saved profile for the first time diff --git a/services/api/app/views/user_notifier/account_is_setup.text.erb b/services/api/app/views/user_notifier/account_is_setup.text.erb index 50d164bfa1..352ee7754e 100644 --- a/services/api/app/views/user_notifier/account_is_setup.text.erb +++ b/services/api/app/views/user_notifier/account_is_setup.text.erb @@ -2,17 +2,4 @@ SPDX-License-Identifier: AGPL-3.0 %> -<% if not @user.full_name.empty? -%> -<%= @user.full_name %>, -<% else -%> -Hi there, -<% end -%> - -Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services.Workbench1.ExternalURL %>at - - <%= Rails.configuration.Services.Workbench1.ExternalURL %><%= "/" if !Rails.configuration.Services.Workbench1.ExternalURL.to_s.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %> - -for connection instructions. - -Thanks, -The Arvados team. +<%= ERB.new(Rails.configuration.Users.UserSetupMailText, 0, "-").result(binding) %> diff --git a/services/api/config/application.rb b/services/api/config/application.rb index 369294e8a7..b28ae0e071 100644 --- a/services/api/config/application.rb +++ b/services/api/config/application.rb @@ -16,7 +16,7 @@ require "sprockets/railtie" require "rails/test_unit/railtie" # Skipping the following: # * ActionCable (new in Rails 5.0) as it adds '/cable' routes that we're not using -# * Skip ActiveStorage (new in Rails 5.1) +# * ActiveStorage (new in Rails 5.1) require 'digest' diff --git a/services/api/config/arvados_config.rb b/services/api/config/arvados_config.rb index 035a3972f8..69b20420ab 100644 --- a/services/api/config/arvados_config.rb +++ b/services/api/config/arvados_config.rb @@ -110,7 +110,9 @@ arvcfg.declare_config "Users.NewInactiveUserNotificationRecipients", Hash, :new_ arvcfg.declare_config "Login.SSO.ProviderAppSecret", String, :sso_app_secret arvcfg.declare_config "Login.SSO.ProviderAppID", String, :sso_app_id arvcfg.declare_config "Login.LoginCluster", String +arvcfg.declare_config "Login.TrustedClients", Hash arvcfg.declare_config "Login.RemoteTokenRefresh", ActiveSupport::Duration +arvcfg.declare_config "Login.TokenLifetime", ActiveSupport::Duration arvcfg.declare_config "TLS.Insecure", Boolean, :sso_insecure arvcfg.declare_config "Services.SSO.ExternalURL", String, :sso_provider_url arvcfg.declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age diff --git a/services/api/db/migrate/20200914203202_public_favorites_project.rb b/services/api/db/migrate/20200914203202_public_favorites_project.rb new file mode 100644 index 0000000000..ef139aa704 --- /dev/null +++ b/services/api/db/migrate/20200914203202_public_favorites_project.rb @@ -0,0 +1,23 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +class PublicFavoritesProject < ActiveRecord::Migration[5.2] + include CurrentApiClient + def up + act_as_system_user do + public_project_group + public_project_read_permission + Link.where(link_class: "star", + owner_uuid: system_user_uuid, + tail_uuid: all_users_group_uuid).each do |ln| + ln.owner_uuid = public_project_uuid + ln.tail_uuid = public_project_uuid + ln.save! + end + end + end + + def down + end +end diff --git a/services/api/db/migrate/20201103170213_refresh_trashed_groups.rb b/services/api/db/migrate/20201103170213_refresh_trashed_groups.rb new file mode 100644 index 0000000000..4e8c245c8a --- /dev/null +++ b/services/api/db/migrate/20201103170213_refresh_trashed_groups.rb @@ -0,0 +1,17 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require '20200501150153_permission_table_constants' + +class RefreshTrashedGroups < ActiveRecord::Migration[5.2] + def change + # The original refresh_trashed query had a bug, it would insert + # all trashed rows, including those with null trash_at times. + # This went unnoticed because null trash_at behaved the same as + # not having those rows at all, but it is inefficient to fetch + # rows we'll never use. That bug is fixed in the original query + # but we need another migration to make sure it runs. + refresh_trashed + end +end diff --git a/services/api/db/migrate/20201105190435_refresh_permissions.rb b/services/api/db/migrate/20201105190435_refresh_permissions.rb new file mode 100644 index 0000000000..1ced9d228b --- /dev/null +++ b/services/api/db/migrate/20201105190435_refresh_permissions.rb @@ -0,0 +1,15 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require '20200501150153_permission_table_constants' + +class RefreshPermissions < ActiveRecord::Migration[5.2] + def change + # There was a report of deadlocks resulting in failing permission + # updates. These failures should not have corrupted permissions + # (the failure should have rolled back the entire update) but we + # will refresh the permissions out of an abundance of caution. + refresh_permissions + end +end diff --git a/services/api/db/migrate/20201202174753_fix_collection_versions_timestamps.rb b/services/api/db/migrate/20201202174753_fix_collection_versions_timestamps.rb new file mode 100644 index 0000000000..4c56d3d752 --- /dev/null +++ b/services/api/db/migrate/20201202174753_fix_collection_versions_timestamps.rb @@ -0,0 +1,17 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require 'fix_collection_versions_timestamps' + +class FixCollectionVersionsTimestamps < ActiveRecord::Migration[5.2] + def up + # Defined in a function for easy testing. + fix_collection_versions_timestamps + end + + def down + # This migration is not reversible. However, the results are + # backwards compatible. + end +end diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql index 83987d0518..12a28c6c72 100644 --- a/services/api/db/structure.sql +++ b/services/api/db/structure.sql @@ -10,20 +10,6 @@ SET check_function_bodies = false; SET xmloption = content; SET client_min_messages = warning; --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - --- - --- COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - -- -- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: - -- @@ -3197,6 +3183,10 @@ INSERT INTO "schema_migrations" (version) VALUES ('20190809135453'), ('20190905151603'), ('20200501150153'), -('20200602141328'); +('20200602141328'), +('20200914203202'), +('20201103170213'), +('20201105190435'), +('20201202174753'); diff --git a/services/api/lib/20200501150153_permission_table_constants.rb b/services/api/lib/20200501150153_permission_table_constants.rb index 6e43a628c7..74c15bc2e9 100644 --- a/services/api/lib/20200501150153_permission_table_constants.rb +++ b/services/api/lib/20200501150153_permission_table_constants.rb @@ -63,7 +63,7 @@ def refresh_trashed INSERT INTO #{TRASHED_GROUPS} select ps.target_uuid as group_uuid, ps.trash_at from groups, lateral project_subtree_with_trash_at(groups.uuid, groups.trash_at) ps - where groups.owner_uuid like '_____-tpzed-_______________' + where groups.owner_uuid like '_____-tpzed-_______________' and ps.trash_at is not NULL }) end end diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb index cf16993ca5..f421fb5b2a 100644 --- a/services/api/lib/config_loader.rb +++ b/services/api/lib/config_loader.rb @@ -147,14 +147,14 @@ class ConfigLoader 'Ki' => 1 << 10, 'M' => 1000000, 'Mi' => 1 << 20, - "G" => 1000000000, - "Gi" => 1 << 30, - "T" => 1000000000000, - "Ti" => 1 << 40, - "P" => 1000000000000000, - "Pi" => 1 << 50, - "E" => 1000000000000000000, - "Ei" => 1 << 60, + "G" => 1000000000, + "Gi" => 1 << 30, + "T" => 1000000000000, + "Ti" => 1 << 40, + "P" => 1000000000000000, + "Pi" => 1 << 50, + "E" => 1000000000000000000, + "Ei" => 1 << 60, }[mt[2]] end end diff --git a/services/api/lib/create_superuser_token.rb b/services/api/lib/create_superuser_token.rb index 57eac048a9..7a18d97058 100755 --- a/services/api/lib/create_superuser_token.rb +++ b/services/api/lib/create_superuser_token.rb @@ -54,7 +54,7 @@ module CreateSuperUserToken end end - api_client_auth.api_token + "v2/" + api_client_auth.uuid + "/" + api_client_auth.api_token end end end diff --git a/services/api/lib/current_api_client.rb b/services/api/lib/current_api_client.rb index 98112c858b..37e86976c1 100644 --- a/services/api/lib/current_api_client.rb +++ b/services/api/lib/current_api_client.rb @@ -9,6 +9,8 @@ $anonymous_user = nil $anonymous_group = nil $anonymous_group_read_permission = nil $empty_collection = nil +$public_project_group = nil +$public_project_group_read_permission = nil module CurrentApiClient def current_user @@ -65,6 +67,12 @@ module CurrentApiClient 'anonymouspublic'].join('-') end + def public_project_uuid + [Rails.configuration.ClusterID, + Group.uuid_prefix, + 'publicfavorites'].join('-') + end + def system_user $system_user = check_cache $system_user do real_current_user = Thread.current[:user] @@ -141,6 +149,9 @@ module CurrentApiClient yield ensure Thread.current[:user] = user_was + if user_was + user_was.forget_cached_group_perms + end end end @@ -189,6 +200,41 @@ module CurrentApiClient end end + def public_project_group + $public_project_group = check_cache $public_project_group do + act_as_system_user do + ActiveRecord::Base.transaction do + Group.where(uuid: public_project_uuid). + first_or_create!(group_class: "project", + name: "Public favorites", + description: "Public favorites") + end + end + end + end + + def public_project_read_permission + $public_project_group_read_permission = + check_cache $public_project_group_read_permission do + act_as_system_user do + Link.where(tail_uuid: anonymous_group.uuid, + head_uuid: public_project_group.uuid, + link_class: "permission", + name: "can_read").first_or_create! + end + end + end + + def system_root_token_api_client + $system_root_token_api_client = check_cache $system_root_token_api_client do + act_as_system_user do + ActiveRecord::Base.transaction do + ApiClient.find_or_create_by!(is_trusted: true, url_prefix: "", name: "SystemRootToken") + end + end + end + end + def empty_collection_pdh 'd41d8cd98f00b204e9800998ecf8427e+0' end diff --git a/services/api/lib/enable_jobs_api.rb b/services/api/lib/enable_jobs_api.rb index 1a96a81ad6..cef76f08a5 100644 --- a/services/api/lib/enable_jobs_api.rb +++ b/services/api/lib/enable_jobs_api.rb @@ -2,16 +2,19 @@ # # SPDX-License-Identifier: AGPL-3.0 -Disable_update_jobs_api_method_list = {"jobs.create"=>{}, - "pipeline_instances.create"=>{}, - "pipeline_templates.create"=>{}, - "jobs.update"=>{}, - "pipeline_instances.update"=>{}, - "pipeline_templates.update"=>{}, - "job_tasks.create"=>{}, - "job_tasks.update"=>{}} +Disable_update_jobs_api_method_list = ConfigLoader.to_OrderedOptions({ + "jobs.create"=>{}, + "pipeline_instances.create"=>{}, + "pipeline_templates.create"=>{}, + "jobs.update"=>{}, + "pipeline_instances.update"=>{}, + "pipeline_templates.update"=>{}, + "job_tasks.create"=>{}, + "job_tasks.update"=>{} + }) -Disable_jobs_api_method_list = {"jobs.create"=>{}, +Disable_jobs_api_method_list = ConfigLoader.to_OrderedOptions({ + "jobs.create"=>{}, "pipeline_instances.create"=>{}, "pipeline_templates.create"=>{}, "jobs.get"=>{}, @@ -36,7 +39,7 @@ Disable_jobs_api_method_list = {"jobs.create"=>{}, "jobs.show"=>{}, "pipeline_instances.show"=>{}, "pipeline_templates.show"=>{}, - "job_tasks.show"=>{}} + "job_tasks.show"=>{}}) def check_enable_legacy_jobs_api # Create/update is permanently disabled (legacy functionality has been removed) diff --git a/services/api/lib/fix_collection_versions_timestamps.rb b/services/api/lib/fix_collection_versions_timestamps.rb new file mode 100644 index 0000000000..61da988c00 --- /dev/null +++ b/services/api/lib/fix_collection_versions_timestamps.rb @@ -0,0 +1,43 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require 'set' + +include CurrentApiClient +include ArvadosModelUpdates + +def fix_collection_versions_timestamps + act_as_system_user do + uuids = [].to_set + # Get UUIDs from collections with more than 1 version + Collection.where(version: 2).find_each(batch_size: 100) do |c| + uuids.add(c.current_version_uuid) + end + uuids.each do |uuid| + first_pair = true + # All versions of a particular collection get fixed together. + ActiveRecord::Base.transaction do + Collection.where(current_version_uuid: uuid).order(version: :desc).each_cons(2) do |c1, c2| + # Skip if the 2 newest versions' modified_at values are separate enough; + # this means that this collection doesn't require fixing, allowing for + # migration re-runs in case of transient problems. + break if first_pair && (c2.modified_at.to_f - c1.modified_at.to_f) > 1 + first_pair = false + # Fix modified_at timestamps by assigning to N-1's value to N. + # Special case: the first version's modified_at will be == to created_at + leave_modified_by_user_alone do + leave_modified_at_alone do + c1.modified_at = c2.modified_at + c1.save!(validate: false) + if c2.version == 1 + c2.modified_at = c2.created_at + c2.save!(validate: false) + end + end + end + end + end + end + end +end \ No newline at end of file diff --git a/services/api/lib/tasks/manage_long_lived_tokens.rake b/services/api/lib/tasks/manage_long_lived_tokens.rake new file mode 100644 index 0000000000..7bcf315b04 --- /dev/null +++ b/services/api/lib/tasks/manage_long_lived_tokens.rake @@ -0,0 +1,61 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Tasks that can be useful when changing token expiration policies by assigning +# a non-zero value to Login.TokenLifetime config. + +require 'set' +require 'current_api_client' + +namespace :db do + desc "Apply expiration policy on long lived tokens" + task fix_long_lived_tokens: :environment do + if Rails.configuration.Login.TokenLifetime == 0 + puts("No expiration policy set on Login.TokenLifetime.") + else + exp_date = Time.now + Rails.configuration.Login.TokenLifetime + puts("Setting token expiration to: #{exp_date}") + token_count = 0 + ll_tokens.each do |auth| + if (auth.user.uuid =~ /-tpzed-000000000000000/).nil? + CurrentApiClientHelper.act_as_system_user do + auth.update_attributes!(expires_at: exp_date) + end + token_count += 1 + end + end + puts("#{token_count} tokens updated.") + end + end + + desc "Show users with long lived tokens" + task check_long_lived_tokens: :environment do + user_ids = Set.new() + token_count = 0 + ll_tokens.each do |auth| + if (auth.user.uuid =~ /-tpzed-000000000000000/).nil? + user_ids.add(auth.user_id) + token_count += 1 + end + end + + if user_ids.size > 0 + puts("Found #{token_count} long-lived tokens from users:") + user_ids.each do |uid| + u = User.find(uid) + puts("#{u.username},#{u.email},#{u.uuid}") if !u.nil? + end + else + puts("No long-lived tokens found.") + end + end + + def ll_tokens + query = ApiClientAuthorization.where(expires_at: nil) + if Rails.configuration.Login.TokenLifetime > 0 + query = query.or(ApiClientAuthorization.where("expires_at > ?", Time.now + Rails.configuration.Login.TokenLifetime)) + end + query + end +end diff --git a/services/api/lib/update_permissions.rb b/services/api/lib/update_permissions.rb index 7b1b900cac..23e60c8ed9 100644 --- a/services/api/lib/update_permissions.rb +++ b/services/api/lib/update_permissions.rb @@ -62,10 +62,12 @@ def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil ActiveRecord::Base.transaction do - # "Conflicts with the ROW EXCLUSIVE, SHARE UPDATE EXCLUSIVE, SHARE - # ROW EXCLUSIVE, EXCLUSIVE, and ACCESS EXCLUSIVE lock modes. This - # mode protects a table against concurrent data changes." - ActiveRecord::Base.connection.execute "LOCK TABLE #{PERMISSION_VIEW} in SHARE MODE" + # "Conflicts with the ROW SHARE, ROW EXCLUSIVE, SHARE UPDATE + # EXCLUSIVE, SHARE, SHARE ROW EXCLUSIVE, EXCLUSIVE, and ACCESS + # EXCLUSIVE lock modes. This mode allows only concurrent ACCESS + # SHARE locks, i.e., only reads from the table can proceed in + # parallel with a transaction holding this lock mode." + ActiveRecord::Base.connection.execute "LOCK TABLE #{PERMISSION_VIEW} in EXCLUSIVE MODE" # Workaround for # BUG #15160: planner overestimates number of rows in join when there are more than 200 rows coming from CTE diff --git a/services/api/script/get_anonymous_user_token.rb b/services/api/script/get_anonymous_user_token.rb index 4bb91e2446..8775ae5959 100755 --- a/services/api/script/get_anonymous_user_token.rb +++ b/services/api/script/get_anonymous_user_token.rb @@ -29,27 +29,37 @@ include ApplicationHelper act_as_system_user def create_api_client_auth(supplied_token=nil) + supplied_token = Rails.configuration.Users["AnonymousUserToken"] - # If token is supplied, see if it exists - if supplied_token - api_client_auth = ApiClientAuthorization. - where(api_token: supplied_token). - first - if !api_client_auth - # fall through to create a token - else - raise "Token exists, aborting!" + if supplied_token.nil? or supplied_token.empty? + puts "Users.AnonymousUserToken is empty. Destroying tokens that belong to anonymous." + # Token is empty. Destroy any anonymous tokens. + ApiClientAuthorization.where(user: anonymous_user).destroy_all + return nil + end + + attr = {user: anonymous_user, + api_client_id: 0, + scopes: ['GET /']} + + secret = supplied_token + + if supplied_token[0..2] == 'v2/' + _, token_uuid, secret, optional = supplied_token.split('/') + if token_uuid[0..4] != Rails.configuration.ClusterID + # Belongs to a different cluster. + puts supplied_token + return nil end + attr[:uuid] = token_uuid end - api_client_auth = ApiClientAuthorization. - new(user: anonymous_user, - api_client_id: 0, - expires_at: Time.now + 100.years, - scopes: ['GET /'], - api_token: supplied_token) - api_client_auth.save! - api_client_auth.reload + attr[:api_token] = secret + + api_client_auth = ApiClientAuthorization.where(attr).first + if !api_client_auth + api_client_auth = ApiClientAuthorization.create!(attr) + end api_client_auth end @@ -67,4 +77,6 @@ if !api_client_auth end # print it to the console -puts api_client_auth.api_token +if api_client_auth + puts "v2/#{api_client_auth.uuid}/#{api_client_auth.api_token}" +end diff --git a/services/api/script/rails b/services/api/script/rails index 901460c701..14dcc9dd73 100755 --- a/services/api/script/rails +++ b/services/api/script/rails @@ -4,7 +4,7 @@ ##### SSL - ward, 2012-10-15 require 'rubygems' -require 'rails/commands/server' +require 'rails/command' require 'rack' require 'webrick' require 'webrick/https' diff --git a/services/api/test/fixtures/api_clients.yml b/services/api/test/fixtures/api_clients.yml index 7b522734ab..9965718f99 100644 --- a/services/api/test/fixtures/api_clients.yml +++ b/services/api/test/fixtures/api_clients.yml @@ -17,3 +17,10 @@ untrusted: name: Untrusted url_prefix: https://untrusted.local/ is_trusted: false + +system_root_token_api_client: + uuid: zzzzz-ozdt8-pbw7foaks3qjyej + owner_uuid: zzzzz-tpzed-000000000000000 + name: SystemRootToken + url_prefix: "" + is_trusted: true diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml index a16ee8763f..1f2eab73af 100644 --- a/services/api/test/fixtures/collections.yml +++ b/services/api/test/fixtures/collections.yml @@ -21,10 +21,10 @@ collection_owned_by_active: portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45 owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz created_at: 2014-02-03T17:22:54Z - modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_client_uuid: zzzzz-ozdt8-teyxzyd8qllg11h modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f - modified_at: 2014-02-03T17:22:54Z - updated_at: 2014-02-03T17:22:54Z + modified_at: 2014-02-03T18:22:54Z + updated_at: 2014-02-03T18:22:54Z manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n" name: owned_by_active version: 2 @@ -43,7 +43,7 @@ collection_owned_by_active_with_file_stats: file_count: 1 file_size_total: 3 name: owned_by_active_with_file_stats - version: 2 + version: 1 collection_owned_by_active_past_version_1: uuid: zzzzz-4zz18-znfnqtbbv4spast @@ -53,8 +53,8 @@ collection_owned_by_active_past_version_1: created_at: 2014-02-03T17:22:54Z modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f - modified_at: 2014-02-03T15:22:54Z - updated_at: 2014-02-03T15:22:54Z + modified_at: 2014-02-03T18:22:54Z + updated_at: 2014-02-03T18:22:54Z manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n" name: owned_by_active_version_1 version: 1 @@ -106,8 +106,8 @@ w_a_z_file: created_at: 2015-02-09T10:53:38Z modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f - modified_at: 2015-02-09T10:53:38Z - updated_at: 2015-02-09T10:53:38Z + modified_at: 2015-02-09T10:55:38Z + updated_at: 2015-02-09T10:55:38Z manifest_text: ". 4c6c2c0ac8aa0696edd7316a3be5ca3c+5 0:5:w\\040\\141\\040z\n" name: "\"w a z\" file" version: 2 @@ -120,8 +120,8 @@ w_a_z_file_version_1: created_at: 2015-02-09T10:53:38Z modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f - modified_at: 2015-02-09T10:53:38Z - updated_at: 2015-02-09T10:53:38Z + modified_at: 2015-02-09T10:55:38Z + updated_at: 2015-02-09T10:55:38Z manifest_text: ". 4d20280d5e516a0109768d49ab0f3318+3 0:3:waz\n" name: "waz file" version: 1 @@ -1031,6 +1031,90 @@ collection_with_uri_prop: properties: "http://schema.org/example": "value1" +log_collection: + uuid: zzzzz-4zz18-logcollection01 + current_version_uuid: zzzzz-4zz18-logcollection01 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-10-29T00:51:44.075594000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-10-29T00:51:44.072109000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: a real log collection for a completed container + +log_collection2: + uuid: zzzzz-4zz18-logcollection02 + current_version_uuid: zzzzz-4zz18-logcollection02 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-10-29T00:51:44.075594000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-10-29T00:51:44.072109000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: another real log collection for a completed container + +diagnostics_request_container_log_collection: + uuid: zzzzz-4zz18-diagcompreqlog1 + current_version_uuid: zzzzz-4zz18-diagcompreqlog1 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:20:44.007557000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:20:44.005381000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: Container log for request zzzzz-xvhdp-diagnostics0001 + +hasher1_log_collection: + uuid: zzzzz-4zz18-dlogcollhash001 + current_version_uuid: zzzzz-4zz18-dlogcollhash001 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:16:55.272606000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:16:55.267006000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: hasher1 log collection + +hasher2_log_collection: + uuid: zzzzz-4zz18-dlogcollhash002 + current_version_uuid: zzzzz-4zz18-dlogcollhash002 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:20:23.547251000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:20:23.545275000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: hasher2 log collection + +hasher3_log_collection: + uuid: zzzzz-4zz18-dlogcollhash003 + current_version_uuid: zzzzz-4zz18-dlogcollhash003 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-02T00:20:38.789204000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-02T00:20:38.787329000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: hasher3 log collection + +diagnostics_request_container_log_collection2: + uuid: zzzzz-4zz18-diagcompreqlog2 + current_version_uuid: zzzzz-4zz18-diagcompreqlog2 + portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + created_at: 2020-11-03T16:17:53.351593000Z + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f + modified_at: 2020-11-03T16:17:53.346969000Z + manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n" + name: Container log for request zzzzz-xvhdp-diagnostics0002 + # Test Helper trims the rest of the file # Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml index ea86dca178..ab0400a678 100644 --- a/services/api/test/fixtures/container_requests.yml +++ b/services/api/test/fixtures/container_requests.yml @@ -94,7 +94,7 @@ completed: output_path: test command: ["echo", "hello"] container_uuid: zzzzz-dz642-compltcontainer - log_uuid: zzzzz-4zz18-y9vne9npefyxh8g + log_uuid: zzzzz-4zz18-logcollection01 output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w runtime_constraints: vcpus: 1 @@ -115,10 +115,238 @@ completed-older: output_path: test command: ["arvados-cwl-runner", "echo", "hello"] container_uuid: zzzzz-dz642-compltcontainr2 + log_uuid: zzzzz-4zz18-logcollection02 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w runtime_constraints: vcpus: 1 ram: 123 +completed_diagnostics: + name: CWL diagnostics hasher + uuid: zzzzz-xvhdp-diagnostics0001 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 1 + created_at: 2020-11-02T00:03:50.229364000Z + modified_at: 2020-11-02T00:20:44.041122000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-diagcompreqlog1 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 1342177280 + API: true + +completed_diagnostics_hasher1: + name: hasher1 + uuid: zzzzz-xvhdp-diag1hasher0001 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:03:50.229364000Z + modified_at: 2020-11-02T00:20:44.041122000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher1 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher1 + requesting_container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-dlogcollhash001 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + +completed_diagnostics_hasher2: + name: hasher2 + uuid: zzzzz-xvhdp-diag1hasher0002 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:17:07.067464000Z + modified_at: 2020-11-02T00:20:23.557498000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher2 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher2 + requesting_container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-dlogcollhash002 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 2 + ram: 2684354560 + API: true + +completed_diagnostics_hasher3: + name: hasher3 + uuid: zzzzz-xvhdp-diag1hasher0003 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:20:30.960251000Z + modified_at: 2020-11-02T00:20:38.799377000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher3 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher3 + requesting_container_uuid: zzzzz-dz642-diagcompreq0001 + log_uuid: zzzzz-4zz18-dlogcollhash003 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + +completed_diagnostics2: + name: Copy of CWL diagnostics hasher + uuid: zzzzz-xvhdp-diagnostics0002 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 1 + created_at: 2020-11-03T15:54:30.098485000Z + modified_at: 2020-11-03T16:17:53.406809000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-diagcompreqlog2 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 1342177280 + API: true + +completed_diagnostics_hasher1_reuse: + name: hasher1 + uuid: zzzzz-xvhdp-diag2hasher0001 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:03:50.229364000Z + modified_at: 2020-11-02T00:20:44.041122000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher1 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher1 + requesting_container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-dlogcollhash001 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + +completed_diagnostics_hasher2_reuse: + name: hasher2 + uuid: zzzzz-xvhdp-diag2hasher0002 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:17:07.067464000Z + modified_at: 2020-11-02T00:20:23.557498000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher2 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher2 + requesting_container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-dlogcollhash002 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 2 + ram: 2684354560 + API: true + +completed_diagnostics_hasher3_reuse: + name: hasher3 + uuid: zzzzz-xvhdp-diag2hasher0003 + owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz + state: Final + priority: 500 + created_at: 2020-11-02T00:20:30.960251000Z + modified_at: 2020-11-02T00:20:38.799377000Z + modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + output_name: Output for step hasher3 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt" + ] + container_uuid: zzzzz-dz642-diagcomphasher3 + requesting_container_uuid: zzzzz-dz642-diagcompreq0002 + log_uuid: zzzzz-4zz18-dlogcollhash003 + output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w + runtime_constraints: + vcpus: 1 + ram: 2684354560 + API: true + requester: uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5 owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz @@ -309,7 +537,7 @@ completed_with_input_mounts: vcpus: 1 ram: 123 container_uuid: zzzzz-dz642-compltcontainer - log_uuid: zzzzz-4zz18-y9vne9npefyxh8g + log_uuid: zzzzz-4zz18-logcollection01 output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w mounts: /var/lib/cwl/cwl.input.json: @@ -758,7 +986,7 @@ cr_in_trashed_project: output_path: test command: ["echo", "hello"] container_uuid: zzzzz-dz642-compltcontainer - log_uuid: zzzzz-4zz18-y9vne9npefyxh8g + log_uuid: zzzzz-4zz18-logcollection01 output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w runtime_constraints: vcpus: 1 diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml index f18adb5dbd..b7d082771a 100644 --- a/services/api/test/fixtures/containers.yml +++ b/services/api/test/fixtures/containers.yml @@ -126,6 +126,153 @@ completed_older: secret_mounts: {} secret_mounts_md5: 99914b932bd37a50b983c5e7c90ae93b +diagnostics_completed_requester: + uuid: zzzzz-dz642-diagcompreq0001 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:03:50.192697000Z + modified_at: 2020-11-02T00:20:43.987275000Z + started_at: 2020-11-02T00:08:07.186711000Z + finished_at: 2020-11-02T00:20:43.975416000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: 6129e376cb05c942f75a0c36083383e8+244 + output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45 + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 1342177280 + vcpus: 1 + +diagnostics_completed_hasher1: + uuid: zzzzz-dz642-diagcomphasher1 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:08:18.829222000Z + modified_at: 2020-11-02T00:16:55.142023000Z + started_at: 2020-11-02T00:16:52.375871000Z + finished_at: 2020-11-02T00:16:55.105985000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: fed8fb19fe8e3a320c29fed0edab12dd+220 + output: d3a687732e84061f3bae15dc7e313483+62 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 268435456 + vcpus: 1 + +diagnostics_completed_hasher2: + uuid: zzzzz-dz642-diagcomphasher2 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:17:07.026493000Z + modified_at: 2020-11-02T00:20:23.505908000Z + started_at: 2020-11-02T00:20:21.513185000Z + finished_at: 2020-11-02T00:20:23.478317000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: 4fc03b95fc2646b0dec7383dbb7d56d8+221 + output: 6bd770f6cf8f83e7647c602eecfaeeb8+62 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 268435456 + vcpus: 2 + +diagnostics_completed_hasher3: + uuid: zzzzz-dz642-diagcomphasher3 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 562948349145881771 + created_at: 2020-11-02T00:20:30.943856000Z + modified_at: 2020-11-02T00:20:38.746541000Z + started_at: 2020-11-02T00:20:36.748957000Z + finished_at: 2020-11-02T00:20:38.732199000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: 1eeaf70de0f65b1346e54c59f09e848d+210 + output: 11b5fdaa380102e760c3eb6de80a9876+62 + output_path: /var/spool/cwl + command: [ + "md5sum", + "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 268435456 + vcpus: 1 + +diagnostics_completed_requester2: + uuid: zzzzz-dz642-diagcompreq0002 + owner_uuid: zzzzz-tpzed-000000000000000 + state: Complete + exit_code: 0 + priority: 1124295487972526 + created_at: 2020-11-03T15:54:36.504661000Z + modified_at: 2020-11-03T16:17:53.242868000Z + started_at: 2020-11-03T16:09:51.123659000Z + finished_at: 2020-11-03T16:17:53.220358000Z + container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261 + cwd: /var/spool/cwl + log: f1933bf5191f576613ea7f65bd0ead53+244 + output: 941b71a57208741ce8742eca62352fb1+123 + output_path: /var/spool/cwl + command: [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--disable-validate", + "--disable-color", + "--eval-timeout=20", + "--thread-count=1", + "--disable-reuse", + "--collection-cache-size=256", + "--on-error=continue", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ] + runtime_constraints: + API: true + keep_cache_ram: 268435456 + ram: 1342177280 + vcpus: 1 + requester: uuid: zzzzz-dz642-requestingcntnr owner_uuid: zzzzz-tpzed-000000000000000 diff --git a/services/api/test/fixtures/groups.yml b/services/api/test/fixtures/groups.yml index ee0d786bbe..31a72f1720 100644 --- a/services/api/test/fixtures/groups.yml +++ b/services/api/test/fixtures/groups.yml @@ -56,6 +56,13 @@ system_group: description: System-owned Group group_class: role +public_favorites_project: + uuid: zzzzz-j7d0g-publicfavorites + owner_uuid: zzzzz-tpzed-000000000000000 + name: Public favorites + description: Public favorites + group_class: project + empty_lonely_group: uuid: zzzzz-j7d0g-jtp06ulmvsezgyu owner_uuid: zzzzz-tpzed-000000000000000 diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml index ee5dcd3421..b7f1aaa1fa 100644 --- a/services/api/test/fixtures/links.yml +++ b/services/api/test/fixtures/links.yml @@ -1125,3 +1125,17 @@ active_manages_viewing_group: name: can_manage head_uuid: zzzzz-j7d0g-futrprojviewgrp properties: {} + +public_favorites_permission_link: + uuid: zzzzz-o0j2j-testpublicfavor + owner_uuid: zzzzz-tpzed-000000000000000 + created_at: 2014-01-24 20:42:26 -0800 + modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr + modified_by_user_uuid: zzzzz-tpzed-000000000000000 + modified_at: 2014-01-24 20:42:26 -0800 + updated_at: 2014-01-24 20:42:26 -0800 + tail_uuid: zzzzz-j7d0g-anonymouspublic + link_class: permission + name: can_read + head_uuid: zzzzz-j7d0g-publicfavorites + properties: {} diff --git a/services/api/test/fixtures/logs.yml b/services/api/test/fixtures/logs.yml index 0785c12a50..25f1efff62 100644 --- a/services/api/test/fixtures/logs.yml +++ b/services/api/test/fixtures/logs.yml @@ -4,51 +4,56 @@ noop: # nothing happened ...to the 'spectator' user id: 1 - uuid: zzzzz-xxxxx-pshmckwoma9plh7 + uuid: zzzzz-57u5n-pshmckwoma9plh7 owner_uuid: zzzzz-tpzed-000000000000000 object_uuid: zzzzz-tpzed-l1s2piq4t4mps8r object_owner_uuid: zzzzz-tpzed-000000000000000 event_at: <%= 1.minute.ago.to_s(:db) %> + created_at: <%= 1.minute.ago.to_s(:db) %> admin_changes_repository2: # admin changes repository2, which is owned by active user id: 2 - uuid: zzzzz-xxxxx-pshmckwoma00002 + uuid: zzzzz-57u5n-pshmckwoma00002 owner_uuid: zzzzz-tpzed-d9tiejq69daie8f # admin user object_uuid: zzzzz-2x53u-382brsig8rp3667 # repository foo object_owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz # active user + created_at: <%= 2.minute.ago.to_s(:db) %> event_at: <%= 2.minute.ago.to_s(:db) %> event_type: update admin_changes_specimen: # admin changes specimen owned_by_spectator id: 3 - uuid: zzzzz-xxxxx-pshmckwoma00003 + uuid: zzzzz-57u5n-pshmckwoma00003 owner_uuid: zzzzz-tpzed-d9tiejq69daie8f # admin user object_uuid: zzzzz-2x53u-3b0xxwzlbzxq5yr # specimen owned_by_spectator object_owner_uuid: zzzzz-tpzed-l1s2piq4t4mps8r # spectator user + created_at: <%= 3.minute.ago.to_s(:db) %> event_at: <%= 3.minute.ago.to_s(:db) %> event_type: update system_adds_foo_file: # foo collection added, readable by active through link id: 4 - uuid: zzzzz-xxxxx-pshmckwoma00004 + uuid: zzzzz-57u5n-pshmckwoma00004 owner_uuid: zzzzz-tpzed-000000000000000 # system user object_uuid: zzzzz-4zz18-znfnqtbbv4spc3w # foo file object_owner_uuid: zzzzz-tpzed-000000000000000 # system user + created_at: <%= 4.minute.ago.to_s(:db) %> event_at: <%= 4.minute.ago.to_s(:db) %> event_type: create system_adds_baz: # baz collection added, readable by active and spectator through group 'all users' group membership id: 5 - uuid: zzzzz-xxxxx-pshmckwoma00005 + uuid: zzzzz-57u5n-pshmckwoma00005 owner_uuid: zzzzz-tpzed-000000000000000 # system user object_uuid: zzzzz-4zz18-y9vne9npefyxh8g # baz file object_owner_uuid: zzzzz-tpzed-000000000000000 # system user + created_at: <%= 5.minute.ago.to_s(:db) %> event_at: <%= 5.minute.ago.to_s(:db) %> event_type: create log_owned_by_active: id: 6 - uuid: zzzzz-xxxxx-pshmckwoma12345 + uuid: zzzzz-57u5n-pshmckwoma12345 owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz # active user object_uuid: zzzzz-2x53u-382brsig8rp3667 # repository foo object_owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz # active user diff --git a/services/api/test/fixtures/workflows.yml b/services/api/test/fixtures/workflows.yml index 2859e375a4..29b76abb45 100644 --- a/services/api/test/fixtures/workflows.yml +++ b/services/api/test/fixtures/workflows.yml @@ -67,3 +67,28 @@ workflow_with_input_defaults: id: ex_string_def default: hello-testing-123 outputs: [] + +workflow_with_wrr: + uuid: zzzzz-7fd4e-validwithinput3 + owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0 + name: Workflow with WorkflowRunnerResources + description: this workflow has WorkflowRunnerResources + created_at: <%= 1.minute.ago.to_s(:db) %> + definition: | + cwlVersion: v1.0 + class: CommandLineTool + hints: + - class: http://arvados.org/cwl#WorkflowRunnerResources + acrContainerImage: arvados/jobs:2.0.4 + ramMin: 1234 + coresMin: 2 + keep_cache: 678 + baseCommand: + - echo + inputs: + - type: string + id: ex_string + - type: string + id: ex_string_def + default: hello-testing-123 + outputs: [] diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb index d8017881d5..1ca2dd1dc1 100644 --- a/services/api/test/functional/arvados/v1/collections_controller_test.rb +++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb @@ -1145,7 +1145,7 @@ EOS end [:admin, :active].each do |user| - test "get trashed collection via filters and #{user} user" do + test "get trashed collection via filters and #{user} user without including its past versions" do uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection authorize_with user get :index, params: { @@ -1388,6 +1388,16 @@ EOS json_response['name'] end + test 'can get old version collection by PDH' do + authorize_with :active + get :show, params: { + id: collections(:collection_owned_by_active_past_version_1).portable_data_hash, + } + assert_response :success + assert_equal collections(:collection_owned_by_active_past_version_1).portable_data_hash, + json_response['portable_data_hash'] + end + test 'version and current_version_uuid are ignored at creation time' do permit_unsigned_manifests authorize_with :active diff --git a/services/api/test/functional/arvados/v1/groups_controller_test.rb b/services/api/test/functional/arvados/v1/groups_controller_test.rb index ff89cd2129..02a4ce9663 100644 --- a/services/api/test/functional/arvados/v1/groups_controller_test.rb +++ b/services/api/test/functional/arvados/v1/groups_controller_test.rb @@ -147,6 +147,39 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase refute_includes found_uuids, specimens(:in_asubproject).uuid, "specimen appeared unexpectedly in home project" end + test "list collections in home project" do + authorize_with :active + get(:contents, params: { + format: :json, + filters: [ + ['uuid', 'is_a', 'arvados#collection'], + ], + limit: 200, + id: users(:active).uuid, + }) + assert_response :success + found_uuids = json_response['items'].collect { |i| i['uuid'] } + assert_includes found_uuids, collections(:collection_owned_by_active).uuid, "collection did not appear in home project" + refute_includes found_uuids, collections(:collection_owned_by_active_past_version_1).uuid, "collection appeared unexpectedly in home project" + end + + test "list collections in home project, including old versions" do + authorize_with :active + get(:contents, params: { + format: :json, + include_old_versions: true, + filters: [ + ['uuid', 'is_a', 'arvados#collection'], + ], + limit: 200, + id: users(:active).uuid, + }) + assert_response :success + found_uuids = json_response['items'].collect { |i| i['uuid'] } + assert_includes found_uuids, collections(:collection_owned_by_active).uuid, "collection did not appear in home project" + assert_includes found_uuids, collections(:collection_owned_by_active_past_version_1).uuid, "old collection version did not appear in home project" + end + test "user with project read permission can see project collections" do authorize_with :project_viewer get :contents, params: { @@ -316,7 +349,7 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase end end - test "Collection contents don't include manifest_text" do + test "Collection contents don't include manifest_text or unsigned_manifest_text" do authorize_with :active get :contents, params: { id: groups(:aproject).uuid, @@ -327,7 +360,9 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase refute(json_response["items"].any? { |c| not c["portable_data_hash"] }, "response included an item without a portable data hash") refute(json_response["items"].any? { |c| c.include?("manifest_text") }, - "response included an item with a manifest text") + "response included an item with manifest_text") + refute(json_response["items"].any? { |c| c.include?("unsigned_manifest_text") }, + "response included an item with unsigned_manifest_text") end test 'get writable_by list for owned group' do @@ -430,7 +465,8 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase end test 'get contents with jobs and pipeline instances disabled' do - Rails.configuration.API.DisabledAPIs = {'jobs.index'=>{}, 'pipeline_instances.index'=>{}} + Rails.configuration.API.DisabledAPIs = ConfigLoader.to_OrderedOptions( + {'jobs.index'=>{}, 'pipeline_instances.index'=>{}}) authorize_with :active get :contents, params: { diff --git a/services/api/test/functional/arvados/v1/schema_controller_test.rb b/services/api/test/functional/arvados/v1/schema_controller_test.rb index 3dd343b13c..89feecb454 100644 --- a/services/api/test/functional/arvados/v1/schema_controller_test.rb +++ b/services/api/test/functional/arvados/v1/schema_controller_test.rb @@ -65,8 +65,8 @@ class Arvados::V1::SchemaControllerTest < ActionController::TestCase end test "non-empty disable_api_methods" do - Rails.configuration.API.DisabledAPIs = - {'jobs.create'=>{}, 'pipeline_instances.create'=>{}, 'pipeline_templates.create'=>{}} + Rails.configuration.API.DisabledAPIs = ConfigLoader.to_OrderedOptions( + {'jobs.create'=>{}, 'pipeline_instances.create'=>{}, 'pipeline_templates.create'=>{}}) get :index assert_response :success discovery_doc = JSON.parse(@response.body) @@ -84,7 +84,7 @@ class Arvados::V1::SchemaControllerTest < ActionController::TestCase group_index_params = discovery_doc['resources']['groups']['methods']['index']['parameters'] group_contents_params = discovery_doc['resources']['groups']['methods']['contents']['parameters'] - assert_equal group_contents_params.keys.sort, (group_index_params.keys - ['select'] + ['uuid', 'recursive', 'include']).sort + assert_equal group_contents_params.keys.sort, (group_index_params.keys - ['select'] + ['uuid', 'recursive', 'include', 'include_old_versions']).sort recursive_param = group_contents_params['recursive'] assert_equal 'boolean', recursive_param['type'] diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb index 0ce9f1137f..e0f7b8970d 100644 --- a/services/api/test/functional/arvados/v1/users_controller_test.rb +++ b/services/api/test/functional/arvados/v1/users_controller_test.rb @@ -609,6 +609,23 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase test "setup user with send notification param true and verify email" do authorize_with :admin + Rails.configuration.Users.UserSetupMailText = %{ +<% if not @user.full_name.empty? -%> +<%= @user.full_name %>, +<% else -%> +Hi there, +<% end -%> + +Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services.Workbench1.ExternalURL %>at + +<%= Rails.configuration.Services.Workbench1.ExternalURL %><%= "/" if !Rails.configuration.Services.Workbench1.ExternalURL.to_s.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %> + +for connection instructions. + +Thanks, +The Arvados team. +} + post :setup, params: { send_notification_email: 'true', user: { @@ -1039,9 +1056,12 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase test "batch update" do existinguuid = 'remot-tpzed-foobarbazwazqux' newuuid = 'remot-tpzed-newnarnazwazqux' + unchanginguuid = 'remot-tpzed-nochangingattrs' act_as_system_user do User.create!(uuid: existinguuid, email: 'root@existing.example.com') + User.create!(uuid: unchanginguuid, email: 'root@unchanging.example.com', prefs: {'foo' => {'bar' => 'baz'}}) end + assert_equal(1, Log.where(object_uuid: unchanginguuid).count) authorize_with(:admin) patch(:batch_update, @@ -1059,6 +1079,10 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase 'email' => 'root@remot.example.com', 'username' => '', }, + unchanginguuid => { + 'email' => 'root@unchanging.example.com', + 'prefs' => {'foo' => {'bar' => 'baz'}}, + }, }}) assert_response(:success) @@ -1070,6 +1094,8 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase assert_equal('noot', User.find_by_uuid(newuuid).first_name) assert_equal('root@remot.example.com', User.find_by_uuid(newuuid).email) + + assert_equal(1, Log.where(object_uuid: unchanginguuid).count) end NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name", diff --git a/services/api/test/functional/user_sessions_controller_test.rb b/services/api/test/functional/user_sessions_controller_test.rb index fc9475692a..d979208d38 100644 --- a/services/api/test/functional/user_sessions_controller_test.rb +++ b/services/api/test/functional/user_sessions_controller_test.rb @@ -14,7 +14,6 @@ class UserSessionsControllerTest < ActionController::TestCase assert_nil assigns(:api_client) end - test "send token when user is already logged in" do authorize_with :inactive api_client_page = 'http://client.example.com/home' @@ -26,6 +25,28 @@ class UserSessionsControllerTest < ActionController::TestCase assert_not_nil assigns(:api_client) end + test "login creates token without expiration by default" do + assert_equal Rails.configuration.Login.TokenLifetime, 0 + authorize_with :inactive + api_client_page = 'http://client.example.com/home' + get :login, params: {return_to: api_client_page} + assert_not_nil assigns(:api_client) + assert_nil assigns(:api_client_auth).expires_at + end + + test "login creates token with configured lifetime" do + token_lifetime = 1.hour + Rails.configuration.Login.TokenLifetime = token_lifetime + authorize_with :inactive + api_client_page = 'http://client.example.com/home' + get :login, params: {return_to: api_client_page} + assert_not_nil assigns(:api_client) + api_client_auth = assigns(:api_client_auth) + assert_in_delta(api_client_auth.expires_at, + api_client_auth.updated_at + token_lifetime, + 1.second) + end + test "login with remote param returns a salted token" do authorize_with :inactive api_client_page = 'http://client.example.com/home' @@ -47,7 +68,7 @@ class UserSessionsControllerTest < ActionController::TestCase test "login to LoginCluster" do Rails.configuration.Login.LoginCluster = 'zbbbb' - Rails.configuration.RemoteClusters['zbbbb'] = {'Host' => 'zbbbb.example.com'} + Rails.configuration.RemoteClusters['zbbbb'] = ConfigLoader.to_OrderedOptions({'Host' => 'zbbbb.example.com'}) api_client_page = 'http://client.example.com/home' get :login, params: {return_to: api_client_page} assert_response :redirect diff --git a/services/api/test/integration/api_client_authorizations_api_test.rb b/services/api/test/integration/api_client_authorizations_api_test.rb index b9bfd3a395..296ab8a2ff 100644 --- a/services/api/test/integration/api_client_authorizations_api_test.rb +++ b/services/api/test/integration/api_client_authorizations_api_test.rb @@ -14,22 +14,40 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest assert_response :success end - test "create token for different user" do - post "/arvados/v1/api_client_authorizations", - params: { - :format => :json, - :api_client_authorization => { - :owner_uuid => users(:spectator).uuid - } - }, - headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"} - assert_response :success + [:admin_trustedclient, :SystemRootToken].each do |tk| + test "create token for different user using #{tk}" do + if tk == :SystemRootToken + token = "xyzzy-SystemRootToken" + Rails.configuration.SystemRootToken = token + else + token = api_client_authorizations(tk).api_token + end + + post "/arvados/v1/api_client_authorizations", + params: { + :format => :json, + :api_client_authorization => { + :owner_uuid => users(:spectator).uuid + } + }, + headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{token}"} + assert_response :success + + get "/arvados/v1/users/current", + params: {:format => :json}, + headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{json_response['api_token']}"} + @json_response = nil + assert_equal json_response['uuid'], users(:spectator).uuid + end + end + test "System root token is system user" do + token = "xyzzy-SystemRootToken" + Rails.configuration.SystemRootToken = token get "/arvados/v1/users/current", - params: {:format => :json}, - headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{json_response['api_token']}"} - @json_response = nil - assert_equal users(:spectator).uuid, json_response['uuid'] + params: {:format => :json}, + headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{token}"} + assert_equal json_response['uuid'], system_user_uuid end test "refuse to create token for different user if not trusted client" do diff --git a/services/api/test/integration/collections_api_test.rb b/services/api/test/integration/collections_api_test.rb index 86195fba75..73cbad6430 100644 --- a/services/api/test/integration/collections_api_test.rb +++ b/services/api/test/integration/collections_api_test.rb @@ -495,4 +495,82 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest assert_equal Hash, json_response['properties'].class, 'Collection properties attribute should be of type hash' assert_equal 'value_1', json_response['properties']['property_1'] end + + test "update collection with versioning enabled and using preserve_version" do + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = -1 # Disable auto versioning + + signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active)) + post "/arvados/v1/collections", + params: { + format: :json, + collection: { + name: 'Test collection', + manifest_text: signed_manifest, + }.to_json, + }, + headers: auth(:active) + assert_response 200 + assert_not_nil json_response['uuid'] + assert_equal 1, json_response['version'] + assert_equal false, json_response['preserve_version'] + + # Versionable update including preserve_version=true should create a new + # version that will also be persisted. + put "/arvados/v1/collections/#{json_response['uuid']}", + params: { + format: :json, + collection: { + name: 'Test collection v2', + preserve_version: true, + }.to_json, + }, + headers: auth(:active) + assert_response 200 + assert_equal 2, json_response['version'] + assert_equal true, json_response['preserve_version'] + + # 2nd versionable update including preserve_version=true should create a new + # version that will also be persisted. + put "/arvados/v1/collections/#{json_response['uuid']}", + params: { + format: :json, + collection: { + name: 'Test collection v3', + preserve_version: true, + }.to_json, + }, + headers: auth(:active) + assert_response 200 + assert_equal 3, json_response['version'] + assert_equal true, json_response['preserve_version'] + + # 3rd versionable update without including preserve_version should create a new + # version that will have its preserve_version attr reset to false. + put "/arvados/v1/collections/#{json_response['uuid']}", + params: { + format: :json, + collection: { + name: 'Test collection v4', + }.to_json, + }, + headers: auth(:active) + assert_response 200 + assert_equal 4, json_response['version'] + assert_equal false, json_response['preserve_version'] + + # 4th versionable update without including preserve_version=true should NOT + # create a new version. + put "/arvados/v1/collections/#{json_response['uuid']}", + params: { + format: :json, + collection: { + name: 'Test collection v5?', + }.to_json, + }, + headers: auth(:active) + assert_response 200 + assert_equal 4, json_response['version'] + assert_equal false, json_response['preserve_version'] + end end diff --git a/services/api/test/integration/remote_user_test.rb b/services/api/test/integration/remote_user_test.rb index 04a45420fd..4323884529 100644 --- a/services/api/test/integration/remote_user_test.rb +++ b/services/api/test/integration/remote_user_test.rb @@ -79,11 +79,12 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest Arvados::V1::SchemaController.any_instance.stubs(:root_url).returns "https://#{@remote_host[0]}" @stub_status = 200 @stub_content = { - uuid: 'zbbbb-tpzed-000000000000000', + uuid: 'zbbbb-tpzed-000000000000001', email: 'foo@example.com', username: 'barney', is_admin: true, is_active: true, + is_invited: true, } end @@ -98,7 +99,7 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest params: {format: 'json'}, headers: auth(remote: 'zbbbb') assert_response :success - assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid'] + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] assert_equal false, json_response['is_admin'] assert_equal false, json_response['is_active'] assert_equal 'foo@example.com', json_response['email'] @@ -153,6 +154,7 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest # revoke original token @stub_content[:is_active] = false + @stub_content[:is_invited] = false # simulate cache expiry ApiClientAuthorization.where( @@ -286,12 +288,12 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest params: {format: 'json'}, headers: auth(remote: 'zbbbb') assert_response :success - assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid'] + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] assert_equal false, json_response['is_admin'] assert_equal false, json_response['is_active'] assert_equal 'foo@example.com', json_response['email'] assert_equal 'barney', json_response['username'] - post '/arvados/v1/users/zbbbb-tpzed-000000000000000/activate', + post '/arvados/v1/users/zbbbb-tpzed-000000000000001/activate', params: {format: 'json'}, headers: auth(remote: 'zbbbb') assert_response 422 @@ -303,7 +305,7 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest params: {format: 'json'}, headers: auth(remote: 'zbbbb') assert_response :success - assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid'] + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] assert_equal false, json_response['is_admin'] assert_equal true, json_response['is_active'] assert_equal 'foo@example.com', json_response['email'] @@ -316,13 +318,111 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest params: {format: 'json'}, headers: auth(remote: 'zbbbb') assert_response :success - assert_equal 'zbbbb-tpzed-000000000000000', json_response['uuid'] + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] assert_equal true, json_response['is_admin'] assert_equal true, json_response['is_active'] assert_equal 'foo@example.com', json_response['email'] assert_equal 'barney', json_response['username'] end + [true, false].each do |trusted| + [true, false].each do |logincluster| + [true, false].each do |admin| + [true, false].each do |active| + [true, false].each do |autosetup| + [true, false].each do |invited| + test "get invited=#{invited}, active=#{active}, admin=#{admin} user from #{if logincluster then "Login" else "peer" end} cluster when AutoSetupNewUsers=#{autosetup} ActivateUsers=#{trusted}" do + Rails.configuration.Login.LoginCluster = 'zbbbb' if logincluster + Rails.configuration.RemoteClusters['zbbbb'].ActivateUsers = trusted + Rails.configuration.Users.AutoSetupNewUsers = autosetup + @stub_content = { + uuid: 'zbbbb-tpzed-000000000000001', + email: 'foo@example.com', + username: 'barney', + is_admin: admin, + is_active: active, + is_invited: invited, + } + get '/arvados/v1/users/current', + params: {format: 'json'}, + headers: auth(remote: 'zbbbb') + assert_response :success + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] + assert_equal (logincluster && admin && invited && active), json_response['is_admin'] + assert_equal (invited and (logincluster || trusted || autosetup)), json_response['is_invited'] + assert_equal (invited and (logincluster || trusted) and active), json_response['is_active'] + assert_equal 'foo@example.com', json_response['email'] + assert_equal 'barney', json_response['username'] + end + end + end + end + end + end + end + + test 'get active user from Login cluster when AutoSetupNewUsers is set' do + Rails.configuration.Login.LoginCluster = 'zbbbb' + Rails.configuration.Users.AutoSetupNewUsers = true + @stub_content = { + uuid: 'zbbbb-tpzed-000000000000001', + email: 'foo@example.com', + username: 'barney', + is_admin: false, + is_active: true, + is_invited: true, + } + get '/arvados/v1/users/current', + params: {format: 'json'}, + headers: auth(remote: 'zbbbb') + assert_response :success + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] + assert_equal false, json_response['is_admin'] + assert_equal true, json_response['is_active'] + assert_equal true, json_response['is_invited'] + assert_equal 'foo@example.com', json_response['email'] + assert_equal 'barney', json_response['username'] + + @stub_content = { + uuid: 'zbbbb-tpzed-000000000000001', + email: 'foo@example.com', + username: 'barney', + is_admin: false, + is_active: false, + is_invited: false, + } + + # Use cached value. User will still be active because we haven't + # re-queried the upstream cluster. + get '/arvados/v1/users/current', + params: {format: 'json'}, + headers: auth(remote: 'zbbbb') + assert_response :success + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] + assert_equal false, json_response['is_admin'] + assert_equal true, json_response['is_active'] + assert_equal true, json_response['is_invited'] + assert_equal 'foo@example.com', json_response['email'] + assert_equal 'barney', json_response['username'] + + # Delete cached value. User should be inactive now. + act_as_system_user do + ApiClientAuthorization.delete_all + end + + get '/arvados/v1/users/current', + params: {format: 'json'}, + headers: auth(remote: 'zbbbb') + assert_response :success + assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid'] + assert_equal false, json_response['is_admin'] + assert_equal false, json_response['is_active'] + assert_equal false, json_response['is_invited'] + assert_equal 'foo@example.com', json_response['email'] + assert_equal 'barney', json_response['username'] + + end + test 'pre-activate remote user' do @stub_content = { uuid: 'zbbbb-tpzed-000000000001234', @@ -330,6 +430,7 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest username: 'barney', is_admin: true, is_active: true, + is_invited: true, } post '/arvados/v1/users', @@ -364,6 +465,7 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest username: 'barney', is_admin: true, is_active: true, + is_invited: true, } get '/arvados/v1/users/current', @@ -412,4 +514,22 @@ class RemoteUsersTest < ActionDispatch::IntegrationTest end end + test 'authenticate with remote token, remote user is system user' do + @stub_content[:uuid] = 'zbbbb-tpzed-000000000000000' + get '/arvados/v1/users/current', + params: {format: 'json'}, + headers: auth(remote: 'zbbbb') + assert_equal 'from cluster zbbbb', json_response['last_name'] + end + + test 'authenticate with remote token, remote user is anonymous user' do + @stub_content[:uuid] = 'zbbbb-tpzed-anonymouspublic' + get '/arvados/v1/users/current', + params: {format: 'json'}, + headers: auth(remote: 'zbbbb') + assert_response :success + assert_equal 'zzzzz-tpzed-anonymouspublic', json_response['uuid'] + end + + end diff --git a/services/api/test/test_helper.rb b/services/api/test/test_helper.rb index c99a57aaff..ee7dac4cd9 100644 --- a/services/api/test/test_helper.rb +++ b/services/api/test/test_helper.rb @@ -62,7 +62,7 @@ class ActiveSupport::TestCase include ArvadosTestSupport include CurrentApiClient - teardown do + setup do Thread.current[:api_client_ip_address] = nil Thread.current[:api_client_authorization] = nil Thread.current[:api_client_uuid] = nil @@ -72,6 +72,14 @@ class ActiveSupport::TestCase restore_configuration end + teardown do + # Confirm that any changed configuration doesn't include non-symbol keys + $arvados_config.keys.each do |conf_name| + conf = Rails.configuration.send(conf_name) + confirm_keys_as_symbols(conf, conf_name) if conf.respond_to?('keys') + end + end + def assert_equal(expect, *args) if expect.nil? assert_nil(*args) @@ -99,6 +107,14 @@ class ActiveSupport::TestCase end end + def confirm_keys_as_symbols(conf, conf_name) + assert(conf.is_a?(ActiveSupport::OrderedOptions), "#{conf_name} should be an OrderedOptions object") + conf.keys.each do |k| + assert(k.is_a?(Symbol), "Key '#{k}' on section '#{conf_name}' should be a Symbol") + confirm_keys_as_symbols(conf[k], "#{conf_name}.#{k}") if conf[k].respond_to?('keys') + end + end + def restore_configuration # Restore configuration settings changed during tests ConfigLoader.copy_into_config $arvados_config, Rails.configuration @@ -107,6 +123,7 @@ class ActiveSupport::TestCase def set_user_from_auth(auth_name) client_auth = api_client_authorizations(auth_name) + client_auth.user.forget_cached_group_perms Thread.current[:api_client_authorization] = client_auth Thread.current[:api_client] = client_auth.api_client Thread.current[:user] = client_auth.user diff --git a/services/api/test/unit/api_client_test.rb b/services/api/test/unit/api_client_test.rb index df082c27fd..bf47cd175b 100644 --- a/services/api/test/unit/api_client_test.rb +++ b/services/api/test/unit/api_client_test.rb @@ -7,25 +7,36 @@ require 'test_helper' class ApiClientTest < ActiveSupport::TestCase include CurrentApiClient - test "configured workbench is trusted" do - Rails.configuration.Services.Workbench1.ExternalURL = URI("http://wb1.example.com") - Rails.configuration.Services.Workbench2.ExternalURL = URI("https://wb2.example.com:443") + [true, false].each do |token_lifetime_enabled| + test "configured workbench is trusted when token lifetime is#{token_lifetime_enabled ? '': ' not'} enabled" do + Rails.configuration.Login.TokenLifetime = token_lifetime_enabled ? 8.hours : 0 + Rails.configuration.Services.Workbench1.ExternalURL = URI("http://wb1.example.com") + Rails.configuration.Services.Workbench2.ExternalURL = URI("https://wb2.example.com:443") + Rails.configuration.Login.TrustedClients = ActiveSupport::OrderedOptions.new + Rails.configuration.Login.TrustedClients[:"https://wb3.example.com"] = ActiveSupport::OrderedOptions.new - act_as_system_user do - [["http://wb0.example.com", false], - ["http://wb1.example.com", true], - ["http://wb2.example.com", false], - ["https://wb2.example.com", true], - ["https://wb2.example.com/", true], - ].each do |pfx, result| - a = ApiClient.create(url_prefix: pfx, is_trusted: false) - assert_equal result, a.is_trusted - end + act_as_system_user do + [["http://wb0.example.com", false], + ["http://wb1.example.com", true], + ["http://wb2.example.com", false], + ["https://wb2.example.com", true], + ["https://wb2.example.com/", true], + ["https://wb3.example.com/", true], + ["https://wb4.example.com/", false], + ].each do |pfx, result| + a = ApiClient.create(url_prefix: pfx, is_trusted: false) + if token_lifetime_enabled + assert_equal false, a.is_trusted, "API client with url prefix '#{pfx}' shouldn't be trusted" + else + assert_equal result, a.is_trusted + end + end - a = ApiClient.create(url_prefix: "http://example.com", is_trusted: true) - a.save! - a.reload - assert a.is_trusted + a = ApiClient.create(url_prefix: "http://example.com", is_trusted: true) + a.save! + a.reload + assert a.is_trusted + end end end end diff --git a/services/api/test/unit/application_test.rb b/services/api/test/unit/application_test.rb index 679dddf223..e1565ec627 100644 --- a/services/api/test/unit/application_test.rb +++ b/services/api/test/unit/application_test.rb @@ -7,7 +7,7 @@ require 'test_helper' class ApplicationTest < ActiveSupport::TestCase include CurrentApiClient - test "test act_as_system_user" do + test "act_as_system_user" do Thread.current[:user] = users(:active) assert_equal users(:active), Thread.current[:user] act_as_system_user do @@ -17,7 +17,7 @@ class ApplicationTest < ActiveSupport::TestCase assert_equal users(:active), Thread.current[:user] end - test "test act_as_system_user is exception safe" do + test "act_as_system_user is exception safe" do Thread.current[:user] = users(:active) assert_equal users(:active), Thread.current[:user] caught = false @@ -33,4 +33,12 @@ class ApplicationTest < ActiveSupport::TestCase assert caught assert_equal users(:active), Thread.current[:user] end + + test "config maps' keys are returned as symbols" do + assert Rails.configuration.Users.AutoSetupUsernameBlacklist.is_a? ActiveSupport::OrderedOptions + assert Rails.configuration.Users.AutoSetupUsernameBlacklist.keys.size > 0 + Rails.configuration.Users.AutoSetupUsernameBlacklist.keys.each do |k| + assert k.is_a? Symbol + end + end end diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb index addea83062..916ca09587 100644 --- a/services/api/test/unit/collection_test.rb +++ b/services/api/test/unit/collection_test.rb @@ -4,6 +4,7 @@ require 'test_helper' require 'sweep_trashed_objects' +require 'fix_collection_versions_timestamps' class CollectionTest < ActiveSupport::TestCase include DbCurrentTime @@ -187,9 +188,9 @@ class CollectionTest < ActiveSupport::TestCase end end - test "preserve_version=false assignment is ignored while being true and not producing a new version" do + test "preserve_version updates" do Rails.configuration.Collections.CollectionVersioning = true - Rails.configuration.Collections.PreserveVersionIfIdle = 3600 + Rails.configuration.Collections.PreserveVersionIfIdle = -1 # disabled act_as_user users(:active) do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -198,28 +199,61 @@ class CollectionTest < ActiveSupport::TestCase assert_equal false, c.preserve_version # This update shouldn't produce a new version, as the idle time is not up c.update_attributes!({ - 'name' => 'bar', - 'preserve_version' => true + 'name' => 'bar' }) c.reload assert_equal 1, c.version assert_equal 'bar', c.name + assert_equal false, c.preserve_version + # This update should produce a new version, even if the idle time is not up + # and also keep the preserve_version=true flag to persist it. + c.update_attributes!({ + 'name' => 'baz', + 'preserve_version' => true + }) + c.reload + assert_equal 2, c.version + assert_equal 'baz', c.name assert_equal true, c.preserve_version # Make sure preserve_version is not disabled after being enabled, unless # a new version is created. + # This is a non-versionable update c.update_attributes!({ 'preserve_version' => false, 'replication_desired' => 2 }) c.reload - assert_equal 1, c.version + assert_equal 2, c.version assert_equal 2, c.replication_desired assert_equal true, c.preserve_version - c.update_attributes!({'name' => 'foobar'}) + # This is a versionable update + c.update_attributes!({ + 'preserve_version' => false, + 'name' => 'foobar' + }) c.reload - assert_equal 2, c.version + assert_equal 3, c.version assert_equal false, c.preserve_version assert_equal 'foobar', c.name + # Flipping only 'preserve_version' to true doesn't create a new version + c.update_attributes!({'preserve_version' => true}) + c.reload + assert_equal 3, c.version + assert_equal true, c.preserve_version + end + end + + test "preserve_version updates don't change modified_at timestamp" do + act_as_user users(:active) do + c = create_collection 'foo', Encoding::US_ASCII + assert c.valid? + assert_equal false, c.preserve_version + modified_at = c.modified_at.to_f + c.update_attributes!({'preserve_version' => true}) + c.reload + assert_equal true, c.preserve_version + assert_equal modified_at, c.modified_at.to_f, + 'preserve_version updates should not trigger modified_at changes' end end @@ -334,6 +368,7 @@ class CollectionTest < ActiveSupport::TestCase # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII assert c.valid? + original_version_modified_at = c.modified_at.to_f # Make changes so that a new version is created c.update_attributes!({'name' => 'bar'}) c.reload @@ -344,9 +379,7 @@ class CollectionTest < ActiveSupport::TestCase version_creation_datetime = c_old.modified_at.to_f assert_equal c.created_at.to_f, c_old.created_at.to_f - # Current version is updated just a few milliseconds before the version is - # saved on the database. - assert_operator c.modified_at.to_f, :<, version_creation_datetime + assert_equal original_version_modified_at, version_creation_datetime # Make update on current version so old version get the attribute synced; # its modified_at should not change. @@ -361,6 +394,29 @@ class CollectionTest < ActiveSupport::TestCase end end + # Bug #17152 - This test relies on fixtures simulating the problem. + test "migration fixing collection versions' modified_at timestamps" do + versioned_collection_fixtures = [ + collections(:w_a_z_file).uuid, + collections(:collection_owned_by_active).uuid + ] + versioned_collection_fixtures.each do |uuid| + cols = Collection.where(current_version_uuid: uuid).order(version: :desc) + assert_equal cols.size, 2 + # cols[0] -> head version // cols[1] -> old version + assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :==, 0 + assert cols[1].modified_at != cols[1].created_at + end + fix_collection_versions_timestamps + versioned_collection_fixtures.each do |uuid| + cols = Collection.where(current_version_uuid: uuid).order(version: :desc) + assert_equal cols.size, 2 + # cols[0] -> head version // cols[1] -> old version + assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :>, 1 + assert_operator cols[1].modified_at, :==, cols[1].created_at + end + end + test "past versions should not be directly updatable" do Rails.configuration.Collections.CollectionVersioning = true Rails.configuration.Collections.PreserveVersionIfIdle = 0 @@ -1044,10 +1100,10 @@ class CollectionTest < ActiveSupport::TestCase end test "create collections with managed properties" do - Rails.configuration.Collections.ManagedProperties = { + Rails.configuration.Collections.ManagedProperties = ConfigLoader.to_OrderedOptions({ 'default_prop1' => {'Value' => 'prop1_value'}, 'responsible_person_uuid' => {'Function' => 'original_owner'} - } + }) # Test collection without initial properties act_as_user users(:active) do c = create_collection 'foo', Encoding::US_ASCII @@ -1076,9 +1132,9 @@ class CollectionTest < ActiveSupport::TestCase end test "update collection with protected managed properties" do - Rails.configuration.Collections.ManagedProperties = { + Rails.configuration.Collections.ManagedProperties = ConfigLoader.to_OrderedOptions({ 'default_prop1' => {'Value' => 'prop1_value', 'Protected' => true}, - } + }) act_as_user users(:active) do c = create_collection 'foo', Encoding::US_ASCII assert c.valid? diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb index b91910d2d6..90de800b2f 100644 --- a/services/api/test/unit/container_request_test.rb +++ b/services/api/test/unit/container_request_test.rb @@ -576,7 +576,7 @@ class ContainerRequestTest < ActiveSupport::TestCase test "Container.resolve_container_image(pdh)" do set_user_from_auth :active [[:docker_image, 'v1'], [:docker_image_1_12, 'v2']].each do |coll, ver| - Rails.configuration.Containers.SupportedDockerImageFormats = {ver=>{}} + Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({ver=>{}}) pdh = collections(coll).portable_data_hash resolved = Container.resolve_container_image(pdh) assert_equal resolved, pdh @@ -602,7 +602,7 @@ class ContainerRequestTest < ActiveSupport::TestCase end test "migrated docker image" do - Rails.configuration.Containers.SupportedDockerImageFormats = {'v2'=>{}} + Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v2'=>{}}) add_docker19_migration_link # Test that it returns only v2 images even though request is for v1 image. @@ -620,7 +620,7 @@ class ContainerRequestTest < ActiveSupport::TestCase end test "use unmigrated docker image" do - Rails.configuration.Containers.SupportedDockerImageFormats = {'v1'=>{}} + Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v1'=>{}}) add_docker19_migration_link # Test that it returns only supported v1 images even though there is a @@ -639,7 +639,7 @@ class ContainerRequestTest < ActiveSupport::TestCase end test "incompatible docker image v1" do - Rails.configuration.Containers.SupportedDockerImageFormats = {'v1'=>{}} + Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v1'=>{}}) add_docker19_migration_link # Don't return unsupported v2 image even if we ask for it directly. @@ -652,7 +652,7 @@ class ContainerRequestTest < ActiveSupport::TestCase end test "incompatible docker image v2" do - Rails.configuration.Containers.SupportedDockerImageFormats = {'v2'=>{}} + Rails.configuration.Containers.SupportedDockerImageFormats = ConfigLoader.to_OrderedOptions({'v2'=>{}}) # No migration link, don't return unsupported v1 image, set_user_from_auth :active diff --git a/services/api/test/unit/create_superuser_token_test.rb b/services/api/test/unit/create_superuser_token_test.rb index e95e0f2264..3c6dcbdbbc 100644 --- a/services/api/test/unit/create_superuser_token_test.rb +++ b/services/api/test/unit/create_superuser_token_test.rb @@ -9,11 +9,11 @@ require 'create_superuser_token' class CreateSuperUserTokenTest < ActiveSupport::TestCase include CreateSuperUserToken - test "create superuser token twice and expect same resutls" do + test "create superuser token twice and expect same results" do # Create a token with some string token1 = create_superuser_token 'atesttoken' assert_not_nil token1 - assert_equal token1, 'atesttoken' + assert_match(/atesttoken$/, token1) # Create token again; this time, we should get the one created earlier token2 = create_superuser_token @@ -25,7 +25,7 @@ class CreateSuperUserTokenTest < ActiveSupport::TestCase # Create a token with some string token1 = create_superuser_token 'atesttoken' assert_not_nil token1 - assert_equal token1, 'atesttoken' + assert_match(/\/atesttoken$/, token1) # Create token again with some other string and expect the existing superuser token back token2 = create_superuser_token 'someothertokenstring' @@ -33,37 +33,26 @@ class CreateSuperUserTokenTest < ActiveSupport::TestCase assert_equal token1, token2 end - test "create superuser token twice and expect same results" do - # Create a token with some string - token1 = create_superuser_token 'atesttoken' - assert_not_nil token1 - assert_equal token1, 'atesttoken' - - # Create token again with that same superuser token and expect it back - token2 = create_superuser_token 'atesttoken' - assert_not_nil token2 - assert_equal token1, token2 - end - test "create superuser token and invoke again with some other valid token" do # Create a token with some string token1 = create_superuser_token 'atesttoken' assert_not_nil token1 - assert_equal token1, 'atesttoken' + assert_match(/\/atesttoken$/, token1) su_token = api_client_authorizations("system_user").api_token token2 = create_superuser_token su_token - assert_equal token2, su_token + assert_equal token2.split('/')[2], su_token end test "create superuser token, expire it, and create again" do # Create a token with some string token1 = create_superuser_token 'atesttoken' assert_not_nil token1 - assert_equal token1, 'atesttoken' + assert_match(/\/atesttoken$/, token1) # Expire this token and call create again; expect a new token created - apiClientAuth = ApiClientAuthorization.where(api_token: token1).first + apiClientAuth = ApiClientAuthorization.where(api_token: 'atesttoken').first + refute_nil apiClientAuth Thread.current[:user] = users(:admin) apiClientAuth.update_attributes expires_at: '2000-10-10' diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb index 0e8cc48538..c529aab8b6 100644 --- a/services/api/test/unit/job_test.rb +++ b/services/api/test/unit/job_test.rb @@ -117,7 +117,7 @@ class JobTest < ActiveSupport::TestCase 'locator' => BAD_COLLECTION, }.each_pair do |spec_type, image_spec| test "Job validation fails with nonexistent Docker image #{spec_type}" do - Rails.configuration.RemoteClusters = {} + Rails.configuration.RemoteClusters = ConfigLoader.to_OrderedOptions({}) job = Job.new job_attrs(runtime_constraints: {'docker_image' => image_spec}) assert(job.invalid?, "nonexistent Docker image #{spec_type} #{image_spec} was valid") diff --git a/services/api/test/unit/log_test.rb b/services/api/test/unit/log_test.rb index 016a0e4eb4..66c8c8d923 100644 --- a/services/api/test/unit/log_test.rb +++ b/services/api/test/unit/log_test.rb @@ -228,6 +228,20 @@ class LogTest < ActiveSupport::TestCase assert_logged(auth, :update) end + test "don't log changes only to Collection.preserve_version" do + set_user_from_auth :admin_trustedclient + col = collections(:collection_owned_by_active) + start_log_count = get_logs_about(col).size + assert_equal false, col.preserve_version + col.preserve_version = true + col.save! + assert_equal(start_log_count, get_logs_about(col).size, + "log count changed after updating Collection.preserve_version") + col.name = 'updated by admin' + col.save! + assert_logged(col, :update) + end + test "token isn't included in ApiClientAuthorization logs" do set_user_from_auth :admin_trustedclient auth = ApiClientAuthorization.new @@ -282,7 +296,7 @@ class LogTest < ActiveSupport::TestCase end test "non-empty configuration.unlogged_attributes" do - Rails.configuration.AuditLogs.UnloggedAttributes = {"manifest_text"=>{}} + Rails.configuration.AuditLogs.UnloggedAttributes = ConfigLoader.to_OrderedOptions({"manifest_text"=>{}}) txt = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n" act_as_system_user do @@ -297,7 +311,7 @@ class LogTest < ActiveSupport::TestCase end test "empty configuration.unlogged_attributes" do - Rails.configuration.AuditLogs.UnloggedAttributes = {} + Rails.configuration.AuditLogs.UnloggedAttributes = ConfigLoader.to_OrderedOptions({}) txt = ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n" act_as_system_user do @@ -319,6 +333,7 @@ class LogTest < ActiveSupport::TestCase def assert_no_logs_deleted logs_before = Log.unscoped.all.count + assert logs_before > 0 yield assert_equal logs_before, Log.unscoped.all.count end @@ -350,34 +365,34 @@ class LogTest < ActiveSupport::TestCase # but 3 minutes suits our test data better (and is test-worthy in # that it's expected to work correctly in production). test 'delete old audit logs with production settings' do - initial_log_count = Log.unscoped.all.count + initial_log_count = remaining_audit_logs.count + assert initial_log_count > 0 AuditLogs.delete_old(max_age: 180, max_batch: 100000) assert_operator remaining_audit_logs.count, :<, initial_log_count end test 'delete all audit logs in multiple batches' do + assert remaining_audit_logs.count > 2 AuditLogs.delete_old(max_age: 0.00001, max_batch: 2) assert_equal [], remaining_audit_logs.collect(&:uuid) end test 'delete old audit logs in thread' do - begin - Rails.configuration.AuditLogs.MaxAge = 20 - Rails.configuration.AuditLogs.MaxDeleteBatch = 100000 - Rails.cache.delete 'AuditLogs' - initial_log_count = Log.unscoped.all.count + 1 - act_as_system_user do - Log.create!() - initial_log_count += 1 - end - deadline = Time.now + 10 - while remaining_audit_logs.count == initial_log_count - if Time.now > deadline - raise "timed out" - end - sleep 0.1 + Rails.configuration.AuditLogs.MaxAge = 20 + Rails.configuration.AuditLogs.MaxDeleteBatch = 100000 + Rails.cache.delete 'AuditLogs' + initial_audit_log_count = remaining_audit_logs.count + assert initial_audit_log_count > 0 + act_as_system_user do + Log.create!() + end + deadline = Time.now + 10 + while remaining_audit_logs.count == initial_audit_log_count + if Time.now > deadline + raise "timed out" end - assert_operator remaining_audit_logs.count, :<, initial_log_count + sleep 0.1 end + assert_operator remaining_audit_logs.count, :<, initial_audit_log_count end end diff --git a/services/api/test/unit/permission_test.rb b/services/api/test/unit/permission_test.rb index 10664474c6..123031b35f 100644 --- a/services/api/test/unit/permission_test.rb +++ b/services/api/test/unit/permission_test.rb @@ -579,4 +579,24 @@ class PermissionTest < ActiveSupport::TestCase assert users(:active).can?(write: prj.uuid) assert users(:active).can?(manage: prj.uuid) end + + [system_user_uuid, anonymous_user_uuid].each do |u| + test "cannot delete system user #{u}" do + act_as_system_user do + assert_raises ArvadosModel::PermissionDeniedError do + User.find_by_uuid(u).destroy + end + end + end + end + + [system_group_uuid, anonymous_group_uuid, public_project_uuid].each do |g| + test "cannot delete system group #{g}" do + act_as_system_user do + assert_raises ArvadosModel::PermissionDeniedError do + Group.find_by_uuid(g).destroy + end + end + end + end end diff --git a/services/api/test/unit/user_notifier_test.rb b/services/api/test/unit/user_notifier_test.rb index da6c7fdb88..c288786c13 100644 --- a/services/api/test/unit/user_notifier_test.rb +++ b/services/api/test/unit/user_notifier_test.rb @@ -9,6 +9,24 @@ class UserNotifierTest < ActionMailer::TestCase # Send the email, then test that it got queued test "account is setup" do user = users :active + + Rails.configuration.Users.UserSetupMailText = %{ +<% if not @user.full_name.empty? -%> +<%= @user.full_name %>, +<% else -%> +Hi there, +<% end -%> + +Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services.Workbench1.ExternalURL %>at + +<%= Rails.configuration.Services.Workbench1.ExternalURL %><%= "/" if !Rails.configuration.Services.Workbench1.ExternalURL.to_s.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %> + +for connection instructions. + +Thanks, +The Arvados team. +} + email = UserNotifier.account_is_setup user assert_not_nil email diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb index 7fcd36d709..f973c6ba1f 100644 --- a/services/api/test/unit/user_test.rb +++ b/services/api/test/unit/user_test.rb @@ -110,7 +110,7 @@ class UserTest < ActiveSupport::TestCase end test "new username set avoiding blacklist" do - Rails.configuration.Users.AutoSetupUsernameBlacklist = {"root"=>{}} + Rails.configuration.Users.AutoSetupUsernameBlacklist = ConfigLoader.to_OrderedOptions({"root"=>{}}) check_new_username_setting("root", "root2") end @@ -340,50 +340,54 @@ class UserTest < ActiveSupport::TestCase assert_equal(user.first_name, 'first_name_for_newly_created_user_updated') end + active_notify_list = ConfigLoader.to_OrderedOptions({"active-notify@example.com"=>{}}) + inactive_notify_list = ConfigLoader.to_OrderedOptions({"inactive-notify@example.com"=>{}}) + empty_notify_list = ConfigLoader.to_OrderedOptions({}) + test "create new user with notifications" do set_user_from_auth :admin - create_user_and_verify_setup_and_notifications true, {'active-notify-address@example.com'=>{}}, {'inactive-notify-address@example.com'=>{}}, nil, nil - create_user_and_verify_setup_and_notifications true, {'active-notify-address@example.com'=>{}}, {}, nil, nil - create_user_and_verify_setup_and_notifications true, {}, [], nil, nil - create_user_and_verify_setup_and_notifications false, {'active-notify-address@example.com'=>{}}, {'inactive-notify-address@example.com'=>{}}, nil, nil - create_user_and_verify_setup_and_notifications false, {}, {'inactive-notify-address@example.com'=>{}}, nil, nil - create_user_and_verify_setup_and_notifications false, {}, {}, nil, nil + create_user_and_verify_setup_and_notifications true, active_notify_list, inactive_notify_list, nil, nil + create_user_and_verify_setup_and_notifications true, active_notify_list, empty_notify_list, nil, nil + create_user_and_verify_setup_and_notifications true, empty_notify_list, empty_notify_list, nil, nil + create_user_and_verify_setup_and_notifications false, active_notify_list, inactive_notify_list, nil, nil + create_user_and_verify_setup_and_notifications false, empty_notify_list, inactive_notify_list, nil, nil + create_user_and_verify_setup_and_notifications false, empty_notify_list, empty_notify_list, nil, nil end [ # Easy inactive user tests. - [false, {}, {}, "inactive-none@example.com", false, false, "inactivenone"], - [false, {}, {}, "inactive-vm@example.com", true, false, "inactivevm"], - [false, {}, {}, "inactive-repo@example.com", false, true, "inactiverepo"], - [false, {}, {}, "inactive-both@example.com", true, true, "inactiveboth"], + [false, empty_notify_list, empty_notify_list, "inactive-none@example.com", false, false, "inactivenone"], + [false, empty_notify_list, empty_notify_list, "inactive-vm@example.com", true, false, "inactivevm"], + [false, empty_notify_list, empty_notify_list, "inactive-repo@example.com", false, true, "inactiverepo"], + [false, empty_notify_list, empty_notify_list, "inactive-both@example.com", true, true, "inactiveboth"], # Easy active user tests. - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "active-none@example.com", false, false, "activenone"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "active-vm@example.com", true, false, "activevm"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "active-repo@example.com", false, true, "activerepo"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "active-both@example.com", true, true, "activeboth"], + [true, active_notify_list, inactive_notify_list, "active-none@example.com", false, false, "activenone"], + [true, active_notify_list, inactive_notify_list, "active-vm@example.com", true, false, "activevm"], + [true, active_notify_list, inactive_notify_list, "active-repo@example.com", false, true, "activerepo"], + [true, active_notify_list, inactive_notify_list, "active-both@example.com", true, true, "activeboth"], # Test users with malformed e-mail addresses. - [false, {}, {}, nil, true, true, nil], - [false, {}, {}, "arvados", true, true, nil], - [false, {}, {}, "@example.com", true, true, nil], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "*!*@example.com", true, false, nil], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "*!*@example.com", false, false, nil], + [false, empty_notify_list, empty_notify_list, nil, true, true, nil], + [false, empty_notify_list, empty_notify_list, "arvados", true, true, nil], + [false, empty_notify_list, empty_notify_list, "@example.com", true, true, nil], + [true, active_notify_list, inactive_notify_list, "*!*@example.com", true, false, nil], + [true, active_notify_list, inactive_notify_list, "*!*@example.com", false, false, nil], # Test users with various username transformations. - [false, {}, {}, "arvados@example.com", false, false, "arvados2"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "arvados@example.com", false, false, "arvados2"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "root@example.com", true, false, "root2"], - [false, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "root@example.com", true, false, "root2"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "roo_t@example.com", false, true, "root2"], - [false, {}, {}, "^^incorrect_format@example.com", true, true, "incorrectformat"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "&4a_d9.@example.com", true, true, "ad9"], - [true, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "&4a_d9.@example.com", false, false, "ad9"], - [false, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "&4a_d9.@example.com", true, true, "ad9"], - [false, {"active-notify@example.com"=>{}}, {"inactive-notify@example.com"=>{}}, "&4a_d9.@example.com", false, false, "ad9"], + [false, empty_notify_list, empty_notify_list, "arvados@example.com", false, false, "arvados2"], + [true, active_notify_list, inactive_notify_list, "arvados@example.com", false, false, "arvados2"], + [true, active_notify_list, inactive_notify_list, "root@example.com", true, false, "root2"], + [false, active_notify_list, inactive_notify_list, "root@example.com", true, false, "root2"], + [true, active_notify_list, inactive_notify_list, "roo_t@example.com", false, true, "root2"], + [false, empty_notify_list, empty_notify_list, "^^incorrect_format@example.com", true, true, "incorrectformat"], + [true, active_notify_list, inactive_notify_list, "&4a_d9.@example.com", true, true, "ad9"], + [true, active_notify_list, inactive_notify_list, "&4a_d9.@example.com", false, false, "ad9"], + [false, active_notify_list, inactive_notify_list, "&4a_d9.@example.com", true, true, "ad9"], + [false, active_notify_list, inactive_notify_list, "&4a_d9.@example.com", false, false, "ad9"], ].each do |active, new_user_recipients, inactive_recipients, email, auto_setup_vm, auto_setup_repo, expect_username| - test "create new user with auto setup #{active} #{email} #{auto_setup_vm} #{auto_setup_repo}" do + test "create new user with auto setup active=#{active} email=#{email} vm=#{auto_setup_vm} repo=#{auto_setup_repo}" do set_user_from_auth :admin Rails.configuration.Users.AutoSetupNewUsers = true @@ -569,7 +573,6 @@ class UserTest < ActiveSupport::TestCase assert_not_nil resp_user, 'expected user object' assert_not_nil resp_user['uuid'], 'expected user object' assert_equal email, resp_user['email'], 'expected email not found' - end def verify_link (link_object, link_class, link_name, tail_uuid, head_uuid) @@ -618,6 +621,7 @@ class UserTest < ActiveSupport::TestCase Rails.configuration.Users.AutoSetupNewUsersWithRepository), named_repo.uuid, user.uuid, "permission", "can_manage") end + # Check for VM login. if (auto_vm_uuid = Rails.configuration.Users.AutoSetupNewUsersWithVmUUID) != "" verify_link_exists(can_setup, auto_vm_uuid, user.uuid, @@ -648,7 +652,7 @@ class UserTest < ActiveSupport::TestCase if not new_user_recipients.empty? then assert_not_nil new_user_email, 'Expected new user email after setup' assert_equal Rails.configuration.Users.UserNotifierEmailFrom, new_user_email.from[0] - assert_equal new_user_recipients.keys.first, new_user_email.to[0] + assert_equal new_user_recipients.stringify_keys.keys.first, new_user_email.to[0] assert_equal new_user_email_subject, new_user_email.subject else assert_nil new_user_email, 'Did not expect new user email after setup' @@ -658,7 +662,7 @@ class UserTest < ActiveSupport::TestCase if not inactive_recipients.empty? then assert_not_nil new_inactive_user_email, 'Expected new inactive user email after setup' assert_equal Rails.configuration.Users.UserNotifierEmailFrom, new_inactive_user_email.from[0] - assert_equal inactive_recipients.keys.first, new_inactive_user_email.to[0] + assert_equal inactive_recipients.stringify_keys.keys.first, new_inactive_user_email.to[0] assert_equal "#{Rails.configuration.Users.EmailSubjectPrefix}New inactive user notification", new_inactive_user_email.subject else assert_nil new_inactive_user_email, 'Did not expect new inactive user email after setup' @@ -667,7 +671,6 @@ class UserTest < ActiveSupport::TestCase assert_nil new_inactive_user_email, 'Expected no inactive user email after setting up active user' end ActionMailer::Base.deliveries = [] - end def verify_link_exists link_exists, head_uuid, tail_uuid, link_class, link_name, property_name=nil, property_value=nil @@ -675,7 +678,7 @@ class UserTest < ActiveSupport::TestCase tail_uuid: tail_uuid, link_class: link_class, name: link_name) - assert_equal link_exists, all_links.any?, "Link #{'not' if link_exists} found for #{link_name} #{link_class} #{property_value}" + assert_equal link_exists, all_links.any?, "Link#{' not' if link_exists} found for #{link_name} #{link_class} #{property_value}" if link_exists && property_name && property_value all_links.each do |link| assert_equal true, all_links.first.properties[property_name].start_with?(property_value), 'Property not found in link' diff --git a/services/arv-git-httpd/auth_handler_test.go b/services/arv-git-httpd/auth_handler_test.go index 4b8f95ef33..4e1a47dcb2 100644 --- a/services/arv-git-httpd/auth_handler_test.go +++ b/services/arv-git-httpd/auth_handler_test.go @@ -44,7 +44,7 @@ func (s *AuthHandlerSuite) SetUpTest(c *check.C) { s.cluster, err = cfg.GetCluster("") c.Assert(err, check.Equals, nil) - s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}} + s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}} s.cluster.TLS.Insecure = true s.cluster.Git.GitCommand = "/usr/bin/git" s.cluster.Git.Repositories = repoRoot diff --git a/services/arv-git-httpd/git_handler_test.go b/services/arv-git-httpd/git_handler_test.go index c14030f95d..dafe5d31d7 100644 --- a/services/arv-git-httpd/git_handler_test.go +++ b/services/arv-git-httpd/git_handler_test.go @@ -28,7 +28,7 @@ func (s *GitHandlerSuite) SetUpTest(c *check.C) { s.cluster, err = cfg.GetCluster("") c.Assert(err, check.Equals, nil) - s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:80"}: arvados.ServiceInstance{}} + s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:80"}: {}} s.cluster.Git.GitoliteHome = "/test/ghh" s.cluster.Git.Repositories = "/" } diff --git a/services/arv-git-httpd/gitolite_test.go b/services/arv-git-httpd/gitolite_test.go index 5f3cc608c3..fb0fc0d783 100644 --- a/services/arv-git-httpd/gitolite_test.go +++ b/services/arv-git-httpd/gitolite_test.go @@ -54,7 +54,7 @@ func (s *GitoliteSuite) SetUpTest(c *check.C) { s.cluster, err = cfg.GetCluster("") c.Assert(err, check.Equals, nil) - s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}} + s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}} s.cluster.TLS.Insecure = true s.cluster.Git.GitCommand = "/usr/share/gitolite3/gitolite-shell" s.cluster.Git.GitoliteHome = s.gitoliteHome diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go index b50c2a2341..12ddc5b770 100644 --- a/services/arv-git-httpd/integration_test.go +++ b/services/arv-git-httpd/integration_test.go @@ -68,7 +68,7 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) { s.cluster, err = cfg.GetCluster("") c.Assert(err, check.Equals, nil) - s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}} + s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}} s.cluster.TLS.Insecure = true s.cluster.Git.GitCommand = "/usr/bin/git" s.cluster.Git.Repositories = s.tmpRepoRoot diff --git a/services/arv-web/README b/services/arv-web/README deleted file mode 100644 index eaf7624dc4..0000000000 --- a/services/arv-web/README +++ /dev/null @@ -1,6 +0,0 @@ -arv-web enables you to run a custom web service using the contents of an -Arvados collection. - -See "Using arv-web" in the Arvados user guide: - -http://doc.arvados.org/user/topics/arv-web.html diff --git a/services/arv-web/arv-web.py b/services/arv-web/arv-web.py deleted file mode 100755 index 55b710a754..0000000000 --- a/services/arv-web/arv-web.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -# arv-web enables you to run a custom web service from the contents of an Arvados collection. -# -# See http://doc.arvados.org/user/topics/arv-web.html - -import arvados -from arvados.safeapi import ThreadSafeApiCache -import subprocess -from arvados_fuse import Operations, CollectionDirectory -import tempfile -import os -import llfuse -import threading -import Queue -import argparse -import logging -import signal -import sys -import functools - -logger = logging.getLogger('arvados.arv-web') -logger.setLevel(logging.INFO) - -class ArvWeb(object): - def __init__(self, project, docker_image, port): - self.project = project - self.loop = True - self.cid = None - self.prev_docker_image = None - self.mountdir = None - self.collection = None - self.override_docker_image = docker_image - self.port = port - self.evqueue = Queue.Queue() - self.api = ThreadSafeApiCache(arvados.config.settings()) - - if arvados.util.group_uuid_pattern.match(project) is None: - raise arvados.errors.ArgumentError("Project uuid is not valid") - - collections = self.api.collections().list(filters=[["owner_uuid", "=", project]], - limit=1, - order='modified_at desc').execute()['items'] - self.newcollection = collections[0]['uuid'] if collections else None - - self.ws = arvados.events.subscribe(self.api, [["object_uuid", "is_a", "arvados#collection"]], self.on_message) - - def check_docker_running(self): - # It would be less hacky to use "docker events" than poll "docker ps" - # but that would require writing a bigger pile of code. - if self.cid: - ps = subprocess.check_output(["docker", "ps", "--no-trunc=true", "--filter=status=running"]) - for l in ps.splitlines(): - if l.startswith(self.cid): - return True - return False - - # Handle messages from Arvados event bus. - def on_message(self, ev): - if 'event_type' in ev: - old_attr = None - if 'old_attributes' in ev['properties'] and ev['properties']['old_attributes']: - old_attr = ev['properties']['old_attributes'] - if self.project not in (ev['properties']['new_attributes']['owner_uuid'], - old_attr['owner_uuid'] if old_attr else None): - return - - et = ev['event_type'] - if ev['event_type'] == 'update': - if ev['properties']['new_attributes']['owner_uuid'] != ev['properties']['old_attributes']['owner_uuid']: - if self.project == ev['properties']['new_attributes']['owner_uuid']: - et = 'add' - else: - et = 'remove' - if ev['properties']['new_attributes']['trash_at'] is not None: - et = 'remove' - - self.evqueue.put((self.project, et, ev['object_uuid'])) - - # Run an arvados_fuse mount under the control of the local process. This lets - # us switch out the contents of the directory without having to unmount and - # remount. - def run_fuse_mount(self): - self.mountdir = tempfile.mkdtemp() - - self.operations = Operations(os.getuid(), os.getgid(), self.api, "utf-8") - self.cdir = CollectionDirectory(llfuse.ROOT_INODE, self.operations.inodes, self.api, 2, self.collection) - self.operations.inodes.add_entry(self.cdir) - - # Initialize the fuse connection - llfuse.init(self.operations, self.mountdir, ['allow_other']) - - t = threading.Thread(None, llfuse.main) - t.start() - - # wait until the driver is finished initializing - self.operations.initlock.wait() - - def mount_collection(self): - if self.newcollection != self.collection: - self.collection = self.newcollection - if not self.mountdir and self.collection: - self.run_fuse_mount() - - if self.mountdir: - with llfuse.lock: - self.cdir.clear() - # Switch the FUSE directory object so that it stores - # the newly selected collection - if self.collection: - logger.info("Mounting %s", self.collection) - else: - logger.info("Mount is empty") - self.cdir.change_collection(self.collection) - - - def stop_docker(self): - if self.cid: - logger.info("Stopping Docker container") - subprocess.call(["docker", "stop", self.cid]) - self.cid = None - - def run_docker(self): - try: - if self.collection is None: - self.stop_docker() - return - - docker_image = None - if self.override_docker_image: - docker_image = self.override_docker_image - else: - try: - with llfuse.lock: - if "docker_image" in self.cdir: - docker_image = self.cdir["docker_image"].readfrom(0, 1024).strip() - except IOError as e: - pass - - has_reload = False - try: - with llfuse.lock: - has_reload = "reload" in self.cdir - except IOError as e: - pass - - if docker_image is None: - logger.error("Collection must contain a file 'docker_image' or must specify --image on the command line.") - self.stop_docker() - return - - if docker_image == self.prev_docker_image and self.cid is not None and has_reload: - logger.info("Running container reload command") - subprocess.check_call(["docker", "exec", self.cid, "/mnt/reload"]) - return - - self.stop_docker() - - logger.info("Starting Docker container %s", docker_image) - self.cid = subprocess.check_output(["docker", "run", - "--detach=true", - "--publish=%i:80" % (self.port), - "--volume=%s:/mnt:ro" % self.mountdir, - docker_image]).strip() - - self.prev_docker_image = docker_image - logger.info("Container id %s", self.cid) - - except subprocess.CalledProcessError: - self.cid = None - - def wait_for_events(self): - if not self.cid: - logger.warning("No service running! Will wait for a new collection to appear in the project.") - else: - logger.info("Waiting for events") - - running = True - self.loop = True - while running: - # Main run loop. Wait on project events, signals, or the - # Docker container stopping. - - try: - # Poll the queue with a 1 second timeout, if we have no - # timeout the Python runtime doesn't have a chance to - # process SIGINT or SIGTERM. - eq = self.evqueue.get(True, 1) - logger.info("%s %s", eq[1], eq[2]) - self.newcollection = self.collection - if eq[1] in ('add', 'update', 'create'): - self.newcollection = eq[2] - elif eq[1] == 'remove': - collections = self.api.collections().list(filters=[["owner_uuid", "=", self.project]], - limit=1, - order='modified_at desc').execute()['items'] - self.newcollection = collections[0]['uuid'] if collections else None - running = False - except Queue.Empty: - pass - - if self.cid and not self.check_docker_running(): - logger.warning("Service has terminated. Will try to restart.") - self.cid = None - running = False - - - def run(self): - try: - while self.loop: - self.loop = False - self.mount_collection() - try: - self.run_docker() - self.wait_for_events() - except (KeyboardInterrupt): - logger.info("Got keyboard interrupt") - self.ws.close() - self.loop = False - except Exception as e: - logger.exception("Caught fatal exception, shutting down") - self.ws.close() - self.loop = False - finally: - self.stop_docker() - - if self.mountdir: - logger.info("Unmounting") - subprocess.call(["fusermount", "-u", self.mountdir]) - os.rmdir(self.mountdir) - - -def main(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--project-uuid', type=str, required=True, help="Project uuid to watch") - parser.add_argument('--port', type=int, default=8080, help="Host port to listen on (default 8080)") - parser.add_argument('--image', type=str, help="Docker image to run") - - args = parser.parse_args(argv) - - signal.signal(signal.SIGTERM, lambda signal, frame: sys.exit(0)) - - try: - arvweb = ArvWeb(args.project_uuid, args.image, args.port) - arvweb.run() - except arvados.errors.ArgumentError as e: - logger.error(e) - return 1 - - return 0 - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/services/arv-web/sample-cgi-app/docker_image b/services/arv-web/sample-cgi-app/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-cgi-app/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-cgi-app/public/.htaccess b/services/arv-web/sample-cgi-app/public/.htaccess deleted file mode 100644 index e5145bd37d..0000000000 --- a/services/arv-web/sample-cgi-app/public/.htaccess +++ /dev/null @@ -1,3 +0,0 @@ -Options +ExecCGI -AddHandler cgi-script .cgi -DirectoryIndex index.cgi diff --git a/services/arv-web/sample-cgi-app/public/index.cgi b/services/arv-web/sample-cgi-app/public/index.cgi deleted file mode 100755 index 57bc2a9a01..0000000000 --- a/services/arv-web/sample-cgi-app/public/index.cgi +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/perl - -print "Content-type: text/html\n\n"; -print "Hello world from perl!"; diff --git a/services/arv-web/sample-cgi-app/tmp/.keepkeep b/services/arv-web/sample-cgi-app/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-rack-app/config.ru b/services/arv-web/sample-rack-app/config.ru deleted file mode 100644 index 65f3c7ca36..0000000000 --- a/services/arv-web/sample-rack-app/config.ru +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -app = proc do |env| - [200, { "Content-Type" => "text/html" }, ["hello world from ruby"]] -end -run app diff --git a/services/arv-web/sample-rack-app/docker_image b/services/arv-web/sample-rack-app/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-rack-app/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-rack-app/public/.keepkeep b/services/arv-web/sample-rack-app/public/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-rack-app/tmp/.keepkeep b/services/arv-web/sample-rack-app/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-static-page/docker_image b/services/arv-web/sample-static-page/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-static-page/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-static-page/public/index.html b/services/arv-web/sample-static-page/public/index.html deleted file mode 100644 index e8608a5ebe..0000000000 --- a/services/arv-web/sample-static-page/public/index.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - arv-web sample - -

Hello world static page

- - diff --git a/services/arv-web/sample-static-page/tmp/.keepkeep b/services/arv-web/sample-static-page/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-wsgi-app/docker_image b/services/arv-web/sample-wsgi-app/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-wsgi-app/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-wsgi-app/passenger_wsgi.py b/services/arv-web/sample-wsgi-app/passenger_wsgi.py deleted file mode 100644 index faec3c23cd..0000000000 --- a/services/arv-web/sample-wsgi-app/passenger_wsgi.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -def application(environ, start_response): - start_response('200 OK', [('Content-Type', 'text/plain')]) - return [b"hello world from python!\n"] diff --git a/services/arv-web/sample-wsgi-app/public/.keepkeep b/services/arv-web/sample-wsgi-app/public/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-wsgi-app/tmp/.keepkeep b/services/arv-web/sample-wsgi-app/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.service b/services/crunch-dispatch-local/crunch-dispatch-local.service new file mode 100644 index 0000000000..692d81e570 --- /dev/null +++ b/services/crunch-dispatch-local/crunch-dispatch-local.service @@ -0,0 +1,29 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 +[Unit] +Description=Arvados Crunch Dispatcher for LOCAL service +Documentation=https://doc.arvados.org/ +After=network.target + +# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section +StartLimitInterval=0 + +# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section +StartLimitIntervalSec=0 + +[Service] +Type=simple +EnvironmentFile=-/etc/arvados/crunch-dispatch-local-credentials +ExecStart=/usr/bin/crunch-dispatch-local -poll-interval=1 -crunch-run-command=/usr/bin/crunch-run +# Set a reasonable default for the open file limit +LimitNOFILE=65536 +Restart=always +RestartSec=1 +LimitNOFILE=1000000 + +# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section +StartLimitInterval=0 + +[Install] +WantedBy=multi-user.target diff --git a/apps/workbench/app/models/application_record.rb b/services/crunch-dispatch-local/fpm-info.sh similarity index 55% rename from apps/workbench/app/models/application_record.rb rename to services/crunch-dispatch-local/fpm-info.sh index 759034da66..6956c4c597 100644 --- a/apps/workbench/app/models/application_record.rb +++ b/services/crunch-dispatch-local/fpm-info.sh @@ -2,6 +2,4 @@ # # SPDX-License-Identifier: AGPL-3.0 -class ApplicationRecord < ActiveRecord::Base - self.abstract_class = true -end \ No newline at end of file +fpm_depends+=(crunch-run) diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go index 4115482d80..a5899ce8a7 100644 --- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go +++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go @@ -202,7 +202,7 @@ var containerUuidPattern = regexp.MustCompile(`^[a-z0-9]{5}-dz642-[a-z0-9]{15}$` // Cancelled or Complete. See https://dev.arvados.org/issues/10979 func (disp *Dispatcher) checkSqueueForOrphans() { for _, uuid := range disp.sqCheck.All() { - if !containerUuidPattern.MatchString(uuid) { + if !containerUuidPattern.MatchString(uuid) || !strings.HasPrefix(uuid, disp.cluster.ClusterID) { continue } err := disp.TrackContainer(uuid) diff --git a/services/crunch-dispatch-slurm/squeue.go b/services/crunch-dispatch-slurm/squeue.go index 5aee7e087b..eae21e62b6 100644 --- a/services/crunch-dispatch-slurm/squeue.go +++ b/services/crunch-dispatch-slurm/squeue.go @@ -23,8 +23,8 @@ type slurmJob struct { hitNiceLimit bool } -// Squeue implements asynchronous polling monitor of the SLURM queue using the -// command 'squeue'. +// SqueueChecker implements asynchronous polling monitor of the SLURM queue +// using the command 'squeue'. type SqueueChecker struct { Logger logger Period time.Duration @@ -102,13 +102,12 @@ func (sqc *SqueueChecker) reniceAll() { sort.Slice(jobs, func(i, j int) bool { if jobs[i].wantPriority != jobs[j].wantPriority { return jobs[i].wantPriority > jobs[j].wantPriority - } else { - // break ties with container uuid -- - // otherwise, the ordering would change from - // one interval to the next, and we'd do many - // pointless slurm queue rearrangements. - return jobs[i].uuid > jobs[j].uuid } + // break ties with container uuid -- + // otherwise, the ordering would change from + // one interval to the next, and we'd do many + // pointless slurm queue rearrangements. + return jobs[i].uuid > jobs[j].uuid }) renice := wantNice(jobs, sqc.PrioritySpread) for i, job := range jobs { diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py index 9aabff4292..38e6f564e7 100644 --- a/services/dockercleaner/arvados_version.py +++ b/services/dockercleaner/arvados_version.py @@ -6,21 +6,41 @@ import subprocess import time import os import re +import sys + +SETUP_DIR = os.path.dirname(os.path.abspath(__file__)) +VERSION_PATHS = { + SETUP_DIR, + os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh")) + } + +def choose_version_from(): + ts = {} + for path in VERSION_PATHS: + ts[subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', path]).strip()] = path + + sorted_ts = sorted(ts.items()) + getver = sorted_ts[-1][1] + print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr) + return getver def git_version_at_commit(): - curdir = os.path.dirname(os.path.abspath(__file__)) + curdir = choose_version_from() myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', '--format=%H', curdir]).strip() - myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode() return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: - return fp.write("__version__ = '%s'\n" % v) + v = v.replace("~dev", ".dev").replace("~rc", "rc") + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: + return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: - return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: + return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") @@ -30,7 +50,8 @@ def get_version(setup_dir, module): else: try: save_version(setup_dir, module, git_version_at_commit()) - except (subprocess.CalledProcessError, OSError): + except (subprocess.CalledProcessError, OSError) as err: + print("ERROR: {0}".format(err), file=sys.stderr) pass return read_version(setup_dir, module) diff --git a/services/dockercleaner/bin/arvados-docker-cleaner b/services/dockercleaner/bin/arvados-docker-cleaner index c00593fbf1..b9dcd79500 100755 --- a/services/dockercleaner/bin/arvados-docker-cleaner +++ b/services/dockercleaner/bin/arvados-docker-cleaner @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 diff --git a/services/dockercleaner/fpm-info.sh b/services/dockercleaner/fpm-info.sh index d678fdfd7a..ccb7a467af 100644 --- a/services/dockercleaner/fpm-info.sh +++ b/services/dockercleaner/fpm-info.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 case "$TARGET" in - debian9 | ubuntu1604) + ubuntu1604) fpm_depends+=() ;; debian* | ubuntu*) diff --git a/services/dockercleaner/gittaggers.py b/services/dockercleaner/gittaggers.py deleted file mode 120000 index a9ad861d81..0000000000 --- a/services/dockercleaner/gittaggers.py +++ /dev/null @@ -1 +0,0 @@ -../../sdk/python/gittaggers.py \ No newline at end of file diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py index 8b12f73e89..db5020cfef 100644 --- a/services/fuse/arvados_fuse/fusedir.py +++ b/services/fuse/arvados_fuse/fusedir.py @@ -7,17 +7,17 @@ from __future__ import division from future.utils import viewitems from future.utils import itervalues from builtins import dict -import logging -import re -import time -import llfuse -import arvados import apiclient +import arvados +import errno import functools +import llfuse +import logging +import re +import sys import threading -from apiclient import errors as apiclient_errors -import errno import time +from apiclient import errors as apiclient_errors from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile from .fresh import FreshBase, convertTime, use_counter, check_update @@ -689,7 +689,6 @@ and the directory will appear if it exists. e = self.inodes.add_entry(ProjectDirectory( self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0])) else: - import sys e = self.inodes.add_entry(CollectionDirectory( self.inode, self.inodes, self.api, self.num_retries, k)) diff --git a/services/fuse/arvados_fuse/unmount.py b/services/fuse/arvados_fuse/unmount.py index 1f06d8c91c..dbfea1f904 100644 --- a/services/fuse/arvados_fuse/unmount.py +++ b/services/fuse/arvados_fuse/unmount.py @@ -6,6 +6,7 @@ import collections import errno import os import subprocess +import sys import time diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py index 0c653694f5..d8eec3d9ee 100644 --- a/services/fuse/arvados_version.py +++ b/services/fuse/arvados_version.py @@ -6,36 +6,42 @@ import subprocess import time import os import re +import sys SETUP_DIR = os.path.dirname(os.path.abspath(__file__)) +VERSION_PATHS = { + SETUP_DIR, + os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")), + os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh")) + } def choose_version_from(): - sdk_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip() - cwl_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', SETUP_DIR]).strip() - if int(sdk_ts) > int(cwl_ts): - getver = os.path.join(SETUP_DIR, "../../sdk/python") - else: - getver = SETUP_DIR + ts = {} + for path in VERSION_PATHS: + ts[subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', path]).strip()] = path + + sorted_ts = sorted(ts.items()) + getver = sorted_ts[-1][1] + print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr) return getver def git_version_at_commit(): curdir = choose_version_from() myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', '--format=%H', curdir]).strip() - myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode() return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: - return fp.write("__version__ = '%s'\n" % v) + v = v.replace("~dev", ".dev").replace("~rc", "rc") + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: + return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: - return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: + return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") @@ -45,7 +51,8 @@ def get_version(setup_dir, module): else: try: save_version(setup_dir, module, git_version_at_commit()) - except (subprocess.CalledProcessError, OSError): + except (subprocess.CalledProcessError, OSError) as err: + print("ERROR: {0}".format(err), file=sys.stderr) pass return read_version(setup_dir, module) diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount index 2663e3def7..019e9644a8 100755 --- a/services/fuse/bin/arv-mount +++ b/services/fuse/bin/arv-mount @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 diff --git a/services/fuse/gittaggers.py b/services/fuse/gittaggers.py deleted file mode 120000 index a9ad861d81..0000000000 --- a/services/fuse/gittaggers.py +++ /dev/null @@ -1 +0,0 @@ -../../sdk/python/gittaggers.py \ No newline at end of file diff --git a/services/fuse/setup.py b/services/fuse/setup.py index 8398670619..545b4bfa01 100644 --- a/services/fuse/setup.py +++ b/services/fuse/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index 2ff2136ed7..eeb78ad905 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -144,14 +144,14 @@ var selectPDH = map[string]interface{}{ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error { c.setupOnce.Do(c.setup) - if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText { + m, err := fs.MarshalManifest(".") + if err != nil || m == coll.ManifestText { return err - } else { - coll.ManifestText = m } + coll.ManifestText = m var updated arvados.Collection defer c.pdhs.Remove(coll.UUID) - err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{ + err = client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{ "collection": map[string]string{ "manifest_text": coll.ManifestText, }, @@ -224,13 +224,12 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo }) } return collection, err - } else { - // PDH changed, but now we know we have - // permission -- and maybe we already have the - // new PDH in the cache. - if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil { - return coll, nil - } + } + // PDH changed, but now we know we have + // permission -- and maybe we already have the + // new PDH in the cache. + if coll := c.lookupCollection(arv.ApiToken + "\000" + current.PortableDataHash); coll != nil { + return coll, nil } } diff --git a/services/keep-web/doc.go b/services/keep-web/doc.go index 8682eac2dd..be81bb68c7 100644 --- a/services/keep-web/doc.go +++ b/services/keep-web/doc.go @@ -91,161 +91,7 @@ // // Download URLs // -// The following "same origin" URL patterns are supported for public -// collections and collections shared anonymously via secret links -// (i.e., collections which can be served by keep-web without making -// use of any implicit credentials like cookies). See "Same-origin -// URLs" below. -// -// http://collections.example.com/c=uuid_or_pdh/path/file.txt -// http://collections.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt -// -// The following "multiple origin" URL patterns are supported for all -// collections: -// -// http://uuid_or_pdh--collections.example.com/path/file.txt -// http://uuid_or_pdh--collections.example.com/t=TOKEN/path/file.txt -// -// In the "multiple origin" form, the string "--" can be replaced with -// "." with identical results (assuming the downstream proxy is -// configured accordingly). These two are equivalent: -// -// http://uuid_or_pdh--collections.example.com/path/file.txt -// http://uuid_or_pdh.collections.example.com/path/file.txt -// -// The first form (with "--" instead of ".") avoids the cost and -// effort of deploying a wildcard TLS certificate for -// *.collections.example.com at sites that already have a wildcard -// certificate for *.example.com. The second form is likely to be -// easier to configure, and more efficient to run, on a downstream -// proxy. -// -// In all of the above forms, the "collections.example.com" part can -// be anything at all: keep-web itself ignores everything after the -// first "." or "--". (Of course, in order for clients to connect at -// all, DNS and any relevant proxies must be configured accordingly.) -// -// In all of the above forms, the "uuid_or_pdh" part can be either a -// collection UUID or a portable data hash with the "+" character -// optionally replaced by "-". (When "uuid_or_pdh" appears in the -// domain name, replacing "+" with "-" is mandatory, because "+" is -// not a valid character in a domain name.) -// -// In all of the above forms, a top level directory called "_" is -// skipped. In cases where the "path/file.txt" part might start with -// "t=" or "c=" or "_/", links should be constructed with a leading -// "_/" to ensure the top level directory is not interpreted as a -// token or collection ID. -// -// Assuming there is a collection with UUID -// zzzzz-4zz18-znfnqtbbv4spc3w and portable data hash -// 1f4b0bc7583c2a7f9102c395f4ffc5e3+45, the following URLs are -// interchangeable: -// -// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt -// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt -// http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt -// -// The following URLs are read-only, but otherwise interchangeable -// with the above: -// -// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt -// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt -// http://collections.example.com/by_id/1f4b0bc7583c2a7f9102c395f4ffc5e3%2B45/foo/bar.txt -// http://collections.example.com/by_id/zzzzz-4zz18-znfnqtbbv4spc3w/foo/bar.txt -// -// If the collection is named "MyCollection" and located in a project -// called "MyProject" which is in the home project of a user with -// username is "bob", the following read-only URL is also available -// when authenticating as bob: -// -// http://collections.example.com/users/bob/MyProject/MyCollection/foo/bar.txt -// -// An additional form is supported specifically to make it more -// convenient to maintain support for existing Workbench download -// links: -// -// http://collections.example.com/collections/download/uuid_or_pdh/TOKEN/foo/bar.txt -// -// A regular Workbench "download" link is also accepted, but -// credentials passed via cookie, header, etc. are ignored. Only -// public data can be served this way: -// -// http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt -// -// Collections can also be accessed (read-only) via "/by_id/X" where X -// is a UUID or portable data hash. -// -// Authorization mechanisms -// -// A token can be provided in an Authorization header: -// -// Authorization: OAuth2 o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK -// -// A base64-encoded token can be provided in a cookie named "api_token": -// -// Cookie: api_token=bzA3ajRweDdSbEpLNEN1TVlwN0MwTERUNEN6UjFKMXFCRTVBdm83ZUNjVWpPVGlreEs= -// -// A token can be provided in an URL-encoded query string: -// -// GET /foo/bar.txt?api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK -// -// A suitably encoded token can be provided in a POST body if the -// request has a content type of application/x-www-form-urlencoded or -// multipart/form-data: -// -// POST /foo/bar.txt -// Content-Type: application/x-www-form-urlencoded -// [...] -// api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK -// -// If a token is provided in a query string or in a POST request, the -// response is an HTTP 303 redirect to an equivalent GET request, with -// the token stripped from the query string and added to a cookie -// instead. -// -// Indexes -// -// Keep-web returns a generic HTML index listing when a directory is -// requested with the GET method. It does not serve a default file -// like "index.html". Directory listings are also returned for WebDAV -// PROPFIND requests. -// -// Compatibility -// -// Client-provided authorization tokens are ignored if the client does -// not provide a Host header. -// -// In order to use the query string or a POST form authorization -// mechanisms, the client must follow 303 redirects; the client must -// accept cookies with a 303 response and send those cookies when -// performing the redirect; and either the client or an intervening -// proxy must resolve a relative URL ("//host/path") if given in a -// response Location header. -// -// Intranet mode -// -// Normally, Keep-web accepts requests for multiple collections using -// the same host name, provided the client's credentials are not being -// used. This provides insufficient XSS protection in an installation -// where the "anonymously accessible" data is not truly public, but -// merely protected by network topology. -// -// In such cases -- for example, a site which is not reachable from -// the internet, where some data is world-readable from Arvados's -// perspective but is intended to be available only to users within -// the local network -- the downstream proxy should configured to -// return 401 for all paths beginning with "/c=". -// -// Same-origin URLs -// -// Without the same-origin protection outlined above, a web page -// stored in collection X could execute JavaScript code that uses the -// current viewer's credentials to download additional data from -// collection Y -- data which is accessible to the current viewer, but -// not to the author of collection X -- from the same origin -// (``https://collections.example.com/'') and upload it to some other -// site chosen by the author of collection X. +// See http://doc.arvados.org/api/keep-web-urls.html // // Attachment-Only host // diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go index 915924e288..2d6fb78f80 100644 --- a/services/keep-web/handler.go +++ b/services/keep-web/handler.go @@ -62,6 +62,9 @@ func parseCollectionIDFromDNSName(s string) string { var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+") +var notFoundMessage = "404 Not found\r\n\r\nThe requested path was not found, or you do not have permission to access it.\r" +var unauthorizedMessage = "401 Unauthorized\r\n\r\nA valid Arvados token must be provided to access this resource.\r" + // parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a // PDH (even if it is a PDH with "+" replaced by " " or "-"); // otherwise "". @@ -185,10 +188,6 @@ var ( func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { h.setupOnce.Do(h.setup) - remoteAddr := r.RemoteAddr - if xff := r.Header.Get("X-Forwarded-For"); xff != "" { - remoteAddr = xff + "," + remoteAddr - } if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" { r.URL.Scheme = xfp } @@ -283,7 +282,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { } if collectionID == "" && !useSiteFS { - w.WriteHeader(http.StatusNotFound) + http.Error(w, notFoundMessage, http.StatusNotFound) return } @@ -297,27 +296,32 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { } formToken := r.FormValue("api_token") - if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" { - // The client provided an explicit token in the POST - // body. The Origin header indicates this *might* be - // an AJAX request, in which case redirect-with-cookie - // won't work: we should just serve the content in the - // POST response. This is safe because: - // - // * We're supplying an attachment, not inline - // content, so we don't need to convert the POST to - // a GET and avoid the "really resubmit form?" - // problem. + origin := r.Header.Get("Origin") + cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host) + safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead) + safeAttachment := attachment && r.URL.Query().Get("api_token") == "" + if formToken == "" { + // No token to use or redact. + } else if safeAjax || safeAttachment { + // If this is a cross-origin request, the URL won't + // appear in the browser's address bar, so + // substituting a clipboard-safe URL is pointless. + // Redirect-with-cookie wouldn't work anyway, because + // it's not safe to allow third-party use of our + // cookie. // - // * The token isn't embedded in the URL, so we don't - // need to worry about bookmarks and copy/paste. + // If we're supplying an attachment, we don't need to + // convert POST to GET to avoid the "really resubmit + // form?" problem, so provided the token isn't + // embedded in the URL, there's no reason to do + // redirect-with-cookie in this case either. reqTokens = append(reqTokens, formToken) - } else if formToken != "" && browserMethod[r.Method] { - // The client provided an explicit token in the query - // string, or a form in POST body. We must put the - // token in an HttpOnly cookie, and redirect to the - // same URL with the query param redacted and method = - // GET. + } else if browserMethod[r.Method] { + // If this is a page view, and the client provided a + // token via query string or POST body, we must put + // the token in an HttpOnly cookie, and redirect to an + // equivalent URL with the query param redacted and + // method = GET. h.seeOtherWithCookie(w, r, "", credentialsOK) return } @@ -392,14 +396,14 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { // for additional credentials would just be // confusing), or we don't even accept // credentials at this path. - w.WriteHeader(http.StatusNotFound) + http.Error(w, notFoundMessage, http.StatusNotFound) return } for _, t := range reqTokens { if tokenResult[t] == 404 { // The client provided valid token(s), but the // collection was not found. - w.WriteHeader(http.StatusNotFound) + http.Error(w, notFoundMessage, http.StatusNotFound) return } } @@ -413,7 +417,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { // data that has been deleted. Allow a referrer to // provide this context somehow? w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"") - w.WriteHeader(http.StatusUnauthorized) + http.Error(w, unauthorizedMessage, http.StatusUnauthorized) return } @@ -483,7 +487,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { openPath := "/" + strings.Join(targetPath, "/") if f, err := fs.Open(openPath); os.IsNotExist(err) { // Requested non-existent path - w.WriteHeader(http.StatusNotFound) + http.Error(w, notFoundMessage, http.StatusNotFound) } else if err != nil { // Some other (unexpected) error http.Error(w, "open: "+err.Error(), http.StatusInternalServerError) @@ -537,7 +541,7 @@ func (h *handler) getClients(reqID, token string) (arv *arvadosclient.ArvadosCli func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) { if len(tokens) == 0 { w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"") - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + http.Error(w, unauthorizedMessage, http.StatusUnauthorized) return } if writeMethod[r.Method] { @@ -769,6 +773,7 @@ func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, loc Value: auth.EncodeTokenCookie([]byte(formToken)), Path: "/", HttpOnly: true, + SameSite: http.SameSiteLaxMode, }) } diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go index f6f3de8877..5291efeb82 100644 --- a/services/keep-web/handler_test.go +++ b/services/keep-web/handler_test.go @@ -122,7 +122,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) { } s.testServer.Handler.ServeHTTP(resp, req) c.Check(resp.Code, check.Equals, http.StatusNotFound) - c.Check(resp.Body.String(), check.Equals, "") + c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n") } } @@ -250,7 +250,11 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori // depending on the authz method. c.Check(code, check.Equals, failCode) } - c.Check(body, check.Equals, "") + if code == 404 { + c.Check(body, check.Equals, notFoundMessage+"\n") + } else { + c.Check(body, check.Equals, unauthorizedMessage+"\n") + } } } } @@ -307,7 +311,7 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) { "", "", http.StatusNotFound, - "", + notFoundMessage+"\n", ) } @@ -321,7 +325,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) "", "", http.StatusUnauthorized, - "", + unauthorizedMessage+"\n", ) } @@ -439,7 +443,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) "application/x-www-form-urlencoded", url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(), http.StatusNotFound, - "", + notFoundMessage+"\n", ) } @@ -463,7 +467,7 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) { "", "", http.StatusNotFound, - "", + notFoundMessage+"\n", ) } @@ -579,6 +583,25 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) { c.Check(resp.Code, check.Equals, http.StatusOK) c.Check(resp.Body.String(), check.Equals, "foo") c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") + + // GET + Origin header is representative of both AJAX GET + // requests and inline images via . + u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2) + req = &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Origin": {"https://origin.example"}, + }, + } + resp = httptest.NewRecorder() + s.testServer.Handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + c.Check(resp.Body.String(), check.Equals, "foo") + c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*") } func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder { diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go index 12e294d933..1c84976d2b 100644 --- a/services/keep-web/s3.go +++ b/services/keep-web/s3.go @@ -5,23 +5,221 @@ package main import ( + "crypto/hmac" + "crypto/sha256" "encoding/xml" "errors" "fmt" + "hash" "io" "net/http" + "net/url" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" + "time" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/AdRoll/goamz/s3" ) -const s3MaxKeys = 1000 +const ( + s3MaxKeys = 1000 + s3SignAlgorithm = "AWS4-HMAC-SHA256" + s3MaxClockSkew = 5 * time.Minute +) + +func hmacstring(msg string, key []byte) []byte { + h := hmac.New(sha256.New, key) + io.WriteString(h, msg) + return h.Sum(nil) +} + +func hashdigest(h hash.Hash, payload string) string { + io.WriteString(h, payload) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// Signing key for given secret key and request attrs. +func s3signatureKey(key, datestamp, regionName, serviceName string) []byte { + return hmacstring("aws4_request", + hmacstring(serviceName, + hmacstring(regionName, + hmacstring(datestamp, []byte("AWS4"+key))))) +} + +// Canonical query string for S3 V4 signature: sorted keys, spaces +// escaped as %20 instead of +, keyvalues joined with &. +func s3querystring(u *url.URL) string { + keys := make([]string, 0, len(u.Query())) + values := make(map[string]string, len(u.Query())) + for k, vs := range u.Query() { + k = strings.Replace(url.QueryEscape(k), "+", "%20", -1) + keys = append(keys, k) + for _, v := range vs { + v = strings.Replace(url.QueryEscape(v), "+", "%20", -1) + if values[k] != "" { + values[k] += "&" + } + values[k] += k + "=" + v + } + } + sort.Strings(keys) + for i, k := range keys { + keys[i] = values[k] + } + return strings.Join(keys, "&") +} + +var reMultipleSlashChars = regexp.MustCompile(`//+`) + +func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) { + timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date") + if timestr == "" { + timefmt, timestr = time.RFC1123, r.Header.Get("Date") + } + t, err := time.Parse(timefmt, timestr) + if err != nil { + return "", fmt.Errorf("invalid timestamp %q: %s", timestr, err) + } + if skew := time.Now().Sub(t); skew < -s3MaxClockSkew || skew > s3MaxClockSkew { + return "", errors.New("exceeded max clock skew") + } + + var canonicalHeaders string + for _, h := range strings.Split(signedHeaders, ";") { + if h == "host" { + canonicalHeaders += h + ":" + r.Host + "\n" + } else { + canonicalHeaders += h + ":" + r.Header.Get(h) + "\n" + } + } + + normalizedURL := *r.URL + normalizedURL.RawPath = "" + normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/") + canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256")) + ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest) + return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil +} + +func s3signature(secretKey, scope, signedHeaders, stringToSign string) (string, error) { + // scope is {datestamp}/{region}/{service}/aws4_request + drs := strings.Split(scope, "/") + if len(drs) != 4 { + return "", fmt.Errorf("invalid scope %q", scope) + } + key := s3signatureKey(secretKey, drs[0], drs[1], drs[2]) + return hashdigest(hmac.New(sha256.New, key), stringToSign), nil +} + +var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`) + +func unescapeKey(key string) string { + if v2tokenUnderscore.MatchString(key) { + // Entire Arvados token, with "/" replaced by "_" to + // avoid colliding with the Authorization header + // format. + return strings.Replace(key, "_", "/", -1) + } else if s, err := url.PathUnescape(key); err == nil { + return s + } else { + return key + } +} + +// checks3signature verifies the given S3 V4 signature and returns the +// Arvados token that corresponds to the given accessKey. An error is +// returned if accessKey is not a valid token UUID or the signature +// does not match. +func (h *handler) checks3signature(r *http.Request) (string, error) { + var key, scope, signedHeaders, signature string + authstring := strings.TrimPrefix(r.Header.Get("Authorization"), s3SignAlgorithm+" ") + for _, cmpt := range strings.Split(authstring, ",") { + cmpt = strings.TrimSpace(cmpt) + split := strings.SplitN(cmpt, "=", 2) + switch { + case len(split) != 2: + // (?) ignore + case split[0] == "Credential": + keyandscope := strings.SplitN(split[1], "/", 2) + if len(keyandscope) == 2 { + key, scope = keyandscope[0], keyandscope[1] + } + case split[0] == "SignedHeaders": + signedHeaders = split[1] + case split[0] == "Signature": + signature = split[1] + } + } + + client := (&arvados.Client{ + APIHost: h.Config.cluster.Services.Controller.ExternalURL.Host, + Insecure: h.Config.cluster.TLS.Insecure, + }).WithRequestID(r.Header.Get("X-Request-Id")) + var aca arvados.APIClientAuthorization + var secret string + var err error + if len(key) == 27 && key[5:12] == "-gj3su-" { + // Access key is the UUID of an Arvados token, secret + // key is the secret part. + ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Config.cluster.SystemRootToken) + err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/"+key, nil, nil) + secret = aca.APIToken + } else { + // Access key and secret key are both an entire + // Arvados token or OIDC access token. + ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key)) + err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil) + secret = key + } + if err != nil { + ctxlog.FromContext(r.Context()).WithError(err).WithField("UUID", key).Info("token lookup failed") + return "", errors.New("invalid access key") + } + stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, r) + if err != nil { + return "", err + } + expect, err := s3signature(secret, scope, signedHeaders, stringToSign) + if err != nil { + return "", err + } else if expect != signature { + return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign) + } + return aca.TokenV2(), nil +} + +func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resource string, code int) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + var errstruct struct { + Code string + Message string + Resource string + RequestId string + } + errstruct.Code = s3code + errstruct.Message = message + errstruct.Resource = resource + errstruct.RequestId = "" + enc := xml.NewEncoder(w) + fmt.Fprint(w, xml.Header) + enc.EncodeElement(errstruct, xml.StartElement{Name: xml.Name{Local: "Error"}}) +} + +var NoSuchKey = "NoSuchKey" +var NoSuchBucket = "NoSuchBucket" +var InvalidArgument = "InvalidArgument" +var InternalError = "InternalError" +var UnauthorizedAccess = "UnauthorizedAccess" +var InvalidRequest = "InvalidRequest" +var SignatureDoesNotMatch = "SignatureDoesNotMatch" // serveS3 handles r and returns true if r is a request from an S3 // client, otherwise it returns false. @@ -30,34 +228,24 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") { split := strings.SplitN(auth[4:], ":", 2) if len(split) < 2 { - w.WriteHeader(http.StatusUnauthorized) + s3ErrorResponse(w, InvalidRequest, "malformed Authorization header", r.URL.Path, http.StatusUnauthorized) return true } - token = split[0] - } else if strings.HasPrefix(auth, "AWS4-HMAC-SHA256 ") { - for _, cmpt := range strings.Split(auth[17:], ",") { - cmpt = strings.TrimSpace(cmpt) - split := strings.SplitN(cmpt, "=", 2) - if len(split) == 2 && split[0] == "Credential" { - keyandscope := strings.Split(split[1], "/") - if len(keyandscope[0]) > 0 { - token = keyandscope[0] - break - } - } - } - if token == "" { - w.WriteHeader(http.StatusBadRequest) - fmt.Println(w, "invalid V4 signature") + token = unescapeKey(split[0]) + } else if strings.HasPrefix(auth, s3SignAlgorithm+" ") { + t, err := h.checks3signature(r) + if err != nil { + s3ErrorResponse(w, SignatureDoesNotMatch, "signature verification failed: "+err.Error(), r.URL.Path, http.StatusForbidden) return true } + token = t } else { return false } _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token) if err != nil { - http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError) + s3ErrorResponse(w, InternalError, "Pool failed: "+h.clientPool.Err().Error(), r.URL.Path, http.StatusInternalServerError) return true } defer release() @@ -65,7 +253,18 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { fs := client.SiteFileSystem(kc) fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution) - objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1 + var objectNameGiven bool + var bucketName string + fspath := "/by_id" + if id := parseCollectionIDFromDNSName(r.Host); id != "" { + fspath += "/" + id + bucketName = id + objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0 + } else { + bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0] + objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1 + } + fspath += reMultipleSlashChars.ReplaceAllString(r.URL.Path, "/") switch { case r.Method == http.MethodGet && !objectNameGiven: @@ -77,20 +276,19 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { fmt.Fprintln(w, ``) } else { // ListObjects - h.s3list(w, r, fs) + h.s3list(bucketName, w, r, fs) } return true case r.Method == http.MethodGet || r.Method == http.MethodHead: - fspath := "/by_id" + r.URL.Path fi, err := fs.Stat(fspath) if r.Method == "HEAD" && !objectNameGiven { // HeadBucket if err == nil && fi.IsDir() { w.WriteHeader(http.StatusOK) } else if os.IsNotExist(err) { - w.WriteHeader(http.StatusNotFound) + s3ErrorResponse(w, NoSuchBucket, "The specified bucket does not exist.", r.URL.Path, http.StatusNotFound) } else { - http.Error(w, err.Error(), http.StatusBadGateway) + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway) } return true } @@ -102,7 +300,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { if os.IsNotExist(err) || (err != nil && err.Error() == "not a directory") || (fi != nil && fi.IsDir()) { - http.Error(w, "not found", http.StatusNotFound) + s3ErrorResponse(w, NoSuchKey, "The specified key does not exist.", r.URL.Path, http.StatusNotFound) return true } // shallow copy r, and change URL path @@ -112,25 +310,24 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { return true case r.Method == http.MethodPut: if !objectNameGiven { - http.Error(w, "missing object name in PUT request", http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest) return true } - fspath := "by_id" + r.URL.Path var objectIsDir bool if strings.HasSuffix(fspath, "/") { if !h.Config.cluster.Collections.S3FolderObjects { - http.Error(w, "invalid object name: trailing slash", http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest) return true } n, err := r.Body.Read(make([]byte, 1)) if err != nil && err != io.EOF { - http.Error(w, fmt.Sprintf("error reading request body: %s", err), http.StatusInternalServerError) + s3ErrorResponse(w, InternalError, fmt.Sprintf("error reading request body: %s", err), r.URL.Path, http.StatusInternalServerError) return true } else if n > 0 { - http.Error(w, "cannot create object with trailing '/' char unless content is empty", http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless content is empty", r.URL.Path, http.StatusBadRequest) return true } else if strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] != "application/x-directory" { - http.Error(w, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", r.URL.Path, http.StatusBadRequest) return true } // Given PUT "foo/bar/", we'll use "foo/bar/." @@ -142,12 +339,12 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { fi, err := fs.Stat(fspath) if err != nil && err.Error() == "not a directory" { // requested foo/bar, but foo is a file - http.Error(w, "object name conflicts with existing object", http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest) return true } if strings.HasSuffix(r.URL.Path, "/") && err == nil && !fi.IsDir() { // requested foo/bar/, but foo/bar is a file - http.Error(w, "object name conflicts with existing object", http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest) return true } // create missing parent/intermediate directories, if any @@ -156,7 +353,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { dir := fspath[:i] if strings.HasSuffix(dir, "/") { err = errors.New("invalid object name (consecutive '/' chars)") - http.Error(w, err.Error(), http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest) return true } err = fs.Mkdir(dir, 0755) @@ -164,11 +361,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { // Cannot create a directory // here. err = fmt.Errorf("mkdir %q failed: %w", dir, err) - http.Error(w, err.Error(), http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest) return true } else if err != nil && !os.IsExist(err) { err = fmt.Errorf("mkdir %q failed: %w", dir, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) return true } } @@ -180,33 +377,81 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } if err != nil { err = fmt.Errorf("open %q failed: %w", r.URL.Path, err) - http.Error(w, err.Error(), http.StatusBadRequest) + s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest) return true } defer f.Close() _, err = io.Copy(f, r.Body) if err != nil { err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err) - http.Error(w, err.Error(), http.StatusBadGateway) + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway) return true } err = f.Close() if err != nil { err = fmt.Errorf("write to %q failed: close: %w", r.URL.Path, err) - http.Error(w, err.Error(), http.StatusBadGateway) + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway) return true } } err = fs.Sync() if err != nil { err = fmt.Errorf("sync failed: %w", err) - http.Error(w, err.Error(), http.StatusInternalServerError) + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) return true } w.WriteHeader(http.StatusOK) return true + case r.Method == http.MethodDelete: + if !objectNameGiven || r.URL.Path == "/" { + s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest) + return true + } + if strings.HasSuffix(fspath, "/") { + fspath = strings.TrimSuffix(fspath, "/") + fi, err := fs.Stat(fspath) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNoContent) + return true + } else if err != nil { + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) + return true + } else if !fi.IsDir() { + // if "foo" exists and is a file, then + // "foo/" doesn't exist, so we say + // delete was successful. + w.WriteHeader(http.StatusNoContent) + return true + } + } else if fi, err := fs.Stat(fspath); err == nil && fi.IsDir() { + // if "foo" is a dir, it is visible via S3 + // only as "foo/", not "foo" -- so we leave + // the dir alone and return 204 to indicate + // that "foo" does not exist. + w.WriteHeader(http.StatusNoContent) + return true + } + err = fs.Remove(fspath) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNoContent) + return true + } + if err != nil { + err = fmt.Errorf("rm failed: %w", err) + s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest) + return true + } + err = fs.Sync() + if err != nil { + err = fmt.Errorf("sync failed: %w", err) + s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError) + return true + } + w.WriteHeader(http.StatusNoContent) + return true default: - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + s3ErrorResponse(w, InvalidRequest, "method not allowed", r.URL.Path, http.StatusMethodNotAllowed) + return true } } @@ -267,15 +512,13 @@ func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path var errDone = errors.New("done") -func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) { +func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) { var params struct { - bucket string delimiter string marker string maxKeys int prefix string } - params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0] params.delimiter = r.FormValue("delimiter") params.marker = r.FormValue("marker") if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys { @@ -285,7 +528,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust } params.prefix = r.FormValue("prefix") - bucketdir := "by_id/" + params.bucket + bucketdir := "by_id/" + bucket // walkpath is the directory (relative to bucketdir) we need // to walk: the innermost directory that is guaranteed to // contain all paths that have the requested prefix. Examples: @@ -300,12 +543,32 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust walkpath = "" } - resp := s3.ListResp{ - Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0], - Prefix: params.prefix, - Delimiter: params.delimiter, - Marker: params.marker, - MaxKeys: params.maxKeys, + type commonPrefix struct { + Prefix string + } + type listResp struct { + XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` + s3.ListResp + // s3.ListResp marshals an empty tag when + // CommonPrefixes is nil, which confuses some clients. + // Fix by using this nested struct instead. + CommonPrefixes []commonPrefix + // Similarly, we need omitempty here, because an empty + // tag confuses some clients (e.g., + // github.com/aws/aws-sdk-net never terminates its + // paging loop). + NextMarker string `xml:"NextMarker,omitempty"` + // ListObjectsV2 has a KeyCount response field. + KeyCount int + } + resp := listResp{ + ListResp: s3.ListResp{ + Name: bucket, + Prefix: params.prefix, + Delimiter: params.delimiter, + Marker: params.marker, + MaxKeys: params.maxKeys, + }, } commonPrefixes := map[string]bool{} err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error { @@ -387,18 +650,16 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust return } if params.delimiter != "" { + resp.CommonPrefixes = make([]commonPrefix, 0, len(commonPrefixes)) for prefix := range commonPrefixes { - resp.CommonPrefixes = append(resp.CommonPrefixes, prefix) - sort.Strings(resp.CommonPrefixes) + resp.CommonPrefixes = append(resp.CommonPrefixes, commonPrefix{prefix}) } + sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix }) } - wrappedResp := struct { - XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` - s3.ListResp - }{"", resp} + resp.KeyCount = len(resp.Contents) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, xml.Header) - if err := xml.NewEncoder(w).Encode(wrappedResp); err != nil { + if err := xml.NewEncoder(w).Encode(resp); err != nil { ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response") } } diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go index 73553ff4d3..52ef795097 100644 --- a/services/keep-web/s3_test.go +++ b/services/keep-web/s3_test.go @@ -7,10 +7,14 @@ package main import ( "bytes" "crypto/rand" + "crypto/sha256" "fmt" "io/ioutil" "net/http" + "net/http/httptest" + "net/url" "os" + "os/exec" "strings" "sync" "time" @@ -70,12 +74,13 @@ func (s *IntegrationSuite) s3setup(c *check.C) s3stage { err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil) c.Assert(err, check.IsNil) - auth := aws.NewAuth(arvadostest.ActiveTokenV2, arvadostest.ActiveTokenV2, "", time.Now().Add(time.Hour)) + auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour)) region := aws.Region{ Name: s.testServer.Addr, S3Endpoint: "http://" + s.testServer.Addr, } client := s3.New(*auth, region) + client.Signature = aws.V4Signature return s3stage{ arv: arv, ac: ac, @@ -104,6 +109,44 @@ func (stage s3stage) teardown(c *check.C) { } } +func (s *IntegrationSuite) TestS3Signatures(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + bucket := stage.collbucket + for _, trial := range []struct { + success bool + signature int + accesskey string + secretkey string + }{ + {true, aws.V2Signature, arvadostest.ActiveToken, "none"}, + {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"}, + {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"}, + {false, aws.V2Signature, "none", "none"}, + {false, aws.V2Signature, "none", arvadostest.ActiveToken}, + + {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken}, + {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken}, + {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)}, + {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)}, + {false, aws.V4Signature, arvadostest.ActiveToken, ""}, + {false, aws.V4Signature, arvadostest.ActiveToken, "none"}, + {false, aws.V4Signature, "none", arvadostest.ActiveToken}, + {false, aws.V4Signature, "none", "none"}, + } { + c.Logf("%#v", trial) + bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour))) + bucket.S3.Signature = trial.signature + _, err := bucket.GetReader("emptyfile") + if trial.success { + c.Check(err, check.IsNil) + } else { + c.Check(err, check.NotNil) + } + } +} + func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) @@ -137,7 +180,9 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix // GetObject rdr, err = bucket.GetReader(prefix + "missingfile") - c.Check(err, check.ErrorMatches, `404 Not Found`) + c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) + c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) + c.Check(err, check.ErrorMatches, `The specified key does not exist.`) // HeadObject exists, err := bucket.Exists(prefix + "missingfile") @@ -154,7 +199,13 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix c.Check(err, check.IsNil) // HeadObject - exists, err = bucket.Exists(prefix + "sailboat.txt") + resp, err := bucket.Head(prefix+"sailboat.txt", nil) + c.Check(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + c.Check(resp.ContentLength, check.Equals, int64(4)) + + // HeadObject with superfluous leading slashes + exists, err = bucket.Exists(prefix + "//sailboat.txt") c.Check(err, check.IsNil) c.Check(exists, check.Equals, true) } @@ -183,6 +234,18 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, path: "newdir/newfile", size: 1 << 26, contentType: "application/octet-stream", + }, { + path: "/aaa", + size: 2, + contentType: "application/octet-stream", + }, { + path: "//bbb", + size: 2, + contentType: "application/octet-stream", + }, { + path: "ccc//", + size: 0, + contentType: "application/x-directory", }, { path: "newdir1/newdir2/newfile", size: 0, @@ -198,7 +261,14 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket, objname := prefix + trial.path _, err := bucket.GetReader(objname) - c.Assert(err, check.ErrorMatches, `404 Not Found`) + if !c.Check(err, check.NotNil) { + continue + } + c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) + c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) + if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) { + continue + } buf := make([]byte, trial.size) rand.Read(buf) @@ -247,16 +317,59 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) { c.Logf("=== %v", trial) _, err := bucket.GetReader(trial.path) - c.Assert(err, check.ErrorMatches, `404 Not Found`) + c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) + c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) + c.Assert(err, check.ErrorMatches, `The specified key does not exist.`) buf := make([]byte, trial.size) rand.Read(buf) err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{}) - c.Check(err, check.ErrorMatches, `400 Bad Request`) + c.Check(err.(*s3.Error).StatusCode, check.Equals, 400) + c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`) + c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`) _, err = bucket.GetReader(trial.path) - c.Assert(err, check.ErrorMatches, `404 Not Found`) + c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) + c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) + c.Assert(err, check.ErrorMatches, `The specified key does not exist.`) + } +} + +func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + s.testS3DeleteObject(c, stage.collbucket, "") +} +func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/") +} +func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) { + s.testServer.Config.cluster.Collections.S3FolderObjects = true + for _, trial := range []struct { + path string + }{ + {"/"}, + {"nonexistentfile"}, + {"emptyfile"}, + {"sailboat.txt"}, + {"sailboat.txt/"}, + {"emptydir"}, + {"emptydir/"}, + } { + objname := prefix + trial.path + comment := check.Commentf("objname %q", objname) + + err := bucket.Del(objname) + if trial.path == "/" { + c.Check(err, check.NotNil) + continue + } + c.Check(err, check.IsNil, comment) + _, err = bucket.GetReader(objname) + c.Check(err, check.NotNil, comment) } } @@ -272,6 +385,7 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) { } func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) { s.testServer.Config.cluster.Collections.S3FolderObjects = false + var wg sync.WaitGroup for _, trial := range []struct { path string @@ -294,8 +408,6 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, path: "/", }, { path: "//", - }, { - path: "foo//bar", }, { path: "", }, @@ -312,13 +424,15 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, rand.Read(buf) err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{}) - if !c.Check(err, check.ErrorMatches, `400 Bad.*`, check.Commentf("PUT %q should fail", objname)) { + if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) { return } if objname != "" && objname != "/" { _, err = bucket.GetReader(objname) - c.Check(err, check.ErrorMatches, `404 Not Found`, check.Commentf("GET %q should return 404", objname)) + c.Check(err.(*s3.Error).StatusCode, check.Equals, 404) + c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`) + c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname)) } }() } @@ -340,11 +454,136 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) { c.Assert(fs.Sync(), check.IsNil) } +func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) { + scope := "20200202/region/service/aws4_request" + signedHeaders := "date" + req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123)) + stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req) + c.Assert(err, check.IsNil) + sig, err := s3signature(secret, scope, signedHeaders, stringToSign) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig) +} + +func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + for _, trial := range []struct { + url string + method string + body string + responseCode int + responseRegexp []string + }{ + { + url: "https://" + stage.collbucket.Name + ".example.com/", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`(?ms).*sailboat\.txt.*`}, + }, + { + url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`(?ms).*sailboat\.txt.*`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`(?ms).*sailboat\.txt.*`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`⛵\n`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep", + method: "PUT", + body: "boop", + responseCode: http.StatusOK, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`boop`}, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop", + method: "GET", + responseCode: http.StatusNotFound, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop", + method: "PUT", + body: "boop", + responseCode: http.StatusOK, + }, + { + url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop", + method: "GET", + responseCode: http.StatusOK, + responseRegexp: []string{`boop`}, + }, + } { + url, err := url.Parse(trial.url) + c.Assert(err, check.IsNil) + req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body))) + c.Assert(err, check.IsNil) + s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken) + rr := httptest.NewRecorder() + s.testServer.Server.Handler.ServeHTTP(rr, req) + resp := rr.Result() + c.Check(resp.StatusCode, check.Equals, trial.responseCode) + body, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + for _, re := range trial.responseRegexp { + c.Check(string(body), check.Matches, re) + } + } +} + +func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + for _, trial := range []struct { + rawPath string + normalizedPath string + }{ + {"/foo", "/foo"}, // boring case + {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped + {"/foo%2fbar", "/foo/bar"}, // / must not be escaped + {"/(foo)", "/%28foo%29"}, // () must be escaped + {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase + } { + date := time.Now().UTC().Format("20060102T150405Z") + scope := "20200202/fakeregion/S3/aws4_request" + canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "") + c.Logf("canonicalRequest %q", canonicalRequest) + expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest)) + c.Logf("expected stringToSign %q", expect) + + req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil) + req.Header.Set("X-Amz-Date", date) + req.Host = "host.example.com" + + obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req) + if !c.Check(err, check.IsNil) { + continue + } + c.Check(obtained, check.Equals, expect) + } +} + func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} { req, err := http.NewRequest("GET", bucket.URL("/"), nil) + c.Check(err, check.IsNil) req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") req.URL.RawQuery = "versioning" resp, err := http.DefaultClient.Do(req) @@ -356,6 +595,59 @@ func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) { } } +// If there are no CommonPrefixes entries, the CommonPrefixes XML tag +// should not appear at all. +func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/" + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`) +} + +// If there is no delimiter in the request, or the results are not +// truncated, the NextMarker XML tag should not appear in the response +// body. +func (s *IntegrationSuite) TestS3ListNoNextMarker(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + for _, query := range []string{"prefix=e&delimiter=/", ""} { + req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = query + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Not(check.Matches), `(?ms).*NextMarker.*`) + } +} + +// List response should include KeyCount field. +func (s *IntegrationSuite) TestS3ListKeyCount(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil) + c.Assert(err, check.IsNil) + req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") + req.URL.RawQuery = "prefix=&delimiter=/" + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + buf, err := ioutil.ReadAll(resp.Body) + c.Assert(err, check.IsNil) + c.Check(string(buf), check.Matches, `(?ms).*2.*`) +} + func (s *IntegrationSuite) TestS3CollectionList(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) @@ -544,3 +836,31 @@ func (s *IntegrationSuite) testS3CollectionListRollup(c *check.C) { c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker) } } + +// TestS3cmd checks compatibility with the s3cmd command line tool, if +// it's installed. As of Debian buster, s3cmd is only in backports, so +// `arvados-server install` don't install it, and this test skips if +// it's not installed. +func (s *IntegrationSuite) TestS3cmd(c *check.C) { + if _, err := exec.LookPath("s3cmd"); err != nil { + c.Skip("s3cmd not found") + return + } + + stage := s.s3setup(c) + defer stage.teardown(c) + + cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection) + buf, err := cmd.CombinedOutput() + c.Check(err, check.IsNil) + c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`) +} + +func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt") + c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`) + c.Check(body, check.Equals, "⛵\n") +} diff --git a/services/keep-web/s3aws_test.go b/services/keep-web/s3aws_test.go new file mode 100644 index 0000000000..d528dbaf79 --- /dev/null +++ b/services/keep-web/s3aws_test.go @@ -0,0 +1,77 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package main + +import ( + "bytes" + "context" + "io/ioutil" + + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/defaults" + "github.com/aws/aws-sdk-go-v2/aws/ec2metadata" + "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds" + "github.com/aws/aws-sdk-go-v2/aws/endpoints" + "github.com/aws/aws-sdk-go-v2/service/s3" + check "gopkg.in/check.v1" +) + +func (s *IntegrationSuite) TestS3AWSSDK(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + + cfg := defaults.Config() + cfg.Credentials = aws.NewChainProvider([]aws.CredentialsProvider{ + aws.NewStaticCredentialsProvider(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, ""), + ec2rolecreds.New(ec2metadata.New(cfg)), + }) + cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + if service == "s3" { + return aws.Endpoint{ + URL: "http://" + s.testServer.Addr, + SigningRegion: "custom-signing-region", + }, nil + } + return endpoints.NewDefaultResolver().ResolveEndpoint(service, region) + }) + client := s3.New(cfg) + client.ForcePathStyle = true + listreq := client.ListObjectsV2Request(&s3.ListObjectsV2Input{ + Bucket: aws.String(arvadostest.FooCollection), + MaxKeys: aws.Int64(100), + Prefix: aws.String(""), + ContinuationToken: nil, + }) + resp, err := listreq.Send(context.Background()) + c.Assert(err, check.IsNil) + c.Check(resp.Contents, check.HasLen, 1) + for _, key := range resp.Contents { + c.Check(*key.Key, check.Equals, "foo") + } + + p := make([]byte, 100000000) + for i := range p { + p[i] = byte('a') + } + putreq := client.PutObjectRequest(&s3.PutObjectInput{ + Body: bytes.NewReader(p), + Bucket: aws.String(stage.collbucket.Name), + ContentType: aws.String("application/octet-stream"), + Key: aws.String("aaaa"), + }) + _, err = putreq.Send(context.Background()) + c.Assert(err, check.IsNil) + + getreq := client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(stage.collbucket.Name), + Key: aws.String("aaaa"), + }) + getresp, err := getreq.Send(context.Background()) + c.Assert(err, check.IsNil) + getdata, err := ioutil.ReadAll(getresp.Body) + c.Assert(err, check.IsNil) + c.Check(bytes.Equal(getdata, p), check.Equals, true) +} diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go index c37852a128..0a1c7d1b3a 100644 --- a/services/keep-web/server_test.go +++ b/services/keep-web/server_test.go @@ -43,17 +43,17 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) { } { hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo") c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`) - c.Check(body, check.Equals, "") + c.Check(body, check.Equals, notFoundMessage+"\n") if token != "" { hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo") c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`) - c.Check(body, check.Equals, "") + c.Check(body, check.Equals, notFoundMessage+"\n") } hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route") c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`) - c.Check(body, check.Equals, "") + c.Check(body, check.Equals, notFoundMessage+"\n") } } @@ -86,7 +86,7 @@ func (s *IntegrationSuite) Test404(c *check.C) { hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri) c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*") if len(body) > 0 { - c.Check(body, check.Equals, "404 page not found\n") + c.Check(body, check.Equals, notFoundMessage+"\n") } } } @@ -257,12 +257,16 @@ func (s *IntegrationSuite) Test200(c *check.C) { } // Return header block and body. -func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) { +func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) { curlArgs := []string{"--silent", "--show-error", "--include"} testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr) curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost) - if token != "" { - curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token) + if strings.Contains(auth, " ") { + // caller supplied entire Authorization header value + curlArgs = append(curlArgs, "-H", "Authorization: "+auth) + } else if auth != "" { + // caller supplied Arvados token + curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth) } curlArgs = append(curlArgs, args...) curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri) @@ -440,6 +444,7 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) { cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{} cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{} cfg.cluster.ManagementToken = arvadostest.ManagementToken + cfg.cluster.SystemRootToken = arvadostest.SystemRootToken cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testServer = &server{Config: cfg} err = s.testServer.Start(ctxlog.TestLogger(c)) diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go index 0191e5ba45..538a061227 100644 --- a/services/keepproxy/keepproxy.go +++ b/services/keepproxy/keepproxy.go @@ -167,43 +167,48 @@ func run(logger log.FieldLogger, cluster *arvados.Cluster) error { return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router))) } -type ApiTokenCache struct { +type APITokenCache struct { tokens map[string]int64 lock sync.Mutex expireTime int64 } -// Cache the token and set an expire time. If we already have an expire time -// on the token, it is not updated. -func (this *ApiTokenCache) RememberToken(token string) { - this.lock.Lock() - defer this.lock.Unlock() +// RememberToken caches the token and set an expire time. If we already have +// an expire time on the token, it is not updated. +func (cache *APITokenCache) RememberToken(token string) { + cache.lock.Lock() + defer cache.lock.Unlock() now := time.Now().Unix() - if this.tokens[token] == 0 { - this.tokens[token] = now + this.expireTime + if cache.tokens[token] == 0 { + cache.tokens[token] = now + cache.expireTime } } -// Check if the cached token is known and still believed to be valid. -func (this *ApiTokenCache) RecallToken(token string) bool { - this.lock.Lock() - defer this.lock.Unlock() +// RecallToken checks if the cached token is known and still believed to be +// valid. +func (cache *APITokenCache) RecallToken(token string) bool { + cache.lock.Lock() + defer cache.lock.Unlock() now := time.Now().Unix() - if this.tokens[token] == 0 { + if cache.tokens[token] == 0 { // Unknown token return false - } else if now < this.tokens[token] { + } else if now < cache.tokens[token] { // Token is known and still valid return true } else { // Token is expired - this.tokens[token] = 0 + cache.tokens[token] = 0 return false } } +// GetRemoteAddress returns a string with the remote address for the request. +// If the X-Forwarded-For header is set and has a non-zero length, it returns a +// string made from a comma separated list of all the remote addresses, +// starting with the one(s) from the X-Forwarded-For header. func GetRemoteAddress(req *http.Request) string { if xff := req.Header.Get("X-Forwarded-For"); xff != "" { return xff + "," + req.RemoteAddr @@ -211,7 +216,7 @@ func GetRemoteAddress(req *http.Request) string { return req.RemoteAddr } -func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) { +func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *APITokenCache, req *http.Request) (pass bool, tok string) { parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2) if len(parts) < 2 || !(parts[0] == "OAuth2" || parts[0] == "Bearer") || len(parts[1]) == 0 { return false, "" @@ -265,7 +270,7 @@ var defaultTransport = *(http.DefaultTransport.(*http.Transport)) type proxyHandler struct { http.Handler *keepclient.KeepClient - *ApiTokenCache + *APITokenCache timeout time.Duration transport *http.Transport } @@ -289,7 +294,7 @@ func MakeRESTRouter(kc *keepclient.KeepClient, timeout time.Duration, mgmtToken KeepClient: kc, timeout: timeout, transport: &transport, - ApiTokenCache: &ApiTokenCache{ + APITokenCache: &APITokenCache{ tokens: make(map[string]int64), expireTime: 300, }, @@ -349,9 +354,9 @@ func (h *proxyHandler) Options(resp http.ResponseWriter, req *http.Request) { SetCorsHeaders(resp) } -var BadAuthorizationHeader = errors.New("Missing or invalid Authorization header") -var ContentLengthMismatch = errors.New("Actual length != expected content length") -var MethodNotSupported = errors.New("Method not supported") +var errBadAuthorizationHeader = errors.New("Missing or invalid Authorization header") +var errContentLengthMismatch = errors.New("Actual length != expected content length") +var errMethodNotSupported = errors.New("Method not supported") var removeHint, _ = regexp.Compile("\\+K@[a-z0-9]{5}(\\+|$)") @@ -379,8 +384,8 @@ func (h *proxyHandler) Get(resp http.ResponseWriter, req *http.Request) { var pass bool var tok string - if pass, tok = CheckAuthorizationHeader(kc, h.ApiTokenCache, req); !pass { - status, err = http.StatusForbidden, BadAuthorizationHeader + if pass, tok = CheckAuthorizationHeader(kc, h.APITokenCache, req); !pass { + status, err = http.StatusForbidden, errBadAuthorizationHeader return } @@ -402,7 +407,7 @@ func (h *proxyHandler) Get(resp http.ResponseWriter, req *http.Request) { defer reader.Close() } default: - status, err = http.StatusNotImplemented, MethodNotSupported + status, err = http.StatusNotImplemented, errMethodNotSupported return } @@ -420,7 +425,7 @@ func (h *proxyHandler) Get(resp http.ResponseWriter, req *http.Request) { case "GET": responseLength, err = io.Copy(resp, reader) if err == nil && expectLength > -1 && responseLength != expectLength { - err = ContentLengthMismatch + err = errContentLengthMismatch } } case keepclient.Error: @@ -436,8 +441,8 @@ func (h *proxyHandler) Get(resp http.ResponseWriter, req *http.Request) { } } -var LengthRequiredError = errors.New(http.StatusText(http.StatusLengthRequired)) -var LengthMismatchError = errors.New("Locator size hint does not match Content-Length header") +var errLengthRequired = errors.New(http.StatusText(http.StatusLengthRequired)) +var errLengthMismatch = errors.New("Locator size hint does not match Content-Length header") func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) { if err := h.checkLoop(resp, req); err != nil { @@ -474,7 +479,7 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) { _, err = fmt.Sscanf(req.Header.Get("Content-Length"), "%d", &expectLength) if err != nil || expectLength < 0 { - err = LengthRequiredError + err = errLengthRequired status = http.StatusLengthRequired return } @@ -485,7 +490,7 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) { status = http.StatusBadRequest return } else if loc.Size > 0 && int64(loc.Size) != expectLength { - err = LengthMismatchError + err = errLengthMismatch status = http.StatusBadRequest return } @@ -493,8 +498,8 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) { var pass bool var tok string - if pass, tok = CheckAuthorizationHeader(kc, h.ApiTokenCache, req); !pass { - err = BadAuthorizationHeader + if pass, tok = CheckAuthorizationHeader(kc, h.APITokenCache, req); !pass { + err = errBadAuthorizationHeader status = http.StatusForbidden return } @@ -507,7 +512,7 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) { // Check if the client specified the number of replicas if req.Header.Get("X-Keep-Desired-Replicas") != "" { var r int - _, err := fmt.Sscanf(req.Header.Get(keepclient.X_Keep_Desired_Replicas), "%d", &r) + _, err := fmt.Sscanf(req.Header.Get(keepclient.XKeepDesiredReplicas), "%d", &r) if err == nil { kc.Want_replicas = r } @@ -527,7 +532,7 @@ func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) { } // Tell the client how many successful PUTs we accomplished - resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", wroteReplicas)) + resp.Header().Set(keepclient.XKeepReplicasStored, fmt.Sprintf("%d", wroteReplicas)) switch err.(type) { case nil: @@ -575,9 +580,9 @@ func (h *proxyHandler) Index(resp http.ResponseWriter, req *http.Request) { }() kc := h.makeKeepClient(req) - ok, token := CheckAuthorizationHeader(kc, h.ApiTokenCache, req) + ok, token := CheckAuthorizationHeader(kc, h.APITokenCache, req) if !ok { - status, err = http.StatusForbidden, BadAuthorizationHeader + status, err = http.StatusForbidden, errBadAuthorizationHeader return } @@ -588,7 +593,7 @@ func (h *proxyHandler) Index(resp http.ResponseWriter, req *http.Request) { // Only GET method is supported if req.Method != "GET" { - status, err = http.StatusNotImplemented, MethodNotSupported + status, err = http.StatusNotImplemented, errMethodNotSupported return } diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go index 94ed05bff1..6a02ab9bd3 100644 --- a/services/keepproxy/keepproxy_test.go +++ b/services/keepproxy/keepproxy_test.go @@ -131,7 +131,7 @@ func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepc cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance) } - cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}} + cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}} listener = nil go func() { diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go index f2973b586a..3c9d5d15e8 100644 --- a/services/keepstore/keepstore.go +++ b/services/keepstore/keepstore.go @@ -8,10 +8,10 @@ import ( "time" ) -// A Keep "block" is 64MB. +// BlockSize for a Keep "block" is 64MB. const BlockSize = 64 * 1024 * 1024 -// A Keep volume must have at least MinFreeKilobytes available +// MinFreeKilobytes is the amount of space a Keep volume must have available // in order to permit writes. const MinFreeKilobytes = BlockSize / 1024 diff --git a/services/keepstore/proxy_remote_test.go b/services/keepstore/proxy_remote_test.go index a244561dfa..00161bf236 100644 --- a/services/keepstore/proxy_remote_test.go +++ b/services/keepstore/proxy_remote_test.go @@ -91,7 +91,7 @@ func (s *ProxyRemoteSuite) SetUpTest(c *check.C) { s.cluster.Collections.BlobSigningKey = knownKey s.cluster.SystemRootToken = arvadostest.SystemRootToken s.cluster.RemoteClusters = map[string]arvados.RemoteCluster{ - s.remoteClusterID: arvados.RemoteCluster{ + s.remoteClusterID: { Host: strings.Split(s.remoteAPI.URL, "//")[1], Proxy: true, Scheme: "http", diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go index b4ccd98282..670fa1a414 100644 --- a/services/keepstore/pull_worker.go +++ b/services/keepstore/pull_worker.go @@ -80,7 +80,7 @@ func (h *handler) pullItemAndProcess(pullRequest PullRequest) error { return writePulledBlock(h.volmgr, vol, readContent, pullRequest.Locator) } -// Fetch the content for the given locator using keepclient. +// GetContent fetches the content for the given locator using keepclient. var GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (io.ReadCloser, int64, string, error) { return keepClient.Get(signedLocator) } @@ -88,8 +88,7 @@ var GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) ( var writePulledBlock = func(volmgr *RRVolumeManager, volume Volume, data []byte, locator string) error { if volume != nil { return volume.Put(context.Background(), locator, data) - } else { - _, err := PutBlock(context.Background(), volmgr, data, locator) - return err } + _, err := PutBlock(context.Background(), volmgr, data, locator) + return err } diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index 235d369b5a..07bb033c9f 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -586,7 +586,10 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { if err != nil { return err } - fmt.Fprintf(writer, "%s+%d %d\n", data.Key, data.Size, t.UnixNano()) + // We truncate sub-second precision here. Otherwise + // timestamps will never match the RFC1123-formatted + // Last-Modified values parsed by Mtime(). + fmt.Fprintf(writer, "%s+%d %d\n", data.Key, data.Size, t.Unix()*1000000000) } return dataL.Error() } diff --git a/services/keepstore/s3aws_volume.go b/services/keepstore/s3aws_volume.go index c9fa7fce5e..8d999e7472 100644 --- a/services/keepstore/s3aws_volume.go +++ b/services/keepstore/s3aws_volume.go @@ -33,7 +33,7 @@ import ( "github.com/sirupsen/logrus" ) -// S3Volume implements Volume using an S3 bucket. +// S3AWSVolume implements Volume using an S3 bucket. type S3AWSVolume struct { arvados.S3VolumeDriverParameters AuthToken string // populated automatically when IAMRole is used @@ -69,10 +69,9 @@ func chooseS3VolumeDriver(cluster *arvados.Cluster, volume arvados.Volume, logge if v.UseAWSS3v2Driver { logger.Debugln("Using AWS S3 v2 driver") return newS3AWSVolume(cluster, volume, logger, metrics) - } else { - logger.Debugln("Using goamz S3 driver") - return newS3Volume(cluster, volume, logger, metrics) } + logger.Debugln("Using goamz S3 driver") + return newS3Volume(cluster, volume, logger, metrics) } const ( @@ -728,7 +727,10 @@ func (v *S3AWSVolume) IndexTo(prefix string, writer io.Writer) error { if err := recentL.Error(); err != nil { return err } - fmt.Fprintf(writer, "%s+%d %d\n", *data.Key, *data.Size, stamp.LastModified.UnixNano()) + // We truncate sub-second precision here. Otherwise + // timestamps will never match the RFC1123-formatted + // Last-Modified values parsed by Mtime(). + fmt.Fprintf(writer, "%s+%d %d\n", *data.Key, *data.Size, stamp.LastModified.Unix()*1000000000) } return dataL.Error() } diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go index 5a277b6007..4d8a0aec7a 100644 --- a/services/keepstore/volume.go +++ b/services/keepstore/volume.go @@ -353,9 +353,8 @@ func (vm *RRVolumeManager) Mounts() []*VolumeMount { func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) *VolumeMount { if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || !mnt.ReadOnly) { return mnt - } else { - return nil } + return nil } // AllReadable returns an array of all readable volumes diff --git a/services/login-sync/arvados-login-sync.gemspec b/services/login-sync/arvados-login-sync.gemspec index b45f8692be..911548bbf9 100644 --- a/services/login-sync/arvados-login-sync.gemspec +++ b/services/login-sync/arvados-login-sync.gemspec @@ -18,6 +18,7 @@ begin else version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip end + version = version.sub("~dev", ".dev").sub("~rc", ".rc") git_timestamp = Time.at(git_timestamp.to_i).utc ensure ENV["GIT_DIR"] = git_dir @@ -31,7 +32,7 @@ Gem::Specification.new do |s| s.summary = "Set up local login accounts for Arvados users" s.description = "Creates and updates local login accounts for Arvados users. Built from git commit #{git_hash}" s.authors = ["Arvados Authors"] - s.email = 'gem-dev@curoverse.com' + s.email = 'packaging@arvados.org' s.licenses = ['AGPL-3.0'] s.files = ["bin/arvados-login-sync", "agpl-3.0.txt"] s.executables << "arvados-login-sync" diff --git a/services/login-sync/bin/arvados-login-sync b/services/login-sync/bin/arvados-login-sync index e00495c04d..8162e22a2f 100755 --- a/services/login-sync/bin/arvados-login-sync +++ b/services/login-sync/bin/arvados-login-sync @@ -36,7 +36,7 @@ begin logins = arv.virtual_machine.logins(:uuid => vm_uuid)[:items] logins = [] if logins.nil? - logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:virtual_machine_uuid] != vm_uuid } + logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:virtual_machine_uuid] != vm_uuid } # No system users uid_min = 1000 @@ -79,48 +79,77 @@ begin logins.each do |l| keys[l[:username]] = Array.new() if not keys.has_key?(l[:username]) key = l[:public_key] - # Handle putty-style ssh public keys - key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1') - key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1') - key.gsub!(/\n/,'') - key.strip - - keys[l[:username]].push(key) if not keys[l[:username]].include?(key) + if !key.nil? + # Handle putty-style ssh public keys + key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1') + key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1') + key.gsub!(/\n/,'') + key.strip + + keys[l[:username]].push(key) if not keys[l[:username]].include?(key) + end end seen = Hash.new() - devnull = open("/dev/null", "w") + + current_user_groups = Hash.new + while (ent = Etc.getgrent()) do + ent.mem.each do |member| + current_user_groups[member] ||= Array.new + current_user_groups[member].push ent.name + end + end + Etc.endgrent() logins.each do |l| next if seen[l[:username]] seen[l[:username]] = true + username = l[:username] + unless pwnam[l[:username]] STDERR.puts "Creating account #{l[:username]}" - groups = l[:groups] || [] - # Adding users to the FUSE group has long been hardcoded behavior. - groups << "fuse" - groups.select! { |g| Etc.getgrnam(g) rescue false } # Create new user unless system("useradd", "-m", - "-c", l[:username], + "-c", username, "-s", "/bin/bash", - "-G", groups.join(","), - l[:username], - out: devnull) + username) STDERR.puts "Account creation failed for #{l[:username]}: #{$?}" next end begin - pwnam[l[:username]] = Etc.getpwnam(l[:username]) + pwnam[username] = Etc.getpwnam(username) rescue => e STDERR.puts "Created account but then getpwnam() failed for #{l[:username]}: #{e}" raise end end - @homedir = pwnam[l[:username]].dir - userdotssh = File.join(@homedir, ".ssh") + existing_groups = current_user_groups[username] || [] + groups = l[:groups] || [] + # Adding users to the FUSE group has long been hardcoded behavior. + groups << "fuse" + groups << username + groups.select! { |g| Etc.getgrnam(g) rescue false } + + groups.each do |addgroup| + if existing_groups.index(addgroup).nil? + # User should be in group, but isn't, so add them. + STDERR.puts "Add user #{username} to #{addgroup} group" + system("adduser", username, addgroup) + end + end + + existing_groups.each do |removegroup| + if groups.index(removegroup).nil? + # User is in a group, but shouldn't be, so remove them. + STDERR.puts "Remove user #{username} from #{removegroup} group" + system("deluser", username, removegroup) + end + end + + homedir = pwnam[l[:username]].dir + userdotssh = File.join(homedir, ".ssh") Dir.mkdir(userdotssh) if !File.exist?(userdotssh) newkeys = "###\n###\n" + keys[l[:username]].join("\n") + "\n###\n###\n" @@ -148,13 +177,41 @@ begin f.write(newkeys) f.close() end + + userdotconfig = File.join(homedir, ".config") + if !File.exist?(userdotconfig) + Dir.mkdir(userdotconfig) + end + + configarvados = File.join(userdotconfig, "arvados") + Dir.mkdir(configarvados) if !File.exist?(configarvados) + + tokenfile = File.join(configarvados, "settings.conf") + + begin + if !File.exist?(tokenfile) + user_token = arv.api_client_authorization.create(api_client_authorization: {owner_uuid: l[:user_uuid], api_client_id: 0}) + f = File.new(tokenfile, 'w') + f.write("ARVADOS_API_HOST=#{ENV['ARVADOS_API_HOST']}\n") + f.write("ARVADOS_API_TOKEN=v2/#{user_token[:uuid]}/#{user_token[:api_token]}\n") + f.close() + end + rescue => e + STDERR.puts "Error setting token for #{l[:username]}: #{e}" + end + FileUtils.chown_R(l[:username], nil, userdotssh) + FileUtils.chown_R(l[:username], nil, userdotconfig) File.chmod(0700, userdotssh) - File.chmod(0750, @homedir) + File.chmod(0700, userdotconfig) + File.chmod(0700, configarvados) + File.chmod(0750, homedir) File.chmod(0600, keysfile) + if File.exist?(tokenfile) + File.chmod(0600, tokenfile) + end end - devnull.close rescue Exception => bang puts "Error: " + bang.to_s puts bang.backtrace.join("\n") diff --git a/services/login-sync/test/test_add_user.rb b/services/login-sync/test/test_add_user.rb index e90c16d64f..db909ac83f 100644 --- a/services/login-sync/test/test_add_user.rb +++ b/services/login-sync/test/test_add_user.rb @@ -16,20 +16,15 @@ class TestAddUser < Minitest::Test File.open(@tmpdir+'/succeed', 'w') do |f| end invoke_sync binstubs: ['new_user'] spied = File.read(@tmpdir+'/spy') - assert_match %r{useradd -m -c active -s /bin/bash -G (fuse)? active}, spied - assert_match %r{useradd -m -c adminroot -s /bin/bash -G #{valid_groups.join(',')} adminroot}, spied + assert_match %r{useradd -m -c active -s /bin/bash active}, spied + assert_match %r{useradd -m -c adminroot -s /bin/bash adminroot}, spied end def test_useradd_success # binstub_new_user/useradd will succeed. File.open(@tmpdir+'/succeed', 'w') do |f| - f.puts 'useradd -m -c active -s /bin/bash -G fuse active' - f.puts 'useradd -m -c active -s /bin/bash -G active' - # Accept either form; see note about groups in test_useradd_error. - f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,fuse adminroot' - f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,admin,fuse adminroot' - f.puts 'useradd -m -c adminroot -s /bin/bash -G docker adminroot' - f.puts 'useradd -m -c adminroot -s /bin/bash -G docker,admin adminroot' + f.puts 'useradd -m -c active -s /bin/bash -G active' + f.puts 'useradd -m -c adminroot -s /bin/bash adminroot' end $stderr.puts "*** Expect crash after getpwnam() fails:" invoke_sync binstubs: ['new_user'] diff --git a/services/ws/doc.go b/services/ws/doc.go index 6a86cbe7a8..a67df15117 100644 --- a/services/ws/doc.go +++ b/services/ws/doc.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0 -// Arvados-ws exposes Arvados APIs (currently just one, the +// Package ws exposes Arvados APIs (currently just one, the // cache-invalidation event feed at "ws://.../websocket") to // websocket clients. // diff --git a/services/ws/service_test.go b/services/ws/service_test.go index 13726836a4..4e68d09da2 100644 --- a/services/ws/service_test.go +++ b/services/ws/service_test.go @@ -72,7 +72,7 @@ func (*serviceSuite) testConfig(c *check.C) (*arvados.Cluster, error) { cluster.TLS.Insecure = client.Insecure cluster.PostgreSQL.Connection = testDBConfig() cluster.PostgreSQL.ConnectionPool = 12 - cluster.Services.Websocket.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":"}: arvados.ServiceInstance{}} + cluster.Services.Websocket.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":"}: {}} cluster.ManagementToken = arvadostest.ManagementToken return cluster, nil } diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox index 5abaa90e36..96f3666cda 100755 --- a/tools/arvbox/bin/arvbox +++ b/tools/arvbox/bin/arvbox @@ -44,10 +44,6 @@ if test -z "$ARVADOS_ROOT" ; then ARVADOS_ROOT="$ARVBOX_DATA/arvados" fi -if test -z "$SSO_ROOT" ; then - SSO_ROOT="$ARVBOX_DATA/sso-devise-omniauth-provider" -fi - if test -z "$COMPOSER_ROOT" ; then COMPOSER_ROOT="$ARVBOX_DATA/composer" fi @@ -64,6 +60,8 @@ PIPCACHE="$ARVBOX_DATA/pip" NPMCACHE="$ARVBOX_DATA/npm" GOSTUFF="$ARVBOX_DATA/gopath" RLIBS="$ARVBOX_DATA/Rlibs" +ARVADOS_CONTAINER_PATH="/var/lib/arvados-arvbox" +GEM_HOME="/var/lib/arvados/lib/ruby/gems/2.5.0" getip() { docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $ARVBOX_CONTAINER @@ -82,7 +80,7 @@ gethost() { } getclusterid() { - docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/api_uuid_prefix + docker exec $ARVBOX_CONTAINER cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix } updateconf() { @@ -99,6 +97,10 @@ EOF fi } +listusers() { + docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py $ARVADOS_CONTAINER_PATH/cluster_config.yml $(getclusterid) list +} + wait_for_arvbox() { FF=/tmp/arvbox-fifo-$$ mkfifo $FF @@ -107,11 +109,11 @@ wait_for_arvbox() { while read line ; do if [[ $line =~ "ok: down: ready:" ]] ; then kill $LOGPID - set +e - wait $LOGPID 2>/dev/null - set -e - else - echo $line + set +e + wait $LOGPID 2>/dev/null + set -e + else + echo $line fi done < $FF rm $FF @@ -125,20 +127,19 @@ wait_for_arvbox() { docker_run_dev() { docker run \ - "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ - "--volume=$SSO_ROOT:/usr/src/sso:rw" \ + "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \ "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \ "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \ "--volume=$PG_DATA:/var/lib/postgresql:rw" \ - "--volume=$VAR_DATA:/var/lib/arvados:rw" \ + "--volume=$VAR_DATA:$ARVADOS_CONTAINER_PATH:rw" \ "--volume=$PASSENGER:/var/lib/passenger:rw" \ - "--volume=$GEMS:/var/lib/gems:rw" \ + "--volume=$GEMS:$GEM_HOME:rw" \ "--volume=$PIPCACHE:/var/lib/pip:rw" \ "--volume=$NPMCACHE:/var/lib/npm:rw" \ "--volume=$GOSTUFF:/var/lib/gopath:rw" \ "--volume=$RLIBS:/var/lib/Rlibs:rw" \ - --label "org.arvados.arvbox_config=$CONFIG" \ - "$@" + --label "org.arvados.arvbox_config=$CONFIG" \ + "$@" } running_config() { @@ -154,10 +155,10 @@ run() { need_setup=1 if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - if [[ $(running_config) != "$CONFIG" ]] ; then - echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" - return 1 - fi + if [[ $(running_config) != "$CONFIG" ]] ; then + echo "Container $ARVBOX_CONTAINER is '$(running_config)' config but requested '$CONFIG'; use restart or reboot" + return 1 + fi if test "$CONFIG" = test -o "$CONFIG" = devenv ; then need_setup=0 else @@ -176,12 +177,12 @@ run() { if test -n "$TAG" then if test $(echo $TAG | cut -c1-1) != '-' ; then - TAG=":$TAG" + TAG=":$TAG" shift else - if [[ $TAG = '-' ]] ; then - shift - fi + if [[ $TAG = '-' ]] ; then + shift + fi unset TAG fi fi @@ -193,7 +194,7 @@ run() { defaultdev=$(/sbin/ip route|awk '/default/ { print $5 }') localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/') fi - echo "Public arvbox will use address $localip" + echo "Public arvbox will use address $localip" iptemp=$(mktemp) echo $localip > $iptemp chmod og+r $iptemp @@ -204,10 +205,12 @@ run() { --publish=8900:8900 --publish=9000:9000 --publish=9002:9002 + --publish=9004:9004 --publish=25101:25101 --publish=8001:8001 --publish=8002:8002 - --publish=45000-45020:45000-45020" + --publish=4202:4202 + --publish=45000-45020:45000-45020" else PUBLIC="" fi @@ -220,7 +223,7 @@ run() { fi if ! (docker ps -a | grep -E "$ARVBOX_CONTAINER-data$" -q) ; then - docker create -v /var/lib/postgresql -v /var/lib/arvados --name $ARVBOX_CONTAINER-data arvados/arvbox-demo /bin/true + docker create -v /var/lib/postgresql -v $ARVADOS_CONTAINER_PATH --name $ARVBOX_CONTAINER-data arvados/arvbox-demo /bin/true fi docker run \ @@ -228,7 +231,7 @@ run() { --name=$ARVBOX_CONTAINER \ --privileged \ --volumes-from $ARVBOX_CONTAINER-data \ - --label "org.arvados.arvbox_config=$CONFIG" \ + --label "org.arvados.arvbox_config=$CONFIG" \ $PUBLIC \ arvados/arvbox-demo$TAG updateconf @@ -239,16 +242,13 @@ run() { if ! test -d "$ARVADOS_ROOT" ; then git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT" fi - if ! test -d "$SSO_ROOT" ; then - git clone https://github.com/arvados/sso-devise-omniauth-provider.git "$SSO_ROOT" - fi if ! test -d "$COMPOSER_ROOT" ; then git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT" git -C "$COMPOSER_ROOT" checkout arvados-fork git -C "$COMPOSER_ROOT" pull fi if ! test -d "$WORKBENCH2_ROOT" ; then - git clone https://github.com/arvados/arvados-workbench2.git "$WORKBENCH2_ROOT" + git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT" fi if [[ "$CONFIG" = test ]] ; then @@ -260,62 +260,52 @@ run() { --detach \ --name=$ARVBOX_CONTAINER \ --privileged \ - "--env=SVDIR=/etc/test-service" \ + "--env=SVDIR=/etc/test-service" \ arvados/arvbox-dev$TAG docker exec -ti \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /usr/local/lib/arvbox/waitforpostgres.sh - - docker exec -ti \ - $ARVBOX_CONTAINER \ - /usr/local/lib/arvbox/runsu.sh \ - /var/lib/arvbox/service/sso/run-service --only-setup - - docker exec -ti \ - $ARVBOX_CONTAINER \ - /usr/local/lib/arvbox/runsu.sh \ - /var/lib/arvbox/service/api/run-service --only-setup fi - interactive="" - if [[ -z "$@" ]] ; then - interactive=--interactive - fi + interactive="" + if [[ -z "$@" ]] ; then + interactive=--interactive + fi docker exec -ti \ -e LINES=$(tput lines) \ -e COLUMNS=$(tput cols) \ -e TERM=$TERM \ -e WORKSPACE=/usr/src/arvados \ - -e GEM_HOME=/var/lib/gems \ - -e CONFIGSRC=/var/lib/arvados/run_tests \ + -e GEM_HOME=$GEM_HOME \ + -e CONFIGSRC=$ARVADOS_CONTAINER_PATH/run_tests \ $ARVBOX_CONTAINER \ /usr/local/lib/arvbox/runsu.sh \ /usr/src/arvados/build/run-tests.sh \ - --temp /var/lib/arvados/test \ - $interactive \ + --temp $ARVADOS_CONTAINER_PATH/test \ + $interactive \ "$@" elif [[ "$CONFIG" = devenv ]] ; then - if [[ $need_setup = 1 ]] ; then - docker_run_dev \ + if [[ $need_setup = 1 ]] ; then + docker_run_dev \ --detach \ - --name=${ARVBOX_CONTAINER} \ - "--env=SVDIR=/etc/devenv-service" \ - "--volume=$HOME:$HOME:rw" \ - --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ - arvados/arvbox-dev$TAG - fi - exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e "ARVBOX_HOME=$HOME" \ - -e "DISPLAY=$DISPLAY" \ - --workdir=$PWD \ - ${ARVBOX_CONTAINER} \ - /usr/local/lib/arvbox/devenv.sh "$@" + --name=${ARVBOX_CONTAINER} \ + "--env=SVDIR=/etc/devenv-service" \ + "--volume=$HOME:$HOME:rw" \ + --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \ + arvados/arvbox-dev$TAG + fi + exec docker exec --interactive --tty \ + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e "ARVBOX_HOME=$HOME" \ + -e "DISPLAY=$DISPLAY" \ + --workdir=$PWD \ + ${ARVBOX_CONTAINER} \ + /usr/local/lib/arvbox/devenv.sh "$@" elif [[ "$CONFIG" =~ dev$ ]] ; then docker_run_dev \ --detach \ @@ -326,7 +316,12 @@ run() { updateconf wait_for_arvbox echo "The Arvados source code is checked out at: $ARVADOS_ROOT" - echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem" + echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem" + if [[ "$(listusers)" =~ ^\{\} ]] ; then + echo "No users defined, use 'arvbox adduser' to add user logins" + else + echo "Use 'arvbox listusers' to see user logins" + fi else echo "Unknown configuration '$CONFIG'" fi @@ -340,7 +335,7 @@ update() { if test -n "$TAG" then if test $(echo $TAG | cut -c1-1) != '-' ; then - TAG=":$TAG" + TAG=":$TAG" shift else unset TAG @@ -348,9 +343,9 @@ update() { fi if echo "$CONFIG" | grep 'demo$' ; then - docker pull arvados/arvbox-demo$TAG + docker pull arvados/arvbox-demo$TAG else - docker pull arvados/arvbox-dev$TAG + docker pull arvados/arvbox-dev$TAG fi } @@ -369,6 +364,7 @@ stop() { } build() { + export DOCKER_BUILDKIT=1 if ! test -f "$ARVBOX_DOCKER/Dockerfile.base" ; then echo "Could not find Dockerfile (expected it at $ARVBOX_DOCKER/Dockerfile.base)" exit 1 @@ -379,15 +375,25 @@ build() { FORCE=-f fi GITHEAD=$(cd $ARVBOX_DOCKER && git log --format=%H -n1 HEAD) - docker build --build-arg=arvados_version=$GITHEAD $NO_CACHE -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$ARVBOX_DOCKER" - docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest + + set +e + if which greadlink >/dev/null 2>/dev/null ; then + LOCAL_ARVADOS_ROOT=$(greadlink -f $(dirname $0)/../../../) + else + LOCAL_ARVADOS_ROOT=$(readlink -f $(dirname $0)/../../../) + fi + set -e + if test "$1" = localdemo -o "$1" = publicdemo ; then - docker build $NO_CACHE -t arvados/arvbox-demo:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.demo" "$ARVBOX_DOCKER" - docker tag $FORCE arvados/arvbox-demo:$GITHEAD arvados/arvbox-demo:latest + BUILDTYPE=demo else - docker build $NO_CACHE -t arvados/arvbox-dev:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.dev" "$ARVBOX_DOCKER" - docker tag $FORCE arvados/arvbox-dev:$GITHEAD arvados/arvbox-dev:latest + BUILDTYPE=dev fi + + docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE --build-arg=arvados_version=$GITHEAD --build-arg=workdir=/tools/arvbox/lib/arvbox/docker -t arvados/arvbox-base:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.base" "$LOCAL_ARVADOS_ROOT" + docker tag $FORCE arvados/arvbox-base:$GITHEAD arvados/arvbox-base:latest + docker build $NO_CACHE -t arvados/arvbox-$BUILDTYPE:$GITHEAD -f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" "$ARVBOX_DOCKER" + docker tag $FORCE arvados/arvbox-$BUILDTYPE:$GITHEAD arvados/arvbox-$BUILDTYPE:latest } check() { @@ -424,26 +430,26 @@ case "$subcmd" in sh*) exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e GEM_HOME=/var/lib/gems \ - $ARVBOX_CONTAINER /bin/bash + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=$GEM_HOME \ + $ARVBOX_CONTAINER /bin/bash ;; ash*) exec docker exec --interactive --tty \ - -e LINES=$(tput lines) \ - -e COLUMNS=$(tput cols) \ - -e TERM=$TERM \ - -e GEM_HOME=/var/lib/gems \ - -u arvbox \ - -w /usr/src/arvados \ - $ARVBOX_CONTAINER /bin/bash --login + -e LINES=$(tput lines) \ + -e COLUMNS=$(tput cols) \ + -e TERM=$TERM \ + -e GEM_HOME=$GEM_HOME \ + -u arvbox \ + -w /usr/src/arvados \ + $ARVBOX_CONTAINER /bin/bash --login ;; pipe) - exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - + exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=$GEM_HOME /bin/bash - ;; stop) @@ -466,7 +472,7 @@ case "$subcmd" in update) check $@ stop - update $@ + update $@ run $@ ;; @@ -485,7 +491,7 @@ case "$subcmd" in status) echo "Container: $ARVBOX_CONTAINER" if docker ps -a --filter "status=running" | grep -E "$ARVBOX_CONTAINER$" -q ; then - echo "Cluster id: $(getclusterid)" + echo "Cluster id: $(getclusterid)" echo "Status: running" echo "Container IP: $(getip)" echo "Published host: $(gethost)" @@ -511,6 +517,7 @@ case "$subcmd" in exit 1 fi set -x + chmod -R u+w "$ARVBOX_DATA" rm -rf "$ARVBOX_DATA" else if test "$1" != -f ; then @@ -565,18 +572,17 @@ case "$subcmd" in clone) if test -n "$2" ; then - mkdir -p "$ARVBOX_BASE/$2" + mkdir -p "$ARVBOX_BASE/$2" cp -a "$ARVBOX_BASE/$1/passenger" \ - "$ARVBOX_BASE/$1/gems" \ - "$ARVBOX_BASE/$1/pip" \ - "$ARVBOX_BASE/$1/npm" \ - "$ARVBOX_BASE/$1/gopath" \ - "$ARVBOX_BASE/$1/Rlibs" \ - "$ARVBOX_BASE/$1/arvados" \ - "$ARVBOX_BASE/$1/sso-devise-omniauth-provider" \ - "$ARVBOX_BASE/$1/composer" \ - "$ARVBOX_BASE/$1/workbench2" \ - "$ARVBOX_BASE/$2" + "$ARVBOX_BASE/$1/gems" \ + "$ARVBOX_BASE/$1/pip" \ + "$ARVBOX_BASE/$1/npm" \ + "$ARVBOX_BASE/$1/gopath" \ + "$ARVBOX_BASE/$1/Rlibs" \ + "$ARVBOX_BASE/$1/arvados" \ + "$ARVBOX_BASE/$1/composer" \ + "$ARVBOX_BASE/$1/workbench2" \ + "$ARVBOX_BASE/$2" echo "Created new arvbox $2" echo "export ARVBOX_CONTAINER=$2" else @@ -586,28 +592,28 @@ case "$subcmd" in ;; root-cert) - CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt - if test -n "$1" ; then - CERT="$1" - fi - docker exec $ARVBOX_CONTAINER cat /var/lib/arvados/root-cert.pem > "$CERT" - echo "Certificate copied to $CERT" - ;; + CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt + if test -n "$1" ; then + CERT="$1" + fi + docker exec $ARVBOX_CONTAINER cat $ARVADOS_CONTAINER_PATH/root-cert.pem > "$CERT" + echo "Certificate copied to $CERT" + ;; psql) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados' + ;; checkpoint) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > $ARVADOS_CONTAINER_PATH/checkpoint.sql' + ;; restore) - exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql' - ;; + exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=$ARVADOS_CONTAINER_PATH/checkpoint.sql' + ;; hotreset) - exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - < [password]" + fi + ;; + + removeuser) + if [[ -n "$1" ]] ; then + docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py $ARVADOS_CONTAINER_PATH/cluster_config.yml.override $(getclusterid) remove $@ + docker exec $ARVBOX_CONTAINER sv restart controller + else + echo "Usage: removeuser " + fi + ;; + + listusers) + listusers + ;; *) echo "Arvados-in-a-box https://doc.arvados.org/install/arvbox.html" @@ -650,9 +675,9 @@ EOF echo "build build arvbox Docker image" echo "reboot stop, build arvbox Docker image, run" echo "rebuild build arvbox Docker image, no layer cache" - echo "checkpoint create database backup" - echo "restore restore checkpoint" - echo "hotreset reset database and restart API without restarting container" + echo "checkpoint create database backup" + echo "restore restore checkpoint" + echo "hotreset reset database and restart API without restarting container" echo "reset delete arvbox arvados data (be careful!)" echo "destroy delete all arvbox code and data (be careful!)" echo "log tail log of specified service" @@ -660,7 +685,12 @@ EOF echo "cat get contents of files inside arvbox" echo "pipe run a bash script piped in from stdin" echo "sv " - echo " change state of service inside arvbox" + echo " change state of service inside arvbox" echo "clone clone dev arvbox" + echo "adduser [password]" + echo " add a user login" + echo "removeuser " + echo " remove user login" + echo "listusers list user logins" ;; esac diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base index b6d6c68e31..79f0d3f4f6 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base @@ -1,98 +1,112 @@ +# syntax = docker/dockerfile:experimental # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 -FROM debian:9 +ARG BUILDTYPE +# We're using poor man's conditionals (see +# https://www.docker.com/blog/advanced-dockerfiles-faster-builds-and-smaller-images-using-buildkit-and-multistage-builds/) +# here to dtrt in the dev/test scenario and the demo scenario. In the dev/test +# scenario, we use the docker context (i.e. the copy of Arvados checked out on +# the host) to build arvados-server. In the demo scenario, we check out a new +# tree, and use the $arvados_version commit (passed in via an argument). + +########################################################################################################### +FROM debian:10-slim as dev ENV DEBIAN_FRONTEND noninteractive +RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list + RUN apt-get update && \ apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ - postgresql-9.6 postgresql-contrib-9.6 git build-essential runit curl libpq-dev \ - libcurl4-openssl-dev libssl1.0-dev zlib1g-dev libpcre3-dev libpam-dev \ - openssh-server python-setuptools netcat-traditional \ - python-epydoc graphviz bzip2 less sudo virtualenv \ - libpython-dev fuse libfuse-dev python-pip python-yaml \ - pkg-config libattr1-dev python-pycurl \ - libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \ - libjson-perl nginx gitolite3 lsof libreadline-dev \ - apt-transport-https ca-certificates \ - linkchecker python3-virtualenv python-virtualenv xvfb iceweasel \ - libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \ - libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \ - python3-setuptools python3-pip openjdk-8-jdk bsdmainutils net-tools \ - ruby2.3 ruby-dev bundler && \ - apt-get clean + golang -t buster-backports -ENV RUBYVERSION_MINOR 2.3 -ENV RUBYVERSION 2.3.5 +RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + build-essential ca-certificates git libpam0g-dev -# Install Ruby from source -# RUN cd /tmp && \ -# curl -f http://cache.ruby-lang.org/pub/ruby/${RUBYVERSION_MINOR}/ruby-${RUBYVERSION}.tar.gz | tar -xzf - && \ -# cd ruby-${RUBYVERSION} && \ -# ./configure --disable-install-doc && \ -# make && \ -# make install && \ -# cd /tmp && \ -# rm -rf ruby-${RUBYVERSION} +ENV GOPATH /var/lib/gopath -ENV GEM_HOME /var/lib/gems -ENV GEM_PATH /var/lib/gems -ENV PATH $PATH:/var/lib/gems/bin +# the --mount option requires the experimental syntax enabled (enables +# buildkit) on the first line of this file. This Dockerfile must also be built +# with the DOCKER_BUILDKIT=1 environment variable set. +RUN --mount=type=bind,target=/usr/src/arvados \ + cd /usr/src/arvados && \ + go mod download && \ + cd cmd/arvados-server && \ + go install -ENV GOVERSION 1.13.6 +########################################################################################################### +FROM debian:10-slim as demo +ENV DEBIAN_FRONTEND noninteractive -# Install golang binary -RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \ - tar -C /usr/local -xzf - +RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/backports.list -ENV PATH ${PATH}:/usr/local/go/bin +RUN apt-get update && \ + apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + golang -t buster-backports -VOLUME /var/lib/docker -VOLUME /var/log/nginx -VOLUME /etc/ssl/private +RUN apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + build-essential ca-certificates git libpam0g-dev -ADD 8D81803C0EBFCD88.asc /tmp/ -RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \ - rm -f /tmp/8D81803C0EBFCD88.asc +ENV GOPATH /var/lib/gopath -RUN mkdir -p /etc/apt/sources.list.d && \ - echo deb https://download.docker.com/linux/debian/ stretch stable > /etc/apt/sources.list.d/docker.list && \ - apt-get update && \ - apt-get -yq --no-install-recommends install docker-ce=17.06.0~ce-0~debian && \ - apt-get clean +ARG arvados_version +RUN echo arvados_version is git commit $arvados_version -RUN rm -rf /var/lib/postgresql && mkdir -p /var/lib/postgresql +RUN cd /usr/src && \ + git clone --no-checkout https://git.arvados.org/arvados.git && \ + git -C arvados checkout ${arvados_version} && \ + cd /usr/src/arvados && \ + go mod download && \ + cd cmd/arvados-server && \ + go install -ENV PJSVERSION=1.9.8 -# bitbucket is the origin, but downloads fail sometimes, so use our own mirror instead. -#ENV PJSURL=https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2 -ENV PJSURL=http://cache.arvados.org/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2 +########################################################################################################### +FROM ${BUILDTYPE} as base -RUN set -e && \ - curl -L -f ${PJSURL} | tar -C /usr/local -xjf - && \ - ln -s ../phantomjs-${PJSVERSION}-linux-x86_64/bin/phantomjs /usr/local/bin +########################################################################################################### +FROM debian:10 +ENV DEBIAN_FRONTEND noninteractive -ENV GDVERSION=v0.23.0 -ENV GDURL=https://github.com/mozilla/geckodriver/releases/download/$GDVERSION/geckodriver-$GDVERSION-linux64.tar.gz -RUN set -e && curl -L -f ${GDURL} | tar -C /usr/local/bin -xzf - geckodriver +# The arvbox-specific dependencies are +# gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less +RUN apt-get update && \ + apt-get -yq --no-install-recommends -o Acquire::Retries=6 install \ + gnupg2 runit python3-pip python3-setuptools python3-yaml shellinabox netcat less && \ + apt-get clean + +ENV GOPATH /var/lib/gopath +RUN echo buildtype is $BUILDTYPE -RUN pip install -U setuptools +RUN mkdir -p $GOPATH/bin/ +COPY --from=base $GOPATH/bin/arvados-server $GOPATH/bin/arvados-server +RUN $GOPATH/bin/arvados-server --version +RUN $GOPATH/bin/arvados-server install -type test -ENV NODEVERSION v8.15.1 +RUN /etc/init.d/postgresql start && \ + su postgres -c 'dropuser arvados' && \ + su postgres -c 'createuser -s arvbox' && \ + /etc/init.d/postgresql stop + +ENV GEM_HOME /var/lib/arvados/lib/ruby/gems/2.5.0 +ENV PATH $PATH:$GEM_HOME/bin + +VOLUME /var/lib/docker +VOLUME /var/log/nginx +VOLUME /etc/ssl/private -# Install nodejs binary -RUN curl -L -f https://nodejs.org/dist/${NODEVERSION}/node-${NODEVERSION}-linux-x64.tar.xz | tar -C /usr/local -xJf - && \ - ln -s ../node-${NODEVERSION}-linux-x64/bin/node ../node-${NODEVERSION}-linux-x64/bin/npm /usr/local/bin +ARG workdir -ENV GRADLEVERSION 5.3.1 +ADD $workdir/8D81803C0EBFCD88.asc /tmp/ +RUN apt-key add --no-tty /tmp/8D81803C0EBFCD88.asc && \ + rm -f /tmp/8D81803C0EBFCD88.asc -RUN cd /tmp && \ - curl -L -O https://services.gradle.org/distributions/gradle-${GRADLEVERSION}-bin.zip && \ - unzip gradle-${GRADLEVERSION}-bin.zip -d /usr/local && \ - ln -s ../gradle-${GRADLEVERSION}/bin/gradle /usr/local/bin && \ - rm gradle-${GRADLEVERSION}-bin.zip +RUN mkdir -p /etc/apt/sources.list.d && \ + echo deb https://download.docker.com/linux/debian/ buster stable > /etc/apt/sources.list.d/docker.list && \ + apt-get update && \ + apt-get -yq --no-install-recommends install docker-ce=5:19.03.13~3-0~debian-buster && \ + apt-get clean # Set UTF-8 locale RUN echo en_US.UTF-8 UTF-8 > /etc/locale.gen && locale-gen @@ -103,18 +117,25 @@ ENV LC_ALL en_US.UTF-8 ARG arvados_version RUN echo arvados_version is git commit $arvados_version -ADD fuse.conf /etc/ +COPY $workdir/fuse.conf /etc/ -ADD gitolite.rc \ - keep-setup.sh common.sh createusers.sh \ - logger runsu.sh waitforpostgres.sh \ - yml_override.py api-setup.sh \ - go-setup.sh devenv.sh cluster-config.sh \ +COPY $workdir/gitolite.rc \ + $workdir/keep-setup.sh $workdir/common.sh $workdir/createusers.sh \ + $workdir/logger $workdir/runsu.sh $workdir/waitforpostgres.sh \ + $workdir/yml_override.py $workdir/api-setup.sh \ + $workdir/go-setup.sh $workdir/devenv.sh $workdir/cluster-config.sh $workdir/edit_users.py \ /usr/local/lib/arvbox/ -ADD runit /etc/runit +COPY $workdir/runit /etc/runit + +# arvbox mounts a docker volume at $ARVADOS_CONTAINER_PATH, make sure that that +# doesn't overlap with the directory where `arvados-server install -type test` +# put everything (/var/lib/arvados) +ENV ARVADOS_CONTAINER_PATH /var/lib/arvados-arvbox + +RUN /bin/ln -s /var/lib/arvados/bin/ruby /usr/local/bin/ # Start the supervisor. ENV SVDIR /etc/service STOPSIGNAL SIGINT -CMD ["/sbin/runit"] +CMD ["/etc/runit/2"] diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo index 34d3845eaf..92d4e70881 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo @@ -4,31 +4,40 @@ FROM arvados/arvbox-base ARG arvados_version -ARG sso_version=master ARG composer_version=arvados-fork ARG workbench2_version=master RUN cd /usr/src && \ - git clone --no-checkout https://github.com/arvados/arvados.git && \ + git clone --no-checkout https://git.arvados.org/arvados.git && \ git -C arvados checkout ${arvados_version} && \ git -C arvados pull && \ - git clone --no-checkout https://github.com/arvados/sso-devise-omniauth-provider.git sso && \ - git -C sso checkout ${sso_version} && \ - git -C sso pull && \ git clone --no-checkout https://github.com/arvados/composer.git && \ git -C composer checkout ${composer_version} && \ git -C composer pull && \ - git clone --no-checkout https://github.com/arvados/arvados-workbench2.git workbench2 && \ + git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \ git -C workbench2 checkout ${workbench2_version} && \ git -C workbench2 pull && \ chown -R 1000:1000 /usr/src +# avoid rebuilding arvados-server, it's already been built as part of the base image +RUN install $GOPATH/bin/arvados-server /usr/local/bin + ADD service/ /var/lib/arvbox/service RUN ln -sf /var/lib/arvbox/service /etc -RUN mkdir -p /var/lib/arvados -RUN echo "production" > /var/lib/arvados/api_rails_env -RUN echo "production" > /var/lib/arvados/sso_rails_env -RUN echo "production" > /var/lib/arvados/workbench_rails_env +RUN mkdir -p $ARVADOS_CONTAINER_PATH +RUN echo "production" > $ARVADOS_CONTAINER_PATH/api_rails_env +RUN echo "production" > $ARVADOS_CONTAINER_PATH/workbench_rails_env + +# for the federation tests, the dev server watches a lot of files, +# and we run three instances of the docker container. Bump up the +# inotify limit from 8192, to avoid errors like +# events.js:183 +# throw er; // Unhandled 'error' event +# ^ +# +# Error: watch /usr/src/workbench2/public ENOSPC +# cf. https://github.com/facebook/jest/issues/3254 +RUN echo fs.inotify.max_user_watches=524288 >> /etc/sysctl.conf RUN /usr/local/lib/arvbox/createusers.sh @@ -36,7 +45,6 @@ RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps RUN sudo -u arvbox /var/lib/arvbox/service/composer/run-service --only-deps RUN sudo -u arvbox /var/lib/arvbox/service/workbench2/run-service --only-deps RUN sudo -u arvbox /var/lib/arvbox/service/keep-web/run-service --only-deps -RUN sudo -u arvbox /var/lib/arvbox/service/sso/run-service --only-deps RUN sudo -u arvbox /var/lib/arvbox/service/workbench/run-service --only-deps RUN sudo -u arvbox /var/lib/arvbox/service/doc/run-service --only-deps RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev index 22668253e1..e9c296a190 100644 --- a/tools/arvbox/lib/arvbox/docker/Dockerfile.dev +++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.dev @@ -7,12 +7,11 @@ ARG arvados_version ADD service/ /var/lib/arvbox/service RUN ln -sf /var/lib/arvbox/service /etc -RUN mkdir -p /var/lib/arvados -RUN echo "development" > /var/lib/arvados/api_rails_env -RUN echo "development" > /var/lib/arvados/sso_rails_env -RUN echo "development" > /var/lib/arvados/workbench_rails_env +RUN mkdir -p $ARVADOS_CONTAINER_PATH +RUN echo "development" > $ARVADOS_CONTAINER_PATH/api_rails_env +RUN echo "development" > $ARVADOS_CONTAINER_PATH/workbench_rails_env RUN mkdir /etc/test-service && \ ln -sf /var/lib/arvbox/service/postgres /etc/test-service && \ ln -sf /var/lib/arvbox/service/certificate /etc/test-service -RUN mkdir /etc/devenv-service \ No newline at end of file +RUN mkdir /etc/devenv-service diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh index 4ed25e03c0..4ad2aed0cc 100755 --- a/tools/arvbox/lib/arvbox/docker/api-setup.sh +++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh @@ -11,36 +11,31 @@ set -ex -o pipefail cd /usr/src/arvados/services/api -if test -s /var/lib/arvados/api_rails_env ; then - export RAILS_ENV=$(cat /var/lib/arvados/api_rails_env) +if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then + export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env) else export RAILS_ENV=development fi set -u -flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh +flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh if test -a /usr/src/arvados/services/api/config/arvados_config.rb ; then rm -f config/application.yml config/database.yml else - uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix) - secret_token=$(cat /var/lib/arvados/api_secret_token) - blob_signing_key=$(cat /var/lib/arvados/blob_signing_key) - management_token=$(cat /var/lib/arvados/management_token) - sso_app_secret=$(cat /var/lib/arvados/sso_app_secret) - database_pw=$(cat /var/lib/arvados/api_database_pw) - vm_uuid=$(cat /var/lib/arvados/vm-uuid) + uuid_prefix=$(cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix) + secret_token=$(cat $ARVADOS_CONTAINER_PATH/api_secret_token) + blob_signing_key=$(cat $ARVADOS_CONTAINER_PATH/blob_signing_key) + management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token) + database_pw=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) + vm_uuid=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid) -cat >config/application.yml <config/application.yml <config/database.yml + (cd config && /usr/local/lib/arvbox/yml_override.py application.yml) + sed "s/password:.*/password: $database_pw/" config/database.yml fi -if ! test -f /var/lib/arvados/api_database_setup ; then - bundle exec rake db:setup - touch /var/lib/arvados/api_database_setup +if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then + flock $GEM_HOME/gems.lock bundle exec rake db:setup + touch $ARVADOS_CONTAINER_PATH/api_database_setup fi -if ! test -s /var/lib/arvados/superuser_token ; then - superuser_tok=$(bundle exec ./script/create_superuser_token.rb) - echo "$superuser_tok" > /var/lib/arvados/superuser_token +if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then + superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb) + echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token fi rm -rf tmp mkdir -p tmp/cache -bundle exec rake db:migrate +flock $GEM_HOME/gems.lock bundle exec rake db:migrate diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh index 4798cb6ccd..708af17d5c 100755 --- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh +++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh @@ -6,7 +6,9 @@ exec 2>&1 set -ex -o pipefail -if [[ -s /etc/arvados/config.yml ]] && [[ /var/lib/arvados/cluster_config.yml.override -ot /etc/arvados/config.yml ]] ; then +export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox + +if [[ -s /etc/arvados/config.yml ]] && [[ $ARVADOS_CONTAINER_PATH/cluster_config.yml.override -ot /etc/arvados/config.yml ]] ; then exit fi @@ -14,63 +16,58 @@ fi set -u -if ! test -s /var/lib/arvados/api_uuid_prefix ; then - ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > /var/lib/arvados/api_uuid_prefix -fi -uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix) - -if ! test -s /var/lib/arvados/api_secret_token ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/api_secret_token +if ! test -s $ARVADOS_CONTAINER_PATH/api_uuid_prefix ; then + ruby -e 'puts "x#{rand(2**64).to_s(36)[0,4]}"' > $ARVADOS_CONTAINER_PATH/api_uuid_prefix fi -secret_token=$(cat /var/lib/arvados/api_secret_token) +uuid_prefix=$(cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix) -if ! test -s /var/lib/arvados/blob_signing_key ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/blob_signing_key +if ! test -s $ARVADOS_CONTAINER_PATH/api_secret_token ; then + ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/api_secret_token fi -blob_signing_key=$(cat /var/lib/arvados/blob_signing_key) +secret_token=$(cat $ARVADOS_CONTAINER_PATH/api_secret_token) -if ! test -s /var/lib/arvados/management_token ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/management_token +if ! test -s $ARVADOS_CONTAINER_PATH/blob_signing_key ; then + ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/blob_signing_key fi -management_token=$(cat /var/lib/arvados/management_token) +blob_signing_key=$(cat $ARVADOS_CONTAINER_PATH/blob_signing_key) -if ! test -s /var/lib/arvados/system_root_token ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/system_root_token +if ! test -s $ARVADOS_CONTAINER_PATH/management_token ; then + ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/management_token fi -system_root_token=$(cat /var/lib/arvados/system_root_token) +management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token) -if ! test -s /var/lib/arvados/sso_app_secret ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_app_secret +if ! test -s $ARVADOS_CONTAINER_PATH/system_root_token ; then + ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/system_root_token fi -sso_app_secret=$(cat /var/lib/arvados/sso_app_secret) +system_root_token=$(cat $ARVADOS_CONTAINER_PATH/system_root_token) -if ! test -s /var/lib/arvados/vm-uuid ; then - echo $uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]') > /var/lib/arvados/vm-uuid +if ! test -s $ARVADOS_CONTAINER_PATH/vm-uuid ; then + echo $uuid_prefix-2x53u-$(ruby -e 'puts rand(2**400).to_s(36)[0,15]') > $ARVADOS_CONTAINER_PATH/vm-uuid fi -vm_uuid=$(cat /var/lib/arvados/vm-uuid) +vm_uuid=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid) -if ! test -f /var/lib/arvados/api_database_pw ; then - ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw +if ! test -f $ARVADOS_CONTAINER_PATH/api_database_pw ; then + ruby -e 'puts rand(2**128).to_s(36)' > $ARVADOS_CONTAINER_PATH/api_database_pw fi -database_pw=$(cat /var/lib/arvados/api_database_pw) +database_pw=$(cat $ARVADOS_CONTAINER_PATH/api_database_pw) if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then psql postgres -c "create user arvados with password '$database_pw'" fi psql postgres -c "ALTER USER arvados WITH SUPERUSER;" -if ! test -s /var/lib/arvados/workbench_secret_token ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/workbench_secret_token +if ! test -s $ARVADOS_CONTAINER_PATH/workbench_secret_token ; then + ruby -e 'puts rand(2**400).to_s(36)' > $ARVADOS_CONTAINER_PATH/workbench_secret_token fi -workbench_secret_key_base=$(cat /var/lib/arvados/workbench_secret_token) +workbench_secret_key_base=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token) -if test -s /var/lib/arvados/api_rails_env ; then - database_env=$(cat /var/lib/arvados/api_rails_env) +if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then + database_env=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env) else database_env=development fi -cat >/var/lib/arvados/cluster_config.yml <$ARVADOS_CONTAINER_PATH/cluster_config.yml </var/lib/arvados/run_tests/config.yml <$ARVADOS_CONTAINER_PATH/run_tests/config.yml </dev/null)" = 0 ; then @@ -59,21 +61,27 @@ fi run_bundler() { if test -f Gemfile.lock ; then + # The 'gem install bundler line below' is cf. + # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html, + # until we get bundler 2.7.10/3.0.0 or higher + flock $GEM_HOME/gems.lock gem install bundler --no-document -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1|tr -d ' ')" frozen=--frozen else frozen="" fi - # if ! test -x /var/lib/gems/bin/bundler ; then + # if ! test -x $GEM_HOME/bin/bundler ; then # bundleversion=2.0.2 # bundlergem=$(ls -r $GEM_HOME/cache/bundler-${bundleversion}.gem 2>/dev/null | head -n1 || true) # if test -n "$bundlergem" ; then - # flock /var/lib/gems/gems.lock gem install --verbose --local --no-document $bundlergem + # flock $GEM_HOME/gems.lock gem install --verbose --local --no-document $bundlergem # else - # flock /var/lib/gems/gems.lock gem install --verbose --no-document bundler --version ${bundleversion} + # flock $GEM_HOME/gems.lock gem install --verbose --no-document bundler --version ${bundleversion} # fi # fi - if ! flock /var/lib/gems/gems.lock bundler install --verbose --path $GEM_HOME --local --no-deployment $frozen "$@" ; then - flock /var/lib/gems/gems.lock bundler install --verbose --path $GEM_HOME --no-deployment $frozen "$@" + # Make sure to put the gem binaries in the right place + flock /var/lib/arvados/lib/ruby/gems/2.5.0/gems.lock bundler config bin $GEM_HOME/bin + if ! flock $GEM_HOME/gems.lock bundler install --verbose --local --no-deployment $frozen "$@" ; then + flock $GEM_HOME/gems.lock bundler install --verbose --no-deployment $frozen "$@" fi } diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh index 58fb413582..7cf58e201d 100755 --- a/tools/arvbox/lib/arvbox/docker/createusers.sh +++ b/tools/arvbox/lib/arvbox/docker/createusers.sh @@ -5,16 +5,19 @@ set -e -o pipefail +export GEM_HOME=/var/lib/arvados/lib/ruby/gems/2.5.0 +export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox + if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then HOSTUID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f4) HOSTGID=$(ls -nd /usr/src/arvados | sed 's/ */ /' | cut -d' ' -f5) - mkdir -p /var/lib/arvados/git /var/lib/gems \ + mkdir -p $ARVADOS_CONTAINER_PATH/git $GEM_HOME \ /var/lib/passenger /var/lib/gopath \ /var/lib/pip /var/lib/npm if test -z "$ARVBOX_HOME" ; then - ARVBOX_HOME=/var/lib/arvados + ARVBOX_HOME=$ARVADOS_CONTAINER_PATH fi groupadd --gid $HOSTGID --non-unique arvbox @@ -25,28 +28,25 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then --groups docker \ --shell /bin/bash \ arvbox - useradd --home-dir /var/lib/arvados/git --uid $HOSTUID --gid $HOSTGID --non-unique git + useradd --home-dir $ARVADOS_CONTAINER_PATH/git --uid $HOSTUID --gid $HOSTGID --non-unique git useradd --groups docker crunch if [[ "$1" != --no-chown ]] ; then - chown arvbox:arvbox -R /usr/local /var/lib/arvados /var/lib/gems \ + chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH $GEM_HOME \ /var/lib/passenger /var/lib/postgresql \ /var/lib/nginx /var/log/nginx /etc/ssl/private \ - /var/lib/gopath /var/lib/pip /var/lib/npm + /var/lib/gopath /var/lib/pip /var/lib/npm \ + /var/lib/arvados fi - mkdir -p /var/lib/gems/ruby - chown arvbox:arvbox -R /var/lib/gems/ruby - mkdir -p /tmp/crunch0 /tmp/crunch1 chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1 echo "arvbox ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers cat < /etc/profile.d/paths.sh -export PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/go/bin:/var/lib/gems/bin:$(ls -d /usr/local/node-*)/bin -export GEM_HOME=/var/lib/gems -export GEM_PATH=/var/lib/gems +export PATH=/var/lib/arvados/bin:/usr/local/bin:/usr/bin:/bin +export GEM_HOME=/var/lib/arvados/lib/ruby/gems/2.5.0 export npm_config_cache=/var/lib/npm export npm_config_cache_min=Infinity export R_LIBS=/var/lib/Rlibs diff --git a/tools/arvbox/lib/arvbox/docker/devenv.sh b/tools/arvbox/lib/arvbox/docker/devenv.sh index 4df5463f1f..b5c57f39fc 100755 --- a/tools/arvbox/lib/arvbox/docker/devenv.sh +++ b/tools/arvbox/lib/arvbox/docker/devenv.sh @@ -3,7 +3,8 @@ # # SPDX-License-Identifier: AGPL-3.0 -flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh --no-chown +export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox +flock $ARVADOS_CONTAINER_PATH/createusers.lock /usr/local/lib/arvbox/createusers.sh --no-chown if [[ -n "$*" ]] ; then exec su --preserve-environment arvbox -c "$*" diff --git a/tools/arvbox/lib/arvbox/docker/edit_users.py b/tools/arvbox/lib/arvbox/docker/edit_users.py new file mode 100755 index 0000000000..ab046b11d4 --- /dev/null +++ b/tools/arvbox/lib/arvbox/docker/edit_users.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +import ruamel.yaml +import sys +import getpass +import os + +def print_help(): + print("%s add [pass]" % (sys.argv[0])) + print("%s remove " % (" " * len(sys.argv[0]))) + print("%s list" % (" " * len(sys.argv[0]))) + exit() + +if len(sys.argv) < 4: + print_help() + +fn = sys.argv[1] +cl = sys.argv[2] +op = sys.argv[3] + +if op == "remove" and len(sys.argv) < 5: + print_help() +if op == "add" and len(sys.argv) < 6: + print_help() + +if op in ("add", "remove"): + user = sys.argv[4] + +if not os.path.exists(fn): + open(fn, "w").close() + +with open(fn, "r") as f: + conf = ruamel.yaml.round_trip_load(f) + +if not conf: + conf = {} + +conf["Clusters"] = conf.get("Clusters", {}) +conf["Clusters"][cl] = conf["Clusters"].get(cl, {}) +conf["Clusters"][cl]["Login"] = conf["Clusters"][cl].get("Login", {}) +conf["Clusters"][cl]["Login"]["Test"] = conf["Clusters"][cl]["Login"].get("Test", {}) +conf["Clusters"][cl]["Login"]["Test"]["Users"] = conf["Clusters"][cl]["Login"]["Test"].get("Users", {}) + +users_obj = conf["Clusters"][cl]["Login"]["Test"]["Users"] + +if op == "add": + email = sys.argv[5] + if len(sys.argv) == 7: + p = sys.argv[6] + else: + p = getpass.getpass("Password for %s: " % user) + + users_obj[user] = { + "Email": email, + "Password": p + } + print("Added %s" % user) +elif op == "remove": + del users_obj[user] + print("Removed %s" % user) +elif op == "list": + print(ruamel.yaml.round_trip_dump(users_obj)) +else: + print("Operations are 'add', 'remove' and 'list'") + +with open(fn, "w") as f: + f.write(ruamel.yaml.round_trip_dump(conf)) diff --git a/tools/arvbox/lib/arvbox/docker/go-setup.sh b/tools/arvbox/lib/arvbox/docker/go-setup.sh index 9bee910448..5bdc5207a3 100644 --- a/tools/arvbox/lib/arvbox/docker/go-setup.sh +++ b/tools/arvbox/lib/arvbox/docker/go-setup.sh @@ -8,10 +8,11 @@ mkdir -p $GOPATH cd /usr/src/arvados if [[ $UID = 0 ]] ; then - /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go mod download - /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server -else - flock /var/lib/gopath/gopath.lock go mod download - flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server + RUNSU="/usr/local/lib/arvbox/runsu.sh" +fi + +if [[ ! -f /usr/local/bin/arvados-server ]]; then + $RUNSU flock /var/lib/gopath/gopath.lock go mod download + $RUNSU flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server + $RUNSU flock /var/lib/gopath/gopath.lock install $GOPATH/bin/arvados-server /usr/local/bin fi -install $GOPATH/bin/arvados-server /usr/local/bin diff --git a/tools/arvbox/lib/arvbox/docker/keep-setup.sh b/tools/arvbox/lib/arvbox/docker/keep-setup.sh index 3bc3899b0b..cb64f8406f 100755 --- a/tools/arvbox/lib/arvbox/docker/keep-setup.sh +++ b/tools/arvbox/lib/arvbox/docker/keep-setup.sh @@ -17,42 +17,6 @@ if test "$1" = "--only-deps" ; then exit fi -mkdir -p /var/lib/arvados/$1 +mkdir -p $ARVADOS_CONTAINER_PATH/$1 -export ARVADOS_API_HOST=$localip:${services[controller-ssl]} -export ARVADOS_API_HOST_INSECURE=1 -export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token) - -set +e -read -rd $'\000' keepservice < /var/lib/arvados/$1-uuid -fi - -management_token=$(cat /var/lib/arvados/management_token) - -set +e -sv hup /var/lib/arvbox/service/keepproxy - -cat >/var/lib/arvados/$1.yml </dev/null + if test -z "$1" ; then exec chpst -u arvbox:arvbox:docker $0-service else diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service index f052b5d636..d2691e7ed6 100755 --- a/tools/arvbox/lib/arvbox/docker/service/api/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service @@ -10,25 +10,27 @@ set -ex -o pipefail cd /usr/src/arvados/services/api -if test -s /var/lib/arvados/api_rails_env ; then - export RAILS_ENV=$(cat /var/lib/arvados/api_rails_env) +if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then + export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env) else export RAILS_ENV=development fi run_bundler --without=development -bundle exec passenger-config build-native-support -bundle exec passenger-config install-standalone-runtime +flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support +flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime if test "$1" = "--only-deps" ; then exit fi -flock /var/lib/arvados/api.lock /usr/local/lib/arvbox/api-setup.sh +flock $ARVADOS_CONTAINER_PATH/api.lock /usr/local/lib/arvbox/api-setup.sh set +u if test "$1" = "--only-setup" ; then exit fi +touch $ARVADOS_CONTAINER_PATH/api.ready + exec bundle exec passenger start --port=${services[api]} diff --git a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service index 5f71e5ab28..b369ff6228 100755 --- a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service @@ -18,7 +18,7 @@ fi export ARVADOS_API_HOST=$localip:${services[controller-ssl]} export ARVADOS_API_HOST_INSECURE=1 -export PATH="$PATH:/var/lib/arvados/git/bin" +export PATH="$PATH:$ARVADOS_CONTAINER_PATH/git/bin" cd ~git exec /usr/local/bin/arv-git-httpd diff --git a/tools/arvbox/lib/arvbox/docker/service/certificate/run b/tools/arvbox/lib/arvbox/docker/service/certificate/run index 6443b01793..2536981a7a 100755 --- a/tools/arvbox/lib/arvbox/docker/service/certificate/run +++ b/tools/arvbox/lib/arvbox/docker/service/certificate/run @@ -8,9 +8,9 @@ set -ex -o pipefail . /usr/local/lib/arvbox/common.sh -/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh +/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh -uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix) +uuid_prefix=$(cat $ARVADOS_CONTAINER_PATH/api_uuid_prefix) if ! openssl verify -CAfile $root_cert $root_cert ; then # req signing request sub-command @@ -74,13 +74,13 @@ if ! openssl verify -CAfile $root_cert $server_cert ; then -extensions x509_ext \ -config <(cat /etc/ssl/openssl.cnf \ <(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \ - -out /var/lib/arvados/server-cert-${localip}.csr \ + -out $ARVADOS_CONTAINER_PATH/server-cert-${localip}.csr \ -keyout $server_cert_key \ -days 365 openssl x509 \ -req \ - -in /var/lib/arvados/server-cert-${localip}.csr \ + -in $ARVADOS_CONTAINER_PATH/server-cert-${localip}.csr \ -CA $root_cert \ -CAkey $root_cert_key \ -out $server_cert \ diff --git a/tools/arvbox/lib/arvbox/docker/service/controller/run b/tools/arvbox/lib/arvbox/docker/service/controller/run index 588e9d2dad..e495e222e1 100755 --- a/tools/arvbox/lib/arvbox/docker/service/controller/run +++ b/tools/arvbox/lib/arvbox/docker/service/controller/run @@ -15,6 +15,6 @@ if test "$1" = "--only-deps" ; then exit fi -/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh +/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh exec /usr/local/bin/arvados-controller diff --git a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service index 6e80d30ab9..821afdce50 100755 --- a/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/crunch-dispatch-local/run-service @@ -25,6 +25,6 @@ chmod +x /usr/local/bin/crunch-run.sh export ARVADOS_API_HOST=$localip:${services[controller-ssl]} export ARVADOS_API_HOST_INSECURE=1 -export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token) +export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token) exec /usr/local/bin/crunch-dispatch-local -crunch-run-command=/usr/local/bin/crunch-run.sh -poll-interval=1 diff --git a/tools/arvbox/lib/arvbox/docker/service/doc/run-service b/tools/arvbox/lib/arvbox/docker/service/doc/run-service index 66a4a28ec5..36566c9d9b 100755 --- a/tools/arvbox/lib/arvbox/docker/service/doc/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/doc/run-service @@ -8,6 +8,11 @@ set -ex -o pipefail . /usr/local/lib/arvbox/common.sh +if test "$1" != "--only-deps" ; then + while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do + sleep 1 + done +fi cd /usr/src/arvados/doc run_bundler --without=development @@ -24,4 +29,4 @@ if test "$1" = "--only-deps" ; then fi cd /usr/src/arvados/doc -bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip +flock $GEM_HOME/gems.lock bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip diff --git a/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service b/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service index 6055efc479..c60c15bfc5 100755 --- a/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service @@ -8,16 +8,22 @@ set -eux -o pipefail . /usr/local/lib/arvbox/common.sh -mkdir -p /var/lib/arvados/git +if test "$1" != "--only-deps" ; then + while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do + sleep 1 + done +fi + +mkdir -p $ARVADOS_CONTAINER_PATH/git export ARVADOS_API_HOST=$localip:${services[controller-ssl]} export ARVADOS_API_HOST_INSECURE=1 -export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token) +export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token) export USER=git export USERNAME=git export LOGNAME=git -export HOME=/var/lib/arvados/git +export HOME=$ARVADOS_CONTAINER_PATH/git cd ~arvbox @@ -33,7 +39,7 @@ if test -s ~arvbox/.ssh/known_hosts ; then ssh-keygen -f ".ssh/known_hosts" -R localhost fi -if ! test -f /var/lib/arvados/gitolite-setup ; then +if ! test -f $ARVADOS_CONTAINER_PATH/gitolite-setup ; then cd ~git # Do a no-op login to populate known_hosts @@ -57,7 +63,7 @@ if ! test -f /var/lib/arvados/gitolite-setup ; then git config push.default simple git push - touch /var/lib/arvados/gitolite-setup + touch $ARVADOS_CONTAINER_PATH/gitolite-setup else # Do a no-op login to populate known_hosts # with the hostkey, so it won't try to ask @@ -68,14 +74,14 @@ fi prefix=$(arv --format=uuid user current | cut -d- -f1) -if ! test -s /var/lib/arvados/arvados-git-uuid ; then +if ! test -s $ARVADOS_CONTAINER_PATH/arvados-git-uuid ; then repo_uuid=$(arv --format=uuid repository create --repository "{\"owner_uuid\":\"$prefix-tpzed-000000000000000\", \"name\":\"arvados\"}") - echo $repo_uuid > /var/lib/arvados/arvados-git-uuid + echo $repo_uuid > $ARVADOS_CONTAINER_PATH/arvados-git-uuid fi -repo_uuid=$(cat /var/lib/arvados/arvados-git-uuid) +repo_uuid=$(cat $ARVADOS_CONTAINER_PATH/arvados-git-uuid) -if ! test -s /var/lib/arvados/arvados-git-link-uuid ; then +if ! test -s $ARVADOS_CONTAINER_PATH/arvados-git-link-uuid ; then all_users_group_uuid="$prefix-j7d0g-fffffffffffffff" set +e @@ -89,19 +95,19 @@ if ! test -s /var/lib/arvados/arvados-git-link-uuid ; then EOF set -e link_uuid=$(arv --format=uuid link create --link "$newlink") - echo $link_uuid > /var/lib/arvados/arvados-git-link-uuid + echo $link_uuid > $ARVADOS_CONTAINER_PATH/arvados-git-link-uuid fi -if ! test -d /var/lib/arvados/git/repositories/$repo_uuid.git ; then - git clone --bare /usr/src/arvados /var/lib/arvados/git/repositories/$repo_uuid.git +if ! test -d $ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git ; then + git clone --bare /usr/src/arvados $ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git else - git --git-dir=/var/lib/arvados/git/repositories/$repo_uuid.git fetch -f /usr/src/arvados master:master + git --git-dir=$ARVADOS_CONTAINER_PATH/git/repositories/$repo_uuid.git fetch -f /usr/src/arvados master:master fi cd /usr/src/arvados/services/api -if test -s /var/lib/arvados/api_rails_env ; then - RAILS_ENV=$(cat /var/lib/arvados/api_rails_env) +if test -s $ARVADOS_CONTAINER_PATH/api_rails_env ; then + RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/api_rails_env) else RAILS_ENV=development fi @@ -110,8 +116,8 @@ git_user_key=$(cat ~git/.ssh/id_rsa.pub) cat > config/arvados-clients.yml < /var/lib/arvados/keepproxy-uuid -fi - exec /usr/local/bin/keepproxy diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/run b/tools/arvbox/lib/arvbox/docker/service/nginx/run index d6fecb4436..991927be70 100755 --- a/tools/arvbox/lib/arvbox/docker/service/nginx/run +++ b/tools/arvbox/lib/arvbox/docker/service/nginx/run @@ -21,9 +21,9 @@ fi openssl verify -CAfile $root_cert $server_cert -cat </var/lib/arvados/nginx.conf +cat <$ARVADOS_CONTAINER_PATH/nginx.conf worker_processes auto; -pid /var/lib/arvados/nginx.pid; +pid $ARVADOS_CONTAINER_PATH/nginx.pid; error_log stderr; daemon off; @@ -144,6 +144,20 @@ http { proxy_redirect off; } } + server { + listen *:${services[keep-web-dl-ssl]} ssl default_server; + server_name keep-web-dl; + ssl_certificate "${server_cert}"; + ssl_certificate_key "${server_cert_key}"; + client_max_body_size 0; + location / { + proxy_pass http://keep-web; + proxy_set_header Host \$http_host; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_redirect off; + } + } upstream keepproxy { server localhost:${services[keepproxy]}; @@ -186,8 +200,53 @@ http { } } + +upstream arvados-webshell { + server localhost:${services[webshell]}; +} +server { + listen ${services[webshell-ssl]} ssl; + server_name arvados-webshell; + + proxy_connect_timeout 90s; + proxy_read_timeout 300s; + + ssl on; + ssl_certificate "${server_cert}"; + ssl_certificate_key "${server_cert_key}"; + + location / { + if (\$request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + if (\$request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + } + if (\$request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + } + + proxy_ssl_session_reuse off; + proxy_read_timeout 90; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host \$http_host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_pass http://arvados-webshell; + } +} } EOF -exec nginx -c /var/lib/arvados/nginx.conf +exec nginx -c $ARVADOS_CONTAINER_PATH/nginx.conf diff --git a/tools/arvbox/lib/arvbox/docker/service/postgres/run b/tools/arvbox/lib/arvbox/docker/service/postgres/run index 3ef78ee455..d8abc4d89d 100755 --- a/tools/arvbox/lib/arvbox/docker/service/postgres/run +++ b/tools/arvbox/lib/arvbox/docker/service/postgres/run @@ -3,7 +3,8 @@ # # SPDX-License-Identifier: AGPL-3.0 -flock /var/lib/arvados/createusers.lock /usr/local/lib/arvbox/createusers.sh +export ARVADOS_CONTAINER_PATH=/var/lib/arvados-arvbox +flock $ARVADOS_CONTAINER_PATH/createusers.lock /usr/local/lib/arvbox/createusers.sh make-ssl-cert generate-default-snakeoil --force-overwrite diff --git a/tools/arvbox/lib/arvbox/docker/service/postgres/run-service b/tools/arvbox/lib/arvbox/docker/service/postgres/run-service index a0771aa6a0..3569fd3126 100755 --- a/tools/arvbox/lib/arvbox/docker/service/postgres/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/postgres/run-service @@ -6,11 +6,10 @@ exec 2>&1 set -eux -o pipefail -PGVERSION=9.6 +PGVERSION=11 if ! test -d /var/lib/postgresql/$PGVERSION/main ; then /usr/lib/postgresql/$PGVERSION/bin/initdb --locale=en_US.UTF-8 -D /var/lib/postgresql/$PGVERSION/main - sh -c "while ! (psql postgres -c'\du' | grep '^ arvbox ') >/dev/null ; do createuser -s arvbox ; sleep 1 ; done" & fi mkdir -p /var/run/postgresql/$PGVERSION-main.pg_stat_tmp diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service index 470d105375..b29dafed70 100755 --- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service @@ -49,9 +49,9 @@ export ARVADOS_API_HOST=$localip:${services[controller-ssl]} export ARVADOS_API_HOST_INSECURE=1 vm_ok=0 -if test -s /var/lib/arvados/vm-uuid -a -s /var/lib/arvados/superuser_token; then - vm_uuid=$(cat /var/lib/arvados/vm-uuid) - export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token) +if test -s $ARVADOS_CONTAINER_PATH/vm-uuid -a -s $ARVADOS_CONTAINER_PATH/superuser_token; then + vm_uuid=$(cat $ARVADOS_CONTAINER_PATH/vm-uuid) + export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token) if (which arv && arv virtual_machine get --uuid $vm_uuid) >/dev/null 2>/dev/null ; then vm_ok=1 fi @@ -63,12 +63,11 @@ fi if ! [[ -z "$waiting" ]] ; then if ps x | grep -v grep | grep "bundle install" > /dev/null; then - gemcount=$(ls /var/lib/gems/ruby/2.1.0/gems 2>/dev/null | wc -l) + gemcount=$(ls $GEM_HOME/gems 2>/dev/null | wc -l) gemlockcount=0 for l in /usr/src/arvados/services/api/Gemfile.lock \ - /usr/src/arvados/apps/workbench/Gemfile.lock \ - /usr/src/sso/Gemfile.lock ; do + /usr/src/arvados/apps/workbench/Gemfile.lock ; do gc=$(cat $l \ | grep -vE "(GEM|PLATFORMS|DEPENDENCIES|BUNDLED|GIT|$^|remote:|specs:|revision:)" \ | sed 's/^ *//' | sed 's/(.*)//' | sed 's/ *$//' | sort | uniq | wc -l) diff --git a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service index 8a36140bcf..d66bf315b1 100755 --- a/tools/arvbox/lib/arvbox/docker/service/sdk/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/sdk/run-service @@ -20,30 +20,16 @@ ln -sf /usr/src/arvados/sdk/cli/binstubs/arv /usr/local/bin/arv export PYCMD=python3 -# Need to install the upstream version of pip because the python-pip package -# shipped with Debian 9 is patched to change behavior in a way that breaks our -# use case. -# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=876145 -# When a non-root user attempts to install system packages, it makes the -# --ignore-installed flag the default (and there is no way to turn it off), -# this has the effect of making it very hard to share dependencies shared among -# multiple packages, because it will blindly install the latest version of each -# dependency requested by each package, even if a compatible package version is -# already installed. -if ! pip3 install --no-index --find-links /var/lib/pip pip==9.0.3 ; then - pip3 install pip==9.0.3 -fi - pip_install wheel cd /usr/src/arvados/sdk/python -python setup.py sdist +$PYCMD setup.py sdist pip_install $(ls dist/arvados-python-client-*.tar.gz | tail -n1) cd /usr/src/arvados/services/fuse -python setup.py sdist +$PYCMD setup.py sdist pip_install $(ls dist/arvados_fuse-*.tar.gz | tail -n1) cd /usr/src/arvados/sdk/cwl -python setup.py sdist +$PYCMD setup.py sdist pip_install $(ls dist/arvados-cwl-runner-*.tar.gz | tail -n1) diff --git a/tools/arvbox/lib/arvbox/docker/service/sso/run b/tools/arvbox/lib/arvbox/docker/service/sso/run deleted file mode 120000 index a388c8b67b..0000000000 --- a/tools/arvbox/lib/arvbox/docker/service/sso/run +++ /dev/null @@ -1 +0,0 @@ -/usr/local/lib/arvbox/runsu.sh \ No newline at end of file diff --git a/tools/arvbox/lib/arvbox/docker/service/sso/run-service b/tools/arvbox/lib/arvbox/docker/service/sso/run-service deleted file mode 100755 index e30e34f7c1..0000000000 --- a/tools/arvbox/lib/arvbox/docker/service/sso/run-service +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -exec 2>&1 -set -ex -o pipefail - -. /usr/local/lib/arvbox/common.sh - -cd /usr/src/sso -if test -s /var/lib/arvados/sso_rails_env ; then - export RAILS_ENV=$(cat /var/lib/arvados/sso_rails_env) -else - export RAILS_ENV=development -fi - -run_bundler --without=development -bundle exec passenger-config build-native-support -bundle exec passenger-config install-standalone-runtime - -if test "$1" = "--only-deps" ; then - exit -fi - -set -u - -uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix) - -if ! test -s /var/lib/arvados/sso_secret_token ; then - ruby -e 'puts rand(2**400).to_s(36)' > /var/lib/arvados/sso_secret_token -fi -secret_token=$(cat /var/lib/arvados/sso_secret_token) - -openssl verify -CAfile $root_cert $server_cert - -cat >config/application.yml < /var/lib/arvados/sso_database_pw -fi -database_pw=$(cat /var/lib/arvados/sso_database_pw) - -if ! (psql postgres -c "\du" | grep "^ arvados_sso ") >/dev/null ; then - psql postgres -c "create user arvados_sso with password '$database_pw'" - psql postgres -c "ALTER USER arvados_sso CREATEDB;" -fi - -sed "s/password:.*/password: $database_pw/" config/database.yml - -if ! test -f /var/lib/arvados/sso_database_setup ; then - bundle exec rake db:setup - - app_secret=$(cat /var/lib/arvados/sso_app_secret) - - bundle exec rails console <&1 +set -ex -o pipefail + +. /usr/local/lib/arvbox/common.sh + +/usr/local/lib/arvbox/runsu.sh $0-service + +cat > /etc/pam.d/shellinabox <&1 +set -ex -o pipefail + +. /usr/local/lib/arvbox/common.sh +. /usr/local/lib/arvbox/go-setup.sh + +flock /var/lib/gopath/gopath.lock go build -buildmode=c-shared -o ${GOPATH}/bin/pam_arvados.so git.arvados.org/arvados.git/lib/pam +install $GOPATH/bin/pam_arvados.so /usr/local/lib \ No newline at end of file diff --git a/tools/arvbox/lib/arvbox/docker/service/websockets/run b/tools/arvbox/lib/arvbox/docker/service/websockets/run index efa2e08a7a..f962c3e8f0 100755 --- a/tools/arvbox/lib/arvbox/docker/service/websockets/run +++ b/tools/arvbox/lib/arvbox/docker/service/websockets/run @@ -15,6 +15,6 @@ if test "$1" = "--only-deps" ; then exit fi -/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh +/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-ws diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run b/tools/arvbox/lib/arvbox/docker/service/workbench/run index e163493781..b8a28fa762 100755 --- a/tools/arvbox/lib/arvbox/docker/service/workbench/run +++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run @@ -15,8 +15,8 @@ rm -rf tmp mkdir tmp chown arvbox:arvbox tmp -if test -s /var/lib/arvados/workbench_rails_env ; then - export RAILS_ENV=$(cat /var/lib/arvados/workbench_rails_env) +if test -s $ARVADOS_CONTAINER_PATH/workbench_rails_env ; then + export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/workbench_rails_env) else export RAILS_ENV=development fi @@ -24,7 +24,7 @@ fi if test "$1" != "--only-deps" ; then openssl verify -CAfile $root_cert $server_cert exec bundle exec passenger start --port=${services[workbench]} \ - --ssl --ssl-certificate=/var/lib/arvados/server-cert-${localip}.pem \ - --ssl-certificate-key=/var/lib/arvados/server-cert-${localip}.key \ + --ssl --ssl-certificate=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem \ + --ssl-certificate-key=$ARVADOS_CONTAINER_PATH/server-cert-${localip}.key \ --user arvbox fi diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service index 06742cf82e..32efea51b1 100755 --- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service @@ -8,17 +8,23 @@ set -ex -o pipefail . /usr/local/lib/arvbox/common.sh +if test "$1" != "--only-deps" ; then + while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do + sleep 1 + done +fi + cd /usr/src/arvados/apps/workbench -if test -s /var/lib/arvados/workbench_rails_env ; then - export RAILS_ENV=$(cat /var/lib/arvados/workbench_rails_env) +if test -s $ARVADOS_CONTAINER_PATH/workbench_rails_env ; then + export RAILS_ENV=$(cat $ARVADOS_CONTAINER_PATH/workbench_rails_env) else export RAILS_ENV=development fi run_bundler --without=development -bundle exec passenger-config build-native-support -bundle exec passenger-config install-standalone-runtime +flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support +flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime mkdir -p /usr/src/arvados/apps/workbench/tmp if test "$1" = "--only-deps" ; then @@ -28,34 +34,14 @@ cat >config/application.yml <config/application.yml < /usr/src/workbench2/public/config.json EOF export ARVADOS_API_HOST=$localip:${services[controller-ssl]} -export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token) +export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token) url_prefix="https://$localip:${services[workbench2-ssl]}/" diff --git a/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh b/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh index 6bda618ab8..9b2eb69f9e 100755 --- a/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh +++ b/tools/arvbox/lib/arvbox/docker/waitforpostgres.sh @@ -9,6 +9,6 @@ while ! psql postgres -c\\du >/dev/null 2>/dev/null ; do sleep 1 done -while ! test -s /var/lib/arvados/server-cert-${localip}.pem ; do +while ! test -s $ARVADOS_CONTAINER_PATH/server-cert-${localip}.pem ; do sleep 1 done diff --git a/tools/arvbox/lib/arvbox/docker/yml_override.py b/tools/arvbox/lib/arvbox/docker/yml_override.py index b44acf4c3a..7f35ac1d68 100755 --- a/tools/arvbox/lib/arvbox/docker/yml_override.py +++ b/tools/arvbox/lib/arvbox/docker/yml_override.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 @@ -20,7 +20,7 @@ with open(fn) as f: def recursiveMerge(a, b): if isinstance(a, dict) and isinstance(b, dict): for k in b: - print k + print(k) a[k] = recursiveMerge(a.get(k), b[k]) return a else: diff --git a/tools/copy-tutorial/copy-tutorial.sh b/tools/copy-tutorial/copy-tutorial.sh new file mode 100755 index 0000000000..e7fac7af48 --- /dev/null +++ b/tools/copy-tutorial/copy-tutorial.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +set -e -o pipefail + +if test -z "$1" ; then + echo "$0: Copies Arvados tutorial resources from public data cluster (jutro)" + echo "Usage: copy-tutorial.sh " + echo " is which tutorial to copy, one of:" + echo " bwa-mem Tutorial from https://doc.arvados.org/user/tutorials/tutorial-workflow-workbench.html" + echo " whole-genome Whole genome variant calling tutorial workflow (large)" + exit +fi + +if test -z "ARVADOS_API_HOST" ; then + echo "Please set ARVADOS_API_HOST to the destination cluster" + exit +fi + +src=jutro +tutorial=$1 + +if ! test -f $HOME/.config/arvados/jutro.conf ; then + # Set it up with the anonymous user token. + echo "ARVADOS_API_HOST=jutro.arvadosapi.com" > $HOME/.config/arvados/jutro.conf + echo "ARVADOS_API_TOKEN=v2/jutro-gj3su-e2o9x84aeg7q005/22idg1m3zna4qe4id3n0b9aw86t72jdw8qu1zj45aboh1mm4ej" >> $HOME/.config/arvados/jutro.conf + exit 1 +fi + +echo +echo "Copying from public data cluster (jutro) to $ARVADOS_API_HOST" +echo + +make_project() { + name="$1" + owner="$2" + if test -z "$owner" ; then + owner=$(arv --format=uuid user current) + fi + project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "'"$name"'"], ["owner_uuid", "=", "'$owner'"]]') + if test -z "$project_uuid" ; then + project_uuid=$(arv --format=uuid group create --group '{"name":"'"$name"'", "group_class": "project", "owner_uuid": "'$owner'"}') + + fi + echo $project_uuid +} + +copy_jobs_image() { + if ! arv-keepdocker | grep "arvados/jobs *latest" ; then + arv-copy --project-uuid=$parent_project jutro-4zz18-sxmit0qs6i9n2s4 + fi +} + +parent_project=$(make_project "Tutorial projects") +copy_jobs_image + +if test "$tutorial" = "bwa-mem" ; then + echo + echo "Copying bwa mem tutorial" + echo + + arv-copy --project-uuid=$parent_project jutro-j7d0g-rehmt1w5v2p2drp + + echo + echo "Finished, data copied to \"User guide resources\" at $parent_project" + echo "You can now go to Workbench and choose 'Run a process' and then select 'bwa-mem.cwl'" + echo +fi + +if test "$tutorial" = "whole-genome" ; then + echo + echo "Copying whole genome variant calling tutorial" + echo + + arv-copy --project-uuid=$parent_project jutro-j7d0g-n2g87m02rsl4cx2 + + echo + echo "Finished, data copied to \"WGS Processing Tutorial\" at $parent_project" + echo "You can now go to Workbench and choose 'Run a process' and then select 'WGS Processing Tutorial'" + echo +fi diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py index 0c653694f5..d8eec3d9ee 100644 --- a/tools/crunchstat-summary/arvados_version.py +++ b/tools/crunchstat-summary/arvados_version.py @@ -6,36 +6,42 @@ import subprocess import time import os import re +import sys SETUP_DIR = os.path.dirname(os.path.abspath(__file__)) +VERSION_PATHS = { + SETUP_DIR, + os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")), + os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh")) + } def choose_version_from(): - sdk_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip() - cwl_ts = subprocess.check_output( - ['git', 'log', '--first-parent', '--max-count=1', - '--format=format:%ct', SETUP_DIR]).strip() - if int(sdk_ts) > int(cwl_ts): - getver = os.path.join(SETUP_DIR, "../../sdk/python") - else: - getver = SETUP_DIR + ts = {} + for path in VERSION_PATHS: + ts[subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', path]).strip()] = path + + sorted_ts = sorted(ts.items()) + getver = sorted_ts[-1][1] + print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr) return getver def git_version_at_commit(): curdir = choose_version_from() myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', '--format=%H', curdir]).strip() - myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode() + myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode() return myversion def save_version(setup_dir, module, v): - with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: - return fp.write("__version__ = '%s'\n" % v) + v = v.replace("~dev", ".dev").replace("~rc", "rc") + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: + return fp.write("__version__ = '%s'\n" % v) def read_version(setup_dir, module): - with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: - return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: + return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] def get_version(setup_dir, module): env_version = os.environ.get("ARVADOS_BUILDING_VERSION") @@ -45,7 +51,8 @@ def get_version(setup_dir, module): else: try: save_version(setup_dir, module, git_version_at_commit()) - except (subprocess.CalledProcessError, OSError): + except (subprocess.CalledProcessError, OSError) as err: + print("ERROR: {0}".format(err), file=sys.stderr) pass return read_version(setup_dir, module) diff --git a/tools/crunchstat-summary/bin/crunchstat-summary b/tools/crunchstat-summary/bin/crunchstat-summary index 0ccb898a15..3c18829189 100755 --- a/tools/crunchstat-summary/bin/crunchstat-summary +++ b/tools/crunchstat-summary/bin/crunchstat-summary @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 diff --git a/tools/crunchstat-summary/gittaggers.py b/tools/crunchstat-summary/gittaggers.py deleted file mode 120000 index a9ad861d81..0000000000 --- a/tools/crunchstat-summary/gittaggers.py +++ /dev/null @@ -1 +0,0 @@ -../../sdk/python/gittaggers.py \ No newline at end of file diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py index 557b6d3f4e..8507990f5a 100755 --- a/tools/crunchstat-summary/setup.py +++ b/tools/crunchstat-summary/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: AGPL-3.0 diff --git a/tools/keep-xref/keep-xref.py b/tools/keep-xref/keep-xref.py index 7bc4158928..d77e593640 100755 --- a/tools/keep-xref/keep-xref.py +++ b/tools/keep-xref/keep-xref.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) The Arvados Authors. All rights reserved. # diff --git a/tools/salt-install/README.md b/tools/salt-install/README.md new file mode 100644 index 0000000000..10d08b414a --- /dev/null +++ b/tools/salt-install/README.md @@ -0,0 +1,20 @@ +[comment]: # (Copyright © The Arvados Authors. All rights reserved.) +[comment]: # () +[comment]: # (SPDX-License-Identifier: CC-BY-SA-3.0) + +# Arvados install with Saltstack + +##### About + +This directory holds a small script to install Arvados on a single node, using the +[Saltstack arvados-formula](https://github.com/saltstack-formulas/arvados-formula) +in master-less mode. + +The fastest way to get it running is to modify the first lines in the `provision.sh` +script to suit your needs, copy it in the host where you want to install Arvados +and run it as root. + +There's an example `Vagrantfile` also, to install it in a vagrant box if you want +to try it locally. + +For more information, please read https://doc.arvados.org/main/install/salt-single-host.html diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile new file mode 100644 index 0000000000..6966ea8345 --- /dev/null +++ b/tools/salt-install/Vagrantfile @@ -0,0 +1,42 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Vagrantfile API/syntax version. Don"t touch unless you know what you"re doing! +VAGRANTFILE_API_VERSION = "2".freeze + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.ssh.insert_key = false + config.ssh.forward_x11 = true + + config.vm.define "arvados" do |arv| + arv.vm.box = "bento/debian-10" + arv.vm.hostname = "vagrant.local" + # CPU/RAM + config.vm.provider :virtualbox do |v| + v.memory = 2048 + v.cpus = 2 + end + + # Networking + arv.vm.network "forwarded_port", guest: 8443, host: 8443 + arv.vm.network "forwarded_port", guest: 25100, host: 25100 + arv.vm.network "forwarded_port", guest: 9002, host: 9002 + arv.vm.network "forwarded_port", guest: 9000, host: 9000 + arv.vm.network "forwarded_port", guest: 8900, host: 8900 + arv.vm.network "forwarded_port", guest: 8002, host: 8002 + arv.vm.network "forwarded_port", guest: 8001, host: 8001 + arv.vm.network "forwarded_port", guest: 8000, host: 8000 + arv.vm.network "forwarded_port", guest: 3001, host: 3001 + arv.vm.provision "shell", + path: "provision.sh", + args: [ + # "--debug", + "--test", + "--vagrant", + "--ssl-port=8443" + ].join(" ") + end +end diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh new file mode 100755 index 0000000000..a4d55c2216 --- /dev/null +++ b/tools/salt-install/provision.sh @@ -0,0 +1,285 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: CC-BY-SA-3.0 + +# If you want to test arvados in a single host, you can run this script, which +# will install it using salt masterless +# This script is run by the Vagrant file when you run it with +# +# vagrant up + +########################################################## +# This section are the basic parameters to configure the installation + +# The 5 letters name you want to give your cluster +CLUSTER="arva2" +DOMAIN="arv.local" + +INITIAL_USER="admin" + +# If not specified, the initial user email will be composed as +# INITIAL_USER@CLUSTER.DOMAIN +INITIAL_USER_EMAIL="${INITIAL_USER}@${CLUSTER}.${DOMAIN}" +INITIAL_USER_PASSWORD="password" + +# The example config you want to use. Currently, only "single_host" is +# available +CONFIG_DIR="single_host" + +# Which release of Arvados repo you want to use +RELEASE="production" +# Which version of Arvados you want to install. Defaults to 'latest' +# in the desired repo +VERSION="latest" + +# Host SSL port where you want to point your browser to access Arvados +# Defaults to 443 for regular runs, and to 8443 when called in Vagrant. +# You can point it to another port if desired +# In Vagrant, make sure it matches what you set in the Vagrantfile +# HOST_SSL_PORT=443 + +# This is a arvados-formula setting. +# If branch is set, the script will switch to it before running salt +# Usually not needed, only used for testing +# BRANCH="master" + +########################################################## +# Usually there's no need to modify things below this line + +# Formulas versions +ARVADOS_TAG="v1.1.3" +POSTGRES_TAG="v0.41.3" +NGINX_TAG="v2.4.0" +DOCKER_TAG="v1.0.0" +LOCALE_TAG="v0.3.4" + +set -o pipefail + +# capture the directory that the script is running from +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +usage() { + echo >&2 + echo >&2 "Usage: ${0} [-h] [-h]" + echo >&2 + echo >&2 "${0} options:" + echo >&2 " -d, --debug Run salt installation in debug mode" + echo >&2 " -p , --ssl-port SSL port to use for the web applications" + echo >&2 " -t, --test Test installation running a CWL workflow" + echo >&2 " -h, --help Display this help and exit" + echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir" + echo >&2 +} + +arguments() { + # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros). + TEMP=$(getopt -o dhp:tv \ + --long debug,help,ssl-port:,test,vagrant \ + -n "${0}" -- "${@}") + + if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi + # Note the quotes around `$TEMP': they are essential! + eval set -- "$TEMP" + + while [ ${#} -ge 1 ]; do + case ${1} in + -d | --debug) + LOG_LEVEL="debug" + shift + ;; + -t | --test) + TEST="yes" + shift + ;; + -v | --vagrant) + VAGRANT="yes" + shift + ;; + -p | --ssl-port) + HOST_SSL_PORT=${2} + shift 2 + ;; + --) + shift + break + ;; + *) + usage + exit 1 + ;; + esac + done +} + +LOG_LEVEL="info" +HOST_SSL_PORT=443 +TESTS_DIR="tests" + +arguments ${@} + +# Salt's dir +## states +S_DIR="/srv/salt" +## formulas +F_DIR="/srv/formulas" +##pillars +P_DIR="/srv/pillars" + +apt-get update +apt-get install -y curl git jq + +dpkg -l |grep salt-minion +if [ ${?} -eq 0 ]; then + echo "Salt already installed" +else + curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh + sh /tmp/bootstrap_salt.sh -XUdfP -x python3 + /bin/systemctl disable salt-minion.service +fi + +# Set salt to masterless mode +cat > /etc/salt/minion << EOFSM +file_client: local +file_roots: + base: + - ${S_DIR} + - ${F_DIR}/* + - ${F_DIR}/*/test/salt/states/examples + +pillar_roots: + base: + - ${P_DIR} +EOFSM + +mkdir -p ${S_DIR} +mkdir -p ${F_DIR} +mkdir -p ${P_DIR} + +# States +cat > ${S_DIR}/top.sls << EOFTSLS +base: + '*': + - single_host.host_entries + - single_host.snakeoil_certs + - locale + - nginx.passenger + - postgres + - docker + - arvados +EOFTSLS + +# Pillars +cat > ${P_DIR}/top.sls << EOFPSLS +base: + '*': + - arvados + - docker + - locale + - nginx_api_configuration + - nginx_controller_configuration + - nginx_keepproxy_configuration + - nginx_keepweb_configuration + - nginx_passenger + - nginx_websocket_configuration + - nginx_webshell_configuration + - nginx_workbench2_configuration + - nginx_workbench_configuration + - postgresql +EOFPSLS + +# Get the formula and dependencies +cd ${F_DIR} || exit 1 +git clone --branch "${ARVADOS_TAG}" https://github.com/saltstack-formulas/arvados-formula.git +git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git +git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git +git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git +git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git + +if [ "x${BRANCH}" != "x" ]; then + cd ${F_DIR}/arvados-formula || exit 1 + git checkout -t origin/"${BRANCH}" + cd - +fi + +if [ "x${VAGRANT}" = "xyes" ]; then + SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}" + TESTS_DIR="/vagrant/${TESTS_DIR}" +else + SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}" + TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}" +fi + +# Replace cluster and domain name in the example pillars and test files +for f in "${SOURCE_PILLARS_DIR}"/*; do + sed "s/__CLUSTER__/${CLUSTER}/g; + s/__DOMAIN__/${DOMAIN}/g; + s/__RELEASE__/${RELEASE}/g; + s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g; + s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g; + s/__INITIAL_USER__/${INITIAL_USER}/g; + s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g; + s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g; + s/__VERSION__/${VERSION}/g" \ + "${f}" > "${P_DIR}"/$(basename "${f}") +done + +mkdir -p /tmp/cluster_tests +# Replace cluster and domain name in the example pillars and test files +for f in "${TESTS_DIR}"/*; do + sed "s/__CLUSTER__/${CLUSTER}/g; + s/__DOMAIN__/${DOMAIN}/g; + s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g; + s/__INITIAL_USER__/${INITIAL_USER}/g; + s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g; + s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \ + ${f} > /tmp/cluster_tests/$(basename ${f}) +done +chmod 755 /tmp/cluster_tests/run-test.sh + +# FIXME! #16992 Temporary fix for psql call in arvados-api-server +if [ -e /root/.psqlrc ]; then + if ! ( grep 'pset pager off' /root/.psqlrc ); then + RESTORE_PSQL="yes" + cp /root/.psqlrc /root/.psqlrc.provision.backup + fi +else + DELETE_PSQL="yes" +fi + +echo '\pset pager off' >> /root/.psqlrc +# END FIXME! #16992 Temporary fix for psql call in arvados-api-server + +# Now run the install +salt-call --local state.apply -l ${LOG_LEVEL} + +# FIXME! #16992 Temporary fix for psql call in arvados-api-server +if [ "x${DELETE_PSQL}" = "xyes" ]; then + echo "Removing .psql file" + rm /root/.psqlrc +fi + +if [ "x${RESTORE_PSQL}" = "xyes" ]; then + echo "Restoring .psql file" + mv -v /root/.psqlrc.provision.backup /root/.psqlrc +fi +# END FIXME! #16992 Temporary fix for psql call in arvados-api-server + +# Leave a copy of the Arvados CA so the user can copy it where it's required +echo "Copying the Arvados CA certificate to the installer dir, so you can import it" +# If running in a vagrant VM, also add default user to docker group +if [ "x${VAGRANT}" = "xyes" ]; then + cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant + + echo "Adding the vagrant user to the docker group" + usermod -a -G docker vagrant +else + cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR} +fi + +# Test that the installation finished correctly +if [ "x${TEST}" = "xyes" ]; then + cd /tmp/cluster_tests + ./run-test.sh +fi diff --git a/tools/salt-install/single_host/arvados.sls b/tools/salt-install/single_host/arvados.sls new file mode 100644 index 0000000000..a06244270c --- /dev/null +++ b/tools/salt-install/single_host/arvados.sls @@ -0,0 +1,159 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# The variables commented out are the default values that the formula uses. +# The uncommented values are REQUIRED values. If you don't set them, running +# this formula will fail. +arvados: + ### GENERAL CONFIG + version: '__VERSION__' + ## It makes little sense to disable this flag, but you can, if you want :) + # use_upstream_repo: true + + ## Repo URL is built with grains values. If desired, it can be completely + ## overwritten with the pillar parameter 'repo_url' + # repo: + # humanname: Arvados Official Repository + + release: __RELEASE__ + + ## IMPORTANT!!!!! + ## api, workbench and shell require some gems, so you need to make sure ruby + ## and deps are installed in order to install and compile the gems. + ## We default to `false` in these two variables as it's expected you already + ## manage OS packages with some other tool and you don't want us messing up + ## with your setup. + ruby: + ## We set these to `true` here for testing purposes. + ## They both default to `false`. + manage_ruby: true + manage_gems_deps: true + # pkg: ruby + # gems_deps: + # - curl + # - g++ + # - gcc + # - git + # - libcurl4 + # - libcurl4-gnutls-dev + # - libpq-dev + # - libxml2 + # - libxml2-dev + # - make + # - python3-dev + # - ruby-dev + # - zlib1g-dev + + # config: + # file: /etc/arvados/config.yml + # user: root + ## IMPORTANT!!!!! + ## If you're intalling any of the rails apps (api, workbench), the group + ## should be set to that of the web server, usually `www-data` + # group: root + # mode: 640 + + ### ARVADOS CLUSTER CONFIG + cluster: + name: __CLUSTER__ + domain: __DOMAIN__ + + database: + # max concurrent connections per arvados server daemon + # connection_pool_max: 32 + name: arvados + host: 127.0.0.1 + password: changeme_arvados + user: arvados + encoding: en_US.utf8 + client_encoding: UTF8 + + tls: + # certificate: '' + # key: '' + # required to test with arvados-snakeoil certs + insecure: true + + ### TOKENS + tokens: + system_root: changemesystemroottoken + management: changememanagementtoken + rails_secret: changemerailssecrettoken + anonymous_user: changemeanonymoususertoken + + ### KEYS + secrets: + blob_signing_key: changemeblobsigningkey + workbench_secret_key: changemeworkbenchsecretkey + dispatcher_access_key: changemedispatcheraccesskey + dispatcher_secret_key: changeme_dispatchersecretkey + keep_access_key: changemekeepaccesskey + keep_secret_key: changemekeepsecretkey + + Login: + Test: + Enable: true + Users: + __INITIAL_USER__: + Email: __INITIAL_USER_EMAIL__ + Password: __INITIAL_USER_PASSWORD__ + + ### VOLUMES + ## This should usually match all your `keepstore` instances + Volumes: + # the volume name will be composed with + # -nyw5e- + __CLUSTER__-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.__CLUSTER__.__DOMAIN__:25107: + ReadOnly: false + Replication: 2 + Driver: Directory + DriverParameters: + Root: /tmp + + Users: + NewUsersAreActive: true + AutoAdminFirstUser: true + AutoSetupNewUsers: true + AutoSetupNewUsersWithRepository: true + + Services: + Controller: + ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ + InternalURLs: + http://controller.internal:8003: {} + DispatchCloud: + InternalURLs: + http://__CLUSTER__.__DOMAIN__:9006: {} + Keepbalance: + InternalURLs: + http://__CLUSTER__.__DOMAIN__:9005: {} + Keepproxy: + ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ + InternalURLs: + http://keep.internal:25100: {} + Keepstore: + InternalURLs: + http://keep0.__CLUSTER__.__DOMAIN__:25107: {} + RailsAPI: + InternalURLs: + http://api.internal:8004: {} + WebDAV: + ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ + InternalURLs: + http://collections.internal:9002: {} + WebDAVDownload: + ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ + WebShell: + ExternalURL: https://webshell.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ + Websocket: + ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket + InternalURLs: + http://ws.internal:8005: {} + Workbench1: + ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ + Workbench2: + ExternalURL: https://workbench2.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ diff --git a/tools/salt-install/single_host/docker.sls b/tools/salt-install/single_host/docker.sls new file mode 100644 index 0000000000..54d2256159 --- /dev/null +++ b/tools/salt-install/single_host/docker.sls @@ -0,0 +1,9 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +docker: + pkg: + docker: + use_upstream: package diff --git a/tools/salt-install/single_host/locale.sls b/tools/salt-install/single_host/locale.sls new file mode 100644 index 0000000000..17f53a2881 --- /dev/null +++ b/tools/salt-install/single_host/locale.sls @@ -0,0 +1,14 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +locale: + present: + - "en_US.UTF-8 UTF-8" + default: + # Note: On debian systems don't write the second 'UTF-8' here or you will + # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8 + # Restart the minion after you corrected this! + name: 'en_US.UTF-8' + requires: 'en_US.UTF-8 UTF-8' diff --git a/tools/salt-install/single_host/nginx_api_configuration.sls b/tools/salt-install/single_host/nginx_api_configuration.sls new file mode 100644 index 0000000000..b2f12c7739 --- /dev/null +++ b/tools/salt-install/single_host/nginx_api_configuration.sls @@ -0,0 +1,28 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### ARVADOS +arvados: + config: + group: www-data + +### NGINX +nginx: + ### SITES + servers: + managed: + arvados_api: + enabled: true + overwrite: true + config: + - server: + - listen: 'api.internal:8004' + - server_name: api + - root: /var/www/arvados-api/current/public + - index: index.html index.htm + - access_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.access.log combined + - error_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.error.log + - passenger_enabled: 'on' + - client_max_body_size: 128m diff --git a/tools/salt-install/single_host/nginx_controller_configuration.sls b/tools/salt-install/single_host/nginx_controller_configuration.sls new file mode 100644 index 0000000000..00c3b3a13e --- /dev/null +++ b/tools/salt-install/single_host/nginx_controller_configuration.sls @@ -0,0 +1,58 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + 'geo $external_client': + default: 1 + '127.0.0.0/8': 0 + upstream controller_upstream: + - server: 'controller.internal:8003 fail_timeout=10s' + + ### SITES + servers: + managed: + ### DEFAULT + arvados_controller_default: + enabled: true + overwrite: true + config: + - server: + - server_name: __CLUSTER__.__DOMAIN__ + - listen: + - 80 default + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_controller_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: __CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://controller_upstream' + - proxy_read_timeout: 300 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_set_header: 'X-External-Client $external_client' + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log + - client_max_body_size: 128m diff --git a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls new file mode 100644 index 0000000000..6554f79a7c --- /dev/null +++ b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls @@ -0,0 +1,57 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + upstream keepproxy_upstream: + - server: 'keep.internal:25100 fail_timeout=10s' + + servers: + managed: + ### DEFAULT + arvados_keepproxy_default: + enabled: true + overwrite: true + config: + - server: + - server_name: keep.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_keepproxy_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: keep.__CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://keepproxy_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_body_buffer_size: 64M + - client_max_body_size: 64M + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/single_host/nginx_keepweb_configuration.sls b/tools/salt-install/single_host/nginx_keepweb_configuration.sls new file mode 100644 index 0000000000..cc871b9da1 --- /dev/null +++ b/tools/salt-install/single_host/nginx_keepweb_configuration.sls @@ -0,0 +1,57 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + upstream collections_downloads_upstream: + - server: 'collections.internal:9002 fail_timeout=10s' + + servers: + managed: + ### DEFAULT + arvados_collections_download_default: + enabled: true + overwrite: true + config: + - server: + - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + ### COLLECTIONS / DOWNLOAD + arvados_collections_download_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://collections_downloads_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_max_body_size: 0 + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/single_host/nginx_passenger.sls b/tools/salt-install/single_host/nginx_passenger.sls new file mode 100644 index 0000000000..6ce75faa70 --- /dev/null +++ b/tools/salt-install/single_host/nginx_passenger.sls @@ -0,0 +1,24 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + install_from_phusionpassenger: true + lookup: + passenger_package: libnginx-mod-http-passenger + passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf + + ### SERVER + server: + config: + include: 'modules-enabled/*.conf' + worker_processes: 4 + + ### SITES + servers: + managed: + # Remove default webserver + default: + enabled: false diff --git a/tools/salt-install/single_host/nginx_webshell_configuration.sls b/tools/salt-install/single_host/nginx_webshell_configuration.sls new file mode 100644 index 0000000000..a0756b7ce5 --- /dev/null +++ b/tools/salt-install/single_host/nginx_webshell_configuration.sls @@ -0,0 +1,74 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + + ### STREAMS + http: + upstream webshell_upstream: + - server: 'shell.internal:4200 fail_timeout=10s' + + ### SITES + servers: + managed: + arvados_webshell_default: + enabled: true + overwrite: true + config: + - server: + - server_name: webshell.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_webshell_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: webshell.__CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /shell.__CLUSTER__.__DOMAIN__: + - proxy_pass: 'http://webshell_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_ssl_session_reuse: 'off' + + - "if ($request_method = 'OPTIONS')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + - add_header: "'Access-Control-Max-Age' 1728000" + - add_header: "'Content-Type' 'text/plain charset=UTF-8'" + - add_header: "'Content-Length' 0" + - return: 204 + + - "if ($request_method = 'POST')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + + - "if ($request_method = 'GET')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log + diff --git a/tools/salt-install/single_host/nginx_websocket_configuration.sls b/tools/salt-install/single_host/nginx_websocket_configuration.sls new file mode 100644 index 0000000000..ebe03f7337 --- /dev/null +++ b/tools/salt-install/single_host/nginx_websocket_configuration.sls @@ -0,0 +1,58 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + upstream websocket_upstream: + - server: 'ws.internal:8005 fail_timeout=10s' + + servers: + managed: + ### DEFAULT + arvados_websocket_default: + enabled: true + overwrite: true + config: + - server: + - server_name: ws.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_websocket_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: ws.__CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://websocket_upstream' + - proxy_read_timeout: 600 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: 'Host $host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'Upgrade $http_upgrade' + - proxy_set_header: 'Connection "upgrade"' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_body_buffer_size: 64M + - client_max_body_size: 64M + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/single_host/nginx_workbench2_configuration.sls b/tools/salt-install/single_host/nginx_workbench2_configuration.sls new file mode 100644 index 0000000000..8930be408c --- /dev/null +++ b/tools/salt-install/single_host/nginx_workbench2_configuration.sls @@ -0,0 +1,48 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### ARVADOS +arvados: + config: + group: www-data + +### NGINX +nginx: + ### SITES + servers: + managed: + ### DEFAULT + arvados_workbench2_default: + enabled: true + overwrite: true + config: + - server: + - server_name: workbench2.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_workbench2_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: workbench2.__CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - root: /var/www/arvados-workbench2/workbench2 + - try_files: '$uri $uri/ /index.html' + - 'if (-f $document_root/maintenance.html)': + - return: 503 + - location /config.json: + - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__"}' ~ "'" }} + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/single_host/nginx_workbench_configuration.sls b/tools/salt-install/single_host/nginx_workbench_configuration.sls new file mode 100644 index 0000000000..be571ca77e --- /dev/null +++ b/tools/salt-install/single_host/nginx_workbench_configuration.sls @@ -0,0 +1,73 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### ARVADOS +arvados: + config: + group: www-data + +### NGINX +nginx: + ### SERVER + server: + config: + + ### STREAMS + http: + upstream workbench_upstream: + - server: 'workbench.internal:9000 fail_timeout=10s' + + ### SITES + servers: + managed: + ### DEFAULT + arvados_workbench_default: + enabled: true + overwrite: true + config: + - server: + - server_name: workbench.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_workbench_ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: workbench.__CLUSTER__.__DOMAIN__ + - listen: + - __HOST_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://workbench_upstream' + - proxy_read_timeout: 300 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - include: 'snippets/arvados-snakeoil.conf' + - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log + + arvados_workbench_upstream: + enabled: true + overwrite: true + config: + - server: + - listen: 'workbench.internal:9000' + - server_name: workbench + - root: /var/www/arvados-workbench/current/public + - index: index.html index.htm + - passenger_enabled: 'on' + # yamllint disable-line rule:line-length + - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.access.log combined + - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.error.log diff --git a/tools/salt-install/single_host/postgresql.sls b/tools/salt-install/single_host/postgresql.sls new file mode 100644 index 0000000000..56b0a42e8b --- /dev/null +++ b/tools/salt-install/single_host/postgresql.sls @@ -0,0 +1,42 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### POSTGRESQL +postgres: + use_upstream_repo: false + pkgs_extra: + - postgresql-contrib + postgresconf: |- + listen_addresses = '*' # listen on all interfaces + acls: + - ['local', 'all', 'postgres', 'peer'] + - ['local', 'all', 'all', 'peer'] + - ['host', 'all', 'all', '127.0.0.1/32', 'md5'] + - ['host', 'all', 'all', '::1/128', 'md5'] + - ['host', 'arvados', 'arvados', '127.0.0.1/32'] + users: + arvados: + ensure: present + password: changeme_arvados + + # tablespaces: + # arvados_tablespace: + # directory: /path/to/some/tbspace/arvados_tbsp + # owner: arvados + + databases: + arvados: + owner: arvados + template: template0 + lc_ctype: en_US.utf8 + lc_collate: en_US.utf8 + # tablespace: arvados_tablespace + schemas: + public: + owner: arvados + extensions: + pg_trgm: + if_not_exists: true + schema: public diff --git a/tools/salt-install/tests/hasher-workflow-job.yml b/tools/salt-install/tests/hasher-workflow-job.yml new file mode 100644 index 0000000000..8e5f61167c --- /dev/null +++ b/tools/salt-install/tests/hasher-workflow-job.yml @@ -0,0 +1,10 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +inputfile: + class: File + path: test.txt +hasher1_outputname: hasher1.md5sum.txt +hasher2_outputname: hasher2.md5sum.txt +hasher3_outputname: hasher3.md5sum.txt diff --git a/tools/salt-install/tests/hasher-workflow.cwl b/tools/salt-install/tests/hasher-workflow.cwl new file mode 100644 index 0000000000..a23a22f91e --- /dev/null +++ b/tools/salt-install/tests/hasher-workflow.cwl @@ -0,0 +1,65 @@ +#!/usr/bin/env cwl-runner +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow + +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" + +inputs: + inputfile: File + hasher1_outputname: string + hasher2_outputname: string + hasher3_outputname: string + +outputs: + hasher_out: + type: File + outputSource: hasher3/hasher_out + +steps: + hasher1: + run: hasher.cwl + in: + inputfile: inputfile + outputname: hasher1_outputname + out: [hasher_out] + hints: + ResourceRequirement: + coresMin: 1 + arv:IntermediateOutput: + outputTTL: 3600 + arv:ReuseRequirement: + enableReuse: false + + hasher2: + run: hasher.cwl + in: + inputfile: hasher1/hasher_out + outputname: hasher2_outputname + out: [hasher_out] + hints: + ResourceRequirement: + coresMin: 1 + arv:IntermediateOutput: + outputTTL: 3600 + arv:ReuseRequirement: + enableReuse: false + + hasher3: + run: hasher.cwl + in: + inputfile: hasher2/hasher_out + outputname: hasher3_outputname + out: [hasher_out] + hints: + ResourceRequirement: + coresMin: 1 + arv:IntermediateOutput: + outputTTL: 3600 + arv:ReuseRequirement: + enableReuse: false diff --git a/tools/salt-install/tests/hasher.cwl b/tools/salt-install/tests/hasher.cwl new file mode 100644 index 0000000000..0a0f64f056 --- /dev/null +++ b/tools/salt-install/tests/hasher.cwl @@ -0,0 +1,24 @@ +#!/usr/bin/env cwl-runner +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool + +baseCommand: md5sum +inputs: + inputfile: + type: File + inputBinding: + position: 1 + outputname: + type: string + +stdout: $(inputs.outputname) + +outputs: + hasher_out: + type: File + outputBinding: + glob: $(inputs.outputname) diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh new file mode 100755 index 0000000000..8d9de6fdf0 --- /dev/null +++ b/tools/salt-install/tests/run-test.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +export ARVADOS_API_TOKEN=changemesystemroottoken +export ARVADOS_API_HOST=__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__ +export ARVADOS_API_HOST_INSECURE=true + +set -o pipefail + +# First, validate that the CA is installed and that we can query it with no errors. +if ! curl -s -o /dev/null https://workbench.${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then + echo "The Arvados CA was not correctly installed. Although some components will work," + echo "others won't. Please verify that the CA cert file was installed correctly and" + echo "retry running these tests." + exit 1 +fi + +# https://doc.arvados.org/v2.0/install/install-jobs-image.html +echo "Creating Arvados Standard Docker Images project" +uuid_prefix=$(arv --format=uuid user current | cut -d- -f1) +project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "Arvados Standard Docker Images"]]') + +if [ "x${project_uuid}" = "x" ]; then + project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}") + + read -rd $'\000' newlink < + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/tools/user-activity/arvados_user_activity/__init__.py b/tools/user-activity/arvados_user_activity/__init__.py new file mode 100644 index 0000000000..e62d75def4 --- /dev/null +++ b/tools/user-activity/arvados_user_activity/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 diff --git a/tools/user-activity/arvados_user_activity/main.py b/tools/user-activity/arvados_user_activity/main.py new file mode 100755 index 0000000000..959f16d898 --- /dev/null +++ b/tools/user-activity/arvados_user_activity/main.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +import argparse +import sys + +import arvados +import arvados.util +import datetime +import ciso8601 + +def parse_arguments(arguments): + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument('--days', type=int, required=True) + args = arg_parser.parse_args(arguments) + return args + +def getowner(arv, uuid, owners): + if uuid is None: + return None + if uuid[6:11] == "tpzed": + return uuid + + if uuid not in owners: + try: + gp = arv.groups().get(uuid=uuid).execute() + owners[uuid] = gp["owner_uuid"] + except: + owners[uuid] = None + + return getowner(arv, owners[uuid], owners) + +def getuserinfo(arv, uuid): + u = arv.users().get(uuid=uuid).execute() + prof = "\n".join(" %s: \"%s\"" % (k, v) for k, v in u["prefs"].get("profile", {}).items() if v) + if prof: + prof = "\n"+prof+"\n" + return "%s %s <%s> (%susers/%s)%s" % (u["first_name"], u["last_name"], u["email"], + arv.config()["Services"]["Workbench1"]["ExternalURL"], + uuid, prof) + +def getname(u): + return "\"%s\" (%s)" % (u["name"], u["uuid"]) + +def main(arguments=None): + if arguments is None: + arguments = sys.argv[1:] + + args = parse_arguments(arguments) + + arv = arvados.api() + + since = datetime.datetime.utcnow() - datetime.timedelta(days=args.days) + + print("User activity on %s between %s and %s\n" % (arv.config()["ClusterID"], + (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat(sep=" ", timespec="minutes"), + datetime.datetime.now().isoformat(sep=" ", timespec="minutes"))) + + events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", since.isoformat()]]) + + users = {} + owners = {} + + for e in events: + owner = getowner(arv, e["object_owner_uuid"], owners) + users.setdefault(owner, []) + event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat(sep=" ", timespec="minutes") + # loguuid = e["uuid"] + loguuid = "" + + if e["event_type"] == "create" and e["object_uuid"][6:11] == "tpzed": + users.setdefault(e["object_uuid"], []) + users[e["object_uuid"]].append("%s User account created" % event_at) + + elif e["event_type"] == "update" and e["object_uuid"][6:11] == "tpzed": + pass + + elif e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp": + if e["properties"]["new_attributes"]["requesting_container_uuid"] is None: + users[owner].append("%s Ran container %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid)) + + elif e["event_type"] == "update" and e["object_uuid"][6:11] == "xvhdp": + pass + + elif e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g": + users[owner].append("%s Created project %s" % (event_at, getname(e["properties"]["new_attributes"]))) + + elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "j7d0g": + users[owner].append("%s Deleted project %s" % (event_at, getname(e["properties"]["old_attributes"]))) + + elif e["event_type"] == "update" and e["object_uuid"][6:11] == "j7d0g": + users[owner].append("%s Updated project %s" % (event_at, getname(e["properties"]["new_attributes"]))) + + elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su": + since_last = None + if len(users[owner]) > 0 and users[owner][-1].endswith("activity"): + sp = users[owner][-1].split(" ") + start = sp[0]+" "+sp[1] + since_last = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(sp[3]+" "+sp[4]) + span = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(start) + + if since_last is not None and since_last < datetime.timedelta(minutes=61): + users[owner][-1] = "%s to %s (%02d:%02d) Account activity" % (start, event_at, span.days*24 + int(span.seconds/3600), int((span.seconds % 3600)/60)) + else: + users[owner].append("%s to %s (0:00) Account activity" % (event_at, event_at)) + + elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j": + if e["properties"]["new_attributes"]["link_class"] == "tag": + users[owner].append("%s Tagged %s" % (event_at, e["properties"]["new_attributes"]["head_uuid"])) + elif e["properties"]["new_attributes"]["link_class"] == "permission": + users[owner].append("%s Shared %s with %s" % (event_at, e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"])) + else: + users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid)) + + elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "o0j2j": + if e["properties"]["old_attributes"]["link_class"] == "tag": + users[owner].append("%s Untagged %s" % (event_at, e["properties"]["old_attributes"]["head_uuid"])) + elif e["properties"]["old_attributes"]["link_class"] == "permission": + users[owner].append("%s Unshared %s with %s" % (event_at, e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"])) + else: + users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid)) + + elif e["event_type"] == "create" and e["object_uuid"][6:11] == "4zz18": + if e["properties"]["new_attributes"]["properties"].get("type") in ("log", "output", "intermediate"): + pass + else: + users[owner].append("%s Created collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid)) + + elif e["event_type"] == "update" and e["object_uuid"][6:11] == "4zz18": + users[owner].append("%s Updated collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid)) + + elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "4zz18": + if e["properties"]["old_attributes"]["properties"].get("type") in ("log", "output", "intermediate"): + pass + else: + users[owner].append("%s Deleted collection %s %s" % (event_at, getname(e["properties"]["old_attributes"]), loguuid)) + + else: + users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid)) + + for k,v in users.items(): + if k is None or k.endswith("-tpzed-000000000000000"): + continue + print(getuserinfo(arv, k)) + for ev in v: + print(" %s" % ev) + print("") + +if __name__ == "__main__": + main() diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py new file mode 100644 index 0000000000..d8eec3d9ee --- /dev/null +++ b/tools/user-activity/arvados_version.py @@ -0,0 +1,58 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +import time +import os +import re +import sys + +SETUP_DIR = os.path.dirname(os.path.abspath(__file__)) +VERSION_PATHS = { + SETUP_DIR, + os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")), + os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh")) + } + +def choose_version_from(): + ts = {} + for path in VERSION_PATHS: + ts[subprocess.check_output( + ['git', 'log', '--first-parent', '--max-count=1', + '--format=format:%ct', path]).strip()] = path + + sorted_ts = sorted(ts.items()) + getver = sorted_ts[-1][1] + print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr) + return getver + +def git_version_at_commit(): + curdir = choose_version_from() + myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent', + '--format=%H', curdir]).strip() + myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode() + return myversion + +def save_version(setup_dir, module, v): + v = v.replace("~dev", ".dev").replace("~rc", "rc") + with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp: + return fp.write("__version__ = '%s'\n" % v) + +def read_version(setup_dir, module): + with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp: + return re.match("__version__ = '(.*)'$", fp.read()).groups()[0] + +def get_version(setup_dir, module): + env_version = os.environ.get("ARVADOS_BUILDING_VERSION") + + if env_version: + save_version(setup_dir, module, env_version) + else: + try: + save_version(setup_dir, module, git_version_at_commit()) + except (subprocess.CalledProcessError, OSError) as err: + print("ERROR: {0}".format(err), file=sys.stderr) + pass + + return read_version(setup_dir, module) diff --git a/tools/user-activity/bin/arv-user-activity b/tools/user-activity/bin/arv-user-activity new file mode 100755 index 0000000000..bc73f84afc --- /dev/null +++ b/tools/user-activity/bin/arv-user-activity @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +import arvados_user_activity.main + +arvados_user_activity.main.main() diff --git a/tools/user-activity/fpm-info.sh b/tools/user-activity/fpm-info.sh new file mode 100644 index 0000000000..0abc6a08ea --- /dev/null +++ b/tools/user-activity/fpm-info.sh @@ -0,0 +1,9 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +case "$TARGET" in + debian* | ubuntu*) + fpm_depends+=(libcurl3-gnutls) + ;; +esac diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py new file mode 100755 index 0000000000..41f8f66a07 --- /dev/null +++ b/tools/user-activity/setup.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +from __future__ import absolute_import +import os +import sys +import re + +from setuptools import setup, find_packages + +SETUP_DIR = os.path.dirname(__file__) or '.' +README = os.path.join(SETUP_DIR, 'README.rst') + +import arvados_version +version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity") + +setup(name='arvados-user-activity', + version=version, + description='Summarize user activity from Arvados audit logs', + author='Arvados', + author_email='info@arvados.org', + url="https://arvados.org", + download_url="https://github.com/arvados/arvados.git", + license='GNU Affero General Public License, version 3.0', + packages=['arvados_user_activity'], + include_package_data=True, + entry_points={"console_scripts": ["arv-user-activity=arvados_user_activity.main:main"]}, + data_files=[ + ('share/doc/arvados_user_activity', ['agpl-3.0.txt']), + ], + install_requires=[ + 'arvados-python-client >= 2.2.0.dev20201118185221', + ], + zip_safe=True, +) diff --git a/tools/vocabulary-migrate/vocabulary-migrate.py b/tools/vocabulary-migrate/vocabulary-migrate.py index 920344b320..89a4f030e8 100644 --- a/tools/vocabulary-migrate/vocabulary-migrate.py +++ b/tools/vocabulary-migrate/vocabulary-migrate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) The Arvados Authors. All rights reserved. #