From: Lucas Di Pentima Date: Tue, 25 Aug 2020 18:00:33 +0000 (-0300) Subject: Merge branch '16678-login-tokens-lifetime-config' X-Git-Tag: 2.1.0~109 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/bd8bdd90055d61263eff5bdb9a953c57319aa83d?hp=67207bade066ddf25b5d0056a5df6201e7ecab2c Merge branch '16678-login-tokens-lifetime-config' Closes #16678 Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- diff --git a/apps/workbench/Gemfile b/apps/workbench/Gemfile index 24bfba383f..d5b416b539 100644 --- a/apps/workbench/Gemfile +++ b/apps/workbench/Gemfile @@ -4,7 +4,7 @@ source 'https://rubygems.org' -gem 'rails', '~> 5.0.0' +gem 'rails', '~> 5.2.0' gem 'arvados', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/ruby/arvados.gemspec' gem 'activerecord-nulldb-adapter', git: 'https://github.com/arvados/nulldb' @@ -14,6 +14,13 @@ gem 'sass' gem 'mime-types' gem 'responders', '~> 2.0' +# Pin sprockets to < 4.0 to avoid issues when upgrading rails to 5.2 +# See: https://github.com/rails/sprockets-rails/issues/443 +gem 'sprockets', '~> 3.0' + +# Fast app boot times +gem 'bootsnap', require: false + # Note: keeping this out of the "group :assets" section "may" allow us # to use Coffescript for UJS responses. It also prevents a # warning/problem when running tests: "WARN: tilt autoloading @@ -31,8 +38,14 @@ group :assets do gem 'therubyracer', :platforms => :ruby end -group :development do +group :development, :test, :performance do gem 'byebug' + # Pinning launchy because 2.5 requires ruby >= 2.4, which arvbox currently + # doesn't have because of SSO. + gem 'launchy', '~> 2.4.0' +end + +group :development do gem 'ruby-debug-passenger' gem 'rack-mini-profiler', require: false gem 'flamegraph', require: false @@ -48,7 +61,6 @@ group :test, :diagnostics, :performance do end group :test, :performance do - gem 'byebug' gem 'rails-perftest' gem 'ruby-prof' gem 'rvm-capistrano' @@ -70,12 +82,6 @@ gem 'angularjs-rails', '~> 1.3.8' gem 'less' gem 'less-rails' - -# Wiselinks hasn't been updated for many years and it's using deprecated methods -# Use our own Wiselinks fork until this PR is accepted: -# https://github.com/igor-alexandrov/wiselinks/pull/116 -# gem 'wiselinks', git: 'https://github.com/arvados/wiselinks.git', branch: 'rails-5.1-compatibility' - gem 'sshkey' # To use ActiveModel has_secure_password diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock index cb4e7ab9e3..e19172cb2e 100644 --- a/apps/workbench/Gemfile.lock +++ b/apps/workbench/Gemfile.lock @@ -30,39 +30,43 @@ GEM remote: https://rubygems.org/ specs: RedCloth (4.3.2) - actioncable (5.0.7.2) - actionpack (= 5.0.7.2) - nio4r (>= 1.2, < 3.0) - websocket-driver (~> 0.6.1) - actionmailer (5.0.7.2) - actionpack (= 5.0.7.2) - actionview (= 5.0.7.2) - activejob (= 5.0.7.2) + actioncable (5.2.4.3) + actionpack (= 5.2.4.3) + nio4r (~> 2.0) + websocket-driver (>= 0.6.1) + actionmailer (5.2.4.3) + actionpack (= 5.2.4.3) + actionview (= 5.2.4.3) + activejob (= 5.2.4.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.0.7.2) - actionview (= 5.0.7.2) - activesupport (= 5.0.7.2) - rack (~> 2.0) - rack-test (~> 0.6.3) + actionpack (5.2.4.3) + actionview (= 5.2.4.3) + activesupport (= 5.2.4.3) + rack (~> 2.0, >= 2.0.8) + rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.0.7.2) - activesupport (= 5.0.7.2) + actionview (5.2.4.3) + activesupport (= 5.2.4.3) builder (~> 3.1) - erubis (~> 2.7.0) + erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.0.7.2) - activesupport (= 5.0.7.2) + activejob (5.2.4.3) + activesupport (= 5.2.4.3) globalid (>= 0.3.6) - activemodel (5.0.7.2) - activesupport (= 5.0.7.2) - activerecord (5.0.7.2) - activemodel (= 5.0.7.2) - activesupport (= 5.0.7.2) - arel (~> 7.0) - activesupport (5.0.7.2) + activemodel (5.2.4.3) + activesupport (= 5.2.4.3) + activerecord (5.2.4.3) + activemodel (= 5.2.4.3) + activesupport (= 5.2.4.3) + arel (>= 9.0) + activestorage (5.2.4.3) + actionpack (= 5.2.4.3) + activerecord (= 5.2.4.3) + marcel (~> 0.3.1) + activesupport (5.2.4.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -71,9 +75,9 @@ GEM public_suffix (>= 2.0.2, < 5.0) andand (1.3.3) angularjs-rails (1.3.15) - arel (7.1.4) - arvados-google-api-client (0.8.7.3) - activesupport (>= 3.2, < 5.1) + arel (9.0.0) + arvados-google-api-client (0.8.7.4) + activesupport (>= 3.2, < 5.3) addressable (~> 2.3) autoparse (~> 0.3) extlib (~> 0.9) @@ -89,6 +93,8 @@ GEM multi_json (>= 1.0.0) autoprefixer-rails (9.5.1.1) execjs + bootsnap (1.4.7) + msgpack (~> 1.0) bootstrap-sass (3.4.1) autoprefixer-rails (>= 5.2.1) sassc (>= 2.0.0) @@ -96,7 +102,7 @@ GEM railties (>= 3.1) bootstrap-x-editable-rails (1.5.1.1) railties (>= 3.0) - builder (3.2.3) + builder (3.2.4) byebug (11.0.1) capistrano (2.15.9) highline @@ -121,11 +127,11 @@ GEM execjs coffee-script-source (1.12.2) commonjs (0.2.7) - concurrent-ruby (1.1.5) - crass (1.0.5) + concurrent-ruby (1.1.6) + crass (1.0.6) deep_merge (1.2.1) docile (1.3.1) - erubis (2.7.0) + erubi (1.9.0) execjs (2.7.0) extlib (0.9.16) faraday (0.15.4) @@ -167,25 +173,29 @@ GEM railties (>= 4) request_store (~> 1.0) logstash-event (1.2.02) - loofah (2.3.1) + loofah (2.6.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) + marcel (0.3.3) + mimemagic (~> 0.3.2) memoist (0.16.2) metaclass (0.0.4) - method_source (0.9.2) + method_source (1.0.0) mime-types (3.2.2) mime-types-data (~> 3.2015) mime-types-data (3.2019.0331) - mini_mime (1.0.1) + mimemagic (0.3.5) + mini_mime (1.0.2) mini_portile2 (2.4.0) minitest (5.10.3) mocha (1.8.0) metaclass (~> 0.0.1) morrisjs-rails (0.5.1.2) railties (> 3.1, < 6) - multi_json (1.14.1) + msgpack (1.3.3) + multi_json (1.15.0) multipart-post (2.1.1) net-scp (2.0.0) net-ssh (>= 2.6.5, < 6.0.0) @@ -194,13 +204,13 @@ GEM net-ssh (5.2.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) - nio4r (2.3.1) - nokogiri (1.10.8) + nio4r (2.5.2) + nokogiri (1.10.10) mini_portile2 (~> 2.4.0) npm-rails (0.2.1) rails (>= 3.2) oj (3.7.12) - os (1.0.1) + os (1.1.1) passenger (6.0.2) rack rake (>= 0.8.1) @@ -213,23 +223,24 @@ GEM cliver (~> 0.3.1) multi_json (~> 1.0) websocket-driver (>= 0.2.0) - public_suffix (4.0.3) + public_suffix (4.0.5) rack (2.2.3) rack-mini-profiler (1.0.2) rack (>= 1.2.0) - rack-test (0.6.3) - rack (>= 1.0) - rails (5.0.7.2) - actioncable (= 5.0.7.2) - actionmailer (= 5.0.7.2) - actionpack (= 5.0.7.2) - actionview (= 5.0.7.2) - activejob (= 5.0.7.2) - activemodel (= 5.0.7.2) - activerecord (= 5.0.7.2) - activesupport (= 5.0.7.2) + rack-test (1.1.0) + rack (>= 1.0, < 3) + rails (5.2.4.3) + actioncable (= 5.2.4.3) + actionmailer (= 5.2.4.3) + actionpack (= 5.2.4.3) + actionview (= 5.2.4.3) + activejob (= 5.2.4.3) + activemodel (= 5.2.4.3) + activerecord (= 5.2.4.3) + activestorage (= 5.2.4.3) + activesupport (= 5.2.4.3) bundler (>= 1.3.0) - railties (= 5.0.7.2) + railties (= 5.2.4.3) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.4) actionpack (>= 5.0.1.x) @@ -238,15 +249,15 @@ GEM rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) + rails-html-sanitizer (1.3.0) + loofah (~> 2.3) rails-perftest (0.0.7) - railties (5.0.7.2) - actionpack (= 5.0.7.2) - activesupport (= 5.0.7.2) + railties (5.2.4.3) + actionpack (= 5.2.4.3) + activesupport (= 5.2.4.3) method_source rake (>= 0.8.7) - thor (>= 0.18.1, < 2.0) + thor (>= 0.19.0, < 2.0) rake (13.0.1) raphael-rails (2.1.2) rb-fsevent (0.10.3) @@ -305,15 +316,15 @@ GEM therubyracer (0.12.3) libv8 (~> 3.16.14.15) ref - thor (0.20.3) + thor (1.0.1) thread_safe (0.3.6) tilt (2.0.9) - tzinfo (1.2.6) + tzinfo (1.2.7) thread_safe (~> 0.1) uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) - websocket-driver (0.6.5) + websocket-driver (0.7.3) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) xpath (2.1.0) @@ -328,6 +339,7 @@ DEPENDENCIES andand angularjs-rails (~> 1.3.8) arvados! + bootsnap bootstrap-sass (~> 3.4.1) bootstrap-tab-history-rails bootstrap-x-editable-rails @@ -339,6 +351,7 @@ DEPENDENCIES headless (~> 1.0.2) httpclient (~> 2.5) jquery-rails + launchy (~> 2.4.0) less less-rails lograge @@ -354,7 +367,7 @@ DEPENDENCIES piwik_analytics poltergeist (~> 1.5.1) rack-mini-profiler - rails (~> 5.0.0) + rails (~> 5.2.0) rails-controller-testing rails-perftest raphael-rails @@ -369,10 +382,11 @@ DEPENDENCIES signet (< 0.12) simplecov (~> 0.7) simplecov-rcov + sprockets (~> 3.0) sshkey themes_for_rails! therubyracer uglifier (~> 2.0) BUNDLED WITH - 1.16.6 + 1.17.3 diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb index 8d6f897bb6..77ec68bdb0 100644 --- a/apps/workbench/app/controllers/application_controller.rb +++ b/apps/workbench/app/controllers/application_controller.rb @@ -29,7 +29,6 @@ class ApplicationController < ActionController::Base begin rescue_from(ActiveRecord::RecordNotFound, ActionController::RoutingError, - ActionController::UnknownController, AbstractController::ActionNotFound, with: :render_not_found) rescue_from(Exception, diff --git a/apps/workbench/app/models/application_record.rb b/apps/workbench/app/models/application_record.rb deleted file mode 100644 index 759034da66..0000000000 --- a/apps/workbench/app/models/application_record.rb +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -class ApplicationRecord < ActiveRecord::Base - self.abstract_class = true -end \ No newline at end of file diff --git a/apps/workbench/app/models/arvados_base.rb b/apps/workbench/app/models/arvados_base.rb index b9162c2aec..c5e1a4ed22 100644 --- a/apps/workbench/app/models/arvados_base.rb +++ b/apps/workbench/app/models/arvados_base.rb @@ -106,6 +106,12 @@ class ArvadosBase end end + # The ActiveModel::Dirty API was changed on Rails 5.2 + # See: https://github.com/rails/rails/commit/c3675f50d2e59b7fc173d7b332860c4b1a24a726#diff-aaddd42c7feb0834b1b5c66af69814d3 + def mutations_from_database + @mutations_from_database ||= ActiveModel::NullMutationTracker.instance + end + def self.columns @discovered_columns = [] if !defined?(@discovered_columns) return @discovered_columns if @discovered_columns.andand.any? diff --git a/apps/workbench/bin/bundle b/apps/workbench/bin/bundle index 9447ba8612..cb10307acd 100755 --- a/apps/workbench/bin/bundle +++ b/apps/workbench/bin/bundle @@ -3,5 +3,5 @@ # # SPDX-License-Identifier: AGPL-3.0 -ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) load Gem.bin_path('bundler', 'bundle') diff --git a/apps/workbench/bin/setup b/apps/workbench/bin/setup index 50c3fa0548..7aed0fb282 100755 --- a/apps/workbench/bin/setup +++ b/apps/workbench/bin/setup @@ -3,12 +3,11 @@ # # SPDX-License-Identifier: AGPL-3.0 -require 'pathname' require 'fileutils' include FileUtils # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") @@ -22,6 +21,9 @@ chdir APP_ROOT do system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') + # Install JavaScript dependencies if using Yarn + # system('bin/yarn') + # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') # cp 'config/database.yml.sample', 'config/database.yml' diff --git a/apps/workbench/bin/update b/apps/workbench/bin/update index b56771ece8..46aa76ca87 100755 --- a/apps/workbench/bin/update +++ b/apps/workbench/bin/update @@ -22,6 +22,9 @@ chdir APP_ROOT do system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') + # Install JavaScript dependencies if using Yarn + # system('bin/yarn') + puts "\n== Updating database ==" system! 'bin/rails db:migrate' diff --git a/apps/workbench/bin/yarn b/apps/workbench/bin/yarn new file mode 100755 index 0000000000..5fc7611952 --- /dev/null +++ b/apps/workbench/bin/yarn @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +APP_ROOT = File.expand_path('..', __dir__) +Dir.chdir(APP_ROOT) do + begin + exec "yarnpkg #{ARGV.join(" ")}" + rescue Errno::ENOENT + $stderr.puts "Yarn executable was not detected in the system." + $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" + exit 1 + end +end diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml index 9456e61455..255ad44f85 100644 --- a/apps/workbench/config/application.default.yml +++ b/apps/workbench/config/application.default.yml @@ -77,7 +77,6 @@ test: action_mailer.delivery_method: :test active_support.deprecation: :stderr profiling_enabled: true - secret_token: <%= rand(2**256).to_s(36) %> secret_key_base: <%= rand(2**256).to_s(36) %> site_name: Workbench:test diff --git a/apps/workbench/config/application.rb b/apps/workbench/config/application.rb index e88229b851..42bf4da24b 100644 --- a/apps/workbench/config/application.rb +++ b/apps/workbench/config/application.rb @@ -2,13 +2,15 @@ # # SPDX-License-Identifier: AGPL-3.0 -require File.expand_path('../boot', __FILE__) +require_relative 'boot' require "rails" # Pick only the frameworks we need: require "active_model/railtie" require "active_job/railtie" require "active_record/railtie" +# Skip ActiveStorage (new in Rails 5.1) +# require "active_storage/engine" require "action_controller/railtie" require "action_mailer/railtie" require "action_view/railtie" @@ -28,6 +30,9 @@ module ArvadosWorkbench require_relative "arvados_config.rb" + # Initialize configuration defaults for originally generated Rails version. + config.load_defaults 5.1 + # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. diff --git a/apps/workbench/config/boot.rb b/apps/workbench/config/boot.rb index 8153266683..6add5911f6 100644 --- a/apps/workbench/config/boot.rb +++ b/apps/workbench/config/boot.rb @@ -8,6 +8,7 @@ require 'rubygems' ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE']) +require 'bootsnap/setup' # Speed up boot time by caching expensive operations. # Use ARVADOS_API_TOKEN environment variable (if set) in console require 'rails' diff --git a/apps/workbench/config/initializers/content_security_policy.rb b/apps/workbench/config/initializers/content_security_policy.rb new file mode 100644 index 0000000000..853ecdeec4 --- /dev/null +++ b/apps/workbench/config/initializers/content_security_policy.rb @@ -0,0 +1,29 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy +# For further information see the following documentation +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + +# Rails.application.config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https + +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end + +# If you are using UJS then enable automatic nonce generation +# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) } + +# Report CSP violations to a specified URI +# For further information see the following documentation: +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only +# Rails.application.config.content_security_policy_report_only = true diff --git a/apps/workbench/config/initializers/new_framework_defaults.rb b/apps/workbench/config/initializers/new_framework_defaults.rb index b8dca33a37..2e2f0b1810 100644 --- a/apps/workbench/config/initializers/new_framework_defaults.rb +++ b/apps/workbench/config/initializers/new_framework_defaults.rb @@ -24,6 +24,3 @@ ActiveSupport.to_time_preserves_timezone = false # Require `belongs_to` associations by default. Previous versions had false. Rails.application.config.active_record.belongs_to_required_by_default = false - -# Do not halt callback chains when a callback returns false. Previous versions had true. -ActiveSupport.halt_callback_chains_on_return_false = true diff --git a/apps/workbench/config/initializers/new_framework_defaults_5_1.rb b/apps/workbench/config/initializers/new_framework_defaults_5_1.rb new file mode 100644 index 0000000000..804ee6f506 --- /dev/null +++ b/apps/workbench/config/initializers/new_framework_defaults_5_1.rb @@ -0,0 +1,18 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 5.1 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Make `form_with` generate non-remote forms. +Rails.application.config.action_view.form_with_generates_remote_forms = false + +# Unknown asset fallback will return the path passed in when the given +# asset is not present in the asset pipeline. +# Rails.application.config.assets.unknown_asset_fallback = false diff --git a/apps/workbench/config/initializers/new_framework_defaults_5_2.rb b/apps/workbench/config/initializers/new_framework_defaults_5_2.rb new file mode 100644 index 0000000000..93a8d52406 --- /dev/null +++ b/apps/workbench/config/initializers/new_framework_defaults_5_2.rb @@ -0,0 +1,42 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 5.2 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Make Active Record use stable #cache_key alongside new #cache_version method. +# This is needed for recyclable cache keys. +# Rails.application.config.active_record.cache_versioning = true + +# Use AES-256-GCM authenticated encryption for encrypted cookies. +# Also, embed cookie expiry in signed or encrypted cookies for increased security. +# +# This option is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 5.2. +# +# Existing cookies will be converted on read then written with the new scheme. +# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true + +# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages +# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true. +# Rails.application.config.active_support.use_authenticated_message_encryption = true + +# Add default protection from forgery to ActionController::Base instead of in +# ApplicationController. +# Rails.application.config.action_controller.default_protect_from_forgery = true + +# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and +# 'f' after migrating old data. +# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true + +# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header. +# Rails.application.config.active_support.use_sha1_digests = true + +# Make `form_with` generate id attributes for any generated HTML tags. +# Rails.application.config.action_view.form_with_generates_ids = true diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb index 718adfd2ed..ffc09ac933 100644 --- a/apps/workbench/config/routes.rb +++ b/apps/workbench/config/routes.rb @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0 -ArvadosWorkbench::Application.routes.draw do +Rails.application.routes.draw do themes_for_rails resources :keep_disks diff --git a/apps/workbench/config/secrets.yml b/apps/workbench/config/secrets.yml index bc8a0d0de5..57399082e8 100644 --- a/apps/workbench/config/secrets.yml +++ b/apps/workbench/config/secrets.yml @@ -11,16 +11,16 @@ # no regular words or you'll be exposed to dictionary attacks. # You can use `rails secret` to generate a secure secret key. -# Make sure the secrets in this file are kept private -# if you're sharing your code publicly. +# NOTE that these get overriden by Arvados' own configuration system. -development: - secret_key_base: 33e2d171ec6c67cf8e9a9fbfadc1071328bdab761297e2fe28b9db7613dd542c1ba3bdb3bd3e636d1d6f74ab73a2d90c4e9c0ecc14fde8ccd153045f94e9cc41 +# development: +# secret_key_base: <%= rand(1<<255).to_s(36) %> -test: - secret_key_base: d4c07cab3530fccf5d86565ecdc359eb2a853b8ede3b06edb2885e4423d7a726f50a3e415bb940fd4861e8fec16459665fd377acc8cdd98ea63294d2e0d12bb2 +# test: +# secret_key_base: <%= rand(1<<255).to_s(36) %> -# Do not keep production secrets in the repository, -# instead read values from the environment. +# In case this doesn't get overriden for some reason, assign a random key +# to gracefully degrade by rejecting cookies instead of by opening a +# vulnerability. production: - secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> + secret_key_base: <%= rand(1<<255).to_s(36) %> diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh index ec8357701d..d1fb2ac670 100755 --- a/build/run-build-docker-jobs-image.sh +++ b/build/run-build-docker-jobs-image.sh @@ -185,28 +185,23 @@ if docker --version |grep " 1\.[0-9]\." ; then FORCE=-f fi -#docker export arvados/jobs:$cwl_runner_version_orig | docker import - arvados/jobs:$cwl_runner_version_orig - if ! [[ -z "$version_tag" ]]; then docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag" -else - docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:latest -fi + ECODE=$? -ECODE=$? + if [[ "$ECODE" != "0" ]]; then + EXITCODE=$(($EXITCODE + $ECODE)) + fi -if [[ "$ECODE" != "0" ]]; then - EXITCODE=$(($EXITCODE + $ECODE)) + checkexit $ECODE "docker tag" + title "docker tag complete (`timer`)" fi -checkexit $ECODE "docker tag" -title "docker tag complete (`timer`)" - title "uploading images" timer_reset -if [[ "$ECODE" != "0" ]]; then +if [[ "$EXITCODE" != "0" ]]; then title "upload arvados images SKIPPED because build or tag failed" else if [[ $upload == true ]]; then @@ -217,7 +212,6 @@ else docker_push arvados/jobs:"$version_tag" else docker_push arvados/jobs:$cwl_runner_version_orig - docker_push arvados/jobs:latest fi title "upload arvados images finished (`timer`)" else diff --git a/build/run-tests.sh b/build/run-tests.sh index 2742540b16..bedc95b2db 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -195,7 +195,7 @@ sanity_checks() { ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \ - || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" + || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" echo Checking dependencies: echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ @@ -373,7 +373,7 @@ if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then fi if [[ $NEED_SDK_R == false ]]; then - echo "R SDK not needed, it will not be installed." + echo "R SDK not needed, it will not be installed." fi checkpidfile() { @@ -414,11 +414,11 @@ start_services() { . "$VENVDIR/bin/activate" echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then - mkdir -p "$WORKSPACE/services/api/log" + mkdir -p "$WORKSPACE/services/api/log" fi # Remove empty api.pid file if it exists if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then - rm -f "$WORKSPACE/tmp/api.pid" + rm -f "$WORKSPACE/tmp/api.pid" fi all_services_stopped= fail=1 @@ -817,19 +817,19 @@ do_test_once() { check_arvados_config() { if [[ "$1" = "env" ]] ; then - return + return fi if [[ -z "$ARVADOS_CONFIG" ]] ; then - # Create config file. The run_test_server script requires PyYAML, - # so virtualenv needs to be active. Downstream steps like - # workbench install which require a valid config.yml. - if [[ ! -s "$VENVDIR/bin/activate" ]] ; then - install_env - fi - . "$VENVDIR/bin/activate" + # Create config file. The run_test_server script requires PyYAML, + # so virtualenv needs to be active. Downstream steps like + # workbench install which require a valid config.yml. + if [[ ! -s "$VENVDIR/bin/activate" ]] ; then + install_env + fi + . "$VENVDIR/bin/activate" cd "$WORKSPACE" - eval $(python sdk/python/tests/run_test_server.py setup_config) - deactivate + eval $(python sdk/python/tests/run_test_server.py setup_config) + deactivate fi } diff --git a/doc/user/topics/arv-web.html.textile.liquid b/doc/user/topics/arv-web.html.textile.liquid deleted file mode 100644 index 9671e97096..0000000000 --- a/doc/user/topics/arv-web.html.textile.liquid +++ /dev/null @@ -1,106 +0,0 @@ ---- -layout: default -navsection: userguide -title: "Using arv-web" -... -{% comment %} -Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: CC-BY-SA-3.0 -{% endcomment %} - -@arv-web@ enables you to run a custom web service from the contents of an Arvados collection. - -{% include 'tutorial_expectations_workstation' %} - -h2. Usage - -@arv-web@ enables you to set up a web service based on the most recent collection in a project. An arv-web application is a reproducible, immutable application bundle where the web app is packaged with both the code to run and the data to serve. Because Arvados Collections can be updated with minimum duplication, it is efficient to produce a new application bundle when the code or data needs to be updated; retaining old application bundles makes it easy to go back and run older versions of your web app. - -
-$ cd $HOME/arvados/services/arv-web
-usage: arv-web.py [-h] --project-uuid PROJECT_UUID [--port PORT]
-                  [--image IMAGE]
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --project-uuid PROJECT_UUID
-                        Project uuid to watch
-  --port PORT           Host port to listen on (default 8080)
-  --image IMAGE         Docker image to run
-
- -At startup, @arv-web@ queries an Arvados project and mounts the most recently modified collection into a temporary directory. It then runs a Docker image with the collection bound to @/mnt@ inside the container. When a new collection is added to the project, or an existing project is updated, it will stop the running Docker container, unmount the old collection, mount the new most recently modified collection, and restart the Docker container with the new mount. - -h2. Docker container - -The @Dockerfile@ in @arvados/docker/arv-web@ builds a Docker image that runs Apache with @/mnt@ as the DocumentRoot. It is configured to run web applications which use Python WSGI, Ruby Rack, or CGI; to serve static HTML; or browse the contents of the @public@ subdirectory of the collection using default Apache index pages. - -To build the Docker image: - - -
~$ cd arvados/docker
-~/arvados/docker$ docker build -t arvados/arv-web arv-web
-
- - -h2. Running sample applications - -First, in Arvados Workbench, create a new project. Copy the project UUID from the URL bar (this is the part of the URL after @projects/...@). - -Now upload a collection containing a "Python WSGI web app:":http://wsgi.readthedocs.org/en/latest/ - - -
~$ cd arvados/services/arv-web
-~/arvados/services/arv-web$ arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-wsgi-app sample-wsgi-app
-0M / 0M 100.0%
-Collection saved as 'sample-wsgi-app'
-zzzzz-4zz18-ebohzfbzh82qmqy
-~/arvados/services/arv-web$ ./arv-web.py --project [zzzzz-j7d0g-yourprojectuuid] --port 8888
-2015-01-30 11:21:00 arvados.arv-web[4897] INFO: Mounting zzzzz-4zz18-ebohzfbzh82qmqy
-2015-01-30 11:21:01 arvados.arv-web[4897] INFO: Starting Docker container arvados/arv-web
-2015-01-30 11:21:02 arvados.arv-web[4897] INFO: Container id e79e70558d585a3e038e4bfbc97e5c511f21b6101443b29a8017bdf3d84689a3
-2015-01-30 11:21:03 arvados.arv-web[4897] INFO: Waiting for events
-
- - -The sample application will be available at @http://localhost:8888@. - -h3. Updating the application - -If you upload a new collection to the same project, arv-web will restart the web service and serve the new collection. For example, uploading a collection containing a "Ruby Rack web app:":https://github.com/rack/rack/wiki - - -
~$ cd arvados/services/arv-web
-~/arvados/services/arv-web$ arv-put --project [zzzzz-j7d0g-yourprojectuuid] --name sample-rack-app sample-rack-app
-0M / 0M 100.0%
-Collection saved as 'sample-rack-app'
-zzzzz-4zz18-dhhm0ay8k8cqkvg
-
- - -@arv-web@ will automatically notice the change, load a new container, and send an update signal (SIGHUP) to the service: - -
-2015-01-30 11:21:03 arvados.arv-web[4897] INFO:Waiting for events
-2015-01-30 11:21:04 arvados.arv-web[4897] INFO:create zzzzz-4zz18-dhhm0ay8k8cqkvg
-2015-01-30 11:21:05 arvados.arv-web[4897] INFO:Mounting zzzzz-4zz18-dhhm0ay8k8cqkvg
-2015-01-30 11:21:06 arvados.arv-web[4897] INFO:Sending refresh signal to container
-2015-01-30 11:21:07 arvados.arv-web[4897] INFO:Waiting for events
-
- -h2. Writing your own applications - -The @arvados/arv-web@ image serves Python and Ruby applications using Phusion Passenger and Apache @mod_passenger@. See "Phusion Passenger users guide for Apache":https://www.phusionpassenger.com/documentation/Users%20guide%20Apache.html for details, and look at the sample apps @arvados/services/arv-web/sample-wsgi-app@ and @arvados/services/arv-web/sample-rack-app@. - -You can serve CGI applications using standard Apache CGI support. See "Apache Tutorial: Dynamic Content with CGI":https://httpd.apache.org/docs/current/howto/cgi.html for details, and look at the sample app @arvados/services/arv-web/sample-cgi-app@. - -You can also serve static content from the @public@ directory of the collection. Look at @arvados/services/arv-web/sample-static-page@ for an example. If no @index.html@ is found in @public/@, it will render default Apache index pages, permitting simple browsing of the collection contents. - -h3. Custom images - -You can provide your own Docker image. The Docker image that will be used create the web application container is specified in the @docker_image@ file in the root of the collection. You can also specify @--image@ on the command @arv-web@ line to choose the docker image (this will override the contents of @docker_image@). - -h3. Reloading the web service - -Stopping the Docker container and starting it again can result in a small amount of downtime. When the collection containing a new or updated web application uses the same Docker image as the currently running web application, it is possible to avoid this downtime by keeping the existing container and only reloading the web server. This is accomplished by providing a file called @reload@ in the root of the collection, which should contain the commands necessary to reload the web server inside the container. diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go index e38a4775e8..3f4fb74822 100644 --- a/lib/boot/supervisor.go +++ b/lib/boot/supervisor.go @@ -601,7 +601,7 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error { } if len(svc.InternalURLs) == 0 { svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{ - arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{}, + {Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: {}, } } } diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go index c26309aca5..ba8a836dd0 100644 --- a/lib/cloud/azure/azure.go +++ b/lib/cloud/azure/azure.go @@ -677,6 +677,10 @@ func (az *azureInstanceSet) manageDisks() { } for ; response.NotDone(); err = response.Next() { + if err != nil { + az.logger.WithError(err).Warn("Error getting next page of disks") + return + } for _, d := range response.Values() { if d.DiskProperties.DiskState == compute.Unattached && d.Name != nil && re.MatchString(*d.Name) && diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index c392d8638d..6c47a068e9 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -689,6 +689,16 @@ Clusters: ProviderAppID: "" ProviderAppSecret: "" + Test: + # Authenticate users listed here in the config file. This + # feature is intended to be used in test environments, and + # should not be used in production. + Enable: false + Users: + SAMPLE: + Email: alice@example.com + Password: xyzzy + # The cluster ID to delegate the user database. When set, # logins on this cluster will be redirected to the login cluster # (login cluster must appear in RemoteClusters with Proxy: true) diff --git a/lib/config/export.go b/lib/config/export.go index 76f00c7652..b203dff26a 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -170,6 +170,9 @@ var whitelist = map[string]bool{ "Login.SSO.Enable": true, "Login.SSO.ProviderAppID": false, "Login.SSO.ProviderAppSecret": false, + "Login.Test": true, + "Login.Test.Enable": true, + "Login.Test.Users": false, "Login.TokenLifetime": false, "Mail": true, "Mail.EmailFrom": false, diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index f5004667b2..cbb73d97b8 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -695,6 +695,16 @@ Clusters: ProviderAppID: "" ProviderAppSecret: "" + Test: + # Authenticate users listed here in the config file. This + # feature is intended to be used in test environments, and + # should not be used in production. + Enable: false + Users: + SAMPLE: + Email: alice@example.com + Password: xyzzy + # The cluster ID to delegate the user database. When set, # logins on this cluster will be redirected to the login cluster # (login cluster must appear in RemoteClusters with Proxy: true) diff --git a/lib/controller/federation/federation_test.go b/lib/controller/federation/federation_test.go index 256afc8e6b..5079b402b7 100644 --- a/lib/controller/federation/federation_test.go +++ b/lib/controller/federation/federation_test.go @@ -38,7 +38,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) { ClusterID: "aaaaa", SystemRootToken: arvadostest.SystemRootToken, RemoteClusters: map[string]arvados.RemoteCluster{ - "aaaaa": arvados.RemoteCluster{ + "aaaaa": { Host: os.Getenv("ARVADOS_API_HOST"), }, }, diff --git a/lib/controller/localdb/login.go b/lib/controller/localdb/login.go index ee1ea56924..1267414842 100644 --- a/lib/controller/localdb/login.go +++ b/lib/controller/localdb/login.go @@ -33,8 +33,13 @@ func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) log wantSSO := cluster.Login.SSO.Enable wantPAM := cluster.Login.PAM.Enable wantLDAP := cluster.Login.LDAP.Enable + wantTest := cluster.Login.Test.Enable switch { - case wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP: + case 1 != countTrue(wantGoogle, wantOpenIDConnect, wantSSO, wantPAM, wantLDAP, wantTest): + return errorLoginController{ + error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, Login.LDAP, and Login.Test must be enabled"), + } + case wantGoogle: return &oidcLoginController{ Cluster: cluster, RailsProxy: railsProxy, @@ -45,7 +50,7 @@ func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) log EmailClaim: "email", EmailVerifiedClaim: "email_verified", } - case !wantGoogle && wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP: + case wantOpenIDConnect: return &oidcLoginController{ Cluster: cluster, RailsProxy: railsProxy, @@ -56,17 +61,29 @@ func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) log EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim, UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim, } - case !wantGoogle && !wantOpenIDConnect && wantSSO && !wantPAM && !wantLDAP: + case wantSSO: return &ssoLoginController{railsProxy} - case !wantGoogle && !wantOpenIDConnect && !wantSSO && wantPAM && !wantLDAP: + case wantPAM: return &pamLoginController{Cluster: cluster, RailsProxy: railsProxy} - case !wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && wantLDAP: + case wantLDAP: return &ldapLoginController{Cluster: cluster, RailsProxy: railsProxy} + case wantTest: + return &testLoginController{Cluster: cluster, RailsProxy: railsProxy} default: return errorLoginController{ - error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, and Login.LDAP must be enabled"), + error: errors.New("BUG: missing case in login controller setup switch"), + } + } +} + +func countTrue(vals ...bool) int { + n := 0 + for _, val := range vals { + if val { + n++ } } + return n } // Login and Logout are passed through to the wrapped railsProxy; diff --git a/lib/controller/localdb/login_ldap_test.go b/lib/controller/localdb/login_ldap_test.go index 700d757c27..bce1ecfcf2 100644 --- a/lib/controller/localdb/login_ldap_test.go +++ b/lib/controller/localdb/login_ldap_test.go @@ -64,7 +64,7 @@ func (s *LDAPSuite) SetUpSuite(c *check.C) { return []*godap.LDAPSimpleSearchResultEntry{} } return []*godap.LDAPSimpleSearchResultEntry{ - &godap.LDAPSimpleSearchResultEntry{ + { DN: "cn=" + req.FilterValue + "," + req.BaseDN, Attrs: map[string]interface{}{ "SN": req.FilterValue, diff --git a/lib/controller/localdb/login_testuser.go b/lib/controller/localdb/login_testuser.go new file mode 100644 index 0000000000..5a3d803b89 --- /dev/null +++ b/lib/controller/localdb/login_testuser.go @@ -0,0 +1,45 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + "errors" + "fmt" + + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/sirupsen/logrus" +) + +type testLoginController struct { + Cluster *arvados.Cluster + RailsProxy *railsProxy +} + +func (ctrl *testLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) { + return noopLogout(ctrl.Cluster, opts) +} + +func (ctrl *testLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) { + return arvados.LoginResponse{}, errors.New("interactive login is not available") +} + +func (ctrl *testLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) { + for username, user := range ctrl.Cluster.Login.Test.Users { + if (opts.Username == username || opts.Username == user.Email) && opts.Password == user.Password { + ctxlog.FromContext(ctx).WithFields(logrus.Fields{ + "username": username, + "email": user.Email, + }).Debug("test authentication succeeded") + return createAPIClientAuthorization(ctx, ctrl.RailsProxy, ctrl.Cluster.SystemRootToken, rpc.UserSessionAuthInfo{ + Username: username, + Email: user.Email, + }) + } + } + return arvados.APIClientAuthorization{}, fmt.Errorf("authentication failed for user %q with password len=%d", opts.Username, len(opts.Password)) +} diff --git a/lib/controller/localdb/login_testuser_test.go b/lib/controller/localdb/login_testuser_test.go new file mode 100644 index 0000000000..d2d651e205 --- /dev/null +++ b/lib/controller/localdb/login_testuser_test.go @@ -0,0 +1,94 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package localdb + +import ( + "context" + + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/lib/controller/rpc" + "git.arvados.org/arvados.git/lib/ctrlctx" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "github.com/jmoiron/sqlx" + check "gopkg.in/check.v1" +) + +var _ = check.Suite(&TestUserSuite{}) + +type TestUserSuite struct { + cluster *arvados.Cluster + ctrl *testLoginController + railsSpy *arvadostest.Proxy + db *sqlx.DB + + // transaction context + ctx context.Context + rollback func() error +} + +func (s *TestUserSuite) SetUpSuite(c *check.C) { + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() + c.Assert(err, check.IsNil) + s.cluster, err = cfg.GetCluster("") + c.Assert(err, check.IsNil) + s.cluster.Login.Test.Enable = true + s.cluster.Login.Test.Users = map[string]arvados.TestUser{ + "valid": {Email: "valid@example.com", Password: "v@l1d"}, + } + s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI) + s.ctrl = &testLoginController{ + Cluster: s.cluster, + RailsProxy: rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider), + } + s.db = arvadostest.DB(c, s.cluster) +} + +func (s *TestUserSuite) SetUpTest(c *check.C) { + tx, err := s.db.Beginx() + c.Assert(err, check.IsNil) + s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx) + s.rollback = tx.Rollback +} + +func (s *TestUserSuite) TearDownTest(c *check.C) { + if s.rollback != nil { + s.rollback() + } +} + +func (s *TestUserSuite) TestLogin(c *check.C) { + for _, trial := range []struct { + success bool + username string + password string + }{ + {false, "foo", "bar"}, + {false, "", ""}, + {false, "valid", ""}, + {false, "", "v@l1d"}, + {true, "valid", "v@l1d"}, + {true, "valid@example.com", "v@l1d"}, + } { + c.Logf("=== %#v", trial) + resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{ + Username: trial.username, + Password: trial.password, + }) + if trial.success { + c.Check(err, check.IsNil) + c.Check(resp.APIToken, check.Not(check.Equals), "") + c.Check(resp.UUID, check.Matches, `zzzzz-gj3su-.*`) + c.Check(resp.Scopes, check.DeepEquals, []string{"all"}) + + authinfo := getCallbackAuthInfo(c, s.railsSpy) + c.Check(authinfo.Email, check.Equals, "valid@example.com") + c.Check(authinfo.AlternateEmails, check.DeepEquals, []string(nil)) + } else { + c.Check(err, check.ErrorMatches, `authentication failed.*`) + } + } +} diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go index aa5f22a501..42decff31d 100644 --- a/lib/dispatchcloud/dispatcher_test.go +++ b/lib/dispatchcloud/dispatcher_test.go @@ -115,6 +115,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) { ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) { return ChooseInstanceType(s.cluster, ctr) }, + Logger: ctxlog.TestLogger(c), } for i := 0; i < 200; i++ { queue.Containers = append(queue.Containers, arvados.Container{ @@ -170,6 +171,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) { stubvm.CrunchRunCrashRate = 0.1 } } + s.stubDriver.Bugf = c.Errorf start := time.Now() go s.disp.run() @@ -303,7 +305,7 @@ func (s *DispatcherSuite) TestInstancesAPI(c *check.C) { time.Sleep(time.Millisecond) } c.Assert(len(sr.Items), check.Equals, 1) - c.Check(sr.Items[0].Instance, check.Matches, "stub.*") + c.Check(sr.Items[0].Instance, check.Matches, "inst.*") c.Check(sr.Items[0].WorkerState, check.Equals, "booting") c.Check(sr.Items[0].Price, check.Equals, 0.123) c.Check(sr.Items[0].LastContainerUUID, check.Equals, "") diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go index 4447f084a9..dddb974b32 100644 --- a/lib/dispatchcloud/scheduler/run_queue.go +++ b/lib/dispatchcloud/scheduler/run_queue.go @@ -88,6 +88,8 @@ tryrun: // a higher-priority container on the // same instance type. Don't let this // one sneak in ahead of it. + } else if sch.pool.KillContainer(ctr.UUID, "about to lock") { + logger.Info("not restarting yet: crunch-run process from previous attempt has not exited") } else if sch.pool.StartContainer(it, ctr) { // Success. } else { diff --git a/lib/dispatchcloud/scheduler/run_queue_test.go b/lib/dispatchcloud/scheduler/run_queue_test.go index 32c6b3b24d..992edddfba 100644 --- a/lib/dispatchcloud/scheduler/run_queue_test.go +++ b/lib/dispatchcloud/scheduler/run_queue_test.go @@ -83,8 +83,9 @@ func (p *stubPool) ForgetContainer(uuid string) { func (p *stubPool) KillContainer(uuid, reason string) bool { p.Lock() defer p.Unlock() - delete(p.running, uuid) - return true + defer delete(p.running, uuid) + t, ok := p.running[uuid] + return ok && t.IsZero() } func (p *stubPool) Shutdown(arvados.InstanceType) bool { p.shutdowns++ diff --git a/lib/dispatchcloud/scheduler/sync.go b/lib/dispatchcloud/scheduler/sync.go index 116ca76431..fc683505f9 100644 --- a/lib/dispatchcloud/scheduler/sync.go +++ b/lib/dispatchcloud/scheduler/sync.go @@ -109,13 +109,17 @@ func (sch *Scheduler) cancel(uuid string, reason string) { } func (sch *Scheduler) kill(uuid string, reason string) { + if !sch.uuidLock(uuid, "kill") { + return + } + defer sch.uuidUnlock(uuid) sch.pool.KillContainer(uuid, reason) sch.pool.ForgetContainer(uuid) } func (sch *Scheduler) requeue(ent container.QueueEnt, reason string) { uuid := ent.Container.UUID - if !sch.uuidLock(uuid, "cancel") { + if !sch.uuidLock(uuid, "requeue") { return } defer sch.uuidUnlock(uuid) diff --git a/lib/dispatchcloud/test/queue.go b/lib/dispatchcloud/test/queue.go index 11d410fb1b..74b84122f2 100644 --- a/lib/dispatchcloud/test/queue.go +++ b/lib/dispatchcloud/test/queue.go @@ -11,6 +11,7 @@ import ( "git.arvados.org/arvados.git/lib/dispatchcloud/container" "git.arvados.org/arvados.git/sdk/go/arvados" + "github.com/sirupsen/logrus" ) // Queue is a test stub for container.Queue. The caller specifies the @@ -23,6 +24,8 @@ type Queue struct { // must not be nil. ChooseType func(*arvados.Container) (arvados.InstanceType, error) + Logger logrus.FieldLogger + entries map[string]container.QueueEnt updTime time.Time subscribers map[<-chan struct{}]chan struct{} @@ -166,13 +169,36 @@ func (q *Queue) Notify(upd arvados.Container) bool { defer q.mtx.Unlock() for i, ctr := range q.Containers { if ctr.UUID == upd.UUID { - if ctr.State != arvados.ContainerStateComplete && ctr.State != arvados.ContainerStateCancelled { + if allowContainerUpdate[ctr.State][upd.State] { q.Containers[i] = upd return true + } else { + if q.Logger != nil { + q.Logger.WithField("ContainerUUID", ctr.UUID).Infof("test.Queue rejected update from %s to %s", ctr.State, upd.State) + } + return false } - return false } } q.Containers = append(q.Containers, upd) return true } + +var allowContainerUpdate = map[arvados.ContainerState]map[arvados.ContainerState]bool{ + arvados.ContainerStateQueued: map[arvados.ContainerState]bool{ + arvados.ContainerStateQueued: true, + arvados.ContainerStateLocked: true, + arvados.ContainerStateCancelled: true, + }, + arvados.ContainerStateLocked: map[arvados.ContainerState]bool{ + arvados.ContainerStateQueued: true, + arvados.ContainerStateLocked: true, + arvados.ContainerStateRunning: true, + arvados.ContainerStateCancelled: true, + }, + arvados.ContainerStateRunning: map[arvados.ContainerState]bool{ + arvados.ContainerStateRunning: true, + arvados.ContainerStateCancelled: true, + arvados.ContainerStateComplete: true, + }, +} diff --git a/lib/dispatchcloud/test/stub_driver.go b/lib/dispatchcloud/test/stub_driver.go index 7a1f423016..f6e06d3f7c 100644 --- a/lib/dispatchcloud/test/stub_driver.go +++ b/lib/dispatchcloud/test/stub_driver.go @@ -34,6 +34,11 @@ type StubDriver struct { // VM's error rate and other behaviors. SetupVM func(*StubVM) + // Bugf, if set, is called if a bug is detected in the caller + // or stub. Typically set to (*check.C)Errorf. If unset, + // logger.Warnf is called instead. + Bugf func(string, ...interface{}) + // StubVM's fake crunch-run uses this Queue to read and update // container state. Queue *Queue @@ -99,6 +104,7 @@ type StubInstanceSet struct { allowCreateCall time.Time allowInstancesCall time.Time + lastInstanceID int } func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID, tags cloud.InstanceTags, cmd cloud.InitCommand, authKey ssh.PublicKey) (cloud.Instance, error) { @@ -120,9 +126,10 @@ func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID, if authKey != nil { ak = append([]ssh.PublicKey{authKey}, ak...) } + sis.lastInstanceID++ svm := &StubVM{ sis: sis, - id: cloud.InstanceID(fmt.Sprintf("stub-%s-%x", it.ProviderType, math_rand.Int63())), + id: cloud.InstanceID(fmt.Sprintf("inst%d,%s", sis.lastInstanceID, it.ProviderType)), tags: copyTags(tags), providerType: it.ProviderType, initCommand: cmd, @@ -263,49 +270,68 @@ func (svm *StubVM) Exec(env map[string]string, command string, stdin io.Reader, }) logger.Printf("[test] starting crunch-run stub") go func() { + var ctr arvados.Container + var started, completed bool + defer func() { + logger.Print("[test] exiting crunch-run stub") + svm.Lock() + defer svm.Unlock() + if svm.running[uuid] != pid { + if !completed { + bugf := svm.sis.driver.Bugf + if bugf == nil { + bugf = logger.Warnf + } + bugf("[test] StubDriver bug or caller bug: pid %d exiting, running[%s]==%d", pid, uuid, svm.running[uuid]) + } + } else { + delete(svm.running, uuid) + } + if !completed { + logger.WithField("State", ctr.State).Print("[test] crashing crunch-run stub") + if started && svm.CrashRunningContainer != nil { + svm.CrashRunningContainer(ctr) + } + } + }() + crashluck := math_rand.Float64() + wantCrash := crashluck < svm.CrunchRunCrashRate + wantCrashEarly := crashluck < svm.CrunchRunCrashRate/2 + ctr, ok := queue.Get(uuid) if !ok { logger.Print("[test] container not in queue") return } - defer func() { - if ctr.State == arvados.ContainerStateRunning && svm.CrashRunningContainer != nil { - svm.CrashRunningContainer(ctr) - } - }() - - if crashluck > svm.CrunchRunCrashRate/2 { - time.Sleep(time.Duration(math_rand.Float64()*20) * time.Millisecond) - ctr.State = arvados.ContainerStateRunning - if !queue.Notify(ctr) { - ctr, _ = queue.Get(uuid) - logger.Print("[test] erroring out because state=Running update was rejected") - return - } - } - time.Sleep(time.Duration(math_rand.Float64()*20) * time.Millisecond) svm.Lock() - defer svm.Unlock() - if svm.running[uuid] != pid { - logger.Print("[test] container was killed") + killed := svm.running[uuid] != pid + svm.Unlock() + if killed || wantCrashEarly { return } - delete(svm.running, uuid) - if crashluck < svm.CrunchRunCrashRate { + ctr.State = arvados.ContainerStateRunning + started = queue.Notify(ctr) + if !started { + ctr, _ = queue.Get(uuid) + logger.Print("[test] erroring out because state=Running update was rejected") + return + } + + if wantCrash { logger.WithField("State", ctr.State).Print("[test] crashing crunch-run stub") - } else { - if svm.ExecuteContainer != nil { - ctr.ExitCode = svm.ExecuteContainer(ctr) - } - logger.WithField("ExitCode", ctr.ExitCode).Print("[test] exiting crunch-run stub") - ctr.State = arvados.ContainerStateComplete - go queue.Notify(ctr) + return + } + if svm.ExecuteContainer != nil { + ctr.ExitCode = svm.ExecuteContainer(ctr) } + logger.WithField("ExitCode", ctr.ExitCode).Print("[test] completing container") + ctr.State = arvados.ContainerStateComplete + completed = queue.Notify(ctr) }() return 0 } diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 86673320da..d98ffd18ed 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -177,6 +177,10 @@ type Cluster struct { ProviderAppID string ProviderAppSecret string } + Test struct { + Enable bool + Users map[string]TestUser + } LoginCluster string RemoteTokenRefresh Duration TokenLifetime Duration @@ -331,6 +335,11 @@ type Service struct { ExternalURL URL } +type TestUser struct { + Email string + Password string +} + // URL is a url.URL that is also usable as a JSON key/value. type URL url.URL diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go index 0edc48162b..060b57b493 100644 --- a/sdk/go/arvados/fs_collection.go +++ b/sdk/go/arvados/fs_collection.go @@ -568,8 +568,6 @@ func (fn *filenode) Write(p []byte, startPtr filenodePtr) (n int, ptr filenodePt seg.Truncate(len(cando)) fn.memsize += int64(len(cando)) fn.segments[cur] = seg - cur++ - prev++ } } @@ -1109,9 +1107,9 @@ func (dn *dirnode) loadManifest(txt string) error { // situation might be rare anyway) segIdx, pos = 0, 0 } - for next := int64(0); segIdx < len(segments); segIdx++ { + for ; segIdx < len(segments); segIdx++ { seg := segments[segIdx] - next = pos + int64(seg.Len()) + next := pos + int64(seg.Len()) if next <= offset || seg.Len() == 0 { pos = next continue diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go index cb2e54bda2..86facd681e 100644 --- a/sdk/go/arvados/fs_project_test.go +++ b/sdk/go/arvados/fs_project_test.go @@ -214,6 +214,7 @@ func (s *SiteFSSuite) TestProjectUpdatedByOther(c *check.C) { // Ensure collection was flushed by Sync var latest Collection err = s.client.RequestAndDecode(&latest, "GET", "arvados/v1/collections/"+oob.UUID, nil, nil) + c.Check(err, check.IsNil) c.Check(latest.ManifestText, check.Matches, `.*:test.txt.*\n`) // Delete test.txt behind s.fs's back by updating the diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index a1801b2145..2604b02b17 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -535,6 +535,7 @@ func (s *StandaloneSuite) TestGetEmptyBlock(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) diff --git a/services/api/config/application.rb b/services/api/config/application.rb index 369294e8a7..b28ae0e071 100644 --- a/services/api/config/application.rb +++ b/services/api/config/application.rb @@ -16,7 +16,7 @@ require "sprockets/railtie" require "rails/test_unit/railtie" # Skipping the following: # * ActionCable (new in Rails 5.0) as it adds '/cable' routes that we're not using -# * Skip ActiveStorage (new in Rails 5.1) +# * ActiveStorage (new in Rails 5.1) require 'digest' diff --git a/services/arv-git-httpd/gitolite_test.go b/services/arv-git-httpd/gitolite_test.go index 5f3cc608c3..fb0fc0d783 100644 --- a/services/arv-git-httpd/gitolite_test.go +++ b/services/arv-git-httpd/gitolite_test.go @@ -54,7 +54,7 @@ func (s *GitoliteSuite) SetUpTest(c *check.C) { s.cluster, err = cfg.GetCluster("") c.Assert(err, check.Equals, nil) - s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}} + s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: "localhost:0"}: {}} s.cluster.TLS.Insecure = true s.cluster.Git.GitCommand = "/usr/share/gitolite3/gitolite-shell" s.cluster.Git.GitoliteHome = s.gitoliteHome diff --git a/services/arv-web/README b/services/arv-web/README deleted file mode 100644 index eaf7624dc4..0000000000 --- a/services/arv-web/README +++ /dev/null @@ -1,6 +0,0 @@ -arv-web enables you to run a custom web service using the contents of an -Arvados collection. - -See "Using arv-web" in the Arvados user guide: - -http://doc.arvados.org/user/topics/arv-web.html diff --git a/services/arv-web/arv-web.py b/services/arv-web/arv-web.py deleted file mode 100755 index 55b710a754..0000000000 --- a/services/arv-web/arv-web.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -# arv-web enables you to run a custom web service from the contents of an Arvados collection. -# -# See http://doc.arvados.org/user/topics/arv-web.html - -import arvados -from arvados.safeapi import ThreadSafeApiCache -import subprocess -from arvados_fuse import Operations, CollectionDirectory -import tempfile -import os -import llfuse -import threading -import Queue -import argparse -import logging -import signal -import sys -import functools - -logger = logging.getLogger('arvados.arv-web') -logger.setLevel(logging.INFO) - -class ArvWeb(object): - def __init__(self, project, docker_image, port): - self.project = project - self.loop = True - self.cid = None - self.prev_docker_image = None - self.mountdir = None - self.collection = None - self.override_docker_image = docker_image - self.port = port - self.evqueue = Queue.Queue() - self.api = ThreadSafeApiCache(arvados.config.settings()) - - if arvados.util.group_uuid_pattern.match(project) is None: - raise arvados.errors.ArgumentError("Project uuid is not valid") - - collections = self.api.collections().list(filters=[["owner_uuid", "=", project]], - limit=1, - order='modified_at desc').execute()['items'] - self.newcollection = collections[0]['uuid'] if collections else None - - self.ws = arvados.events.subscribe(self.api, [["object_uuid", "is_a", "arvados#collection"]], self.on_message) - - def check_docker_running(self): - # It would be less hacky to use "docker events" than poll "docker ps" - # but that would require writing a bigger pile of code. - if self.cid: - ps = subprocess.check_output(["docker", "ps", "--no-trunc=true", "--filter=status=running"]) - for l in ps.splitlines(): - if l.startswith(self.cid): - return True - return False - - # Handle messages from Arvados event bus. - def on_message(self, ev): - if 'event_type' in ev: - old_attr = None - if 'old_attributes' in ev['properties'] and ev['properties']['old_attributes']: - old_attr = ev['properties']['old_attributes'] - if self.project not in (ev['properties']['new_attributes']['owner_uuid'], - old_attr['owner_uuid'] if old_attr else None): - return - - et = ev['event_type'] - if ev['event_type'] == 'update': - if ev['properties']['new_attributes']['owner_uuid'] != ev['properties']['old_attributes']['owner_uuid']: - if self.project == ev['properties']['new_attributes']['owner_uuid']: - et = 'add' - else: - et = 'remove' - if ev['properties']['new_attributes']['trash_at'] is not None: - et = 'remove' - - self.evqueue.put((self.project, et, ev['object_uuid'])) - - # Run an arvados_fuse mount under the control of the local process. This lets - # us switch out the contents of the directory without having to unmount and - # remount. - def run_fuse_mount(self): - self.mountdir = tempfile.mkdtemp() - - self.operations = Operations(os.getuid(), os.getgid(), self.api, "utf-8") - self.cdir = CollectionDirectory(llfuse.ROOT_INODE, self.operations.inodes, self.api, 2, self.collection) - self.operations.inodes.add_entry(self.cdir) - - # Initialize the fuse connection - llfuse.init(self.operations, self.mountdir, ['allow_other']) - - t = threading.Thread(None, llfuse.main) - t.start() - - # wait until the driver is finished initializing - self.operations.initlock.wait() - - def mount_collection(self): - if self.newcollection != self.collection: - self.collection = self.newcollection - if not self.mountdir and self.collection: - self.run_fuse_mount() - - if self.mountdir: - with llfuse.lock: - self.cdir.clear() - # Switch the FUSE directory object so that it stores - # the newly selected collection - if self.collection: - logger.info("Mounting %s", self.collection) - else: - logger.info("Mount is empty") - self.cdir.change_collection(self.collection) - - - def stop_docker(self): - if self.cid: - logger.info("Stopping Docker container") - subprocess.call(["docker", "stop", self.cid]) - self.cid = None - - def run_docker(self): - try: - if self.collection is None: - self.stop_docker() - return - - docker_image = None - if self.override_docker_image: - docker_image = self.override_docker_image - else: - try: - with llfuse.lock: - if "docker_image" in self.cdir: - docker_image = self.cdir["docker_image"].readfrom(0, 1024).strip() - except IOError as e: - pass - - has_reload = False - try: - with llfuse.lock: - has_reload = "reload" in self.cdir - except IOError as e: - pass - - if docker_image is None: - logger.error("Collection must contain a file 'docker_image' or must specify --image on the command line.") - self.stop_docker() - return - - if docker_image == self.prev_docker_image and self.cid is not None and has_reload: - logger.info("Running container reload command") - subprocess.check_call(["docker", "exec", self.cid, "/mnt/reload"]) - return - - self.stop_docker() - - logger.info("Starting Docker container %s", docker_image) - self.cid = subprocess.check_output(["docker", "run", - "--detach=true", - "--publish=%i:80" % (self.port), - "--volume=%s:/mnt:ro" % self.mountdir, - docker_image]).strip() - - self.prev_docker_image = docker_image - logger.info("Container id %s", self.cid) - - except subprocess.CalledProcessError: - self.cid = None - - def wait_for_events(self): - if not self.cid: - logger.warning("No service running! Will wait for a new collection to appear in the project.") - else: - logger.info("Waiting for events") - - running = True - self.loop = True - while running: - # Main run loop. Wait on project events, signals, or the - # Docker container stopping. - - try: - # Poll the queue with a 1 second timeout, if we have no - # timeout the Python runtime doesn't have a chance to - # process SIGINT or SIGTERM. - eq = self.evqueue.get(True, 1) - logger.info("%s %s", eq[1], eq[2]) - self.newcollection = self.collection - if eq[1] in ('add', 'update', 'create'): - self.newcollection = eq[2] - elif eq[1] == 'remove': - collections = self.api.collections().list(filters=[["owner_uuid", "=", self.project]], - limit=1, - order='modified_at desc').execute()['items'] - self.newcollection = collections[0]['uuid'] if collections else None - running = False - except Queue.Empty: - pass - - if self.cid and not self.check_docker_running(): - logger.warning("Service has terminated. Will try to restart.") - self.cid = None - running = False - - - def run(self): - try: - while self.loop: - self.loop = False - self.mount_collection() - try: - self.run_docker() - self.wait_for_events() - except (KeyboardInterrupt): - logger.info("Got keyboard interrupt") - self.ws.close() - self.loop = False - except Exception as e: - logger.exception("Caught fatal exception, shutting down") - self.ws.close() - self.loop = False - finally: - self.stop_docker() - - if self.mountdir: - logger.info("Unmounting") - subprocess.call(["fusermount", "-u", self.mountdir]) - os.rmdir(self.mountdir) - - -def main(argv): - parser = argparse.ArgumentParser() - parser.add_argument('--project-uuid', type=str, required=True, help="Project uuid to watch") - parser.add_argument('--port', type=int, default=8080, help="Host port to listen on (default 8080)") - parser.add_argument('--image', type=str, help="Docker image to run") - - args = parser.parse_args(argv) - - signal.signal(signal.SIGTERM, lambda signal, frame: sys.exit(0)) - - try: - arvweb = ArvWeb(args.project_uuid, args.image, args.port) - arvweb.run() - except arvados.errors.ArgumentError as e: - logger.error(e) - return 1 - - return 0 - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/services/arv-web/sample-cgi-app/docker_image b/services/arv-web/sample-cgi-app/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-cgi-app/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-cgi-app/public/.htaccess b/services/arv-web/sample-cgi-app/public/.htaccess deleted file mode 100644 index e5145bd37d..0000000000 --- a/services/arv-web/sample-cgi-app/public/.htaccess +++ /dev/null @@ -1,3 +0,0 @@ -Options +ExecCGI -AddHandler cgi-script .cgi -DirectoryIndex index.cgi diff --git a/services/arv-web/sample-cgi-app/public/index.cgi b/services/arv-web/sample-cgi-app/public/index.cgi deleted file mode 100755 index 57bc2a9a01..0000000000 --- a/services/arv-web/sample-cgi-app/public/index.cgi +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/perl - -print "Content-type: text/html\n\n"; -print "Hello world from perl!"; diff --git a/services/arv-web/sample-cgi-app/tmp/.keepkeep b/services/arv-web/sample-cgi-app/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-rack-app/config.ru b/services/arv-web/sample-rack-app/config.ru deleted file mode 100644 index 65f3c7ca36..0000000000 --- a/services/arv-web/sample-rack-app/config.ru +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -app = proc do |env| - [200, { "Content-Type" => "text/html" }, ["hello world from ruby"]] -end -run app diff --git a/services/arv-web/sample-rack-app/docker_image b/services/arv-web/sample-rack-app/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-rack-app/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-rack-app/public/.keepkeep b/services/arv-web/sample-rack-app/public/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-rack-app/tmp/.keepkeep b/services/arv-web/sample-rack-app/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-static-page/docker_image b/services/arv-web/sample-static-page/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-static-page/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-static-page/public/index.html b/services/arv-web/sample-static-page/public/index.html deleted file mode 100644 index e8608a5ebe..0000000000 --- a/services/arv-web/sample-static-page/public/index.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - arv-web sample - -

Hello world static page

- - diff --git a/services/arv-web/sample-static-page/tmp/.keepkeep b/services/arv-web/sample-static-page/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-wsgi-app/docker_image b/services/arv-web/sample-wsgi-app/docker_image deleted file mode 100644 index 57f344fcd7..0000000000 --- a/services/arv-web/sample-wsgi-app/docker_image +++ /dev/null @@ -1 +0,0 @@ -arvados/arv-web \ No newline at end of file diff --git a/services/arv-web/sample-wsgi-app/passenger_wsgi.py b/services/arv-web/sample-wsgi-app/passenger_wsgi.py deleted file mode 100644 index faec3c23cd..0000000000 --- a/services/arv-web/sample-wsgi-app/passenger_wsgi.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -def application(environ, start_response): - start_response('200 OK', [('Content-Type', 'text/plain')]) - return [b"hello world from python!\n"] diff --git a/services/arv-web/sample-wsgi-app/public/.keepkeep b/services/arv-web/sample-wsgi-app/public/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/arv-web/sample-wsgi-app/tmp/.keepkeep b/services/arv-web/sample-wsgi-app/tmp/.keepkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go index 915924e288..963948cc6b 100644 --- a/services/keep-web/handler.go +++ b/services/keep-web/handler.go @@ -185,10 +185,6 @@ var ( func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) { h.setupOnce.Do(h.setup) - remoteAddr := r.RemoteAddr - if xff := r.Header.Get("X-Forwarded-For"); xff != "" { - remoteAddr = xff + "," + remoteAddr - } if xfp := r.Header.Get("X-Forwarded-Proto"); xfp != "" && xfp != "http" { r.URL.Scheme = xfp } diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go index 12e294d933..01bc8b7047 100644 --- a/services/keep-web/s3.go +++ b/services/keep-web/s3.go @@ -205,6 +205,54 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool { } w.WriteHeader(http.StatusOK) return true + case r.Method == http.MethodDelete: + if !objectNameGiven || r.URL.Path == "/" { + http.Error(w, "missing object name in DELETE request", http.StatusBadRequest) + return true + } + fspath := "by_id" + r.URL.Path + if strings.HasSuffix(fspath, "/") { + fspath = strings.TrimSuffix(fspath, "/") + fi, err := fs.Stat(fspath) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNoContent) + return true + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } else if !fi.IsDir() { + // if "foo" exists and is a file, then + // "foo/" doesn't exist, so we say + // delete was successful. + w.WriteHeader(http.StatusNoContent) + return true + } + } else if fi, err := fs.Stat(fspath); err == nil && fi.IsDir() { + // if "foo" is a dir, it is visible via S3 + // only as "foo/", not "foo" -- so we leave + // the dir alone and return 204 to indicate + // that "foo" does not exist. + w.WriteHeader(http.StatusNoContent) + return true + } + err = fs.Remove(fspath) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNoContent) + return true + } + if err != nil { + err = fmt.Errorf("rm failed: %w", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return true + } + err = fs.Sync() + if err != nil { + err = fmt.Errorf("sync failed: %w", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + w.WriteHeader(http.StatusNoContent) + return true default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return true diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go index 73553ff4d3..b82f1efd78 100644 --- a/services/keep-web/s3_test.go +++ b/services/keep-web/s3_test.go @@ -260,6 +260,43 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) { } } +func (s *IntegrationSuite) TestS3CollectionDeleteObject(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + s.testS3DeleteObject(c, stage.collbucket, "") +} +func (s *IntegrationSuite) TestS3ProjectDeleteObject(c *check.C) { + stage := s.s3setup(c) + defer stage.teardown(c) + s.testS3DeleteObject(c, stage.projbucket, stage.coll.Name+"/") +} +func (s *IntegrationSuite) testS3DeleteObject(c *check.C, bucket *s3.Bucket, prefix string) { + s.testServer.Config.cluster.Collections.S3FolderObjects = true + for _, trial := range []struct { + path string + }{ + {"/"}, + {"nonexistentfile"}, + {"emptyfile"}, + {"sailboat.txt"}, + {"sailboat.txt/"}, + {"emptydir"}, + {"emptydir/"}, + } { + objname := prefix + trial.path + comment := check.Commentf("objname %q", objname) + + err := bucket.Del(objname) + if trial.path == "/" { + c.Check(err, check.NotNil) + continue + } + c.Check(err, check.IsNil, comment) + _, err = bucket.GetReader(objname) + c.Check(err, check.NotNil, comment) + } +} + func (s *IntegrationSuite) TestS3CollectionPutObjectFailure(c *check.C) { stage := s.s3setup(c) defer stage.teardown(c) @@ -345,6 +382,7 @@ func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) { defer stage.teardown(c) for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} { req, err := http.NewRequest("GET", bucket.URL("/"), nil) + c.Check(err, check.IsNil) req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none") req.URL.RawQuery = "versioning" resp, err := http.DefaultClient.Do(req)