From: Peter Amstutz <peter.amstutz@curoverse.com>
Date: Fri, 28 Apr 2017 14:40:06 +0000 (-0400)
Subject: Merge branch '11549-cwl-smarter-mounts' closes #11549
X-Git-Tag: 1.1.0~283
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/e39a7d5704932cbec606e85cdca50aa38a1ed053?hp=f5f5de0ae41e12738a380c422417d5a5e5af7f09

Merge branch '11549-cwl-smarter-mounts' closes #11549
---

diff --git a/sdk/go/arvadosclient/arvadosclient_test.go b/sdk/go/arvadosclient/arvadosclient_test.go
index 8e32efe4f9..54591d30ba 100644
--- a/sdk/go/arvadosclient/arvadosclient_test.go
+++ b/sdk/go/arvadosclient/arvadosclient_test.go
@@ -2,13 +2,14 @@ package arvadosclient
 
 import (
 	"fmt"
-	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
-	. "gopkg.in/check.v1"
 	"net"
 	"net/http"
 	"os"
 	"testing"
 	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	. "gopkg.in/check.v1"
 )
 
 // Gocheck boilerplate
@@ -168,7 +169,7 @@ func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
 			Dict{"log": Dict{"bogus_attr": "foo"}},
 			&getback)
 		c.Assert(err, ErrorMatches, "arvados API server error: .*")
-		c.Assert(err, ErrorMatches, ".*unknown attribute: bogus_attr.*")
+		c.Assert(err, ErrorMatches, ".*unknown attribute(: | ')bogus_attr.*")
 		c.Assert(err, FitsTypeOf, APIServerError{})
 		c.Assert(err.(APIServerError).HttpStatusCode, Equals, 422)
 	}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index d10e60c22f..4c6c2d3b9d 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -314,10 +314,7 @@ def run(leave_running_atexit=False):
     env = os.environ.copy()
     env['RAILS_ENV'] = 'test'
     env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
-    if env.get('ARVADOS_TEST_EXPERIMENTAL_WS'):
-        env.pop('ARVADOS_WEBSOCKETS', None)
-    else:
-        env['ARVADOS_WEBSOCKETS'] = 'yes'
+    env.pop('ARVADOS_WEBSOCKETS', None)
     env.pop('ARVADOS_TEST_API_HOST', None)
     env.pop('ARVADOS_API_HOST', None)
     env.pop('ARVADOS_API_HOST_INSECURE', None)
diff --git a/sdk/ruby/lib/arvados.rb b/sdk/ruby/lib/arvados.rb
index 7a3f4b4226..8b0406539f 100644
--- a/sdk/ruby/lib/arvados.rb
+++ b/sdk/ruby/lib/arvados.rb
@@ -11,18 +11,7 @@ ActiveSupport::Inflector.inflections do |inflect|
   inflect.irregular 'human', 'humans'
 end
 
-module Kernel
-  def suppress_warnings
-    original_verbosity = $VERBOSE
-    $VERBOSE = nil
-    result = yield
-    $VERBOSE = original_verbosity
-    return result
-  end
-end
-
 class Arvados
-
   class TransactionFailedError < StandardError
   end
 
@@ -262,4 +251,16 @@ class Arvados
       @attributes = j
     end
   end
+
+  protected
+
+  def suppress_warnings
+    original_verbosity = $VERBOSE
+    begin
+      $VERBOSE = nil
+      yield
+    ensure
+      $VERBOSE = original_verbosity
+    end
+  end
 end
diff --git a/sdk/ruby/lib/arvados/keep.rb b/sdk/ruby/lib/arvados/keep.rb
index 489eeeeebb..00f4f36d2b 100644
--- a/sdk/ruby/lib/arvados/keep.rb
+++ b/sdk/ruby/lib/arvados/keep.rb
@@ -35,7 +35,7 @@ module Keep
     def self.parse(tok)
       begin
         Locator.parse!(tok)
-      rescue ArgumentError => e
+      rescue ArgumentError
         nil
       end
     end
@@ -112,7 +112,7 @@ module Keep
         stream_name = nil
         block_tokens = []
         file_tokens = []
-        line.scan /\S+/ do |token|
+        line.scan(/\S+/) do |token|
           if stream_name.nil?
             stream_name = unescape token
           elsif file_tokens.empty? and Locator.valid? token
@@ -152,7 +152,7 @@ module Keep
       @text.each_line do |line|
         stream_name = nil
         in_file_tokens = false
-        line.scan /\S+/ do |token|
+        line.scan(/\S+/) do |token|
           if stream_name.nil?
             stream_name = unescape token
           elsif in_file_tokens or not Locator.valid? token
diff --git a/services/api/.gitignore b/services/api/.gitignore
index 29eb939002..da4c39d901 100644
--- a/services/api/.gitignore
+++ b/services/api/.gitignore
@@ -2,8 +2,7 @@
 /db/*.sqlite3
 
 # Ignore all logfiles and tempfiles.
-/log/*.log
-/log/*.log.gz
+/log
 /tmp
 
 # Sensitive files and local configuration
diff --git a/services/api/Gemfile b/services/api/Gemfile
index 39f217f100..4474c49e6a 100644
--- a/services/api/Gemfile
+++ b/services/api/Gemfile
@@ -1,9 +1,8 @@
 source 'https://rubygems.org'
 
-gem 'rails', '~> 3.2'
-
-# Bundle edge Rails instead:
-# gem 'rails',     :git => 'git://github.com/rails/rails.git'
+gem 'rails', '~> 4.0'
+gem 'responders', '~> 2.0'
+gem 'protected_attributes'
 
 group :test, :development do
   gem 'factory_girl_rails'
@@ -21,45 +20,26 @@ end
 # pg is the only supported database driver.
 gem 'pg'
 
-# Start using multi_json once we are on Rails 3.2;
-# Rails 3.1 has a dependency on multi_json < 1.3.0 but we need version 1.3.4 to
-# fix bug https://github.com/collectiveidea/json_spec/issues/27
 gem 'multi_json'
 gem 'oj'
+gem 'oj_mimic_json'
 
-# Gems used only for assets and not required
-# in production environments by default.
-group :assets do
-  gem 'sass-rails',   '~> 3.2'
-  gem 'coffee-rails', '~> 3.2'
-
-  # See https://github.com/sstephenson/execjs#readme for more supported runtimes
-  gem 'therubyracer'
-
-  gem 'uglifier', '~> 2.0'
-end
+# for building assets
+gem 'sass-rails',   '~> 4.0'
+gem 'coffee-rails', '~> 4.0'
+gem 'therubyracer'
+gem 'uglifier', '~> 2.0'
 
 gem 'jquery-rails'
 
-# To use ActiveModel has_secure_password
-# gem 'bcrypt-ruby', '~> 3.0.0'
-
-# Use unicorn as the web server
-# gem 'unicorn'
-
-# Deploy with Capistrano
-# gem 'capistrano'
-
-# To use debugger
-# gem 'ruby-debug'
-
 gem 'rvm-capistrano', :group => :test
 
 gem 'acts_as_api'
 
 gem 'passenger'
 
-gem 'omniauth', '~> 1.1'
+# Restricted because omniauth >= 1.5.0 requires Ruby >= 2.1.9:
+gem 'omniauth', '~> 1.4.0'
 gem 'omniauth-oauth2', '~> 1.1'
 
 gem 'andand'
@@ -69,21 +49,20 @@ gem 'test_after_commit', :group => :test
 gem 'trollop'
 gem 'faye-websocket'
 
-gem 'themes_for_rails'
+gem 'themes_for_rails', git: 'https://github.com/curoverse/themes_for_rails'
 
 gem 'arvados', '>= 0.1.20150615153458'
 gem 'arvados-cli', '>= 0.1.20161017193526'
 
-# pg_power lets us use partial indexes in schema.rb in Rails 3
-gem 'pg_power'
-
 gem 'puma', '~> 2.0'
 gem 'sshkey'
 gem 'safe_yaml'
 gem 'lograge'
 gem 'logstash-event'
 
+gem 'rails-observers'
+
 # Install any plugin gems
-Dir.glob(File.join(File.dirname(__FILE__), 'lib', '**', "Gemfile")) do |gemfile|
-    eval(IO.read(gemfile), binding)
+Dir.glob(File.join(File.dirname(__FILE__), 'lib', '**', "Gemfile")) do |f|
+    eval(IO.read(f), binding)
 end
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 9c9c4ae9e5..c59e59d63c 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -1,49 +1,65 @@
+GIT
+  remote: https://github.com/curoverse/themes_for_rails
+  revision: 61154877047d2346890bda0b7be5827cf51a6a76
+  specs:
+    themes_for_rails (0.5.1)
+      rails (>= 3.0.0)
+
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (3.2.22.5)
-      actionpack (= 3.2.22.5)
-      mail (~> 2.5.4)
-    actionpack (3.2.22.5)
-      activemodel (= 3.2.22.5)
-      activesupport (= 3.2.22.5)
-      builder (~> 3.0.0)
+    actionmailer (4.2.5.2)
+      actionpack (= 4.2.5.2)
+      actionview (= 4.2.5.2)
+      activejob (= 4.2.5.2)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.5.2)
+      actionview (= 4.2.5.2)
+      activesupport (= 4.2.5.2)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (4.2.5.2)
+      activesupport (= 4.2.5.2)
+      builder (~> 3.1)
       erubis (~> 2.7.0)
-      journey (~> 1.0.4)
-      rack (~> 1.4.5)
-      rack-cache (~> 1.2)
-      rack-test (~> 0.6.1)
-      sprockets (~> 2.2.1)
-    activemodel (3.2.22.5)
-      activesupport (= 3.2.22.5)
-      builder (~> 3.0.0)
-    activerecord (3.2.22.5)
-      activemodel (= 3.2.22.5)
-      activesupport (= 3.2.22.5)
-      arel (~> 3.0.2)
-      tzinfo (~> 0.3.29)
-    activeresource (3.2.22.5)
-      activemodel (= 3.2.22.5)
-      activesupport (= 3.2.22.5)
-    activesupport (3.2.22.5)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
-    acts_as_api (0.4.3)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activejob (4.2.5.2)
+      activesupport (= 4.2.5.2)
+      globalid (>= 0.3.0)
+    activemodel (4.2.5.2)
+      activesupport (= 4.2.5.2)
+      builder (~> 3.1)
+    activerecord (4.2.5.2)
+      activemodel (= 4.2.5.2)
+      activesupport (= 4.2.5.2)
+      arel (~> 6.0)
+    activesupport (4.2.5.2)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    acts_as_api (1.0.0)
       activemodel (>= 3.0.0)
       activesupport (>= 3.0.0)
       rack (>= 1.1.0)
-    addressable (2.4.0)
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
     andand (1.3.3)
-    arel (3.0.3)
-    arvados (0.1.20160513152536)
+    arel (6.0.4)
+    arvados (0.1.20170215224121)
       activesupport (>= 3, < 4.2.6)
       andand (~> 1.3, >= 1.3.3)
       google-api-client (>= 0.7, < 0.8.9)
       i18n (~> 0)
       json (~> 1.7, >= 1.7.7)
       jwt (>= 0.1.5, < 2)
-    arvados-cli (0.1.20161017193526)
-      activesupport (~> 3.2, >= 3.2.13)
+    arvados-cli (0.1.20170322173355)
+      activesupport (>= 3.2.13, < 5)
       andand (~> 1.3, >= 1.3.3)
       arvados (~> 0.1, >= 0.1.20150128223554)
       curb (~> 0.8)
@@ -55,36 +71,38 @@ GEM
       addressable (>= 2.3.1)
       extlib (>= 0.9.15)
       multi_json (>= 1.0.0)
-    builder (3.0.4)
+    builder (3.2.3)
     capistrano (2.15.9)
       highline
       net-scp (>= 1.0.0)
       net-sftp (>= 2.0.0)
       net-ssh (>= 2.0.14)
       net-ssh-gateway (>= 1.1.0)
-    coffee-rails (3.2.2)
+    coffee-rails (4.2.1)
       coffee-script (>= 2.2.0)
-      railties (~> 3.2.0)
+      railties (>= 4.0.0, < 5.2.x)
     coffee-script (2.4.1)
       coffee-script-source
       execjs
-    coffee-script-source (1.10.0)
+    coffee-script-source (1.12.2)
     curb (0.9.3)
     database_cleaner (1.5.3)
     erubis (2.7.0)
-    eventmachine (1.2.0.1)
+    eventmachine (1.2.3)
     execjs (2.7.0)
     extlib (0.9.16)
-    factory_girl (4.7.0)
+    factory_girl (4.8.0)
       activesupport (>= 3.0.0)
-    factory_girl_rails (4.7.0)
-      factory_girl (~> 4.7.0)
+    factory_girl_rails (4.8.0)
+      factory_girl (~> 4.8.0)
       railties (>= 3.0.0)
-    faraday (0.9.2)
+    faraday (0.11.0)
       multipart-post (>= 1.2, < 3)
-    faye-websocket (0.10.4)
+    faye-websocket (0.10.7)
       eventmachine (>= 0.12.0)
       websocket-driver (>= 0.5.1)
+    globalid (0.3.7)
+      activesupport (>= 4.1.0)
     google-api-client (0.8.7)
       activesupport (>= 3.2, < 5.0)
       addressable (~> 2.3)
@@ -104,106 +122,121 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    hashie (3.4.6)
+    hashie (3.5.5)
     highline (1.7.8)
     hike (1.2.3)
-    i18n (0.7.0)
-    journey (1.0.4)
-    jquery-rails (3.1.4)
-      railties (>= 3.0, < 5.0)
+    i18n (0.8.1)
+    jquery-rails (4.2.2)
+      rails-dom-testing (>= 1, < 3)
+      railties (>= 4.2.0)
       thor (>= 0.14, < 2.0)
-    json (1.8.3)
+    json (1.8.6)
     jwt (1.5.6)
     launchy (2.4.3)
       addressable (~> 2.3)
-    libv8 (3.16.14.15)
+    libv8 (3.16.14.19)
     little-plugger (1.1.4)
-    logging (2.1.0)
+    logging (2.2.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    lograge (0.3.6)
-      actionpack (>= 3)
-      activesupport (>= 3)
-      railties (>= 3)
+    lograge (0.4.1)
+      actionpack (>= 4, < 5.1)
+      activesupport (>= 4, < 5.1)
+      railties (>= 4, < 5.1)
     logstash-event (1.2.02)
-    mail (2.5.4)
-      mime-types (~> 1.16)
-      treetop (~> 1.4.8)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    mail (2.6.4)
+      mime-types (>= 1.16, < 4)
     memoist (0.15.0)
     metaclass (0.0.4)
-    mime-types (1.25.1)
-    mocha (1.2.0)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.1.0)
+    minitest (5.10.1)
+    mocha (1.2.1)
       metaclass (~> 0.0.1)
     multi_json (1.12.1)
-    multi_xml (0.5.5)
+    multi_xml (0.6.0)
     multipart-post (2.0.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
-    net-ssh (3.2.0)
-    net-ssh-gateway (1.2.0)
-      net-ssh (>= 2.6.5)
-    oauth2 (1.2.0)
-      faraday (>= 0.8, < 0.10)
+    net-ssh (4.1.0)
+    net-ssh-gateway (2.0.0)
+      net-ssh (>= 4.0.0)
+    nokogiri (1.7.1)
+      mini_portile2 (~> 2.1.0)
+    oauth2 (1.3.1)
+      faraday (>= 0.8, < 0.12)
       jwt (~> 1.0)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (>= 1.2, < 3)
-    oj (2.15.0)
-    omniauth (1.3.1)
+    oj (2.18.5)
+    oj_mimic_json (1.0.1)
+    omniauth (1.4.2)
       hashie (>= 1.2, < 4)
       rack (>= 1.0, < 3)
     omniauth-oauth2 (1.4.0)
       oauth2 (~> 1.0)
       omniauth (~> 1.2)
     os (0.9.6)
-    passenger (5.0.30)
+    passenger (5.1.2)
       rack
       rake (>= 0.8.1)
-    pg (0.19.0)
-    pg_power (1.6.4)
-      pg
-      rails (~> 3.1)
-    polyglot (0.3.5)
-    power_assert (0.3.1)
+    pg (0.20.0)
+    power_assert (1.0.1)
+    protected_attributes (1.1.3)
+      activemodel (>= 4.0.1, < 5.0)
+    public_suffix (2.0.5)
     puma (2.16.0)
-    rack (1.4.7)
-    rack-cache (1.6.1)
-      rack (>= 0.4)
-    rack-ssl (1.3.4)
-      rack
+    rack (1.6.5)
     rack-test (0.6.3)
       rack (>= 1.0)
-    rails (3.2.22.5)
-      actionmailer (= 3.2.22.5)
-      actionpack (= 3.2.22.5)
-      activerecord (= 3.2.22.5)
-      activeresource (= 3.2.22.5)
-      activesupport (= 3.2.22.5)
-      bundler (~> 1.0)
-      railties (= 3.2.22.5)
-    railties (3.2.22.5)
-      actionpack (= 3.2.22.5)
-      activesupport (= 3.2.22.5)
-      rack-ssl (~> 1.3.2)
+    rails (4.2.5.2)
+      actionmailer (= 4.2.5.2)
+      actionpack (= 4.2.5.2)
+      actionview (= 4.2.5.2)
+      activejob (= 4.2.5.2)
+      activemodel (= 4.2.5.2)
+      activerecord (= 4.2.5.2)
+      activesupport (= 4.2.5.2)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.2.5.2)
+      sprockets-rails
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.8)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    rails-observers (0.1.2)
+      activemodel (~> 4.0)
+    railties (4.2.5.2)
+      actionpack (= 4.2.5.2)
+      activesupport (= 4.2.5.2)
       rake (>= 0.8.7)
-      rdoc (~> 3.4)
-      thor (>= 0.14.6, < 2.0)
-    rake (11.3.0)
-    rdoc (3.12.2)
-      json (~> 1.4)
+      thor (>= 0.18.1, < 2.0)
+    rake (12.0.0)
     ref (2.0.0)
+    responders (2.3.0)
+      railties (>= 4.2.0, < 5.1)
     retriable (1.4.1)
     ruby-prof (0.16.2)
     rvm-capistrano (1.5.6)
       capistrano (~> 2.15.4)
     safe_yaml (1.0.4)
-    sass (3.4.22)
-    sass-rails (3.2.6)
-      railties (~> 3.2.0)
-      sass (>= 3.1.10)
-      tilt (~> 1.3)
+    sass (3.2.19)
+    sass-rails (4.0.5)
+      railties (>= 4.0.0, < 5.0)
+      sass (~> 3.2.2)
+      sprockets (~> 2.8, < 3.0)
+      sprockets-rails (~> 2.0)
     signet (0.7.3)
       addressable (~> 2.3)
       faraday (~> 0.9)
@@ -215,32 +248,33 @@ GEM
     simplecov-html (0.7.1)
     simplecov-rcov (0.2.3)
       simplecov (>= 0.4.1)
-    sprockets (2.2.3)
+    sprockets (2.12.4)
       hike (~> 1.2)
       multi_json (~> 1.0)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
-    sshkey (1.8.0)
-    test-unit (3.2.1)
+    sprockets-rails (2.3.3)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (>= 2.8, < 4.0)
+    sshkey (1.9.0)
+    test-unit (3.2.3)
       power_assert
     test_after_commit (1.1.0)
       activerecord (>= 3.2)
-    themes_for_rails (0.5.1)
-      rails (>= 3.0.0)
-    therubyracer (0.12.2)
-      libv8 (~> 3.16.14.0)
+    therubyracer (0.12.3)
+      libv8 (~> 3.16.14.15)
       ref
-    thor (0.19.1)
+    thor (0.19.4)
+    thread_safe (0.3.6)
     tilt (1.4.1)
-    treetop (1.4.15)
-      polyglot
-      polyglot (>= 0.3.1)
     trollop (2.1.2)
-    tzinfo (0.3.51)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
     uglifier (2.7.2)
       execjs (>= 0.3.0)
       json (>= 1.8.0)
-    websocket-driver (0.6.4)
+    websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.2)
 
@@ -252,7 +286,7 @@ DEPENDENCIES
   andand
   arvados (>= 0.1.20150615153458)
   arvados-cli (>= 0.1.20161017193526)
-  coffee-rails (~> 3.2)
+  coffee-rails (~> 4.0)
   database_cleaner
   factory_girl_rails
   faye-websocket
@@ -262,26 +296,29 @@ DEPENDENCIES
   mocha
   multi_json
   oj
-  omniauth (~> 1.1)
+  oj_mimic_json
+  omniauth (~> 1.4.0)
   omniauth-oauth2 (~> 1.1)
   passenger
   pg
-  pg_power
+  protected_attributes
   puma (~> 2.0)
-  rails (~> 3.2)
+  rails (~> 4.0)
+  rails-observers
+  responders (~> 2.0)
   ruby-prof
   rvm-capistrano
   safe_yaml
-  sass-rails (~> 3.2)
+  sass-rails (~> 4.0)
   simplecov (~> 0.7.1)
   simplecov-rcov
   sshkey
   test-unit (~> 3.0)
   test_after_commit
-  themes_for_rails
+  themes_for_rails!
   therubyracer
   trollop
   uglifier (~> 2.0)
 
 BUNDLED WITH
-   1.13.6
+   1.14.3
diff --git a/services/api/Rakefile b/services/api/Rakefile
index 925e6c2d1d..67e4202ba2 100644
--- a/services/api/Rakefile
+++ b/services/api/Rakefile
@@ -4,12 +4,6 @@
 
 require File.expand_path('../config/application', __FILE__)
 
-begin
-  ok = PgPower
-rescue
-  abort "Hm, pg_power is missing. Make sure you use 'bundle exec rake ...'"
-end
-
 Server::Application.load_tasks
 
 namespace :test do
diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb
index 71fb365fc6..831bcceee7 100644
--- a/services/api/app/controllers/application_controller.rb
+++ b/services/api/app/controllers/application_controller.rb
@@ -18,8 +18,8 @@ end
 require 'load_param'
 
 class ApplicationController < ActionController::Base
-  include CurrentApiClient
   include ThemesForRails::ActionController
+  include CurrentApiClient
   include LoadParam
   include DbCurrentTime
 
@@ -47,7 +47,7 @@ class ApplicationController < ActionController::Base
   before_filter(:render_404_if_no_object,
                 except: [:index, :create] + ERROR_ACTIONS)
 
-  theme :select_theme
+  theme Rails.configuration.arvados_theme
 
   attr_writer :resource_attrs
 
@@ -83,7 +83,9 @@ class ApplicationController < ActionController::Base
   end
 
   def index
-    @objects.uniq!(&:id) if @select.nil? or @select.include? "id"
+    if @select.nil? || @select.include?("id")
+      @objects = @objects.uniq(&:id)
+    end
     if params[:eager] and params[:eager] != '0' and params[:eager] != 0 and params[:eager] != ''
       @objects.each(&:eager_load_associations)
     end
@@ -488,7 +490,7 @@ class ApplicationController < ActionController::Base
 
   def remote_ip
     # Caveat: this is highly dependent on the proxy setup. YMMV.
-    if request.headers.has_key?('HTTP_X_REAL_IP') then
+    if request.headers.key?('HTTP_X_REAL_IP') then
       # We're behind a reverse proxy
       @remote_ip = request.headers['HTTP_X_REAL_IP']
     else
@@ -562,8 +564,4 @@ class ApplicationController < ActionController::Base
     end
     super(*opts)
   end
-
-  def select_theme
-    return Rails.configuration.arvados_theme
-  end
 end
diff --git a/services/api/app/controllers/arvados/v1/nodes_controller.rb b/services/api/app/controllers/arvados/v1/nodes_controller.rb
index b308c18329..2f6c6504b7 100644
--- a/services/api/app/controllers/arvados/v1/nodes_controller.rb
+++ b/services/api/app/controllers/arvados/v1/nodes_controller.rb
@@ -6,8 +6,8 @@ class Arvados::V1::NodesController < ApplicationController
   include DbCurrentTime
 
   def update
-    if resource_attrs[:job_uuid]
-      @object.job_readable = readable_job_uuids(resource_attrs[:job_uuid]).any?
+    if resource_attrs[:job_uuid].is_a? String
+      @object.job_readable = readable_job_uuids([resource_attrs[:job_uuid]]).any?
     end
     super
   end
@@ -57,7 +57,7 @@ class Arvados::V1::NodesController < ApplicationController
 
   protected
 
-  def readable_job_uuids(*uuids)
+  def readable_job_uuids(uuids)
     Job.readable_by(*@read_users).select(:uuid).where(uuid: uuids).map(&:uuid)
   end
 end
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 443c6503c6..11269d2556 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -38,6 +38,7 @@ class Arvados::V1::SchemaController < ApplicationController
         blobSignatureTtl: Rails.application.config.blob_signature_ttl,
         maxRequestSize: Rails.application.config.max_request_size,
         dockerImageFormats: Rails.application.config.docker_image_formats,
+        websocketUrl: Rails.application.config.websocket_address,
         parameters: {
           alt: {
             type: "string",
@@ -83,12 +84,6 @@ class Arvados::V1::SchemaController < ApplicationController
         resources: {}
       }
 
-      if Rails.application.config.websocket_address
-        discovery[:websocketUrl] = Rails.application.config.websocket_address
-      elsif ENV['ARVADOS_WEBSOCKETS']
-        discovery[:websocketUrl] = root_url.sub(/^http/, 'ws') + "websocket"
-      end
-
       ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
         begin
           ctl_class = "Arvados::V1::#{k.to_s.pluralize}Controller".constantize
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index db5e7bd952..7a1f699911 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -123,7 +123,7 @@ class Arvados::V1::UsersController < ApplicationController
 
     # setup succeeded. send email to user
     if params[:send_notification_email] == true || params[:send_notification_email] == 'true'
-      UserNotifier.account_is_setup(@object).deliver
+      UserNotifier.account_is_setup(@object).deliver_now
     end
 
     send_json kind: "arvados#HashList", items: @response.as_api_response(nil)
diff --git a/services/api/app/controllers/database_controller.rb b/services/api/app/controllers/database_controller.rb
index 6699f7363b..458766d3e5 100644
--- a/services/api/app/controllers/database_controller.rb
+++ b/services/api/app/controllers/database_controller.rb
@@ -51,8 +51,8 @@ class DatabaseController < ApplicationController
 
         # create_fixtures() is a no-op for cached fixture sets, so
         # uncache them all.
-        ActiveRecord::Fixtures.reset_cache
-        ActiveRecord::Fixtures.
+        ActiveRecord::FixtureSet.reset_cache
+        ActiveRecord::FixtureSet.
           create_fixtures(Rails.root.join('test', 'fixtures'), fixturesets)
 
         # Dump cache of permissions etc.
diff --git a/services/api/app/controllers/user_sessions_controller.rb b/services/api/app/controllers/user_sessions_controller.rb
index 8bb27a705e..c5507045c4 100644
--- a/services/api/app/controllers/user_sessions_controller.rb
+++ b/services/api/app/controllers/user_sessions_controller.rb
@@ -127,7 +127,8 @@ class UserSessionsController < ApplicationController
     # Stub: automatically register all new API clients
     api_client_url_prefix = callback_url.match(%r{^.*?://[^/]+})[0] + '/'
     act_as_system_user do
-      @api_client = ApiClient.find_or_create_by_url_prefix api_client_url_prefix
+      @api_client = ApiClient.
+        find_or_create_by(url_prefix: api_client_url_prefix)
     end
 
     api_client_auth = ApiClientAuthorization.
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index f7985a986a..b0c2f31e4f 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -61,7 +61,13 @@ class ApiClientAuthorization < ArvadosModel
   end
 
   def scopes_allow_request?(request)
-    scopes_allow? [request.request_method, request.path].join(' ')
+    method = request.request_method
+    if method == 'HEAD'
+      (scopes_allow?(['HEAD', request.path].join(' ')) ||
+       scopes_allow?(['GET', request.path].join(' ')))
+    else
+      scopes_allow?([method, request.path].join(' '))
+    end
   end
 
   def logged_attributes
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 96dc85e1e4..84897d04ef 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -9,10 +9,6 @@ class ArvadosModel < ActiveRecord::Base
   include DbCurrentTime
   extend RecordFilters
 
-  attr_protected :created_at
-  attr_protected :modified_by_user_uuid
-  attr_protected :modified_by_client_uuid
-  attr_protected :modified_at
   after_initialize :log_start_state
   before_save :ensure_permission_to_save
   before_save :ensure_owner_uuid_is_permitted
@@ -27,17 +23,16 @@ class ArvadosModel < ActiveRecord::Base
   after_find :convert_serialized_symbols_to_strings
   before_validation :normalize_collection_uuids
   before_validation :set_default_owner
-  validate :ensure_serialized_attribute_type
   validate :ensure_valid_uuids
 
   # Note: This only returns permission links. It does not account for
   # permissions obtained via user.is_admin or
   # user.uuid==object.owner_uuid.
   has_many(:permissions,
+           ->{where(link_class: 'permission')},
            foreign_key: :head_uuid,
            class_name: 'Link',
-           primary_key: :uuid,
-           conditions: "link_class = 'permission'")
+           primary_key: :uuid)
 
   class PermissionDeniedError < StandardError
     def http_status
@@ -77,6 +72,40 @@ class ArvadosModel < ActiveRecord::Base
     "#{current_api_base}/#{self.class.to_s.pluralize.underscore}/#{self.uuid}"
   end
 
+  def self.permit_attribute_params raw_params
+    # strong_parameters does not provide security: permissions are
+    # implemented with before_save hooks.
+    #
+    # The following permit! is necessary even with
+    # "ActionController::Parameters.permit_all_parameters = true",
+    # because permit_all does not permit nested attributes.
+    if raw_params
+      serialized_attributes.each do |colname, coder|
+        param = raw_params[colname.to_sym]
+        if param.nil?
+          # ok
+        elsif !param.is_a?(coder.object_class)
+          raise ArgumentError.new("#{colname} parameter must be #{coder.object_class}, not #{param.class}")
+        elsif has_nonstring_keys?(param)
+          raise ArgumentError.new("#{colname} parameter cannot have non-string hash keys")
+        end
+      end
+    end
+    ActionController::Parameters.new(raw_params).permit!
+  end
+
+  def initialize raw_params={}, *args
+    super(self.class.permit_attribute_params(raw_params), *args)
+  end
+
+  def self.create raw_params={}, *args
+    super(permit_attribute_params(raw_params), *args)
+  end
+
+  def update_attributes raw_params={}, *args
+    super(self.class.permit_attribute_params(raw_params), *args)
+  end
+
   def self.selectable_attributes(template=:user)
     # Return an array of attribute name strings that can be selected
     # in the given template.
@@ -458,6 +487,7 @@ class ArvadosModel < ActiveRecord::Base
 
   def update_modified_by_fields
     current_time = db_current_time
+    self.created_at = created_at_was || current_time
     self.updated_at = current_time
     self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
     self.modified_at = current_time
@@ -466,6 +496,19 @@ class ArvadosModel < ActiveRecord::Base
     true
   end
 
+  def self.has_nonstring_keys? x
+    if x.is_a? Hash
+      x.each do |k,v|
+        return true if !(k.is_a?(String) || k.is_a?(Symbol)) || has_nonstring_keys?(v)
+      end
+    elsif x.is_a? Array
+      x.each do |v|
+        return true if has_nonstring_keys?(v)
+      end
+    end
+    false
+  end
+
   def self.has_symbols? x
     if x.is_a? Hash
       x.each do |k,v|
@@ -502,8 +545,15 @@ class ArvadosModel < ActiveRecord::Base
   end
 
   def self.where_serialized(colname, value)
-    sorted = deep_sort_hash(value)
-    where("#{colname.to_s} IN (?)", [sorted.to_yaml, SafeJSON.dump(sorted)])
+    if value.empty?
+      # rails4 stores as null, rails3 stored as serialized [] or {}
+      sql = "#{colname.to_s} is null or #{colname.to_s} IN (?)"
+      sorted = value
+    else
+      sql = "#{colname.to_s} IN (?)"
+      sorted = deep_sort_hash(value)
+    end
+    where(sql, [sorted.to_yaml, SafeJSON.dump(sorted)])
   end
 
   Serializer = {
@@ -512,25 +562,18 @@ class ArvadosModel < ActiveRecord::Base
   }
 
   def self.serialize(colname, type)
-    super(colname, Serializer[type])
+    coder = Serializer[type]
+    @serialized_attributes ||= {}
+    @serialized_attributes[colname.to_s] = coder
+    super(colname, coder)
   end
 
-  def ensure_serialized_attribute_type
-    # Specifying a type in the "serialize" declaration causes rails to
-    # raise an exception if a different data type is retrieved from
-    # the database during load().  The validation preventing such
-    # crash-inducing records from being inserted in the database in
-    # the first place seems to have been left as an exercise to the
-    # developer.
-    self.class.serialized_attributes.each do |colname, attr|
-      if attr.object_class
-        if self.attributes[colname].class != attr.object_class
-          self.errors.add colname.to_sym, "must be a #{attr.object_class.to_s}, not a #{self.attributes[colname].class.to_s}"
-        elsif self.class.has_symbols? attributes[colname]
-          self.errors.add colname.to_sym, "must not contain symbols: #{attributes[colname].inspect}"
-        end
-      end
-    end
+  def self.serialized_attributes
+    @serialized_attributes ||= {}
+  end
+
+  def serialized_attributes
+    self.class.serialized_attributes
   end
 
   def convert_serialized_symbols_to_strings
@@ -543,8 +586,8 @@ class ArvadosModel < ActiveRecord::Base
     self.class.serialized_attributes.each do |colname, attr|
       if self.class.has_symbols? attributes[colname]
         attributes[colname] = self.class.recursive_stringify attributes[colname]
-        self.send(colname + '=',
-                  self.class.recursive_stringify(attributes[colname]))
+        send(colname + '=',
+             self.class.recursive_stringify(attributes[colname]))
       end
     end
   end
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 4a1e610303..33f6bc2524 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -24,7 +24,7 @@ class Collection < ArvadosModel
   before_save :set_file_names
 
   # Query only untrashed collections by default.
-  default_scope where("is_trashed = false")
+  default_scope { where("is_trashed = false") }
 
   api_accessible :user, extend: :common do |t|
     t.add :name
diff --git a/services/api/app/models/link.rb b/services/api/app/models/link.rb
index 649a6f80c2..d38ea59083 100644
--- a/services/api/app/models/link.rb
+++ b/services/api/app/models/link.rb
@@ -20,11 +20,6 @@ class Link < ArvadosModel
     t.add :properties
   end
 
-  def properties
-    @properties ||= Hash.new
-    super
-  end
-
   def head_kind
     if k = ArvadosModel::resource_class_for_uuid(head_uuid)
       k.kind
diff --git a/services/api/app/models/log.rb b/services/api/app/models/log.rb
index eedf06a976..44984255f8 100644
--- a/services/api/app/models/log.rb
+++ b/services/api/app/models/log.rb
@@ -102,6 +102,6 @@ class Log < ArvadosModel
   end
 
   def send_notify
-    connection.execute "NOTIFY logs, '#{self.id}'"
+    ActiveRecord::Base.connection.execute "NOTIFY logs, '#{self.id}'"
   end
 end
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 78ec7bea1e..742db4c9b0 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -10,7 +10,7 @@ class User < ArvadosModel
   has_many :api_client_authorizations
   validates(:username,
             format: {
-              with: /^[A-Za-z][A-Za-z0-9]*$/,
+              with: /\A[A-Za-z][A-Za-z0-9]*\z/,
               message: "must begin with a letter and contain only alphanumerics",
             },
             uniqueness: true,
@@ -475,9 +475,9 @@ class User < ArvadosModel
 
   # Send admin notifications
   def send_admin_notifications
-    AdminNotifier.new_user(self).deliver
+    AdminNotifier.new_user(self).deliver_now
     if not self.is_active then
-      AdminNotifier.new_inactive_user(self).deliver
+      AdminNotifier.new_inactive_user(self).deliver_now
     end
   end
 
@@ -502,7 +502,7 @@ class User < ArvadosModel
     if self.prefs_changed?
       if self.prefs_was.andand.empty? || !self.prefs_was.andand['profile']
         profile_notification_address = Rails.configuration.user_profile_notification_address
-        ProfileNotifier.profile_created(self, profile_notification_address).deliver if profile_notification_address
+        ProfileNotifier.profile_created(self, profile_notification_address).deliver_now if profile_notification_address
       end
     end
   end
diff --git a/services/api/app/models/virtual_machine.rb b/services/api/app/models/virtual_machine.rb
index 094591e6cc..6fbbddfb5d 100644
--- a/services/api/app/models/virtual_machine.rb
+++ b/services/api/app/models/virtual_machine.rb
@@ -3,7 +3,11 @@ class VirtualMachine < ArvadosModel
   include KindAndEtag
   include CommonApiTemplate
 
-  has_many :login_permissions, :foreign_key => :head_uuid, :class_name => 'Link', :primary_key => :uuid, :conditions => "link_class = 'permission' and name = 'can_login'"
+  has_many(:login_permissions,
+           -> { where("link_class = 'permission' and name = 'can_login'") },
+           foreign_key: :head_uuid,
+           class_name: 'Link',
+           primary_key: :uuid)
 
   api_accessible :user, extend: :common do |t|
     t.add :hostname
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 5241cb4378..85955be4e2 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -46,28 +46,16 @@ common:
   # to log in.
   workbench_address: false
 
-  # The ARVADOS_WEBSOCKETS environment variable determines whether to
-  # serve http, websockets, or both.
+  # Client-facing URI for websocket service. Nginx should be
+  # configured to proxy this URI to arvados-ws; see
+  # http://doc.arvados.org/install/install-ws.html
   #
-  # If ARVADOS_WEBSOCKETS="true", http and websockets are both served
-  # from the same process.
+  # If websocket_address is false (which is the default), no websocket
+  # server will be advertised to clients. This configuration is not
+  # supported.
   #
-  # If ARVADOS_WEBSOCKETS="ws-only", only websockets is served.
-  #
-  # If ARVADOS_WEBSOCKETS="false" or not set at all, only http is
-  # served. In this case, you should have a separate process serving
-  # websockets, and the address of that service should be given here
-  # as websocket_address.
-  #
-  # If websocket_address is false (which is the default), the
-  # discovery document will tell clients to use the current server as
-  # the websocket service, or (if the current server does not have
-  # websockets enabled) not to use websockets at all.
-  #
-  # Example: Clients will connect to the specified endpoint.
-  #websocket_address: wss://127.0.0.1:3333/websocket
-  # Default: Clients will connect to this server if it's running
-  # websockets, otherwise none at all.
+  # Example:
+  #websocket_address: wss://ws.zzzzz.arvadosapi.com/websocket
   websocket_address: false
 
   # Maximum number of websocket connections allowed
@@ -441,7 +429,6 @@ development:
   action_mailer.perform_deliveries: false
   active_support.deprecation: :log
   action_dispatch.best_standards_support: :builtin
-  active_record.mass_assignment_sanitizer: :strict
   active_record.auto_explain_threshold_in_seconds: 0.5
   assets.compress: false
   assets.debug: true
@@ -451,7 +438,7 @@ production:
   cache_classes: true
   consider_all_requests_local: false
   action_controller.perform_caching: true
-  serve_static_assets: false
+  serve_static_files: false
   assets.compress: true
   assets.compile: false
   assets.digest: true
@@ -459,7 +446,7 @@ production:
 test:
   force_ssl: false
   cache_classes: true
-  serve_static_assets: true
+  serve_static_files: true
   static_cache_control: public, max-age=3600
   whiny_nils: true
   consider_all_requests_local: true
@@ -468,7 +455,6 @@ test:
   action_controller.allow_forgery_protection: false
   action_mailer.delivery_method: :test
   active_support.deprecation: :stderr
-  active_record.mass_assignment_sanitizer: :strict
   uuid_prefix: zzzzz
   sso_app_id: arvados-server
   sso_app_secret: <%= rand(2**512).to_s(36) %>
@@ -479,6 +465,6 @@ test:
   workbench_address: https://localhost:3001/
   git_repositories_dir: <%= Rails.root.join 'tmp', 'git', 'test' %>
   git_internal_dir: <%= Rails.root.join 'tmp', 'internal.git' %>
-  websocket_address: <% if ENV['ARVADOS_TEST_EXPERIMENTAL_WS'] %>"wss://0.0.0.0:<%= ENV['ARVADOS_TEST_WSS_PORT'] %>/websocket"<% else %>false<% end %>
+  websocket_address: "wss://0.0.0.0:<%= ENV['ARVADOS_TEST_WSS_PORT'] %>/websocket"
   trash_sweep_interval: -1
   docker_image_formats: ["v1"]
diff --git a/services/api/config/application.rb b/services/api/config/application.rb
index f3f6424b2d..0d6aa1d9f9 100644
--- a/services/api/config/application.rb
+++ b/services/api/config/application.rb
@@ -3,11 +3,25 @@ require File.expand_path('../boot', __FILE__)
 require 'rails/all'
 require 'digest'
 
+module Kernel
+  def suppress_warnings
+    verbose_orig = $VERBOSE
+    begin
+      $VERBOSE = nil
+      yield
+    ensure
+      $VERBOSE = verbose_orig
+    end
+  end
+end
+
 if defined?(Bundler)
-  # If you precompile assets before deploying to production, use this line
-  Bundler.require(*Rails.groups(:assets => %w(development test)))
-  # If you want your assets lazily compiled in production, use this line
-  # Bundler.require(:default, :assets, Rails.env)
+  suppress_warnings do
+    # If you precompile assets before deploying to production, use this line
+    Bundler.require(*Rails.groups(:assets => %w(development test)))
+    # If you want your assets lazily compiled in production, use this line
+    # Bundler.require(:default, :assets, Rails.env)
+  end
 end
 
 module Server
@@ -34,6 +48,13 @@ module Server
     # Configure sensitive parameters which will be filtered from the log file.
     config.filter_parameters += [:password]
 
+    # Load entire application at startup.
+    config.eager_load = true
+
+    config.active_record.raise_in_transactional_callbacks = true
+
+    config.active_support.test_order = :sorted
+
     I18n.enforce_available_locales = false
 
     # Before using the filesystem backend for Rails.cache, check
diff --git a/services/api/config/environments/development.rb.example b/services/api/config/environments/development.rb.example
index b6c4c92871..449d05ab18 100644
--- a/services/api/config/environments/development.rb.example
+++ b/services/api/config/environments/development.rb.example
@@ -23,9 +23,6 @@ Server::Application.configure do
   # Only use best-standards-support built into browsers
   config.action_dispatch.best_standards_support = :builtin
 
-  # Raise exception on mass assignment protection for Active Record models
-  config.active_record.mass_assignment_sanitizer = :strict
-
   # Log the query plan for queries taking more than this (works
   # with SQLite, MySQL, and PostgreSQL)
   config.active_record.auto_explain_threshold_in_seconds = 0.5
diff --git a/services/api/config/environments/production.rb.example b/services/api/config/environments/production.rb.example
index c1092d3fc7..2b91d3de87 100644
--- a/services/api/config/environments/production.rb.example
+++ b/services/api/config/environments/production.rb.example
@@ -9,7 +9,7 @@ Server::Application.configure do
   config.action_controller.perform_caching = true
 
   # Disable Rails's static asset server (Apache or nginx will already do this)
-  config.serve_static_assets = false
+  config.serve_static_files = false
 
   # Compress JavaScripts and CSS
   config.assets.compress = true
diff --git a/services/api/config/environments/test.rb.example b/services/api/config/environments/test.rb.example
index 5baf09d207..f21a6d41c3 100644
--- a/services/api/config/environments/test.rb.example
+++ b/services/api/config/environments/test.rb.example
@@ -8,7 +8,7 @@ Server::Application.configure do
   config.cache_classes = true
 
   # Configure static asset server for tests with Cache-Control for performance
-  config.serve_static_assets = true
+  config.serve_static_files = true
   config.static_cache_control = "public, max-age=3600"
 
   # Log error messages when you accidentally call methods on nil
@@ -37,9 +37,6 @@ Server::Application.configure do
   # Print deprecation notices to the stderr
   config.active_support.deprecation = :stderr
 
-  # Raise exception on mass assignment protection for Active Record models
-  config.active_record.mass_assignment_sanitizer = :strict
-
   # No need for SSL while testing
   config.force_ssl = false
 
diff --git a/services/api/config/initializers/eventbus.rb b/services/api/config/initializers/eventbus.rb
index ea1c210385..ad0120f658 100644
--- a/services/api/config/initializers/eventbus.rb
+++ b/services/api/config/initializers/eventbus.rb
@@ -1,19 +1,27 @@
-require 'eventbus'
+if ENV['ARVADOS_WEBSOCKETS']
+  Server::Application.configure do
+    Rails.logger.error "Built-in websocket server is disabled. See note (2017-03-23, e8cc0d7) at https://dev.arvados.org/projects/arvados/wiki/Upgrading_to_master"
 
-# See application.yml for details about configuring the websocket service.
+    class EventBusRemoved
+      def overloaded?
+        false
+      end
+      def on_connect ws
+        ws.on :open do |e|
+          EM::Timer.new 1 do
+            ws.send(SafeJSON.dump({status: 501, message: "Server misconfigured? see http://doc.arvados.org/install/install-ws.html"}))
+          end
+          EM::Timer.new 3 do
+            ws.close
+          end
+        end
+      end
+    end
 
-Server::Application.configure do
-  # Enables websockets if ARVADOS_WEBSOCKETS is defined with any value.  If
-  # ARVADOS_WEBSOCKETS=ws-only, server will only accept websocket connections
-  # and return an error response for all other requests.
-  if ENV['ARVADOS_WEBSOCKETS']
-    config.middleware.insert_after ArvadosApiToken, RackSocket, {
-      :handler => EventBus,
-      :mount => "/websocket",
-      :websocket_only => (ENV['ARVADOS_WEBSOCKETS'] == "ws-only")
-    }
-    Rails.logger.info "Websockets #{ENV['ARVADOS_WEBSOCKETS']}, running at /websocket"
-  else
-    Rails.logger.info "Websockets disabled"
+    config.middleware.insert_after(ArvadosApiToken, RackSocket, {
+                                     handler: EventBusRemoved,
+                                     mount: "/websocket",
+                                     websocket_only: (ENV['ARVADOS_WEBSOCKETS'] == "ws-only")
+                                   })
   end
 end
diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb
index fd3c977393..787469cca3 100644
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/initializers/load_config.rb
@@ -69,4 +69,5 @@ config/application.yml:
 
 EOS
   end
+  config.secret_key_base = config.secret_token
 end
diff --git a/services/api/config/initializers/noop_deep_munge.rb b/services/api/config/initializers/noop_deep_munge.rb
index abfa7e5820..b1f84e9520 100644
--- a/services/api/config/initializers/noop_deep_munge.rb
+++ b/services/api/config/initializers/noop_deep_munge.rb
@@ -2,6 +2,7 @@ module ActionDispatch
   class Request < Rack::Request
     # This Rails method messes with valid JSON, for example turning the empty
     # array [] into 'nil'.  We don't want that, so turn it into a no-op.
+    remove_method :deep_munge
     def deep_munge(hash)
       hash
     end
diff --git a/services/api/config/initializers/permit_all_parameters.rb b/services/api/config/initializers/permit_all_parameters.rb
new file mode 100644
index 0000000000..051e13b358
--- /dev/null
+++ b/services/api/config/initializers/permit_all_parameters.rb
@@ -0,0 +1 @@
+ActionController::Parameters.permit_all_parameters = true
diff --git a/services/api/config/initializers/time_format.rb b/services/api/config/initializers/time_format.rb
index ee7c91fffd..b2a2448a8e 100644
--- a/services/api/config/initializers/time_format.rb
+++ b/services/api/config/initializers/time_format.rb
@@ -1,10 +1,12 @@
 class ActiveSupport::TimeWithZone
+  remove_method :as_json
   def as_json *args
     strftime "%Y-%m-%dT%H:%M:%S.%NZ"
   end
 end
 
 class Time
+  remove_method :as_json
   def as_json *args
     strftime "%Y-%m-%dT%H:%M:%S.%NZ"
   end
diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index 9cb53fee30..77e5372a15 100644
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -1,15 +1,13 @@
 Server::Application.routes.draw do
   themes_for_rails
 
-  # See http://guides.rubyonrails.org/routing.html
-
   # OPTIONS requests are not allowed at routes that use cookies.
   ['/auth/*a', '/login', '/logout'].each do |nono|
-    match nono, :to => 'user_sessions#cross_origin_forbidden', :via => 'OPTIONS'
+    match nono, to: 'user_sessions#cross_origin_forbidden', via: 'OPTIONS'
   end
   # OPTIONS at discovery and API paths get an empty response with CORS headers.
-  match '/discovery/v1/*a', :to => 'static#empty', :via => 'OPTIONS'
-  match '/arvados/v1/*a', :to => 'static#empty', :via => 'OPTIONS'
+  match '/discovery/v1/*a', to: 'static#empty', via: 'OPTIONS'
+  match '/arvados/v1/*a', to: 'static#empty', via: 'OPTIONS'
 
   namespace :arvados do
     namespace :v1 do
@@ -79,7 +77,7 @@ Server::Application.routes.draw do
         get 'logins', on: :member
         get 'get_all_logins', on: :collection
       end
-      get '/permissions/:uuid', :to => 'links#get_permissions'
+      get '/permissions/:uuid', to: 'links#get_permissions'
     end
   end
 
@@ -88,22 +86,22 @@ Server::Application.routes.draw do
   end
 
   # omniauth
-  match '/auth/:provider/callback', :to => 'user_sessions#create'
-  match '/auth/failure', :to => 'user_sessions#failure'
+  match '/auth/:provider/callback', to: 'user_sessions#create', via: [:get, :post]
+  match '/auth/failure', to: 'user_sessions#failure', via: [:get, :post]
   # not handled by omniauth provider -> 403 with no CORS headers.
-  get '/auth/*a', :to => 'user_sessions#cross_origin_forbidden'
+  get '/auth/*a', to: 'user_sessions#cross_origin_forbidden'
 
   # Custom logout
-  match '/login', :to => 'user_sessions#login'
-  match '/logout', :to => 'user_sessions#logout'
+  match '/login', to: 'user_sessions#login', via: [:get, :post]
+  match '/logout', to: 'user_sessions#logout', via: [:get, :post]
 
-  match '/discovery/v1/apis/arvados/v1/rest', :to => 'arvados/v1/schema#index'
+  match '/discovery/v1/apis/arvados/v1/rest', to: 'arvados/v1/schema#index', via: [:get, :post]
 
-  match '/static/login_failure', :to => 'static#login_failure', :as => :login_failure
+  match '/static/login_failure', to: 'static#login_failure', as: :login_failure, via: [:get, :post]
 
   # Send unroutable requests to an arbitrary controller
   # (ends up at ApplicationController#render_not_found)
-  match '*a', :to => 'static#render_not_found'
+  match '*a', to: 'static#render_not_found', via: [:get, :post, :put, :patch, :delete, :options]
 
-  root :to => 'static#home'
+  root to: 'static#home'
 end
diff --git a/services/api/db/migrate/20170319063406_serialized_columns_accept_null.rb b/services/api/db/migrate/20170319063406_serialized_columns_accept_null.rb
new file mode 100644
index 0000000000..564586eb12
--- /dev/null
+++ b/services/api/db/migrate/20170319063406_serialized_columns_accept_null.rb
@@ -0,0 +1,5 @@
+class SerializedColumnsAcceptNull < ActiveRecord::Migration
+  def change
+    change_column :api_client_authorizations, :scopes, :text, null: true, default: '["all"]'
+  end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index d6729428be..3e1fa3fae4 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -44,9 +44,7 @@ CREATE TABLE api_client_authorizations (
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
     default_owner_uuid character varying(255),
-    scopes text DEFAULT '---
-- all
-'::text NOT NULL,
+    scopes text DEFAULT '["all"]'::text,
     uuid character varying(255) NOT NULL
 );
 
@@ -2775,6 +2773,8 @@ INSERT INTO schema_migrations (version) VALUES ('20170216170823');
 
 INSERT INTO schema_migrations (version) VALUES ('20170301225558');
 
+INSERT INTO schema_migrations (version) VALUES ('20170319063406');
+
 INSERT INTO schema_migrations (version) VALUES ('20170328215436');
 
 INSERT INTO schema_migrations (version) VALUES ('20170330012505');
@@ -2783,4 +2783,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170419173031');
 
 INSERT INTO schema_migrations (version) VALUES ('20170419173712');
 
-INSERT INTO schema_migrations (version) VALUES ('20170419175801');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20170419175801');
+
diff --git a/services/api/lib/can_be_an_owner.rb b/services/api/lib/can_be_an_owner.rb
index 16a878347a..75a63509c2 100644
--- a/services/api/lib/can_be_an_owner.rb
+++ b/services/api/lib/can_be_an_owner.rb
@@ -14,7 +14,7 @@ module CanBeAnOwner
       base.has_many(t.to_sym,
                     foreign_key: :owner_uuid,
                     primary_key: :uuid,
-                    dependent: :restrict)
+                    dependent: :restrict_with_exception)
     end
     # We need custom protection for changing an owner's primary
     # key. (Apart from this restriction, admins are allowed to change
diff --git a/services/api/lib/create_superuser_token.rb b/services/api/lib/create_superuser_token.rb
index 84100c2efd..da67a32c79 100755
--- a/services/api/lib/create_superuser_token.rb
+++ b/services/api/lib/create_superuser_token.rb
@@ -26,7 +26,9 @@ module CreateSuperUserToken
       # need to create a token
       if !api_client_auth
         # Get (or create) trusted api client
-        apiClient =  ApiClient.find_or_create_by_url_prefix_and_is_trusted("ssh://root@localhost/", true)
+        apiClient =  ApiClient.
+          find_or_create_by(url_prefix: "ssh://root@localhost/",
+                            is_trusted: true)
 
         # Check if there is an unexpired superuser token corresponding to this api client
         api_client_auth =
diff --git a/services/api/lib/eventbus.rb b/services/api/lib/eventbus.rb
deleted file mode 100644
index 11b178d985..0000000000
--- a/services/api/lib/eventbus.rb
+++ /dev/null
@@ -1,358 +0,0 @@
-# If any threads raise an unhandled exception, make them all die.
-# We trust a supervisor like runit to restart the server in this case.
-Thread.abort_on_exception = true
-
-require 'eventmachine'
-require 'faye/websocket'
-require 'load_param'
-require 'oj'
-require 'record_filters'
-require 'safe_json'
-require 'set'
-require 'thread'
-
-# Patch in user, last_log_id and filters fields into the Faye::Websocket class.
-module Faye
-  class WebSocket
-    attr_accessor :user
-    attr_accessor :last_log_id
-    attr_accessor :filters
-    attr_accessor :sent_ids
-    attr_accessor :queue
-    attr_accessor :frame_mtx
-  end
-end
-
-module WebSocket
-  class Driver
-
-    class Server
-      alias_method :_write, :write
-
-      def write(data)
-        # Most of the sending activity will be from the thread set up in
-        # on_connect.  However, there is also some automatic activity in the
-        # form of ping/pong messages, so ensure that the write method used to
-        # send one complete message to the underlying socket can only be
-        # called by one thread at a time.
-        self.frame_mtx.synchronize do
-          _write(data)
-        end
-      end
-    end
-  end
-end
-
-# Store the filters supplied by the user that will be applied to the logs table
-# to determine which events to return to the listener.
-class Filter
-  include LoadParam
-
-  attr_accessor :filters
-
-  def initialize p
-    @params = p
-    load_filters_param
-  end
-
-  def params
-    @params
-  end
-end
-
-# Manages websocket connections, accepts subscription messages and publishes
-# log table events.
-class EventBus
-  include CurrentApiClient
-  include RecordFilters
-
-  # used in RecordFilters
-  def model_class
-    Log
-  end
-
-  # Initialize EventBus.  Takes no parameters.
-  def initialize
-    @channel = EventMachine::Channel.new
-    @mtx = Mutex.new
-    @bgthread = false
-    @connection_count = 0
-  end
-
-  def send_message(ws, obj)
-    ws.send(SafeJSON.dump(obj))
-  end
-
-  # Push out any pending events to the connection +ws+
-  # +notify_id+  the id of the most recent row in the log table, may be nil
-  #
-  # This accepts a websocket and a notify_id (this is the row id from Postgres
-  # LISTEN/NOTIFY, it may be nil if called from somewhere else)
-  #
-  # It queries the database for log rows that are either
-  #  a) greater than ws.last_log_id, which is the last log id which was a candidate to be sent out
-  #  b) if ws.last_log_id is nil, then it queries the row notify_id
-  #
-  # Regular Arvados permissions are applied using readable_by() and filters using record_filters().
-  def push_events ws, notify_id
-    begin
-      # Must have at least one filter set up to receive events
-      if ws.filters.length > 0
-        # Start with log rows readable by user
-        logs = Log.readable_by(ws.user)
-
-        cond_id = nil
-        cond_out = []
-        param_out = []
-
-        if not ws.last_log_id.nil?
-          # We are catching up from some starting point.
-          cond_id = "logs.id > ?"
-          param_out << ws.last_log_id
-        elsif not notify_id.nil?
-          # Get next row being notified.
-          cond_id = "logs.id = ?"
-          param_out << notify_id
-        else
-          # No log id to start from, nothing to do, return
-          return
-        end
-
-        # Now build filters provided by client
-        ws.filters.each do |filter|
-          ft = record_filters filter.filters, Log
-          if ft[:cond_out].any?
-            # Join the clauses within a single subscription filter with AND
-            # so it is consistent with regular queries
-            cond_out << "(#{ft[:cond_out].join ') AND ('})"
-            param_out += ft[:param_out]
-          end
-        end
-
-        # Add filters to query
-        if cond_out.any?
-          # Join subscriptions with OR
-          logs = logs.where(cond_id + " AND ((#{cond_out.join ') OR ('}))", *param_out)
-        else
-          logs = logs.where(cond_id, *param_out)
-        end
-
-        # Execute query and actually send the matching log rows. Load
-        # the full log records only when we're ready to send them,
-        # though: otherwise, (1) postgres has to build the whole
-        # result set and return it to us before we can send the first
-        # event, and (2) we store lots of records in memory while
-        # waiting to spool them out to the client. Both of these are
-        # troublesome when log records are large (e.g., a collection
-        # update contains both old and new manifest_text).
-        #
-        # Note: find_each implies order('id asc'), which is what we
-        # want.
-        logs.select('logs.id').find_each do |l|
-          if not ws.sent_ids.include?(l.id)
-            # only send if not a duplicate
-            send_message(ws, Log.find(l.id).as_api_response)
-          end
-          if not ws.last_log_id.nil?
-            # record ids only when sending "catchup" messages, not notifies
-            ws.sent_ids << l.id
-          end
-        end
-        ws.last_log_id = nil
-      end
-    rescue ArgumentError => e
-      # There was some kind of user error.
-      Rails.logger.warn "Error publishing event: #{$!}"
-      send_message(ws, {status: 500, message: $!})
-      ws.close
-    rescue => e
-      Rails.logger.warn "Error publishing event: #{$!}"
-      Rails.logger.warn "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
-      send_message(ws, {status: 500, message: $!})
-      ws.close
-      # These exceptions typically indicate serious server trouble:
-      # out of memory issues, database connection problems, etc.  Go ahead and
-      # crash; we expect that a supervisor service like runit will restart us.
-      raise
-    end
-  end
-
-  # Handle inbound subscribe or unsubscribe message.
-  def handle_message ws, event
-    begin
-      begin
-        # Parse event data as JSON
-        p = SafeJSON.load(event.data).symbolize_keys
-        filter = Filter.new(p)
-      rescue Oj::Error => e
-        send_message(ws, {status: 400, message: "malformed request"})
-        return
-      end
-
-      if p[:method] == 'subscribe'
-        # Handle subscribe event
-
-        if p[:last_log_id]
-          # Set or reset the last_log_id.  The event bus only reports events
-          # for rows that come after last_log_id.
-          ws.last_log_id = p[:last_log_id].to_i
-          # Reset sent_ids for consistency
-          # (always re-deliver all matching messages following last_log_id)
-          ws.sent_ids = Set.new
-        end
-
-        if ws.filters.length < Rails.configuration.websocket_max_filters
-          # Add a filter.  This gets the :filters field which is the same
-          # format as used for regular index queries.
-          ws.filters << filter
-          send_message(ws, {status: 200, message: 'subscribe ok', filter: p})
-
-          # Send any pending events
-          push_events ws, nil
-        else
-          send_message(ws, {status: 403, message: "maximum of #{Rails.configuration.websocket_max_filters} filters allowed per connection"})
-        end
-
-      elsif p[:method] == 'unsubscribe'
-        # Handle unsubscribe event
-
-        len = ws.filters.length
-        ws.filters.select! { |f| not ((f.filters == p[:filters]) or (f.filters.empty? and p[:filters].nil?)) }
-        if ws.filters.length < len
-          send_message(ws, {status: 200, message: 'unsubscribe ok'})
-        else
-          send_message(ws, {status: 404, message: 'filter not found'})
-        end
-
-      else
-        send_message(ws, {status: 400, message: "missing or unrecognized method"})
-      end
-    rescue => e
-      Rails.logger.warn "Error handling message: #{$!}"
-      Rails.logger.warn "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
-      send_message(ws, {status: 500, message: 'error'})
-      ws.close
-    end
-  end
-
-  def overloaded?
-    @mtx.synchronize do
-      @connection_count >= Rails.configuration.websocket_max_connections
-    end
-  end
-
-  # Called by RackSocket when a new websocket connection has been established.
-  def on_connect ws
-    # Disconnect if no valid API token.
-    # current_user is included from CurrentApiClient
-    if not current_user
-      send_message(ws, {status: 401, message: "Valid API token required"})
-      # Wait for the handshake to complete before closing the
-      # socket. Otherwise, nginx responds with HTTP 502 Bad gateway,
-      # and the client never sees our real error message.
-      ws.on :open do |event|
-        ws.close
-      end
-      return
-    end
-
-    # Initialize our custom fields on the websocket connection object.
-    ws.user = current_user
-    ws.filters = []
-    ws.last_log_id = nil
-    ws.sent_ids = Set.new
-    ws.queue = Queue.new
-    ws.frame_mtx = Mutex.new
-
-    @mtx.synchronize do
-      @connection_count += 1
-    end
-
-    # Subscribe to internal postgres notifications through @channel and
-    # forward them to the thread associated with the connection.
-    sub = @channel.subscribe do |msg|
-      if ws.queue.length > Rails.configuration.websocket_max_notify_backlog
-        send_message(ws, {status: 500, message: 'Notify backlog too long'})
-        ws.close
-        @channel.unsubscribe sub
-        ws.queue.clear
-      else
-        ws.queue << [:notify, msg]
-      end
-    end
-
-    # Set up callback for inbound message dispatch.
-    ws.on :message do |event|
-      ws.queue << [:message, event]
-    end
-
-    # Set up socket close callback
-    ws.on :close do |event|
-      @channel.unsubscribe sub
-      ws.queue.clear
-      ws.queue << [:close, nil]
-    end
-
-    # Spin off a new thread to handle sending events to the client.  We need a
-    # separate thread per connection so that a slow client doesn't interfere
-    # with other clients.
-    #
-    # We don't want the loop in the request thread because on a TERM signal,
-    # Puma waits for outstanding requests to complete, and long-lived websocket
-    # connections may not complete in a timely manner.
-    Thread.new do
-      # Loop and react to socket events.
-      begin
-        loop do
-          eventType, msg = ws.queue.pop
-          if eventType == :message
-            handle_message ws, msg
-          elsif eventType == :notify
-            push_events ws, msg
-          elsif eventType == :close
-            break
-          end
-        end
-      ensure
-        @mtx.synchronize do
-          @connection_count -= 1
-        end
-        ActiveRecord::Base.connection.close
-      end
-    end
-
-    # Start up thread to monitor the Postgres database, if none exists already.
-    @mtx.synchronize do
-      unless @bgthread
-        @bgthread = true
-        Thread.new do
-          # from http://stackoverflow.com/questions/16405520/postgres-listen-notify-rails
-          ActiveRecord::Base.connection_pool.with_connection do |connection|
-            conn = connection.instance_variable_get(:@connection)
-            begin
-              conn.async_exec "LISTEN logs"
-              while true
-                # wait_for_notify will block until there is a change
-                # notification from Postgres about the logs table, then push
-                # the notification into the EventMachine channel.  Each
-                # websocket connection subscribes to the other end of the
-                # channel and calls #push_events to actually dispatch the
-                # events to the client.
-                conn.wait_for_notify do |channel, pid, payload|
-                  @channel.push payload.to_i
-                end
-              end
-            ensure
-              # Don't want the connection to still be listening once we return
-              # it to the pool - could result in weird behavior for the next
-              # thread to check it out.
-              conn.async_exec "UNLISTEN *"
-            end
-          end
-          @bgthread = false
-        end
-      end
-    end
-
-  end
-end
diff --git a/services/api/lib/has_uuid.rb b/services/api/lib/has_uuid.rb
index 36c087986e..74d09e94e9 100644
--- a/services/api/lib/has_uuid.rb
+++ b/services/api/lib/has_uuid.rb
@@ -7,8 +7,18 @@ module HasUuid
     base.validate :validate_uuid
     base.before_create :assign_uuid
     base.before_destroy :destroy_permission_links
-    base.has_many :links_via_head, class_name: 'Link', foreign_key: :head_uuid, primary_key: :uuid, conditions: "not (link_class = 'permission')", dependent: :destroy
-    base.has_many :links_via_tail, class_name: 'Link', foreign_key: :tail_uuid, primary_key: :uuid, conditions: "not (link_class = 'permission')", dependent: :destroy
+    base.has_many(:links_via_head,
+                  -> { where("not (link_class = 'permission')") },
+                  class_name: 'Link',
+                  foreign_key: :head_uuid,
+                  primary_key: :uuid,
+                  dependent: :destroy)
+    base.has_many(:links_via_tail,
+                  -> { where("not (link_class = 'permission')") },
+                  class_name: 'Link',
+                  foreign_key: :tail_uuid,
+                  primary_key: :uuid,
+                  dependent: :destroy)
   end
 
   module ClassMethods
diff --git a/services/api/lib/load_param.rb b/services/api/lib/load_param.rb
index 3e1e8b5eba..e3de05f24f 100644
--- a/services/api/lib/load_param.rb
+++ b/services/api/lib/load_param.rb
@@ -153,7 +153,7 @@ module LoadParam
       # Any ordering columns must be selected when doing select,
       # otherwise it is an SQL error, so filter out invaliding orderings.
       @orders.select! { |o|
-        col, dir = o.split
+        col, _ = o.split
         # match select column against order array entry
         @select.select { |s| col == "#{table_name}.#{s}" }.any?
       }
diff --git a/services/api/lib/serializers.rb b/services/api/lib/serializers.rb
index 41379f308f..e412f63f62 100644
--- a/services/api/lib/serializers.rb
+++ b/services/api/lib/serializers.rb
@@ -1,7 +1,13 @@
 require 'safe_json'
 
 class Serializer
+  class TypeMismatch < ArgumentError
+  end
+
   def self.dump(val)
+    if !val.is_a?(object_class)
+      raise TypeMismatch.new("cannot serialize #{val.class} as #{object_class}")
+    end
     SafeJSON.dump(val)
   end
 
diff --git a/services/api/lib/tasks/delete_old_container_logs.rake b/services/api/lib/tasks/delete_old_container_logs.rake
index 3421fb8b96..33fe47d8d1 100644
--- a/services/api/lib/tasks/delete_old_container_logs.rake
+++ b/services/api/lib/tasks/delete_old_container_logs.rake
@@ -7,7 +7,7 @@ namespace :db do
   desc "Remove old container log entries from the logs table"
 
   task delete_old_container_logs: :environment do
-    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND containers.finished_at < '#{Rails.configuration.clean_container_log_rows_after.ago}')"
+    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND clock_timestamp() - containers.finished_at > interval '#{Rails.configuration.clean_container_log_rows_after} seconds')"
 
     ActiveRecord::Base.connection.execute(delete_sql)
   end
diff --git a/services/api/lib/tasks/delete_old_job_logs.rake b/services/api/lib/tasks/delete_old_job_logs.rake
index 18a5f02277..dec5f7243b 100644
--- a/services/api/lib/tasks/delete_old_job_logs.rake
+++ b/services/api/lib/tasks/delete_old_job_logs.rake
@@ -5,7 +5,7 @@
 namespace :db do
   desc "Remove old job stderr entries from the logs table"
   task delete_old_job_logs: :environment do
-    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND jobs.finished_at < '#{Rails.configuration.clean_job_log_rows_after.ago}')"
+    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.clean_job_log_rows_after} seconds')"
 
     ActiveRecord::Base.connection.execute(delete_sql)
   end
diff --git a/services/api/lib/whitelist_update.rb b/services/api/lib/whitelist_update.rb
index 8fccd0f45c..039d983476 100644
--- a/services/api/lib/whitelist_update.rb
+++ b/services/api/lib/whitelist_update.rb
@@ -1,12 +1,23 @@
 module WhitelistUpdate
   def check_update_whitelist permitted_fields
     attribute_names.each do |field|
-      if not permitted_fields.include? field.to_sym and self.send((field.to_s + "_changed?").to_sym)
-        errors.add field, "cannot be modified in this state"
+      if !permitted_fields.include?(field.to_sym) && really_changed(field)
+        errors.add field, "cannot be modified in this state (#{send(field+"_was").inspect}, #{send(field).inspect})"
       end
     end
   end
 
+  def really_changed(attr)
+    return false if !send(attr+"_changed?")
+    old = send(attr+"_was")
+    new = send(attr)
+    if (old.nil? || old == [] || old == {}) && (new.nil? || new == [] || new == {})
+      false
+    else
+      old != new
+    end
+  end
+
   def validate_state_change
     if self.state_changed?
       unless state_transitions[self.state_was].andand.include? self.state
diff --git a/services/api/log/.gitkeep b/services/api/log/.gitkeep
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 2391fc19b7..a31ad8af03 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -283,12 +283,12 @@ EOS
       assert_response :success
       assert_not_nil assigns(:object)
       resp = assigns(:object)
-      assert_equal foo_collection[:portable_data_hash], resp['portable_data_hash']
-      assert_signed_manifest resp['manifest_text']
+      assert_equal foo_collection[:portable_data_hash], resp[:portable_data_hash]
+      assert_signed_manifest resp[:manifest_text]
 
       # The manifest in the response will have had permission hints added.
       # Remove any permission hints in the response before comparing it to the source.
-      stripped_manifest = resp['manifest_text'].gsub(/\+A[A-Za-z0-9@_-]+/, '')
+      stripped_manifest = resp[:manifest_text].gsub(/\+A[A-Za-z0-9@_-]+/, '')
       assert_equal foo_collection[:manifest_text], stripped_manifest
     end
   end
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index 98643a9e74..f98e482dd8 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -8,6 +8,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
   setup do
     @initial_link_count = Link.count
     @vm_uuid = virtual_machines(:testvm).uuid
+    ActionMailer::Base.deliveries = []
   end
 
   test "activate a user after signing UA" do
diff --git a/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb b/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb
index 9b805af8e3..626d7f04b0 100644
--- a/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb
@@ -59,8 +59,8 @@ class Arvados::V1::VirtualMachinesControllerTest < ActionController::TestCase
     get :logins, id: vm.uuid
     assert_response :success
     assert_equal 1, json_response['items'].length
-    assert_equal nil, json_response['items'][0]['public_key']
-    assert_equal nil, json_response['items'][0]['authorized_key_uuid']
+    assert_nil json_response['items'][0]['public_key']
+    assert_nil json_response['items'][0]['authorized_key_uuid']
     assert_equal u.uuid, json_response['items'][0]['user_uuid']
     assert_equal 'bobblogin', json_response['items'][0]['username']
   end
diff --git a/services/api/test/functional/database_controller_test.rb b/services/api/test/functional/database_controller_test.rb
index 4bda0d0f11..324a7ffb39 100644
--- a/services/api/test/functional/database_controller_test.rb
+++ b/services/api/test/functional/database_controller_test.rb
@@ -15,7 +15,7 @@ class DatabaseControllerTest < ActionController::TestCase
     begin
       Rails.env = 'production'
       Rails.application.reload_routes!
-      assert_raises ActionController::RoutingError do
+      assert_raises ActionController::UrlGenerationError do
         post :reset
       end
     ensure
diff --git a/services/api/test/integration/api_client_authorizations_scopes_test.rb b/services/api/test/integration/api_client_authorizations_scopes_test.rb
index beef1953ae..a1a20d428e 100644
--- a/services/api/test/integration/api_client_authorizations_scopes_test.rb
+++ b/services/api/test/integration/api_client_authorizations_scopes_test.rb
@@ -4,11 +4,11 @@
 
 require 'test_helper'
 
-class Arvados::V1::ApiTokensScopeTest < ActionController::IntegrationTest
+class ApiTokensScopeTest < ActionDispatch::IntegrationTest
   fixtures :all
 
   def v1_url(*parts)
-    (['arvados', 'v1'] + parts).join('/')
+    (['', 'arvados', 'v1'] + parts).join('/')
   end
 
   test "user list token can only list users" do
@@ -29,6 +29,8 @@ class Arvados::V1::ApiTokensScopeTest < ActionController::IntegrationTest
     assert_response 403
     get(v1_url('specimens', specimens(:owned_by_active_user).uuid), *get_args)
     assert_response :success
+    head(v1_url('specimens', specimens(:owned_by_active_user).uuid), *get_args)
+    assert_response :success
     get(v1_url('specimens', specimens(:owned_by_spectator).uuid), *get_args)
     assert_includes(403..404, @response.status)
   end
diff --git a/services/api/test/integration/crunch_dispatch_test.rb b/services/api/test/integration/crunch_dispatch_test.rb
index a6f937bbda..552fd37ad5 100644
--- a/services/api/test/integration/crunch_dispatch_test.rb
+++ b/services/api/test/integration/crunch_dispatch_test.rb
@@ -1,7 +1,7 @@
 require 'test_helper'
 require 'helpers/git_test_helper'
 
-class CrunchDispatchTest < ActionDispatch::IntegrationTest
+class CrunchDispatchIntegrationTest < ActionDispatch::IntegrationTest
   include GitTestHelper
 
   fixtures :all
diff --git a/services/api/test/integration/database_reset_test.rb b/services/api/test/integration/database_reset_test.rb
index 029e37cbbf..a9b64b335f 100644
--- a/services/api/test/integration/database_reset_test.rb
+++ b/services/api/test/integration/database_reset_test.rb
@@ -1,8 +1,6 @@
 require 'test_helper'
 
 class DatabaseResetTest < ActionDispatch::IntegrationTest
-  self.use_transactional_fixtures = false
-
   slow_test "reset fails when Rails.env != 'test'" do
     rails_env_was = Rails.env
     begin
diff --git a/services/api/test/integration/errors_test.rb b/services/api/test/integration/errors_test.rb
index 984f81fe51..1bd17dc782 100644
--- a/services/api/test/integration/errors_test.rb
+++ b/services/api/test/integration/errors_test.rb
@@ -19,7 +19,7 @@ class ErrorsTest < ActionDispatch::IntegrationTest
       # Generally, new routes should appear under /arvados/v1/. If
       # they appear elsewhere, that might have been caused by default
       # rails generator behavior that we don't want.
-      assert_match(/^\/(|\*a|arvados\/v1\/.*|auth\/.*|login|logout|database\/reset|discovery\/.*|static\/.*|themes\/.*)(\(\.:format\))?$/,
+      assert_match(/^\/(|\*a|arvados\/v1\/.*|auth\/.*|login|logout|database\/reset|discovery\/.*|static\/.*|themes\/.*|assets)(\(\.:format\))?$/,
                    route.path.spec.to_s,
                    "Unexpected new route: #{route.path.spec}")
     end
diff --git a/services/api/test/integration/pipeline_test.rb b/services/api/test/integration/pipeline_test.rb
index a550246b13..6a264bbe79 100644
--- a/services/api/test/integration/pipeline_test.rb
+++ b/services/api/test/integration/pipeline_test.rb
@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class PipelineTest < ActionDispatch::IntegrationTest
+class PipelineIntegrationTest < ActionDispatch::IntegrationTest
   # These tests simulate the workflow of arv-run-pipeline-instance
   # and other pipeline-running code.
 
diff --git a/services/api/test/integration/reader_tokens_test.rb b/services/api/test/integration/reader_tokens_test.rb
index 6ed8461c62..23dd42f302 100644
--- a/services/api/test/integration/reader_tokens_test.rb
+++ b/services/api/test/integration/reader_tokens_test.rb
@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class Arvados::V1::ReaderTokensTest < ActionController::IntegrationTest
+class ReaderTokensTest < ActionDispatch::IntegrationTest
   fixtures :all
 
   def spectator_specimen
diff --git a/services/api/test/integration/websocket_test.rb b/services/api/test/integration/websocket_test.rb
deleted file mode 100644
index 549bbc6f99..0000000000
--- a/services/api/test/integration/websocket_test.rb
+++ /dev/null
@@ -1,742 +0,0 @@
-require 'database_cleaner'
-require 'safe_json'
-require 'test_helper'
-
-DatabaseCleaner.strategy = :deletion
-
-class WebsocketTest < ActionDispatch::IntegrationTest
-  self.use_transactional_fixtures = false
-
-  setup do
-    DatabaseCleaner.start
-  end
-
-  teardown do
-    DatabaseCleaner.clean
-  end
-
-  def self.startup
-    s = TCPServer.new('0.0.0.0', 0)
-    @@port = s.addr[1]
-    s.close
-    @@pidfile = "tmp/pids/passenger.#{@@port}.pid"
-    DatabaseCleaner.start
-    Dir.chdir(Rails.root) do |apidir|
-      # Only passenger seems to be able to run the websockets server
-      # successfully.
-      _system('passenger', 'start', '-d',
-              "-p#{@@port}",
-              "--log-file", "/dev/stderr",
-              "--pid-file", @@pidfile)
-      timeout = Time.now.tv_sec + 10
-      begin
-        sleep 0.2
-        begin
-          server_pid = IO.read(@@pidfile).to_i
-          good_pid = (server_pid > 0) and (Process.kill(0, pid) rescue false)
-        rescue Errno::ENOENT
-          good_pid = false
-        end
-      end while (not good_pid) and (Time.now.tv_sec < timeout)
-      if not good_pid
-        raise RuntimeError, "could not find API server Rails pid"
-      end
-      STDERR.puts "Started websocket server on port #{@@port} with pid #{server_pid}"
-    end
-  end
-
-  def self.shutdown
-    Dir.chdir(Rails.root) do
-      _system('passenger', 'stop', "-p#{@@port}",
-              "--pid-file", @@pidfile)
-    end
-    # DatabaseCleaner leaves the database empty. Prefer to leave it full.
-    dc = DatabaseController.new
-    dc.define_singleton_method :render do |*args| end
-    dc.reset
-  end
-
-  def self._system(*cmd)
-    Bundler.with_clean_env do
-      env = {
-        'ARVADOS_WEBSOCKETS' => 'ws-only',
-        'RAILS_ENV' => 'test',
-      }
-      if not system(env, *cmd)
-        raise RuntimeError, "Command exited #{$?}: #{cmd.inspect}"
-      end
-    end
-  end
-
-  def ws_helper(token: nil, timeout: 8)
-    opened = false
-    close_status = nil
-    too_long = false
-
-    EM.run do
-      if token
-        ws = Faye::WebSocket::Client.new("ws://localhost:#{@@port}/websocket?api_token=#{api_client_authorizations(token).api_token}")
-      else
-        ws = Faye::WebSocket::Client.new("ws://localhost:#{@@port}/websocket")
-      end
-
-      ws.on :open do |event|
-        opened = true
-        if timeout
-          EM::Timer.new(timeout) do
-            too_long = true if close_status.nil?
-            EM.stop_event_loop
-          end
-        end
-      end
-
-      ws.on :error do |event|
-        STDERR.puts "websocket client error: #{event.inspect}"
-      end
-
-      ws.on :close do |event|
-        close_status = [:close, event.code, event.reason]
-        EM.stop_event_loop
-      end
-
-      yield ws
-    end
-
-    assert opened, "Should have opened web socket"
-    assert (not too_long), "Test took too long"
-    assert_equal 1000, close_status[1], "Connection closed unexpectedly (check log for errors)"
-  end
-
-  test "connect with no token" do
-    status = nil
-
-    ws_helper do |ws|
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        status = d["status"]
-        ws.close
-      end
-    end
-
-    assert_equal 401, status
-  end
-
-  test "connect, subscribe and get response" do
-    status = nil
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        status = d["status"]
-        ws.close
-      end
-    end
-
-    assert_equal 200, status
-  end
-
-  def subscribe_test
-    state = 1
-    spec = nil
-    ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          spec = Specimen.create
-          state = 2
-        when 2
-          ev_uuid = d["object_uuid"]
-          ws.close
-        end
-      end
-
-    end
-
-    assert_not_nil spec
-    assert_equal spec.uuid, ev_uuid
-  end
-
-  test "connect, subscribe, get event" do
-    subscribe_test()
-  end
-
-  test "connect, subscribe, get two events" do
-    state = 1
-    spec = nil
-    human = nil
-    spec_ev_uuid = nil
-    human_ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          spec = Specimen.create
-          human = Human.create
-          state = 2
-        when 2
-          spec_ev_uuid = d["object_uuid"]
-          state = 3
-        when 3
-          human_ev_uuid = d["object_uuid"]
-          state = 4
-          ws.close
-        when 4
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_not_nil spec
-    assert_not_nil human
-    assert_equal spec.uuid, spec_ev_uuid
-    assert_equal human.uuid, human_ev_uuid
-  end
-
-  test "connect, subscribe, filter events" do
-    state = 1
-    human = nil
-    human_ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          Specimen.create
-          human = Human.create
-          state = 2
-        when 2
-          human_ev_uuid = d["object_uuid"]
-          state = 3
-          ws.close
-        when 3
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_not_nil human
-    assert_equal human.uuid, human_ev_uuid
-  end
-
-
-  test "connect, subscribe, multiple filters" do
-    state = 1
-    spec = nil
-    human = nil
-    spec_ev_uuid = nil
-    human_ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
-        ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#specimen']]}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          state = 2
-        when 2
-          assert_equal 200, d["status"]
-          spec = Specimen.create
-          Trait.create # not part of filters, should not be received
-          human = Human.create
-          state = 3
-        when 3
-          spec_ev_uuid = d["object_uuid"]
-          state = 4
-        when 4
-          human_ev_uuid = d["object_uuid"]
-          state = 5
-          ws.close
-        when 5
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_not_nil spec
-    assert_not_nil human
-    assert_equal spec.uuid, spec_ev_uuid
-    assert_equal human.uuid, human_ev_uuid
-  end
-
-
-  test "connect, subscribe, compound filter" do
-    state = 1
-    t1 = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#trait'], ['event_type', '=', 'update']]}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          t1 = Trait.create("name" => "foo")
-          t1.name = "bar"
-          t1.save!
-          state = 2
-         when 2
-          assert_equal 'update', d['event_type']
-          state = 3
-          ws.close
-        when 3
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_equal 3, state
-    assert_not_nil t1
-  end
-
-  test "connect, subscribe, ask events starting at seq num" do
-    state = 1
-
-    authorize_with :active
-
-    lastid = logs(:admin_changes_specimen).id
-    l1 = nil
-    l2 = nil
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe', last_log_id: lastid}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          state = 2
-        when 2
-          l1 = d["object_uuid"]
-          assert_not_nil l1, "Unexpected message: #{d}"
-          state = 3
-        when 3
-          l2 = d["object_uuid"]
-          assert_not_nil l2, "Unexpected message: #{d}"
-          state = 4
-          ws.close
-        when 4
-          assert false, "Should not get any more events"
-        end
-      end
-    end
-
-    expect_next_logs = Log.where('id > ?', lastid).order('id asc')
-    assert_equal expect_next_logs[0].object_uuid, l1
-    assert_equal expect_next_logs[1].object_uuid, l2
-  end
-
-  slow_test "connect, subscribe, get event, unsubscribe" do
-    state = 1
-    spec = nil
-    spec_ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active, timeout: false) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-        EM::Timer.new 3 do
-          # Set a time limit on the test because after unsubscribing the server
-          # still has to process the next event (and then hopefully correctly
-          # decides not to send it because we unsubscribed.)
-          ws.close
-        end
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          spec = Specimen.create
-          state = 2
-        when 2
-          spec_ev_uuid = d["object_uuid"]
-          ws.send ({method: 'unsubscribe'}.to_json)
-
-          EM::Timer.new 1 do
-            Specimen.create
-          end
-
-          state = 3
-        when 3
-          assert_equal 200, d["status"]
-          state = 4
-        when 4
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_not_nil spec
-    assert_equal spec.uuid, spec_ev_uuid
-  end
-
-  slow_test "connect, subscribe, get event, unsubscribe with filter" do
-    state = 1
-    spec = nil
-    spec_ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active, timeout: false) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
-        EM::Timer.new 6 do
-          # Set a time limit on the test because after unsubscribing the server
-          # still has to process the next event (and then hopefully correctly
-          # decides not to send it because we unsubscribed.)
-          ws.close
-        end
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          spec = Human.create
-          state = 2
-        when 2
-          spec_ev_uuid = d["object_uuid"]
-          ws.send ({method: 'unsubscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
-
-          EM::Timer.new 1 do
-            Human.create
-          end
-
-          state = 3
-        when 3
-          assert_equal 200, d["status"]
-          state = 4
-        when 4
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_not_nil spec
-    assert_equal spec.uuid, spec_ev_uuid
-  end
-
-
-  slow_test "connect, subscribe, get event, try to unsubscribe with bogus filter" do
-    state = 1
-    spec = nil
-    spec_ev_uuid = nil
-    human = nil
-    human_ev_uuid = nil
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          spec = Specimen.create
-          state = 2
-        when 2
-          spec_ev_uuid = d["object_uuid"]
-          ws.send ({method: 'unsubscribe', filters: [['foo', 'bar', 'baz']]}.to_json)
-
-          EM::Timer.new 1 do
-            human = Human.create
-          end
-
-          state = 3
-        when 3
-          assert_equal 404, d["status"]
-          state = 4
-        when 4
-          human_ev_uuid = d["object_uuid"]
-          state = 5
-          ws.close
-        when 5
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_not_nil spec
-    assert_not_nil human
-    assert_equal spec.uuid, spec_ev_uuid
-    assert_equal human.uuid, human_ev_uuid
-  end
-
-  slow_test "connected, not subscribed, no event" do
-    authorize_with :active
-
-    ws_helper(token: :active, timeout: false) do |ws|
-      ws.on :open do |event|
-        EM::Timer.new 1 do
-          Specimen.create
-        end
-
-        EM::Timer.new 3 do
-          ws.close
-        end
-      end
-
-      ws.on :message do |event|
-        assert false, "Should not get any messages, message was #{event.data}"
-      end
-    end
-  end
-
-  slow_test "connected, not authorized to see event" do
-    state = 1
-
-    authorize_with :admin
-
-    ws_helper(token: :active, timeout: false) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-
-        EM::Timer.new 3 do
-          ws.close
-        end
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          Specimen.create
-          state = 2
-        when 2
-          assert false, "Should not get any messages, message was #{event.data}"
-        end
-      end
-
-    end
-
-  end
-
-  test "connect, try bogus method" do
-    status = nil
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({method: 'frobnabble'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        status = d["status"]
-        ws.close
-      end
-    end
-
-    assert_equal 400, status
-  end
-
-  test "connect, missing method" do
-    status = nil
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send ({fizzbuzz: 'frobnabble'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        status = d["status"]
-        ws.close
-      end
-    end
-
-    assert_equal 400, status
-  end
-
-  test "connect, send malformed request" do
-    status = nil
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        ws.send '<XML4EVER></XML4EVER>'
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        status = d["status"]
-        ws.close
-      end
-    end
-
-    assert_equal 400, status
-  end
-
-
-  test "connect, try subscribe too many filters" do
-    state = 1
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        (1..17).each do |i|
-          ws.send ({method: 'subscribe', filters: [['object_uuid', '=', i]]}.to_json)
-        end
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when (1..Rails.configuration.websocket_max_filters)
-          assert_equal 200, d["status"]
-          state += 1
-        when (Rails.configuration.websocket_max_filters+1)
-          assert_equal 403, d["status"]
-          ws.close
-        end
-      end
-
-    end
-
-    assert_equal Rails.configuration.websocket_max_filters+1, state
-
-  end
-
-  slow_test "connect, subscribe, lots of events" do
-    state = 1
-    event_count = 0
-    log_start = Log.order(:id).last.id
-
-    authorize_with :active
-
-    ws_helper(token: :active, timeout: false) do |ws|
-      EM::Timer.new 45 do
-        # Needs a longer timeout than the default
-        ws.close
-      end
-
-      ws.on :open do |event|
-        ws.send ({method: 'subscribe'}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          ActiveRecord::Base.transaction do
-            (1..202).each do
-              Specimen.create
-            end
-          end
-          state = 2
-        when 2
-          event_count += 1
-          assert_equal d['id'], event_count+log_start
-          if event_count == 202
-            ws.close
-          end
-        end
-      end
-
-    end
-
-    assert_equal 202, event_count
-  end
-
-
-  test "connect, subscribe with invalid filter" do
-    state = 1
-
-    authorize_with :active
-
-    ws_helper(token: :active) do |ws|
-      ws.on :open do |event|
-        # test that #6451 is fixed (invalid filter crashes websockets)
-        ws.send ({method: 'subscribe', filters: [['object_blarg', 'is_a', 'arvados#human']]}.to_json)
-      end
-
-      ws.on :message do |event|
-        d = SafeJSON.load event.data
-        case state
-        when 1
-          assert_equal 200, d["status"]
-          Specimen.create
-          Human.create
-          state = 2
-        when 2
-          assert_equal 500, d["status"]
-          state = 3
-          ws.close
-        when 3
-          assert false, "Should not get any more events"
-        end
-      end
-
-    end
-
-    assert_equal 3, state
-
-    # Try connecting again, ensure that websockets server is still running and
-    # didn't crash per #6451
-    subscribe_test()
-
-  end
-
-
-end
diff --git a/services/api/test/test_helper.rb b/services/api/test/test_helper.rb
index 86bc2397c5..38f44c5d9b 100644
--- a/services/api/test/test_helper.rb
+++ b/services/api/test/test_helper.rb
@@ -1,8 +1,14 @@
 ENV["RAILS_ENV"] = "test"
 unless ENV["NO_COVERAGE_TEST"]
   begin
-    require 'simplecov'
-    require 'simplecov-rcov'
+    verbose_orig = $VERBOSE
+    begin
+      $VERBOSE = nil
+      require 'simplecov'
+      require 'simplecov-rcov'
+    ensure
+      $VERBOSE = verbose_orig
+    end
     class SimpleCov::Formatter::MergedFormatter
       def format(result)
         SimpleCov::Formatter::HTMLFormatter.new.format(result)
@@ -23,6 +29,7 @@ end
 require File.expand_path('../../config/environment', __FILE__)
 require 'rails/test_help'
 require 'mocha'
+require 'mocha/mini_test'
 
 module ArvadosTestSupport
   def json_response
@@ -49,10 +56,6 @@ class ActiveSupport::TestCase
   include ArvadosTestSupport
   include CurrentApiClient
 
-  setup do
-    Rails.logger.warn "\n\n#{'=' * 70}\n#{self.class}\##{method_name}\n#{'-' * 70}\n\n"
-  end
-
   teardown do
     Thread.current[:api_client_ip_address] = nil
     Thread.current[:api_client_authorization] = nil
@@ -60,6 +63,15 @@ class ActiveSupport::TestCase
     Thread.current[:api_client] = nil
     Thread.current[:user] = nil
     restore_configuration
+    User.invalidate_permissions_cache
+  end
+
+  def assert_equal(expect, *args)
+    if expect.nil?
+      assert_nil(*args)
+    else
+      super
+    end
   end
 
   def assert_not_allowed
@@ -122,8 +134,6 @@ class ActiveSupport::TestCase
   def self.slow_test(name, &block)
     define_method(name, block) unless skip_slow_tests?
   end
-
-  alias_method :skip, :omit
 end
 
 class ActionController::TestCase
diff --git a/services/api/test/unit/arvados_model_test.rb b/services/api/test/unit/arvados_model_test.rb
index 676581470c..d83f583bf3 100644
--- a/services/api/test/unit/arvados_model_test.rb
+++ b/services/api/test/unit/arvados_model_test.rb
@@ -53,15 +53,25 @@ class ArvadosModelTest < ActiveSupport::TestCase
     {'a' => {'foo' => {:bar => 'baz'}}},
     {'a' => {'foo' => {'bar' => :baz}}},
     {'a' => {'foo' => ['bar', :baz]}},
+  ].each do |x|
+    test "prevent symbol keys in serialized db columns: #{x.inspect}" do
+      set_user_from_auth :active
+      link = Link.create!(link_class: 'test',
+                          properties: x)
+      raw = ActiveRecord::Base.connection.
+          select_value("select properties from links where uuid='#{link.uuid}'")
+      refute_match(/:[fb]/, raw)
+    end
+  end
+
+  [ {['foo'] => 'bar'},
+    {'a' => {['foo', :foo] => 'bar'}},
+    {'a' => {{'foo' => 'bar'} => 'bar'}},
     {'a' => {['foo', :foo] => ['bar', 'baz']}},
   ].each do |x|
-    test "refuse symbol keys in serialized attribute: #{x.inspect}" do
-      set_user_from_auth :admin_trustedclient
-      assert_nothing_raised do
-        Link.create!(link_class: 'test',
-                     properties: {})
-      end
-      assert_raises ActiveRecord::RecordInvalid do
+    test "refuse non-string keys in serialized db columns: #{x.inspect}" do
+      set_user_from_auth :active
+      assert_raises(ArgumentError) do
         Link.create!(link_class: 'test',
                      properties: x)
       end
@@ -81,10 +91,11 @@ class ArvadosModelTest < ActiveSupport::TestCase
 
   test "No HashWithIndifferentAccess in database" do
     set_user_from_auth :admin_trustedclient
-    assert_raises ActiveRecord::RecordInvalid do
-      Link.create!(link_class: 'test',
-                   properties: {'foo' => 'bar'}.with_indifferent_access)
-    end
+    link = Link.create!(link_class: 'test',
+                        properties: {'foo' => 'bar'}.with_indifferent_access)
+    raw = ActiveRecord::Base.connection.
+      select_value("select properties from links where uuid='#{link.uuid}'")
+    assert_equal '{"foo":"bar"}', raw
   end
 
   test "store long string" do
@@ -147,23 +158,46 @@ class ArvadosModelTest < ActiveSupport::TestCase
   end
 
   test "full text search index exists on models" do
+    indexes = {}
+    conn = ActiveRecord::Base.connection
+    conn.exec_query("SELECT i.relname as indname,
+      i.relowner as indowner,
+      idx.indrelid::regclass::text as table,
+      am.amname as indam,
+      idx.indkey,
+      ARRAY(
+            SELECT pg_get_indexdef(idx.indexrelid, k + 1, true)
+                   FROM generate_subscripts(idx.indkey, 1) as k
+                   ORDER BY k
+                   ) as keys,
+      idx.indexprs IS NOT NULL as indexprs,
+      idx.indpred IS NOT NULL as indpred
+      FROM   pg_index as idx
+      JOIN   pg_class as i
+      ON     i.oid = idx.indexrelid
+      JOIN   pg_am as am
+      ON     i.relam = am.oid
+      JOIN   pg_namespace as ns
+      ON     ns.oid = i.relnamespace
+      AND    ns.nspname = ANY(current_schemas(false))").each do |idx|
+      if idx['keys'].match(/to_tsvector/)
+        indexes[idx['table']] ||= []
+        indexes[idx['table']] << idx
+      end
+    end
     fts_tables =  ["collections", "container_requests", "groups", "jobs",
                    "pipeline_instances", "pipeline_templates", "workflows"]
     fts_tables.each do |table|
       table_class = table.classify.constantize
       if table_class.respond_to?('full_text_searchable_columns')
-        fts_index_columns = table_class.full_text_searchable_columns
-        index_columns = nil
-        indexes = ActiveRecord::Base.connection.indexes(table)
-        fts_index_by_columns = indexes.select do |index|
-          if index.columns.first.match(/to_tsvector/)
-            index_columns = index.columns.first.scan(/\((?<columns>[A-Za-z_]+)\,/).flatten!
-            index_columns.sort == fts_index_columns.sort
-          else
-            false
+        expect = table_class.full_text_searchable_columns
+        ok = false
+        indexes[table].andand.each do |idx|
+          if expect == idx['keys'].scan(/COALESCE\(([A-Za-z_]+)/).flatten
+            ok = true
           end
         end
-        assert !fts_index_by_columns.empty?, "#{table} has no FTS index with columns #{fts_index_columns}. Instead found FTS index with columns #{index_columns}"
+        assert ok, "#{table} has no full-text index\nexpect: #{expect.inspect}\nfound: #{indexes[table].inspect}"
       end
     end
   end
diff --git a/services/api/test/unit/create_superuser_token_test.rb b/services/api/test/unit/create_superuser_token_test.rb
index 122ae51c2d..45446ec85b 100644
--- a/services/api/test/unit/create_superuser_token_test.rb
+++ b/services/api/test/unit/create_superuser_token_test.rb
@@ -90,7 +90,7 @@ class CreateSuperUserTokenTest < ActiveSupport::TestCase
     active_user_token = api_client_authorizations("admin_vm").api_token
     ApiClientAuthorization.
       where(user_id: system_user.id).
-      update_all(scopes: SafeJSON.dump(["GET /"]))
+      update_all(scopes: ["GET /"])
     fixture_tokens = ApiClientAuthorization.all.collect(&:api_token)
     new_token = create_superuser_token
     refute_includes(fixture_tokens, new_token)
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index 5677776cd4..9241465d49 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -176,28 +176,35 @@ class JobTest < ActiveSupport::TestCase
   [
    {script_parameters: ""},
    {script_parameters: []},
-   {script_parameters: {symbols: :are_not_allowed_here}},
+   {script_parameters: {["foo"] => ["bar"]}},
    {runtime_constraints: ""},
    {runtime_constraints: []},
    {tasks_summary: ""},
    {tasks_summary: []},
-   {script_version: "no/branch/could/ever/possibly/have/this/name"},
   ].each do |invalid_attrs|
     test "validation failures set error messages: #{invalid_attrs.to_json}" do
       # Ensure valid_attrs doesn't produce errors -- otherwise we will
       # not know whether errors reported below are actually caused by
       # invalid_attrs.
-      Job.create! job_attrs
+      Job.new(job_attrs).save!
 
-      job = Job.create job_attrs(invalid_attrs)
-      assert_raises(ActiveRecord::RecordInvalid, ArgumentError,
-                    "save! did not raise the expected exception") do
-        job.save!
+      err = assert_raises(ArgumentError) do
+        Job.new(job_attrs(invalid_attrs)).save!
       end
-      assert_not_empty job.errors, "validation failure did not provide errors"
+      assert_match /parameters|constraints|summary/, err.message
     end
   end
 
+  test "invalid script_version" do
+    invalid = {
+      script_version: "no/branch/could/ever/possibly/have/this/name",
+    }
+    err = assert_raises(ActiveRecord::RecordInvalid) do
+      Job.new(job_attrs(invalid)).save!
+    end
+    assert_match /Script version .* does not resolve to a commit/, err.message
+  end
+
   [
     # Each test case is of the following format
     # Array of parameters where each parameter is of the format:
diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb
index 3bd6ed4003..742deda0c6 100644
--- a/services/api/test/unit/user_test.rb
+++ b/services/api/test/unit/user_test.rb
@@ -162,8 +162,8 @@ class UserTest < ActiveSupport::TestCase
       if auto_admin_first_user_config
         # This test requires no admin users exist (except for the system user)
         users(:admin).delete
-        @all_users = User.where("uuid not like '%-000000000000000'").where(:is_admin => true).find(:all)
-        assert_equal 0, @all_users.size, "No admin users should exist (except for the system user)"
+        @all_users = User.where("uuid not like '%-000000000000000'").where(:is_admin => true)
+        assert_equal 0, @all_users.count, "No admin users should exist (except for the system user)"
       end
 
       Rails.configuration.auto_admin_first_user = auto_admin_first_user_config
@@ -285,7 +285,7 @@ class UserTest < ActiveSupport::TestCase
   end
 
   test "find user method checks" do
-    User.find(:all).each do |user|
+    User.all.each do |user|
       assert_not_nil user.uuid, "non-null uuid expected for " + user.full_name
     end
 
@@ -313,14 +313,14 @@ class UserTest < ActiveSupport::TestCase
   test "create new user" do
     set_user_from_auth :admin
 
-    @all_users = User.find(:all)
+    @all_users = User.all.to_a
 
     user = User.new
     user.first_name = "first_name_for_newly_created_user"
     user.save
 
     # verify there is one extra user in the db now
-    assert_equal @all_users.size+1, User.find(:all).size
+    assert_equal @all_users.size+1, User.all.count
 
     user = User.find(user.id)   # get the user back
     assert_equal(user.first_name, 'first_name_for_newly_created_user')
@@ -422,7 +422,7 @@ class UserTest < ActiveSupport::TestCase
     @active_user.delete
 
     found_deleted_user = false
-    User.find(:all).each do |user|
+    User.all.each do |user|
       if user.uuid == active_user_uuid
         found_deleted_user = true
         break
diff --git a/services/api/test/unit/workflow_test.rb b/services/api/test/unit/workflow_test.rb
index c7c528828d..29ca09bdab 100644
--- a/services/api/test/unit/workflow_test.rb
+++ b/services/api/test/unit/workflow_test.rb
@@ -85,8 +85,8 @@ class WorkflowTest < ActiveSupport::TestCase
     definition = "more: etc"
     w.update_attributes!(definition: definition)
     w.reload
-    assert_equal nil, w.name
-    assert_equal nil, w.description
+    assert_nil w.name
+    assert_nil w.description
 
     # Workflow name and desc set using definition yaml should be cleared
     # if definition yaml is cleared
@@ -96,8 +96,8 @@ class WorkflowTest < ActiveSupport::TestCase
     definition = nil
     w.update_attributes!(definition: definition)
     w.reload
-    assert_equal nil, w.name
-    assert_equal nil, w.description
+    assert_nil w.name
+    assert_nil w.description
 
     # Workflow name and desc should be set to provided custom values
     definition = "name: test name 3\ndescription: test desc 3\nother: some more"