From: Peter Amstutz Date: Fri, 2 Dec 2016 18:17:33 +0000 (-0500) Subject: Merge branch '10655-arvbash' closes #10655 X-Git-Tag: 1.1.0~567 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/d25dedaec8ea386c18b7f61c08a3097ba3c4f26c?hp=2993bc9aea20c2295d1b2b3cbecf3b1f7e06bb17 Merge branch '10655-arvbash' closes #10655 --- diff --git a/apps/workbench/Gemfile b/apps/workbench/Gemfile index 20d64d17a1..eac4fdf601 100644 --- a/apps/workbench/Gemfile +++ b/apps/workbench/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'rails', '~> 4.1.0' +gem 'rails', '~> 4.1' gem 'arvados', '>= 0.1.20150511150219' gem 'activerecord-nulldb-adapter' @@ -19,7 +19,7 @@ gem 'coffee-rails' # in production environments by default. group :assets do gem 'sass-rails' - gem 'uglifier', '>= 1.0.3' + gem 'uglifier', '~> 2.0' # See https://github.com/sstephenson/execjs#readme for more supported runtimes gem 'therubyracer', :platforms => :ruby @@ -33,7 +33,7 @@ group :development do end group :test, :diagnostics, :performance do - gem 'minitest', '>= 5.0.0' + gem 'minitest', '~> 5.0' gem 'selenium-webdriver' gem 'capybara' gem 'poltergeist' diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock index a8431a7dfd..0abe868ccf 100644 --- a/apps/workbench/Gemfile.lock +++ b/apps/workbench/Gemfile.lock @@ -92,7 +92,7 @@ GEM deep_merge (1.0.1) docile (1.1.5) erubis (2.7.0) - execjs (2.2.2) + execjs (2.7.0) extlib (0.9.16) faraday (0.9.2) multipart-post (>= 1.2, < 3) @@ -123,7 +123,7 @@ GEM signet (~> 0.7) headless (1.0.2) highline (1.6.21) - httpclient (2.6.0.1) + httpclient (2.8.2.4) i18n (0.7.0) jquery-rails (3.1.2) railties (>= 3.0, < 5.0) @@ -158,7 +158,7 @@ GEM metaclass (~> 0.0.1) morrisjs-rails (0.5.1) railties (> 3.1, < 5) - multi_json (1.12.0) + multi_json (1.12.1) multipart-post (2.0.0) net-scp (1.2.1) net-ssh (>= 2.6.5) @@ -257,7 +257,7 @@ GEM tilt (1.4.1) tzinfo (1.2.2) thread_safe (~> 0.1) - uglifier (2.7.0) + uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) websocket (1.2.2) @@ -292,7 +292,7 @@ DEPENDENCIES less-rails lograge logstash-event - minitest (>= 5.0.0) + minitest (~> 5.0) mocha morrisjs-rails multi_json @@ -301,7 +301,7 @@ DEPENDENCIES piwik_analytics poltergeist rack-mini-profiler - rails (~> 4.1.0) + rails (~> 4.1) rails-perftest raphael-rails ruby-debug-passenger @@ -316,8 +316,8 @@ DEPENDENCIES sshkey themes_for_rails! therubyracer - uglifier (>= 1.0.3) + uglifier (~> 2.0) wiselinks BUNDLED WITH - 1.12.1 + 1.13.2 diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb index c9ce8ce0b7..ee3ac4d681 100644 --- a/apps/workbench/app/controllers/application_controller.rb +++ b/apps/workbench/app/controllers/application_controller.rb @@ -907,7 +907,7 @@ class ApplicationController < ActionController::Base # from the top three levels. # That is: get toplevel projects under home, get subprojects of # these projects, and so on until we hit the limit. - def my_wanted_projects user, page_size=100 + def my_wanted_projects(user, page_size=100) return @my_wanted_projects if @my_wanted_projects from_top = [] @@ -922,7 +922,7 @@ class ApplicationController < ActionController::Base break if current_level.results.size == 0 @too_many_projects = true if current_level.items_available > current_level.results.size from_top.concat current_level.results - uuids = current_level.results.collect { |x| x.uuid } + uuids = current_level.results.collect(&:uuid) depth += 1 if depth >= 3 @reached_level_limit = true @@ -933,12 +933,12 @@ class ApplicationController < ActionController::Base end helper_method :my_wanted_projects_tree - def my_wanted_projects_tree user, page_size=100 - build_my_wanted_projects_tree user, page_size + def my_wanted_projects_tree(user, page_size=100) + build_my_wanted_projects_tree(user, page_size) [@my_wanted_projects_tree, @too_many_projects, @reached_level_limit] end - def build_my_wanted_projects_tree user, page_size=100 + def build_my_wanted_projects_tree(user, page_size=100) return @my_wanted_projects_tree if @my_wanted_projects_tree parent_of = {user.uuid => 'me'} diff --git a/apps/workbench/test/controllers/projects_controller_test.rb b/apps/workbench/test/controllers/projects_controller_test.rb index d31d6e3458..d0b1e287ff 100644 --- a/apps/workbench/test/controllers/projects_controller_test.rb +++ b/apps/workbench/test/controllers/projects_controller_test.rb @@ -490,27 +490,28 @@ class ProjectsControllerTest < ActionController::TestCase ["user1_with_load", 2, ["project_with_10_collections"], "project_with_2_pipelines_and_60_crs"], ["admin", 5, ["anonymously_accessible_project", "subproject_in_anonymous_accessible_project"], "aproject"], ].each do |user, page_size, tree_segment, unexpected| + # Note: this test is sensitive to database collation. It passes + # with en_US.UTF-8. test "build my projects tree for #{user} user and verify #{unexpected} is omitted" do use_token user - ctrl = ProjectsController.new - - current_user = User.find(api_fixture('users')[user]['uuid']) - my_tree = ctrl.send :my_wanted_projects_tree, current_user, page_size + tree, _, _ = @controller.send(:my_wanted_projects_tree, + User.current, + page_size) tree_segment_at_depth_1 = api_fixture('groups')[tree_segment[0]] tree_segment_at_depth_2 = api_fixture('groups')[tree_segment[1]] if tree_segment[1] - tree_nodes = {} - my_tree[0].each do |x| - tree_nodes[x[:object]['uuid']] = x[:depth] + node_depth = {} + tree.each do |x| + node_depth[x[:object]['uuid']] = x[:depth] end - assert_equal(1, tree_nodes[tree_segment_at_depth_1['uuid']]) - assert_equal(2, tree_nodes[tree_segment_at_depth_2['uuid']]) if tree_segment[1] + assert_equal(1, node_depth[tree_segment_at_depth_1['uuid']]) + assert_equal(2, node_depth[tree_segment_at_depth_2['uuid']]) if tree_segment[1] unexpected_project = api_fixture('groups')[unexpected] - assert_nil(tree_nodes[unexpected_project['uuid']]) + assert_nil(node_depth[unexpected_project['uuid']], node_depth.inspect) end end diff --git a/build/README b/build/README index 418254457d..4c67839a10 100644 --- a/build/README +++ b/build/README @@ -23,7 +23,11 @@ run-build-packages-python-and-ruby.sh Build Python and Ruby packages suitable run-build-docker-images.sh Build arvbox Docker images. -run-build-docker-jobs-image.sh Build arvados/jobs Docker image. +run-build-docker-jobs-image.sh Build arvados/jobs Docker image + (uses published debian packages) + +build-dev-docker-jobs-image.sh Build developer arvados/jobs Docker image + (uses local git tree) run-library.sh A library of functions shared by the various scripts in this diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh new file mode 100755 index 0000000000..5a6e77717e --- /dev/null +++ b/build/build-dev-docker-jobs-image.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +read -rd "\000" helpmessage <=0.3.0'. Force the version. Ward, 2016-03-25 fpm_build rdflib-jsonld "" "" python 0.3.0 diff --git a/build/run-tests.sh b/build/run-tests.sh index 8959cfbe09..d08568cde5 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -156,6 +156,12 @@ sanity_checks() { echo -n 'virtualenv: ' virtualenv --version \ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" + echo -n 'ruby: ' + ruby -v \ + || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)" + echo -n 'bundler: ' + bundle version \ + || fatal "No bundler. Try: gem install bundler" echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py index 7bdae5a1bd..c755565135 100644 --- a/sdk/cwl/arvados_cwl/done.py +++ b/sdk/cwl/arvados_cwl/done.py @@ -35,7 +35,7 @@ def done(self, record, tmpdir, outdir, keepdir): }, ensure_unique_name=True).execute( num_retries=self.arvrunner.num_retries) - done_outputs(self, record, tmpdir, outdir, keepdir) + return done_outputs(self, record, tmpdir, outdir, keepdir) def done_outputs(self, record, tmpdir, outdir, keepdir): self.builder.outdir = outdir diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile new file mode 100644 index 0000000000..bd80a0c1af --- /dev/null +++ b/sdk/dev-jobs.dockerfile @@ -0,0 +1,38 @@ +# Dockerfile for building an arvados/jobs Docker image from local git tree. +# +# Intended for use by developers working on arvados-python-client or +# arvados-cwl-runner and need to run a crunch job with a custom package +# version. +# +# Use arvados/build/build-dev-docker-jobs-image.sh to build. +# +# (This dockerfile file must be located in the arvados/sdk/ directory because +# of the docker build root.) + +FROM debian:jessie +MAINTAINER Ward Vandewege + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs + +RUN pip install -U setuptools + +ARG sdk +ARG runner +ARG cwltool + +ADD python/dist/$sdk /tmp/ +ADD cwl/cwltool_dist/$cwltool /tmp/ +ADD cwl/dist/$runner /tmp/ + +RUN cd /tmp/arvados-python-client-* && python setup.py install +RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && python setup.py install ; fi +RUN cd /tmp/arvados-cwl-runner-* && python setup.py install + +# Install dependencies and set up system. +RUN /usr/sbin/adduser --disabled-password \ + --gecos 'Crunch execution user' crunch && \ + /usr/bin/install --directory --owner=crunch --group=crunch --mode=0700 /keep /tmp/crunch-src /tmp/crunch-job + +USER crunch diff --git a/services/api/Gemfile b/services/api/Gemfile index 5d9b031e02..88237c9063 100644 --- a/services/api/Gemfile +++ b/services/api/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'rails', '~> 3.2.0' +gem 'rails', '~> 3.2' # Bundle edge Rails instead: # gem 'rails', :git => 'git://github.com/rails/rails.git' @@ -12,14 +12,13 @@ group :test, :development do # Note: "require: false" here tells bunder not to automatically # 'require' the packages during application startup. Installation is # still mandatory. + gem 'test-unit', '~> 3.0', require: false gem 'simplecov', '~> 0.7.1', require: false gem 'simplecov-rcov', require: false gem 'mocha', require: false end -# This might not be needed in :test and :development, but we load it -# anyway to make sure it always gets in Gemfile.lock and to help -# reveal install problems sooner rather than later. +# pg is the only supported database driver. gem 'pg' # Start using multi_json once we are on Rails 3.2; @@ -31,13 +30,13 @@ gem 'oj' # Gems used only for assets and not required # in production environments by default. group :assets do - gem 'sass-rails', '>= 3.2.0' - gem 'coffee-rails', '~> 3.2.0' + gem 'sass-rails', '~> 3.2' + gem 'coffee-rails', '~> 3.2' # See https://github.com/sstephenson/execjs#readme for more supported runtimes gem 'therubyracer' - gem 'uglifier', '>= 1.0.3' + gem 'uglifier', '~> 2.0' end gem 'jquery-rails' @@ -60,8 +59,8 @@ gem 'acts_as_api' gem 'passenger' -gem 'omniauth', '1.1.1' -gem 'omniauth-oauth2', '1.1.1' +gem 'omniauth', '~> 1.1' +gem 'omniauth-oauth2', '~> 1.1' gem 'andand' @@ -78,7 +77,7 @@ gem 'arvados-cli', '>= 0.1.20161017193526' # pg_power lets us use partial indexes in schema.rb in Rails 3 gem 'pg_power' -gem 'puma' +gem 'puma', '~> 2.0' gem 'sshkey' gem 'safe_yaml' gem 'lograge' diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock index 6f7875163b..9c9c4ae9e5 100644 --- a/services/api/Gemfile.lock +++ b/services/api/Gemfile.lock @@ -1,12 +1,12 @@ GEM remote: https://rubygems.org/ specs: - actionmailer (3.2.17) - actionpack (= 3.2.17) + actionmailer (3.2.22.5) + actionpack (= 3.2.22.5) mail (~> 2.5.4) - actionpack (3.2.17) - activemodel (= 3.2.17) - activesupport (= 3.2.17) + actionpack (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) builder (~> 3.0.0) erubis (~> 2.7.0) journey (~> 1.0.4) @@ -14,31 +14,31 @@ GEM rack-cache (~> 1.2) rack-test (~> 0.6.1) sprockets (~> 2.2.1) - activemodel (3.2.17) - activesupport (= 3.2.17) + activemodel (3.2.22.5) + activesupport (= 3.2.22.5) builder (~> 3.0.0) - activerecord (3.2.17) - activemodel (= 3.2.17) - activesupport (= 3.2.17) + activerecord (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) arel (~> 3.0.2) tzinfo (~> 0.3.29) - activeresource (3.2.17) - activemodel (= 3.2.17) - activesupport (= 3.2.17) - activesupport (3.2.17) + activeresource (3.2.22.5) + activemodel (= 3.2.22.5) + activesupport (= 3.2.22.5) + activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - acts_as_api (0.4.2) + acts_as_api (0.4.3) activemodel (>= 3.0.0) activesupport (>= 3.0.0) rack (>= 1.1.0) addressable (2.4.0) andand (1.3.3) arel (3.0.3) - arvados (0.1.20160420143004) + arvados (0.1.20160513152536) activesupport (>= 3, < 4.2.6) andand (~> 1.3, >= 1.3.3) - google-api-client (>= 0.7, < 0.9) + google-api-client (>= 0.7, < 0.8.9) i18n (~> 0) json (~> 1.7, >= 1.7.7) jwt (>= 0.1.5, < 2) @@ -56,62 +56,71 @@ GEM extlib (>= 0.9.15) multi_json (>= 1.0.0) builder (3.0.4) - capistrano (2.15.5) + capistrano (2.15.9) highline net-scp (>= 1.0.0) net-sftp (>= 2.0.0) net-ssh (>= 2.0.14) net-ssh-gateway (>= 1.1.0) - coffee-rails (3.2.1) + coffee-rails (3.2.2) coffee-script (>= 2.2.0) - railties (~> 3.2.0.beta) - coffee-script (2.2.0) + railties (~> 3.2.0) + coffee-script (2.4.1) coffee-script-source execjs - coffee-script-source (1.7.0) + coffee-script-source (1.10.0) curb (0.9.3) - daemon_controller (1.2.0) - database_cleaner (1.2.0) + database_cleaner (1.5.3) erubis (2.7.0) - eventmachine (1.0.3) - execjs (2.0.2) + eventmachine (1.2.0.1) + execjs (2.7.0) extlib (0.9.16) - factory_girl (4.4.0) + factory_girl (4.7.0) activesupport (>= 3.0.0) - factory_girl_rails (4.4.1) - factory_girl (~> 4.4.0) + factory_girl_rails (4.7.0) + factory_girl (~> 4.7.0) railties (>= 3.0.0) faraday (0.9.2) multipart-post (>= 1.2, < 3) - faye-websocket (0.7.2) + faye-websocket (0.10.4) eventmachine (>= 0.12.0) - websocket-driver (>= 0.3.1) - google-api-client (0.7.1) - addressable (>= 2.3.2) - autoparse (>= 0.3.3) - extlib (>= 0.9.15) - faraday (>= 0.9.0) - jwt (>= 0.1.5) - launchy (>= 2.1.1) - multi_json (>= 1.0.0) - retriable (>= 1.4) - signet (>= 0.5.0) - uuidtools (>= 2.1.0) - hashie (1.2.0) - highline (1.6.21) + websocket-driver (>= 0.5.1) + google-api-client (0.8.7) + activesupport (>= 3.2, < 5.0) + addressable (~> 2.3) + autoparse (~> 0.3) + extlib (~> 0.9) + faraday (~> 0.9) + googleauth (~> 0.3) + launchy (~> 2.4) + multi_json (~> 1.10) + retriable (~> 1.4) + signet (~> 0.6) + googleauth (0.5.1) + faraday (~> 0.9) + jwt (~> 1.4) + logging (~> 2.0) + memoist (~> 0.12) + multi_json (~> 1.11) + os (~> 0.9) + signet (~> 0.7) + hashie (3.4.6) + highline (1.7.8) hike (1.2.3) - httpauth (0.2.1) i18n (0.7.0) journey (1.0.4) - jquery-rails (3.1.0) + jquery-rails (3.1.4) railties (>= 3.0, < 5.0) thor (>= 0.14, < 2.0) json (1.8.3) - jwt (0.1.13) - multi_json (>= 1.5) + jwt (1.5.6) launchy (2.4.3) addressable (~> 2.3) - libv8 (3.16.14.3) + libv8 (3.16.14.15) + little-plugger (1.1.4) + logging (2.1.0) + little-plugger (~> 1.1) + multi_json (~> 1.10) lograge (0.3.6) actionpack (>= 3) activesupport (>= 3) @@ -120,100 +129,105 @@ GEM mail (2.5.4) mime-types (~> 1.16) treetop (~> 1.4.8) + memoist (0.15.0) metaclass (0.0.4) mime-types (1.25.1) - mocha (1.1.0) + mocha (1.2.0) metaclass (~> 0.0.1) - multi_json (1.12.0) + multi_json (1.12.1) + multi_xml (0.5.5) multipart-post (2.0.0) - net-scp (1.2.0) + net-scp (1.2.1) net-ssh (>= 2.6.5) net-sftp (2.1.2) net-ssh (>= 2.6.5) - net-ssh (2.8.0) + net-ssh (3.2.0) net-ssh-gateway (1.2.0) net-ssh (>= 2.6.5) - oauth2 (0.8.1) - faraday (~> 0.8) - httpauth (~> 0.1) - jwt (~> 0.1.4) - multi_json (~> 1.0) - rack (~> 1.2) + oauth2 (1.2.0) + faraday (>= 0.8, < 0.10) + jwt (~> 1.0) + multi_json (~> 1.3) + multi_xml (~> 0.5) + rack (>= 1.2, < 3) oj (2.15.0) - omniauth (1.1.1) - hashie (~> 1.2) - rack - omniauth-oauth2 (1.1.1) - oauth2 (~> 0.8.0) - omniauth (~> 1.0) - passenger (4.0.41) - daemon_controller (>= 1.2.0) + omniauth (1.3.1) + hashie (>= 1.2, < 4) + rack (>= 1.0, < 3) + omniauth-oauth2 (1.4.0) + oauth2 (~> 1.0) + omniauth (~> 1.2) + os (0.9.6) + passenger (5.0.30) rack rake (>= 0.8.1) - pg (0.17.1) + pg (0.19.0) pg_power (1.6.4) pg rails (~> 3.1) - polyglot (0.3.4) - puma (2.8.2) - rack (>= 1.1, < 2.0) - rack (1.4.5) - rack-cache (1.2) + polyglot (0.3.5) + power_assert (0.3.1) + puma (2.16.0) + rack (1.4.7) + rack-cache (1.6.1) rack (>= 0.4) rack-ssl (1.3.4) rack - rack-test (0.6.2) + rack-test (0.6.3) rack (>= 1.0) - rails (3.2.17) - actionmailer (= 3.2.17) - actionpack (= 3.2.17) - activerecord (= 3.2.17) - activeresource (= 3.2.17) - activesupport (= 3.2.17) + rails (3.2.22.5) + actionmailer (= 3.2.22.5) + actionpack (= 3.2.22.5) + activerecord (= 3.2.22.5) + activeresource (= 3.2.22.5) + activesupport (= 3.2.22.5) bundler (~> 1.0) - railties (= 3.2.17) - railties (3.2.17) - actionpack (= 3.2.17) - activesupport (= 3.2.17) + railties (= 3.2.22.5) + railties (3.2.22.5) + actionpack (= 3.2.22.5) + activesupport (= 3.2.22.5) rack-ssl (~> 1.3.2) rake (>= 0.8.7) rdoc (~> 3.4) thor (>= 0.14.6, < 2.0) - rake (10.2.2) + rake (11.3.0) rdoc (3.12.2) json (~> 1.4) - ref (1.0.5) - retriable (2.1.0) - ruby-prof (0.15.2) - rvm-capistrano (1.5.1) + ref (2.0.0) + retriable (1.4.1) + ruby-prof (0.16.2) + rvm-capistrano (1.5.6) capistrano (~> 2.15.4) safe_yaml (1.0.4) - sass (3.3.4) + sass (3.4.22) sass-rails (3.2.6) railties (~> 3.2.0) sass (>= 3.1.10) tilt (~> 1.3) - signet (0.5.1) - addressable (>= 2.2.3) - faraday (>= 0.9.0.rc5) - jwt (>= 0.1.5) - multi_json (>= 1.0.0) + signet (0.7.3) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (~> 1.5) + multi_json (~> 1.10) simplecov (0.7.1) multi_json (~> 1.0) simplecov-html (~> 0.7.1) simplecov-html (0.7.1) simplecov-rcov (0.2.3) simplecov (>= 0.4.1) - sprockets (2.2.2) + sprockets (2.2.3) hike (~> 1.2) multi_json (~> 1.0) rack (~> 1.0) tilt (~> 1.1, != 1.3.0) - sshkey (1.6.1) - test_after_commit (0.2.3) + sshkey (1.8.0) + test-unit (3.2.1) + power_assert + test_after_commit (1.1.0) + activerecord (>= 3.2) themes_for_rails (0.5.1) rails (>= 3.0.0) - therubyracer (0.12.1) + therubyracer (0.12.2) libv8 (~> 3.16.14.0) ref thor (0.19.1) @@ -222,12 +236,13 @@ GEM polyglot polyglot (>= 0.3.1) trollop (2.1.2) - tzinfo (0.3.39) - uglifier (2.5.0) + tzinfo (0.3.51) + uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) - uuidtools (2.1.5) - websocket-driver (0.3.2) + websocket-driver (0.6.4) + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.2) PLATFORMS ruby @@ -237,7 +252,7 @@ DEPENDENCIES andand arvados (>= 0.1.20150615153458) arvados-cli (>= 0.1.20161017193526) - coffee-rails (~> 3.2.0) + coffee-rails (~> 3.2) database_cleaner factory_girl_rails faye-websocket @@ -247,22 +262,26 @@ DEPENDENCIES mocha multi_json oj - omniauth (= 1.1.1) - omniauth-oauth2 (= 1.1.1) + omniauth (~> 1.1) + omniauth-oauth2 (~> 1.1) passenger pg pg_power - puma - rails (~> 3.2.0) + puma (~> 2.0) + rails (~> 3.2) ruby-prof rvm-capistrano safe_yaml - sass-rails (>= 3.2.0) + sass-rails (~> 3.2) simplecov (~> 0.7.1) simplecov-rcov sshkey + test-unit (~> 3.0) test_after_commit themes_for_rails therubyracer trollop - uglifier (>= 1.0.3) + uglifier (~> 2.0) + +BUNDLED WITH + 1.13.6 diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb index 776f7e190e..d58c432b6c 100644 --- a/services/api/app/controllers/application_controller.rb +++ b/services/api/app/controllers/application_controller.rb @@ -46,7 +46,7 @@ class ApplicationController < ActionController::Base theme :select_theme - attr_accessor :resource_attrs + attr_writer :resource_attrs begin rescue_from(Exception, @@ -59,6 +59,18 @@ class ApplicationController < ActionController::Base :with => :render_not_found) end + def initialize *args + super + @object = nil + @objects = nil + @offset = nil + @limit = nil + @select = nil + @distinct = nil + @response_resource_name = nil + @attrs = nil + end + def default_url_options if Rails.configuration.host {:host => Rails.configuration.host} @@ -420,7 +432,7 @@ class ApplicationController < ActionController::Base end def find_object_by_uuid - if params[:id] and params[:id].match /\D/ + if params[:id] and params[:id].match(/\D/) params[:uuid] = params.delete :id end @where = { uuid: params[:uuid] } @@ -567,7 +579,7 @@ class ApplicationController < ActionController::Base } end end - super *opts + super(*opts) end def select_theme diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb index 922cf7dac1..017c023db2 100644 --- a/services/api/app/controllers/arvados/v1/collections_controller.rb +++ b/services/api/app/controllers/arvados/v1/collections_controller.rb @@ -125,9 +125,9 @@ class Arvados::V1::CollectionsController < ApplicationController visited[uuid] = job.as_api_response if direction == :search_up # Follow upstream collections referenced in the script parameters - find_collections(visited, job) do |hash, uuid| + find_collections(visited, job) do |hash, col_uuid| search_edges(visited, hash, :search_up) if hash - search_edges(visited, uuid, :search_up) if uuid + search_edges(visited, col_uuid, :search_up) if col_uuid end elsif direction == :search_down # Follow downstream job output diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb index 2c55b15068..5f43ba8af8 100644 --- a/services/api/app/controllers/arvados/v1/schema_controller.rb +++ b/services/api/app/controllers/arvados/v1/schema_controller.rb @@ -85,7 +85,7 @@ class Arvados::V1::SchemaController < ApplicationController if Rails.application.config.websocket_address discovery[:websocketUrl] = Rails.application.config.websocket_address elsif ENV['ARVADOS_WEBSOCKETS'] - discovery[:websocketUrl] = (root_url.sub /^http/, 'ws') + "websocket" + discovery[:websocketUrl] = root_url.sub(/^http/, 'ws') + "websocket" end ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k| @@ -377,21 +377,21 @@ class Arvados::V1::SchemaController < ApplicationController method = d_methods[action.to_sym] end if ctl_class.respond_to? "_#{action}_requires_parameters".to_sym - ctl_class.send("_#{action}_requires_parameters".to_sym).each do |k, v| + ctl_class.send("_#{action}_requires_parameters".to_sym).each do |l, v| if v.is_a? Hash - method[:parameters][k] = v + method[:parameters][l] = v else - method[:parameters][k] = {} + method[:parameters][l] = {} end - if !method[:parameters][k][:default].nil? + if !method[:parameters][l][:default].nil? # The JAVA SDK is sensitive to all values being strings - method[:parameters][k][:default] = method[:parameters][k][:default].to_s + method[:parameters][l][:default] = method[:parameters][l][:default].to_s end - method[:parameters][k][:type] ||= 'string' - method[:parameters][k][:description] ||= '' - method[:parameters][k][:location] = (route.segment_keys.include?(k) ? 'path' : 'query') - if method[:parameters][k][:required].nil? - method[:parameters][k][:required] = v != false + method[:parameters][l][:type] ||= 'string' + method[:parameters][l][:description] ||= '' + method[:parameters][l][:location] = (route.segment_keys.include?(l) ? 'path' : 'query') + if method[:parameters][l][:required].nil? + method[:parameters][l][:required] = v != false end end end diff --git a/services/api/app/controllers/arvados/v1/user_agreements_controller.rb b/services/api/app/controllers/arvados/v1/user_agreements_controller.rb index 32adde9507..f23cd98c35 100644 --- a/services/api/app/controllers/arvados/v1/user_agreements_controller.rb +++ b/services/api/app/controllers/arvados/v1/user_agreements_controller.rb @@ -17,7 +17,6 @@ class Arvados::V1::UserAgreementsController < ApplicationController # use this installation. @objects = [] else - current_user_uuid = current_user.uuid act_as_system_user do uuids = Link.where("owner_uuid = ? and link_class = ? and name = ? and tail_uuid = ? and head_uuid like ?", system_user_uuid, @@ -25,7 +24,7 @@ class Arvados::V1::UserAgreementsController < ApplicationController 'require', system_user_uuid, Collection.uuid_like_pattern). - collect &:head_uuid + collect(&:head_uuid) @objects = Collection.where('uuid in (?)', uuids) end end diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb index 03efed999f..db5e7bd952 100644 --- a/services/api/app/controllers/arvados/v1/users_controller.rb +++ b/services/api/app/controllers/arvados/v1/users_controller.rb @@ -159,7 +159,7 @@ class Arvados::V1::UsersController < ApplicationController end def apply_filters(model_class=nil) - return super if @read_users.any? &:is_admin + return super if @read_users.any?(&:is_admin) if params[:uuid] != current_user.andand.uuid # Non-admin index/show returns very basic information about readable users. safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name"] diff --git a/services/api/app/controllers/arvados/v1/virtual_machines_controller.rb b/services/api/app/controllers/arvados/v1/virtual_machines_controller.rb index e6474aa4e0..99b663da43 100644 --- a/services/api/app/controllers/arvados/v1/virtual_machines_controller.rb +++ b/services/api/app/controllers/arvados/v1/virtual_machines_controller.rb @@ -23,7 +23,7 @@ class Arvados::V1::VirtualMachinesController < ApplicationController @users = {} User.eager_load(:authorized_keys). where('users.uuid in (?)', - @vms.map { |vm| vm.login_permissions.map &:tail_uuid }.flatten.uniq). + @vms.map { |vm| vm.login_permissions.map(&:tail_uuid) }.flatten.uniq). each do |u| @users[u.uuid] = u end diff --git a/services/api/app/controllers/database_controller.rb b/services/api/app/controllers/database_controller.rb index 21c8e4710c..6699f7363b 100644 --- a/services/api/app/controllers/database_controller.rb +++ b/services/api/app/controllers/database_controller.rb @@ -11,7 +11,7 @@ class DatabaseController < ApplicationController # we can tell they're not valuable. user_uuids = User. where('email is null or email not like ?', '%@example.com'). - collect &:uuid + collect(&:uuid) fixture_uuids = YAML::load_file(File.expand_path('../../../test/fixtures/users.yml', __FILE__)). diff --git a/services/api/app/middlewares/arvados_api_token.rb b/services/api/app/middlewares/arvados_api_token.rb index d8c04a1adb..2487f2ecb7 100644 --- a/services/api/app/middlewares/arvados_api_token.rb +++ b/services/api/app/middlewares/arvados_api_token.rb @@ -7,7 +7,7 @@ class ArvadosApiToken # Create a new ArvadosApiToken handler # +app+ The next layer of the Rack stack. def initialize(app = nil, options = nil) - @app = app if app.respond_to?(:call) + @app = app.respond_to?(:call) ? app : nil end def call env diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb index 18d5647cc9..aed0309591 100644 --- a/services/api/app/models/arvados_model.rb +++ b/services/api/app/models/arvados_model.rb @@ -239,7 +239,7 @@ class ArvadosModel < ActiveRecord::Base end def logged_attributes - attributes.except *Rails.configuration.unlogged_attributes + attributes.except(*Rails.configuration.unlogged_attributes) end def self.full_text_searchable_columns @@ -490,7 +490,7 @@ class ArvadosModel < ActiveRecord::Base end def foreign_key_attributes - attributes.keys.select { |a| a.match /_uuid$/ } + attributes.keys.select { |a| a.match(/_uuid$/) } end def skip_uuid_read_permission_check @@ -505,7 +505,7 @@ class ArvadosModel < ActiveRecord::Base foreign_key_attributes.each do |attr| attr_value = send attr if attr_value.is_a? String and - attr_value.match /^[0-9a-f]{32,}(\+[@\w]+)*$/ + attr_value.match(/^[0-9a-f]{32,}(\+[@\w]+)*$/) begin send "#{attr}=", Collection.normalize_uuid(attr_value) rescue @@ -584,13 +584,12 @@ class ArvadosModel < ActiveRecord::Base unless uuid.is_a? String return nil end - resource_class = nil uuid.match HasUuid::UUID_REGEX do |re| return uuid_prefixes[re[1]] if uuid_prefixes[re[1]] end - if uuid.match /.+@.+/ + if uuid.match(/.+@.+/) return Email end diff --git a/services/api/app/models/blob.rb b/services/api/app/models/blob.rb index 41d5b27093..00c2501865 100644 --- a/services/api/app/models/blob.rb +++ b/services/api/app/models/blob.rb @@ -64,9 +64,9 @@ class Blob # Return value: true if the locator has a valid signature, false otherwise # Arguments: signed_blob_locator, opts # - def self.verify_signature *args + def self.verify_signature(*args) begin - self.verify_signature! *args + self.verify_signature!(*args) true rescue Blob::InvalidSignatureError false diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb index 8579509de7..901084c763 100644 --- a/services/api/app/models/collection.rb +++ b/services/api/app/models/collection.rb @@ -32,6 +32,11 @@ class Collection < ArvadosModel t.add :expires_at end + after_initialize do + @signatures_checked = false + @computed_pdh_for_manifest_text = false + end + def self.attributes_required_columns super.merge( # If we don't list manifest_text explicitly, the @@ -61,7 +66,9 @@ class Collection < ArvadosModel # subsequent passes without checking any signatures. This is # important because the signatures have probably been stripped off # by the time we get to a second validation pass! - return true if @signatures_checked and @signatures_checked == computed_pdh + if @signatures_checked && @signatures_checked == computed_pdh + return true + end if self.manifest_text_changed? # Check permissions on the collection manifest. @@ -197,7 +204,7 @@ class Collection < ArvadosModel utf8 = manifest_text utf8.force_encoding Encoding::UTF_8 if utf8.valid_encoding? and utf8 == manifest_text.encode(Encoding::UTF_8) - manifest_text = utf8 + self.manifest_text = utf8 return true end rescue @@ -283,10 +290,10 @@ class Collection < ArvadosModel hash_part = nil size_part = nil uuid.split('+').each do |token| - if token.match /^[0-9a-f]{32,}$/ + if token.match(/^[0-9a-f]{32,}$/) raise "uuid #{uuid} has multiple hash parts" if hash_part hash_part = token - elsif token.match /^\d+$/ + elsif token.match(/^\d+$/) raise "uuid #{uuid} has multiple size parts" if size_part size_part = token end diff --git a/services/api/app/models/commit_ancestor.rb b/services/api/app/models/commit_ancestor.rb index 71ea57fb95..419eca2e01 100644 --- a/services/api/app/models/commit_ancestor.rb +++ b/services/api/app/models/commit_ancestor.rb @@ -16,13 +16,13 @@ class CommitAncestor < ActiveRecord::Base @gitdirbase = Rails.configuration.git_repositories_dir self.is = nil Dir.foreach @gitdirbase do |repo| - next if repo.match /^\./ + next if repo.match(/^\./) git_dir = repo.match(/\.git$/) ? repo : File.join(repo, '.git') repo_name = repo.sub(/\.git$/, '') ENV['GIT_DIR'] = File.join(@gitdirbase, git_dir) - IO.foreach("|git rev-list --format=oneline '#{self.descendant.gsub /[^0-9a-f]/,""}'") do |line| + IO.foreach("|git rev-list --format=oneline '#{self.descendant.gsub(/[^0-9a-f]/,"")}'") do |line| self.is = false - sha1, message = line.strip.split(" ", 2) + sha1, _ = line.strip.split(" ", 2) if sha1 == self.ancestor self.is = true break diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb index 30ca7f8cb2..ef3d0b5e10 100644 --- a/services/api/app/models/job.rb +++ b/services/api/app/models/job.rb @@ -67,6 +67,10 @@ class Job < ArvadosModel (Complete = 'Complete'), ] + after_initialize do + @need_crunch_dispatch_trigger = false + end + def assert_finished update_attributes(finished_at: finished_at || db_current_time, success: success.nil? ? false : success, @@ -336,7 +340,7 @@ class Job < ArvadosModel assign_uuid Commit.tag_in_internal_repository repository, script_version, uuid rescue - uuid = uuid_was + self.uuid = uuid_was raise end end @@ -565,24 +569,6 @@ class Job < ArvadosModel end def ensure_no_collection_uuids_in_script_params - # recursive_hash_search searches recursively through hashes and - # arrays in 'thing' for string fields matching regular expression - # 'pattern'. Returns true if pattern is found, false otherwise. - def recursive_hash_search thing, pattern - if thing.is_a? Hash - thing.each do |k, v| - return true if recursive_hash_search v, pattern - end - elsif thing.is_a? Array - thing.each do |k| - return true if recursive_hash_search k, pattern - end - elsif thing.is_a? String - return true if thing.match pattern - end - false - end - # Fail validation if any script_parameters field includes a string containing a # collection uuid pattern. if self.script_parameters_changed? @@ -593,4 +579,22 @@ class Job < ArvadosModel end true end + + # recursive_hash_search searches recursively through hashes and + # arrays in 'thing' for string fields matching regular expression + # 'pattern'. Returns true if pattern is found, false otherwise. + def recursive_hash_search thing, pattern + if thing.is_a? Hash + thing.each do |k, v| + return true if recursive_hash_search v, pattern + end + elsif thing.is_a? Array + thing.each do |k| + return true if recursive_hash_search k, pattern + end + elsif thing.is_a? String + return true if thing.match pattern + end + false + end end diff --git a/services/api/app/models/link.rb b/services/api/app/models/link.rb index 24872b21ec..649a6f80c2 100644 --- a/services/api/app/models/link.rb +++ b/services/api/app/models/link.rb @@ -8,7 +8,6 @@ class Link < ArvadosModel after_update :maybe_invalidate_permissions_cache after_create :maybe_invalidate_permissions_cache after_destroy :maybe_invalidate_permissions_cache - attr_accessor :head_kind, :tail_kind validate :name_links_are_obsolete api_accessible :user, extend: :common do |t| diff --git a/services/api/app/models/log.rb b/services/api/app/models/log.rb index 7eab402609..3207d1f288 100644 --- a/services/api/app/models/log.rb +++ b/services/api/app/models/log.rb @@ -4,7 +4,6 @@ class Log < ArvadosModel include CommonApiTemplate serialize :properties, Hash before_validation :set_default_event_at - attr_accessor :object, :object_kind after_save :send_notify api_accessible :user, extend: :common do |t| diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb index e470e4c2bd..1855020466 100644 --- a/services/api/app/models/node.rb +++ b/services/api/app/models/node.rb @@ -32,6 +32,10 @@ class Node < ArvadosModel t.add lambda { |x| Rails.configuration.compute_node_nameservers }, :as => :nameservers end + after_initialize do + @bypass_arvados_authorization = false + end + def domain super || Rails.configuration.compute_node_domain end @@ -226,7 +230,7 @@ class Node < ArvadosModel (0..Rails.configuration.max_compute_nodes-1).each do |slot_number| hostname = hostname_for_slot(slot_number) hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf" - if !File.exists? hostfile + if !File.exist? hostfile n = Node.where(:slot_number => slot_number).first if n.nil? or n.ip_address.nil? dns_server_update(hostname, UNUSED_NODE_IP) diff --git a/services/api/app/models/repository.rb b/services/api/app/models/repository.rb index f361a49db5..13b00df544 100644 --- a/services/api/app/models/repository.rb +++ b/services/api/app/models/repository.rb @@ -86,7 +86,7 @@ class Repository < ArvadosModel prefix_match = Regexp.escape(owner.username + "/") errmsg_start = "must be the owner's username, then '/', then" end - if not /^#{prefix_match}[A-Za-z][A-Za-z0-9]*$/.match(name) + if not (/^#{prefix_match}[A-Za-z][A-Za-z0-9]*$/.match(name)) errors.add(:name, "#{errmsg_start} a letter followed by alphanumerics") false diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb index 9363cc4f02..964de38d0b 100644 --- a/services/api/app/models/user.rb +++ b/services/api/app/models/user.rb @@ -64,7 +64,7 @@ class User < ArvadosModel def is_invited !!(self.is_active || Rails.configuration.new_users_are_active || - self.groups_i_can(:read).select { |x| x.match /-f+$/ }.first) + self.groups_i_can(:read).select { |x| x.match(/-f+$/) }.first) end def groups_i_can(verb) @@ -242,7 +242,7 @@ class User < ArvadosModel # delete "All users" group read permissions for this user group = Group.where(name: 'All users').select do |g| - g[:uuid].match /-f+$/ + g[:uuid].match(/-f+$/) end.first Link.destroy_all(tail_uuid: self.uuid, head_uuid: group[:uuid], diff --git a/services/api/config/boot.rb b/services/api/config/boot.rb index 4489e58688..f2830ae316 100644 --- a/services/api/config/boot.rb +++ b/services/api/config/boot.rb @@ -3,4 +3,4 @@ require 'rubygems' # Set up gems listed in the Gemfile. ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) -require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE']) +require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) diff --git a/services/api/config/initializers/inflections.rb b/services/api/config/initializers/inflections.rb index 79bca3af38..b6dadf7e2e 100644 --- a/services/api/config/initializers/inflections.rb +++ b/services/api/config/initializers/inflections.rb @@ -10,8 +10,8 @@ # end ActiveSupport::Inflector.inflections do |inflect| - inflect.plural /^([Ss]pecimen)$/i, '\1s' - inflect.singular /^([Ss]pecimen)s?/i, '\1' - inflect.plural /^([Hh]uman)$/i, '\1s' - inflect.singular /^([Hh]uman)s?/i, '\1' + inflect.plural(/^([Ss]pecimen)$/i, '\1s') + inflect.singular(/^([Ss]pecimen)s?/i, '\1') + inflect.plural(/^([Hh]uman)$/i, '\1s') + inflect.singular(/^([Hh]uman)s?/i, '\1') end diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb index 76234d3e4b..fd3c977393 100644 --- a/services/api/config/initializers/load_config.rb +++ b/services/api/config/initializers/load_config.rb @@ -6,7 +6,7 @@ rescue LoadError # configured by application.yml (i.e., here!) instead. end -if (File.exists?(File.expand_path '../omniauth.rb', __FILE__) and +if (File.exist?(File.expand_path '../omniauth.rb', __FILE__) and not defined? WARNED_OMNIAUTH_CONFIG) Rails.logger.warn <<-EOS DEPRECATED CONFIGURATION: @@ -26,7 +26,7 @@ $application_config = {} %w(application.default application).each do |cfgfile| path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml" - if File.exists? path + if File.exist? path yaml = ERB.new(IO.read path).result(binding) confs = YAML.load(yaml, deserialize_symbols: true) # Ignore empty YAML file: diff --git a/services/api/config/initializers/preload_all_models.rb b/services/api/config/initializers/preload_all_models.rb index 7e26123774..1ae531c169 100644 --- a/services/api/config/initializers/preload_all_models.rb +++ b/services/api/config/initializers/preload_all_models.rb @@ -7,6 +7,6 @@ require_relative 'load_config.rb' if Rails.env == 'development' Dir.foreach("#{Rails.root}/app/models") do |model_file| - require_dependency model_file if model_file.match /\.rb$/ + require_dependency model_file if model_file.match(/\.rb$/) end end diff --git a/services/api/lib/crunch_dispatch.rb b/services/api/lib/crunch_dispatch.rb index ce94f737a2..48b0eb5983 100644 --- a/services/api/lib/crunch_dispatch.rb +++ b/services/api/lib/crunch_dispatch.rb @@ -27,7 +27,7 @@ class CrunchDispatch @cgroup_root = ENV['CRUNCH_CGROUP_ROOT'] @arvados_internal = Rails.configuration.git_internal_dir - if not File.exists? @arvados_internal + if not File.exist? @arvados_internal $stderr.puts `mkdir -p #{@arvados_internal.shellescape} && git init --bare #{@arvados_internal.shellescape}` raise "No internal git repository available" unless ($? == 0) end @@ -73,7 +73,7 @@ class CrunchDispatch # into multiple rows with one hostname each. `#{cmd} --noheader -o '%N:#{outfmt}'`.each_line do |line| tokens = line.chomp.split(":", max_fields) - if (re = tokens[0].match /^(.*?)\[([-,\d]+)\]$/) + if (re = tokens[0].match(/^(.*?)\[([-,\d]+)\]$/)) tokens.shift re[2].split(",").each do |range| range = range.split("-").collect(&:to_i) @@ -105,7 +105,7 @@ class CrunchDispatch end def update_node_status - return unless Server::Application.config.crunch_job_wrapper.to_s.match /^slurm/ + return unless Server::Application.config.crunch_job_wrapper.to_s.match(/^slurm/) slurm_status.each_pair do |hostname, slurmdata| next if @node_state[hostname] == slurmdata begin @@ -169,7 +169,7 @@ class CrunchDispatch end usable_nodes << node if usable_nodes.count >= min_node_count - return usable_nodes.map { |node| node.hostname } + return usable_nodes.map { |n| n.hostname } end end nil @@ -512,8 +512,6 @@ class CrunchDispatch def read_pipes @running.each do |job_uuid, j| - job = j[:job] - now = Time.now if now > j[:log_throttle_reset_time] # It has been more than throttle_period seconds since the last diff --git a/services/api/lib/current_api_client.rb b/services/api/lib/current_api_client.rb index fbd4ef5f0c..97348d5fb4 100644 --- a/services/api/lib/current_api_client.rb +++ b/services/api/lib/current_api_client.rb @@ -1,3 +1,11 @@ +$system_user = nil +$system_group = nil +$all_users_group = nil +$anonymous_user = nil +$anonymous_group = nil +$anonymous_group_read_permission = nil +$empty_collection = nil + module CurrentApiClient def current_user Thread.current[:user] @@ -83,9 +91,7 @@ module CurrentApiClient User.all.collect(&:uuid).each do |user_uuid| Link.create!(link_class: 'permission', name: 'can_manage', - tail_kind: 'arvados#group', tail_uuid: system_group_uuid, - head_kind: 'arvados#user', head_uuid: user_uuid) end end diff --git a/services/api/lib/eventbus.rb b/services/api/lib/eventbus.rb index 16bb030941..cb65c7f30c 100644 --- a/services/api/lib/eventbus.rb +++ b/services/api/lib/eventbus.rb @@ -78,6 +78,10 @@ class EventBus @connection_count = 0 end + def send_message(ws, obj) + ws.send(Oj.dump(obj, mode: :compat)) + end + # Push out any pending events to the connection +ws+ # +notify_id+ the id of the most recent row in the log table, may be nil # @@ -146,7 +150,7 @@ class EventBus logs.select('logs.id').find_each do |l| if not ws.sent_ids.include?(l.id) # only send if not a duplicate - ws.send(Log.find(l.id).as_api_response.to_json) + send_message(ws, Log.find(l.id).as_api_response) end if not ws.last_log_id.nil? # record ids only when sending "catchup" messages, not notifies @@ -158,12 +162,12 @@ class EventBus rescue ArgumentError => e # There was some kind of user error. Rails.logger.warn "Error publishing event: #{$!}" - ws.send ({status: 500, message: $!}.to_json) + send_message(ws, {status: 500, message: $!}) ws.close rescue => e Rails.logger.warn "Error publishing event: #{$!}" Rails.logger.warn "Backtrace:\n\t#{e.backtrace.join("\n\t")}" - ws.send ({status: 500, message: $!}.to_json) + send_message(ws, {status: 500, message: $!}) ws.close # These exceptions typically indicate serious server trouble: # out of memory issues, database connection problems, etc. Go ahead and @@ -180,7 +184,7 @@ class EventBus p = (Oj.strict_load event.data).symbolize_keys filter = Filter.new(p) rescue Oj::Error => e - ws.send ({status: 400, message: "malformed request"}.to_json) + send_message(ws, {status: 400, message: "malformed request"}) return end @@ -200,12 +204,12 @@ class EventBus # Add a filter. This gets the :filters field which is the same # format as used for regular index queries. ws.filters << filter - ws.send ({status: 200, message: 'subscribe ok', filter: p}.to_json) + send_message(ws, {status: 200, message: 'subscribe ok', filter: p}) # Send any pending events push_events ws, nil else - ws.send ({status: 403, message: "maximum of #{Rails.configuration.websocket_max_filters} filters allowed per connection"}.to_json) + send_message(ws, {status: 403, message: "maximum of #{Rails.configuration.websocket_max_filters} filters allowed per connection"}) end elsif p[:method] == 'unsubscribe' @@ -214,18 +218,18 @@ class EventBus len = ws.filters.length ws.filters.select! { |f| not ((f.filters == p[:filters]) or (f.filters.empty? and p[:filters].nil?)) } if ws.filters.length < len - ws.send ({status: 200, message: 'unsubscribe ok'}.to_json) + send_message(ws, {status: 200, message: 'unsubscribe ok'}) else - ws.send ({status: 404, message: 'filter not found'}.to_json) + send_message(ws, {status: 404, message: 'filter not found'}) end else - ws.send ({status: 400, message: "missing or unrecognized method"}.to_json) + send_message(ws, {status: 400, message: "missing or unrecognized method"}) end rescue => e Rails.logger.warn "Error handling message: #{$!}" Rails.logger.warn "Backtrace:\n\t#{e.backtrace.join("\n\t")}" - ws.send ({status: 500, message: 'error'}.to_json) + send_message(ws, {status: 500, message: 'error'}) ws.close end end @@ -241,8 +245,13 @@ class EventBus # Disconnect if no valid API token. # current_user is included from CurrentApiClient if not current_user - ws.send ({status: 401, message: "Valid API token required"}.to_json) - ws.close + send_message(ws, {status: 401, message: "Valid API token required"}) + # Wait for the handshake to complete before closing the + # socket. Otherwise, nginx responds with HTTP 502 Bad gateway, + # and the client never sees our real error message. + ws.on :open do |event| + ws.close + end return end @@ -262,7 +271,7 @@ class EventBus # forward them to the thread associated with the connection. sub = @channel.subscribe do |msg| if ws.queue.length > Rails.configuration.websocket_max_notify_backlog - ws.send ({status: 500, message: 'Notify backlog too long'}.to_json) + send_message(ws, {status: 500, message: 'Notify backlog too long'}) ws.close @channel.unsubscribe sub ws.queue.clear diff --git a/services/api/lib/load_param.rb b/services/api/lib/load_param.rb index 5b22274d07..3bab33f9a9 100644 --- a/services/api/lib/load_param.rb +++ b/services/api/lib/load_param.rb @@ -92,11 +92,11 @@ module LoadParam # has used set_table_name to use an alternate table name from the Rails standard. # I could not find a perfect way to handle this well, but ActiveRecord::Base.send(:descendants) # would be a place to start if this ever becomes necessary. - if attr.match /^[a-z][_a-z0-9]+$/ and + if attr.match(/^[a-z][_a-z0-9]+$/) and model_class.columns.collect(&:name).index(attr) and ['asc','desc'].index direction.downcase @orders << "#{table_name}.#{attr} #{direction.downcase}" - elsif attr.match /^([a-z][_a-z0-9]+)\.([a-z][_a-z0-9]+)$/ and + elsif attr.match(/^([a-z][_a-z0-9]+)\.([a-z][_a-z0-9]+)$/) and ['asc','desc'].index(direction.downcase) and ActiveRecord::Base.connection.tables.include?($1) and $1.classify.constantize.columns.collect(&:name).index($2) diff --git a/services/api/lib/salvage_collection.rb b/services/api/lib/salvage_collection.rb index 2011f812d5..c6664b1ec9 100755 --- a/services/api/lib/salvage_collection.rb +++ b/services/api/lib/salvage_collection.rb @@ -29,7 +29,7 @@ module SalvageCollection def salvage_collection_locator_data manifest locators = [] size = 0 - manifest.scan /(^|[^[:xdigit:]])([[:xdigit:]]{32})((\+\d+)(\+|\b))?/ do |_, hash, _, sizehint, _| + manifest.scan(/(^|[^[:xdigit:]])([[:xdigit:]]{32})((\+\d+)(\+|\b))?/) do |_, hash, _, sizehint, _| if sizehint locators << hash.downcase + sizehint size += sizehint.to_i diff --git a/services/api/script/arvados-git-sync.rb b/services/api/script/arvados-git-sync.rb index 3a8ed2724f..b785534917 100755 --- a/services/api/script/arvados-git-sync.rb +++ b/services/api/script/arvados-git-sync.rb @@ -22,7 +22,7 @@ DEBUG = 1 # load and merge in the environment-specific application config info # if present, overriding base config parameters as specified path = File.absolute_path('../../config/arvados-clients.yml', __FILE__) -if File.exists?(path) then +if File.exist?(path) then cp_config = YAML.load_file(path)[ENV['RAILS_ENV']] else puts "Please create a\n #{path}\n file" @@ -214,7 +214,7 @@ end begin # Get our local gitolite-admin repo up to snuff - if not File.exists?(gitolite_admin) then + if not File.exist?(gitolite_admin) then ensure_directory(gitolite_tmpdir, 0700) Dir.chdir(gitolite_tmpdir) `git clone #{gitolite_url}` diff --git a/services/api/script/migrate-gitolite-to-uuid-storage.rb b/services/api/script/migrate-gitolite-to-uuid-storage.rb index 8db1a0edad..169509f63b 100755 --- a/services/api/script/migrate-gitolite-to-uuid-storage.rb +++ b/services/api/script/migrate-gitolite-to-uuid-storage.rb @@ -35,7 +35,7 @@ DEBUG = 1 # load and merge in the environment-specific application config info # if present, overriding base config parameters as specified path = File.dirname(__FILE__) + '/config/arvados-clients.yml' -if File.exists?(path) then +if File.exist?(path) then cp_config = YAML.load_file(path)[ENV['RAILS_ENV']] else puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file" @@ -186,7 +186,7 @@ end begin # Get our local gitolite-admin repo up to snuff - if not File.exists?(gitolite_admin) then + if not File.exist?(gitolite_admin) then ensure_directory(gitolite_tmpdir, 0700) Dir.chdir(gitolite_tmpdir) `git clone #{gitolite_url}` diff --git a/services/api/test/factories/user.rb b/services/api/test/factories/user.rb index 6ec9e9f05d..a0e5ad95dc 100644 --- a/services/api/test/factories/user.rb +++ b/services/api/test/factories/user.rb @@ -4,7 +4,7 @@ end FactoryGirl.define do factory :user do - ignore do + transient do join_groups [] end after :create do |user, evaluator| diff --git a/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb b/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb index 37e690e0b2..6c09d8e9f5 100644 --- a/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb +++ b/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb @@ -46,7 +46,7 @@ class Arvados::V1::ApiClientAuthorizationsControllerTest < ActionController::Tes get :index, search_params assert_response :success got_tokens = JSON.parse(@response.body)['items'] - .map { |auth| auth['api_token'] } + .map { |a| a['api_token'] } assert_equal(expected_tokens.sort, got_tokens.sort, "wrong results for #{search_params.inspect}") end diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb index c85cc1979f..b96e22ed65 100644 --- a/services/api/test/functional/arvados/v1/collections_controller_test.rb +++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb @@ -794,11 +794,11 @@ EOS [2**8, :success], [2**18, 422], ].each do |description_size, expected_response| - test "create collection with description size #{description_size} + # Descriptions are not part of search indexes. Skip until + # full-text search is implemented, at which point replace with a + # search in description. + skip "create collection with description size #{description_size} and expect response #{expected_response}" do - skip "(Descriptions are not part of search indexes. Skip until full-text search - is implemented, at which point replace with a search in description.)" - authorize_with :active description = 'here is a collection with a very large description' diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb index 9344b0bc75..2b1b675323 100644 --- a/services/api/test/functional/arvados/v1/filters_test.rb +++ b/services/api/test/functional/arvados/v1/filters_test.rb @@ -32,7 +32,7 @@ class Arvados::V1::FiltersTest < ActionController::TestCase filters: [['uuid', '@@', 'abcdef']], } assert_response 422 - assert_match /not supported/, json_response['errors'].join(' ') + assert_match(/not supported/, json_response['errors'].join(' ')) end test 'difficult characters in full text search' do @@ -52,7 +52,7 @@ class Arvados::V1::FiltersTest < ActionController::TestCase filters: [['any', '@@', ['abc', 'def']]], } assert_response 422 - assert_match /not supported/, json_response['errors'].join(' ') + assert_match(/not supported/, json_response['errors'].join(' ')) end test 'api responses provide timestamps with nanoseconds' do @@ -65,7 +65,7 @@ class Arvados::V1::FiltersTest < ActionController::TestCase %w(created_at modified_at).each do |attr| # Pass fixtures with null timestamps. next if item[attr].nil? - assert_match /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d{9}Z$/, item[attr] + assert_match(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d{9}Z$/, item[attr]) end end end diff --git a/services/api/test/functional/arvados/v1/jobs_controller_test.rb b/services/api/test/functional/arvados/v1/jobs_controller_test.rb index 3c11b3e009..8808a82c45 100644 --- a/services/api/test/functional/arvados/v1/jobs_controller_test.rb +++ b/services/api/test/functional/arvados/v1/jobs_controller_test.rb @@ -97,7 +97,7 @@ class Arvados::V1::JobsControllerTest < ActionController::TestCase 'server should correct bogus cancelled_at ' + job['cancelled_at']) assert_equal(true, - File.exists?(Rails.configuration.crunch_refresh_trigger), + File.exist?(Rails.configuration.crunch_refresh_trigger), 'trigger file should be created when job is cancelled') end diff --git a/services/api/test/functional/arvados/v1/links_controller_test.rb b/services/api/test/functional/arvados/v1/links_controller_test.rb index 1345701b43..6a19bdf4ad 100644 --- a/services/api/test/functional/arvados/v1/links_controller_test.rb +++ b/services/api/test/functional/arvados/v1/links_controller_test.rb @@ -305,8 +305,8 @@ class Arvados::V1::LinksControllerTest < ActionController::TestCase assert_response 404 end - test "retrieve all permissions using generic links index api" do - skip "(not implemented)" + # not implemented + skip "retrieve all permissions using generic links index api" do # Links.readable_by() does not return the full set of permission # links that are visible to a user (i.e., all permission links # whose head_uuid references an object for which the user has diff --git a/services/api/test/functional/arvados/v1/repositories_controller_test.rb b/services/api/test/functional/arvados/v1/repositories_controller_test.rb index 71b528e72a..56dd57ce7c 100644 --- a/services/api/test/functional/arvados/v1/repositories_controller_test.rb +++ b/services/api/test/functional/arvados/v1/repositories_controller_test.rb @@ -43,9 +43,8 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase end test "get_all_permissions takes into account is_active flag" do - r = nil act_as_user users(:active) do - r = Repository.create! name: 'active/testrepo' + Repository.create! name: 'active/testrepo' end act_as_system_user do u = users(:active) @@ -170,19 +169,19 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase u = User.find_by_uuid(user_uuid) if perms['can_read'] assert u.can? read: repo['uuid'] - assert_match /R/, perms['gitolite_permissions'] + assert_match(/R/, perms['gitolite_permissions']) else - refute_match /R/, perms['gitolite_permissions'] + refute_match(/R/, perms['gitolite_permissions']) end if perms['can_write'] assert u.can? write: repo['uuid'] - assert_match /RW\+/, perms['gitolite_permissions'] + assert_match(/RW\+/, perms['gitolite_permissions']) else - refute_match /W/, perms['gitolite_permissions'] + refute_match(/W/, perms['gitolite_permissions']) end if perms['can_manage'] assert u.can? manage: repo['uuid'] - assert_match /RW\+/, perms['gitolite_permissions'] + assert_match(/RW\+/, perms['gitolite_permissions']) end end end diff --git a/services/api/test/functional/arvados/v1/schema_controller_test.rb b/services/api/test/functional/arvados/v1/schema_controller_test.rb index 2e370ec9cd..7101821746 100644 --- a/services/api/test/functional/arvados/v1/schema_controller_test.rb +++ b/services/api/test/functional/arvados/v1/schema_controller_test.rb @@ -32,7 +32,7 @@ class Arvados::V1::SchemaControllerTest < ActionController::TestCase get :index assert_response :success discovery_doc = JSON.parse(@response.body) - assert_match /^[0-9a-f]+(-modified)?$/, discovery_doc['source_version'] + assert_match(/^[0-9a-f]+(-modified)?$/, discovery_doc['source_version']) end test "discovery document overrides source_version with config" do diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb index 157e487859..579b8cc6d0 100644 --- a/services/api/test/functional/arvados/v1/users_controller_test.rb +++ b/services/api/test/functional/arvados/v1/users_controller_test.rb @@ -603,7 +603,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase active_user = User.find_by_uuid(users(:active).uuid) readable_groups = active_user.groups_i_can(:read) - all_users_group = Group.all.collect(&:uuid).select { |g| g.match /-f+$/ } + all_users_group = Group.all.collect(&:uuid).select { |g| g.match(/-f+$/) } refute_includes(readable_groups, all_users_group, "active user can read All Users group after being deactivated") assert_equal(false, active_user.is_invited, @@ -842,14 +842,12 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase end def verify_num_links (original_links, expected_additional_links) - links_now = Link.all assert_equal expected_additional_links, Link.all.size-original_links.size, "Expected #{expected_additional_links.inspect} more links" end def find_obj_in_resp (response_items, object_type, head_kind=nil) return_obj = nil - response_items response_items.each { |x| if !x next diff --git a/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb b/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb index 329bc1589a..9b805af8e3 100644 --- a/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb +++ b/services/api/test/functional/arvados/v1/virtual_machines_controller_test.rb @@ -33,7 +33,6 @@ class Arvados::V1::VirtualMachinesControllerTest < ActionController::TestCase test "groups is an empty list by default" do get_logins_for(:testvm2) active_login = find_login(:active) - perm = links(:active_can_login_to_testvm2) assert_equal([], active_login["groups"]) end diff --git a/services/api/test/helpers/users_test_helper.rb b/services/api/test/helpers/users_test_helper.rb index 2a618204c6..4cf70cfbc6 100644 --- a/services/api/test/helpers/users_test_helper.rb +++ b/services/api/test/helpers/users_test_helper.rb @@ -74,7 +74,7 @@ module UsersTestHelper end group = Group.where(name: 'All users').select do |g| - g[:uuid].match /-f+$/ + g[:uuid].match(/-f+$/) end.first group_read_perms = Link.where(tail_uuid: uuid, head_uuid: group[:uuid], diff --git a/services/api/test/integration/collections_api_test.rb b/services/api/test/integration/collections_api_test.rb index 0bedc0726a..5f55f5eaec 100644 --- a/services/api/test/integration/collections_api_test.rb +++ b/services/api/test/integration/collections_api_test.rb @@ -21,7 +21,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest :filters => ['uuid', '=', 'ad02e37b6a7f45bbe2ead3c29a109b8a+54'].to_json }, auth(:active) assert_response 422 - assert_match /nvalid element.*not an array/, json_response['errors'].join(' ') + assert_match(/nvalid element.*not an array/, json_response['errors'].join(' ')) end test "get index with invalid filters (unsearchable column) responds 422" do @@ -30,7 +30,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest :filters => [['this_column_does_not_exist', '=', 'bogus']].to_json }, auth(:active) assert_response 422 - assert_match /nvalid attribute/, json_response['errors'].join(' ') + assert_match(/nvalid attribute/, json_response['errors'].join(' ')) end test "get index with invalid filters (invalid operator) responds 422" do @@ -39,7 +39,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest :filters => [['uuid', ':-(', 'displeased']].to_json }, auth(:active) assert_response 422 - assert_match /nvalid operator/, json_response['errors'].join(' ') + assert_match(/nvalid operator/, json_response['errors'].join(' ')) end test "get index with invalid filters (invalid operand type) responds 422" do @@ -48,7 +48,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest :filters => [['uuid', '=', {foo: 'bar'}]].to_json }, auth(:active) assert_response 422 - assert_match /nvalid operand type/, json_response['errors'].join(' ') + assert_match(/nvalid operand type/, json_response['errors'].join(' ')) end test "get index with where= (empty string)" do @@ -73,7 +73,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest :select => ['bogus'].to_json }, auth(:active) assert_response 422 - assert_match /Invalid attribute.*bogus/, json_response['errors'].join(' ') + assert_match(/Invalid attribute.*bogus/, json_response['errors'].join(' ')) end test "get index with select= (invalid attribute type) responds 422" do @@ -82,7 +82,7 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest :select => [['bogus']].to_json }, auth(:active) assert_response 422 - assert_match /Invalid attribute.*bogus/, json_response['errors'].join(' ') + assert_match(/Invalid attribute.*bogus/, json_response['errors'].join(' ')) end test "controller 404 response is json" do @@ -243,8 +243,6 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest assert_response :success assert_equal true, json_response['manifest_text'].include?('file4_in_subdir4.txt') - created = json_response - # search using the filename search_using_full_text_search 'subdir2', 0 search_using_full_text_search 'subdir2:*', 1 diff --git a/services/api/test/integration/collections_performance_test.rb b/services/api/test/integration/collections_performance_test.rb index a952c202cb..f6f39fe526 100644 --- a/services/api/test/integration/collections_performance_test.rb +++ b/services/api/test/integration/collections_performance_test.rb @@ -5,8 +5,7 @@ require 'helpers/time_block' class CollectionsApiPerformanceTest < ActionDispatch::IntegrationTest include ManifestExamples - test "crud cycle for a collection with a big manifest" do - slow_test + slow_test "crud cycle for a collection with a big manifest" do bigmanifest = time_block 'make example' do make_manifest(streams: 100, files_per_stream: 100, @@ -39,8 +38,7 @@ class CollectionsApiPerformanceTest < ActionDispatch::IntegrationTest end end - test "memory usage" do - slow_test + slow_test "memory usage" do hugemanifest = make_manifest(streams: 1, files_per_stream: 2000, blocks_per_file: 200, diff --git a/services/api/test/integration/cross_origin_test.rb b/services/api/test/integration/cross_origin_test.rb index ebe7ce7a67..28c1b81dab 100644 --- a/services/api/test/integration/cross_origin_test.rb +++ b/services/api/test/integration/cross_origin_test.rb @@ -70,7 +70,7 @@ class CrossOriginTest < ActionDispatch::IntegrationTest def assert_no_cors_headers response.headers.keys.each do |h| - assert_no_match /^Access-Control-/i, h + assert_no_match(/^Access-Control-/i, h) end end end diff --git a/services/api/test/integration/database_reset_test.rb b/services/api/test/integration/database_reset_test.rb index ecb2f2a058..029e37cbbf 100644 --- a/services/api/test/integration/database_reset_test.rb +++ b/services/api/test/integration/database_reset_test.rb @@ -3,8 +3,7 @@ require 'test_helper' class DatabaseResetTest < ActionDispatch::IntegrationTest self.use_transactional_fixtures = false - test "reset fails when Rails.env != 'test'" do - slow_test + slow_test "reset fails when Rails.env != 'test'" do rails_env_was = Rails.env begin Rails.env = 'production' @@ -22,8 +21,7 @@ class DatabaseResetTest < ActionDispatch::IntegrationTest assert_response 403 end - test "database reset doesn't break basic CRUD operations" do - slow_test + slow_test "database reset doesn't break basic CRUD operations" do active_auth = auth(:active) admin_auth = auth(:admin) @@ -49,8 +47,7 @@ class DatabaseResetTest < ActionDispatch::IntegrationTest assert_response 404 end - test "roll back database change" do - slow_test + slow_test "roll back database change" do active_auth = auth(:active) admin_auth = auth(:admin) diff --git a/services/api/test/integration/user_sessions_test.rb b/services/api/test/integration/user_sessions_test.rb index 7a9f9176d3..a46a4d1bc2 100644 --- a/services/api/test/integration/user_sessions_test.rb +++ b/services/api/test/integration/user_sessions_test.rb @@ -110,7 +110,7 @@ class UserSessionsApiTest < ActionDispatch::IntegrationTest (repos.collect(&:name) + vm_links.collect { |link| link.properties['username'] } ).each do |name| - r = name.match /^(.{#{prefix.length}})(\d+)$/ + r = name.match(/^(.{#{prefix.length}})(\d+)$/) assert_not_nil r, "#{name.inspect} does not match {prefix}\\d+" assert_equal(prefix, r[1], "#{name.inspect} was not {#{prefix.inspect} plus digits}") diff --git a/services/api/test/integration/websocket_test.rb b/services/api/test/integration/websocket_test.rb index 99ca7ac960..a9993b2fc3 100644 --- a/services/api/test/integration/websocket_test.rb +++ b/services/api/test/integration/websocket_test.rb @@ -1,5 +1,4 @@ require 'test_helper' -require 'websocket_runner' require 'oj' require 'database_cleaner' @@ -16,35 +15,92 @@ class WebsocketTest < ActionDispatch::IntegrationTest DatabaseCleaner.clean end - def ws_helper (token = nil, timeout = true) + def self.startup + s = TCPServer.new('0.0.0.0', 0) + @@port = s.addr[1] + s.close + @@pidfile = "tmp/pids/passenger.#{@@port}.pid" + DatabaseCleaner.start + Dir.chdir(Rails.root) do |apidir| + # Only passenger seems to be able to run the websockets server + # successfully. + _system('passenger', 'start', '-d', + "-p#{@@port}", + "--log-file", "/dev/stderr", + "--pid-file", @@pidfile) + timeout = Time.now.tv_sec + 10 + begin + sleep 0.2 + begin + server_pid = IO.read(@@pidfile).to_i + good_pid = (server_pid > 0) and (Process.kill(0, pid) rescue false) + rescue Errno::ENOENT + good_pid = false + end + end while (not good_pid) and (Time.now.tv_sec < timeout) + if not good_pid + raise RuntimeError, "could not find API server Rails pid" + end + STDERR.puts "Started websocket server on port #{@@port} with pid #{server_pid}" + end + end + + def self.shutdown + Dir.chdir(Rails.root) do + _system('passenger', 'stop', "-p#{@@port}", + "--pid-file", @@pidfile) + end + # DatabaseCleaner leaves the database empty. Prefer to leave it full. + dc = DatabaseController.new + dc.define_singleton_method :render do |*args| end + dc.reset + end + + def self._system(*cmd) + Bundler.with_clean_env do + env = { + 'ARVADOS_WEBSOCKETS' => 'ws-only', + 'RAILS_ENV' => 'test', + } + if not system(env, *cmd) + raise RuntimeError, "Command exited #{$?}: #{cmd.inspect}" + end + end + end + + def ws_helper(token: nil, timeout: 8) opened = false close_status = nil too_long = false - EM.run { + EM.run do if token - ws = Faye::WebSocket::Client.new("ws://localhost:#{WEBSOCKET_PORT}/websocket?api_token=#{api_client_authorizations(token).api_token}") + ws = Faye::WebSocket::Client.new("ws://localhost:#{@@port}/websocket?api_token=#{api_client_authorizations(token).api_token}") else - ws = Faye::WebSocket::Client.new("ws://localhost:#{WEBSOCKET_PORT}/websocket") + ws = Faye::WebSocket::Client.new("ws://localhost:#{@@port}/websocket") end ws.on :open do |event| opened = true if timeout - EM::Timer.new 8 do + EM::Timer.new(timeout) do too_long = true if close_status.nil? EM.stop_event_loop end end end + ws.on :error do |event| + STDERR.puts "websocket client error: #{event.inspect}" + end + ws.on :close do |event| close_status = [:close, event.code, event.reason] EM.stop_event_loop end yield ws - } + end assert opened, "Should have opened web socket" assert (not too_long), "Test took too long" @@ -65,11 +121,10 @@ class WebsocketTest < ActionDispatch::IntegrationTest assert_equal 401, status end - test "connect, subscribe and get response" do status = nil - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe'}.to_json) end @@ -91,7 +146,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe'}.to_json) end @@ -128,7 +183,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe'}.to_json) end @@ -168,7 +223,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json) end @@ -206,7 +261,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json) ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#specimen']]}.to_json) @@ -251,7 +306,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#trait'], ['event_type', '=', 'update']]}.to_json) end @@ -282,8 +337,6 @@ class WebsocketTest < ActionDispatch::IntegrationTest test "connect, subscribe, ask events starting at seq num" do state = 1 - human = nil - human_ev_uuid = nil authorize_with :active @@ -291,7 +344,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest l1 = nil l2 = nil - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe', last_log_id: lastid}.to_json) end @@ -322,16 +375,14 @@ class WebsocketTest < ActionDispatch::IntegrationTest assert_equal expect_next_logs[1].object_uuid, l2 end - test "connect, subscribe, get event, unsubscribe" do - slow_test + slow_test "connect, subscribe, get event, unsubscribe" do state = 1 spec = nil spec_ev_uuid = nil - filter_id = nil authorize_with :active - ws_helper :active, false do |ws| + ws_helper(token: :active, timeout: false) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe'}.to_json) EM::Timer.new 3 do @@ -372,15 +423,14 @@ class WebsocketTest < ActionDispatch::IntegrationTest assert_equal spec.uuid, spec_ev_uuid end - test "connect, subscribe, get event, unsubscribe with filter" do - slow_test + slow_test "connect, subscribe, get event, unsubscribe with filter" do state = 1 spec = nil spec_ev_uuid = nil authorize_with :active - ws_helper :active, false do |ws| + ws_helper(token: :active, timeout: false) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json) EM::Timer.new 6 do @@ -422,8 +472,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest end - test "connect, subscribe, get event, try to unsubscribe with bogus filter" do - slow_test + slow_test "connect, subscribe, get event, try to unsubscribe with bogus filter" do state = 1 spec = nil spec_ev_uuid = nil @@ -432,7 +481,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe'}.to_json) end @@ -473,13 +522,10 @@ class WebsocketTest < ActionDispatch::IntegrationTest assert_equal human.uuid, human_ev_uuid end - - - test "connected, not subscribed, no event" do - slow_test + slow_test "connected, not subscribed, no event" do authorize_with :active - ws_helper :active, false do |ws| + ws_helper(token: :active, timeout: false) do |ws| ws.on :open do |event| EM::Timer.new 1 do Specimen.create @@ -496,13 +542,12 @@ class WebsocketTest < ActionDispatch::IntegrationTest end end - test "connected, not authorized to see event" do - slow_test + slow_test "connected, not authorized to see event" do state = 1 authorize_with :admin - ws_helper :active, false do |ws| + ws_helper(token: :active, timeout: false) do |ws| ws.on :open do |event| ws.send ({method: 'subscribe'}.to_json) @@ -530,7 +575,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest test "connect, try bogus method" do status = nil - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({method: 'frobnabble'}.to_json) end @@ -548,7 +593,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest test "connect, missing method" do status = nil - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send ({fizzbuzz: 'frobnabble'}.to_json) end @@ -566,7 +611,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest test "connect, send malformed request" do status = nil - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| ws.send '' end @@ -587,7 +632,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| (1..17).each do |i| ws.send ({method: 'subscribe', filters: [['object_uuid', '=', i]]}.to_json) @@ -612,15 +657,14 @@ class WebsocketTest < ActionDispatch::IntegrationTest end - test "connect, subscribe, lots of events" do - slow_test + slow_test "connect, subscribe, lots of events" do state = 1 event_count = 0 log_start = Log.order(:id).last.id authorize_with :active - ws_helper :active, false do |ws| + ws_helper(token: :active, timeout: false) do |ws| EM::Timer.new 45 do # Needs a longer timeout than the default ws.close @@ -637,7 +681,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest assert_equal 200, d["status"] ActiveRecord::Base.transaction do (1..202).each do - spec = Specimen.create + Specimen.create end end state = 2 @@ -658,12 +702,10 @@ class WebsocketTest < ActionDispatch::IntegrationTest test "connect, subscribe with invalid filter" do state = 1 - human = nil - human_ev_uuid = nil authorize_with :active - ws_helper :active do |ws| + ws_helper(token: :active) do |ws| ws.on :open do |event| # test that #6451 is fixed (invalid filter crashes websockets) ws.send ({method: 'subscribe', filters: [['object_blarg', 'is_a', 'arvados#human']]}.to_json) @@ -675,7 +717,7 @@ class WebsocketTest < ActionDispatch::IntegrationTest when 1 assert_equal 200, d["status"] Specimen.create - human = Human.create + Human.create state = 2 when 2 assert_equal 500, d["status"] diff --git a/services/api/test/test_helper.rb b/services/api/test/test_helper.rb index 417ddf6bee..86bc2397c5 100644 --- a/services/api/test/test_helper.rb +++ b/services/api/test/test_helper.rb @@ -22,7 +22,7 @@ end require File.expand_path('../../config/environment', __FILE__) require 'rails/test_help' -require 'mocha/mini_test' +require 'mocha' module ArvadosTestSupport def json_response @@ -84,7 +84,7 @@ class ActiveSupport::TestCase def restore_configuration # Restore configuration settings changed during tests $application_config.each do |k,v| - if k.match /^[^.]*$/ + if k.match(/^[^.]*$/) Rails.configuration.send (k + '='), v end end @@ -112,9 +112,18 @@ class ActiveSupport::TestCase "HTTP_AUTHORIZATION" => "OAuth2 #{t}") end - def slow_test - skip "RAILS_TEST_SHORT is set" unless (ENV['RAILS_TEST_SHORT'] || '').empty? + def self.skip_slow_tests? + !(ENV['RAILS_TEST_SHORT'] || '').empty? end + + def self.skip(*args, &block) + end + + def self.slow_test(name, &block) + define_method(name, block) unless skip_slow_tests? + end + + alias_method :skip, :omit end class ActionController::TestCase @@ -135,6 +144,21 @@ class ActionController::TestCase super action, *args end end + + def self.suite + s = super + def s.run(*args) + @test_case.startup() + begin + super + ensure + @test_case.shutdown() + end + end + s + end + def self.startup; end + def self.shutdown; end end class ActionDispatch::IntegrationTest diff --git a/services/api/test/unit/app_version_test.rb b/services/api/test/unit/app_version_test.rb index 3e9b16757d..2e585051ad 100644 --- a/services/api/test/unit/app_version_test.rb +++ b/services/api/test/unit/app_version_test.rb @@ -20,16 +20,19 @@ class AppVersionTest < ActiveSupport::TestCase end end - test 'override with configuration' do + test 'override with configuration "foobar"' do Rails.configuration.source_version = 'foobar' assert_equal 'foobar', AppVersion.hash + end + + test 'override with configuration false' do Rails.configuration.source_version = false assert_not_equal 'foobar', AppVersion.hash end test 'override with file' do path = Rails.root.join 'git-commit.version' - assert(!File.exists?(path), + assert(!File.exist?(path), "Packaged version file found in source tree: #{path}") begin File.open(path, 'w') do |f| diff --git a/services/api/test/unit/authorized_key_test.rb b/services/api/test/unit/authorized_key_test.rb index 5a661785bd..25801bb9b6 100644 --- a/services/api/test/unit/authorized_key_test.rb +++ b/services/api/test/unit/authorized_key_test.rb @@ -25,7 +25,7 @@ class AuthorizedKeyTest < ActiveSupport::TestCase ak2 = AuthorizedKey.new(name: "bar", public_key: TEST_KEY, authorized_user_uuid: u2.uuid) refute ak2.valid? refute ak2.save - assert_match /already exists/, ak2.errors.full_messages.to_s + assert_match(/already exists/, ak2.errors.full_messages.to_s) end end diff --git a/services/api/test/unit/collection_performance_test.rb b/services/api/test/unit/collection_performance_test.rb index 1c6e4f2db2..57beddbe6d 100644 --- a/services/api/test/unit/collection_performance_test.rb +++ b/services/api/test/unit/collection_performance_test.rb @@ -17,8 +17,7 @@ class CollectionModelPerformanceTest < ActiveSupport::TestCase end # "crrud" == "create read render update delete", not a typo - test "crrud cycle for a collection with a big manifest)" do - slow_test + slow_test "crrud cycle for a collection with a big manifest)" do bigmanifest = time_block 'make example' do make_manifest(streams: 100, files_per_stream: 100, @@ -44,7 +43,7 @@ class CollectionModelPerformanceTest < ActiveSupport::TestCase c.signed_manifest_text end time_block 'sign + render' do - resp = c.as_api_response(nil) + c.as_api_response(nil) end loc = Blob.sign_locator(Digest::MD5.hexdigest('foo') + '+3', api_token: api_token(:active)) diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb index 91568927ae..1c85a716e3 100644 --- a/services/api/test/unit/collection_test.rb +++ b/services/api/test/unit/collection_test.rb @@ -28,7 +28,7 @@ class CollectionTest < ActiveSupport::TestCase c = create_collection "f\xc8o", Encoding::UTF_8 assert !c.valid? assert_equal [:manifest_text], c.errors.messages.keys - assert_match /UTF-8/, c.errors.messages[:manifest_text].first + assert_match(/UTF-8/, c.errors.messages[:manifest_text].first) end end @@ -37,7 +37,7 @@ class CollectionTest < ActiveSupport::TestCase c = create_collection "f\xc8o", Encoding::ASCII_8BIT assert !c.valid? assert_equal [:manifest_text], c.errors.messages.keys - assert_match /UTF-8/, c.errors.messages[:manifest_text].first + assert_match(/UTF-8/, c.errors.messages[:manifest_text].first) end end @@ -107,11 +107,11 @@ class CollectionTest < ActiveSupport::TestCase assert c.valid? created_file_names = c.file_names assert created_file_names - assert_match /foo.txt/, c.file_names + assert_match(/foo.txt/, c.file_names) c.update_attribute 'manifest_text', ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo2.txt\n" assert_not_equal created_file_names, c.file_names - assert_match /foo2.txt/, c.file_names + assert_match(/foo2.txt/, c.file_names) end end @@ -134,11 +134,11 @@ class CollectionTest < ActiveSupport::TestCase assert c.valid? assert c.file_names - assert_match /veryverylongfilename0000000000001.txt/, c.file_names - assert_match /veryverylongfilename0000000000002.txt/, c.file_names + assert_match(/veryverylongfilename0000000000001.txt/, c.file_names) + assert_match(/veryverylongfilename0000000000002.txt/, c.file_names) if not allow_truncate - assert_match /veryverylastfilename/, c.file_names - assert_match /laststreamname/, c.file_names + assert_match(/veryverylastfilename/, c.file_names) + assert_match(/laststreamname/, c.file_names) end end end diff --git a/services/api/test/unit/commit_test.rb b/services/api/test/unit/commit_test.rb index b57c23b453..a8594169fb 100644 --- a/services/api/test/unit/commit_test.rb +++ b/services/api/test/unit/commit_test.rb @@ -18,7 +18,7 @@ class CommitTest < ActiveSupport::TestCase test 'find_commit_range does not bypass permissions' do authorize_with :inactive assert_raises ArgumentError do - c = Commit.find_commit_range 'foo', nil, 'master', [] + Commit.find_commit_range 'foo', nil, 'master', [] end end @@ -68,10 +68,10 @@ class CommitTest < ActiveSupport::TestCase authorize_with :active gitint = "git --git-dir #{Rails.configuration.git_internal_dir}" IO.read("|#{gitint} tag -d testtag 2>/dev/null") # "no such tag", fine - assert_match /^fatal: /, IO.read("|#{gitint} show testtag 2>&1") + assert_match(/^fatal: /, IO.read("|#{gitint} show testtag 2>&1")) refute $?.success? Commit.tag_in_internal_repository 'active/foo', '31ce37fe365b3dc204300a3e4c396ad333ed0556', 'testtag' - assert_match /^commit 31ce37f/, IO.read("|#{gitint} show testtag") + assert_match(/^commit 31ce37f/, IO.read("|#{gitint} show testtag")) assert $?.success? end @@ -183,34 +183,34 @@ class CommitTest < ActiveSupport::TestCase Dir.mktmpdir do |touchdir| # invalid input to maximum a = Commit.find_commit_range('active/foo', nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", nil) - assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable" + assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable" assert_equal [], a # invalid input to maximum a = Commit.find_commit_range('active/foo', nil, "$(uname>#{touchdir}/uh_oh)", nil) - assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable" + assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable" assert_equal [], a # invalid input to minimum a = Commit.find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil) - assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable" + assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable" assert_equal [], a # invalid input to minimum a = Commit.find_commit_range('active/foo', "$(uname>#{touchdir}/uh_oh)", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil) - assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable" + assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable" assert_equal [], a # invalid input to 'excludes' # complains "fatal: bad object 077ba2ad3ea24a929091a9e6ce545c93199b8e57" a = Commit.find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["4fe459abe02d9b365932b8f5dc419439ab4e2577 ; touch #{touchdir}/uh_oh"]) - assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable" + assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable" assert_equal [], a # invalid input to 'excludes' # complains "fatal: bad object 077ba2ad3ea24a929091a9e6ce545c93199b8e57" a = Commit.find_commit_range('active/foo', "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["$(uname>#{touchdir}/uh_oh)"]) - assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable" + assert !File.exist?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable" assert_equal [], a end end diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb index 4fd9f8e759..a175533285 100644 --- a/services/api/test/unit/container_test.rb +++ b/services/api/test/unit/container_test.rb @@ -102,8 +102,9 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select higher priority queued container" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}}) - c_low_priority, _ = minimal_new(common_attrs.merge({priority:1})) - c_high_priority, _ = minimal_new(common_attrs.merge({priority:2})) + c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1})) + c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2})) + assert_not_equal c_low_priority.uuid, c_high_priority.uuid assert_equal Container::Queued, c_low_priority.state assert_equal Container::Queued, c_high_priority.state reused = Container.find_reusable(common_attrs) @@ -121,8 +122,9 @@ class ContainerTest < ActiveSupport::TestCase output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45' } - c_older, _ = minimal_new(common_attrs) - c_recent, _ = minimal_new(common_attrs) + c_older, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_recent, _ = minimal_new(common_attrs.merge({use_existing: false})) + assert_not_equal c_older.uuid, c_recent.uuid set_user_from_auth :dispatch1 c_older.update_attributes!({state: Container::Locked}) @@ -151,6 +153,7 @@ class ContainerTest < ActiveSupport::TestCase c_output1 = Container.create common_attrs c_output2 = Container.create common_attrs + assert_not_equal c_output1.uuid, c_output2.uuid cr = ContainerRequest.new common_attrs cr.state = ContainerRequest::Committed @@ -177,9 +180,11 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select running container by start date" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}}) - c_slower, _ = minimal_new(common_attrs) - c_faster_started_first, _ = minimal_new(common_attrs) - c_faster_started_second, _ = minimal_new(common_attrs) + c_slower, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false})) + # Confirm the 3 container UUIDs are different. + assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length set_user_from_auth :dispatch1 c_slower.update_attributes!({state: Container::Locked}) c_slower.update_attributes!({state: Container::Running, @@ -199,9 +204,11 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select running container by progress" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}}) - c_slower, _ = minimal_new(common_attrs) - c_faster_started_first, _ = minimal_new(common_attrs) - c_faster_started_second, _ = minimal_new(common_attrs) + c_slower, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false})) + # Confirm the 3 container UUIDs are different. + assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length set_user_from_auth :dispatch1 c_slower.update_attributes!({state: Container::Locked}) c_slower.update_attributes!({state: Container::Running, @@ -221,9 +228,11 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select locked container most likely to start sooner" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}}) - c_low_priority, _ = minimal_new(common_attrs) - c_high_priority_older, _ = minimal_new(common_attrs) - c_high_priority_newer, _ = minimal_new(common_attrs) + c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false})) + # Confirm the 3 container UUIDs are different. + assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length set_user_from_auth :dispatch1 c_low_priority.update_attributes!({state: Container::Locked, priority: 1}) @@ -239,8 +248,9 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select running over failed container" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}}) - c_failed, _ = minimal_new(common_attrs) - c_running, _ = minimal_new(common_attrs) + c_failed, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_running, _ = minimal_new(common_attrs.merge({use_existing: false})) + assert_not_equal c_failed.uuid, c_running.uuid set_user_from_auth :dispatch1 c_failed.update_attributes!({state: Container::Locked}) c_failed.update_attributes!({state: Container::Running}) @@ -259,8 +269,9 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select complete over running container" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}}) - c_completed, _ = minimal_new(common_attrs) - c_running, _ = minimal_new(common_attrs) + c_completed, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_running, _ = minimal_new(common_attrs.merge({use_existing: false})) + assert_not_equal c_completed.uuid, c_running.uuid set_user_from_auth :dispatch1 c_completed.update_attributes!({state: Container::Locked}) c_completed.update_attributes!({state: Container::Running}) @@ -279,8 +290,9 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select running over locked container" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}}) - c_locked, _ = minimal_new(common_attrs) - c_running, _ = minimal_new(common_attrs) + c_locked, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_running, _ = minimal_new(common_attrs.merge({use_existing: false})) + assert_not_equal c_running.uuid, c_locked.uuid set_user_from_auth :dispatch1 c_locked.update_attributes!({state: Container::Locked}) c_running.update_attributes!({state: Container::Locked}) @@ -294,8 +306,9 @@ class ContainerTest < ActiveSupport::TestCase test "find_reusable method should select locked over queued container" do set_user_from_auth :active common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}}) - c_locked, _ = minimal_new(common_attrs) - c_queued, _ = minimal_new(common_attrs) + c_locked, _ = minimal_new(common_attrs.merge({use_existing: false})) + c_queued, _ = minimal_new(common_attrs.merge({use_existing: false})) + assert_not_equal c_queued.uuid, c_locked.uuid set_user_from_auth :dispatch1 c_locked.update_attributes!({state: Container::Locked}) reused = Container.find_reusable(common_attrs) diff --git a/services/api/test/unit/fail_jobs_test.rb b/services/api/test/unit/fail_jobs_test.rb index c390b3213e..1f5847aea6 100644 --- a/services/api/test/unit/fail_jobs_test.rb +++ b/services/api/test/unit/fail_jobs_test.rb @@ -60,13 +60,13 @@ class FailJobsTest < ActiveSupport::TestCase test 'command line help' do cmd = Rails.root.join('script/fail-jobs.rb').to_s - assert_match /Options:.*--before=/m, File.popen([cmd, '--help']).read + assert_match(/Options:.*--before=/m, File.popen([cmd, '--help']).read) end protected def assert_end_states - @job.values.map &:reload + @job.values.map(&:reload) assert_equal 'Failed', @job[:before_reboot].state assert_equal false, @job[:before_reboot].running assert_equal false, @job[:before_reboot].success diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb index 3da2c836ed..1f80ea50f2 100644 --- a/services/api/test/unit/job_test.rb +++ b/services/api/test/unit/job_test.rb @@ -185,7 +185,7 @@ class JobTest < ActiveSupport::TestCase # Ensure valid_attrs doesn't produce errors -- otherwise we will # not know whether errors reported below are actually caused by # invalid_attrs. - dummy = Job.create! job_attrs + Job.create! job_attrs job = Job.create job_attrs(invalid_attrs) assert_raises(ActiveRecord::RecordInvalid, ArgumentError, @@ -223,7 +223,7 @@ class JobTest < ActiveSupport::TestCase parameters.each do |parameter| expectations = parameter[2] - if parameter[1] == 'use_current_user_uuid' + if 'use_current_user_uuid' == parameter[1] parameter[1] = Thread.current[:user].uuid end @@ -411,7 +411,7 @@ class JobTest < ActiveSupport::TestCase } assert_raises(ActiveRecord::RecordInvalid, "created job with a collection uuid in script_parameters") do - job = Job.create!(job_attrs(bad_params)) + Job.create!(job_attrs(bad_params)) end end diff --git a/services/api/test/unit/log_test.rb b/services/api/test/unit/log_test.rb index efbb189c9f..92976e0053 100644 --- a/services/api/test/unit/log_test.rb +++ b/services/api/test/unit/log_test.rb @@ -263,7 +263,7 @@ class LogTest < ActiveSupport::TestCase # appear too, but only if they are _not_ listed in known_logs # (i.e., we do not make any assertions about logs not mentioned in # either "known" or "expected".) - result_ids = result.collect &:id + result_ids = result.collect(&:id) expected_logs.each do |want| assert_includes result_ids, logs(want).id end diff --git a/services/api/test/unit/node_test.rb b/services/api/test/unit/node_test.rb index 6eb1df56d1..df8c22baf4 100644 --- a/services/api/test/unit/node_test.rb +++ b/services/api/test/unit/node_test.rb @@ -33,7 +33,7 @@ class NodeTest < ActiveSupport::TestCase conffile = Rails.root.join 'tmp', 'compute65535.conf' File.unlink conffile rescue nil assert Node.dns_server_update 'compute65535', '127.0.0.1' - assert_match /\"1\.0\.0\.127\.in-addr\.arpa\. IN PTR compute65535\.zzzzz\.arvadosapi\.com\"/, IO.read(conffile) + assert_match(/\"1\.0\.0\.127\.in-addr\.arpa\. IN PTR compute65535\.zzzzz\.arvadosapi\.com\"/, IO.read(conffile)) File.unlink conffile end diff --git a/services/api/test/unit/owner_test.rb b/services/api/test/unit/owner_test.rb index c7f9776ac6..6fcc316528 100644 --- a/services/api/test/unit/owner_test.rb +++ b/services/api/test/unit/owner_test.rb @@ -27,7 +27,7 @@ class OwnerTest < ActiveSupport::TestCase test "create object with non-existent #{o_class} owner" do assert_raises(ActiveRecord::RecordInvalid, "create should fail with random owner_uuid") do - i = Specimen.create!(owner_uuid: o_class.generate_uuid) + Specimen.create!(owner_uuid: o_class.generate_uuid) end i = Specimen.create(owner_uuid: o_class.generate_uuid) @@ -89,7 +89,6 @@ class OwnerTest < ActiveSupport::TestCase o = eval ofixt assert_equal(true, Specimen.where(owner_uuid: o.uuid).any?, "need something to be owned by #{o.uuid} for this test") - old_uuid = o.uuid new_uuid = o.uuid.sub(/..........$/, rand(2**256).to_s(36)[0..9]) assert(!o.update_attributes(uuid: new_uuid), "should not change uuid of #{ofixt} that owns objects") diff --git a/services/api/test/unit/permission_test.rb b/services/api/test/unit/permission_test.rb index 79fc1f29c7..df11054998 100644 --- a/services/api/test/unit/permission_test.rb +++ b/services/api/test/unit/permission_test.rb @@ -125,10 +125,10 @@ class PermissionTest < ActiveSupport::TestCase sp_grp = Group.create! sp = Specimen.create!(owner_uuid: sp_grp.uuid) - manage_perm = Link.create!(link_class: 'permission', - name: 'can_manage', - tail_uuid: owner_grp.uuid, - head_uuid: sp_grp.uuid) + Link.create!(link_class: 'permission', + name: 'can_manage', + tail_uuid: owner_grp.uuid, + head_uuid: sp_grp.uuid) # active user owns owner_grp, which has can_manage permission on sp_grp # user should be able to add permissions on sp. @@ -137,14 +137,12 @@ class PermissionTest < ActiveSupport::TestCase head_uuid: sp.uuid, link_class: 'permission', name: 'can_write') - test_uuid = test_perm.uuid assert test_perm.save, "could not save new permission on target object" assert test_perm.destroy, "could not delete new permission on target object" end - # TODO(twp): fix bug #3091, which should fix this test. - test "can_manage permission on a non-group object" do - skip + # bug #3091 + skip "can_manage permission on a non-group object" do set_user_from_auth :admin ob = Specimen.create! diff --git a/services/api/test/unit/pipeline_instance_test.rb b/services/api/test/unit/pipeline_instance_test.rb index 93354f8b1e..fc40d06b2c 100644 --- a/services/api/test/unit/pipeline_instance_test.rb +++ b/services/api/test/unit/pipeline_instance_test.rb @@ -91,7 +91,6 @@ class PipelineInstanceTest < ActiveSupport::TestCase component2 = {'script_parameters' => {"something_else" => "xxxad4b39ca5a924e481008009d94e32+210", "input_missing" => {"required" => true}}} pi.components['first'] = component1 pi.components['second'] = component2 - components = pi.components Thread.current[:user] = users(:admin) pi.update_attribute 'components', pi.components diff --git a/services/api/test/unit/salvage_collection_test.rb b/services/api/test/unit/salvage_collection_test.rb index a269078b73..1381c8f538 100644 --- a/services/api/test/unit/salvage_collection_test.rb +++ b/services/api/test/unit/salvage_collection_test.rb @@ -60,7 +60,7 @@ class SalvageCollectionTest < ActiveSupport::TestCase updated_name = updated_src_collection.name assert_equal true, updated_name.include?(src_collection.name) - match = updated_name.match /^test collection.*salvaged data at (.*)\)$/ + match = updated_name.match(/^test collection.*salvaged data at (.*)\)$/) assert_not_nil match assert_not_nil match[1] assert_empty updated_src_collection.manifest_text @@ -68,7 +68,7 @@ class SalvageCollectionTest < ActiveSupport::TestCase # match[1] is the uuid of the new collection created from src_collection's salvaged data # use this to get the new collection and verify new_collection = Collection.find_by_uuid match[1] - match = new_collection.name.match /^salvaged from (.*),.*/ + match = new_collection.name.match(/^salvaged from (.*),.*/) assert_not_nil match assert_equal src_collection.uuid, match[1] @@ -80,7 +80,7 @@ class SalvageCollectionTest < ActiveSupport::TestCase end test "salvage collection with no uuid required argument" do - e = assert_raises RuntimeError do + assert_raises RuntimeError do salvage_collection nil end end @@ -107,7 +107,7 @@ class SalvageCollectionTest < ActiveSupport::TestCase e = assert_raises RuntimeError do salvage_collection collections('user_agreement').uuid end - assert_match /Error during arv-put: pid \d+ exit \d+ \(cmd was \"arv-put .*\"\)/, e.message + assert_match(/Error during arv-put: pid \d+ exit \d+ \(cmd was \"arv-put .*\"\)/, e.message) end # This test uses BAD_MANIFEST, which has the following flaws: @@ -146,7 +146,7 @@ class SalvageCollectionTest < ActiveSupport::TestCase updated_name = updated_src_collection.name assert_equal true, updated_name.include?(src_collection.name) - match = updated_name.match /^test collection.*salvaged data at (.*)\)$/ + match = updated_name.match(/^test collection.*salvaged data at (.*)\)$/) assert_not_nil match assert_not_nil match[1] assert_empty updated_src_collection.manifest_text @@ -154,7 +154,7 @@ class SalvageCollectionTest < ActiveSupport::TestCase # match[1] is the uuid of the new collection created from src_collection's salvaged data # use this to get the new collection and verify new_collection = Collection.find_by_uuid match[1] - match = new_collection.name.match /^salvaged from (.*),.*/ + match = new_collection.name.match(/^salvaged from (.*),.*/) assert_not_nil match assert_equal src_collection.uuid, match[1] # verify the new collection's manifest includes the bad locators diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb index 4df6cc0b36..0d66a4b384 100644 --- a/services/api/test/unit/user_test.rb +++ b/services/api/test/unit/user_test.rb @@ -136,7 +136,6 @@ class UserTest < ActiveSupport::TestCase test "admin can't clear username when user owns repositories" do set_user_from_auth :admin user = users(:active) - start_username = user.username user.username = nil assert_not_allowed { user.save } refute_empty(user.errors[:username]) diff --git a/services/api/test/websocket_runner.rb b/services/api/test/websocket_runner.rb deleted file mode 100644 index be32a0f299..0000000000 --- a/services/api/test/websocket_runner.rb +++ /dev/null @@ -1,53 +0,0 @@ -require 'bundler' -require 'socket' - -$ARV_API_SERVER_DIR = File.expand_path('../..', __FILE__) - -s = TCPServer.new('0.0.0.0', 0) -WEBSOCKET_PORT = s.addr[1] -s.close -SERVER_PID_PATH = "tmp/pids/passenger.#{WEBSOCKET_PORT}.pid" - -class WebsocketTestRunner < MiniTest::Unit - def _system(*cmd) - Bundler.with_clean_env do - if not system({'ARVADOS_WEBSOCKETS' => 'ws-only', 'RAILS_ENV' => 'test'}, *cmd) - raise RuntimeError, "Command failed with exit status #{$?}: #{cmd.inspect}" - end - end - end - - def _run(args=[]) - server_pid = Dir.chdir($ARV_API_SERVER_DIR) do |apidir| - # Only passenger seems to be able to run the websockets server successfully. - _system('passenger', 'start', '-d', "-p#{WEBSOCKET_PORT}") - timeout = Time.now.tv_sec + 10 - begin - sleep 0.2 - begin - server_pid = IO.read(SERVER_PID_PATH).to_i - good_pid = (server_pid > 0) and (Process.kill(0, pid) rescue false) - rescue Errno::ENOENT - good_pid = false - end - end while (not good_pid) and (Time.now.tv_sec < timeout) - if not good_pid - raise RuntimeError, "could not find API server Rails pid" - end - server_pid - end - begin - super(args) - ensure - Dir.chdir($ARV_API_SERVER_DIR) do - _system('passenger', 'stop', "-p#{WEBSOCKET_PORT}") - end - # DatabaseCleaner leaves the database empty. Prefer to leave it full. - dc = DatabaseController.new - dc.define_singleton_method :render do |*args| end - dc.reset - end - end -end - -MiniTest::Unit.runner = WebsocketTestRunner.new diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service index a36205c967..5f72f1c613 100755 --- a/tools/arvbox/lib/arvbox/docker/service/api/run-service +++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service @@ -9,7 +9,8 @@ cd /usr/src/arvados/services/api export RAILS_ENV=development run_bundler --without=development -bundle exec passenger start --runtime-check-only --runtime-dir=/var/lib/passenger +bundle exec passenger-config build-native-support +bundle exec passenger-config install-standalone-runtime if test "$1" = "--only-deps" ; then exit @@ -23,6 +24,5 @@ if test "$1" = "--only-setup" ; then fi exec bundle exec passenger start --port=${services[api]} \ - --runtime-dir=/var/lib/passenger \ --ssl --ssl-certificate=/var/lib/arvados/self-signed.pem \ --ssl-certificate-key=/var/lib/arvados/self-signed.key