From: Peter Amstutz Date: Mon, 25 Mar 2019 20:39:23 +0000 (-0400) Subject: 13996: Fixing test WIP X-Git-Tag: 1.4.0~65^2~23 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/f9ae5b90a5e04477133ca7a7d34bd3eebf862474 13996: Fixing test WIP Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb index 1ec921b8ad..5c97735f24 100644 --- a/services/api/app/controllers/application_controller.rb +++ b/services/api/app/controllers/application_controller.rb @@ -81,10 +81,12 @@ class ApplicationController < ActionController::Base def default_url_options options = {} - exturl = URI.parse(Rails.configuration.Services["Controller"]["ExternalURL"]) - options[:host] = exturl.host - options[:port] = exturl.port - options[:protocol] = exturl.scheme + if Rails.configuration.Services["Controller"]["ExternalURL"] != "" + exturl = URI.parse(Rails.configuration.Services["Controller"]["ExternalURL"]) + options[:host] = exturl.host + options[:port] = exturl.port + options[:protocol] = exturl.scheme + end options end diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb index ced2f16207..5f042877de 100644 --- a/services/api/app/controllers/arvados/v1/schema_controller.rb +++ b/services/api/app/controllers/arvados/v1/schema_controller.rb @@ -46,27 +46,27 @@ class Arvados::V1::SchemaController < ApplicationController rootUrl: root_url, servicePath: "arvados/v1/", batchPath: "batch", - uuidPrefix: Rails.application.config.ClusterID, - defaultTrashLifetime: Rails.application.config.Collections["DefaultTrashLifetime"], - blobSignatureTtl: Rails.application.config.Collections["BlobSigningTTL"], - maxRequestSize: Rails.application.config.API["MaxRequestSize"], - maxItemsPerResponse: Rails.application.API["MaxItemsPerResponse"], - dockerImageFormats: Rails.application.config.Containers["SupportedDockerImageFormats"], - crunchLogBytesPerEvent: Rails.application.config.Containers["Logging"]["LogBytesPerEvent"], - crunchLogSecondsBetweenEvents: Rails.application.config.Containers["Logging"]["LogSecondsBetweenEvents"], - crunchLogThrottlePeriod: Rails.application.config.Containers["Logging"]["LogThrottlePeriod"], - crunchLogThrottleBytes: Rails.application.config.Containers["Logging"]["LogThrottleBytes"], - crunchLogThrottleLines: Rails.application.config.Containers["Logging"]["LogThrottleLines"], - crunchLimitLogBytesPerJob: Rails.application.config.Containers["Logging"]["LimitLogBytesPerJob"], - crunchLogPartialLineThrottlePeriod: Rails.application.config.Containers["Logging"]["LogPartialLineThrottlePeriod"], - crunchLogUpdatePeriod: Rails.application.config.Containers["Logging"]["LogUpdatePeriod"], - crunchLogUpdateSize: Rails.application.config.Containers["Logging"]["LogUpdateSize"], - remoteHosts: Rails.application.config.RemoteClusters.map {|k,v| v.Host}, - remoteHostsViaDNS: Rails.application.config.RemoteClusters["*"]["Proxy"], - websocketUrl: Rails.application.config.Services["Websocket"]["ExternalURL"], - workbenchUrl: Rails.application.config.Services["Workbench1"]["ExternalURL"], - keepWebServiceUrl: Rails.application.config.Services["WebDAV"]["ExternalURL"], - gitUrl: Rails.application.config.Services["GitHTTP"]["ExternalURL"], + uuidPrefix: Rails.configuration.ClusterID, + defaultTrashLifetime: Rails.configuration.Collections["DefaultTrashLifetime"], + blobSignatureTtl: Rails.configuration.Collections["BlobSigningTTL"], + maxRequestSize: Rails.configuration.API["MaxRequestSize"], + maxItemsPerResponse: Rails.configuration.API["MaxItemsPerResponse"], + dockerImageFormats: Rails.configuration.Containers["SupportedDockerImageFormats"], + crunchLogBytesPerEvent: Rails.configuration.Containers["Logging"]["LogBytesPerEvent"], + crunchLogSecondsBetweenEvents: Rails.configuration.Containers["Logging"]["LogSecondsBetweenEvents"], + crunchLogThrottlePeriod: Rails.configuration.Containers["Logging"]["LogThrottlePeriod"], + crunchLogThrottleBytes: Rails.configuration.Containers["Logging"]["LogThrottleBytes"], + crunchLogThrottleLines: Rails.configuration.Containers["Logging"]["LogThrottleLines"], + crunchLimitLogBytesPerJob: Rails.configuration.Containers["Logging"]["LimitLogBytesPerJob"], + crunchLogPartialLineThrottlePeriod: Rails.configuration.Containers["Logging"]["LogPartialLineThrottlePeriod"], + crunchLogUpdatePeriod: Rails.configuration.Containers["Logging"]["LogUpdatePeriod"], + crunchLogUpdateSize: Rails.configuration.Containers["Logging"]["LogUpdateSize"], + remoteHosts: Rails.configuration.RemoteClusters.map {|k,v| v["Host"]}, + remoteHostsViaDNS: Rails.configuration.RemoteClusters["*"]["Proxy"], + websocketUrl: Rails.configuration.Services["Websocket"]["ExternalURL"], + workbenchUrl: Rails.configuration.Services["Workbench1"]["ExternalURL"], + keepWebServiceUrl: Rails.configuration.Services["WebDAV"]["ExternalURL"], + gitUrl: Rails.configuration.Services["GitHTTP"]["ExternalURL"], parameters: { alt: { type: "string", diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb index 46da37afd4..7f7a320074 100644 --- a/services/api/app/models/collection.rb +++ b/services/api/app/models/collection.rb @@ -518,7 +518,9 @@ class Collection < ArvadosModel if loc = Keep::Locator.parse(search_term) loc.strip_hints! coll_match = readable_by(*readers).where(portable_data_hash: loc.to_s).limit(1) - if coll_match.any? or Rails.configuration.RemoteClusters.length > 1 + rc = Rails.configuration.RemoteClusters.select{ |k| + k != "*" && k != Rails.configuration.ClusterID} + if coll_match.any? or rc.length == 0 return get_compatible_images(readers, pattern, coll_match) else # Allow bare pdh that doesn't exist in the local database so diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb index deaa8ff71d..b24df8cbd3 100644 --- a/services/api/app/models/job.rb +++ b/services/api/app/models/job.rb @@ -491,7 +491,7 @@ class Job < ArvadosModel end def find_docker_image_locator - if runtime_constraints.is_a? Hash + if runtime_constraints.is_a? Hash and Rails.configuration.Containers["JobsAPI"]["DefaultDockerImage"] != "" runtime_constraints['docker_image'] ||= Rails.configuration.Containers["JobsAPI"]["DefaultDockerImage"] end diff --git a/services/api/app/models/repository.rb b/services/api/app/models/repository.rb index ac89ecf6b0..7cb36dcb92 100644 --- a/services/api/app/models/repository.rb +++ b/services/api/app/models/repository.rb @@ -49,7 +49,7 @@ class Repository < ArvadosModel # prefers bare repositories over checkouts. [["%s.git"], ["%s", ".git"]].each do |repo_base, *join_args| [:uuid, :name].each do |path_attr| - git_dir = File.join(Rails.configuration.Containers["Git"]["Repositories"], + git_dir = File.join(Rails.configuration.Git["Repositories"], repo_base % send(path_attr), *join_args) return git_dir if File.exist?(git_dir) end @@ -98,21 +98,27 @@ class Repository < ArvadosModel end def ssh_clone_url - _clone_url Rails.configuration.Services["GitSSH"]["ExternalURL"], 'git@git.%s.arvadosapi.com:' + _clone_url Rails.configuration.Services["GitSSH"].andand["ExternalURL"], 'ssh://git@git.%s.arvadosapi.com' end def https_clone_url - _clone_url Rails.configuration.Services["GitHTTP"]["ExternalURL"], 'https://git.%s.arvadosapi.com/' + _clone_url Rails.configuration.Services["GitHTTP"].andand["ExternalURL"], 'https://git.%s.arvadosapi.com/' end def _clone_url config_var, default_base_fmt - configured_base = config_var + if not config_var + return "" + end prefix = new_record? ? Rails.configuration.ClusterID : uuid[0,5] - if prefix == Rails.configuration.ClusterID and configured_base != true - base = configured_base + if prefix == Rails.configuration.ClusterID and config_var != URI("") + base = config_var + else + base = URI(default_base_fmt % prefix) + end + if base.scheme == "ssh" + '%s@%s:%s.git' % [base.user, base.host, name] else - base = default_base_fmt % prefix + '%s%s.git' % [base, name] end - '%s%s.git' % [base, name] end end diff --git a/services/api/app/views/user_notifier/account_is_setup.text.erb b/services/api/app/views/user_notifier/account_is_setup.text.erb index 417698c38b..357fd91ce8 100644 --- a/services/api/app/views/user_notifier/account_is_setup.text.erb +++ b/services/api/app/views/user_notifier/account_is_setup.text.erb @@ -10,7 +10,7 @@ Hi there, Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services["Workbench1"]["ExternalURL"] %>at - <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %><%= "/" if !Rails.configuration.Services["Workbench1"]["ExternalURL"].end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %> + <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %><%= "/" if !Rails.configuration.Services["Workbench1"]["ExternalURL"].to_s.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %> for connection instructions. diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb index 23a5eb5a95..bc58dbfc58 100644 --- a/services/api/config/initializers/load_config.rb +++ b/services/api/config/initializers/load_config.rb @@ -42,7 +42,7 @@ $arvados_config = {} end declare_config "ClusterID", NonemptyString, :uuid_prefix -declare_config "ManagementToken", NonemptyString, :ManagementToken +declare_config "ManagementToken", String, :ManagementToken declare_config "Git.Repositories", String, :git_repositories_dir declare_config "API.DisabledAPIs", Array, :disable_api_methods declare_config "API.MaxRequestSize", Integer, :max_request_size @@ -115,7 +115,7 @@ declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address declare_config "Services.Websocket.ExternalURL", URI, :websocket_address declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base -declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base +declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" } declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) { h = {} v.each do |clusterid, host| @@ -159,7 +159,6 @@ end coercion_and_check $arvados_config Server::Application.configure do - nils = [] copy_into_config $arvados_config, config copy_into_config $remaining_config, config config.secret_key_base = config.secret_token diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb index 1a4135e842..fbb7213fae 100644 --- a/services/api/lib/config_loader.rb +++ b/services/api/lib/config_loader.rb @@ -109,6 +109,10 @@ def coercion_and_check check_cfg cfg[k] = "" end + if cfgtype == String and cfg[k].is_a? Symbol + cfg[k] = cfg[k].to_s + end + if cfgtype == NonemptyString if (!cfg[k] || cfg[k] == "") raise "#{cfgkey} cannot be empty" diff --git a/services/api/lib/crunch_dispatch.rb b/services/api/lib/crunch_dispatch.rb index eceada5a7a..f3e17ffa20 100644 --- a/services/api/lib/crunch_dispatch.rb +++ b/services/api/lib/crunch_dispatch.rb @@ -110,7 +110,7 @@ class CrunchDispatch end def update_node_status - return unless Server::Application.config.crunch_job_wrapper.to_s.match(/^slurm/) + return unless Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"].to_s.match(/^slurm/) slurm_status.each_pair do |hostname, slurmdata| next if @node_state[hostname] == slurmdata begin @@ -337,7 +337,7 @@ class CrunchDispatch next if @running[job.uuid] cmd_args = nil - case Server::Application.config.crunch_job_wrapper + case Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"] when :none if @running.size > 0 # Don't run more than one at a time. @@ -361,7 +361,7 @@ class CrunchDispatch "--job-name=#{job.uuid}", "--nodelist=#{nodelist.join(',')}"] else - raise "Unknown crunch_job_wrapper: #{Server::Application.config.crunch_job_wrapper}" + raise "Unknown crunch_job_wrapper: #{Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"]}" end cmd_args = sudo_preface + cmd_args @@ -902,9 +902,9 @@ class CrunchDispatch end def sudo_preface - return [] if not Server::Application.config.crunch_job_user + return [] if not Rails.configuration.Containers["JobsAPI"]["CrunchJobUser"] ["sudo", "-E", "-u", - Server::Application.config.crunch_job_user, + Rails.configuration.Containers["JobsAPI"]["CrunchJobUser"], "LD_LIBRARY_PATH=#{ENV['LD_LIBRARY_PATH']}", "PATH=#{ENV['PATH']}", "PERLLIB=#{ENV['PERLLIB']}", @@ -957,7 +957,7 @@ class CrunchDispatch # An array of job_uuids in squeue def squeue_jobs - if Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"].to_sym == :slurm_immediate + if Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"] == "slurm_immediate" p = IO.popen(['squeue', '-a', '-h', '-o', '%j']) begin p.readlines.map {|line| line.strip} diff --git a/services/api/lib/current_api_client.rb b/services/api/lib/current_api_client.rb index 49638677b1..c7b48c0cdd 100644 --- a/services/api/lib/current_api_client.rb +++ b/services/api/lib/current_api_client.rb @@ -42,25 +42,25 @@ module CurrentApiClient end def system_user_uuid - [Server::Application.config.uuid_prefix, + [Rails.configuration.ClusterID, User.uuid_prefix, '000000000000000'].join('-') end def system_group_uuid - [Server::Application.config.uuid_prefix, + [Rails.configuration.ClusterID, Group.uuid_prefix, '000000000000000'].join('-') end def anonymous_group_uuid - [Server::Application.config.uuid_prefix, + [Rails.configuration.ClusterID, Group.uuid_prefix, 'anonymouspublic'].join('-') end def anonymous_user_uuid - [Server::Application.config.uuid_prefix, + [Rails.configuration.ClusterID, User.uuid_prefix, 'anonymouspublic'].join('-') end @@ -105,7 +105,7 @@ module CurrentApiClient end def all_users_group_uuid - [Server::Application.config.uuid_prefix, + [Rails.configuration.ClusterID, Group.uuid_prefix, 'fffffffffffffff'].join('-') end diff --git a/services/api/lib/enable_jobs_api.rb b/services/api/lib/enable_jobs_api.rb index 4da6188d15..9d87092066 100644 --- a/services/api/lib/enable_jobs_api.rb +++ b/services/api/lib/enable_jobs_api.rb @@ -30,7 +30,8 @@ Disable_jobs_api_method_list = ["jobs.create", "jobs.show", "job_tasks.show"] - if Rails.configuration.Containers["JobsAPI"]["Enable"] == false || +def check_enable_legacy_jobs_api + if Rails.configuration.Containers["JobsAPI"]["Enable"] == "false" || (Rails.configuration.Containers["JobsAPI"]["Enable"] == "auto" && Job.count == 0) Rails.configuration.API["DisabledAPIs"] += Disable_jobs_api_method_list diff --git a/services/api/lib/has_uuid.rb b/services/api/lib/has_uuid.rb index 60db53d5e6..2074566941 100644 --- a/services/api/lib/has_uuid.rb +++ b/services/api/lib/has_uuid.rb @@ -30,7 +30,7 @@ module HasUuid Digest::MD5.hexdigest(self.to_s).to_i(16).to_s(36)[-5..-1] end def generate_uuid - [Server::Application.config.uuid_prefix, + [Rails.configuration.ClusterID, self.uuid_prefix, rand(2**256).to_s(36)[-15..-1]]. join '-' diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb index ef3198bc63..7d67668c62 100644 --- a/services/api/test/functional/arvados/v1/collections_controller_test.rb +++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb @@ -1176,7 +1176,7 @@ EOS assert_response 200 c = Collection.find_by_uuid(uuid) assert_operator c.trash_at, :<, db_current_time - assert_equal c.delete_at, c.trash_at + Rails.configuration.Collection["BlobSigningTTL"] + assert_equal c.delete_at, c.trash_at + Rails.configuration.Collections["BlobSigningTTL"] end test 'delete long-trashed collection immediately using http DELETE verb' do diff --git a/services/api/test/functional/arvados/v1/repositories_controller_test.rb b/services/api/test/functional/arvados/v1/repositories_controller_test.rb index d986ff937b..d92561fc10 100644 --- a/services/api/test/functional/arvados/v1/repositories_controller_test.rb +++ b/services/api/test/functional/arvados/v1/repositories_controller_test.rb @@ -200,15 +200,15 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase end [ - {cfg: "Services.GitSSH.ExternalURL", cfgval: "git@example.com:", match: %r"^git@example.com:"}, - {cfg: "Services.GitSSH.ExternalURL", cfgval: true, match: %r"^git@git.zzzzz.arvadosapi.com:"}, - {cfg: "Services.GitSSH.ExternalURL", cfgval: false, refute: /^git@/ }, - {cfg: "Services.GitHTTP.ExternalURL", cfgval: "https://example.com/", match: %r"^https://example.com/"}, - {cfg: "Services.GitHTTP.ExternalURL", cfgval: true, match: %r"^https://git.zzzzz.arvadosapi.com/"}, - {cfg: "Services.GitHTTP.ExternalURL", cfgval: false, refute: /^http/ }, + {cfg: "GitSSH.ExternalURL", cfgval: URI("ssh://git@example.com"), match: %r"^git@example.com:"}, + {cfg: "GitSSH.ExternalURL", cfgval: URI(""), match: %r"^git@git.zzzzz.arvadosapi.com:"}, + {cfg: "GitSSH", cfgval: false, refute: /^git@/ }, + {cfg: "GitHTTP.ExternalURL", cfgval: URI("https://example.com/"), match: %r"^https://example.com/"}, + {cfg: "GitHTTP.ExternalURL", cfgval: URI(""), match: %r"^https://git.zzzzz.arvadosapi.com/"}, + {cfg: "GitHTTP", cfgval: false, refute: /^http/ }, ].each do |expect| test "set #{expect[:cfg]} to #{expect[:cfgval]}" do - set_cfg Rails.configuration, expect[:cfg].to_s, expect[:cfgval] + set_cfg Rails.configuration.Services, expect[:cfg].to_s, expect[:cfgval] authorize_with :active get :index assert_response :success diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb index 5d998a2442..c8dcfef238 100644 --- a/services/api/test/unit/container_request_test.rb +++ b/services/api/test/unit/container_request_test.rb @@ -525,6 +525,7 @@ class ContainerRequestTest < ActiveSupport::TestCase 'ENOEXIST', 'arvados/apitestfixture:ENOEXIST', ].each do |img| + puts "RC", Rails.configuration.RemoteClusters test "container_image_for_container(#{img.inspect}) => 422" do set_user_from_auth :active assert_raises(ArvadosModel::UnresolvableContainerError) do @@ -884,7 +885,7 @@ class ContainerRequestTest < ActiveSupport::TestCase [false, ActiveRecord::RecordInvalid], [true, nil], ].each do |preemptible_conf, expected| - test "having Rails.configuration.Containers["UsePreemptibleInstances"]=#{preemptible_conf}, create preemptible container request and verify #{expected}" do + test "having Rails.configuration.Containers['UsePreemptibleInstances']=#{preemptible_conf}, create preemptible container request and verify #{expected}" do sp = {"preemptible" => true} common_attrs = {cwd: "test", priority: 1, @@ -946,7 +947,7 @@ class ContainerRequestTest < ActiveSupport::TestCase [false, 'zzzzz-dz642-runningcontainr', nil], [false, nil, nil], ].each do |preemptible_conf, requesting_c, schedule_preemptible| - test "having Rails.configuration.Containers["UsePreemptibleInstances"]=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do + test "having Rails.configuration.Containers['UsePreemptibleInstances']=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do common_attrs = {cwd: "test", priority: 1, command: ["echo", "hello"], diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb index 65725f3301..5a88262563 100644 --- a/services/api/test/unit/job_test.rb +++ b/services/api/test/unit/job_test.rb @@ -130,7 +130,7 @@ class JobTest < ActiveSupport::TestCase Rails.configuration.RemoteClusters = {} job = Job.new job_attrs(runtime_constraints: {'docker_image' => image_spec}) - assert(job.invalid?, "nonexistent Docker image #{spec_type} was valid") + assert(job.invalid?, "nonexistent Docker image #{spec_type} #{image_spec} was valid") end end diff --git a/services/api/test/unit/user_notifier_test.rb b/services/api/test/unit/user_notifier_test.rb index b5688fcc69..79d3e15ef6 100644 --- a/services/api/test/unit/user_notifier_test.rb +++ b/services/api/test/unit/user_notifier_test.rb @@ -19,7 +19,7 @@ class UserNotifierTest < ActionMailer::TestCase assert_equal 'Welcome to Arvados - shell account enabled', email.subject assert (email.body.to_s.include? 'Your Arvados shell account has been set up'), 'Expected Your Arvados shell account has been set up in email body' - assert (email.body.to_s.include? Rails.configuration.Services["Workbench1"]["ExternalURL"]), + assert (email.body.to_s.include? Rails.configuration.Services["Workbench1"]["ExternalURL"].to_s), 'Expected workbench url in email body' end