13996: Fixing test WIP
authorPeter Amstutz <pamstutz@veritasgenetics.com>
Mon, 25 Mar 2019 20:39:23 +0000 (16:39 -0400)
committerPeter Amstutz <pamstutz@veritasgenetics.com>
Mon, 8 Apr 2019 15:09:53 +0000 (11:09 -0400)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

17 files changed:
services/api/app/controllers/application_controller.rb
services/api/app/controllers/arvados/v1/schema_controller.rb
services/api/app/models/collection.rb
services/api/app/models/job.rb
services/api/app/models/repository.rb
services/api/app/views/user_notifier/account_is_setup.text.erb
services/api/config/initializers/load_config.rb
services/api/lib/config_loader.rb
services/api/lib/crunch_dispatch.rb
services/api/lib/current_api_client.rb
services/api/lib/enable_jobs_api.rb
services/api/lib/has_uuid.rb
services/api/test/functional/arvados/v1/collections_controller_test.rb
services/api/test/functional/arvados/v1/repositories_controller_test.rb
services/api/test/unit/container_request_test.rb
services/api/test/unit/job_test.rb
services/api/test/unit/user_notifier_test.rb

index 1ec921b8ad5b8c548b801280a6cd947ce6eabb0f..5c97735f24b77a783fdc905f5ebd3ccb93124ffd 100644 (file)
@@ -81,10 +81,12 @@ class ApplicationController < ActionController::Base
 
   def default_url_options
     options = {}
-    exturl = URI.parse(Rails.configuration.Services["Controller"]["ExternalURL"])
-    options[:host] = exturl.host
-    options[:port] = exturl.port
-    options[:protocol] = exturl.scheme
+    if Rails.configuration.Services["Controller"]["ExternalURL"] != ""
+      exturl = URI.parse(Rails.configuration.Services["Controller"]["ExternalURL"])
+      options[:host] = exturl.host
+      options[:port] = exturl.port
+      options[:protocol] = exturl.scheme
+    end
     options
   end
 
index ced2f16207f3138325a8b419edbd71ac6fe91712..5f042877de2afa5e3a8fa62cd0004cb75e0aa22e 100644 (file)
@@ -46,27 +46,27 @@ class Arvados::V1::SchemaController < ApplicationController
         rootUrl: root_url,
         servicePath: "arvados/v1/",
         batchPath: "batch",
-        uuidPrefix: Rails.application.config.ClusterID,
-        defaultTrashLifetime: Rails.application.config.Collections["DefaultTrashLifetime"],
-        blobSignatureTtl: Rails.application.config.Collections["BlobSigningTTL"],
-        maxRequestSize: Rails.application.config.API["MaxRequestSize"],
-        maxItemsPerResponse: Rails.application.API["MaxItemsPerResponse"],
-        dockerImageFormats: Rails.application.config.Containers["SupportedDockerImageFormats"],
-        crunchLogBytesPerEvent: Rails.application.config.Containers["Logging"]["LogBytesPerEvent"],
-        crunchLogSecondsBetweenEvents: Rails.application.config.Containers["Logging"]["LogSecondsBetweenEvents"],
-        crunchLogThrottlePeriod: Rails.application.config.Containers["Logging"]["LogThrottlePeriod"],
-        crunchLogThrottleBytes: Rails.application.config.Containers["Logging"]["LogThrottleBytes"],
-        crunchLogThrottleLines: Rails.application.config.Containers["Logging"]["LogThrottleLines"],
-        crunchLimitLogBytesPerJob: Rails.application.config.Containers["Logging"]["LimitLogBytesPerJob"],
-        crunchLogPartialLineThrottlePeriod: Rails.application.config.Containers["Logging"]["LogPartialLineThrottlePeriod"],
-        crunchLogUpdatePeriod: Rails.application.config.Containers["Logging"]["LogUpdatePeriod"],
-        crunchLogUpdateSize: Rails.application.config.Containers["Logging"]["LogUpdateSize"],
-        remoteHosts: Rails.application.config.RemoteClusters.map {|k,v| v.Host},
-        remoteHostsViaDNS: Rails.application.config.RemoteClusters["*"]["Proxy"],
-        websocketUrl: Rails.application.config.Services["Websocket"]["ExternalURL"],
-        workbenchUrl: Rails.application.config.Services["Workbench1"]["ExternalURL"],
-        keepWebServiceUrl: Rails.application.config.Services["WebDAV"]["ExternalURL"],
-        gitUrl: Rails.application.config.Services["GitHTTP"]["ExternalURL"],
+        uuidPrefix: Rails.configuration.ClusterID,
+        defaultTrashLifetime: Rails.configuration.Collections["DefaultTrashLifetime"],
+        blobSignatureTtl: Rails.configuration.Collections["BlobSigningTTL"],
+        maxRequestSize: Rails.configuration.API["MaxRequestSize"],
+        maxItemsPerResponse: Rails.configuration.API["MaxItemsPerResponse"],
+        dockerImageFormats: Rails.configuration.Containers["SupportedDockerImageFormats"],
+        crunchLogBytesPerEvent: Rails.configuration.Containers["Logging"]["LogBytesPerEvent"],
+        crunchLogSecondsBetweenEvents: Rails.configuration.Containers["Logging"]["LogSecondsBetweenEvents"],
+        crunchLogThrottlePeriod: Rails.configuration.Containers["Logging"]["LogThrottlePeriod"],
+        crunchLogThrottleBytes: Rails.configuration.Containers["Logging"]["LogThrottleBytes"],
+        crunchLogThrottleLines: Rails.configuration.Containers["Logging"]["LogThrottleLines"],
+        crunchLimitLogBytesPerJob: Rails.configuration.Containers["Logging"]["LimitLogBytesPerJob"],
+        crunchLogPartialLineThrottlePeriod: Rails.configuration.Containers["Logging"]["LogPartialLineThrottlePeriod"],
+        crunchLogUpdatePeriod: Rails.configuration.Containers["Logging"]["LogUpdatePeriod"],
+        crunchLogUpdateSize: Rails.configuration.Containers["Logging"]["LogUpdateSize"],
+        remoteHosts: Rails.configuration.RemoteClusters.map {|k,v| v["Host"]},
+        remoteHostsViaDNS: Rails.configuration.RemoteClusters["*"]["Proxy"],
+        websocketUrl: Rails.configuration.Services["Websocket"]["ExternalURL"],
+        workbenchUrl: Rails.configuration.Services["Workbench1"]["ExternalURL"],
+        keepWebServiceUrl: Rails.configuration.Services["WebDAV"]["ExternalURL"],
+        gitUrl: Rails.configuration.Services["GitHTTP"]["ExternalURL"],
         parameters: {
           alt: {
             type: "string",
index 46da37afd4876a191c70de2554b7de7ff8f9aba8..7f7a320074b58ed9a13c711bf29a0a1b8e9f7a29 100644 (file)
@@ -518,7 +518,9 @@ class Collection < ArvadosModel
     if loc = Keep::Locator.parse(search_term)
       loc.strip_hints!
       coll_match = readable_by(*readers).where(portable_data_hash: loc.to_s).limit(1)
-      if coll_match.any? or Rails.configuration.RemoteClusters.length > 1
+      rc = Rails.configuration.RemoteClusters.select{ |k|
+        k != "*" && k != Rails.configuration.ClusterID}
+      if coll_match.any? or rc.length == 0
         return get_compatible_images(readers, pattern, coll_match)
       else
         # Allow bare pdh that doesn't exist in the local database so
index deaa8ff71d027f529437577f08a80a8b8ba87731..b24df8cbd3c9ef5c938c49d50cea996431720665 100644 (file)
@@ -491,7 +491,7 @@ class Job < ArvadosModel
   end
 
   def find_docker_image_locator
-    if runtime_constraints.is_a? Hash
+    if runtime_constraints.is_a? Hash and Rails.configuration.Containers["JobsAPI"]["DefaultDockerImage"] != ""
       runtime_constraints['docker_image'] ||=
         Rails.configuration.Containers["JobsAPI"]["DefaultDockerImage"]
     end
index ac89ecf6b06e5b3df40cd4e1728448f86474dd77..7cb36dcb92d8485b4f9696f8fd8917648bcc1ecb 100644 (file)
@@ -49,7 +49,7 @@ class Repository < ArvadosModel
     # prefers bare repositories over checkouts.
     [["%s.git"], ["%s", ".git"]].each do |repo_base, *join_args|
       [:uuid, :name].each do |path_attr|
-        git_dir = File.join(Rails.configuration.Containers["Git"]["Repositories"],
+        git_dir = File.join(Rails.configuration.Git["Repositories"],
                             repo_base % send(path_attr), *join_args)
         return git_dir if File.exist?(git_dir)
       end
@@ -98,21 +98,27 @@ class Repository < ArvadosModel
   end
 
   def ssh_clone_url
-    _clone_url Rails.configuration.Services["GitSSH"]["ExternalURL"], 'git@git.%s.arvadosapi.com:'
+    _clone_url Rails.configuration.Services["GitSSH"].andand["ExternalURL"], 'ssh://git@git.%s.arvadosapi.com'
   end
 
   def https_clone_url
-    _clone_url Rails.configuration.Services["GitHTTP"]["ExternalURL"], 'https://git.%s.arvadosapi.com/'
+    _clone_url Rails.configuration.Services["GitHTTP"].andand["ExternalURL"], 'https://git.%s.arvadosapi.com/'
   end
 
   def _clone_url config_var, default_base_fmt
-    configured_base = config_var
+    if not config_var
+      return ""
+    end
     prefix = new_record? ? Rails.configuration.ClusterID : uuid[0,5]
-    if prefix == Rails.configuration.ClusterID and configured_base != true
-      base = configured_base
+    if prefix == Rails.configuration.ClusterID and config_var != URI("")
+      base = config_var
+    else
+      base = URI(default_base_fmt % prefix)
+    end
+    if base.scheme == "ssh"
+      '%s@%s:%s.git' % [base.user, base.host, name]
     else
-      base = default_base_fmt % prefix
+      '%s%s.git' % [base, name]
     end
-    '%s%s.git' % [base, name]
   end
 end
index 417698c38b4fb86f88a2d145f0a9ef8b6dcc7f38..357fd91ce8a87ee9256871e4a93e5454513c6bbb 100644 (file)
@@ -10,7 +10,7 @@ Hi there,
 
 Your Arvados shell account has been set up. Please visit the virtual machines page <% if Rails.configuration.Services["Workbench1"]["ExternalURL"] %>at
 
-  <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %><%= "/" if !Rails.configuration.Services["Workbench1"]["ExternalURL"].end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %>
+  <%= Rails.configuration.Services["Workbench1"]["ExternalURL"] %><%= "/" if !Rails.configuration.Services["Workbench1"]["ExternalURL"].to_s.end_with?("/") %>users/<%= @user.uuid%>/virtual_machines <% else %><% end %>
 
 for connection instructions.
 
index 23a5eb5a957cc6cbd7e7e1de2784a05fe2425326..bc58dbfc58a7f8c40e757d9365a869a7a9f9bbff 100644 (file)
@@ -42,7 +42,7 @@ $arvados_config = {}
 end
 
 declare_config "ClusterID", NonemptyString, :uuid_prefix
-declare_config "ManagementToken", NonemptyString, :ManagementToken
+declare_config "ManagementToken", String, :ManagementToken
 declare_config "Git.Repositories", String, :git_repositories_dir
 declare_config "API.DisabledAPIs", Array, :disable_api_methods
 declare_config "API.MaxRequestSize", Integer, :max_request_size
@@ -115,7 +115,7 @@ declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
 declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
 declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url
 declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base
-declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base
+declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" }
 declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
   h = {}
   v.each do |clusterid, host|
@@ -159,7 +159,6 @@ end
 coercion_and_check $arvados_config
 
 Server::Application.configure do
-  nils = []
   copy_into_config $arvados_config, config
   copy_into_config $remaining_config, config
   config.secret_key_base = config.secret_token
index 1a4135e84280345ed7626c76c1981f4568370edd..fbb7213fae3d0e5294485b3cab3c365de81a8504 100644 (file)
@@ -109,6 +109,10 @@ def coercion_and_check check_cfg
       cfg[k] = ""
     end
 
+    if cfgtype == String and cfg[k].is_a? Symbol
+      cfg[k] = cfg[k].to_s
+    end
+
     if cfgtype == NonemptyString
       if (!cfg[k] || cfg[k] == "")
         raise "#{cfgkey} cannot be empty"
index eceada5a7a17a7d92c82e322038a81c15ea02d00..f3e17ffa201ef911f154194447178b4372553230 100644 (file)
@@ -110,7 +110,7 @@ class CrunchDispatch
   end
 
   def update_node_status
-    return unless Server::Application.config.crunch_job_wrapper.to_s.match(/^slurm/)
+    return unless Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"].to_s.match(/^slurm/)
     slurm_status.each_pair do |hostname, slurmdata|
       next if @node_state[hostname] == slurmdata
       begin
@@ -337,7 +337,7 @@ class CrunchDispatch
       next if @running[job.uuid]
 
       cmd_args = nil
-      case Server::Application.config.crunch_job_wrapper
+      case Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"]
       when :none
         if @running.size > 0
             # Don't run more than one at a time.
@@ -361,7 +361,7 @@ class CrunchDispatch
                     "--job-name=#{job.uuid}",
                     "--nodelist=#{nodelist.join(',')}"]
       else
-        raise "Unknown crunch_job_wrapper: #{Server::Application.config.crunch_job_wrapper}"
+        raise "Unknown crunch_job_wrapper: #{Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"]}"
       end
 
       cmd_args = sudo_preface + cmd_args
@@ -902,9 +902,9 @@ class CrunchDispatch
   end
 
   def sudo_preface
-    return [] if not Server::Application.config.crunch_job_user
+    return [] if not Rails.configuration.Containers["JobsAPI"]["CrunchJobUser"]
     ["sudo", "-E", "-u",
-     Server::Application.config.crunch_job_user,
+     Rails.configuration.Containers["JobsAPI"]["CrunchJobUser"],
      "LD_LIBRARY_PATH=#{ENV['LD_LIBRARY_PATH']}",
      "PATH=#{ENV['PATH']}",
      "PERLLIB=#{ENV['PERLLIB']}",
@@ -957,7 +957,7 @@ class CrunchDispatch
 
   # An array of job_uuids in squeue
   def squeue_jobs
-    if Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"].to_sym == :slurm_immediate
+    if Rails.configuration.Containers["JobsAPI"]["CrunchJobWrapper"] == "slurm_immediate"
       p = IO.popen(['squeue', '-a', '-h', '-o', '%j'])
       begin
         p.readlines.map {|line| line.strip}
index 49638677b18cddaeb9f27501c9d02f58d34a93f2..c7b48c0cdd6ff1ed0056a32e49f65c106afc100c 100644 (file)
@@ -42,25 +42,25 @@ module CurrentApiClient
   end
 
   def system_user_uuid
-    [Server::Application.config.uuid_prefix,
+    [Rails.configuration.ClusterID,
      User.uuid_prefix,
      '000000000000000'].join('-')
   end
 
   def system_group_uuid
-    [Server::Application.config.uuid_prefix,
+    [Rails.configuration.ClusterID,
      Group.uuid_prefix,
      '000000000000000'].join('-')
   end
 
   def anonymous_group_uuid
-    [Server::Application.config.uuid_prefix,
+    [Rails.configuration.ClusterID,
      Group.uuid_prefix,
      'anonymouspublic'].join('-')
   end
 
   def anonymous_user_uuid
-    [Server::Application.config.uuid_prefix,
+    [Rails.configuration.ClusterID,
      User.uuid_prefix,
      'anonymouspublic'].join('-')
   end
@@ -105,7 +105,7 @@ module CurrentApiClient
   end
 
   def all_users_group_uuid
-    [Server::Application.config.uuid_prefix,
+    [Rails.configuration.ClusterID,
      Group.uuid_prefix,
      'fffffffffffffff'].join('-')
   end
index 4da6188d1549ea571cd1e857e6d8fc8134127340..9d8709206613bc9ba3a3ed2d226cbbac36cf9fe6 100644 (file)
@@ -30,7 +30,8 @@ Disable_jobs_api_method_list = ["jobs.create",
                                                "jobs.show",
                                                "job_tasks.show"]
 
-  if Rails.configuration.Containers["JobsAPI"]["Enable"] == false ||
+def check_enable_legacy_jobs_api
+  if Rails.configuration.Containers["JobsAPI"]["Enable"] == "false" ||
      (Rails.configuration.Containers["JobsAPI"]["Enable"] == "auto" &&
       Job.count == 0)
     Rails.configuration.API["DisabledAPIs"] += Disable_jobs_api_method_list
index 60db53d5e687440f4297b0ea29fc5893e52cf68c..2074566941fec7c6a79903df712b82541e9a5853 100644 (file)
@@ -30,7 +30,7 @@ module HasUuid
       Digest::MD5.hexdigest(self.to_s).to_i(16).to_s(36)[-5..-1]
     end
     def generate_uuid
-      [Server::Application.config.uuid_prefix,
+      [Rails.configuration.ClusterID,
        self.uuid_prefix,
        rand(2**256).to_s(36)[-15..-1]].
         join '-'
index ef3198bc635507adebeb57bb15b6610905e8020a..7d67668c627b505d0d856396d5a83eb1294bfd67 100644 (file)
@@ -1176,7 +1176,7 @@ EOS
     assert_response 200
     c = Collection.find_by_uuid(uuid)
     assert_operator c.trash_at, :<, db_current_time
-    assert_equal c.delete_at, c.trash_at + Rails.configuration.Collection["BlobSigningTTL"]
+    assert_equal c.delete_at, c.trash_at + Rails.configuration.Collections["BlobSigningTTL"]
   end
 
   test 'delete long-trashed collection immediately using http DELETE verb' do
index d986ff937b6438d830d44eb5ae1cb20ee6f6991a..d92561fc102c142e023d2c37ad909083148459db 100644 (file)
@@ -200,15 +200,15 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase
   end
 
   [
-    {cfg: "Services.GitSSH.ExternalURL", cfgval: "git@example.com:", match: %r"^git@example.com:"},
-    {cfg: "Services.GitSSH.ExternalURL", cfgval: true, match: %r"^git@git.zzzzz.arvadosapi.com:"},
-    {cfg: "Services.GitSSH.ExternalURL", cfgval: false, refute: /^git@/ },
-    {cfg: "Services.GitHTTP.ExternalURL", cfgval: "https://example.com/", match: %r"^https://example.com/"},
-    {cfg: "Services.GitHTTP.ExternalURL", cfgval: true, match: %r"^https://git.zzzzz.arvadosapi.com/"},
-    {cfg: "Services.GitHTTP.ExternalURL", cfgval: false, refute: /^http/ },
+    {cfg: "GitSSH.ExternalURL", cfgval: URI("ssh://git@example.com"), match: %r"^git@example.com:"},
+    {cfg: "GitSSH.ExternalURL", cfgval: URI(""), match: %r"^git@git.zzzzz.arvadosapi.com:"},
+    {cfg: "GitSSH", cfgval: false, refute: /^git@/ },
+    {cfg: "GitHTTP.ExternalURL", cfgval: URI("https://example.com/"), match: %r"^https://example.com/"},
+    {cfg: "GitHTTP.ExternalURL", cfgval: URI(""), match: %r"^https://git.zzzzz.arvadosapi.com/"},
+    {cfg: "GitHTTP", cfgval: false, refute: /^http/ },
   ].each do |expect|
     test "set #{expect[:cfg]} to #{expect[:cfgval]}" do
-      set_cfg Rails.configuration, expect[:cfg].to_s, expect[:cfgval]
+      set_cfg Rails.configuration.Services, expect[:cfg].to_s, expect[:cfgval]
       authorize_with :active
       get :index
       assert_response :success
index 5d998a24425dfe9740c36e599c8185721019239d..c8dcfef238450157c1eb95d780be2deb08da7681 100644 (file)
@@ -525,6 +525,7 @@ class ContainerRequestTest < ActiveSupport::TestCase
    'ENOEXIST',
    'arvados/apitestfixture:ENOEXIST',
   ].each do |img|
+    puts "RC", Rails.configuration.RemoteClusters
     test "container_image_for_container(#{img.inspect}) => 422" do
       set_user_from_auth :active
       assert_raises(ArvadosModel::UnresolvableContainerError) do
@@ -884,7 +885,7 @@ class ContainerRequestTest < ActiveSupport::TestCase
     [false, ActiveRecord::RecordInvalid],
     [true, nil],
   ].each do |preemptible_conf, expected|
-    test "having Rails.configuration.Containers["UsePreemptibleInstances"]=#{preemptible_conf}, create preemptible container request and verify #{expected}" do
+    test "having Rails.configuration.Containers['UsePreemptibleInstances']=#{preemptible_conf}, create preemptible container request and verify #{expected}" do
       sp = {"preemptible" => true}
       common_attrs = {cwd: "test",
                       priority: 1,
@@ -946,7 +947,7 @@ class ContainerRequestTest < ActiveSupport::TestCase
     [false, 'zzzzz-dz642-runningcontainr', nil],
     [false, nil, nil],
   ].each do |preemptible_conf, requesting_c, schedule_preemptible|
-    test "having Rails.configuration.Containers["UsePreemptibleInstances"]=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do
+    test "having Rails.configuration.Containers['UsePreemptibleInstances']=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do
       common_attrs = {cwd: "test",
                       priority: 1,
                       command: ["echo", "hello"],
index 65725f3301c62fbc793ea518834d4738bafdff88..5a882625636502fe6f214a40c3365c9f9e871d21 100644 (file)
@@ -130,7 +130,7 @@ class JobTest < ActiveSupport::TestCase
       Rails.configuration.RemoteClusters = {}
       job = Job.new job_attrs(runtime_constraints:
                               {'docker_image' => image_spec})
-      assert(job.invalid?, "nonexistent Docker image #{spec_type} was valid")
+      assert(job.invalid?, "nonexistent Docker image #{spec_type} #{image_spec} was valid")
     end
   end
 
index b5688fcc69bf9f6084b68958d4baef2a8cbb0986..79d3e15ef66b2bf4a728838064f030ee0f26f425 100644 (file)
@@ -19,7 +19,7 @@ class UserNotifierTest < ActionMailer::TestCase
     assert_equal 'Welcome to Arvados - shell account enabled', email.subject
     assert (email.body.to_s.include? 'Your Arvados shell account has been set up'),
         'Expected Your Arvados shell account has been set up in email body'
-    assert (email.body.to_s.include? Rails.configuration.Services["Workbench1"]["ExternalURL"]),
+    assert (email.body.to_s.include? Rails.configuration.Services["Workbench1"]["ExternalURL"].to_s),
         'Expected workbench url in email body'
   end