From: Lucas Di Pentima <lucas@curoverse.com>
Date: Wed, 19 Jul 2017 21:21:27 +0000 (-0300)
Subject: 11167: Removed arv-get calling code from show_file.
X-Git-Tag: 1.1.0~92^2~21
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/f291a43f7154d634a417bff32b9b81c197feb461?ds=sidebyside

11167: Removed arv-get calling code from show_file.
Adjusted/deleted related tests.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas@curoverse.com>
---

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index f8fcf5108f..8da1a20b33 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -162,45 +162,6 @@ class CollectionsController < ApplicationController
       opts[:disposition] = params[:disposition] if params[:disposition]
       return redirect_to keep_web_url(params[:uuid], params[:file], opts)
     end
-
-    # No keep-web server available. Get the file data with arv-get,
-    # and serve it through Rails.
-
-    file_name = params[:file].andand.sub(/^(\.\/|\/|)/, './')
-    if file_name.nil? or not coll.manifest.has_file?(file_name)
-      return render_not_found
-    end
-
-    opts = params.merge(arvados_api_token: usable_token)
-
-    # Handle Range requests. Currently we support only 'bytes=0-....'
-    if request.headers.include? 'HTTP_RANGE'
-      if m = /^bytes=0-(\d+)/.match(request.headers['HTTP_RANGE'])
-        opts[:maxbytes] = m[1]
-        size = params[:size] || '*'
-        self.response.status = 206
-        self.response.headers['Content-Range'] = "bytes 0-#{m[1]}/#{size}"
-      end
-    end
-
-    ext = File.extname(params[:file])
-    self.response.headers['Content-Type'] =
-      Rack::Mime::MIME_TYPES[ext] || 'application/octet-stream'
-    if params[:size]
-      size = params[:size].to_i
-      if opts[:maxbytes]
-        size = [size, opts[:maxbytes].to_i].min
-      end
-      self.response.headers['Content-Length'] = size.to_s
-    end
-    self.response.headers['Content-Disposition'] = params[:disposition] if params[:disposition]
-    begin
-      file_enumerator(opts).each do |bytes|
-        response.stream.write bytes
-      end
-    ensure
-      response.stream.close
-    end
   end
 
   def sharing_scopes
@@ -468,43 +429,4 @@ class CollectionsController < ApplicationController
 
     uri.to_s
   end
-
-  # Note: several controller and integration tests rely on stubbing
-  # file_enumerator to return fake file content.
-  def file_enumerator opts
-    FileStreamer.new opts
-  end
-
-  class FileStreamer
-    include ArvadosApiClientHelper
-    def initialize(opts={})
-      @opts = opts
-    end
-    def each
-      return unless @opts[:uuid] && @opts[:file]
-
-      env = Hash[ENV].dup
-
-      require 'uri'
-      u = URI.parse(arvados_api_client.arvados_v1_base)
-      env['ARVADOS_API_HOST'] = "#{u.host}:#{u.port}"
-      env['ARVADOS_API_TOKEN'] = @opts[:arvados_api_token]
-      env['ARVADOS_API_HOST_INSECURE'] = "true" if Rails.configuration.arvados_insecure_https
-
-      bytesleft = @opts[:maxbytes].andand.to_i || 2**16
-      io = IO.popen([env, 'arv-get', "#{@opts[:uuid]}/#{@opts[:file]}"], 'rb')
-      while bytesleft > 0 && (buf = io.read([bytesleft, 2**16].min)) != nil
-        # shrink the bytesleft count, if we were given a maximum byte
-        # count to read
-        if @opts.include? :maxbytes
-          bytesleft = bytesleft - buf.length
-        end
-        yield buf
-      end
-      io.close
-      # "If ios is opened by IO.popen, close sets $?."
-      # http://www.ruby-doc.org/core-2.1.3/IO.html#method-i-close
-      Rails.logger.warn("#{@opts[:uuid]}/#{@opts[:file]}: #{$?}") if $? != 0
-    end
-  end
 end
diff --git a/apps/workbench/test/controllers/collections_controller_test.rb b/apps/workbench/test/controllers/collections_controller_test.rb
index 26d6fda85e..5f67837146 100644
--- a/apps/workbench/test/controllers/collections_controller_test.rb
+++ b/apps/workbench/test/controllers/collections_controller_test.rb
@@ -23,15 +23,6 @@ class CollectionsControllerTest < ActionController::TestCase
       end
   end
 
-  def stub_file_content
-    # For the duration of the current test case, stub file download
-    # content with a randomized (but recognizable) string. Return the
-    # string, the test case can use it in assertions.
-    txt = 'the quick brown fox ' + rand(2**32).to_s
-    @controller.stubs(:file_enumerator).returns([txt])
-    txt
-  end
-
   def collection_params(collection_name, file_name=nil)
     uuid = api_fixture('collections')[collection_name.to_s]['uuid']
     params = {uuid: uuid, id: uuid}
@@ -75,17 +66,14 @@ class CollectionsControllerTest < ActionController::TestCase
   end
 
   test "download a file with spaces in filename" do
+    setup_for_keep_web
     collection = api_fixture('collections')['w_a_z_file']
-    fakepipe = IO.popen(['echo', '-n', 'w a z'], 'rb')
-    IO.expects(:popen).with { |cmd, mode|
-      cmd.include? "#{collection['uuid']}/w a z"
-    }.returns(fakepipe)
     get :show_file, {
       uuid: collection['uuid'],
       file: 'w a z'
     }, session_for(:active)
-    assert_response :success
-    assert_equal 'w a z', response.body
+    assert_response :redirect
+    assert_match /w%20a%20z/, response.redirect_url
   end
 
   test "viewing a collection fetches related projects" do
@@ -143,14 +131,13 @@ class CollectionsControllerTest < ActionController::TestCase
   end
 
   test "fetching collection file with reader token" do
-    expected = stub_file_content
+    setup_for_keep_web
     params = collection_params(:foo_file, "foo")
     params[:reader_token] = api_fixture("api_client_authorizations",
                                         "active_all_collections", "api_token")
     get(:show_file, params)
-    assert_response :success
-    assert_equal(expected, @response.body,
-                 "failed to fetch a Collection file with a reader token")
+    assert_response :redirect
+    assert_match /foo/, response.redirect_url
     assert_no_session
   end
 
@@ -163,24 +150,23 @@ class CollectionsControllerTest < ActionController::TestCase
   end
 
   test "getting a file from Keep" do
+    setup_for_keep_web
     params = collection_params(:foo_file, 'foo')
     sess = session_for(:active)
-    expect_content = stub_file_content
     get(:show_file, params, sess)
-    assert_response :success
-    assert_equal(expect_content, @response.body,
-                 "failed to get a correct file from Keep")
+    assert_response :redirect
+    assert_match /foo/, response.redirect_url
   end
 
   test 'anonymous download' do
+    setup_for_keep_web
     config_anonymous true
-    expect_content = stub_file_content
     get :show_file, {
       uuid: api_fixture('collections')['user_agreement_in_anonymously_accessible_project']['uuid'],
       file: 'GNU_General_Public_License,_version_3.pdf',
     }
-    assert_response :success
-    assert_equal expect_content, response.body
+    assert_response :redirect
+    assert_match /GNU_General_Public_License/, response.redirect_url
   end
 
   test "can't get a file from Keep without permission" do
@@ -190,22 +176,14 @@ class CollectionsControllerTest < ActionController::TestCase
     assert_response 404
   end
 
-  test "trying to get a nonexistent file from Keep returns a 404" do
-    params = collection_params(:foo_file, 'gone')
-    sess = session_for(:admin)
-    get(:show_file, params, sess)
-    assert_response 404
-  end
-
   test "getting a file from Keep with a good reader token" do
+    setup_for_keep_web
     params = collection_params(:foo_file, 'foo')
     read_token = api_fixture('api_client_authorizations')['active']['api_token']
     params[:reader_token] = read_token
-    expect_content = stub_file_content
     get(:show_file, params)
-    assert_response :success
-    assert_equal(expect_content, @response.body,
-                 "failed to get a correct file from Keep using a reader token")
+    assert_response :redirect
+    assert_match /foo/, response.redirect_url
     assert_not_equal(read_token, session[:arvados_api_token],
                      "using a reader token set the session's API token")
   end
@@ -229,25 +207,22 @@ class CollectionsControllerTest < ActionController::TestCase
   end
 
   test "can get a file with an unpermissioned auth but in-scope reader token" do
+    setup_for_keep_web
     params = collection_params(:foo_file, 'foo')
     sess = session_for(:expired)
     read_token = api_fixture('api_client_authorizations')['active']['api_token']
     params[:reader_token] = read_token
-    expect_content = stub_file_content
     get(:show_file, params, sess)
-    assert_response :success
-    assert_equal(expect_content, @response.body,
-                 "failed to get a correct file from Keep using a reader token")
+    assert_response :redirect
     assert_not_equal(read_token, session[:arvados_api_token],
                      "using a reader token set the session's API token")
   end
 
   test "inactive user can retrieve user agreement" do
+    setup_for_keep_web
     ua_collection = api_fixture('collections')['user_agreement']
     # Here we don't test whether the agreement can be retrieved from
-    # Keep. We only test that show_file decides to send file content,
-    # so we use the file content stub.
-    stub_file_content
+    # Keep. We only test that show_file decides to send file content.
     get :show_file, {
       uuid: ua_collection['uuid'],
       file: ua_collection['manifest_text'].match(/ \d+:\d+:(\S+)/)[1]
@@ -255,7 +230,7 @@ class CollectionsControllerTest < ActionController::TestCase
     assert_nil(assigns(:unsigned_user_agreements),
                "Did not skip check_user_agreements filter " +
                "when showing the user agreement.")
-    assert_response :success
+    assert_response :redirect
   end
 
   test "requesting nonexistent Collection returns 404" do
@@ -263,37 +238,12 @@ class CollectionsControllerTest < ActionController::TestCase
                     :active, 404)
   end
 
-  test "use a reasonable read buffer even if client requests a huge range" do
-    fakefiledata = mock
-    IO.expects(:popen).returns(fakefiledata)
-    fakefiledata.expects(:read).twice.with() do |length|
-      # Fail the test if read() is called with length>1MiB:
-      length < 2**20
-      ## Force the ActionController::Live thread to lose the race to
-      ## verify that @response.body.length actually waits for the
-      ## response (see below):
-      # sleep 3
-    end.returns("foo\n", nil)
-    fakefiledata.expects(:close)
-    foo_file = api_fixture('collections')['foo_file']
-    @request.headers['Range'] = 'bytes=0-4294967296/*'
-    get :show_file, {
-      uuid: foo_file['uuid'],
-      file: foo_file['manifest_text'].match(/ \d+:\d+:(\S+)/)[1]
-    }, session_for(:active)
-    # Wait for the whole response to arrive before deciding whether
-    # mocks' expectations were met. Otherwise, Mocha will fail the
-    # test depending on how slowly the ActionController::Live thread
-    # runs.
-    @response.body.length
-  end
-
   test "show file in a subdirectory of a collection" do
+    setup_for_keep_web
     params = collection_params(:collection_with_files_in_subdir, 'subdir2/subdir3/subdir4/file1_in_subdir4.txt')
-    expect_content = stub_file_content
     get(:show_file, params, session_for(:user1_with_load))
-    assert_response :success
-    assert_equal(expect_content, @response.body, "failed to get a correct file from Keep")
+    assert_response :redirect
+    assert_match /subdir2\/subdir3\/subdir4\/file1_in_subdir4\.txt/, response.redirect_url
   end
 
   test 'provenance graph' do
@@ -521,7 +471,6 @@ class CollectionsControllerTest < ActionController::TestCase
   def setup_for_keep_web cfg='https://%{uuid_or_pdh}.example', dl_cfg=false
     Rails.configuration.keep_web_url = cfg
     Rails.configuration.keep_web_download_url = dl_cfg
-    @controller.expects(:file_enumerator).never
   end
 
   %w(uuid portable_data_hash).each do |id_type|
diff --git a/apps/workbench/test/integration/anonymous_access_test.rb b/apps/workbench/test/integration/anonymous_access_test.rb
index 6141cb3ca1..92e3cf5a5e 100644
--- a/apps/workbench/test/integration/anonymous_access_test.rb
+++ b/apps/workbench/test/integration/anonymous_access_test.rb
@@ -116,16 +116,6 @@ class AnonymousAccessTest < ActionDispatch::IntegrationTest
     end
   end
 
-  test 'view file' do
-    magic = rand(2**512).to_s 36
-    CollectionsController.any_instance.stubs(:file_enumerator).returns([magic])
-    collection = api_fixture('collections')['public_text_file']
-    visit '/collections/' + collection['uuid']
-    find('tr,li', text: 'Hello world.txt').
-      find('a[title~=View]').click
-    assert_text magic
-  end
-
   [
     'running anonymously accessible cr',
     'pipelineInstance'
diff --git a/apps/workbench/test/integration/collections_test.rb b/apps/workbench/test/integration/collections_test.rb
index 8619858dfe..27481c061a 100644
--- a/apps/workbench/test/integration/collections_test.rb
+++ b/apps/workbench/test/integration/collections_test.rb
@@ -52,38 +52,6 @@ class CollectionsTest < ActionDispatch::IntegrationTest
     end
   end
 
-  test "can download an entire collection with a reader token" do
-    Capybara.current_driver = :rack_test
-    CollectionsController.any_instance.
-      stubs(:file_enumerator).returns(["foo\n", "file\n"])
-    uuid = api_fixture('collections')['foo_file']['uuid']
-    token = api_fixture('api_client_authorizations')['active_all_collections']['api_token']
-    url_head = "/collections/download/#{uuid}/#{token}/"
-    visit url_head
-    # It seems that Capybara can't inspect tags outside the body, so this is
-    # a very blunt approach.
-    assert_no_match(/<\s*meta[^>]+\bnofollow\b/i, page.html,
-                    "wget prohibited from recursing the collection page")
-    # Look at all the links that wget would recurse through using our
-    # recommended options, and check that it's exactly the file list.
-    hrefs = page.all('a').map do |anchor|
-      link = anchor[:href] || ''
-      if link.start_with? url_head
-        link[url_head.size .. -1]
-      elsif link.start_with? '/'
-        nil
-      else
-        link
-      end
-    end
-    assert_equal(['foo'], hrefs.compact.sort,
-                 "download page did provide strictly file links")
-    within "#collection_files" do
-      click_link "foo"
-      assert_equal("foo\nfile\n", page.html)
-    end
-  end
-
   test "combine selected collections into new collection" do
     foo_collection = api_fixture('collections')['foo_file']
     bar_collection = api_fixture('collections')['bar_file']
diff --git a/apps/workbench/test/integration/jobs_test.rb b/apps/workbench/test/integration/jobs_test.rb
index 8a60a84459..f427537abb 100644
--- a/apps/workbench/test/integration/jobs_test.rb
+++ b/apps/workbench/test/integration/jobs_test.rb
@@ -39,39 +39,6 @@ class JobsTest < ActionDispatch::IntegrationTest
     assert_selector 'a[href="/"]', text: 'Go to dashboard'
   end
 
-  test "view job log" do
-    job = api_fixture('jobs')['job_with_real_log']
-
-    IO.expects(:popen).returns(fakepipe_with_log_data)
-
-    visit page_with_token("active", "/jobs/#{job['uuid']}")
-    assert page.has_text? job['script_version']
-
-    find(:xpath, "//a[@href='#Log']").click
-    wait_for_ajax
-    assert page.has_text? 'Started at'
-    assert page.has_text? 'Finished at'
-    assert page.has_text? 'log message 1'
-    assert page.has_text? 'log message 2'
-    assert page.has_text? 'log message 3'
-    assert page.has_no_text? 'Showing only 100 bytes of this log'
-  end
-
-  test 'view partial job log' do
-    # This config will be restored during teardown by ../test_helper.rb:
-    Rails.configuration.log_viewer_max_bytes = 100
-
-    IO.expects(:popen).returns(fakepipe_with_log_data)
-    job = api_fixture('jobs')['job_with_real_log']
-
-    visit page_with_token("active", "/jobs/#{job['uuid']}")
-    assert page.has_text? job['script_version']
-
-    find(:xpath, "//a[@href='#Log']").click
-    wait_for_ajax
-    assert page.has_text? 'Showing only 100 bytes of this log'
-  end
-
   test 'view log via keep-web redirect' do
     use_keep_web_config
 
diff --git a/apps/workbench/test/integration_helper.rb b/apps/workbench/test/integration_helper.rb
index 5d2cefe447..efe6bd2b33 100644
--- a/apps/workbench/test/integration_helper.rb
+++ b/apps/workbench/test/integration_helper.rb
@@ -163,7 +163,6 @@ module KeepWebConfig
     @kwdport = getport 'keep-web-dl-ssl'
     Rails.configuration.keep_web_url = "https://localhost:#{@kwport}/c=%{uuid_or_pdh}"
     Rails.configuration.keep_web_download_url = "https://localhost:#{@kwdport}/c=%{uuid_or_pdh}"
-    CollectionsController.any_instance.expects(:file_enumerator).never
   end
 end