services/api/config/arvados-clients.yml
*#*
.DS_Store
+.vscode
end
def show_file_links
- if Rails.configuration.keep_web_url || Rails.configuration.keep_web_download_url
- # show_file will redirect to keep-web's directory listing
- return show_file
- end
- Thread.current[:reader_tokens] = [params[:reader_token]]
- return if false.equal?(find_object_by_uuid)
- render layout: false
+ return show_file
end
def show_file
- # We pipe from arv-get to send the file to the user. Before we start it,
- # we ask the API server if the file actually exists. This serves two
- # purposes: it lets us return a useful status code for common errors, and
- # helps us figure out which token to provide to arv-get.
# The order of searched tokens is important: because the anonymous user
# token is passed along with every API request, we have to check it first.
# Otherwise, it's impossible to know whether any other request succeeded
return
end
- # If we are configured to use a keep-web server, just redirect to
- # the appropriate URL.
- if Rails.configuration.keep_web_url or
- Rails.configuration.keep_web_download_url
- opts = {}
- if usable_token == params[:reader_token]
- opts[:path_token] = usable_token
- elsif usable_token == Rails.configuration.anonymous_user_token
- # Don't pass a token at all
- else
- # We pass the current user's real token only if it's necessary
- # to read the collection.
- opts[:query_token] = usable_token
- end
- opts[:disposition] = params[:disposition] if params[:disposition]
- return redirect_to keep_web_url(params[:uuid], params[:file], opts)
- end
-
- # No keep-web server available. Get the file data with arv-get,
- # and serve it through Rails.
-
- file_name = params[:file].andand.sub(/^(\.\/|\/|)/, './')
- if file_name.nil? or not coll.manifest.has_file?(file_name)
- return render_not_found
- end
-
- opts = params.merge(arvados_api_token: usable_token)
-
- # Handle Range requests. Currently we support only 'bytes=0-....'
- if request.headers.include? 'HTTP_RANGE'
- if m = /^bytes=0-(\d+)/.match(request.headers['HTTP_RANGE'])
- opts[:maxbytes] = m[1]
- size = params[:size] || '*'
- self.response.status = 206
- self.response.headers['Content-Range'] = "bytes 0-#{m[1]}/#{size}"
- end
- end
-
- ext = File.extname(params[:file])
- self.response.headers['Content-Type'] =
- Rack::Mime::MIME_TYPES[ext] || 'application/octet-stream'
- if params[:size]
- size = params[:size].to_i
- if opts[:maxbytes]
- size = [size, opts[:maxbytes].to_i].min
- end
- self.response.headers['Content-Length'] = size.to_s
- end
- self.response.headers['Content-Disposition'] = params[:disposition] if params[:disposition]
- begin
- file_enumerator(opts).each do |bytes|
- response.stream.write bytes
- end
- ensure
- response.stream.close
+ opts = {}
+ if usable_token == params[:reader_token]
+ opts[:path_token] = usable_token
+ elsif usable_token == Rails.configuration.anonymous_user_token
+ # Don't pass a token at all
+ else
+ # We pass the current user's real token only if it's necessary
+ # to read the collection.
+ opts[:query_token] = usable_token
end
+ opts[:disposition] = params[:disposition] if params[:disposition]
+ return redirect_to keep_web_url(params[:uuid], params[:file], opts)
end
def sharing_scopes
def download_link
token = @search_sharing.first.api_token
- if Rails.configuration.keep_web_url || Rails.configuration.keep_web_download_url
- keep_web_url(@object.uuid, nil, {path_token: token})
- else
- collections_url + "/download/#{@object.uuid}/#{token}/"
- end
+ keep_web_url(@object.uuid, nil, {path_token: token})
end
def share
uri.to_s
end
-
- # Note: several controller and integration tests rely on stubbing
- # file_enumerator to return fake file content.
- def file_enumerator opts
- FileStreamer.new opts
- end
-
- class FileStreamer
- include ArvadosApiClientHelper
- def initialize(opts={})
- @opts = opts
- end
- def each
- return unless @opts[:uuid] && @opts[:file]
-
- env = Hash[ENV].dup
-
- require 'uri'
- u = URI.parse(arvados_api_client.arvados_v1_base)
- env['ARVADOS_API_HOST'] = "#{u.host}:#{u.port}"
- env['ARVADOS_API_TOKEN'] = @opts[:arvados_api_token]
- env['ARVADOS_API_HOST_INSECURE'] = "true" if Rails.configuration.arvados_insecure_https
-
- bytesleft = @opts[:maxbytes].andand.to_i || 2**16
- io = IO.popen([env, 'arv-get', "#{@opts[:uuid]}/#{@opts[:file]}"], 'rb')
- while bytesleft > 0 && (buf = io.read([bytesleft, 2**16].min)) != nil
- # shrink the bytesleft count, if we were given a maximum byte
- # count to read
- if @opts.include? :maxbytes
- bytesleft = bytesleft - buf.length
- end
- yield buf
- end
- io.close
- # "If ios is opened by IO.popen, close sets $?."
- # http://www.ruby-doc.org/core-2.1.3/IO.html#method-i-close
- Rails.logger.warn("#{@opts[:uuid]}/#{@opts[:file]}: #{$?}") if $? != 0
- end
- end
end
$("#log-viewer-download-pane").show();
var headers = {};
if (log_size > log_maxbytes) {
- headers['Range'] = 'bytes=0-' + log_maxbytes;
+ headers['Range'] = 'bytes=0-' + (log_maxbytes - 1);
}
var ajax_opts = { dataType: 'text', headers: headers };
load_log();
profiling_enabled: true
secret_token: <%= rand(2**256).to_s(36) %>
secret_key_base: <%= rand(2**256).to_s(36) %>
+ # This setting is to allow workbench start when running tests, it should be
+ # set to a correct value when testing relevant features.
+ keep_web_url: http://example.com/c=%{uuid_or_pdh}
# When you run the Workbench's integration tests, it starts the API
# server as a dependency. These settings should match the API
shell_in_a_box_url: false
# Format of preview links. If false, use keep_web_download_url
- # instead, and disable inline preview. If both are false, use
- # Workbench's built-in file download/preview mechanism.
+ # instead, and disable inline preview.
+ # If both are false, Workbench won't start, this is a mandatory configuration.
#
# Examples:
# keep_web_url: https://%{uuid_or_pdh}.collections.uuid_prefix.arvadosapi.com
arvados_v1_base: https://arvados.local:3030/arvados/v1
arvados_insecure_https: true
+ # You need to configure at least one of these:
+ keep_web_url: false
+ keep_web_download_url: false
+
production:
# At minimum, you need a nice long randomly generated secret_token here.
secret_token: ~
arvados_login_base: https://arvados.local:3030/login
arvados_v1_base: https://arvados.local:3030/arvados/v1
arvados_insecure_https: false
+
+ # You need to configure at least one of these:
+ keep_web_url: false
+ keep_web_download_url: false
config/application.yml:
* #{nils.join "\n* "}
+EOS
+ end
+ # Refuse to start if keep-web isn't configured
+ if not (config.keep_web_url or config.keep_web_download_url)
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+Keep-web service must be configured in config/application.yml:
+* keep_web_url
+* keep_web_download_url
+
EOS
end
end
end
end
- def stub_file_content
- # For the duration of the current test case, stub file download
- # content with a randomized (but recognizable) string. Return the
- # string, the test case can use it in assertions.
- txt = 'the quick brown fox ' + rand(2**32).to_s
- @controller.stubs(:file_enumerator).returns([txt])
- txt
- end
-
def collection_params(collection_name, file_name=nil)
uuid = api_fixture('collections')[collection_name.to_s]['uuid']
params = {uuid: uuid, id: uuid}
end
test "download a file with spaces in filename" do
+ setup_for_keep_web
collection = api_fixture('collections')['w_a_z_file']
- fakepipe = IO.popen(['echo', '-n', 'w a z'], 'rb')
- IO.expects(:popen).with { |cmd, mode|
- cmd.include? "#{collection['uuid']}/w a z"
- }.returns(fakepipe)
get :show_file, {
uuid: collection['uuid'],
file: 'w a z'
}, session_for(:active)
- assert_response :success
- assert_equal 'w a z', response.body
+ assert_response :redirect
+ assert_match /w%20a%20z/, response.redirect_url
end
test "viewing a collection fetches related projects" do
params[:reader_token] = api_fixture("api_client_authorizations",
"active_all_collections", "api_token")
get(:show_file_links, params)
- assert_response :success
- assert_equal([['.', 'foo', 3]], assigns(:object).files)
+ assert_response :redirect
assert_no_session
end
test "fetching collection file with reader token" do
- expected = stub_file_content
+ setup_for_keep_web
params = collection_params(:foo_file, "foo")
params[:reader_token] = api_fixture("api_client_authorizations",
"active_all_collections", "api_token")
get(:show_file, params)
- assert_response :success
- assert_equal(expected, @response.body,
- "failed to fetch a Collection file with a reader token")
+ assert_response :redirect
+ assert_match /foo/, response.redirect_url
assert_no_session
end
end
test "getting a file from Keep" do
+ setup_for_keep_web
params = collection_params(:foo_file, 'foo')
sess = session_for(:active)
- expect_content = stub_file_content
get(:show_file, params, sess)
- assert_response :success
- assert_equal(expect_content, @response.body,
- "failed to get a correct file from Keep")
+ assert_response :redirect
+ assert_match /foo/, response.redirect_url
end
test 'anonymous download' do
+ setup_for_keep_web
config_anonymous true
- expect_content = stub_file_content
get :show_file, {
uuid: api_fixture('collections')['user_agreement_in_anonymously_accessible_project']['uuid'],
file: 'GNU_General_Public_License,_version_3.pdf',
}
- assert_response :success
- assert_equal expect_content, response.body
+ assert_response :redirect
+ assert_match /GNU_General_Public_License/, response.redirect_url
end
test "can't get a file from Keep without permission" do
assert_response 404
end
- test "trying to get a nonexistent file from Keep returns a 404" do
- params = collection_params(:foo_file, 'gone')
- sess = session_for(:admin)
- get(:show_file, params, sess)
- assert_response 404
- end
-
test "getting a file from Keep with a good reader token" do
+ setup_for_keep_web
params = collection_params(:foo_file, 'foo')
read_token = api_fixture('api_client_authorizations')['active']['api_token']
params[:reader_token] = read_token
- expect_content = stub_file_content
get(:show_file, params)
- assert_response :success
- assert_equal(expect_content, @response.body,
- "failed to get a correct file from Keep using a reader token")
+ assert_response :redirect
+ assert_match /foo/, response.redirect_url
assert_not_equal(read_token, session[:arvados_api_token],
"using a reader token set the session's API token")
end
end
test "can get a file with an unpermissioned auth but in-scope reader token" do
+ setup_for_keep_web
params = collection_params(:foo_file, 'foo')
sess = session_for(:expired)
read_token = api_fixture('api_client_authorizations')['active']['api_token']
params[:reader_token] = read_token
- expect_content = stub_file_content
get(:show_file, params, sess)
- assert_response :success
- assert_equal(expect_content, @response.body,
- "failed to get a correct file from Keep using a reader token")
+ assert_response :redirect
assert_not_equal(read_token, session[:arvados_api_token],
"using a reader token set the session's API token")
end
test "inactive user can retrieve user agreement" do
+ setup_for_keep_web
ua_collection = api_fixture('collections')['user_agreement']
# Here we don't test whether the agreement can be retrieved from
- # Keep. We only test that show_file decides to send file content,
- # so we use the file content stub.
- stub_file_content
+ # Keep. We only test that show_file decides to send file content.
get :show_file, {
uuid: ua_collection['uuid'],
file: ua_collection['manifest_text'].match(/ \d+:\d+:(\S+)/)[1]
assert_nil(assigns(:unsigned_user_agreements),
"Did not skip check_user_agreements filter " +
"when showing the user agreement.")
- assert_response :success
+ assert_response :redirect
end
test "requesting nonexistent Collection returns 404" do
:active, 404)
end
- test "use a reasonable read buffer even if client requests a huge range" do
- fakefiledata = mock
- IO.expects(:popen).returns(fakefiledata)
- fakefiledata.expects(:read).twice.with() do |length|
- # Fail the test if read() is called with length>1MiB:
- length < 2**20
- ## Force the ActionController::Live thread to lose the race to
- ## verify that @response.body.length actually waits for the
- ## response (see below):
- # sleep 3
- end.returns("foo\n", nil)
- fakefiledata.expects(:close)
- foo_file = api_fixture('collections')['foo_file']
- @request.headers['Range'] = 'bytes=0-4294967296/*'
- get :show_file, {
- uuid: foo_file['uuid'],
- file: foo_file['manifest_text'].match(/ \d+:\d+:(\S+)/)[1]
- }, session_for(:active)
- # Wait for the whole response to arrive before deciding whether
- # mocks' expectations were met. Otherwise, Mocha will fail the
- # test depending on how slowly the ActionController::Live thread
- # runs.
- @response.body.length
- end
-
test "show file in a subdirectory of a collection" do
+ setup_for_keep_web
params = collection_params(:collection_with_files_in_subdir, 'subdir2/subdir3/subdir4/file1_in_subdir4.txt')
- expect_content = stub_file_content
get(:show_file, params, session_for(:user1_with_load))
- assert_response :success
- assert_equal(expect_content, @response.body, "failed to get a correct file from Keep")
+ assert_response :redirect
+ assert_match /subdir2\/subdir3\/subdir4\/file1_in_subdir4\.txt/, response.redirect_url
end
test 'provenance graph' do
def setup_for_keep_web cfg='https://%{uuid_or_pdh}.example', dl_cfg=false
Rails.configuration.keep_web_url = cfg
Rails.configuration.keep_web_download_url = dl_cfg
- @controller.expects(:file_enumerator).never
end
%w(uuid portable_data_hash).each do |id_type|
require 'integration_helper'
class AnonymousAccessTest < ActionDispatch::IntegrationTest
+ include KeepWebConfig
+
# These tests don't do state-changing API calls. Save some time by
# skipping the database reset.
reset_api_fixtures :after_each_test, false
end
test 'view file' do
+ use_keep_web_config
+
magic = rand(2**512).to_s 36
- CollectionsController.any_instance.stubs(:file_enumerator).returns([magic])
- collection = api_fixture('collections')['public_text_file']
- visit '/collections/' + collection['uuid']
+ owner = api_fixture('groups')['anonymously_accessible_project']['uuid']
+ col = upload_data_and_get_collection(magic, 'admin', "Hello\\040world.txt", owner)
+ visit '/collections/' + col.uuid
find('tr,li', text: 'Hello world.txt').
find('a[title~=View]').click
assert_text magic
require_relative 'integration_test_utils'
class CollectionsTest < ActionDispatch::IntegrationTest
+ include KeepWebConfig
+
setup do
need_javascript
end
test "creating and uncreating a sharing link" do
coll_uuid = api_fixture("collections", "collection_owned_by_active", "uuid")
download_link_re =
- Regexp.new(Regexp.escape("/collections/download/#{coll_uuid}/"))
+ Regexp.new(Regexp.escape("/c=#{coll_uuid}/"))
visit page_with_token("active_trustedclient", "/collections/#{coll_uuid}")
within "#sharing-button" do
check_sharing(:on, download_link_re)
end
test "can download an entire collection with a reader token" do
- Capybara.current_driver = :rack_test
- CollectionsController.any_instance.
- stubs(:file_enumerator).returns(["foo\n", "file\n"])
- uuid = api_fixture('collections')['foo_file']['uuid']
+ use_keep_web_config
+
+ token = api_fixture('api_client_authorizations')['active']['api_token']
+ data = "foo\nfile\n"
+ datablock = `echo -n #{data.shellescape} | ARVADOS_API_TOKEN=#{token.shellescape} arv-put --no-progress --raw -`.strip
+ assert $?.success?, $?
+
+ col = nil
+ use_token 'active' do
+ mtxt = ". #{datablock} 0:#{data.length}:foo\n"
+ col = Collection.create(manifest_text: mtxt)
+ end
+
+ uuid = col.uuid
token = api_fixture('api_client_authorizations')['active_all_collections']['api_token']
url_head = "/collections/download/#{uuid}/#{token}/"
visit url_head
end
assert_equal(['foo'], hrefs.compact.sort,
"download page did provide strictly file links")
- within "#collection_files" do
- click_link "foo"
- assert_equal("foo\nfile\n", page.html)
- end
+ click_link "foo"
+ assert_text "foo\nfile\n"
end
test "combine selected collections into new collection" do
assert_selector 'a[href="/"]', text: 'Go to dashboard'
end
- test "view job log" do
- job = api_fixture('jobs')['job_with_real_log']
-
- IO.expects(:popen).returns(fakepipe_with_log_data)
-
- visit page_with_token("active", "/jobs/#{job['uuid']}")
- assert page.has_text? job['script_version']
-
- find(:xpath, "//a[@href='#Log']").click
- wait_for_ajax
- assert page.has_text? 'Started at'
- assert page.has_text? 'Finished at'
- assert page.has_text? 'log message 1'
- assert page.has_text? 'log message 2'
- assert page.has_text? 'log message 3'
- assert page.has_no_text? 'Showing only 100 bytes of this log'
- end
-
test 'view partial job log' do
+ need_selenium 'to be able to see the CORS response headers (PhantomJS 1.9.8 does not)'
+ use_keep_web_config
+
# This config will be restored during teardown by ../test_helper.rb:
Rails.configuration.log_viewer_max_bytes = 100
- IO.expects(:popen).returns(fakepipe_with_log_data)
- job = api_fixture('jobs')['job_with_real_log']
-
- visit page_with_token("active", "/jobs/#{job['uuid']}")
- assert page.has_text? job['script_version']
-
- find(:xpath, "//a[@href='#Log']").click
+ logdata = fakepipe_with_log_data.read
+ job_uuid = api_fixture('jobs')['running']['uuid']
+ logcollection = upload_data_and_get_collection(logdata, 'active', "#{job_uuid}.log.txt")
+ job = nil
+ use_token 'active' do
+ job = Job.find job_uuid
+ job.update_attributes log: logcollection.portable_data_hash
+ end
+ visit page_with_token 'active', '/jobs/'+job.uuid
+ find('a[href="#Log"]').click
wait_for_ajax
- assert page.has_text? 'Showing only 100 bytes of this log'
+ assert_text 'Showing only 100 bytes of this log'
end
test 'view log via keep-web redirect' do
@kwdport = getport 'keep-web-dl-ssl'
Rails.configuration.keep_web_url = "https://localhost:#{@kwport}/c=%{uuid_or_pdh}"
Rails.configuration.keep_web_download_url = "https://localhost:#{@kwdport}/c=%{uuid_or_pdh}"
- CollectionsController.any_instance.expects(:file_enumerator).never
end
end
end
end
end
+
+def upload_data_and_get_collection(data, user, filename, owner_uuid=nil)
+ token = api_fixture('api_client_authorizations')[user]['api_token']
+ datablock = `echo -n #{data.shellescape} | ARVADOS_API_TOKEN=#{token.shellescape} arv-put --no-progress --raw -`.strip
+ assert $?.success?, $?
+ col = nil
+ use_token user do
+ mtxt = ". #{datablock} 0:#{data.length}:#{filename}\n"
+ if owner_uuid
+ col = Collection.create(manifest_text: mtxt, owner_uuid: owner_uuid)
+ else
+ col = Collection.create(manifest_text: mtxt)
+ end
+ end
+ return col
+end
\cp config/application.yml.example config/application.yml -f
\cp config/environments/production.rb.example config/environments/production.rb -f
sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+ sed -i 's/keep_web_url: false/keep_web_url: exampledotcom/' config/application.yml
RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
// SSL certificates. See
// http://www.w3.org/TR/cors/#user-credentials).
w.Header().Set("Access-Control-Allow-Origin", "*")
+ w.Header().Set("Access-Control-Expose-Headers", "Content-Range")
}
arv := h.clientPool.Get()
projects / arvados.git / commitdiff