sharing_popup
end
- def update
- @updates ||= params[@object.resource_param_name.to_sym]
- if @updates && (@updates.keys - ["name", "description"]).empty?
- # exclude manifest_text since only name or description is being updated
- @object.manifest_text = nil
- end
- super
- end
-
protected
def find_usable_token(token_list)
# Use system CA certificates
@api_client.ssl_config.add_trust_ca('/etc/ssl/certs')
end
+ if Rails.configuration.api_response_compression
+ @api_client.transparent_gzip_decompression = true
+ end
end
end
end
header = {"Accept" => "application/json"}
- if Rails.configuration.include_accept_encoding_header_in_api_requests
- header["Accept-Encoding"] = "gzip, deflate"
- end
- profile_checkpoint { "Prepare request #{url} #{query[:uuid]} #{query[:where]} #{query[:filters]} #{query[:order]}" }
+ profile_checkpoint { "Prepare request #{query["_method"] or "POST"} #{url} #{query[:uuid]} #{query.inspect[0,256]}" }
msg = @client_mtx.synchronize do
begin
@api_client.post(url, query, header: header)
end
end
profile_checkpoint 'API transaction'
+ if @@profiling_enabled
+ if msg.headers['X-Runtime']
+ Rails.logger.info "API server: #{msg.headers['X-Runtime']} runtime reported"
+ end
+ Rails.logger.info "Content-Encoding #{msg.headers['Content-Encoding'].inspect}, Content-Length #{msg.headers['Content-Length'].inspect}, actual content size #{msg.content.size}"
+ end
begin
resp = Oj.load(msg.content, :symbol_keys => true)
'modified_by_client_uuid' => '203',
'uuid' => '999',
}
+ @loaded_attributes = {}
end
def self.columns
@columns << column(k, :text)
serialize k, coldef[:type].constantize
end
+ define_method k do
+ unless new_record? or @loaded_attributes.include? k
+ raise ActiveModel::MissingAttributeError, "missing attribute: #{k}"
+ end
+ super
+ end
@attribute_info[k] = coldef
end
end
def save
obdata = {}
self.class.columns.each do |col|
- unless self.send(col.name.to_sym).nil? and !self.changed.include?(col.name)
- obdata[col.name.to_sym] = self.send(col.name.to_sym)
+ # Non-nil serialized values must be sent because we can't tell
+ # whether they've changed. Other than that, any given attribute
+ # is either unchanged (in which case there's no need to send its
+ # old value in the update/create command) or has been added to
+ # #changed by ActiveRecord's #attr= method.
+ if changed.include? col.name or
+ (self.class.serialized_attributes.include? col.name and
+ @loaded_attributes[col.name])
+ obdata[col.name.to_sym] = self.send col.name
end
end
obdata.delete :id
end
@new_record = false
+ changes_applied
self
end
hash = arvados_api_client.api(self.class, '/' + uuid_or_hash)
end
hash.each do |k,v|
+ @loaded_attributes[k.to_s] = true
if self.respond_to?(k.to_s + '=')
self.send(k.to_s + '=', v)
else
end
@all_links = nil
@new_record = false
+ changes_applied
self
end
# in the directory where your API server is running.
anonymous_user_token: false
- # Enable response payload compression in Arvados API requests.
- include_accept_encoding_header_in_api_requests: true
+ # Ask Arvados API server to compress its response payloads.
+ api_response_compression: true
}, session_for(:active)
assert_response :success
assert_not_nil assigns(:object)
+ # Ensure the Workbench response still has the original manifest_text
assert_equal 'test description update', assigns(:object).description
assert_equal collection['manifest_text'], assigns(:object).manifest_text
+ # Ensure the API server still has the original manifest_text after
+ # we called arvados.v1.collections.update
+ use_token :active do
+ assert_equal(Collection.find(collection['uuid']).manifest_text,
+ collection['manifest_text'])
+ end
end
test "view collection and verify none of the file types listed are disabled" do
--- /dev/null
+../../../../services/api/test/helpers/manifest_examples.rb
\ No newline at end of file
--- /dev/null
+../../../../services/api/test/helpers/time_block.rb
\ No newline at end of file
--- /dev/null
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class Blob
+end
+
+class BigCollectionTest < ActiveSupport::TestCase
+ include ManifestExamples
+
+ setup do
+ Blob.stubs(:sign_locator).returns 'd41d8cd98f00b204e9800998ecf8427e+0'
+ end
+
+ teardown do
+ Thread.current[:arvados_api_client] = nil
+ end
+
+ # You can try with compress=false here too, but at last check it
+ # didn't make a significant difference.
+ [true].each do |compress|
+ test "crud cycle for collection with big manifest (compress=#{compress})" do
+ Rails.configuration.api_response_compression = compress
+ Thread.current[:arvados_api_client] = nil
+ crudtest
+ end
+ end
+
+ def crudtest
+ use_token :active
+ bigmanifest = time_block 'build example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 0)
+ end
+ c = time_block "new (manifest size = #{bigmanifest.length>>20}MiB)" do
+ Collection.new manifest_text: bigmanifest
+ end
+ time_block 'create' do
+ c.save!
+ end
+ time_block 'read' do
+ Collection.find c.uuid
+ end
+ time_block 'read(cached)' do
+ Collection.find c.uuid
+ end
+ time_block 'list' do
+ list = Collection.select(['uuid', 'manifest_text']).filter [['uuid','=',c.uuid]]
+ assert_equal 1, list.count
+ assert_equal c.uuid, list.first.uuid
+ assert_not_nil list.first.manifest_text
+ end
+ time_block 'update(name-only)' do
+ manifest_text_length = c.manifest_text.length
+ c.update_attributes name: 'renamed during test case'
+ assert_equal c.manifest_text.length, manifest_text_length
+ end
+ time_block 'update' do
+ c.manifest_text += ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:empty.txt\n"
+ c.save!
+ end
+ time_block 'delete' do
+ c.destroy
+ end
+ time_block 'read(404)' do
+ assert_empty Collection.filter([['uuid','=',c.uuid]])
+ end
+ end
+end
--- /dev/null
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class Blob
+end
+
+class BigCollectionsControllerTest < ActionController::TestCase
+ include ManifestExamples
+
+ setup do
+ Blob.stubs(:sign_locator).returns 'd41d8cd98f00b204e9800998ecf8427e+0'
+ end
+
+ test "combine two big and two small collections" do
+ @controller = ActionsController.new
+ bigmanifest1 = time_block 'build example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 0)
+ end
+ bigmanifest2 = bigmanifest1.gsub '.txt', '.txt2'
+ smallmanifest1 = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:small1.txt\n"
+ smallmanifest2 = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:small2.txt\n"
+ totalsize = bigmanifest1.length + bigmanifest2.length +
+ smallmanifest1.length + smallmanifest2.length
+ parts = time_block "create (total #{totalsize>>20}MiB)" do
+ use_token :active do
+ {
+ big1: Collection.create(manifest_text: bigmanifest1),
+ big2: Collection.create(manifest_text: bigmanifest2),
+ small1: Collection.create(manifest_text: smallmanifest1),
+ small2: Collection.create(manifest_text: smallmanifest2),
+ }
+ end
+ end
+ time_block 'combine' do
+ post :combine_selected_files_into_collection, {
+ selection: [parts[:big1].uuid,
+ parts[:big2].uuid,
+ parts[:small1].uuid + '/small1.txt',
+ parts[:small2].uuid + '/small2.txt',
+ ],
+ format: :html
+ }, session_for(:active)
+ end
+ assert_response :redirect
+ end
+
+ [:json, :html].each do |format|
+ test "show collection with big manifest (#{format})" do
+ bigmanifest = time_block 'build example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 0)
+ end
+ @controller = CollectionsController.new
+ c = time_block "create (manifest size #{bigmanifest.length>>20}MiB)" do
+ use_token :active do
+ Collection.create(manifest_text: bigmanifest)
+ end
+ end
+ time_block 'show' do
+ get :show, {id: c.uuid, format: format}, session_for(:active)
+ end
+ assert_response :success
+ end
+ end
+end
end
module ApiMockHelpers
- def stub_api_calls_with_body body, status_code=200
+ def fake_api_response body, status_code, headers
resp = mock
- stubbed_client = ArvadosApiClient.new
- stubbed_client.instance_eval do
- resp.responds_like_instance_of HTTP::Message
- resp.stubs(:content).returns body
- resp.stubs(:status_code).returns status_code
+ resp.responds_like_instance_of HTTP::Message
+ resp.stubs(:headers).returns headers
+ resp.stubs(:content).returns body
+ resp.stubs(:status_code).returns status_code
+ resp
+ end
+
+ def stub_api_calls_with_body body, status_code=200, headers={}
+ stub_api_calls
+ resp = fake_api_response body, status_code, headers
+ stub_api_client.stubs(:post).returns resp
+ end
+
+ def stub_api_calls
+ @stubbed_client = ArvadosApiClient.new
+ @stubbed_client.instance_eval do
@api_client = HTTPClient.new
- @api_client.stubs(:post).returns resp
end
- ArvadosApiClient.stubs(:new_or_current).returns(stubbed_client)
+ ArvadosApiClient.stubs(:new_or_current).returns(@stubbed_client)
end
def stub_api_calls_with_invalid_json
stub_api_calls_with_body ']"omg,bogus"['
end
+
+ # Return the HTTPClient mock used by the ArvadosApiClient mock. You
+ # must have called stub_api_calls first.
+ def stub_api_client
+ @stubbed_client.instance_eval do
+ @api_client
+ end
+ end
end
class ActiveSupport::TestCase
--- /dev/null
+require 'test_helper'
+
+class ArvadosBaseTest < ActiveSupport::TestCase
+ test '#save does not send unchanged string attributes' do
+ use_token :active do
+ fixture = api_fixture("collections")["foo_collection_in_aproject"]
+ c = Collection.find(fixture['uuid'])
+
+ new_name = 'name changed during test'
+
+ got_query = nil
+ stub_api_calls
+ stub_api_client.expects(:post).with do |url, query, opts={}|
+ got_query = query
+ true
+ end.returns fake_api_response('{}', 200, {})
+ c.name = new_name
+ c.save
+
+ updates = JSON.parse got_query['collection']
+ assert_equal updates['name'], new_name
+ refute_includes updates, 'description'
+ refute_includes updates, 'manifest_text'
+ end
+ end
+
+ test '#save does not send unchanged attributes missing because of select' do
+ use_token :active do
+ fixture = api_fixture("collections")["foo_collection_in_aproject"]
+ c = Collection.
+ filter([['uuid','=',fixture['uuid']]]).
+ select(['uuid']).
+ first
+ assert_equal nil, c.properties
+
+ got_query = nil
+ stub_api_calls
+ stub_api_client.expects(:post).with do |url, query, opts={}|
+ got_query = query
+ true
+ end.returns fake_api_response('{}', 200, {})
+ c.name = 'foo'
+ c.save
+
+ updates = JSON.parse got_query['collection']
+ assert_includes updates, 'name'
+ refute_includes updates, 'description'
+ refute_includes updates, 'properties'
+ end
+ end
+
+ [false,
+ {},
+ {'foo' => 'bar'},
+ ].each do |init_props|
+ test "#save sends serialized attributes if changed from #{init_props}" do
+ use_token :active do
+ fixture = api_fixture("collections")["foo_collection_in_aproject"]
+ c = Collection.find(fixture['uuid'])
+
+ if init_props
+ c.properties = init_props if init_props
+ c.save!
+ end
+
+ got_query = nil
+ stub_api_calls
+ stub_api_client.expects(:post).with do |url, query, opts={}|
+ got_query = query
+ true
+ end.returns fake_api_response('{"etag":"fake","uuid":"fake"}', 200, {})
+
+ c.properties['baz'] = 'qux'
+ c.save!
+
+ updates = JSON.parse got_query['collection']
+ assert_includes updates, 'properties'
+ end
+ end
+ end
+end
# Blob.sign_locator: return a signed and timestamped blob locator.
#
# The 'opts' argument should include:
- # [required] :key - the Arvados server-side blobstore key
- # [required] :api_token - user's API token
+ # [required] :api_token - API token (signatures only work for this token)
+ # [optional] :key - the Arvados server-side blobstore key
# [optional] :ttl - number of seconds before signature should expire
# [optional] :expire - unix timestamp when signature should expire
#
end
timestamp = opts[:expire]
else
- timestamp = db_current_time.to_i + (opts[:ttl] || 1209600)
+ timestamp = db_current_time.to_i +
+ (opts[:ttl] || Rails.configuration.blob_signature_ttl)
end
timestamp_hex = timestamp.to_s(16)
# => "53163cb4"
# Generate a signature.
signature =
- generate_signature opts[:key], blob_hash, opts[:api_token], timestamp_hex
+ generate_signature((opts[:key] or Rails.configuration.blob_signing_key),
+ blob_hash, opts[:api_token], timestamp_hex)
blob_locator + '+A' + signature + '@' + timestamp_hex
end
end
my_signature =
- generate_signature opts[:key], blob_hash, opts[:api_token], timestamp
+ generate_signature((opts[:key] or Rails.configuration.blob_signing_key),
+ blob_hash, opts[:api_token], timestamp)
if my_signature != given_signature
raise Blob::InvalidSignatureError.new 'Signature is invalid.'
# subsequent passes without checking any signatures. This is
# important because the signatures have probably been stripped off
# by the time we get to a second validation pass!
- return true if @signatures_checked and @signatures_checked == compute_pdh
+ computed_pdh = compute_pdh
+ return true if @signatures_checked and @signatures_checked == computed_pdh
if self.manifest_text_changed?
# Check permissions on the collection manifest.
# which will return 403 Permission denied to the client.
api_token = current_api_client_authorization.andand.api_token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: api_token,
now: db_current_time.to_i,
}
end
end
end
- @signatures_checked = compute_pdh
+ @signatures_checked = computed_pdh
end
def strip_manifest_text
def self.sign_manifest manifest, token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: token,
expire: db_current_time.to_i + Rails.configuration.blob_signature_ttl,
}
--- /dev/null
+module ManifestExamples
+ def make_manifest opts={}
+ opts = {
+ bytes_per_block: 1,
+ blocks_per_file: 1,
+ files_per_stream: 1,
+ streams: 1,
+ }.merge(opts)
+ datablip = "x" * opts[:bytes_per_block]
+ locator = Blob.sign_locator(Digest::MD5.hexdigest(datablip) +
+ '+' + datablip.length.to_s,
+ api_token: opts[:api_token])
+ filesize = datablip.length * opts[:blocks_per_file]
+ txt = ''
+ (1..opts[:streams]).each do |s|
+ streamtoken = "./stream#{s}"
+ streamsize = 0
+ blocktokens = []
+ filetokens = []
+ (1..opts[:files_per_stream]).each do |f|
+ filetokens << " #{streamsize}:#{filesize}:file#{f}.txt"
+ (1..opts[:blocks_per_file]).each do |b|
+ blocktokens << locator
+ end
+ streamsize += filesize
+ end
+ txt << ([streamtoken] + blocktokens + filetokens).join(' ') + "\n"
+ end
+ txt
+ end
+end
--- /dev/null
+class ActiveSupport::TestCase
+ def time_block label
+ t0 = Time.now
+ begin
+ yield
+ ensure
+ t1 = Time.now
+ $stderr.puts "#{t1 - t0}s #{label}"
+ end
+ end
+end
--- /dev/null
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class CollectionsApiPerformanceTest < ActionDispatch::IntegrationTest
+ include ManifestExamples
+
+ test "crud cycle for a collection with a big manifest" do
+ bigmanifest = time_block 'make example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 2**26,
+ api_token: api_token(:active))
+ end
+ json = time_block "JSON encode #{bigmanifest.length>>20}MiB manifest" do
+ Oj.dump({manifest_text: bigmanifest})
+ end
+ time_block 'create' do
+ post '/arvados/v1/collections', {collection: json}, auth(:active)
+ assert_response :success
+ end
+ uuid = json_response['uuid']
+ time_block 'read' do
+ get '/arvados/v1/collections/' + uuid, {}, auth(:active)
+ assert_response :success
+ end
+ time_block 'list' do
+ get '/arvados/v1/collections', {select: ['manifest_text'], filters: [['uuid', '=', uuid]].to_json}, auth(:active)
+ assert_response :success
+ end
+ time_block 'update' do
+ put '/arvados/v1/collections/' + uuid, {collection: json}, auth(:active)
+ assert_response :success
+ end
+ time_block 'delete' do
+ delete '/arvados/v1/collections/' + uuid, {}, auth(:active)
+ end
+ end
+end
--- /dev/null
+require 'test_helper'
+require 'helpers/manifest_examples'
+require 'helpers/time_block'
+
+class CollectionModelPerformanceTest < ActiveSupport::TestCase
+ include ManifestExamples
+
+ setup do
+ # The Collection model needs to have a current token, not just a
+ # current user, to sign & verify manifests:
+ Thread.current[:api_client_authorization] =
+ api_client_authorizations(:active)
+ end
+
+ teardown do
+ Thread.current[:api_client_authorization] = nil
+ end
+
+ # "crrud" == "create read render update delete", not a typo
+ test "crrud cycle for a collection with a big manifest)" do
+ bigmanifest = time_block 'make example' do
+ make_manifest(streams: 100,
+ files_per_stream: 100,
+ blocks_per_file: 20,
+ bytes_per_block: 2**26,
+ api_token: api_token(:active))
+ end
+ act_as_user users(:active) do
+ c = time_block "new (manifest_text is #{bigmanifest.length>>20}MiB)" do
+ Collection.new manifest_text: bigmanifest.dup
+ end
+ time_block 'check signatures' do
+ c.check_signatures
+ end
+ time_block 'check signatures + save' do
+ c.save!
+ end
+ c = time_block 'read' do
+ Collection.find_by_uuid(c.uuid)
+ end
+ time_block 'sign' do
+ c.signed_manifest_text
+ end
+ time_block 'sign + render' do
+ resp = c.as_api_response(nil)
+ end
+ loc = Blob.sign_locator(Digest::MD5.hexdigest('foo') + '+3',
+ api_token: api_token(:active))
+ # Note Collection's strip_manifest_text method has now removed
+ # the signatures from c.manifest_text, so we have to start from
+ # bigmanifest again here instead of just appending with "+=".
+ c.manifest_text = bigmanifest.dup + ". #{loc} 0:3:foo.txt\n"
+ time_block 'update' do
+ c.save!
+ end
+ time_block 'delete' do
+ c.destroy
+ end
+ end
+ end
+end
projects / arvados.git / commitdiff