X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/96fefe6c6fc2e3bd7e4efbb0b6a7dedb9221aff7..40dc04d6fa45b6b727e94986491c06132db4a582:/services/api/test/unit/collection_test.rb diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb index 8deedee018..e7134a5be5 100644 --- a/services/api/test/unit/collection_test.rb +++ b/services/api/test/unit/collection_test.rb @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 require 'test_helper' -require 'sweep_trashed_objects' +require 'fix_collection_versions_timestamps' class CollectionTest < ActiveSupport::TestCase include DbCurrentTime @@ -157,8 +157,8 @@ class CollectionTest < ActiveSupport::TestCase end test "auto-create version after idle setting" do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 600 # 10 minutes + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 600 # 10 minutes act_as_user users(:active) do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -184,12 +184,29 @@ class CollectionTest < ActiveSupport::TestCase c.reload assert_equal 'foobar', c.name assert_equal 2, c.version + # Simulate a keep-balance run and trigger a new versionable update + # This tests bug #18005 + assert_nil c.replication_confirmed + assert_nil c.replication_confirmed_at + # Updates without validations/callbacks + c.update_column('modified_at', fifteen_min_ago) + c.update_column('replication_confirmed_at', Time.now) + c.update_column('replication_confirmed', 2) + c.reload + assert_equal fifteen_min_ago.to_i, c.modified_at.to_i + assert_not_nil c.replication_confirmed_at + assert_not_nil c.replication_confirmed + # Make the versionable update + c.update_attributes!({'name' => 'foobarbaz'}) + c.reload + assert_equal 'foobarbaz', c.name + assert_equal 3, c.version end end - test "preserve_version=false assignment is ignored while being true and not producing a new version" do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 3600 + test "preserve_version updates" do + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = -1 # disabled act_as_user users(:active) do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -198,28 +215,61 @@ class CollectionTest < ActiveSupport::TestCase assert_equal false, c.preserve_version # This update shouldn't produce a new version, as the idle time is not up c.update_attributes!({ - 'name' => 'bar', - 'preserve_version' => true + 'name' => 'bar' }) c.reload assert_equal 1, c.version assert_equal 'bar', c.name + assert_equal false, c.preserve_version + # This update should produce a new version, even if the idle time is not up + # and also keep the preserve_version=true flag to persist it. + c.update_attributes!({ + 'name' => 'baz', + 'preserve_version' => true + }) + c.reload + assert_equal 2, c.version + assert_equal 'baz', c.name assert_equal true, c.preserve_version # Make sure preserve_version is not disabled after being enabled, unless # a new version is created. + # This is a non-versionable update c.update_attributes!({ 'preserve_version' => false, 'replication_desired' => 2 }) c.reload - assert_equal 1, c.version + assert_equal 2, c.version assert_equal 2, c.replication_desired assert_equal true, c.preserve_version - c.update_attributes!({'name' => 'foobar'}) + # This is a versionable update + c.update_attributes!({ + 'preserve_version' => false, + 'name' => 'foobar' + }) c.reload - assert_equal 2, c.version + assert_equal 3, c.version assert_equal false, c.preserve_version assert_equal 'foobar', c.name + # Flipping only 'preserve_version' to true doesn't create a new version + c.update_attributes!({'preserve_version' => true}) + c.reload + assert_equal 3, c.version + assert_equal true, c.preserve_version + end + end + + test "preserve_version updates don't change modified_at timestamp" do + act_as_user users(:active) do + c = create_collection 'foo', Encoding::US_ASCII + assert c.valid? + assert_equal false, c.preserve_version + modified_at = c.modified_at.to_f + c.update_attributes!({'preserve_version' => true}) + c.reload + assert_equal true, c.preserve_version + assert_equal modified_at, c.modified_at.to_f, + 'preserve_version updates should not trigger modified_at changes' end end @@ -244,8 +294,8 @@ class CollectionTest < ActiveSupport::TestCase end test "uuid updates on current version make older versions update their pointers" do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_system_user do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -266,13 +316,75 @@ class CollectionTest < ActiveSupport::TestCase end end + # This test exposes a bug related to JSONB attributes, see #15725. + test "recently loaded collection shouldn't list changed attributes" do + col = Collection.where("properties != '{}'::jsonb").limit(1).first + refute col.properties_changed?, 'Properties field should not be seen as changed' + end + + [ + [ + true, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {:foo=>:bar, :lst=>[1, 3, 5, 7], :hsh=>{'baz'=>'qux', :foobar=>true, 'hsh'=>{:nested=>true}}, :delete_at=>nil}, + ], + [ + true, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {'delete_at'=>nil, 'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}}, + ], + [ + true, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {'delete_at'=>nil, 'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'foobar'=>true, 'hsh'=>{'nested'=>true}, 'baz'=>'qux'}}, + ], + [ + false, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 42], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + ], + [ + false, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {'foo'=>'bar', 'lst'=>[1, 3, 7, 5], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + ], + [ + false, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>false}}, 'delete_at'=>nil}, + ], + [ + false, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>nil}, + {'foo'=>'bar', 'lst'=>[1, 3, 5, 7], 'hsh'=>{'baz'=>'qux', 'foobar'=>true, 'hsh'=>{'nested'=>true}}, 'delete_at'=>1234567890}, + ], + ].each do |should_be_equal, value_1, value_2| + test "JSONB properties #{value_1} is#{should_be_equal ? '' : ' not'} equal to #{value_2}" do + act_as_user users(:active) do + # Set up initial collection + c = create_collection 'foo', Encoding::US_ASCII + assert c.valid? + c.update_attributes!({'properties' => value_1}) + c.reload + assert c.changes.keys.empty? + c.properties = value_2 + if should_be_equal + assert c.changes.keys.empty?, "Properties #{value_1.inspect} should be equal to #{value_2.inspect}" + else + refute c.changes.keys.empty?, "Properties #{value_1.inspect} should not be equal to #{value_2.inspect}" + end + end + end + end + test "older versions' modified_at indicate when they're created" do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_user users(:active) do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII assert c.valid? + original_version_modified_at = c.modified_at.to_f # Make changes so that a new version is created c.update_attributes!({'name' => 'bar'}) c.reload @@ -283,9 +395,7 @@ class CollectionTest < ActiveSupport::TestCase version_creation_datetime = c_old.modified_at.to_f assert_equal c.created_at.to_f, c_old.created_at.to_f - # Current version is updated just a few milliseconds before the version is - # saved on the database. - assert_operator c.modified_at.to_f, :<, version_creation_datetime + assert_equal original_version_modified_at, version_creation_datetime # Make update on current version so old version get the attribute synced; # its modified_at should not change. @@ -300,9 +410,32 @@ class CollectionTest < ActiveSupport::TestCase end end + # Bug #17152 - This test relies on fixtures simulating the problem. + test "migration fixing collection versions' modified_at timestamps" do + versioned_collection_fixtures = [ + collections(:w_a_z_file).uuid, + collections(:collection_owned_by_active).uuid + ] + versioned_collection_fixtures.each do |uuid| + cols = Collection.where(current_version_uuid: uuid).order(version: :desc) + assert_equal cols.size, 2 + # cols[0] -> head version // cols[1] -> old version + assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :==, 0 + assert cols[1].modified_at != cols[1].created_at + end + fix_collection_versions_timestamps + versioned_collection_fixtures.each do |uuid| + cols = Collection.where(current_version_uuid: uuid).order(version: :desc) + assert_equal cols.size, 2 + # cols[0] -> head version // cols[1] -> old version + assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :>, 1 + assert_operator cols[1].modified_at, :==, cols[1].created_at + end + end + test "past versions should not be directly updatable" do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_system_user do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -324,7 +457,7 @@ class CollectionTest < ActiveSupport::TestCase assert c_old.invalid? c_old.reload # Now disable collection versioning, it should behave the same way - Rails.configuration.collection_versioning = false + Rails.configuration.Collections.CollectionVersioning = false c_old.name = 'this was foo' assert c_old.invalid? end @@ -334,11 +467,10 @@ class CollectionTest < ActiveSupport::TestCase ['owner_uuid', 'zzzzz-tpzed-d9tiejq69daie8f', 'zzzzz-tpzed-xurymjxw79nv3jz'], ['replication_desired', 2, 3], ['storage_classes_desired', ['hot'], ['archive']], - ['is_trashed', true, false], ].each do |attr, first_val, second_val| test "sync #{attr} with older versions" do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_system_user do # Set up initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -379,8 +511,8 @@ class CollectionTest < ActiveSupport::TestCase [false, 'replication_desired', 5, false], ].each do |versioning, attr, val, new_version_expected| test "update #{attr} with versioning #{versioning ? '' : 'not '}enabled should #{new_version_expected ? '' : 'not '}create a new version" do - Rails.configuration.collection_versioning = versioning - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = versioning + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_user users(:active) do # Create initial collection c = create_collection 'foo', Encoding::US_ASCII @@ -414,8 +546,8 @@ class CollectionTest < ActiveSupport::TestCase end test 'current_version_uuid is ignored during update' do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_user users(:active) do # Create 1st collection col1 = create_collection 'foo', Encoding::US_ASCII @@ -439,8 +571,8 @@ class CollectionTest < ActiveSupport::TestCase end test 'with versioning enabled, simultaneous updates increment version correctly' do - Rails.configuration.collection_versioning = true - Rails.configuration.preserve_version_if_idle = 0 + Rails.configuration.Collections.CollectionVersioning = true + Rails.configuration.Collections.PreserveVersionIfIdle = 0 act_as_user users(:active) do # Create initial collection col = create_collection 'foo', Encoding::US_ASCII @@ -577,6 +709,19 @@ class CollectionTest < ActiveSupport::TestCase end end + test "storage_classes_desired default respects config" do + saved = Rails.configuration.DefaultStorageClasses + Rails.configuration.DefaultStorageClasses = ["foo"] + begin + act_as_user users(:active) do + c = Collection.create! + assert_equal ["foo"], c.storage_classes_desired + end + ensure + Rails.configuration.DefaultStorageClasses = saved + end + end + test "storage_classes_desired cannot be empty" do act_as_user users(:active) do c = collections(:collection_owned_by_active) @@ -654,7 +799,7 @@ class CollectionTest < ActiveSupport::TestCase [0, 2, 4, nil].each do |ask| test "set replication_desired to #{ask.inspect}" do - Rails.configuration.default_collection_replication = 2 + Rails.configuration.Collections.DefaultReplication = 2 act_as_user users(:active) do c = collections(:replication_undesired_unconfirmed) c.update_attributes replication_desired: ask @@ -710,7 +855,7 @@ class CollectionTest < ActiveSupport::TestCase test "clear replication_confirmed* when introducing a new block in manifest" do c = collections(:replication_desired_2_confirmed_2) act_as_user users(:active) do - assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text) + assert c.update_attributes(manifest_text: collections(:user_agreement).signed_manifest_text_only_for_tests) assert_nil c.replication_confirmed assert_nil c.replication_confirmed_at end @@ -719,7 +864,7 @@ class CollectionTest < ActiveSupport::TestCase test "don't clear replication_confirmed* when just renaming a file" do c = collections(:replication_desired_2_confirmed_2) act_as_user users(:active) do - new_manifest = c.signed_manifest_text.sub(':bar', ':foo') + new_manifest = c.signed_manifest_text_only_for_tests.sub(':bar', ':foo') assert c.update_attributes(manifest_text: new_manifest) assert_equal 2, c.replication_confirmed assert_not_nil c.replication_confirmed_at @@ -729,13 +874,13 @@ class CollectionTest < ActiveSupport::TestCase test "don't clear replication_confirmed* when just deleting a data block" do c = collections(:replication_desired_2_confirmed_2) act_as_user users(:active) do - new_manifest = c.signed_manifest_text + new_manifest = c.signed_manifest_text_only_for_tests new_manifest.sub!(/ \S+:bar/, '') new_manifest.sub!(/ acbd\S+/, '') # Confirm that we did just remove a block from the manifest (if # not, this test would pass without testing the relevant case): - assert_operator new_manifest.length+40, :<, c.signed_manifest_text.length + assert_operator new_manifest.length+40, :<, c.signed_manifest_text_only_for_tests.length assert c.update_attributes(manifest_text: new_manifest) assert_equal 2, c.replication_confirmed @@ -749,7 +894,7 @@ class CollectionTest < ActiveSupport::TestCase c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n", name: 'foo') c.update_attributes! trash_at: (t0 + 1.hours) c.reload - sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i + sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i assert_operator sig_exp.to_i, :<=, (t0 + 1.hours).to_i end end @@ -759,8 +904,8 @@ class CollectionTest < ActiveSupport::TestCase c = Collection.create!(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:x\n", name: 'foo', trash_at: db_current_time + 1.years) - sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text)[1].to_i - expect_max_sig_exp = db_current_time.to_i + Rails.configuration.blob_signature_ttl + sig_exp = /\+A[0-9a-f]{40}\@([0-9]+)/.match(c.signed_manifest_text_only_for_tests)[1].to_i + expect_max_sig_exp = db_current_time.to_i + Rails.configuration.Collections.BlobSigningTTL.to_i assert_operator c.trash_at.to_i, :>, expect_max_sig_exp assert_operator sig_exp.to_i, :<=, expect_max_sig_exp end @@ -849,7 +994,7 @@ class CollectionTest < ActiveSupport::TestCase test test_name do act_as_user users(:active) do min_exp = (db_current_time + - Rails.configuration.blob_signature_ttl.seconds) + Rails.configuration.Collections.BlobSigningTTL) if fixture_name == :expired_collection # Fixture-finder shorthand doesn't find trashed collections # because they're not in the default scope. @@ -890,7 +1035,7 @@ class CollectionTest < ActiveSupport::TestCase end test 'default trash interval > blob signature ttl' do - Rails.configuration.default_trash_lifetime = 86400 * 21 # 3 weeks + Rails.configuration.Collections.DefaultTrashLifetime = 86400 * 21 # 3 weeks start = db_current_time act_as_user users(:active) do c = Collection.create!(manifest_text: '', name: 'foo') @@ -912,44 +1057,102 @@ class CollectionTest < ActiveSupport::TestCase assert_includes(coll_uuids, collections(:docker_image).uuid) end - test "move collections to trash in SweepTrashedObjects" do - c = collections(:trashed_on_next_sweep) - refute_empty Collection.where('uuid=? and is_trashed=false', c.uuid) - assert_raises(ActiveRecord::RecordNotUnique) do - act_as_user users(:active) do - Collection.create!(owner_uuid: c.owner_uuid, - name: c.name) + test "empty names are exempt from name uniqueness" do + act_as_user users(:active) do + c1 = Collection.new(name: nil, manifest_text: '', owner_uuid: groups(:aproject).uuid) + assert c1.save + c2 = Collection.new(name: '', manifest_text: '', owner_uuid: groups(:aproject).uuid) + assert c2.save + c3 = Collection.new(name: '', manifest_text: '', owner_uuid: groups(:aproject).uuid) + assert c3.save + c4 = Collection.new(name: 'c4', manifest_text: '', owner_uuid: groups(:aproject).uuid) + assert c4.save + c5 = Collection.new(name: 'c4', manifest_text: '', owner_uuid: groups(:aproject).uuid) + assert_raises(ActiveRecord::RecordNotUnique) do + c5.save end end - SweepTrashedObjects.sweep_now - c = Collection.where('uuid=? and is_trashed=true', c.uuid).first - assert c + end + + test "create collections with managed properties" do + Rails.configuration.Collections.ManagedProperties = ConfigLoader.to_OrderedOptions({ + 'default_prop1' => {'Value' => 'prop1_value'}, + 'responsible_person_uuid' => {'Function' => 'original_owner'} + }) + # Test collection without initial properties + act_as_user users(:active) do + c = create_collection 'foo', Encoding::US_ASCII + assert c.valid? + assert_not_empty c.properties + assert_equal 'prop1_value', c.properties['default_prop1'] + assert_equal users(:active).uuid, c.properties['responsible_person_uuid'] + end + # Test collection with default_prop1 property already set act_as_user users(:active) do - assert Collection.create!(owner_uuid: c.owner_uuid, - name: c.name) + c = Collection.create(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n", + properties: {'default_prop1' => 'custom_value'}) + assert c.valid? + assert_not_empty c.properties + assert_equal 'custom_value', c.properties['default_prop1'] + assert_equal users(:active).uuid, c.properties['responsible_person_uuid'] + end + # Test collection inside a sub project + act_as_user users(:active) do + c = Collection.create(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n", + owner_uuid: groups(:asubproject).uuid) + assert c.valid? + assert_not_empty c.properties + assert_equal users(:active).uuid, c.properties['responsible_person_uuid'] end end - test "delete collections in SweepTrashedObjects" do - uuid = 'zzzzz-4zz18-3u1p5umicfpqszp' # deleted_on_next_sweep - assert_not_empty Collection.where(uuid: uuid) - SweepTrashedObjects.sweep_now - assert_empty Collection.where(uuid: uuid) + test "update collection with protected managed properties" do + Rails.configuration.Collections.ManagedProperties = ConfigLoader.to_OrderedOptions({ + 'default_prop1' => {'Value' => 'prop1_value', 'Protected' => true}, + }) + act_as_user users(:active) do + c = create_collection 'foo', Encoding::US_ASCII + assert c.valid? + assert_not_empty c.properties + assert_equal 'prop1_value', c.properties['default_prop1'] + # Add new property + c.properties['prop2'] = 'value2' + c.save! + c.reload + assert_equal 'value2', c.properties['prop2'] + # Try to change protected property's value + c.properties['default_prop1'] = 'new_value' + assert_raises(ArvadosModel::PermissionDeniedError) do + c.save! + end + # Admins are allowed to change protected properties + act_as_system_user do + c.properties['default_prop1'] = 'new_value' + c.save! + c.reload + assert_equal 'new_value', c.properties['default_prop1'] + end + end end - test "delete referring links in SweepTrashedObjects" do - uuid = collections(:trashed_on_next_sweep).uuid - act_as_system_user do - Link.create!(head_uuid: uuid, - tail_uuid: system_user_uuid, - link_class: 'whatever', - name: 'something') - end - past = db_current_time - Collection.where(uuid: uuid). - update_all(is_trashed: true, trash_at: past, delete_at: past) - assert_not_empty Collection.where(uuid: uuid) - SweepTrashedObjects.sweep_now - assert_empty Collection.where(uuid: uuid) + test "collection names must be displayable in a filesystem" do + set_user_from_auth :active + ["", "{SOLIDUS}"].each do |subst| + Rails.configuration.Collections.ForwardSlashNameSubstitution = subst + c = Collection.create + [[nil, true], + ["", true], + [".", false], + ["..", false], + ["...", true], + ["..z..", true], + ["foo/bar", subst != ""], + ["../..", subst != ""], + ["/", subst != ""], + ].each do |name, valid| + c.name = name + assert_equal valid, c.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}" + end + end end end