From 2e1892bbd21d7ac8432c923fa84eb8526cfae558 Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Fri, 30 Mar 2018 18:29:23 -0300 Subject: [PATCH] 12414: Add project deletion (with its contents) when delete_at is past Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- services/api/app/models/collection.rb | 4 +- services/api/lib/sweep_trashed_collections.rb | 41 ---------- services/api/lib/sweep_trashed_objects.rb | 74 +++++++++++++++++++ services/api/test/unit/collection_test.rb | 14 ++-- 4 files changed, 83 insertions(+), 50 deletions(-) delete mode 100644 services/api/lib/sweep_trashed_collections.rb create mode 100644 services/api/lib/sweep_trashed_objects.rb diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb index a088d48e68..4772768c8f 100644 --- a/services/api/app/models/collection.rb +++ b/services/api/app/models/collection.rb @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 require 'arvados/keep' -require 'sweep_trashed_collections' +require 'sweep_trashed_objects' require 'trashable' class Collection < ArvadosModel @@ -448,7 +448,7 @@ class Collection < ArvadosModel end def self.where *args - SweepTrashedCollections.sweep_if_stale + SweepTrashedObjects.sweep_if_stale super end diff --git a/services/api/lib/sweep_trashed_collections.rb b/services/api/lib/sweep_trashed_collections.rb deleted file mode 100644 index a899191db0..0000000000 --- a/services/api/lib/sweep_trashed_collections.rb +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -require 'current_api_client' - -module SweepTrashedCollections - extend CurrentApiClient - - def self.sweep_now - act_as_system_user do - Collection. - where('delete_at is not null and delete_at < statement_timestamp()'). - destroy_all - Collection. - where('is_trashed = false and trash_at < statement_timestamp()'). - update_all('is_trashed = true') - end - end - - def self.sweep_if_stale - return if Rails.configuration.trash_sweep_interval <= 0 - exp = Rails.configuration.trash_sweep_interval.seconds - need = false - Rails.cache.fetch('SweepTrashedCollections', expires_in: exp) do - need = true - end - if need - Thread.new do - Thread.current.abort_on_exception = false - begin - sweep_now - rescue => e - Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}" - ensure - ActiveRecord::Base.connection.close - end - end - end - end -end diff --git a/services/api/lib/sweep_trashed_objects.rb b/services/api/lib/sweep_trashed_objects.rb new file mode 100644 index 0000000000..1dc45a0ca1 --- /dev/null +++ b/services/api/lib/sweep_trashed_objects.rb @@ -0,0 +1,74 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require 'current_api_client' + +module SweepTrashedObjects + extend CurrentApiClient + + def delete_project_and_contents(p_uuid) + p = Group.find_by_uuid(p_uuid) + if !p || p.group_class != 'project' + raise "can't sweep group '#{p_uuid}', it may not exist or not be a project" + end + # First delete sub projects + Group.where({group_class: 'project', owner_uuid: p_uuid}).each do |sub_project| + delete_project_and_contents(sub_project.uuid) + end + # Next, iterate over all tables which have owner_uuid fields, with some + # exceptions, and delete records owned by this project + skipped_classes = ['Group', 'User'] + ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass| + if !skipped_classes.include?(klass.name) && klass.columns.collect{|c| c.name}.include?('owner_uuid') + klass.where({owner_uuid: p_uuid}).destroy_all + end + end + # Finally delete the project itself + p.destroy + end + + def self.sweep_now + act_as_system_user do + # Sweep trashed collections + Collection. + where('delete_at is not null and delete_at < statement_timestamp()'). + destroy_all + Collection. + where('is_trashed = false and trash_at < statement_timestamp()'). + update_all('is_trashed = true') + + # Sweep trashed projects and their contents + Group. + where({group_class: 'project'}). + where('delete_at is not null and delete_at < statement_timestamp()').each do |project| + delete_project_and_contents(project.uuid) + end + Group. + where({group_class: 'project'}). + where('is_trashed = false and trash_at < statement_timestamp()'). + update_all('is_trashed = true') + end + end + + def self.sweep_if_stale + return if Rails.configuration.trash_sweep_interval <= 0 + exp = Rails.configuration.trash_sweep_interval.seconds + need = false + Rails.cache.fetch('SweepTrashedObjects', expires_in: exp) do + need = true + end + if need + Thread.new do + Thread.current.abort_on_exception = false + begin + sweep_now + rescue => e + Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}" + ensure + ActiveRecord::Base.connection.close + end + end + end + end +end diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb index d425bc63c0..8b8c48fe1c 100644 --- a/services/api/test/unit/collection_test.rb +++ b/services/api/test/unit/collection_test.rb @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 require 'test_helper' -require 'sweep_trashed_collections' +require 'sweep_trashed_objects' class CollectionTest < ActiveSupport::TestCase include DbCurrentTime @@ -556,7 +556,7 @@ class CollectionTest < ActiveSupport::TestCase assert_includes(coll_uuids, collections(:docker_image).uuid) end - test "move to trash in SweepTrashedCollections" do + test "move collections to trash in SweepTrashedObjects" do c = collections(:trashed_on_next_sweep) refute_empty Collection.where('uuid=? and is_trashed=false', c.uuid) assert_raises(ActiveRecord::RecordNotUnique) do @@ -565,7 +565,7 @@ class CollectionTest < ActiveSupport::TestCase name: c.name) end end - SweepTrashedCollections.sweep_now + SweepTrashedObjects.sweep_now c = Collection.where('uuid=? and is_trashed=true', c.uuid).first assert c act_as_user users(:active) do @@ -574,14 +574,14 @@ class CollectionTest < ActiveSupport::TestCase end end - test "delete in SweepTrashedCollections" do + test "delete collections in SweepTrashedObjects" do uuid = 'zzzzz-4zz18-3u1p5umicfpqszp' # deleted_on_next_sweep assert_not_empty Collection.where(uuid: uuid) - SweepTrashedCollections.sweep_now + SweepTrashedObjects.sweep_now assert_empty Collection.where(uuid: uuid) end - test "delete referring links in SweepTrashedCollections" do + test "delete referring links in SweepTrashedObjects" do uuid = collections(:trashed_on_next_sweep).uuid act_as_system_user do Link.create!(head_uuid: uuid, @@ -593,7 +593,7 @@ class CollectionTest < ActiveSupport::TestCase Collection.where(uuid: uuid). update_all(is_trashed: true, trash_at: past, delete_at: past) assert_not_empty Collection.where(uuid: uuid) - SweepTrashedCollections.sweep_now + SweepTrashedObjects.sweep_now assert_empty Collection.where(uuid: uuid) end end -- 2.30.2