9514: add rake task to delete logs for old containers whose log is already stored...

diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 99d2a104279db1558ebc380b9ed3fb2e623ac7ce..96e7596e8bda000c65d25393fe70d91761b9c04e 100644 (file)
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -220,6 +220,12 @@ common:
   # stderr logs from the logs table.
   clean_job_log_rows_after: <%= 30.days %>
 
+  # When you run the db:delete_old_container_logs task, it will find
+  # containers that have been finished for at least this many seconds,
+  # and delete their stdout, stderr, arv-mount, crunch-run, and
+  # crunchstat logs from the logs table.
+  clean_container_log_rows_after: <%= 30.days %>
+
   # The maximum number of compute nodes that can be in use simultaneously
   # If this limit is reduced, any existing nodes with slot number >= new limit
   # will not be counted against the new limit. In other words, the new limit

diff --git a/services/api/lib/tasks/delete_old_container_logs.rake b/services/api/lib/tasks/delete_old_container_logs.rake
new file mode 100644 (file)
index 0000000..8c32331
--- /dev/null
+++ b/services/api/lib/tasks/delete_old_container_logs.rake
@@ -0,0 +1,17 @@
+# This task finds containers that have been finished for at least as long as
+# the duration specified in the `clean_container_log_rows_after` config setting,
+# and deletes their stdout, stderr, arv-mount, crunch-run, and  crunchstat logs
+# from the logs table.
+
+namespace :db do
+  desc "Remove old container log entries from the logs table"
+  task delete_old_container_logs: :environment do
+    Log.select("logs.id").
+        joins("JOIN containers ON object_uuid = containers.uuid").
+        where("event_type in ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND containers.finished_at < :age",
+              age: Rails.configuration.clean_container_log_rows_after.ago).
+        find_in_batches do |old_log_ids|
+      Log.where(id: old_log_ids.map(&:id)).delete_all
+    end
+  end
+end

diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index 8e1a3ff6799354334f768d5b4b581af1b7291c9e..87098dcf53e96e693635bfd8ce355dbb2f012b36 100644 (file)
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -136,7 +136,7 @@ requester_container:
   auth_uuid: zzzzz-gj3su-077z32aux8dg2s1
 
 failed_container:
-  uuid: zzzzz-dz642-failedcntnr
+  uuid: zzzzz-dz642-failedcontainr1
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   state: Complete
   exit_code: 33
@@ -151,3 +151,56 @@ failed_container:
   runtime_constraints:
     ram: 12000000000
     vcpus: 4
+
+ancient_container_with_logs:
+  uuid: zzzzz-dz642-logscontainer01
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  state: Complete
+  exit_code: 0
+  priority: 1
+  created_at: <%= 2.year.ago.to_s(:db) %>
+  updated_at: <%= 2.year.ago.to_s(:db) %>
+  container_image: test
+  cwd: test
+  output_path: test
+  command: ["echo", "hello"]
+  runtime_constraints:
+    ram: 12000000000
+    vcpus: 4
+  finished_at: <%= 2.year.ago.to_s(:db) %>
+  log: ea10d51bcf88862dbcc36eb292017dfd+45
+  output: test
+
+previous_container_with_logs:
+  uuid: zzzzz-dz642-logscontainer02
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  state: Complete
+  exit_code: 0
+  priority: 1
+  created_at: <%= 1.month.ago.to_s(:db) %>
+  updated_at: <%= 1.month.ago.to_s(:db) %>
+  container_image: test
+  cwd: test
+  output_path: test
+  command: ["echo", "hello"]
+  runtime_constraints:
+    ram: 12000000000
+    vcpus: 4
+  finished_at: <%= 1.month.ago.to_s(:db) %>
+  log: ea10d51bcf88862dbcc36eb292017dfd+45
+  output: test
+
+running_container_with_logs:
+  uuid: zzzzz-dz642-logscontainer03
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  state: Running
+  priority: 1
+  created_at: <%= 1.hour.ago.to_s(:db) %>
+  updated_at: <%= 1.hour.ago.to_s(:db) %>
+  container_image: test
+  cwd: test
+  output_path: test
+  command: ["echo", "hello"]
+  runtime_constraints:
+    ram: 12000000000
+    vcpus: 4

diff --git a/services/api/test/fixtures/logs.yml b/services/api/test/fixtures/logs.yml
index 2f45d69fd29903b9df8c4481fa6c4308e49a2c87..d83cf967e5b4c46c42b3713b581c768612443a75 100644 (file)
--- a/services/api/test/fixtures/logs.yml
+++ b/services/api/test/fixtures/logs.yml
@@ -141,3 +141,117 @@ crunchstat_for_ancient_job:
   updated_at: 2013-11-07 23:33:42.347455000 Z
   modified_at: 2013-11-07 23:33:42.347455000 Z
   object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+stderr_for_ancient_container:
+  id: 12
+  uuid: zzzzz-57u5n-containerlog001
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  object_uuid: zzzzz-dz642-logscontainer01
+  event_at: <%= 2.year.ago.to_s(:db) %>
+  event_type: stderr
+  summary: ~
+  properties:
+    text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+      cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+      0.9900 sys'
+  created_at: <%= 2.year.ago.to_s(:db) %>
+  updated_at: <%= 2.year.ago.to_s(:db) %>
+  modified_at: <%= 2.year.ago.to_s(:db) %>
+  object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_ancient_container:
+  id: 13
+  uuid: zzzzz-57u5n-containerlog002
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  object_uuid: zzzzz-dz642-logscontainer01
+  event_at: <%= 2.year.ago.to_s(:db) %>
+  event_type: crunchstat
+  summary: ~
+  properties:
+    text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+      cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+      0.9900 sys'
+  created_at: <%= 2.year.ago.to_s(:db) %>
+  updated_at: <%= 2.year.ago.to_s(:db) %>
+  modified_at: <%= 2.year.ago.to_s(:db) %>
+  object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+stderr_for_previous_container:
+  id: 14
+  uuid: zzzzz-57u5n-containerlog003
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  object_uuid: zzzzz-dz642-logscontainer02
+  event_at: <%= 1.month.ago.to_s(:db) %>
+  event_type: stderr
+  summary: ~
+  properties:
+    text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+      cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+      0.9900 sys'
+  created_at: <%= 1.month.ago.to_s(:db) %>
+  updated_at: <%= 1.month.ago.to_s(:db) %>
+  modified_at: <%= 1.month.ago.to_s(:db) %>
+  object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_previous_container:
+  id: 15
+  uuid: zzzzz-57u5n-containerlog004
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  object_uuid: zzzzz-dz642-logscontainer02
+  event_at: <%= 1.month.ago.to_s(:db) %>
+  event_type: crunchstat
+  summary: ~
+  properties:
+    text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+      cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+      0.9900 sys'
+  created_at: <%= 1.month.ago.to_s(:db) %>
+  updated_at: <%= 1.month.ago.to_s(:db) %>
+  modified_at: <%= 1.month.ago.to_s(:db) %>
+  object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+stderr_for_running_container:
+  id: 16
+  uuid: zzzzz-57u5n-containerlog005
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  object_uuid: zzzzz-dz642-logscontainer03
+  event_at: <%= 1.hour.ago.to_s(:db) %>
+  event_type: crunchstat
+  summary: ~
+  properties:
+    text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+      cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+      0.9900 sys'
+  created_at: <%= 1.hour.ago.to_s(:db) %>
+  updated_at: <%= 1.hour.ago.to_s(:db) %>
+  modified_at: <%= 1.hour.ago.to_s(:db) %>
+  object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_running_container:
+  id: 17
+  uuid: zzzzz-57u5n-containerlog006
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  object_uuid: zzzzz-dz642-logscontainer03
+  event_at: <%= 1.hour.ago.to_s(:db) %>
+  event_type: crunchstat
+  summary: ~
+  properties:
+    text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+      cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+      0.9900 sys'
+  created_at: <%= 1.hour.ago.to_s(:db) %>
+  updated_at: <%= 1.hour.ago.to_s(:db) %>
+  modified_at: <%= 1.hour.ago.to_s(:db) %>
+  object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz

diff --git a/services/api/test/tasks/delete_old_container_logs_test.rb b/services/api/test/tasks/delete_old_container_logs_test.rb
new file mode 100644 (file)
index 0000000..82a5752
--- /dev/null
+++ b/services/api/test/tasks/delete_old_container_logs_test.rb
@@ -0,0 +1,50 @@
+require 'test_helper'
+require 'rake'
+
+Rake.application.rake_require "tasks/delete_old_container_logs"
+Rake::Task.define_task(:environment)
+
+class DeleteOldContainerLogsTaskTest < ActiveSupport::TestCase
+  TASK_NAME = "db:delete_old_container_logs"
+
+  def log_uuids(*fixture_names)
+    fixture_names.map { |name| logs(name).uuid }
+  end
+
+  def run_with_expiry(clean_after)
+    Rails.configuration.clean_container_log_rows_after = clean_after
+    Rake::Task[TASK_NAME].reenable
+    Rake.application.invoke_task TASK_NAME
+  end
+
+  def check_log_existence(test_method, fixture_uuids)
+    uuids_now = Log.where("object_uuid LIKE :pattern AND event_type in ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat')", pattern: "%-dz642-%").map(&:uuid)
+    fixture_uuids.each do |expect_uuid|
+      send(test_method, uuids_now, expect_uuid)
+    end
+  end
+
+  test "delete all finished logs" do
+    uuids_to_keep = log_uuids(:stderr_for_running_container,
+                              :crunchstat_for_running_container)
+    uuids_to_clean = log_uuids(:stderr_for_previous_container,
+                               :crunchstat_for_previous_container,
+                               :stderr_for_ancient_container,
+                               :crunchstat_for_ancient_container)
+    run_with_expiry(1)
+    check_log_existence(:assert_includes, uuids_to_keep)
+    check_log_existence(:refute_includes, uuids_to_clean)
+  end
+
+  test "delete old finished logs" do
+    uuids_to_keep = log_uuids(:stderr_for_running_container,
+                              :crunchstat_for_running_container,
+                              :stderr_for_previous_container,
+                              :crunchstat_for_previous_container)
+    uuids_to_clean = log_uuids(:stderr_for_ancient_container,
+                               :crunchstat_for_ancient_container)
+    run_with_expiry(360.days)
+    check_log_existence(:assert_includes, uuids_to_keep)
+    check_log_existence(:refute_includes, uuids_to_clean)
+  end
+end