gem 'jquery-rails'
-gem 'rvm-capistrano', :group => :test
-
gem 'acts_as_api'
gem 'passenger'
multi_json (>= 1.0.0)
builder (3.2.4)
byebug (11.0.1)
- capistrano (2.15.9)
- highline
- net-scp (>= 1.0.0)
- net-sftp (>= 2.0.0)
- net-ssh (>= 2.0.14)
- net-ssh-gateway (>= 1.1.0)
concurrent-ruby (1.1.9)
crass (1.0.6)
erubi (1.10.0)
multi_json (~> 1.11)
os (>= 0.9, < 2.0)
signet (~> 0.7)
- highline (2.0.1)
httpclient (2.8.3)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
metaclass (~> 0.0.1)
multi_json (1.15.0)
multipart-post (2.1.1)
- net-scp (2.0.0)
- net-ssh (>= 2.6.5, < 6.0.0)
- net-sftp (2.1.2)
- net-ssh (>= 2.6.5)
- net-ssh (5.2.0)
- net-ssh-gateway (2.0.0)
- net-ssh (>= 4.0.0)
nio4r (2.5.8)
nokogiri (1.13.3)
mini_portile2 (~> 2.8.0)
railties (>= 4.2.0, < 6.0)
retriable (1.4.1)
ruby-prof (0.15.9)
- rvm-capistrano (1.5.6)
- capistrano (~> 2.15.4)
safe_yaml (1.0.5)
signet (0.11.0)
addressable (~> 2.3)
rails-perftest
responders (~> 2.0)
ruby-prof (~> 0.15.0)
- rvm-capistrano
safe_yaml
signet (< 0.12)
simplecov (~> 0.7.1)
desc "Remove old container log entries from the logs table"
task delete_old_container_logs: :environment do
- delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND clock_timestamp() - containers.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
+ delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND now() - containers.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
ActiveRecord::Base.connection.execute(delete_sql)
end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This task finds jobs that have been finished for at least as long as
-# the duration specified in the `clean_job_log_rows_after`
-# configuration setting, and deletes their stderr logs from the logs table.
-
-namespace :db do
- desc "Remove old job stderr entries from the logs table"
- task delete_old_job_logs: :environment do
- delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
-
- ActiveRecord::Base.connection.execute(delete_sql)
- end
-end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-# This is needed instead of just including CurrentApiClient so that its
-# methods don't get imported as Object's class methods; this is a problem because
-# the methods would be imported only on test environment. See #15716 for more info.
-class CurrentApiClientHelper
- extend CurrentApiClient
-end
-
-def has_symbols? x
- if x.is_a? Hash
- x.each do |k,v|
- return true if has_symbols?(k) or has_symbols?(v)
- end
- elsif x.is_a? Array
- x.each do |k|
- return true if has_symbols?(k)
- end
- elsif x.is_a? Symbol
- return true
- elsif x.is_a? String
- return true if x.start_with?(':') && !x.start_with?('::')
- end
- false
-end
-
-def check_for_serialized_symbols rec
- jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
- (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
- if has_symbols? rec.attributes[colname]
- st = recursive_stringify rec.attributes[colname]
- puts "Found value potentially containing Ruby symbols in #{colname} attribute of #{rec.uuid}, current value is\n#{rec.attributes[colname].to_s[0..1024]}\nrake symbols:stringify will update it to:\n#{st.to_s[0..1024]}\n\n"
- end
- end
-end
-
-def recursive_stringify x
- if x.is_a? Hash
- Hash[x.collect do |k,v|
- [recursive_stringify(k), recursive_stringify(v)]
- end]
- elsif x.is_a? Array
- x.collect do |k|
- recursive_stringify k
- end
- elsif x.is_a? Symbol
- x.to_s
- elsif x.is_a? String and x.start_with?(':') and !x.start_with?('::')
- x[1..-1]
- else
- x
- end
-end
-
-def stringify_serialized_symbols rec
- # ensure_serialized_attribute_type should prevent symbols from
- # getting into the database in the first place. If someone managed
- # to get them into the database (perhaps using an older version)
- # we'll convert symbols to strings when loading from the
- # database. (Otherwise, loading and saving an object with existing
- # symbols in a serialized field will crash.)
- jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
- (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
- if has_symbols? rec.attributes[colname]
- begin
- st = recursive_stringify rec.attributes[colname]
- puts "Updating #{colname} attribute of #{rec.uuid} from\n#{rec.attributes[colname].to_s[0..1024]}\nto\n#{st.to_s[0..1024]}\n\n"
- rec.write_attribute(colname, st)
- rec.save!
- rescue => e
- puts "Failed to update #{rec.uuid}: #{e}"
- end
- end
- end
-end
-
-namespace :symbols do
- desc 'Warn about serialized values starting with ":" that may be symbols'
- task check: :environment do
- [ApiClientAuthorization, ApiClient,
- AuthorizedKey, Collection,
- Container, ContainerRequest, Group,
- Human, Job, JobTask, KeepDisk, KeepService, Link,
- Node, PipelineInstance, PipelineTemplate,
- Repository, Specimen, Trait, User, VirtualMachine,
- Workflow].each do |klass|
- CurrentApiClientHelper.act_as_system_user do
- klass.all.each do |c|
- check_for_serialized_symbols c
- end
- end
- end
- end
-
- task stringify: :environment do
- [ApiClientAuthorization, ApiClient,
- AuthorizedKey, Collection,
- Container, ContainerRequest, Group,
- Human, Job, JobTask, KeepDisk, KeepService, Link,
- Node, PipelineInstance, PipelineTemplate,
- Repository, Specimen, Trait, User, VirtualMachine,
- Workflow].each do |klass|
- CurrentApiClientHelper.act_as_system_user do
- klass.all.each do |c|
- stringify_serialized_symbols c
- end
- end
- end
- end
-end
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'test_helper'
-require 'rake'
-
-Rake.application.rake_require "tasks/delete_old_job_logs"
-Rake::Task.define_task(:environment)
-
-class DeleteOldJobLogsTaskTest < ActiveSupport::TestCase
- TASK_NAME = "db:delete_old_job_logs"
-
- def log_uuids(*fixture_names)
- fixture_names.map { |name| logs(name).uuid }
- end
-
- def run_with_expiry(clean_after)
- Rails.configuration.Containers.Logging.MaxAge = clean_after
- Rake::Task[TASK_NAME].reenable
- Rake.application.invoke_task TASK_NAME
- end
-
- def job_stderr_logs
- Log.where("object_uuid LIKE :pattern AND event_type = :etype",
- pattern: "_____-8i9sb-_______________",
- etype: "stderr")
- end
-
- def check_existence(test_method, fixture_uuids)
- uuids_now = job_stderr_logs.map(&:uuid)
- fixture_uuids.each do |expect_uuid|
- send(test_method, uuids_now, expect_uuid)
- end
- end
-
- test "delete all logs" do
- uuids_to_keep = log_uuids(:crunchstat_for_running_job)
- uuids_to_clean = log_uuids(:crunchstat_for_previous_job,
- :crunchstat_for_ancient_job)
- run_with_expiry(1)
- check_existence(:assert_includes, uuids_to_keep)
- check_existence(:refute_includes, uuids_to_clean)
- end
-
- test "delete only old logs" do
- uuids_to_keep = log_uuids(:crunchstat_for_running_job,
- :crunchstat_for_previous_job)
- uuids_to_clean = log_uuids(:crunchstat_for_ancient_job)
- run_with_expiry(360.days)
- check_existence(:assert_includes, uuids_to_keep)
- check_existence(:refute_includes, uuids_to_clean)
- end
-end
if [[ ! -f /usr/local/bin/arvados-server ]]; then
$RUNSU flock /var/lib/gopath/gopath.lock go mod download
+ $RUNSU flock /var/lib/gopath/gopath.lock go mod vendor
$RUNSU flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
$RUNSU flock /var/lib/gopath/gopath.lock install $GOPATH/bin/arvados-server /usr/local/bin
fi
exit 0
fi
-# FIXME! #16992 Temporary fix for psql call in arvados-api-server
-if [ -e /root/.psqlrc ]; then
- if ! ( grep 'pset pager off' /root/.psqlrc ); then
- RESTORE_PSQL="yes"
- cp /root/.psqlrc /root/.psqlrc.provision.backup
- fi
-else
- DELETE_PSQL="yes"
-fi
-
-echo '\pset pager off' >> /root/.psqlrc
-# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
-
# Now run the install
salt-call --local state.apply -l ${LOG_LEVEL}
sed -i 's/^manage_etc_hosts: true/#manage_etc_hosts: true/g' /etc/cloud/cloud.cfg.d/*
fi
-# FIXME! #16992 Temporary fix for psql call in arvados-api-server
-if [ "x${DELETE_PSQL}" = "xyes" ]; then
- echo "Removing .psql file"
- rm /root/.psqlrc
-fi
-
-if [ "x${RESTORE_PSQL}" = "xyes" ]; then
- echo "Restoring .psql file"
- mv -v /root/.psqlrc.provision.backup /root/.psqlrc
-fi
-# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
-
# Leave a copy of the Arvados CA so the user can copy it where it's required
if [ "$DEV_MODE" = "yes" ]; then
echo "Copying the Arvados CA certificate to the installer dir, so you can import it"