From 0a08f54c405dff0dbda5d6dbc14c1f1c6eeecd39 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Thu, 23 Dec 2021 16:59:12 -0500 Subject: [PATCH] 18562: Rename config to AlwaysUsePreemptibleInstances. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- doc/admin/spot-instances.html.textile.liquid | 6 +- lib/config/config.default.yml | 2 +- lib/config/export.go | 174 +++++++++--------- lib/config/generated_config.go | 2 +- sdk/go/arvados/config.go | 34 ++-- services/api/app/models/container_request.rb | 2 +- services/api/config/arvados_config.rb | 2 +- .../api/test/unit/container_request_test.rb | 10 +- 8 files changed, 116 insertions(+), 116 deletions(-) diff --git a/doc/admin/spot-instances.html.textile.liquid b/doc/admin/spot-instances.html.textile.liquid index 5d4b677413..7ca57df0ab 100644 --- a/doc/admin/spot-instances.html.textile.liquid +++ b/doc/admin/spot-instances.html.textile.liquid @@ -16,13 +16,13 @@ Currently Arvados supports preemptible instances using AWS and Azure spot instan h2. Configuration -To use preemptible instances, ensure they are not disabled in your configuration file (they are enabled by default, but can be disabled with @UsePreemptibleInstances: false@), and add entries to @InstanceTypes@ that have @Preemptible: true@. Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example: +First, ensure automatic selection of preemptible instances is not disabled in your configuration file (this is enabled by default, but can be disabled with @AlwaysUsePreemptibleInstances: false@), and add entries to @InstanceTypes@ that have @Preemptible: true@. Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type. The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower. For example: 
 Clusters:
   ClusterID: 
     Containers:
-      UsePreemptibleInstances: true
+      AlwaysUsePreemptibleInstances: true
     InstanceTypes:
       m4.large:
         Preemptible: false
@@ -40,7 +40,7 @@ Clusters:
         Price: 0.1
-When @UsePreemptibleInstances@ is enabled, child containers (workflow steps) will automatically be made preemptible. Note that because preempting the workflow runner would cancel the entire workflow, the workflow runner runs in a reserved (non-preemptible) instance. +When @AlwaysUsePreemptibleInstances@ is enabled, child containers (workflow steps) will automatically be made preemptible. Note that because preempting the workflow runner would cancel the entire workflow, the workflow runner runs in a reserved (non-preemptible) instance. No additional configuration is required, "arvados-dispatch-cloud":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html will now start preemptible instances where appropriate. diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index 87d38ffe4b..d16f9f25f7 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -912,7 +912,7 @@ Clusters: # # This flag is ignored if no preemptible instance types are # configured, and has no effect on top-level containers. - UsePreemptibleInstances: true + AlwaysUsePreemptibleInstances: true # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the # cloud dispatcher for executing containers on worker VMs. diff --git a/lib/config/export.go b/lib/config/export.go index cfc4cb6272..9fe7d6cf27 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -59,93 +59,93 @@ func ExportJSON(w io.Writer, cluster *arvados.Cluster) error { // exists. var whitelist = map[string]bool{ // | sort -t'"' -k2,2 - "API": true, - "API.AsyncPermissionsUpdateInterval": false, - "API.DisabledAPIs": false, - "API.KeepServiceRequestTimeout": false, - "API.MaxConcurrentRequests": false, - "API.MaxIndexDatabaseRead": false, - "API.MaxItemsPerResponse": true, - "API.MaxKeepBlobBuffers": false, - "API.MaxRequestAmplification": false, - "API.MaxRequestSize": true, - "API.MaxTokenLifetime": false, - "API.RequestTimeout": true, - "API.SendTimeout": true, - "API.VocabularyPath": false, - "API.WebsocketClientEventQueue": false, - "API.WebsocketServerEventQueue": false, - "AuditLogs": false, - "AuditLogs.MaxAge": false, - "AuditLogs.MaxDeleteBatch": false, - "AuditLogs.UnloggedAttributes": false, - "ClusterID": true, - "Collections": true, - "Collections.BalanceCollectionBatch": false, - "Collections.BalanceCollectionBuffers": false, - "Collections.BalancePeriod": false, - "Collections.BalanceTimeout": false, - "Collections.BalanceUpdateLimit": false, - "Collections.BlobDeleteConcurrency": false, - "Collections.BlobMissingReport": false, - "Collections.BlobReplicateConcurrency": false, - "Collections.BlobSigning": true, - "Collections.BlobSigningKey": false, - "Collections.BlobSigningTTL": true, - "Collections.BlobTrash": false, - "Collections.BlobTrashCheckInterval": false, - "Collections.BlobTrashConcurrency": false, - "Collections.BlobTrashLifetime": false, - "Collections.CollectionVersioning": true, - "Collections.DefaultReplication": true, - "Collections.DefaultTrashLifetime": true, - "Collections.ForwardSlashNameSubstitution": true, - "Collections.ManagedProperties": true, - "Collections.ManagedProperties.*": true, - "Collections.ManagedProperties.*.*": true, - "Collections.PreserveVersionIfIdle": true, - "Collections.S3FolderObjects": true, - "Collections.TrashSweepInterval": false, - "Collections.TrustAllContent": true, - "Collections.WebDAVCache": false, - "Collections.KeepproxyPermission": false, - "Collections.WebDAVPermission": false, - "Collections.WebDAVLogEvents": false, - "Containers": true, - "Containers.CloudVMs": false, - "Containers.CrunchRunArgumentsList": false, - "Containers.CrunchRunCommand": false, - "Containers.DefaultKeepCacheRAM": true, - "Containers.DispatchPrivateKey": false, - "Containers.JobsAPI": true, - "Containers.JobsAPI.Enable": true, - "Containers.JobsAPI.GitInternalDir": false, - "Containers.LocalKeepBlobBuffersPerVCPU": false, - "Containers.LocalKeepLogsToContainerLog": false, - "Containers.Logging": false, - "Containers.LogReuseDecisions": false, - "Containers.LSF": false, - "Containers.MaxComputeVMs": false, - "Containers.MaxDispatchAttempts": false, - "Containers.MaxRetryAttempts": true, - "Containers.MinRetryPeriod": true, - "Containers.ReserveExtraRAM": true, - "Containers.RuntimeEngine": true, - "Containers.ShellAccess": true, - "Containers.ShellAccess.Admin": true, - "Containers.ShellAccess.User": true, - "Containers.SLURM": false, - "Containers.StaleLockTimeout": false, - "Containers.SupportedDockerImageFormats": true, - "Containers.SupportedDockerImageFormats.*": true, - "Containers.UsePreemptibleInstances": true, - "Git": false, - "InstanceTypes": true, - "InstanceTypes.*": true, - "InstanceTypes.*.*": true, - "Login": true, - "Login.Google": true, - "Login.Google.AlternateEmailAddresses": false, + "API": true, + "API.AsyncPermissionsUpdateInterval": false, + "API.DisabledAPIs": false, + "API.KeepServiceRequestTimeout": false, + "API.MaxConcurrentRequests": false, + "API.MaxIndexDatabaseRead": false, + "API.MaxItemsPerResponse": true, + "API.MaxKeepBlobBuffers": false, + "API.MaxRequestAmplification": false, + "API.MaxRequestSize": true, + "API.MaxTokenLifetime": false, + "API.RequestTimeout": true, + "API.SendTimeout": true, + "API.VocabularyPath": false, + "API.WebsocketClientEventQueue": false, + "API.WebsocketServerEventQueue": false, + "AuditLogs": false, + "AuditLogs.MaxAge": false, + "AuditLogs.MaxDeleteBatch": false, + "AuditLogs.UnloggedAttributes": false, + "ClusterID": true, + "Collections": true, + "Collections.BalanceCollectionBatch": false, + "Collections.BalanceCollectionBuffers": false, + "Collections.BalancePeriod": false, + "Collections.BalanceTimeout": false, + "Collections.BalanceUpdateLimit": false, + "Collections.BlobDeleteConcurrency": false, + "Collections.BlobMissingReport": false, + "Collections.BlobReplicateConcurrency": false, + "Collections.BlobSigning": true, + "Collections.BlobSigningKey": false, + "Collections.BlobSigningTTL": true, + "Collections.BlobTrash": false, + "Collections.BlobTrashCheckInterval": false, + "Collections.BlobTrashConcurrency": false, + "Collections.BlobTrashLifetime": false, + "Collections.CollectionVersioning": true, + "Collections.DefaultReplication": true, + "Collections.DefaultTrashLifetime": true, + "Collections.ForwardSlashNameSubstitution": true, + "Collections.ManagedProperties": true, + "Collections.ManagedProperties.*": true, + "Collections.ManagedProperties.*.*": true, + "Collections.PreserveVersionIfIdle": true, + "Collections.S3FolderObjects": true, + "Collections.TrashSweepInterval": false, + "Collections.TrustAllContent": true, + "Collections.WebDAVCache": false, + "Collections.KeepproxyPermission": false, + "Collections.WebDAVPermission": false, + "Collections.WebDAVLogEvents": false, + "Containers": true, + "Containers.CloudVMs": false, + "Containers.CrunchRunArgumentsList": false, + "Containers.CrunchRunCommand": false, + "Containers.DefaultKeepCacheRAM": true, + "Containers.DispatchPrivateKey": false, + "Containers.JobsAPI": true, + "Containers.JobsAPI.Enable": true, + "Containers.JobsAPI.GitInternalDir": false, + "Containers.LocalKeepBlobBuffersPerVCPU": false, + "Containers.LocalKeepLogsToContainerLog": false, + "Containers.Logging": false, + "Containers.LogReuseDecisions": false, + "Containers.LSF": false, + "Containers.MaxComputeVMs": false, + "Containers.MaxDispatchAttempts": false, + "Containers.MaxRetryAttempts": true, + "Containers.MinRetryPeriod": true, + "Containers.ReserveExtraRAM": true, + "Containers.RuntimeEngine": true, + "Containers.ShellAccess": true, + "Containers.ShellAccess.Admin": true, + "Containers.ShellAccess.User": true, + "Containers.SLURM": false, + "Containers.StaleLockTimeout": false, + "Containers.SupportedDockerImageFormats": true, + "Containers.SupportedDockerImageFormats.*": true, + "Containers.AlwaysUsePreemptibleInstances": true, + "Git": false, + "InstanceTypes": true, + "InstanceTypes.*": true, + "InstanceTypes.*.*": true, + "Login": true, + "Login.Google": true, + "Login.Google.AlternateEmailAddresses": false, "Login.Google.AuthenticationRequestParameters": false, "Login.Google.ClientID": false, "Login.Google.ClientSecret": false, diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index 4187717f8d..402d74bde1 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -918,7 +918,7 @@ Clusters: # # This flag is ignored if no preemptible instance types are # configured, and has no effect on top-level containers. - UsePreemptibleInstances: true + AlwaysUsePreemptibleInstances: true # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the # cloud dispatcher for executing containers on worker VMs. diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index 6d5156e765..e0e0a30f72 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -424,23 +424,23 @@ type InstanceType struct { } type ContainersConfig struct { - CloudVMs CloudVMsConfig - CrunchRunCommand string - CrunchRunArgumentsList []string - DefaultKeepCacheRAM ByteSize - DispatchPrivateKey string - LogReuseDecisions bool - MaxComputeVMs int - MaxDispatchAttempts int - MaxRetryAttempts int - MinRetryPeriod Duration - ReserveExtraRAM ByteSize - StaleLockTimeout Duration - SupportedDockerImageFormats StringSet - UsePreemptibleInstances bool - RuntimeEngine string - LocalKeepBlobBuffersPerVCPU int - LocalKeepLogsToContainerLog string + CloudVMs CloudVMsConfig + CrunchRunCommand string + CrunchRunArgumentsList []string + DefaultKeepCacheRAM ByteSize + DispatchPrivateKey string + LogReuseDecisions bool + MaxComputeVMs int + MaxDispatchAttempts int + MaxRetryAttempts int + MinRetryPeriod Duration + ReserveExtraRAM ByteSize + StaleLockTimeout Duration + SupportedDockerImageFormats StringSet + AlwaysUsePreemptibleInstances bool + RuntimeEngine string + LocalKeepBlobBuffersPerVCPU int + LocalKeepLogsToContainerLog string JobsAPI struct { Enable string diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb index 3186df0905..440c3ff355 100644 --- a/services/api/app/models/container_request.rb +++ b/services/api/app/models/container_request.rb @@ -328,7 +328,7 @@ class ContainerRequest < ArvadosModel def set_preemptible if (new_record? || state_changed?) && state == Committed && - Rails.configuration.Containers.UsePreemptibleInstances && + Rails.configuration.Containers.AlwaysUsePreemptibleInstances && get_requesting_container_uuid() && self.class.any_preemptible_instances? self.scheduling_parameters['preemptible'] = true diff --git a/services/api/config/arvados_config.rb b/services/api/config/arvados_config.rb index c5db6e6b9e..8a96c432a8 100644 --- a/services/api/config/arvados_config.rb +++ b/services/api/config/arvados_config.rb @@ -124,7 +124,7 @@ arvcfg.declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisi arvcfg.declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram arvcfg.declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts arvcfg.declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max -arvcfg.declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances +arvcfg.declare_config "Containers.AlwaysUsePreemptibleInstances", Boolean, :preemptible_instances arvcfg.declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes arvcfg.declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event arvcfg.declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb index b7a1a7636e..9b35769ef2 100644 --- a/services/api/test/unit/container_request_test.rb +++ b/services/api/test/unit/container_request_test.rb @@ -984,15 +984,15 @@ class ContainerRequestTest < ActiveSupport::TestCase [false, true, false, false, false], [true, true, false, false, false], # client requests non-preemptible for child container, preemptible - # is enabled anyway if UsePreemptibleInstances and instance types + # is enabled anyway if AlwaysUsePreemptibleInstances and instance types # are configured. [false, false, true, false, false], [true, false, true, false, false], [false, true, true, false, false], [true, true, true, false, true], ].each do |use_preemptible, have_preemptible, is_child, ask, expect| - test "with UsePreemptibleInstances=#{use_preemptible} and preemptible types #{have_preemptible ? '' : 'not '}configured, create #{is_child ? 'child' : 'top-level'} container request with preemptible=#{ask} and expect #{expect}" do - Rails.configuration.Containers.UsePreemptibleInstances = use_preemptible + test "with AlwaysUsePreemptibleInstances=#{use_preemptible} and preemptible types #{have_preemptible ? '' : 'not '}configured, create #{is_child ? 'child' : 'top-level'} container request with preemptible=#{ask} and expect #{expect}" do + Rails.configuration.Containers.AlwaysUsePreemptibleInstances = use_preemptible if have_preemptible configure_preemptible_instance_type end @@ -1044,12 +1044,12 @@ class ContainerRequestTest < ActiveSupport::TestCase with_container_auth(parent) do configure_preemptible_instance_type - Rails.configuration.Containers.UsePreemptibleInstances = false + Rails.configuration.Containers.AlwaysUsePreemptibleInstances = false expect[true].push create_minimal_req!(attrs_p) expect[false].push create_minimal_req!(attrs_nonp) - Rails.configuration.Containers.UsePreemptibleInstances = true + Rails.configuration.Containers.AlwaysUsePreemptibleInstances = true expect[true].push create_minimal_req!(attrs_p) expect[true].push create_minimal_req!(attrs_nonp) -- 2.30.2