From: Peter Amstutz Date: Fri, 22 Mar 2019 14:42:51 +0000 (-0400) Subject: 13996: Parsing durations wip X-Git-Tag: 1.4.0~65^2~26 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/09a1ecf507df8ca110e6620efeb3593bc0d90192 13996: Parsing durations wip Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/lib/config/config.defaults.yml b/lib/config/config.defaults.yml index 70162ee5ff..c22e2df1e1 100644 --- a/lib/config/config.defaults.yml +++ b/lib/config/config.defaults.yml @@ -308,7 +308,7 @@ Clusters: # containers that have been finished for at least this many seconds, # and delete their stdout, stderr, arv-mount, crunch-run, and # crunchstat logs from the logs table. - MaxAge: 30d + MaxAge: 720h # These two settings control how frequently log events are flushed to the # database. Log lines are buffered until either crunch_log_bytes_per_event @@ -388,12 +388,12 @@ Clusters: AssignNodeHostname: "compute%d" JobsAPI: - # Enable the legacy Jobs API. - # auto -- (default) enable the Jobs API only if it has been used before + # Enable the legacy Jobs API. This value must be a string. + # 'auto' -- (default) enable the Jobs API only if it has been used before # (i.e., there are job records in the database) - # true -- enable the Jobs API despite lack of existing records. - # false -- disable the Jobs API despite presence of existing records. - Enable: auto + # 'true' -- enable the Jobs API despite lack of existing records. + # 'false' -- disable the Jobs API despite presence of existing records. + Enable: 'auto' # Git repositories must be readable by api server, or you won't be # able to submit crunch jobs. To pass the test suites, put a clone @@ -425,11 +425,11 @@ Clusters: # original job reuse behavior, and is still the default). ReuseJobIfOutputsDiffer: false - Mail: - MailchimpAPIKey: "" - MailchimpListID: "" - SendUserSetupNotificationEmail: "" - IssueReporterEmailFrom: "" - IssueReporterEmailTo: "" - SupportEmailAddress: "" - EmailFrom: "" + Mail: + MailchimpAPIKey: "" + MailchimpListID: "" + SendUserSetupNotificationEmail: "" + IssueReporterEmailFrom: "" + IssueReporterEmailTo: "" + SupportEmailAddress: "" + EmailFrom: "" diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml index 66e09f6719..aca0669354 100644 --- a/services/api/config/application.default.yml +++ b/services/api/config/application.default.yml @@ -14,17 +14,6 @@ common: - # When you run the db:delete_old_job_logs task, it will find jobs that - # have been finished for at least this many seconds, and delete their - # stderr logs from the logs table. - clean_job_log_rows_after: <%= 30.days %> - - # When you run the db:delete_old_container_logs task, it will find - # containers that have been finished for at least this many seconds, - # and delete their stdout, stderr, arv-mount, crunch-run, and - # crunchstat logs from the logs table. - clean_container_log_rows_after: <%= 30.days %> - ## Set Time.zone default to the specified zone and make Active ## Record auto-convert to this zone. Run "rake -D time" for a list ## of tasks for finding time zone names. Default is UTC. diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb index 0a99b1afcd..80198d2db5 100644 --- a/services/api/config/initializers/load_config.rb +++ b/services/api/config/initializers/load_config.rb @@ -56,9 +56,9 @@ end $config_migrate_map = {} $config_types = {} -def declare_config(assign_to, configtype, migrate_from=nil) +def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil) if migrate_from - $config_migrate_map[migrate_from] = ->(cfg, k, v) { + $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) { set_cfg cfg, assign_to, v } end @@ -129,7 +129,9 @@ declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_s declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname -declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api +declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { + set_cfg cfg, "Containers.JobsAPI.Enable", if v.is_a? Boolean then v.to_s else v end +} declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger @@ -165,6 +167,8 @@ application_config.each do |k, v| end end +duration_re = /(\d+(\.\d+)?)(ms|s|m|h)/ + $config_types.each do |cfgkey, cfgtype| cfg = $arvados_config k = cfgkey @@ -176,6 +180,11 @@ $config_types.each do |cfgkey, cfgtype| break end end + + if cfg.nil? + raise "missing #{cfgkey}" + end + if cfgtype == String and !cfg[k] cfg[k] = "" end @@ -183,14 +192,15 @@ $config_types.each do |cfgkey, cfgtype| if cfg[k].is_a? Integer cfg[k] = cfg[k].seconds elsif cfg[k].is_a? String - # TODO handle suffixes + mt = duration_re.match cfg[k] + if !mt + raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are ms, s, m, h" + end + multiplier = {ms: 0.001, s: 1, m: 60, h: 3600} + cfg[k] = (Float(mt[1]) * multiplier[mt[3].to_sym]).seconds end end - if cfg.nil? - raise "missing #{cfgkey}" - end - if !cfg[k].is_a? cfgtype raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}" end