From 04e0f89fe89bccdd0f103b149aed4054a3af53c9 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Mon, 7 Oct 2013 14:12:12 -0400 Subject: [PATCH 1/1] rename jobs.resource_limits to runtime_constraints --- apps/workbench/app/views/jobs/index.html.erb | 2 +- doc/api/JobTasks.textile | 2 +- doc/api/Jobs.textile | 17 +++++++++++++---- doc/examples/pipeline_templates/rtg-fq-snp.json | 2 +- sdk/cli/bin/arv-run-pipeline-instance | 2 +- sdk/cli/bin/crunch-job | 8 ++++---- sdk/python/arvados.py | 4 ++-- .../controllers/arvados/v1/jobs_controller.rb | 2 +- services/api/app/models/job.rb | 4 ++-- ...me_resource_limits_to_runtime_constraints.rb | 5 +++++ services/api/db/schema.rb | 4 ++-- services/api/script/crunch-dispatch.rb | 4 ++-- 12 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 services/api/db/migrate/20131007180607_rename_resource_limits_to_runtime_constraints.rb diff --git a/apps/workbench/app/views/jobs/index.html.erb b/apps/workbench/app/views/jobs/index.html.erb index d03f557dcf..2188a3bcf6 100644 --- a/apps/workbench/app/views/jobs/index.html.erb +++ b/apps/workbench/app/views/jobs/index.html.erb @@ -123,7 +123,7 @@ - <% j.resource_limits.sort.each do |k,v| %> + <% j.runtime_constraints.sort.each do |k,v| %> <% end %>
<%= v %><%= k %>
diff --git a/doc/api/JobTasks.textile b/doc/api/JobTasks.textile index bc2ece1d10..e950afdcdd 100644 --- a/doc/api/JobTasks.textile +++ b/doc/api/JobTasks.textile @@ -35,7 +35,7 @@ table(table table-bordered table-condensed). A step cannot be run until all steps with lower sequence numbers have completed. Job steps with the same sequence number can be run in any order.| |parameters{}|list|| -|resource_limits{}|list|Constraints that must be satisfied by the job/task scheduler in order to run the task (see below).| +|runtime_constraints{}|list|Constraints that must be satisfied by the job/task scheduler in order to run the task (see "Jobs":Jobs.html).| |output|string|| |progress|float|| |success|boolean|Is null if the task has neither completed successfully nor failed permanently.| diff --git a/doc/api/Jobs.textile b/doc/api/Jobs.textile index f0f7529e0d..5ae74ca1cf 100644 --- a/doc/api/Jobs.textile +++ b/doc/api/Jobs.textile @@ -46,7 +46,7 @@ table(table table-bordered table-condensed). |*Property name*|*Type*|*Description*|*Notes*| |submit_id|string|Unique ID provided by client when job was submitted|Optional. Can be used by client to ensure idempotence| |priority|string||| -|resource_limits{}|list|Constraints that must be satisfied by the job/task scheduler in order to run the job.|See below.| +|runtime_constraints{}|list|Constraints that must be satisfied by the job/task scheduler in order to run the job.|See below.| |script_name|string|Name of MapReduce script|| |script_parameters{}|list|Parameters passed to MapReduce script|| |script_version|string|git commit/tree used when running the job|If the job has not started, this can be a tag/branch instead of an unambiguous commit id.| @@ -59,12 +59,21 @@ table(table table-bordered table-condensed). |success|boolean|Whether the job indicated successful completion|Is null if job has not finished| |is_locked_by_uuid|string|UUID of the user who has locked this job|Is null if job is not locked. The system user locks the job when starting the job, in order to prevent job attributes from being altered.| -h3. Resource limits +h3. Runtime constraints table(table table-bordered table-condensed). |*Key*|*Type*|*Description*|*Implemented*| -|min_nodes|integer||✓| +|dependencies{}|list|[ + { + "name": "freebayes", + "package_type": "git", + "origin": "https://github.com/ekg/freebayes.git", + "version": "011561f4a96619125d4388b66b2e82b173f3de7a" + }, + ... +]|| +|min_nodes|integer||✓| |max_nodes|integer||| -|max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓| +|max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓| |min_ram_per_task|integer|Minimum real memory (KiB) per task|| |min_ram_per_node|integer|Minimum real memory (KiB) per node|| \ No newline at end of file diff --git a/doc/examples/pipeline_templates/rtg-fq-snp.json b/doc/examples/pipeline_templates/rtg-fq-snp.json index 1235455a6d..c951c4c433 100644 --- a/doc/examples/pipeline_templates/rtg-fq-snp.json +++ b/doc/examples/pipeline_templates/rtg-fq-snp.json @@ -52,7 +52,7 @@ "optional":false } }, - "resource_limits":{ + "runtime_constraints":{ "max_tasks_per_node":1 } }, diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance index 23a3d0283f..47c69022b5 100755 --- a/sdk/cli/bin/arv-run-pipeline-instance +++ b/sdk/cli/bin/arv-run-pipeline-instance @@ -448,7 +448,7 @@ class WhRunPipelineInstance debuglog "component #{cname} new job." job = JobCache.create(:script => c[:script], :script_parameters => c[:script_parameters], - :resource_limits => c[:resource_limits] || {}, + :runtime_constraints => c[:runtime_constraints] || {}, :script_version => c[:script_version] || 'master') if job debuglog "component #{cname} new job #{job[:uuid]}" diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index 10acb8cbbe..e23492aba4 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -170,9 +170,9 @@ $job_id = $Job->{'uuid'}; -$Job->{'resource_limits'} ||= {}; -$Job->{'resource_limits'}->{'max_tasks_per_node'} ||= 0; -my $max_ncpus = $Job->{'resource_limits'}->{'max_tasks_per_node'}; +$Job->{'runtime_constraints'} ||= {}; +$Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0; +my $max_ncpus = $Job->{'runtime_constraints'}->{'max_tasks_per_node'}; Log (undef, "check slurm allocation"); @@ -452,7 +452,7 @@ else -foreach (qw (script script_version script_parameters resource_limits)) +foreach (qw (script script_version script_parameters runtime_constraints)) { Log (undef, "$_ " . diff --git a/sdk/python/arvados.py b/sdk/python/arvados.py index 6e06f5a589..2e49916ef5 100644 --- a/sdk/python/arvados.py +++ b/sdk/python/arvados.py @@ -83,8 +83,8 @@ def api(): return service class JobTask: - def __init__(self, parameters=dict(), resource_limits=dict()): - print "init jobtask %s %s" % (parameters, resource_limits) + def __init__(self, parameters=dict(), runtime_constraints=dict()): + print "init jobtask %s %s" % (parameters, runtime_constraints) class job_setup: @staticmethod diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb index c330da5122..514379e859 100644 --- a/services/api/app/controllers/arvados/v1/jobs_controller.rb +++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb @@ -1,6 +1,6 @@ class Arvados::V1::JobsController < ApplicationController accept_attribute_as_json :script_parameters, Hash - accept_attribute_as_json :resource_limits, Hash + accept_attribute_as_json :runtime_constraints, Hash accept_attribute_as_json :tasks_summary, Hash skip_before_filter :find_object_by_uuid, :only => :queue diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb index c299d9d73f..154e2724db 100644 --- a/services/api/app/models/job.rb +++ b/services/api/app/models/job.rb @@ -3,7 +3,7 @@ class Job < ArvadosModel include KindAndEtag include CommonApiTemplate serialize :script_parameters, Hash - serialize :resource_limits, Hash + serialize :runtime_constraints, Hash serialize :tasks_summary, Hash before_create :ensure_unique_submit_id before_create :ensure_script_version_is_commit @@ -30,7 +30,7 @@ class Job < ArvadosModel t.add :running t.add :is_locked_by_uuid t.add :log - t.add :resource_limits + t.add :runtime_constraints t.add :tasks_summary t.add :dependencies end diff --git a/services/api/db/migrate/20131007180607_rename_resource_limits_to_runtime_constraints.rb b/services/api/db/migrate/20131007180607_rename_resource_limits_to_runtime_constraints.rb new file mode 100644 index 0000000000..8a6a08a985 --- /dev/null +++ b/services/api/db/migrate/20131007180607_rename_resource_limits_to_runtime_constraints.rb @@ -0,0 +1,5 @@ +class RenameResourceLimitsToRuntimeConstraints < ActiveRecord::Migration + def change + rename_column :jobs, :resource_limits, :runtime_constraints + end +end diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb index fe92304aff..6a032d0d19 100644 --- a/services/api/db/schema.rb +++ b/services/api/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20130724153034) do +ActiveRecord::Schema.define(:version => 20131007180607) do create_table "api_client_authorizations", :force => true do |t| t.string "api_token", :null => false @@ -188,7 +188,7 @@ ActiveRecord::Schema.define(:version => 20130724153034) do t.string "is_locked_by_uuid" t.string "log" t.text "tasks_summary" - t.text "resource_limits" + t.text "runtime_constraints" end add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at" diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb index 388ef6bcea..26baeb02fd 100755 --- a/services/api/script/crunch-dispatch.rb +++ b/services/api/script/crunch-dispatch.rb @@ -56,8 +56,8 @@ class Dispatcher min_nodes = 1 begin - if job.resource_limits['min_nodes'] - min_nodes = begin job.resource_limits['min_nodes'].to_i rescue 1 end + if job.runtime_constraints['min_nodes'] + min_nodes = begin job.runtime_constraints['min_nodes'].to_i rescue 1 end end end next if @idle_slurm_nodes and @idle_slurm_nodes < min_nodes -- 2.30.2