gem 'morrisjs-rails'
gem 'raphael-rails'
+
+gem 'lograge'
+gem 'logstash-event'
logging (2.1.0)
little-plugger (~> 1.1)
multi_json (~> 1.10)
+ lograge (0.3.6)
+ actionpack (>= 3)
+ activesupport (>= 3)
+ railties (>= 3)
+ logstash-event (1.2.02)
mail (2.6.3)
mime-types (>= 1.16, < 3)
memoist (0.14.0)
jquery-rails
less
less-rails
+ lograge
+ logstash-event
minitest (>= 5.0.0)
mocha
morrisjs-rails
therubyracer
uglifier (>= 1.0.3)
wiselinks
+
+BUNDLED WITH
+ 1.12.1
@permissions = Link.limit(RELATION_LIMIT).order("modified_at DESC")
.where(head_uuid: @object.uuid, link_class: 'permission',
name: 'can_read').results
- @logs = Log.limit(RELATION_LIMIT).order("created_at DESC")
- .select(%w(uuid event_type object_uuid event_at summary))
- .where(object_uuid: @object.uuid).results
- @is_persistent = Link.limit(1)
- .where(head_uuid: @object.uuid, tail_uuid: current_user.uuid,
- link_class: 'resources', name: 'wants')
- .results.any?
@search_sharing = search_scopes
if params["tab_pane"] == "Used_by"
<div class="row row-fill-height">
- <div class="col-md-6">
+ <div class="col-md-7">
<div class="panel panel-info">
<div class="panel-heading">
<h3 class="panel-title">
</div>
</div>
</div>
- <div class="col-md-3">
- <div class="panel panel-default">
- <div class="panel-heading">
- <h3 class="panel-title">
- Activity
- </h3>
- </div>
- <div class="panel-body smaller-text">
- <!--
- <input type="text" class="form-control" placeholder="Search"/>
- -->
- <div style="height:0.5em;"></div>
- <% name_or_object = @name_link.andand.uuid ? @name_link : @object %>
- <% if name_or_object.created_at and not @logs.andand.any? %>
- <p>
- Created: <%= name_or_object.created_at.to_s(:long) if name_or_object.created_at %>
- </p>
- <p>
- Last modified: <%= name_or_object.modified_at.to_s(:long) if name_or_object.modified_at %> by <%= link_to_if_arvados_object name_or_object.modified_by_user_uuid, friendly_name: true %>
- </p>
- <% else %>
- <%= render_arvados_object_list_start(@logs, 'Show all activity',
- logs_path(filters: [['object_uuid','=',name_or_object.uuid]].to_json)) do |log| %>
- <p>
- <%= time_ago_in_words(log.event_at) rescue 'unknown time' %> ago: <%= log.summary %>
- <% if log.object_uuid %>
- <%= link_to_if_arvados_object log.object_uuid, link_text: raw('<i class="fa fa-hand-o-right"></i>') %>
- <% end %>
- </p>
- <% end %>
- <% end %>
- </div>
- </div>
- </div>
<% if current_user %>
- <div class="col-md-3">
+ <div class="col-md-5">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">
</div>
</div>
<% else %>
- <div class="col-md-3">
+ <div class="col-md-5">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">
--- /dev/null
+ArvadosWorkbench::Application.configure do
+ config.lograge.enabled = true
+ config.lograge.formatter = Lograge::Formatters::Logstash.new
+ config.lograge.custom_options = lambda do |event|
+ exceptions = %w(controller action format id)
+ params = event.payload[:params].except(*exceptions)
+ params_s = Oj.dump(params)
+ if params_s.length > 1000
+ { params_truncated: params_s[0..1000] + "[...]" }
+ else
+ { params: params }
+ end
+ end
+end
"controller did not find logger job")
end
- test "viewing a collection fetches logs about it" do
- show_collection(:foo_file, :active)
- assert_includes(assigns(:logs).map(&:uuid),
- api_fixture('logs')['system_adds_foo_file']['uuid'],
- "controller did not find related log")
- end
-
test "sharing auths available to admin" do
show_collection("collection_owned_by_active", "admin_trustedclient")
assert_not_nil assigns(:search_sharing)
visit page_with_token('active', "/collections/#{new_collection.uuid}")
Rails.logger.info "Done visiting collection at #{Time.now.to_f}"
- assert_text new_collection.uuid
+ assert_selector "input[value=\"#{new_collection.uuid}\"]"
assert(page.has_link?('collection_file_name_with_prefix_0'), "Collection page did not include file link")
end
end
visit page_with_token('active', "/collections/#{new_collection.uuid}")
Rails.logger.info "Done visiting collection at #{Time.now.to_f}"
- assert_text new_collection.uuid
+ assert_selector "input[value=\"#{new_collection.uuid}\"]"
assert(page.has_link?('collection_file_name_with_prefix_0'), "Collection page did not include file link")
# edit description
--- /dev/null
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this postinst script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "${0}")"
+pkg="${script%.postinst}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ configure)
+ if [ -e /run/systemd/system ]; then
+ eval "$(systemctl -p UnitFileState show "${systemd_unit}")"
+ case "${UnitFileState}" in
+ disabled)
+ # Failing to enable or start the service is not a
+ # package error, so don't let errors here
+ # propagate up.
+ systemctl enable "${systemd_unit}" || true
+ systemctl start "${systemd_unit}" || true
+ ;;
+ enabled)
+ systemctl daemon-reload || true
+ systemctl reload-or-try-restart "${systemd_unit}" || true
+ ;;
+ esac
+ fi
+ ;;
+esac
--- /dev/null
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this prerm script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "${0}")"
+pkg="${script%.prerm}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ remove)
+ if [ -e /run/systemd/system ]; then
+ systemctl stop "${systemd_unit}" || true
+ systemctl disable "${systemd_unit}" || true
+ fi
+ ;;
+esac
# clean up the docker build environment
cd "$WORKSPACE"
-tools/arvbox/bin/arvbox build dev
+tools/arvbox/bin/arvbox rebuild localdemo
ECODE=$?
if [[ "$ECODE" != "0" ]]; then
EXITCODE=$(($EXITCODE + $ECODE))
fi
-tools/arvbox/bin/arvbox build localdemo
+tools/arvbox/bin/arvbox build dev
ECODE=$?
if [[ "$ECODE" != "0" ]]; then
title "upload arvados images SKIPPED because build failed"
else
- if [[ $upload == true ]]; then
- ## 20150526 nico -- *sometimes* dockerhub needs re-login
+ if [[ $upload == true ]]; then
+ ## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
cd $WORKSPACE/packages/$TARGET
go get "git.curoverse.com/arvados.git/$src_path"
- fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file"
+
+ declare -a switches=()
+ systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
+ if [[ -e "${systemd_unit}" ]]; then
+ switches+=(
+ --after-install "${WORKSPACE}/build/go-package-scripts/postinst"
+ --before-remove "${WORKSPACE}/build/go-package-scripts/prerm"
+ "${systemd_unit}=/lib/systemd/system/${prog}.service")
+ fi
+ switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
+
+ fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" 'Curoverse, Inc.' dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
default_iteration() {
cat >/tmp/cwltest/arv-cwl-jobs <<EOF2
#!/bin/sh
-exec arvados-cwl-runner --api=jobs \\\$@
+exec arvados-cwl-runner --api=jobs --compute-checksum \\\$@
EOF2
chmod +x /tmp/cwltest/arv-cwl-jobs
cat >/tmp/cwltest/arv-cwl-containers <<EOF2
#!/bin/sh
-exec arvados-cwl-runner --api=containers \\\$@
+exec arvados-cwl-runner --api=containers --compute-checksum \\\$@
EOF2
chmod +x /tmp/cwltest/arv-cwl-containers
gem 'puma'
gem 'sshkey'
gem 'safe_yaml'
+gem 'lograge'
+gem 'logstash-event'
launchy (2.4.3)
addressable (~> 2.3)
libv8 (3.16.14.3)
+ lograge (0.3.6)
+ actionpack (>= 3)
+ activesupport (>= 3)
+ railties (>= 3)
+ logstash-event (1.2.02)
mail (2.5.4)
mime-types (~> 1.16)
treetop (~> 1.4.8)
factory_girl_rails
faye-websocket
jquery-rails
+ lograge
+ logstash-event
mocha
multi_json
oj
therubyracer
trollop
uglifier (>= 1.0.3)
+
+BUNDLED WITH
+ 1.12.1
params[:find_or_create] = !resource_attrs.delete(:no_reuse)
end
- if params[:find_or_create]
- return if false.equal?(load_filters_param)
- if @filters.empty? # Translate older creation parameters into filters.
- @filters =
- [["repository", "=", resource_attrs[:repository]],
- ["script", "=", resource_attrs[:script]],
- ["script_version", "not in git", params[:exclude_script_versions]],
- ].reject { |filter| filter.last.nil? or filter.last.empty? }
- if !params[:minimum_script_version].blank?
- @filters << ["script_version", "in git",
- params[:minimum_script_version]]
- else
- add_default_git_filter("script_version", resource_attrs[:repository],
- resource_attrs[:script_version])
- end
- if image_search = resource_attrs[:runtime_constraints].andand["docker_image"]
- if image_tag = resource_attrs[:runtime_constraints]["docker_image_tag"]
- image_search += ":#{image_tag}"
- end
- image_locator = Collection.
- for_latest_docker_image(image_search).andand.portable_data_hash
- else
- image_locator = nil
- end
- @filters << ["docker_image_locator", "=", image_locator]
- if sdk_version = resource_attrs[:runtime_constraints].andand["arvados_sdk_version"]
- add_default_git_filter("arvados_sdk_version", "arvados", sdk_version)
- end
- begin
- load_job_specific_filters
- rescue ArgumentError => error
- return send_error(error.message)
+ return super if !params[:find_or_create]
+ return if !load_filters_param
+
+ if @filters.empty? # Translate older creation parameters into filters.
+ @filters =
+ [["repository", "=", resource_attrs[:repository]],
+ ["script", "=", resource_attrs[:script]],
+ ["script_version", "not in git", params[:exclude_script_versions]],
+ ].reject { |filter| filter.last.nil? or filter.last.empty? }
+ if !params[:minimum_script_version].blank?
+ @filters << ["script_version", "in git",
+ params[:minimum_script_version]]
+ else
+ add_default_git_filter("script_version", resource_attrs[:repository],
+ resource_attrs[:script_version])
+ end
+ if image_search = resource_attrs[:runtime_constraints].andand["docker_image"]
+ if image_tag = resource_attrs[:runtime_constraints]["docker_image_tag"]
+ image_search += ":#{image_tag}"
end
+ image_locator = Collection.
+ for_latest_docker_image(image_search).andand.portable_data_hash
+ else
+ image_locator = nil
+ end
+ @filters << ["docker_image_locator", "=", image_locator]
+ if sdk_version = resource_attrs[:runtime_constraints].andand["arvados_sdk_version"]
+ add_default_git_filter("arvados_sdk_version", "arvados", sdk_version)
+ end
+ begin
+ load_job_specific_filters
+ rescue ArgumentError => error
+ return send_error(error.message)
+ end
+ end
+
+ # Check specified filters for some reasonableness.
+ filter_names = @filters.map { |f| f.first }.uniq
+ ["repository", "script"].each do |req_filter|
+ if not filter_names.include?(req_filter)
+ return send_error("#{req_filter} filter required")
end
+ end
- # Check specified filters for some reasonableness.
- filter_names = @filters.map { |f| f.first }.uniq
- ["repository", "script"].each do |req_filter|
- if not filter_names.include?(req_filter)
- return send_error("#{req_filter} filter required")
- end
+ # Search for a reusable Job, and return it if found.
+ @objects = Job.
+ readable_by(current_user).
+ where('state = ? or (owner_uuid = ? and state in (?))',
+ Job::Complete, current_user.uuid, [Job::Queued, Job::Running]).
+ where('script_parameters_digest = ?', Job.sorted_hash_digest(resource_attrs[:script_parameters])).
+ where('nondeterministic is distinct from ?', true).
+ order('state desc, created_at') # prefer Running jobs over Queued
+ apply_filters
+ @object = nil
+ incomplete_job = nil
+ @objects.each do |j|
+ if j.state != Job::Complete
+ # We'll use this if we don't find a job that has completed
+ incomplete_job ||= j
+ next
end
- # Search for a reusable Job, and return it if found.
- @objects = Job.readable_by(current_user)
- apply_filters
- @object = nil
- incomplete_job = nil
- @objects.each do |j|
- if j.nondeterministic != true and
- ["Queued", "Running", "Complete"].include?(j.state) and
- j.script_parameters == resource_attrs[:script_parameters]
- if j.state != "Complete" && j.owner_uuid == current_user.uuid
- # We'll use this if we don't find a job that has completed
- incomplete_job ||= j
- else
- if Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
- # Record the first job in the list
- if !@object
- @object = j
- end
- # Ensure that all candidate jobs actually did produce the same output
- if @object.output != j.output
- @object = nil
- break
- end
- end
- end
- end
- @object ||= incomplete_job
- if @object
- return show
+ if @object == false
+ # We have already decided not to reuse any completed job
+ next
+ elsif @object
+ if @object.output != j.output
+ # If two matching jobs produced different outputs, run a new
+ # job (or use one that's already running/queued) instead of
+ # choosing one arbitrarily.
+ @object = false
end
+ # ...and that's the only thing we need to do once we've chosen
+ # an @object to reuse.
+ elsif !Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
+ # As soon as the output we will end up returning (if any) is
+ # decided, check whether it will be visible to the user; if
+ # not, any further investigation of reusable jobs is futile.
+ return super
+ else
+ @object = j
end
end
- super
+ @object ||= incomplete_job
+ if @object
+ show
+ else
+ super
+ end
end
def cancel
load_limit_offset_order_params
load_where_param
@where.merge!({state: Job::Queued})
- return if false.equal?(load_filters_param)
+ return if !load_filters_param
find_objects_for_index
index
end
rescue ArgumentError => error
send_error(error.message)
false
+ else
+ true
end
end
end
after_commit :trigger_crunch_dispatch_if_cancelled, :on => :update
before_validation :set_priority
before_validation :update_state_from_old_state_attrs
+ before_validation :update_script_parameters_digest
validate :ensure_script_version_is_commit
validate :find_docker_image_locator
validate :find_arvados_sdk_version
end
end
+ def update_script_parameters_digest
+ self.script_parameters_digest = self.class.sorted_hash_digest(script_parameters)
+ end
+
+ def self.searchable_columns operator
+ super - ["script_parameters_digest"]
+ end
+
protected
+ def self.sorted_hash_digest h
+ Digest::MD5.hexdigest(Oj.dump(deep_sort_hash(h)))
+ end
+
+ def self.deep_sort_hash h
+ return h unless h.is_a? Hash
+ h.sort.collect do |k, v|
+ [k, deep_sort_hash(v)]
+ end.to_h
+ end
+
def foreign_key_attributes
super + %w(output log)
end
path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
if File.exists? path
yaml = ERB.new(IO.read path).result(binding)
- confs = YAML.load(yaml)
+ confs = YAML.load(yaml, deserialize_symbols: true)
# Ignore empty YAML file:
next if confs == false
$application_config.merge!(confs['common'] || {})
--- /dev/null
+Server::Application.configure do
+ config.lograge.enabled = true
+ config.lograge.formatter = Lograge::Formatters::Logstash.new
+ config.lograge.custom_options = lambda do |event|
+ exceptions = %w(controller action format id)
+ params = event.payload[:params].except(*exceptions)
+ params_s = Oj.dump(params)
+ if params_s.length > 1000
+ { params_truncated: params_s[0..1000] + "[...]" }
+ else
+ { params: params }
+ end
+ end
+end
--- /dev/null
+class AddScriptParametersDigestToJobs < ActiveRecord::Migration
+ def change
+ add_column :jobs, :script_parameters_digest, :string
+ add_index :jobs, :script_parameters_digest
+ end
+end
--- /dev/null
+class PopulateScriptParametersDigest < ActiveRecord::Migration
+ def up
+ done = false
+ while !done
+ done = true
+ Job.
+ where('script_parameters_digest is null').
+ select([:id, :script_parameters, :script_parameters_digest]).
+ limit(200).
+ each do |j|
+ done = false
+ Job.
+ where('id=? or script_parameters=?', j.id, j.script_parameters.to_yaml).
+ update_all(script_parameters_digest: j.update_script_parameters_digest)
+ end
+ end
+ end
+
+ def down
+ end
+end
description character varying(524288),
state character varying(255),
arvados_sdk_version character varying(255),
- components text
+ components text,
+ script_parameters_digest character varying(255)
);
CREATE INDEX index_jobs_on_script ON jobs USING btree (script);
+--
+-- Name: index_jobs_on_script_parameters_digest; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX index_jobs_on_script_parameters_digest ON jobs USING btree (script_parameters_digest);
+
+
--
-- Name: index_jobs_on_started_at; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
INSERT INTO schema_migrations (version) VALUES ('20160808151459');
-INSERT INTO schema_migrations (version) VALUES ('20160808151559');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20160808151559');
+
+INSERT INTO schema_migrations (version) VALUES ('20160819195557');
+
+INSERT INTO schema_migrations (version) VALUES ('20160819195725');
\ No newline at end of file
done: 1
runtime_constraints: {}
state: Running
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
running_cancelled:
uuid: zzzzz-8i9sb-4cf0nhn6xte809j
done: 1
runtime_constraints: {}
state: Cancelled
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
uses_nonexistent_script_version:
uuid: zzzzz-8i9sb-7m339pu0x9mla88
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
foobar:
uuid: zzzzz-8i9sb-aceg2bnq7jt7kon
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 03a43a7d84f7fb022467b876c2950acd
barbaz:
uuid: zzzzz-8i9sb-cjs4pklxxjykyuq
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: c3d19d3ec50ac0914baa56b149640f73
runningbarbaz:
uuid: zzzzz-8i9sb-cjs4pklxxjykyuj
done: 0
runtime_constraints: {}
state: Running
+ script_parameters_digest: c3d19d3ec50ac0914baa56b149640f73
previous_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
log: d41d8cd98f00b204e9800998ecf8427e+0
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
previous_ancient_job_run:
uuid: zzzzz-8i9sb-ahd7cie8jah9qui
log: d41d8cd98f00b204e9800998ecf8427e+0
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
+ script_parameters_digest: 174dd339d44f2b259fadbab7ebdb8df9
previous_docker_job_run:
uuid: zzzzz-8i9sb-k6emstgk4kw4yhi
output: ea10d51bcf88862dbcc36eb292017dfd+45
docker_image_locator: fa3c1a9cb6783f85f2ecda037e07b8c3+167
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
previous_ancient_docker_image_job_run:
uuid: zzzzz-8i9sb-t3b460aolxxuldl
output: ea10d51bcf88862dbcc36eb292017dfd+45
docker_image_locator: b519d9cb706a29fc7ea24dbea2f05851+93
state: Complete
+ script_parameters_digest: 174dd339d44f2b259fadbab7ebdb8df9
previous_job_run_with_arvados_sdk_version:
uuid: zzzzz-8i9sb-eoo0321or2dw2jg
success: true
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
previous_job_run_no_output:
uuid: zzzzz-8i9sb-cjs4pklxxjykppp
success: true
output: ~
state: Complete
+ script_parameters_digest: 174dd339d44f2b259fadbab7ebdb8df9
previous_job_run_superseded_by_hash_branch:
# This supplied_script_version is a branch name with later commits.
success: true
output: d41d8cd98f00b204e9800998ecf8427e+0
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
nondeterminisic_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykyyy
success: true
nondeterministic: true
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
nearly_finished_job:
uuid: zzzzz-8i9sb-2gx6rz0pjl033w3
done: 0
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 7ea26d58a79b7f5db9f90fb1e33d3006
queued:
uuid: zzzzz-8i9sb-grx15v5mjnsyxk7
tasks_summary: {}
runtime_constraints: {}
state: Queued
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
# A job with a log collection that can be parsed by the log viewer.
job_with_real_log:
log: 0b9a7787660e1fce4a93f33e01376ba6+81
script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
cancelled:
uuid: zzzzz-8i9sb-4cf0abc123e809j
done: 1
runtime_constraints: {}
state: Cancelled
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
job_in_subproject:
uuid: zzzzz-8i9sb-subprojectjob01
script: hash
script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
running_will_be_completed:
uuid: zzzzz-8i9sb-rshmckwoma9pjh8
done: 1
runtime_constraints: {}
state: Running
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
graph_stage1:
uuid: zzzzz-8i9sb-graphstage10000
script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
state: Complete
output: fa7aeb5140e2848d39b416daeef4ffc5+45
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
graph_stage2:
uuid: zzzzz-8i9sb-graphstage20000
input: fa7aeb5140e2848d39b416daeef4ffc5+45
input2: "stuff"
output: 65b17c95fdbc9800fc48acda4e9dcd0b+93
+ script_parameters_digest: 4900033ec5cfaf8a63566f3664aeaa70
graph_stage3:
uuid: zzzzz-8i9sb-graphstage30000
input: fa7aeb5140e2848d39b416daeef4ffc5+45
input2: "stuff2"
output: ea10d51bcf88862dbcc36eb292017dfd+45
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
job_with_latest_version:
uuid: zzzzz-8i9sb-nj8ioxnrvjtyk2b
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 03a43a7d84f7fb022467b876c2950acd
running_job_in_publicly_accessible_project:
uuid: zzzzz-8i9sb-n7omg50bvt0m1nf
script_parameters:
input: fa7aeb5140e2848d39b416daeef4ffc5+45
input2: "stuff2"
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
completed_job_in_publicly_accessible_project:
uuid: zzzzz-8i9sb-jyq01m7in1jlofj
input2: "stuff2"
log: zzzzz-4zz18-4en62shvi99lxd4
output: b519d9cb706a29fc7ea24dbea2f05851+93
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
job_in_publicly_accessible_project_but_other_objects_elsewhere:
uuid: zzzzz-8i9sb-jyq01muyhgr4ofj
input2: "stuff2"
log: zzzzz-4zz18-fy296fx3hot09f7
output: zzzzz-4zz18-bv31uwvy3neko21
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
running_job_with_components:
uuid: zzzzz-8i9sb-with2components
components:
component1: zzzzz-8i9sb-jyq01m7in1jlofj
component2: zzzzz-d1hrv-partdonepipelin
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
repository: "active/foo",
script_parameters: {
- input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
- an_integer: '1'
+ an_integer: '1',
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45'
}
}
assert_response :success
assert_equal('077ba2ad3ea24a929091a9e6ce545c93199b8e57',
internal_tag(j.uuid))
end
+
+ test 'script_parameters_digest is independent of key order' do
+ j1 = Job.new(job_attrs(script_parameters: {'a' => 'a', 'ddee' => {'d' => 'd', 'e' => 'e'}}))
+ j2 = Job.new(job_attrs(script_parameters: {'ddee' => {'e' => 'e', 'd' => 'd'}, 'a' => 'a'}))
+ assert j1.valid?
+ assert j2.valid?
+ assert_equal(j1.script_parameters_digest, j2.script_parameters_digest)
+ end
+
+ test 'job fixtures have correct script_parameters_digest' do
+ Job.all.each do |j|
+ d = j.script_parameters_digest
+ assert_equal(j.update_script_parameters_digest, d,
+ "wrong script_parameters_digest for #{j.uuid}")
+ end
+ end
end
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/dispatch"
+ "github.com/coreos/go-systemd/daemon"
"io"
"io/ioutil"
"log"
// Config used by crunch-dispatch-slurm
type Config struct {
- arvados.Client
+ Client arvados.Client
SbatchArguments []string
PollPeriod arvados.Duration
PollInterval: time.Duration(config.PollPeriod),
DoneProcessing: make(chan struct{})}
+ if _, err := daemon.SdNotify("READY=1"); err != nil {
+ log.Printf("Error notifying init daemon: %v", err)
+ }
+
err = dispatcher.RunDispatcher()
if err != nil {
return err
--- /dev/null
+[Unit]
+Description=Arvados Crunch Dispatcher for SLURM
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/crunch-dispatch-slurm
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
projects / arvados.git / commitdiff