end
def can_cancel?
- @proxied.is_a?(ContainerRequest) && state_label.in?(["Queued", "Locked", "Running"]) && priority > 0
+ @proxied.is_a?(ContainerRequest) && @proxied.state == "Committed" && @proxied.priority > 0 && @proxied.editable?
end
def container_uuid
get(:container_uuid)
end
+ def priority
+ @proxied.priority
+ end
+
# For the following properties, use value from the @container if exists
# This applies to a ContainerRequest with container_uuid
get_combined(:runtime_constraints)
end
- def priority
- get_combined(:priority)
- end
-
def log_collection
get_combined(:log)
end
[get_combined(:uuid), get(:uuid)].uniq
end
- def live_log_lines(limit=2000)
- event_types = ["stdout", "stderr", "arv-mount", "crunch-run"]
- log_lines = Log.where(event_type: event_types, object_uuid: log_object_uuids).order("id DESC").limit(limit)
- log_lines.results.reverse.
- flat_map { |log| log.properties[:text].split("\n") rescue [] }
- end
-
def render_log
collection = Collection.find(log_collection) rescue nil
if collection
end
end
- # End combined propeties
+ # End combined properties
protected
def get_combined key
end
def stderr_log_query(limit=nil)
- query = Log.where(event_type: "stderr", object_uuid: self.uuid)
- .order("id DESC")
+ query = Log.where(object_uuid: self.uuid).order("created_at DESC")
query = query.limit(limit) if limit
query
end
[uuid]
end
+ def live_log_lines(limit)
+ Log.where(object_uuid: log_object_uuids).
+ order("created_at DESC").
+ limit(limit).
+ select { |log| log.properties[:text].is_a? String }.
+ reverse.
+ flat_map { |log| log.properties[:text].split("\n") }
+ end
+
protected
def get key, obj=@proxied
<div class="panel panel-default">
<div class="panel-heading">
- <div class="container-fluid">
- <div class="row-fluid">
+ <div class="row">
<div class="col-md-2" style="word-break:break-all;">
<h4 class="panel-title">
<a data-toggle="collapse" href="#collapse<%= i %>">
</div>
<% end %>
</div>
- </div>
</div>
<div id="collapse<%= i %>" class="panel-collapse collapse <%= if expanded then 'in' end %>">
<%# Work unit status %>
-<div class="container-fluid>
- <div class="row-fluid">
- <%# Need additional handling for main object display %>
- <% if @object.uuid == wu.uuid %>
- <div class="container-fluid">
- <div class="pull-right">
- <div class="container-fluid">
- <div class="row-fulid pipeline-instance-spacing">
- <div class="col-md-7">
- <% if wu.is_running? and wu.child_summary_str %>
- <%= wu.child_summary_str %>
- <% end %>
- </div>
- <div class="col-md-3">
- <%= render partial: 'work_units/progress', locals: {wu: wu} %>
- </div>
- <div class="col-md-1">
- <% if wu.can_cancel? and @object.editable? %>
- <%= form_tag "#{wu.uri}/cancel", remote: true, style: "display:inline; padding-left: 1em" do |f| %>
- <%= hidden_field_tag :return_to, url_for(@object) %>
- <%= button_tag "Cancel", {class: 'btn btn-xs btn-danger', id: "cancel-obj-button"} %>
- <% end %>
- <% end %>
- </div>
- </div>
- </div>
- </div>
- </div>
+<div class="row">
+ <div class="col-md-4">
+ <% if wu.is_paused? %>
+ <p>
+ This <%= wu.title %> is paused. Children that are already running
+ will continue to run, but no new processes will be submitted.
+ </p>
<% end %>
- <div class="col-md-10" >
- <% if wu.is_paused? %>
- <p>
- This <%= wu.title %> is paused. Children that are already running
- will continue to run, but no new processes will be submitted.
- </p>
+ <%= raw(wu.show_runtime) %>
+ </div>
+ <%# Need additional handling for main object display %>
+ <% if @object.uuid == wu.uuid %>
+ <div class="col-md-3">
+ <% if wu.is_running? and wu.child_summary_str %>
+ <%= wu.child_summary_str %>
<% end %>
-
- <%= raw(wu.show_runtime) %>
</div>
- </div>
+ <div class="col-md-3">
+ <%= render partial: 'work_units/progress', locals: {wu: wu} %>
+ </div>
+ <div class="col-md-2">
+ <% if wu.can_cancel? and @object.editable? %>
+ <%= form_tag "#{wu.uri}/cancel", remote: true, style: "display:inline; padding-left: 1em" do |f| %>
+ <%= hidden_field_tag :return_to, url_for(@object) %>
+ <%= button_tag "Cancel", {class: 'btn btn-xs btn-danger', id: "cancel-obj-button"} %>
+ <% end %>
+ <% end %>
+ </div>
+ <% end %>
+</div>
<p>
<%= render(partial: 'work_units/component_detail', locals: {current_obj: wu}) %>
end
end
end
-
- [
- ['jobs', 'running_job_with_components', true],
- ['pipeline_instances', 'components_is_jobspec', false],
- ['containers', 'running', false],
- ['container_requests', 'running', true],
- ].each do |type, fixture, cancelable|
- test "cancel button for #{type}/#{fixture}" do
- if cancelable
- need_selenium 'to cancel'
- end
-
- obj = api_fixture(type)[fixture]
- visit page_with_token "active", "/#{type}/#{obj['uuid']}"
-
- assert_text 'created_at'
- if cancelable
- assert page.has_button?('Cancel'), 'No Cancel button'
- click_button 'Cancel'
- wait_for_ajax
- assert page.has_no_button?('Cancel'), 'Cancel button not expected after clicking'
- else
- assert page.has_no_button?('Cancel'), 'Cancel button not expected'
- end
- end
- end
-
- [
- ['jobs', 'running_job_with_components'],
- ['pipeline_instances', 'has_component_with_completed_jobs'],
- ['container_requests', 'running'],
- ['container_requests', 'completed'],
- ].each do |type, fixture|
- test "edit description for #{type}/#{fixture}" do
- obj = api_fixture(type)[fixture]
- visit page_with_token "active", "/#{type}/#{obj['uuid']}"
-
- within('.arv-description-as-subtitle') do
- find('.fa-pencil').click
- find('.editable-input textarea').set('*Textile description for object*')
- find('.editable-submit').click
- end
- wait_for_ajax
-
- # verify description
- assert page.has_no_text? '*Textile description for object*'
- assert page.has_text? 'Textile description for object'
- end
- end
-
- [
- ['Two Part Pipeline Template', 'part-one', 'Provide a value for the following'],
- ['Workflow with input specifications', 'this workflow has inputs specified', 'Provide a value for the following'],
- ].each do |template_name, preview_txt, process_txt|
- test "run a process using template #{template_name} from dashboard" do
- visit page_with_token('admin')
- assert_text 'Recent pipelines and processes' # seeing dashboard now
-
- within('.recent-processes-actions') do
- assert page.has_link?('All processes')
- find('a', text: 'Run a pipeline').click
- end
-
- # in the chooser, verify preview and click Next button
- within('.modal-dialog') do
- find('.selectable', text: template_name).click
- assert_text preview_txt
- find('.btn', text: 'Next: choose inputs').click
- end
-
- # in the process page now
- assert_text process_txt
- assert_selector 'a', text: template_name
- end
- end
end
assert_text 'This workflow does not need any further inputs'
click_link "Run"
wait_for_ajax
- assert_text 'This container is committed'
+ assert_text 'This container is queued'
end
end
page.assert_no_selector 'a.disabled,button.disabled', text: 'Run'
click_link "Run"
wait_for_ajax
- assert_text 'This container is committed'
+ assert_text 'This container is queued'
end
end
class WebsocketTest < ActionDispatch::IntegrationTest
setup do
need_selenium "to make websockets work"
+ @dispatch_client = ArvadosApiClient.new
+ end
+
+ def dispatch_log(body)
+ use_token :dispatch1 do
+ @dispatch_client.api('logs', '', log: body)
+ end
end
test "test page" do
- visit(page_with_token("admin", "/websockets"))
+ visit(page_with_token("active", "/websockets"))
fill_in("websocket-message-content", :with => "Stuff")
click_button("Send")
assert_text '"status":400'
end
- test "test live logging" do
- visit(page_with_token("admin", "/pipeline_instances/zzzzz-d1hrv-9fm8l10i9z2kqc6"))
- click_link("Log")
- assert_no_text '123 hello'
-
- api = ArvadosApiClient.new
+ [
+ ['pipeline_instances', 'pipeline_in_running_state', api_fixture('jobs')['running']],
+ ['jobs', 'running'],
+ ['containers', 'running'],
+ ['container_requests', 'running', api_fixture('containers')['running']],
+ ].each do |controller, view_fixture_name, log_target_fixture|
+ view_fixture = api_fixture(controller)[view_fixture_name]
+ log_target_fixture ||= view_fixture
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
- api.api("logs", "", {log: {
- object_uuid: "zzzzz-d1hrv-9fm8l10i9z2kqc6",
- event_type: "stderr",
- properties: {"text" => "123 hello"}}})
- assert_text '123 hello'
- end
+ test "test live logging and scrolling for #{controller}" do
- [
- ["pipeline_instances", api_fixture("pipeline_instances")['pipeline_with_newer_template']['uuid']],
- ["jobs", api_fixture("jobs")['running']['uuid']],
- ["containers", api_fixture("containers")['running']['uuid']],
- ["container_requests", api_fixture("container_requests")['running']['uuid'], api_fixture("containers")['running']['uuid']],
- ].each do |c|
- test "test live logging scrolling #{c[0]}" do
-
- controller = c[0]
- uuid = c[1]
- log_uuid = c[2] || c[1]
-
- visit(page_with_token("admin", "/#{controller}/#{uuid}"))
- click_link("Log")
+ visit(page_with_token("active", "/#{controller}/#{view_fixture['uuid']}\#Log"))
assert_no_text '123 hello'
- api = ArvadosApiClient.new
-
text = ""
(1..1000).each do |i|
text << "#{i} hello\n"
end
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
- api.api("logs", "", {log: {
- object_uuid: log_uuid,
- event_type: "stderr",
- properties: {"text" => text}}})
+ dispatch_log(owner_uuid: log_target_fixture['owner_uuid'],
+ object_uuid: log_target_fixture['uuid'],
+ event_type: "stderr",
+ properties: {"text" => text})
assert_text '1000 hello'
# First test that when we're already at the bottom of the page, it scrolls down
# when a new line is added.
old_top = page.evaluate_script("$('#event_log_div').scrollTop()")
- api.api("logs", "", {log: {
- object_uuid: log_uuid,
- event_type: "stderr",
- properties: {"text" => "1001 hello\n"}}})
+ dispatch_log(owner_uuid: log_target_fixture['owner_uuid'],
+ object_uuid: log_target_fixture['uuid'],
+ event_type: "dispatch",
+ properties: {"text" => "1001 hello\n"})
assert_text '1001 hello'
# Check that new value of scrollTop is greater than the old one
page.execute_script "$('#event_log_div').scrollTop(30)"
assert_equal 30, page.evaluate_script("$('#event_log_div').scrollTop()")
- api.api("logs", "", {log: {
- object_uuid: log_uuid,
- event_type: "stderr",
- properties: {"text" => "1002 hello\n"}}})
+ dispatch_log(owner_uuid: log_target_fixture['owner_uuid'],
+ object_uuid: log_target_fixture['uuid'],
+ event_type: "stdout",
+ properties: {"text" => "1002 hello\n"})
assert_text '1002 hello'
# Check that we haven't changed scroll position
end
test "pipeline instance arv-refresh-on-log-event" do
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
# Do something and check that the pane reloads.
- p = PipelineInstance.create({state: "RunningOnServer",
- components: {
- c1: {
- script: "test_hash.py",
- script_version: "1de84a854e2b440dc53bf42f8548afa4c17da332"
- }
- }
- })
-
- visit(page_with_token("admin", "/pipeline_instances/#{p.uuid}"))
+ p = use_token :active do
+ PipelineInstance.create(state: "RunningOnServer",
+ components: {
+ c1: {
+ script: "test_hash.py",
+ script_version: "1de84a854e2b440dc53bf42f8548afa4c17da332"
+ }
+ })
+ end
+ visit(page_with_token("active", "/pipeline_instances/#{p.uuid}"))
assert_text 'Active'
assert page.has_link? 'Pause'
assert_no_text 'Complete'
assert page.has_no_link? 'Re-run with latest'
- p.state = "Complete"
- p.save!
+ use_token :dispatch1 do
+ p.update_attributes!(state: 'Complete')
+ end
assert_no_text 'Active'
assert page.has_no_link? 'Pause'
end
test "job arv-refresh-on-log-event" do
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
# Do something and check that the pane reloads.
- p = Job.where(uuid: api_fixture('jobs')['running_will_be_completed']['uuid']).results.first
-
- visit(page_with_token("admin", "/jobs/#{p.uuid}"))
+ uuid = api_fixture('jobs')['running_will_be_completed']['uuid']
+ visit(page_with_token("active", "/jobs/#{uuid}"))
assert_no_text 'complete'
assert_no_text 'Re-run job'
- p.state = "Complete"
- p.save!
+ use_token :dispatch1 do
+ Job.find(uuid).update_attributes!(state: 'Complete')
+ end
assert_text 'complete'
assert_text 'Re-run job'
end
test "dashboard arv-refresh-on-log-event" do
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
-
- visit(page_with_token("admin", "/"))
+ visit(page_with_token("active", "/"))
assert_no_text 'test dashboard arv-refresh-on-log-event'
# Do something and check that the pane reloads.
- p = PipelineInstance.create({state: "RunningOnServer",
- name: "test dashboard arv-refresh-on-log-event",
- components: {
- }
- })
+ use_token :active do
+ p = PipelineInstance.create({state: "RunningOnServer",
+ name: "test dashboard arv-refresh-on-log-event",
+ components: {
+ }
+ })
+ end
assert_text 'test dashboard arv-refresh-on-log-event'
end
text = "2014-11-07_23:33:51 #{uuid} 31708 1 stderr crunchstat: cpu 1970.8200 user 60.2700 sys 8 cpus -- interval 10.0002 seconds 35.3900 user 0.8600 sys"
assert_triggers_dom_event 'arv-log-event' do
- use_token :active do
- api = ArvadosApiClient.new
- api.api("logs", "", {log: {
- object_uuid: uuid,
- event_type: "stderr",
- properties: {"text" => text}}})
- end
+ dispatch_log(owner_uuid: api_fixture('jobs')['running']['owner_uuid'],
+ object_uuid: uuid,
+ event_type: "stderr",
+ properties: {"text" => text})
end
# Graph should have appeared (even if it hadn't above). It's
end
test "test running job with just a few previous log records" do
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
- job = Job.where(uuid: api_fixture("jobs")['running']['uuid']).results.first
- visit page_with_token("admin", "/jobs/#{job.uuid}")
-
- api = ArvadosApiClient.new
+ job = api_fixture("jobs")['running']
# Create just one old log record
- api.api("logs", "", {log: {
- object_uuid: job.uuid,
- event_type: "stderr",
- properties: {"text" => "Historic log message"}}})
+ dispatch_log(owner_uuid: job['owner_uuid'],
+ object_uuid: job['uuid'],
+ event_type: "stderr",
+ properties: {"text" => "Historic log message"})
- click_link("Log")
+ visit page_with_token("active", "/jobs/#{job['uuid']}\#Log")
# Expect "all" historic log records because we have less than
# default Rails.configuration.running_job_log_records_to_fetch count
assert_text 'Historic log message'
# Create new log record and expect it to show up in log tab
- api.api("logs", "", {log: {
- object_uuid: job.uuid,
- event_type: "stderr",
- properties: {"text" => "Log message after subscription"}}})
+ dispatch_log(owner_uuid: job['owner_uuid'],
+ object_uuid: job['uuid'],
+ event_type: "stderr",
+ properties: {"text" => "Log message after subscription"})
assert_text 'Log message after subscription'
end
test "test running job with too many previous log records" do
- Rails.configuration.running_job_log_records_to_fetch = 5
-
- Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
- job = Job.where(uuid: api_fixture("jobs")['running']['uuid']).results.first
-
- visit page_with_token("admin", "/jobs/#{job.uuid}")
-
- api = ArvadosApiClient.new
-
- # Create Rails.configuration.running_job_log_records_to_fetch + 1 log records
- (0..Rails.configuration.running_job_log_records_to_fetch).each do |count|
- api.api("logs", "", {log: {
- object_uuid: job.uuid,
- event_type: "stderr",
- properties: {"text" => "Old log message #{count}"}}})
+ max = 5
+ Rails.configuration.running_job_log_records_to_fetch = max
+ job = api_fixture("jobs")['running']
+
+ # Create max+1 log records
+ (0..max).each do |count|
+ dispatch_log(owner_uuid: job['owner_uuid'],
+ object_uuid: job['uuid'],
+ event_type: "stderr",
+ properties: {"text" => "Old log message #{count}"})
end
- # Go to log tab, which results in subscribing to websockets
- click_link("Log")
+ visit page_with_token("active", "/jobs/#{job['uuid']}\#Log")
# Expect all but the first historic log records,
# because that was one too many than fetch count.
- (1..Rails.configuration.running_job_log_records_to_fetch).each do |count|
+ (1..max).each do |count|
assert_text "Old log message #{count}"
end
assert_no_text 'Old log message 0'
# Create one more log record after subscription
- api.api("logs", "", {log: {
- object_uuid: job.uuid,
- event_type: "stderr",
- properties: {"text" => "Life goes on!"}}})
+ dispatch_log(owner_uuid: job['owner_uuid'],
+ object_uuid: job['uuid'],
+ event_type: "stderr",
+ properties: {"text" => "Life goes on!"})
+
# Expect it to show up in log tab
assert_text 'Life goes on!'
end
assert_no_selector "a[href=\"#{link}\"]"
end
end
+
+ [
+ ['jobs', 'running_job_with_components', true],
+ ['pipeline_instances', 'components_is_jobspec', false],
+ ['containers', 'running', false],
+ ['container_requests', 'running', true],
+ ].each do |type, fixture, cancelable|
+ test "cancel button for #{type}/#{fixture}" do
+ if cancelable
+ need_selenium 'to cancel'
+ end
+
+ obj = api_fixture(type)[fixture]
+ visit page_with_token "active", "/#{type}/#{obj['uuid']}"
+
+ assert_text 'created_at'
+ if cancelable
+ assert_selector 'button', text: 'Cancel'
+ click_button 'Cancel'
+ wait_for_ajax
+ end
+ assert_no_selector 'button', text: 'Cancel'
+ end
+ end
+
+ [
+ ['jobs', 'running_job_with_components'],
+ ['pipeline_instances', 'has_component_with_completed_jobs'],
+ ['container_requests', 'running'],
+ ['container_requests', 'completed'],
+ ].each do |type, fixture|
+ test "edit description for #{type}/#{fixture}" do
+ obj = api_fixture(type)[fixture]
+ visit page_with_token "active", "/#{type}/#{obj['uuid']}"
+
+ within('.arv-description-as-subtitle') do
+ find('.fa-pencil').click
+ find('.editable-input textarea').set('*Textile description for object*')
+ find('.editable-submit').click
+ end
+ wait_for_ajax
+
+ # verify description
+ assert page.has_no_text? '*Textile description for object*'
+ assert page.has_text? 'Textile description for object'
+ end
+ end
+
+ [
+ ['Two Part Pipeline Template', 'part-one', 'Provide a value for the following'],
+ ['Workflow with input specifications', 'this workflow has inputs specified', 'Provide a value for the following'],
+ ].each do |template_name, preview_txt, process_txt|
+ test "run a process using template #{template_name} from dashboard" do
+ visit page_with_token('admin')
+ assert_text 'Recent pipelines and processes' # seeing dashboard now
+
+ within('.recent-processes-actions') do
+ assert page.has_link?('All processes')
+ find('a', text: 'Run a pipeline').click
+ end
+
+ # in the chooser, verify preview and click Next button
+ within('.modal-dialog') do
+ find('.selectable', text: template_name).click
+ assert_text preview_txt
+ find('.btn', text: 'Next: choose inputs').click
+ end
+
+ # in the process page now
+ assert_text process_txt
+ assert_selector 'a', text: template_name
+ end
+ end
end
require 'yaml'
def available_port for_what
- Addrinfo.tcp("0.0.0.0", 0).listen do |srv|
- port = srv.connect_address.ip_port
- STDERR.puts "Using port #{port} for #{for_what}"
- return port
+ begin
+ Addrinfo.tcp("0.0.0.0", 0).listen do |srv|
+ port = srv.connect_address.ip_port
+ # Selenium needs an additional locking port, check if it's available
+ # and retry if necessary.
+ if for_what == 'selenium'
+ locking_port = port - 1
+ Addrinfo.tcp("0.0.0.0", locking_port).listen.close
+ end
+ STDERR.puts "Using port #{port} for #{for_what}"
+ return port
+ end
+ rescue Errno::EADDRINUSE, Errno::EACCES
+ retry
end
end
# Note: You'll currently still have to declare fixtures explicitly
# in integration tests -- they do not yet inherit this setting
fixtures :all
- def use_token token_name
- was = Thread.current[:arvados_api_token]
+ def use_token(token_name)
+ user_was = Thread.current[:user]
+ token_was = Thread.current[:arvados_api_token]
auth = api_fixture('api_client_authorizations')[token_name.to_s]
Thread.current[:arvados_api_token] = auth['api_token']
if block_given?
begin
yield
ensure
- Thread.current[:arvados_api_token] = was
+ Thread.current[:user] = user_was
+ Thread.current[:arvados_api_token] = token_was
end
end
end
end
end
end
+
+ test 'can_cancel?' do
+ use_token 'active' do
+ assert find_fixture(Job, 'running').work_unit.can_cancel?
+ refute find_fixture(Container, 'running').work_unit.can_cancel?
+ assert find_fixture(ContainerRequest, 'running').work_unit.can_cancel?
+ end
+ use_token 'spectator' do
+ refute find_fixture(ContainerRequest, 'running_anonymous_accessible').work_unit.can_cancel?
+ end
+ use_token 'admin' do
+ assert find_fixture(ContainerRequest, 'running_anonymous_accessible').work_unit.can_cancel?
+ end
+ end
end
FORMAT=deb
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ rsa uritemplate httplib2 ws4py pykka six \
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol cwltest)
+ cachecontrol)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
;;
debian8)
FORMAT=deb
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ rsa uritemplate httplib2 ws4py pykka six \
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol cwltest)
+ cachecontrol)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
;;
ubuntu1204)
FORMAT=deb
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ rsa uritemplate httplib2 ws4py pykka six \
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro isodate ruamel.ordereddict
- cachecontrol cwltest)
+ cachecontrol)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
;;
ubuntu1404)
google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol cwltest)
+ cachecontrol)
PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client)
;;
centos6)
PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ rsa uritemplate httplib2 ws4py pykka six \
ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
- python-daemon lockfile llfuse==0.41.1 'pbr<1.0' pyyaml \
+ python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
'rdflib>=4.2.0' shellescape mistune typing avro requests \
isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
- ruamel.ordereddict cachecontrol cwltest)
+ ruamel.ordereddict cachecontrol)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
export PYCURL_SSL_LIBRARY=nss
;;
PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \
oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py pykka pyexecjs jsonschema \
+ rsa uritemplate httplib2 ws4py pykka \
ciso8601 pycrypto 'pycurl<7.21.5' \
python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
'rdflib>=4.2.0' shellescape mistune typing avro \
isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
- ruamel.ordereddict cachecontrol cwltest)
+ ruamel.ordereddict cachecontrol)
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client)
export PYCURL_SSL_LIBRARY=nss
;;
rm -rf "$WORKSPACE/sdk/cwl/build"
fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --iteration 3
+fpm_build lockfile "" "" python 0.12.2 --epoch 1
+
# schema_salad. This is a python dependency of arvados-cwl-runner,
# but we can't use the usual PYTHONPACKAGES way to build this package due to the
# intricacies of how version numbers get generated in setup.py: we need version
# So we build this thing separately.
#
# Ward, 2016-03-17
-fpm_build schema_salad "" "" python 1.17.20160820171034
+fpm_build schema_salad "" "" python 1.18.20160907135919 --depends "python-lockfile >= 1:0.12.2-2"
# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
# Ward, 2016-05-26
fpm_build ruamel.yaml "" "" python 0.12.4 --python-setup-py-arguments "--single-version-externally-managed"
+# Dependency of cwltool. Fpm doesn't produce a package with the correct version
+# number unless we build it explicitly
+fpm_build cwltest "" "" python 1.0.20160907111242
+
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
-fpm_build cwltool "" "" python 1.0.20160901133827
+fpm_build cwltool "" "" python 1.0.20160907141844
# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25
fpm_build rdflib-jsonld "" "" python 0.3.0
# Make sure to update arvados/build/run-build-packages.sh as well
# when updating the cwltool version pin.
install_requires=[
- 'cwltool==1.0.20160901133827',
- 'arvados-python-client>=0.1.20160714204738',
+ 'cwltool==1.0.20160907141844',
+ 'arvados-python-client>=0.1.20160714204738'
],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
accept_attribute_as_json :mounts, Hash
accept_attribute_as_json :runtime_constraints, Hash
accept_attribute_as_json :command, Array
+ accept_attribute_as_json :filters, Array
end
return self
end
- # Collect the uuids for each user and any groups readable by each user.
+ # Collect the UUIDs of the authorized users.
user_uuids = users_list.map { |u| u.uuid }
- uuid_list = user_uuids + users_list.flat_map { |u| u.groups_i_can(:read) }
+
+ # Collect the UUIDs of all groups readable by any of the
+ # authorized users. If one of these (or the UUID of one of the
+ # authorized users themselves) is an object's owner_uuid, that
+ # object is readable.
+ owner_uuids = user_uuids + users_list.flat_map { |u| u.groups_i_can(:read) }
+ owner_uuids.uniq!
+
sql_conds = []
- sql_params = []
sql_table = kwargs.fetch(:table_name, table_name)
- or_object_uuid = ''
-
- # This row is owned by a member of users_list, or owned by a group
- # readable by a member of users_list
- # or
- # This row uuid is the uuid of a member of users_list
- # or
- # A permission link exists ('write' and 'manage' implicitly include
- # 'read') from a member of users_list, or a group readable by users_list,
- # to this row, or to the owner of this row (see join() below).
- sql_conds += ["#{sql_table}.uuid in (?)"]
- sql_params += [user_uuids]
-
- if uuid_list.any?
- sql_conds += ["#{sql_table}.owner_uuid in (?)"]
- sql_params += [uuid_list]
-
- sanitized_uuid_list = uuid_list.
- collect { |uuid| sanitize(uuid) }.join(', ')
- permitted_uuids = "(SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (#{sanitized_uuid_list}))"
- sql_conds += ["#{sql_table}.uuid IN #{permitted_uuids}"]
- end
-
- if sql_table == "links" and users_list.any?
- # This row is a 'permission' or 'resources' link class
- # The uuid for a member of users_list is referenced in either the head
- # or tail of the link
- sql_conds += ["(#{sql_table}.link_class in (#{sanitize 'permission'}, #{sanitize 'resources'}) AND (#{sql_table}.head_uuid IN (?) OR #{sql_table}.tail_uuid IN (?)))"]
- sql_params += [user_uuids, user_uuids]
- end
-
- if sql_table == "logs" and users_list.any?
- # Link head points to the object described by this row
- sql_conds += ["#{sql_table}.object_uuid IN #{permitted_uuids}"]
-
- # This object described by this row is owned by this user, or owned by a group readable by this user
- sql_conds += ["#{sql_table}.object_owner_uuid in (?)"]
- sql_params += [uuid_list]
- end
-
- # Link head points to this row, or to the owner of this row (the
- # thing to be read)
- #
- # Link tail originates from this user, or a group that is readable
- # by this user (the identity with authorization to read)
- #
- # Link class is 'permission' ('write' and 'manage' implicitly
- # include 'read')
- where(sql_conds.join(' OR '), *sql_params)
+
+ # Match any object (evidently a group or user) whose UUID is
+ # listed explicitly in owner_uuids.
+ sql_conds += ["#{sql_table}.uuid in (:owner_uuids)"]
+
+ # Match any object whose owner is listed explicitly in
+ # owner_uuids.
+ sql_conds += ["#{sql_table}.owner_uuid IN (:owner_uuids)"]
+
+ # Match the head of any permission link whose tail is listed
+ # explicitly in owner_uuids.
+ sql_conds += ["#{sql_table}.uuid IN (SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (:owner_uuids))"]
+
+ if sql_table == "links"
+ # Match any permission link that gives one of the authorized
+ # users some permission _or_ gives anyone else permission to
+ # view one of the authorized users.
+ sql_conds += ["(#{sql_table}.link_class in (:permission_link_classes) AND "+
+ "(#{sql_table}.head_uuid IN (:user_uuids) OR #{sql_table}.tail_uuid IN (:user_uuids)))"]
+ end
+
+ where(sql_conds.join(' OR '),
+ owner_uuids: owner_uuids,
+ user_uuids: user_uuids,
+ permission_link_classes: ['permission', 'resources'])
end
def logged_attributes
end
end
+ def self.readable_by(*users_list)
+ if users_list.select { |u| u.is_admin }.any?
+ return self
+ end
+ user_uuids = users_list.map { |u| u.uuid }
+ uuid_list = user_uuids + users_list.flat_map { |u| u.groups_i_can(:read) }
+ uuid_list.uniq!
+ permitted = "(SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (:uuids))"
+ joins(:container_requests).
+ where("container_requests.uuid IN #{permitted} OR "+
+ "container_requests.owner_uuid IN (:uuids)",
+ uuids: uuid_list)
+ end
+
protected
def fill_field_defaults
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ extend CurrentApiClient
serialize :components, Hash
attr_protected :arvados_sdk_version, :docker_image_locator
serialize :script_parameters, Hash
self
end
+ def self.readable_by(*users_list)
+ if users_list.select { |u| u.is_admin }.any?
+ return self
+ end
+ user_uuids = users_list.map { |u| u.uuid }
+ uuid_list = user_uuids + users_list.flat_map { |u| u.groups_i_can(:read) }
+ uuid_list.uniq!
+ permitted = "(SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (:uuids))"
+ joins("LEFT JOIN container_requests ON container_requests.container_uuid=logs.object_uuid").
+ where("logs.object_uuid IN #{permitted} OR "+
+ "container_requests.uuid IN (:uuids) OR "+
+ "container_requests.owner_uuid IN (:uuids) OR "+
+ "logs.object_uuid IN (:uuids) OR "+
+ "logs.owner_uuid IN (:uuids) OR "+
+ "logs.object_owner_uuid IN (:uuids)",
+ uuids: uuid_list)
+ end
+
protected
def permission_to_create
#
# Note: find_each implies order('id asc'), which is what we
# want.
- logs.select(:id).find_each do |l|
+ logs.select('logs.id').find_each do |l|
if not ws.sent_ids.include?(l.id)
# only send if not a duplicate
ws.send(Log.find(l.id).as_api_response.to_json)
+require 'current_api_client'
+
module SimulateJobLog
+ include CurrentApiClient
def replay(filename, multiplier = 1, simulated_job_uuid = nil)
raise "Environment must be development or test" unless [ 'test', 'development' ].include? ENV['RAILS_ENV']
factory :api_client do
is_trusted false
to_create do |instance|
- act_as_system_user do
+ CurrentApiClientHelper.act_as_system_user do
instance.save!
end
end
end
to_create do |instance|
- act_as_user instance.user do
+ CurrentApiClientHelper.act_as_user instance.user do
instance.save!
end
end
-include CurrentApiClient
+class CurrentApiClientHelper
+ extend CurrentApiClient
+end
FactoryGirl.define do
factory :user do
join_groups []
end
after :create do |user, evaluator|
- act_as_system_user do
+ CurrentApiClientHelper.act_as_system_user do
evaluator.join_groups.each do |g|
Link.create!(tail_uuid: user.uuid,
head_uuid: g.uuid,
factory :active_user do
is_active true
after :create do |user|
- act_as_system_user do
+ CurrentApiClientHelper.act_as_system_user do
Link.create!(tail_uuid: user.uuid,
head_uuid: Group.where('uuid ~ ?', '-f+$').first.uuid,
link_class: 'permission',
end
end
to_create do |instance|
- act_as_system_user do
+ CurrentApiClientHelper.act_as_system_user do
instance.save!
end
end
vcpus: 1
ram: 123
-running-older:
+running_older:
uuid: zzzzz-xvhdp-cr4runningcntn2
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
name: running
vcpus: 1
ram: 123
+requester:
+ uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: requester
+ state: Committed
+ priority: 1
+ created_at: 2016-01-11 11:11:11.111111111 Z
+ updated_at: 2016-01-11 11:11:11.111111111 Z
+ modified_at: 2016-01-11 11:11:11.111111111 Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: test
+ cwd: /
+ output_path: /output
+ command: ["request-another-container", "echo", "hello"]
+ container_uuid: zzzzz-dz642-requestingcntnr
+ runtime_constraints:
+ vcpus: 1
+ ram: 123
+
cr_for_requester:
uuid: zzzzz-xvhdp-cr4requestercnt
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
queued:
uuid: zzzzz-dz642-queuedcontainer
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Queued
priority: 1
created_at: 2016-01-11 11:11:11.111111111 Z
running:
uuid: zzzzz-dz642-runningcontainr
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- state: Running
+ owner_uuid: zzzzz-tpzed-000000000000000
priority: 1
created_at: <%= 1.minute.ago.to_s(:db) %>
updated_at: <%= 1.minute.ago.to_s(:db) %>
vcpus: 4
auth_uuid: zzzzz-gj3su-077z32aux8dg2s1
-running-older:
+running_older:
uuid: zzzzz-dz642-runningcontain2
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Running
priority: 1
created_at: <%= 2.minute.ago.to_s(:db) %>
locked:
uuid: zzzzz-dz642-lockedcontainer
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Locked
priority: 2
created_at: <%= 2.minute.ago.to_s(:db) %>
completed:
uuid: zzzzz-dz642-compltcontainer
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Complete
exit_code: 0
priority: 1
completed_older:
uuid: zzzzz-dz642-compltcontainr2
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Complete
exit_code: 0
priority: 1
requester:
uuid: zzzzz-dz642-requestingcntnr
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Complete
exit_code: 0
priority: 1
requester_container:
uuid: zzzzz-dz642-requestercntnr1
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
state: Complete
exit_code: 0
priority: 1
-noop:
+noop: # nothing happened ...to the 'spectator' user
id: 1
uuid: zzzzz-xxxxx-pshmckwoma9plh7
+ owner_uuid: zzzzz-tpzed-000000000000000
object_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ object_owner_uuid: zzzzz-tpzed-000000000000000
event_at: <%= 1.minute.ago.to_s(:db) %>
admin_changes_repository2: # admin changes repository2, which is owned by active user
require 'test_helper'
class PermissionsTest < ActionDispatch::IntegrationTest
+ include DbCurrentTime
include CurrentApiClient # for empty_collection
fixtures :users, :groups, :api_client_authorizations, :collections
+ teardown do
+ User.invalidate_permissions_cache db_current_time.to_i
+ end
+
test "adding and removing direct can_read links" do
# try to read collection as spectator
get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
end
- test "get_permissions returns 404 for unreadable uuid" do
- get "/arvados/v1/permissions/#{groups(:public).uuid}", nil, auth(:active)
- assert_response 404
- end
-
test "get_permissions returns 403 if user can read but not manage" do
post "/arvados/v1/links", {
:link => {
test "connect, subscribe and get response" do
status = nil
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe'}.to_json)
end
spec = nil
ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe'}.to_json)
end
spec_ev_uuid = nil
human_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe'}.to_json)
end
human = nil
human_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
end
spec_ev_uuid = nil
human_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#specimen']]}.to_json)
state = 1
t1 = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#trait'], ['event_type', '=', 'update']]}.to_json)
end
human = nil
human_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
lastid = logs(:admin_changes_specimen).id
l1 = nil
l2 = nil
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe', last_log_id: lastid}.to_json)
end
spec_ev_uuid = nil
filter_id = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin, false do |ws|
+ ws_helper :active, false do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe'}.to_json)
EM::Timer.new 3 do
spec = nil
spec_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin, false do |ws|
+ ws_helper :active, false do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe', filters: [['object_uuid', 'is_a', 'arvados#human']]}.to_json)
EM::Timer.new 6 do
human = nil
human_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'subscribe'}.to_json)
end
test "connected, not subscribed, no event" do
slow_test
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin, false do |ws|
+ ws_helper :active, false do |ws|
ws.on :open do |event|
EM::Timer.new 1 do
Specimen.create
test "connect, try bogus method" do
status = nil
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({method: 'frobnabble'}.to_json)
end
test "connect, missing method" do
status = nil
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send ({fizzbuzz: 'frobnabble'}.to_json)
end
test "connect, send malformed request" do
status = nil
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
ws.send '<XML4EVER></XML4EVER>'
end
test "connect, try subscribe too many filters" do
state = 1
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
(1..17).each do |i|
ws.send ({method: 'subscribe', filters: [['object_uuid', '=', i]]}.to_json)
event_count = 0
log_start = Log.order(:id).last.id
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin, false do |ws|
+ ws_helper :active, false do |ws|
EM::Timer.new 45 do
# Needs a longer timeout than the default
ws.close
human = nil
human_ev_uuid = nil
- authorize_with :admin
+ authorize_with :active
- ws_helper :admin do |ws|
+ ws_helper :active do |ws|
ws.on :open do |event|
# test that #6451 is fixed (invalid filter crashes websockets)
ws.send ({method: 'subscribe', filters: [['object_blarg', 'is_a', 'arvados#human']]}.to_json)
fixtures :all
include ArvadosTestSupport
+ include CurrentApiClient
setup do
Rails.logger.warn "\n\n#{'=' * 70}\n#{self.class}\##{method_name}\n#{'-' * 70}\n\n"
end
end
end
+
+ test "requestor can retrieve container owned by dispatch" do
+ assert_not_empty Container.readable_by(users(:admin)).where(uuid: containers(:running).uuid)
+ assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid)
+ assert_empty Container.readable_by(users(:spectator)).where(uuid: containers(:running).uuid)
+ end
end
:crunchstat_for_running_job] # log & job owned by active
c = Log.readable_by(users(:spectator)).order("id asc").each.to_a
- assert_log_result c, known_logs, [:admin_changes_specimen, # owned by spectator
+ assert_log_result c, known_logs, [:noop, # object_uuid is spectator
+ :admin_changes_specimen, # object_uuid is a specimen owned by spectator
:system_adds_baz] # readable via 'all users' group
end
ob.update_attributes!(owner_uuid: groups(:aproject).uuid)
end
end
+
+ def container_logs(container, user)
+ Log.readable_by(users(user)).
+ where(object_uuid: containers(container).uuid, event_type: "test")
+ end
+
+ test "container logs created by dispatch are visible to container requestor" do
+ set_user_from_auth :dispatch1
+ Log.create!(object_uuid: containers(:running).uuid,
+ event_type: "test")
+
+ assert_not_empty container_logs(:running, :admin)
+ assert_not_empty container_logs(:running, :active)
+ assert_empty container_logs(:running, :spectator)
+ end
+
+ test "container logs created by dispatch are public if container request is public" do
+ set_user_from_auth :dispatch1
+ Log.create!(object_uuid: containers(:running_older).uuid,
+ event_type: "test")
+
+ assert_not_empty container_logs(:running_older, :anonymous)
+ end
end
projects / arvados.git / commitdiff