*.pyc
docker/*/generated/*
docker/config.yml
-doc/_site/*
doc/.site/*
-doc/sdk/python/arvados
\ No newline at end of file
+doc/sdk/python/arvados
+sdk/perl/MYMETA.*
+sdk/perl/Makefile
+sdk/perl/blib/*
+sdk/perl/pm_to_blib
gem 'selenium-webdriver'
gem 'capybara'
gem 'poltergeist'
+ gem 'capybara-webkit'
end
gem 'jquery-rails'
gem 'piwik_analytics'
gem 'httpclient'
gem 'themes_for_rails'
-gem "deep_merge", :require => 'deep_merge/rails_compat'
\ No newline at end of file
+gem "deep_merge", :require => 'deep_merge/rails_compat'
rack (>= 1.0.0)
rack-test (>= 0.5.4)
xpath (~> 2.0)
+ capybara-webkit (1.1.0)
+ capybara (~> 2.0, >= 2.0.2)
+ json
childprocess (0.5.1)
ffi (~> 1.0, >= 1.0.11)
cliver (0.3.2)
bootstrap-sass (~> 3.1.0)
bootstrap-x-editable-rails
capybara
+ capybara-webkit
coffee-rails (~> 3.2.0)
deep_merge
httpclient
The Workbench application includes a series of integration tests. When you run these, it starts the API server in a test environment, with all of its fixtures loaded, then tests Workbench by starting that server and making requests against it.
+Before running @bundle install@, make sure you install QT development dependencies (otherwise, capybara-webkit installation will fail). For example, on a Debian or Ubuntu system:
+
+<pre>
+arvados/apps/workbench$ sudo apt-get install qt4-qmake libqt4-dev
+arvados/apps/workbench$ RAILS_ENV=test bundle install
+</pre>
+
In addition to bundled gems, running the integration tests requires "PhantomJS":http://phantomjs.org/download.html to test JavaScript elements. The simplest way to get started is to download one of the binary builds provided, and install the executable into one of the directories in your @$PATH@.
+<pre>
+$ cd /tmp
+/tmp$ wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2
+/tmp$ tar xjf phantomjs-1.9.7-linux-x86_64.tar.bz2
+/tmp$ sudo cp -ip phantomjs-1.9.7-linux-x86_64/bin/phantomjs /usr/local/bin/
+</pre>
+
+If you install the Workbench Bundle in deployment mode, you must also install the API server Bundle in deployment mode, and vice versa. If your Bundle installs have mismatched modes, the integration tests will fail with "Gem not found" errors.
+
h2. Writing tests
Integration tests are written with Capybara, which drives a fully-featured Web browser to interact with Workbench exactly as a user would.
//= require bootstrap/tooltip
//= require bootstrap/popover
//= require bootstrap/collapse
+//= require bootstrap/modal
//= require bootstrap3-editable/bootstrap-editable
//= require_tree .
def breadcrumb_page_name
(@breadcrumb_page_name ||
- (@object.friendly_link_name if @object.respond_to? :friendly_link_name))
+ (@object.friendly_link_name if @object.respond_to? :friendly_link_name) ||
+ action_name)
end
def index_pane_list
p.components.each do |k, v|
j = v[:job] || next
+ # The graph is interested in whether the component is
+ # indicated as persistent, more than whether the job
+ # satisfying it (which could have been reused, or someone
+ # else's) is.
+ j[:output_is_persistent] = v[:output_is_persistent]
+
uuid = j[:uuid].intern
provenance[uuid] = j
pips[uuid] = 0 unless pips[uuid] != nil
class UsersController < ApplicationController
- skip_before_filter :find_object_by_uuid, :only => :welcome
+ skip_before_filter :find_object_by_uuid, :only => [:welcome, :activity]
skip_around_filter :thread_with_mandatory_api_token, :only => :welcome
- before_filter :ensure_current_user_is_admin, only: :sudo
+ before_filter :ensure_current_user_is_admin, only: [:sudo, :unsetup, :setup]
def welcome
if current_user
end
end
+ def activity
+ @breadcrumb_page_name = nil
+ @users = User.limit(params[:limit] || 1000).all
+ @user_activity = {}
+ @activity = {
+ logins: {},
+ jobs: {},
+ pipeline_instances: {}
+ }
+ @total_activity = {}
+ @spans = [['This week', Time.now.beginning_of_week, Time.now],
+ ['Last week',
+ Time.now.beginning_of_week.advance(weeks:-1),
+ Time.now.beginning_of_week],
+ ['This month', Time.now.beginning_of_month, Time.now],
+ ['Last month',
+ 1.month.ago.beginning_of_month,
+ Time.now.beginning_of_month]]
+ @spans.each do |span, threshold_start, threshold_end|
+ @activity[:logins][span] = Log.
+ filter([[:event_type, '=', 'login'],
+ [:object_kind, '=', 'arvados#user'],
+ [:created_at, '>=', threshold_start],
+ [:created_at, '<', threshold_end]])
+ @activity[:jobs][span] = Job.
+ filter([[:created_at, '>=', threshold_start],
+ [:created_at, '<', threshold_end]])
+ @activity[:pipeline_instances][span] = PipelineInstance.
+ filter([[:created_at, '>=', threshold_start],
+ [:created_at, '<', threshold_end]])
+ @activity.each do |type, act|
+ records = act[span]
+ @users.each do |u|
+ @user_activity[u.uuid] ||= {}
+ @user_activity[u.uuid][span + ' ' + type.to_s] ||= 0
+ end
+ records.each do |record|
+ @user_activity[record.modified_by_user_uuid] ||= {}
+ @user_activity[record.modified_by_user_uuid][span + ' ' + type.to_s] ||= 0
+ @user_activity[record.modified_by_user_uuid][span + ' ' + type.to_s] += 1
+ @total_activity[span + ' ' + type.to_s] ||= 0
+ @total_activity[span + ' ' + type.to_s] += 1
+ end
+ end
+ end
+ @users = @users.sort_by do |a|
+ [-@user_activity[a.uuid].values.inject(:+), a.full_name]
+ end
+ # Prepend a "Total" pseudo-user to the sorted list
+ @user_activity[nil] = @total_activity
+ @users = [OpenStruct.new(uuid: nil)] + @users
+ end
+
def show_pane_list
if current_user.andand.is_admin
super | %w(Admin)
end
end
+ def index_pane_list
+ if current_user.andand.is_admin
+ super | %w(Activity)
+ else
+ super
+ end
+ end
+
def sudo
resp = $arvados_api_client.api(ApiClientAuthorization, '', {
api_client_authorization: {
f.html { render template: 'users/home' }
end
end
+
+ def unsetup
+ if current_user.andand.is_admin
+ @object.unsetup
+ end
+ show
+ end
+
+ def setup
+ respond_to do |format|
+ if current_user.andand.is_admin
+ setup_params = {}
+ if params['user_uuid'] && params['user_uuid'].size>0
+ setup_params[:uuid] = params['user_uuid']
+ end
+ if params['email'] && params['email'].size>0
+ user = {email: params['email']}
+ setup_params[:user] = user
+ end
+ if params['openid_prefix'] && params['openid_prefix'].size>0
+ setup_params[:openid_prefix] = params['openid_prefix']
+ end
+ if params['repo_name'] && params['repo_name'].size>0
+ setup_params[:repo_name] = params['repo_name']
+ end
+ if params['vm_uuid'] && params['vm_uuid'].size>0
+ setup_params[:vm_uuid] = params['vm_uuid']
+ end
+
+ if User.setup setup_params
+ format.js
+ else
+ self.render_error status: 422
+ end
+ else
+ self.render_error status: 422
+ end
+ end
+ end
+
+ def setup_popup
+ @vms = VirtualMachine.all.results
+
+ @current_selections = find_current_links @object
+
+ respond_to do |format|
+ format.html
+ format.js
+ end
+ end
+
+ protected
+
+ def find_current_links user
+ current_selections = {}
+
+ if !user
+ return current_selections
+ end
+
+ # oid login perm
+ oid_login_perms = Link.where(tail_uuid: user.email,
+ head_kind: 'arvados#user',
+ link_class: 'permission',
+ name: 'can_login')
+
+ if oid_login_perms.any?
+ prefix_properties = oid_login_perms.first.properties
+ current_selections[:identity_url_prefix] = prefix_properties[:identity_url_prefix]
+ end
+
+ # repo perm
+ repo_perms = Link.where(tail_uuid: user.uuid,
+ head_kind: 'arvados#repository',
+ link_class: 'permission',
+ name: 'can_write')
+ if repo_perms.any?
+ repo_uuid = repo_perms.first.head_uuid
+ repos = Repository.where(head_uuid: repo_uuid)
+ if repos.any?
+ repo_name = repos.first.name
+ current_selections[:repo_name] = repo_name
+ end
+ end
+
+ # vm login perm
+ vm_login_perms = Link.where(tail_uuid: user.uuid,
+ head_kind: 'arvados#virtualMachine',
+ link_class: 'permission',
+ name: 'can_login')
+ if vm_login_perms.any?
+ vm_uuid = vm_login_perms.first.head_uuid
+ current_selections[:vm_uuid] = vm_uuid
+ end
+
+ return current_selections
+ end
+
end
link_name = link_uuid
if opts[:friendly_name]
- begin
- link_name = resource_class.find(link_uuid).friendly_link_name
- rescue RuntimeError
- # If that lookup failed, the link will too. So don't make one.
- return attrvalue
+ if attrvalue.respond_to? :friendly_link_name
+ link_name = attrvalue.friendly_link_name
+ else
+ begin
+ link_name = resource_class.find(link_uuid).friendly_link_name
+ rescue RuntimeError
+ # If that lookup failed, the link will too. So don't make one.
+ return attrvalue
+ end
end
end
if opts[:with_class_name]
@opts = opts
@visited = {}
@jobs = {}
+ @node_extra = {}
end
def self.collection_uuid(uuid)
else
return m[1]
end
- # Collection.where(uuid: ['contains', m[1]]).each do |u|
- # puts "fixup #{uuid} to #{u.uuid}"
- # return u.uuid
- # end
- #end
else
nil
end
def describe_node(uuid)
uuid = uuid.to_sym
- bgcolor = determine_fillcolor @opts[:pips][uuid] if @opts[:pips]
+ bgcolor = determine_fillcolor @opts[:pips].andand[uuid]
rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
if rsc
#"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
if rsc == Collection
- #puts uuid
- if uuid == :"d41d8cd98f00b204e9800998ecf8427e+0"
+ if Collection.is_empty_blob_locator? uuid.to_s
# special case
- #puts "empty!"
return "\"#{uuid}\" [label=\"(empty collection)\"];\n"
end
- puts "#{uuid.class} #{@pdata[uuid]}"
if @pdata[uuid]
- #puts @pdata[uuid]
if @pdata[uuid][:name]
return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
else
if i < files.length
label += "\\n⋮"
end
- #puts "#{uuid} #{label} #{files}"
- return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
+ extra_s = @node_extra[uuid].andand.map { |k,v|
+ "#{k}=\"#{v}\""
+ }.andand.join ","
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor},#{extra_s}];\n"
end
end
end
end
unless node == ""
node += "']"
- #puts node
- #id = "#{job[:uuid]}_#{prefix}"
gr += "\"#{node}\" [label=\"#{node}\"];\n"
gr += edge(job_uuid(job), node, {:label => prefix})
end
when String
return '' if sp.empty?
m = GenerateGraph::collection_uuid(sp)
- #puts "#{m} pdata is #{@pdata[m.intern]}"
if m and (@pdata[m.intern] or (not @opts[:pdata_only]))
gr += edge(job_uuid(job), m, {:label => prefix})
gr += generate_provenance_edges(m)
elsif @opts[:all_script_parameters]
- #id = "#{job[:uuid]}_#{prefix}"
gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
gr += edge(job_uuid(job), sp, {:label => prefix})
end
uuid = uuid.intern if uuid
if (not uuid) or uuid.empty? or @visited[uuid]
-
- #puts "already @visited #{uuid}"
return ""
end
@visited[uuid] = true
end
- #puts "visiting #{uuid}"
-
- if m
+ if m
# uuid is a collection
- gr += describe_node(uuid)
-
- if m == :"d41d8cd98f00b204e9800998ecf8427e+0"
- # empty collection, don't follow any further
- return gr
- end
-
- @pdata.each do |k, job|
- if job[:output] == uuid.to_s
- gr += edge(uuid, job_uuid(job), {:label => "output"})
- gr += generate_provenance_edges(job[:uuid])
- end
- if job[:log] == uuid.to_s
- gr += edge(uuid, job_uuid(job), {:label => "log"})
- gr += generate_provenance_edges(job[:uuid])
+ if not Collection.is_empty_blob_locator? uuid.to_s
+ @pdata.each do |k, job|
+ if job[:output] == uuid.to_s
+ extra = { label: 'output' }
+ if job[:output_is_persistent]
+ extra[:label] += ' (persistent)'
+ @node_extra[uuid] ||= {}
+ @node_extra[uuid][:penwidth] = 4
+ end
+ gr += edge(uuid, job_uuid(job), extra)
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ if job[:log] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "log"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
end
end
+ gr += describe_node(uuid)
else
# uuid is something else
rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
end
end
- #puts "finished #{uuid}"
-
gr
end
gr += "edge [dir=back];"
end
- #puts "@pdata is #{pdata}"
-
g = GenerateGraph.new(pdata, opts)
pdata.each do |k, v|
gr += "}"
svg = ""
- puts gr
-
require 'open3'
Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
@@client_mtx = Mutex.new
@@api_client = nil
- @@profiling_enabled = Rails.configuration.profiling_enabled rescue false
+ @@profiling_enabled = Rails.configuration.profiling_enabled
def api(resources_kind, action, data=nil)
profile_checkpoint
ArvadosResourceList.new(self).order(*args)
end
+ def self.filter(*args)
+ ArvadosResourceList.new(self).filter(*args)
+ end
+
def self.where(*args)
ArvadosResourceList.new(self).where(*args)
end
self
end
+ def filter _filters
+ @filters ||= []
+ @filters += _filters
+ self
+ end
+
def where(cond)
cond = cond.dup
cond.keys.each do |uuid_key|
api_params[:limit] = @limit if @limit
api_params[:offset] = @offset if @offset
api_params[:order] = @orderby_spec if @orderby_spec
+ api_params[:filters] = @filters if @filters
res = $arvados_api_client.api @resource_class, '', api_params
@results = $arvados_api_client.unpack_api_response res
self
class Collection < ArvadosBase
+ MD5_EMPTY = 'd41d8cd98f00b204e9800998ecf8427e'
+
+ # Return true if the given string is the locator of a zero-length blob
+ def self.is_empty_blob_locator? locator
+ !!locator.to_s.match("^#{MD5_EMPTY}(\\+.*)?\$")
+ end
+
def total_bytes
if files
tot = 0
def friendly_link_name
[self.first_name, self.last_name].compact.join ' '
end
+
+ def unsetup
+ self.private_reload($arvados_api_client.api(self.class,
+ "/#{self.uuid}/unsetup",
+ {}))
+ end
+
+ def self.setup params
+ $arvados_api_client.api(self, "/setup", params)
+ end
+
end
<% content_for :tab_line_buttons do %>
-<% if controller.model_class.creatable? %>
-<%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
- { action: 'create', return_to: request.url },
- { class: 'btn btn-primary pull-right' } %>
-<% end %>
+ <% if controller.model_class.creatable? %>
+
+ <% if controller.model_class.name == 'User' %>
+ <%= link_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}", setup_user_popup_path,
+ {class: 'btn btn-primary pull-right', :remote => true, 'data-toggle' => "modal",
+ 'data-target' => '#user-setup-modal-window', return_to: request.url} %>
+ <div id="user-setup-modal-window" class="modal fade" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true"></div>
+ <% else %>
+ <%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
+ { action: 'create', return_to: request.url },
+ { class: 'btn btn-primary pull-right' } %>
+ <% end %>
+
+ <% end %>
<% end %>
-->
<li class="dropdown notification-menu">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="collections-menu">
<span class="glyphicon glyphicon-paperclip"></span>
<span class="badge" id="persistent-selection-count"></span>
<span class="caret"></span>
<% if current_user.is_active %>
<li class="dropdown notification-menu">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="notifications-menu">
<span class="glyphicon glyphicon-envelope"></span>
<span class="badge badge-alert notification-count"><%= @notification_count %></span>
<span class="caret"></span>
</div>
</div>
+ <%= yield :footer_html %>
<%= piwik_tracking_tag %>
<%= javascript_tag do %>
<%= yield :footer_js %>
<% template = PipelineTemplate.find(@object.pipeline_template_uuid) rescue nil %>
<%= content_for :content_top do %>
+ <h2>
+ <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => 'Unnamed pipeline', 'data-mode' => 'inline' } %>
+ </h2>
<% if template %>
- <h2><%= template.name %></h2>
+ <h4>
+ From template:
+ <%= link_to_if_arvados_object template, friendly_name: true %>
+ </h4>
<% end %>
<% end %>
<% render_pipeline_jobs.each do |pj| %>
<tr>
<td>
- <% job_status = render(partial: 'job_status_label',
+ <% job_status = render(partial: 'job_status_label',
locals: { :j => pj[:job], :title => pj[:name] }) %>
<% if pj[:job].andand[:uuid] %>
<%= link_to(job_status, job_url(id: pj[:job][:uuid])) %>
</td><td>
<%= pj[:progress_bar] %>
</td><td>
- <%= render(partial: 'job_status_label',
+ <%= render(partial: 'job_status_label',
locals: { :j => pj[:job] }) %>
</td><td>
<%= link_to_if_arvados_object pj[:output] %>
<% content_for :js do %>
setInterval(function(){$('a.refresh').click()}, 15000);
<% end %>
+
+<% content_for :tab_line_buttons do %>
+ <%= form_tag @object, :method => :put do |f| %>
+
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => false %>
+
+ <%= button_tag "Stop pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
+ <% end %>
+<% end %>
+
<% end %>
<% else %>
<% content_for :tab_line_buttons do %>
<%= form_tag @object, :method => :put do |f| %>
-
+
<%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => true %>
<%= button_tag "Run pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
<% end %>
<%= render partial: 'pipeline_templates/show_components_template', locals: {:template => template, :obj => @object} %>
-
+
<% end %>
<% self.formats = [:html] %>
var new_content = "<%= escape_javascript(render template: 'pipeline_instances/show') %>";
-if ($('div.body-content').html() != new_content)
- $('div.body-content').html(new_content);
+var selected_tab_hrefs = [];
+if ($('div.body-content').html() != new_content) {
+ $('.nav-tabs li.active a').each(function() {
+ selected_tab_hrefs.push($(this).attr('href'));
+ });
+
+ $('div.body-content').html(new_content);
+
+ // Show the same tabs that were active before we rewrote body-content
+ $.each(selected_tab_hrefs, function(i, href) {
+ $('.nav-tabs li a[href="' + href + '"]').tab('show');
+ });
+}
$(document).trigger('ajax:complete');
--- /dev/null
+<div class="modal-dialog">
+ <div class="modal-content">
+
+ <%= form_tag setup_user_path, {id: 'setup_form', name: 'setup_form', method: 'get',
+ class: 'form-search', remote: true} do %>
+
+ <div class="modal-header">
+ <button type="button" class="close" onClick="reset_form()" data-dismiss="modal" aria-hidden="true">×</button>
+ <h4 class="modal-title">Setup User</h4>
+ </div>
+
+ <div class="modal-body">
+ <% if @object%>
+ <% uuid = @object.uuid %>
+ <% email = @object.email %>
+ <% end %>
+ <% disable_email = uuid != nil %>
+ <% identity_url_prefix = @current_selections[:identity_url_prefix] %>
+ <% disable_url_prefix = identity_url_prefix != nil %>
+ <% selected_repo = @current_selections[:repo_name] %>
+ <% selected_vm = @current_selections[:vm_uuid] %>
+
+ <input id="user_uuid" maxlength="250" name="user_uuid" type="hidden" value="<%=uuid%>">
+ <div class="form-group">
+ <label for="email">Email</label>
+ <% if disable_email %>
+ <input class="form-control" id="email" maxlength="250" name="email" type="text" value="<%=email%>" disabled>
+ <% else %>
+ <input class="form-control" id="email" maxlength="250" name="email" type="text">
+ <% end %>
+ </div>
+ <div class="form-group">
+ <label for="openid_prefix">Identity URL Prefix</label>
+ <% if disable_url_prefix %>
+ <input class="form-control" id="openid_prefix" maxlength="250" name="openid_prefix" type="text"
+ value="<%=identity_url_prefix%>" disabled=true>
+ <% else %>
+ <input class="form-control" id="openid_prefix" maxlength="250" name="openid_prefix" type="text"
+ value="<%= Rails.configuration.default_openid_prefix %>">
+ <% end %>
+ </div>
+ <div class="form-group">
+ <label for="repo_name">Repository Name</label>
+ <input class="form-control" id="repo_name" maxlength="250" name="repo_name" type="text" value="<%=selected_repo%>">
+ </div>
+ <div class="form-group">
+ <label for="vm_uuid">Virtual Machine</label>
+ <select class="form-control" name="vm_uuid">
+ <option value="" <%= 'selected' unless selected_vm %>>
+ Choose One:
+ </option>
+ <% @vms.each do |vm| %>
+ <option value="<%=vm.uuid%>"
+ <%= 'selected' if selected_vm == vm.uuid %>>
+ <%= vm.hostname %>
+ </option>
+ <% end %>
+ </select>
+ </div>
+ </div>
+
+ <div class="modal-footer">
+ <button type="submit" id="register" class="btn btn-primary" autofocus>Submit</button>
+ <button class="btn btn-default" onClick="reset_form()" data-dismiss="modal" aria-hidden="true">Cancel</button>
+ </div>
+
+ <% end #form %>
+ </div>
+</div>
--- /dev/null
+<p>
+ As an admin user, you can <%= link_to "view recent user activity", activity_users_url %>.
+</p>
+
<blockquote>
<%= button_to "Log in as #{@object.full_name}", sudo_user_url(id: @object.uuid), class: 'btn btn-primary' %>
</blockquote>
+
+<p>As an admin, you can setup this user. Please input a VM and repository for the user. If you had previously provided any of these items, they are pre-filled for you and you can leave them as is if you would like to reuse them.</p>
+
+<blockquote>
+<%= link_to "Setup #{@object.full_name}", setup_popup_user_url(id: @object.uuid), {class: 'btn btn-primary', :remote => true, 'data-toggle' => "modal", 'data-target' => '#user-setup-modal-window'} %>
+</blockquote>
+
+<p>As an admin, you can deactivate and reset this user. This will remove all repository/VM permissions for the user. If you "setup" the user again, the user will have to sign the user agreement again.</p>
+
+<blockquote>
+<%= button_to "Deactivate #{@object.full_name}", unsetup_user_url(id: @object.uuid), class: 'btn btn-primary', confirm: "Are you sure you want to deactivate #{@object.full_name}?"%>
+</blockquote>
+
+<% content_for :footer_html do %>
+<div id="user-setup-modal-window" class="modal fade" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true"></div>
+<% end %>
<td>
<small>
- <% if j.success %>
+ <% if j.success and j.output %>
<a href="<%= collection_path(j.output) %>">
<% Collection.limit(1).where(uuid: j.output).each do |c| %>
<span class="glyphicon glyphicon-search"></span>
<% end %>
</span>
- </div>
+ </div>
<% end %>
</div>
<% if not current_user.andand.is_active or @my_collections.empty? %>
<p>
Your account must be activated by an Arvados administrator. If this
is your first time accessing Arvados and would like to request
- access, or you believe you are seeing the page in error, please
- <%= link_to "contact us", Rails.configuration.activation_contact_link %>.
+ access, or you believe you are seeing the page in error, please
+ <%= link_to "contact us", Rails.configuration.activation_contact_link %>.
You should receive an email at the address you used to log in when
- your account is activated. In the mean time, you can
+ your account is activated. In the mean time, you can
<%= link_to "learn more about Arvados", "https://arvados.org/projects/arvados/wiki/Introduction_to_Arvados" %>,
and <%= link_to "read the Arvados user guide", "http://doc.arvados.org/user" %>.
</p>
--- /dev/null
+<% content_for :css do %>
+table#users-activity-table th {
+ overflow-x: hidden;
+}
+table#users-activity-table .cell-for-span-This-month,
+table#users-activity-table .cell-for-span-Last-month {
+ background: #eee;
+}
+<% end %>
+<table class="table table-condensed arv-index" id="users-activity-table">
+ <colgroup>
+ <col width="28%" />
+ </colgroup>
+ <% @spans.each do |_| %>
+ <colgroup>
+ <% 3.times do %>
+ <col width="<%= (72 / @spans.count / 3).floor %>%" />
+ <% end %>
+ </colgroup>
+ <% end %>
+
+ <tr>
+ <th rowspan="2">User</th>
+ <% @spans.each do |span, start_at, end_at| %>
+ <th colspan="3" class="cell-for-span-<%= span.gsub ' ','-' %>">
+ <%= span %>
+ <br />
+ <%= start_at.strftime('%b %-d') %>
+ -
+ <%= (end_at-1.second).strftime('%b %-d') %>
+ </th>
+ <% end %>
+ </tr>
+ <tr>
+ <% @spans.each do |span, _| %>
+ <th class="cell-for-span-<%= span.gsub ' ','-' %>">Logins</th>
+ <th class="cell-for-span-<%= span.gsub ' ','-' %>">Jobs</th>
+ <th class="cell-for-span-<%= span.gsub ' ','-' %>">Pipelines</th>
+ <% end %>
+ </tr>
+
+ <% @users.each do |user| %>
+ <tr>
+ <td>
+ <small>
+ <% if user.uuid %>
+ <%= link_to_if_arvados_object user, friendly_name: true %>
+ <% else %>
+ <b>Total</b>
+ <% end %>
+ </small>
+ </td>
+
+ <% @spans.each do |span, _| %>
+ <% ['logins', 'jobs', 'pipeline_instances'].each do |type| %>
+ <td class="cell-for-span-<%= span.gsub ' ','-' %>">
+ <small>
+ <%= @user_activity[user.uuid][span + " " + type].to_s %>
+ </small>
+ </td>
+ <% end %>
+ <% end %>
+ </tr>
+ <% end %>
+</table>
+
+<% content_for :footer_js do %>
+$('#users-activity-table td small').each(function(){
+ if ($(this).html().trim() == '0')
+ $(this).css('opacity', '0.3');
+});
+<% end %>
--- /dev/null
+$("#user-setup-modal-window").modal("hide");
+document.location.reload();
--- /dev/null
+$("#user-setup-modal-window").html("<%= escape_javascript(render partial: 'setup_popup') %>");
+
+// disable the submit button on load
+var $input = $('input:text'),
+$register = $('#register');
+
+var email_disabled = document.forms["setup_form"]["email"].disabled;
+var email_value = document.forms["setup_form"]["email"].value;
+var prefix_value = document.forms["setup_form"]["openid_prefix"].value;
+if ((email_disabled == false) && (email_value == null || email_value == "" ||
+ prefix_value == null || prefix_value == "")) {
+ $register.attr('disabled', true);
+}
+
+// capture events to enable submit button when applicable
+$input.on('keyup paste mouseleave', function() {
+ var trigger = false;
+
+ var email_disabled = document.forms["setup_form"]["email"].disabled;
+ var email_value = document.forms["setup_form"]["email"].value;
+ var prefix_value = document.forms["setup_form"]["openid_prefix"].value;
+
+ var emailRegExp = /^([\w-\.]+@([\w-]+\.)+[\w-]{2,4})?$/;
+ var validEmail = false;
+
+ if (emailRegExp.test(email_value )) {
+ validEmail = true;
+ }
+
+ if ((email_disabled == false) && (!validEmail || email_value == null ||
+ email_value == "" || prefix_value == null || prefix_value == "")){
+ trigger = true;
+ }
+
+ trigger ? $register.attr('disabled', true) : $register.removeAttr('disabled');
+});
+
+// reset form input fields, for the next time around
+function reset_form() {
+ $('#email').val("");
+ $('#openid_prefix').val("");
+ $('#repo_name').val("");
+ $('select').val('')
+}
arvados_theme: default
show_user_agreement_inline: false
secret_token: ~
+ default_openid_prefix: https://www.google.com/accounts/o8/id
+++ /dev/null
-$arvados_api_client = ArvadosApiClient.new
+# This file must be loaded _after_ secret_token.rb if secret_token is
+# defined there instead of in config/application.yml.
+
$application_config = {}
%w(application.default application).each do |cfgfile|
--- /dev/null
+# The client object must be instantiated _after_ zza_load_config.rb
+# runs, because it relies on configuration settings.
+#
+if not $application_config
+ raise "Fatal: Config must be loaded before instantiating ArvadosApiClient."
+end
+
+$arvados_api_client = ArvadosApiClient.new
get 'signatures', on: :collection
end
get '/user_agreements/signatures' => 'user_agreements#signatures'
+ get "users/setup_popup" => 'users#setup_popup', :as => :setup_user_popup
+ get "users/setup" => 'users#setup', :as => :setup_user
resources :nodes
resources :humans
resources :traits
resources :users do
get 'home', :on => :member
get 'welcome', :on => :collection
+ get 'activity', :on => :collection
post 'sudo', :on => :member
+ post 'unsetup', :on => :member
+ get 'setup_popup', :on => :member
end
resources :logs
resources :factory_jobs
+++ /dev/null
-Use this README file to introduce your application and point to useful places in the API for learning more.
-Run "rake doc:app" to generate API documentation for your models, controllers, helpers, and libraries.
desc 'Ensure site configuration has all required settings'
task check: :environment do
$application_config.sort.each do |k, v|
- $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ if ENV.has_key?('QUIET') then
+ # Make sure we still check for the variable to exist
+ eval("Rails.configuration.#{k}")
+ else
+ if /(password|secret)/.match(k) then
+ # Make sure we still check for the variable to exist, but don't print the value
+ eval("Rails.configuration.#{k}")
+ $stderr.puts "%-32s %s" % [k, '*********']
+ else
+ $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ end
+ end
end
end
end
+++ /dev/null
-require 'integration_helper'
-
-class ApiClientAuthorizationsTest < ActionDispatch::IntegrationTest
- test "try loading Manage API tokens page" do
- Capybara.current_driver = Capybara.javascript_driver
- visit page_with_token('admin_trustedclient')
- click_link 'user-menu'
- click_link 'Manage API tokens'
- assert_equal 200, status_code
- end
-end
class LoginsTest < ActionDispatch::IntegrationTest
test "login with api_token works after redirect" do
visit page_with_token('active_trustedclient')
- assert page.has_text? 'Recent jobs'
+ assert page.has_text?('Recent jobs'), "Missing 'Recent jobs' from page"
assert_no_match(/\bapi_token=/, current_path)
end
--- /dev/null
+require 'integration_helper'
+require 'uri'
+
+class SmokeTest < ActionDispatch::IntegrationTest
+ def assert_visit_success(allowed=[200])
+ assert_includes(allowed, status_code,
+ "#{current_url} returned #{status_code}, not one of " +
+ allowed.inspect)
+ end
+
+ def all_links_in(find_spec, text_regexp=//)
+ find(find_spec).all('a').collect { |tag|
+ if tag[:href].nil? or tag[:href].empty? or (tag.text !~ text_regexp)
+ nil
+ else
+ url = URI(tag[:href])
+ url.host.nil? ? url.path : nil
+ end
+ }.compact
+ end
+
+ test "all first-level links succeed" do
+ visit page_with_token('active_trustedclient', '/')
+ assert_visit_success
+ click_link 'user-menu'
+ urls = [all_links_in('.arvados-nav'),
+ all_links_in('.navbar', /^Manage /)].flatten
+ seen_urls = ['/']
+ while not (url = urls.shift).nil?
+ next if seen_urls.include? url
+ visit url
+ seen_urls << url
+ assert_visit_success
+ # Uncommenting the line below lets you crawl the entire site for a
+ # more thorough test.
+ # urls += all_links_in('body')
+ end
+ end
+end
--- /dev/null
+require 'integration_helper'
+
+class UsersTest < ActionDispatch::IntegrationTest
+ test "login as active user but not admin" do
+ Capybara.current_driver = Capybara.javascript_driver
+ visit page_with_token('active_trustedclient')
+
+ assert page.has_no_link? 'Users' 'Found Users link for non-admin user'
+ end
+
+ test "login as admin user and verify active user data" do
+ Capybara.current_driver = Capybara.javascript_driver
+ visit page_with_token('admin_trustedclient')
+
+ # go to Users list page
+ click_link 'Users'
+
+ # check active user attributes in the list page
+ page.within(:xpath, '//tr[@data-object-uuid="zzzzz-tpzed-xurymjxw79nv3jz"]') do
+ assert (text.include? 'true false'), 'Expected is_active'
+ end
+
+ click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+ assert page.has_text? 'Attributes'
+ assert page.has_text? 'Metadata'
+ assert page.has_text? 'Admin'
+
+ # go to the Attributes tab
+ click_link 'Attributes'
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "true", text, "Expected user's is_active to be true"
+ end
+ page.within(:xpath, '//a[@data-name="is_admin"]') do
+ assert_equal "false", text, "Expected user's is_admin to be false"
+ end
+
+ end
+
+ test "create a new user" do
+ Capybara.current_driver = :webkit
+ visit page_with_token('admin_trustedclient')
+
+ click_link 'Users'
+
+ assert page.has_text? 'zzzzz-tpzed-d9tiejq69daie8f'
+
+ click_link 'Add a new user'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ assert has_text? 'Virtual Machine'
+ fill_in "email", :with => "foo@example.com"
+ fill_in "repo_name", :with => "test_repo"
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+
+ # verify that the new user showed up in the users page
+ assert page.has_text? 'foo@example.com'
+
+ page.within(:xpath, '//tr[@data-object-uuid][1]') do
+ assert (text.include? 'foo@example.com false'), 'Expected email'
+ new_user_uuid = text.split[0]
+
+ # go to the new user's page
+ click_link new_user_uuid
+ end
+
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "false", text, "Expected new user's is_active to be false"
+ end
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: test_repo)'
+ assert !(page.has_text? '(VirtualMachine:)')
+ end
+
+ test "setup the active user" do
+ Capybara.current_driver = :webkit
+ visit page_with_token('admin_trustedclient')
+
+ click_link 'Users'
+
+ assert page.has_link? 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # click on active user
+ click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # Setup user
+ click_link 'Admin'
+ assert page.has_text? 'As an admin, you can setup'
+
+ click_link 'Setup Active User'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ assert has_text? 'Virtual Machine'
+ fill_in "repo_name", :with => "test_repo"
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+ assert page.has_text? 'modified_by_client_uuid'
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: test_repo)'
+ assert !(page.has_text? '(VirtualMachine:)')
+
+ # Click on Setup button again and this time also choose a VM
+ click_link 'Admin'
+ click_link 'Setup Active User'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ fill_in "repo_name", :with => "second_test_repo"
+ select("testvm.shell", :from => 'vm_uuid')
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+ assert page.has_text? 'modified_by_client_uuid'
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: second_test_repo)'
+ assert page.has_text? '(VirtualMachine: testvm.shell)'
+ end
+
+ test "unsetup active user" do
+ Capybara.current_driver = :webkit
+
+ visit page_with_token('admin_trustedclient')
+
+ click_link 'Users'
+
+ assert page.has_link? 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # click on active user
+ click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # Verify that is_active is set
+ click_link 'Attributes'
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "true", text, "Expected user's is_active to be true"
+ end
+
+ # go to Admin tab
+ click_link 'Admin'
+ assert page.has_text? 'As an admin, you can deactivate and reset this user'
+
+ # unsetup user and verify all the above links are deleted
+ click_link 'Admin'
+ click_button 'Deactivate Active User'
+ sleep(0.1)
+
+ # Should now be back in the Attributes tab for the user
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "false", text, "Expected user's is_active to be false after unsetup"
+ end
+
+ click_link 'Metadata'
+ assert !(page.has_text? '(Repository: test_repo)')
+ assert !(page.has_text? '(Repository: second_test_repo)')
+ assert !(page.has_text? '(VirtualMachine: testvm.shell)')
+
+ # setup user again and verify links present
+ click_link 'Admin'
+ click_link 'Setup Active User'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ fill_in "repo_name", :with => "second_test_repo"
+ select("testvm.shell", :from => 'vm_uuid')
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+ assert page.has_text? 'modified_by_client_uuid'
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: second_test_repo)'
+ assert page.has_text? '(VirtualMachine: testvm.shell)'
+ end
+
+end
# Make a hash that unsets Bundle's environment variables.
# We'll use this environment when we launch Bundle commands in the API
# server. Otherwise, those commands will try to use Workbench's gems, etc.
- @@APIENV = ENV.map { |(key, val)| (key =~ /^BUNDLE_/) ? [key, nil] : nil }.
- compact.to_h
+ @@APIENV = Hash[ENV.map { |key, val|
+ (key =~ /^BUNDLE_/) ? [key, nil] : nil
+ }.compact]
def _system(*cmd)
if not system(@@APIENV, *cmd)
_system('bundle', 'exec', 'rake', 'db:test:load')
_system('bundle', 'exec', 'rake', 'db:fixtures:load')
_system('bundle', 'exec', 'rails', 'server', '-d')
- timeout = Time.now.tv_sec + 5
- while (not File.exists? SERVER_PID_PATH) and (Time.now.tv_sec < timeout)
+ timeout = Time.now.tv_sec + 10
+ begin
sleep 0.2
+ begin
+ server_pid = IO.read(SERVER_PID_PATH).to_i
+ good_pid = (server_pid > 0) and (Process.kill(0, pid) rescue false)
+ rescue Errno::ENOENT
+ good_pid = false
+ end
+ end while (not good_pid) and (Time.now.tv_sec < timeout)
+ if not good_pid
+ raise RuntimeError, "could not find API server Rails pid"
end
- IO.read(SERVER_PID_PATH).to_i
+ server_pid
end
begin
super(args)
require 'test_helper'
class CollectionTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ test 'recognize empty blob locator' do
+ ['d41d8cd98f00b204e9800998ecf8427e+0',
+ 'd41d8cd98f00b204e9800998ecf8427e',
+ 'd41d8cd98f00b204e9800998ecf8427e+0+Xyzzy'].each do |x|
+ assert_equal true, Collection.is_empty_blob_locator?(x)
+ end
+ ['d41d8cd98f00b204e9800998ecf8427e0',
+ 'acbd18db4cc2f85cedef654fccc4a4d8+3',
+ 'acbd18db4cc2f85cedef654fccc4a4d8+0'].each do |x|
+ assert_equal false, Collection.is_empty_blob_locator?(x)
+ end
+ end
end
+++ /dev/null
-Arvados Documentation
-
-0. Install dependencies
-
- $ bundle install
-
-
-1. To build or update documentation:
- $ rake generate
-
-
-2. To view documentation:
- $ rake run
-[2014-03-10 09:03:41] INFO WEBrick 1.3.1
-[2014-03-10 09:03:41] INFO ruby 2.1.1 (2014-02-24) [x86_64-linux]
-[2014-03-10 09:03:41] INFO WEBrick::HTTPServer#start: pid=8926 port=8000
-
- Then go to http://localhost:8000
-
-
-2. You can set 'baseurl' (the URL prefix for all internal links),
-'arvados_api_host' and 'arvados_workbench_host' without changing _config.yml:
-
- $ rake generate baseurl=/example arvados_api_host=example.com
-
-
-4. To delete generated files:
- $ rake realclean
--- /dev/null
+h1. Arvados documentation
+
+This is the source code for "doc.arvados.org":http://doc.arvados.org.
+
+Here's how to build the HTML pages locally so you can preview your updates before you commit and push.
+
+Additional information is available on the "'Documentation' page on the Arvados wiki":https://arvados.org/projects/arvados/wiki/Documentation.
+
+h2. Install dependencies
+
+<pre>
+arvados/doc$ bundle install
+</pre>
+
+h2. Generate HTML pages
+
+<pre>
+arvados/doc$ rake
+</pre>
+
+Alternately, to make the documentation browsable on the local filesystem:
+
+<pre>
+arvados/doc$ rake generate baseurl=$PWD/.site
+</pre>
+
+h2. Run linkchecker
+
+If you have "Linkchecker":http://wummel.github.io/linkchecker/ installed on
+your system, you can run it against the documentation:
+
+<pre>
+arvados/doc$ rake linkchecker baseurl=file://$PWD/.site
+</pre>
+
+Please note that this will regenerate your $PWD/.site directory.
+
+h2. Preview HTML pages
+
+<pre>
+arvados/doc$ rake run
+[2014-03-10 09:03:41] INFO WEBrick 1.3.1
+[2014-03-10 09:03:41] INFO ruby 2.1.1 (2014-02-24) [x86_64-linux]
+[2014-03-10 09:03:41] INFO WEBrick::HTTPServer#start: pid=8926 port=8000
+</pre>
+
+Preview the rendered pages at "http://localhost:8000":http://localhost:8000.
+
+h2. Publish HTML pages inside Workbench
+
+(or some other web site)
+
+You can set @baseurl@ (the URL prefix for all internal links), @arvados_api_host@ and @arvados_workbench_host@ without changing @_config.yml@:
+
+<pre>
+arvados/doc$ rake generate baseurl=/doc arvados_api_host=xyzzy.arvadosapi.com
+</pre>
+
+Make the docs appear at {workbench_host}/doc by creating a symbolic link in Workbench's @public@ directory, pointing to the generated HTML tree.
+
+<pre>
+arvados/doc$ ln -sn ../../../doc/.site ../apps/workbench/public/doc
+</pre>
+
+h2. Delete generated files
+
+<pre>
+arvados/doc$ rake realclean
+</pre>
require "rubygems"
require "colorize"
-task :generate do
+task :generate => [ :realclean, 'sdk/python/arvados/index.html' ] do
vars = ['baseurl', 'arvados_api_host', 'arvados_workbench_host']
vars.each do |v|
if ENV[v]
end
end
-require "zenweb/tasks"
-load "zenweb-textile.rb"
-load "zenweb-liquid.rb"
-
file "sdk/python/arvados/index.html" do |t|
`which epydoc`
if $? == 0
- `epydoc --html -o sdk/python/arvados arvados`
- Dir["sdk/python/arvados/*"].each do |f|
- puts f
- $website.pages[f] = Zenweb::Page.new($website, f)
- end
+ `epydoc --html --parse-only -o sdk/python/arvados ../sdk/python/arvados/`
else
puts "Warning: epydoc not found, Python documentation will not be generated".colorize(:light_red)
end
end
+task :linkchecker => [ :generate ] do
+ Dir.chdir(".site") do
+ `which linkchecker`
+ if $? == 0
+ system "linkchecker index.html --ignore-url='!file://'"
+ else
+ puts "Warning: linkchecker not found, skipping run".colorize(:light_red)
+ end
+ end
+end
+
+task :clean do
+ rm_rf "sdk/python/arvados"
+end
+
+require "zenweb/tasks"
+load "zenweb-textile.rb"
+load "zenweb-liquid.rb"
+
task :extra_wirings do
$website.pages["sdk/python/python.html.textile.liquid"].depends_on("sdk/python/arvados/index.html")
end
- Reference:
- user/reference/api-tokens.html.textile.liquid
- user/reference/sdk-cli.html.textile.liquid
+ - user/reference/job-and-pipeline-reference.html.textile.liquid
- Arvados License:
- user/copying/copying.html.textile.liquid
- user/copying/agpl-3.0.html
- sdk/python/sdk-python.html.textile.liquid
- sdk/python/python.html.textile.liquid
- sdk/python/crunch-utility-libraries.html.textile.liquid
+ - Perl:
+ - sdk/perl/index.html.textile.liquid
+ - Ruby:
+ - sdk/ruby/index.html.textile.liquid
+ - CLI:
+ - sdk/cli/index.html.textile.liquid
api:
- Concepts:
- api/index.html.textile.liquid
{% if nx == 1 %}
<hr>
{% if prev != "" %}
- <a href="{{ site.baseurl }}{{ prev.url }}" class="pull-left">Previous: {{ prev.title }}</a></li>
+ <a href="{{ site.baseurl }}{{ prev.url }}" class="pull-left">Previous: {{ prev.title }}</a>
{% endif %}
- <a href="{{ site.baseurl }}{{ p.url }}" class="pull-right">Next: {{ p.title }}</a></li>
+ <a href="{{ site.baseurl }}{{ p.url }}" class="pull-right">Next: {{ p.title }}</a>
{% assign nx = 0 %}
{% assign n = 1 %}
{% endif %}
{% endfor %}
{% if n == 0 && prev != "" %}
<hr>
- <a href="{{ site.baseurl }}{{ prev.url }}" class="pull-left">Previous: {{ prev.title }}</a></li>
+ <a href="{{ site.baseurl }}{{ prev.url }}" class="pull-left">Previous: {{ prev.title }}</a>
{% assign n = 1 %}
{% endif %}
\ No newline at end of file
A user (person) is permitted to act on an object if there is a path (series of permission Links) from the acting user to the object in which
-* Every intervening object is a Group, and
+* Every intervening object is a Group or a User, and
* Every intervening permission Link allows the current action
Each object has exactly one _owner_, which can be either a User or a Group.
Three lab members are working together on a project. All Specimens, Links, Jobs, etc. can be modified by any of the three lab members. _Other_ lab members, who are not working on this project, can view but not modify these objects.
-h3. 4. Segregated roles
+h3. 4. Group-level administrator
+
+The Ashton Lab administrator, Alison, manages user accounts within her lab. She can enable and disable accounts, and exercise any permission that her lab members have.
+
+George has read-only access to the same set of accounts. This lets him see things like user activity and resource usage reports, without worrying about accidentally messing up anyone's data.
+
+table(table table-bordered table-condensed).
+|Tail |Permission |Head |Effect|
+|Group: Ashton Lab Admin|can_manage |User: Lab Member 1 |Lab member 1 is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: Lab Member 2 |Lab member 2 is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: Lab Member 3 |Lab member 3 is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: Alison |Alison is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: George |George is in this administrative group|
+|Alison |can_manage |Group: Ashton Lab Admin |Alison can do everything the above lab members can do|
+|George |can_read |Group: Ashton Lab Admin |George can read everything the above lab members can read|
+
+h3. 5. Segregated roles
Granwyth, at the Hulatberi Lab, sets up a Factory Robot which uses a hosted Arvados site to do work for the Hulatberi Lab.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_clients@
h2. Resources
-Each ApiClient has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each ApiClient has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorizations@
navsection: api
navmenu: Schema
title: AuthorizedKey
-
...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_keys@
h2. Resources
-Each AuthorizedKey has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each AuthorizedKey has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
-This resource concerns metadata, usage accounting, and integrity checks for data stored on the cloud. Reading and writing the data _per se_ is achieved by the "Keep":/user/tutorials/tutorial-keep.html storage system.
+This resource concerns metadata, usage accounting, and integrity checks for data stored on the cloud. Reading and writing the data _per se_ is achieved by the "Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html storage system.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/links@
h2. Resource
-Each collection has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each collection has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commits@
h2. Resources
-Each Commit has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Commit has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestors@
h2. Resources
-Each CommitAncestor has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each CommitAncestor has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/groups@
h2. Resources
-Each Group has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Group has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/humans@
h2. Resources
-Each Human has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Human has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/jobs@
→ Job resource list
-This method is equivalent to the "index method":/api/methods.html#index, except that the results are restricted to queued jobs (i.e., jobs that have not yet been started or cancelled) and order defaults to queue priority.
+This method is equivalent to the "index method":{{site.baseurl}}/api/methods.html#index, except that the results are restricted to queued jobs (i.e., jobs that have not yet been started or cancelled) and order defaults to queue priority.
h2. Resource
-Each job has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each job has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Notes|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/job_tasks@
h2. Resources
-Each JobTask has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each JobTask has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disks@
h2. Resources
-Each KeepDisk has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each KeepDisk has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/links@
h2. Resource
-Each link has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each link has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/logs@
h2. Resources
-Each Log has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Log has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/nodes@
h2. Resources
-Each Node has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Node has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instances@
h2. Resources
-Each PipelineInstance has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each PipelineInstance has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_templates@
h2. Resources
-Each PipelineTemplate has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each PipelineTemplate has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repositories@
h2. Resources
-Each Repository has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Repository has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimens@
h2. Resources
-Each Specimen has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Specimen has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/traits@
h2. Resources
-Each Trait has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Trait has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/users@
h2. Resources
-Each User has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each User has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machines@
h2. Resources
-Each VirtualMachine has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each VirtualMachine has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
"name":"GATK / exome PE fastq to snp",
"components":{
"extract-reference":{
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"file-select",
"script_parameters":{
"names":[
],
"input":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi"
},
- "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153"
+ "output_is_persistent":false
},
"bwa-index":{
+ "repository":"arvados",
"script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"bwa-index",
"script_parameters":{
"value":"8b6e2c4916133e1d859c9e812861ce13+70",
"required":true
}
- }
+ },
+ "output_is_persistent":false
},
"bwa-aln":{
+ "repository":"arvados",
"script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"bwa-aln",
"script_parameters":{
},
"runtime_constraints":{
"max_tasks_per_node":1
- }
+ },
+ "output_is_persistent":false
},
"picard-gatk2-prep":{
+ "repository":"arvados",
"script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"picard-gatk2-prep",
"script_parameters":{
},
"runtime_constraints":{
"max_tasks_per_node":1
- }
+ },
+ "output_is_persistent":false
},
"GATK2-realign":{
+ "repository":"arvados",
"script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"GATK2-realign",
"script_parameters":{
},
"runtime_constraints":{
"max_tasks_per_node":2
- }
+ },
+ "output_is_persistent":false
},
"GATK2-bqsr":{
+ "repository":"arvados",
"script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"GATK2-bqsr",
"script_parameters":{
"value":"7e0a277d6d2353678a11f56bab3b13f2+87",
"required":true
}
- }
+ },
+ "output_is_persistent":false
},
"GATK2-merge-call":{
+ "repository":"arvados",
"script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"GATK2-merge-call",
"script_parameters":{
"200"
]
}
- }
+ },
+ "output_is_persistent":true
}
}
}
navsection: api
navmenu: Schema
title: {resource}
-navorder: {navorder}
---
h1. {resource}
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+ See "REST methods for working with Arvados resources":{{{{site.baseurl}}}}/api/methods.html
API endpoint base: @https://{{{{ site.arvados_api_host }}}}/arvados/v1/{res_api_endpoint}@
<p>manuals, guides, and references</p>
</div>
<div class="col-sm-6">
- <img src="images/dax.png" style="max-height: 10em"></img>
+ <img src="images/dax-reading-book.png" style="max-height: 10em" alt="Dax reading a book" />
</div>
</div>
</div>
</p>
<p>
<a href="{{ site.baseurl }}/install/index.html">Install Guide</a> — How to install Arvados on a cloud platform.
- </p>
+ </p>
</div>
</div>
</div>
{% include 'notebox_end' %}
notextile. <pre><code>$ <span class="userinput">sudo gem install arvados arvados-cli</span></code></pre>
+
+h3. Perl
+
+{% include 'notebox_begin' %}
+The Perl client library includes the @Arvados.pm@ module and submodules.
+{% include 'notebox_end' %}
+
+<notextile>
+<pre><code>$ <span class="userinput">cd arvados/sdk/perl</span>
+$ <span class="userinput">perl Makefile.PL</span>
+$ <span class="userinput">sudo make install</span>
+</code></pre>
+</notextile>
+
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: CLI
+title: "Command line SDK"
+
+...
+
+The @arv@ CLI tool provides a generic set of wrappers so you can make API calls easily. It performs some validation before connecting to the API server: for example, it refuses to do an API call if a required parameter is missing.
+
+It also provides access to Keep storage services with the @arv keep@ subcommand.
+
+h3. Usage
+
+See the "command line interface":{{site.baseurl}}/user/reference/sdk-cli.html page in the user guide.
+
+h3. Installation
+
+If you are logged in to an Arvados VM, the @arv@ should be installed.
+
+To use @arv@ elsewhere, you can either install the @arvados-cli@ gem via RubyGems or build and install the package using the arvados source tree.
+
+h4. Prerequisites: Ruby >= 2.0.0 and curl libraries
+
+You can use "RVM":http://rvm.io/rvm/install to install and manage Ruby versions.
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo apt-get install curl</code>
+$ <code class="userinput">sudo sh -c 'curl -sSL https://get.rvm.io | bash -s stable'</code>
+$ <code class="userinput">source /etc/profile.d/rvm.sh</code>
+</pre>
+</notextile>
+
+Install curl libraries with your system's package manager. For example, with Debian or Ubuntu:
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo apt-get install libcurl3 libcurl3-gnutls libcurl4-openssl-dev</code>
+</pre>
+</notextile>
+
+h4. Option 1: install with RubyGems
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo gem install arvados-cli</code>
+</pre>
+</notextile>
+
+h4. Option 2: build and install from source
+
+<notextile>
+<pre>
+$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+$ <code class="userinput">cd arvados/sdk/cli</code>
+$ <code class="userinput">gem build arvados-cli.gemspec</code>
+$ <code class="userinput">sudo gem install arvados-cli-*.gem</code>
+</pre>
+</notextile>
This section documents how to access the Arvados API and Keep using various programming languages.
-* "Python SDK":python/sdk-python.html
+* "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
+* "Perl SDK":{{site.baseurl}}/sdk/perl/index.html
+* "Ruby SDK":{{site.baseurl}}/sdk/ruby/index.html
+* "Command line SDK":{{site.baseurl}}/sdk/cli/index.html ("arv")
+
+SDKs not yet implemented:
+
+* Rails SDK: Workbench uses an ActiveRecord-like interface to Arvados. This hasn't yet been extracted from Workbench and packaged as a gem.
+* R and Java: We plan to support these, but they have not been implemented yet.
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: Perl
+title: "Perl SDK"
+
+...
+
+The Perl SDK provides a generic set of wrappers so you can make API calls easily.
+
+It should be treated as alpha/experimental. Currently, limitations include:
+* Verbose syntax.
+* No native Keep client.
+* No CPAN package.
+
+h3. Installation
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo apt-get install libjson-perl libio-socket-ssl-perl libwww-perl</code>
+$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+$ <code class="userinput">cd arvados/sdk/perl</code>
+$ <code class="userinput">perl Makefile.PL</code>
+$ <code class="userinput">sudo make install</code>
+</pre>
+</notextile>
+
+h4. Test installation
+
+If the SDK is installed, @perl -MArvados -e ''@ should produce no errors.
+
+If your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), the following test script should work:
+
+<notextile>
+<pre>$ <code class="userinput">perl <<'EOF'
+use Arvados;
+my $arv = Arvados->new('apiVersion' => 'v1');
+my $me = $arv->{'users'}->{'current'}->execute;
+print ("arvados.v1.users.current.full_name = '", $me->{'full_name'}, "'\n");
+EOF</code>
+arvados.v1.users.current.full_name = 'Your Name'
+</pre>
+</notextile>
+
+h3. Examples
+
+Set up an API client user agent:
+
+<notextile>
+<pre><code class="userinput">my $arv = Arvados->new('apiVersion' => 'v1');
+</code></pre>
+</notextile>
+
+Get the User object for the current user:
+
+<notextile>
+<pre><code class="userinput">my $current_user = $arv->{'users'}->{'current'}->execute;
+</code></pre>
+</notextile>
+
+Get the UUID of an object that was retrieved using the SDK:
+
+<notextile>
+<pre><code class="userinput">my $current_user_uuid = $current_user->{'uuid'}
+</code></pre>
+</notextile>
+
+Retrieve an object by ID:
+
+<notextile>
+<pre><code class="userinput">my $some_user = $arv->{'users'}->{'get'}->execute('uuid' => $current_user_uuid);
+</code></pre>
+</notextile>
+
+Create an object:
+
+<notextile>
+<pre><code class="userinput">my $test_link = $arv->{'links'}->{'create'}->execute('link' => { 'link_class' => 'test', 'name' => 'test' });
+</code></pre>
+</notextile>
+
+Update an object:
+
+<notextile>
+<pre><code class="userinput">my $test_link = $arv->{'links'}->{'update'}->execute(
+ 'uuid' => $test_link->{'uuid'},
+ 'link' => { 'properties' => { 'foo' => 'bar' } });
+</code></pre>
+</notextile>
+
+Get a list of objects:
+
+<notextile>
+<pre><code class="userinput">my $repos = $arv->{'repositories'}->{'list'}->execute;
+print ("UUID of first repo returned is ", $repos->{'items'}->[0], "\n");
+</code></pre>
+</notextile>
+
+The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version.
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: Ruby
+title: "Ruby SDK"
+
+...
+
+The Ruby SDK provides a generic set of wrappers so you can make API calls easily.
+
+h3. Installation
+
+If you are logged in to an Arvados VM, the Ruby SDK should be installed.
+
+To use it elsewhere, you can either install the @arvados@ gem via RubyGems or build and install the package using the arvados source tree.
+
+h4. Prerequisites: Ruby >= 2.0.0
+
+You can use "RVM":http://rvm.io/rvm/install to install and manage Ruby versions.
+
+h4. Option 1: install with RubyGems
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo gem install arvados</code>
+</pre>
+</notextile>
+
+h4. Option 2: build and install from source
+
+<notextile>
+<pre>
+$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+$ <code class="userinput">cd arvados/sdk/cli</code>
+$ <code class="userinput">gem build arvados.gemspec</code>
+$ <code class="userinput">sudo gem install arvados-*.gem</code>
+</pre>
+</notextile>
+
+h4. Test installation
+
+If the SDK is installed, @ruby -r arvados -e 'puts "OK!"'@ should produce no errors.
+
+If your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), the following test script should work:
+
+<notextile>
+<pre>$ <code class="userinput">ruby -r arvados <<'EOF'
+arv = Arvados.new api_version: 'v1'
+my_full_name = arv.user.current[:full_name]
+puts "arvados.v1.users.current.full_name = '#{my_full_name}'"
+EOF</code>
+arvados.v1.users.current.full_name = 'Your Name'
+</pre>
+</notextile>
+
+h3. Examples
+
+Import the module (we skipped this step above by using "ruby -r arvados"):
+
+<notextile>
+<pre><code class="userinput">require 'arvados'
+</code></pre>
+</notextile>
+
+Set up an API client user agent:
+
+<notextile>
+<pre><code class="userinput">arv = Arvados.new(apiVersion: 'v1')
+</code></pre>
+</notextile>
+
+Get the User object for the current user:
+
+<notextile>
+<pre><code class="userinput">current_user = arv.user.current
+</code></pre>
+</notextile>
+
+Get the UUID of an object that was retrieved using the SDK:
+
+<notextile>
+<pre><code class="userinput">current_user_uuid = current_user[:uuid]
+</code></pre>
+</notextile>
+
+Retrieve an object by ID:
+
+<notextile>
+<pre><code class="userinput">some_user = arv.user.get(uuid: current_user_uuid)
+</code></pre>
+</notextile>
+
+Create an object:
+
+<notextile>
+<pre><code class="userinput">new_link = arv.link.create(link: {link_class: 'test', name: 'test'})
+</code></pre>
+</notextile>
+
+Update an object:
+
+<notextile>
+<pre><code class="userinput">updated_link = arv.link.update(uuid: new_link[:uuid],
+ link: {properties: {foo: 'bar'}})
+</code></pre>
+</notextile>
+
+Delete an object:
+
+<notextile>
+<pre><code class="userinput">arv.link.delete(uuid: new_link[:uuid])
+</code></pre>
+</notextile>
+
+Get a list of objects:
+
+<notextile>
+<pre><code class="userinput">repos = arv.repository.list
+first_repo = repos[:items][0]
+puts "UUID of first repo returned is #{first_repo[:uuid]}"</code>
+UUID of first repo returned is qr1hi-s0uqq-b1bnybpx3u5temz
+</pre>
+</notextile>
+
+The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version.
<p style="text-align: center;">Version 3, 19 November 2007</p>
<p>Copyright © 2007 Free Software Foundation,
-Inc. <<a href="http://fsf.org/">http://fsf.org/</a>>
+Inc. <<a href="http://www.fsf.org/">http://fsf.org/</a>>
<br />
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.</p>
title: Accessing an Arvados VM over ssh
...
-Arvados requires a public @ssh@ key in order to securely log in to an Arvados VM instance, or to access an Arvados @git@ repository.
+Arvados requires a public ssh key in order to securely log in to an Arvados VM instance, or to access an Arvados @git@ repository.
This document is divided up into three sections.
notextile. <pre><code>$ <span class="userinput">ls ~/.ssh/id_rsa.pub</span></code></pre>
-If the file @id_rsa.pub@ exists, then you may use your existing key. Copy the contents of @~/.ssh/id_rsa.pub@ onto the clipboard (this is your public key). Proceed to "adding your key to the Arvados Workbench.":#workbench
+If the file @id_rsa.pub@ exists, then you may use your existing key. Copy the contents of @~/.ssh/id_rsa.pub@ onto the clipboard (this is your public key). You can skip this step and proceed by "adding your key to the Arvados Workbench.":#workbench
If there is no file @~/.ssh/id_rsa.pub@, you must generate a new key. Use @ssh-keygen@ to do this:
</code></pre>
</notextile>
-Now you can set up @ssh-agent@ (next) or proceed to "adding your key to the Arvados Workbench.":#workbench
+Now you can set up @ssh-agent@ (next) or proceed with "adding your key to the Arvados Workbench.":#workbench
h3. Setting up ssh-agent (recommended)
notextile. <pre><code>$ <span class="userinput">eval $(ssh-agent -s)</span></code></pre>
-* @ssh-agent -s@ prints out values for environment variables SSH_AUTH_SOCK and SSH_AGENT_PID and then runs in the background. Using "eval" on the output as shown here causes those variables to be set in the current shell environment so that subsequent calls to @ssh@ can discover how to access the @ssh-agent@ daemon.
+@ssh-agent -s@ prints out values for environment variables SSH_AUTH_SOCK and SSH_AGENT_PID and then runs in the background. Using "eval" on the output as shown here causes those variables to be set in the current shell environment so that subsequent calls to @ssh@ can discover how to access the @ssh-agent@ daemon.
-After running @ssh-agent@, or if @ssh-add -l@ prints "The agent has no identities", then you will need to add your key using the following command. The passphrase to decrypt the key is the same used to protect the key when it was created with @ssh-keygen@:
+After running @ssh-agent@, or if @ssh-add -l@ prints "The agent has no identities", then you will need to add your key using the following command. The passphrase to decrypt the key is the same used to protect the key when it was created with @ssh-keygen@:
<notextile>
<pre><code>$ <span class="userinput">ssh-add</span>
(Note: if you are using the @ssh@ client that comes with "Cygwin":http://cygwin.com you should follow the "Unix":#unix instructions).
-"PuTTY":http://www.putty.org/ is a free (MIT-licensed) Win32 Telnet and SSH client. PuTTy includes all the tools a windows user needs to set up Private Keys and to set up and use SSH connections to your virtual machines in the Arvados Cloud.
+"PuTTY":http://www.chiark.greenend.org.uk/~sgtatham/putty/ is a free (MIT-licensed) Win32 Telnet and SSH client. PuTTY includes all the tools a Windows user needs to create private keys and make ssh connections to your virtual machines in the Arvados Cloud.
-You can use PuTTY to create public/private keys, which are how you’ll ensure that that access to Arvados cloud is secure. You can also use PuTTY as an SSH client to access your virtual machine in an Arvados cloud and work with the Arvados Command Line Interface (CLI) client.
-
-You may download putty from "http://www.putty.org/":http://www.putty.org/ .
-
-Note that you should download the installer or .zip file with all of the PuTTY tools (PuTTYtel is not required).
+You can "download PuTTY from its Web site":http://www.chiark.greenend.org.uk/~sgtatham/putty/. Note that you should download the installer or .zip file with all of the PuTTY tools (PuTTYtel is not required).
h3. Step 1 - Adding PuTTY to the PATH
# Open the Control Panel.
# Select _Advanced System Settings_, and choose _Environment Variables_.
# Under system variables, find and edit @PATH@.
-# Add the following to the end of PATH (make sure to include semi colon and quotation marks):
+# If you installed PuTTY in @C:\Program Files\PuTTY\@, add the following to the end of PATH (make sure to include semicolon and quotation marks):
+<code>;\"C:\Program Files\PuTTY\"</code>
+If you installed PuTTY in @C:\Program Files (x86)\PuTTY\@, add the following to the end of PATH (make sure to include semicolon and quotation marks):
<code>;\"C:\Program Files (x86)\PuTTY\"</code>
# Click through the OKs to close all the dialogs you’ve opened.
# At the bottom of the window, make sure the ‘Number of bits in a generated key’ field is set to 4096.
# Click Generate and follow the instructions to generate a key.
# Click to save the Public Key.
-# Click to save the Private Key (we recommend using a strong passphrase) .
+# Click to save the Private Key (we recommend using a strong passphrase).
# Select the text of the Public Key and copy it to the clipboard.
h3. Step 3 - Set up Pageant
-Note: Pageant is a PuTTY utility that manages your private keys so is not necessary to enter your private key passphrase every time you need to make a new ssh connection.
+Pageant is a PuTTY utility that manages your private keys so is not necessary to enter your private key passphrase every time you make a new ssh connection.
# Start Pageant from the Start Menu or the folder where it was installed.
# Pageant will now be running in the system tray. Click the Pageant icon to configure.
# Choose _Add Key_ and add the private key which you created in the previous step.
-You are now ready to proceed to "adding your key to the Arvados Workbench":#workbench .
-
-_Note: We recommend you do not delete the “Default” Saved Session._
+You are now ready to proceed to "adding your key to the Arvados Workbench.":#workbench
h1(#workbench). Adding your key to Arvados Workbench
-h3. From the workbench dashboard
+h3. From the Workbench dashboard
-If you have no @ssh@ keys registered, there should be a notification asking you to provide your @ssh@ public key. On the Workbench dashboard (in this guide, this is "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/ ), look for the envelope icon <span class="glyphicon glyphicon-envelope"></span> <span class="badge badge-alert">1</span> in upper right corner (the number indicates there are new notifications). Click on this icon and a dropdown menu should appear with a message asking you to add your public key. Paste your public key into the text area provided and click on the check button to submit the key. You are now ready to "log into an Arvados VM":#login.
+If you have no ssh keys registered, there should be a notification asking you to provide your ssh public key. On the Workbench dashboard, look for the envelope icon <span class="glyphicon glyphicon-envelope"></span> <span class="badge badge-alert">1</span> in upper right corner (the number indicates there are new notifications). Click on this icon and a dropdown menu should appear with a message asking you to add your public key. Paste your public key into the text area provided and click on the check button to submit the key. You are now ready to "log into an Arvados VM":#login.
h3. Alternate way to add ssh keys
-If you want to add additional @ssh@ keys, click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item _Manage ssh keys_ to go to the Authorized keys page.
+If you want to add additional ssh keys, click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item *Manage ssh keys* to go to the Authorized keys page.
-On _Authorized keys_ page, the click on the button <span class="btn btn-primary disabled">Add a new authorized key</span> in the upper right corner.
+On the *Authorized keys* page, the click on the button <span class="btn btn-primary disabled">Add a new authorized key</span> in the upper right corner.
-The page will reload with a new row of information. Under the *public_key* column heading, click on the cell +none+ . This will open an editing popup as shown in this screenshot:
+The page will reload with a new row of information. Under the *public_key* column heading, click on the cell +none+. This will open an editing popup as shown in this screenshot:
!{{ site.baseurl }}/images/ssh-adding-public-key.png!
-Paste the public key from the previous section into the popup text box and click on the check mark to save it. This should refresh the page with the public key that you just added now listed under the *public_key* column. You are now ready to "log into an Arvados VM":#login.
+Paste the public key that you copied to the cliboard in the previous section into the popup text box, then click on the check mark to save it. This should refresh the page with the public key that you just added now listed under the *public_key* column. You are now ready to "log into an Arvados VM":#login.
h1(#login). Using ssh to log into an Arvados VM
-To see a list of virtual machines that you have access to and determine the name and login information, click on Compute %(rarr)→% Virtual machines. Once on the "virtual machines" page, The *hostname* columns lists the name of each available VM. The *logins* column will have a value in the form of @["you"]@. Ignore the square brackets and quotes to get your login name. In this guide the hostname will be _shell_ and the login will be _you_. Replace these with your hostname and login as appropriate.
+To see a list of virtual machines that you have access to and determine the name and login information, click on Compute %(rarr)→% Virtual machines. Once on the *Virtual machines* page, The *hostname* columns lists the name of each available VM. The *logins* column will have a value in the form of @["you"]@. Your login name is the text inside the quotes. In this guide the hostname will be _shell_ and the login will be _you_. Replace these with your hostname and login name as appropriate.
This section consists of two sets of instructions, depending on whether you will be logging in using a "Unix":#unixvm (Linux, OS X, Cygwin) or "Windows":#windowsvm client.
h2(#unixvm). Logging in using command line ssh (Unix)
-h3. Connecting to the VM
+h3. Connecting to the virtual machine
-Use the following command to connect to the "shell" VM instance as "you". Replace *<code>you@shell</code>* at the end of the following command with your *login* and *hostname* from Workbench:
+Use the following command to connect to the _shell_ VM instance as _you_. Replace *<code>you@shell</code>* at the end of the following command with your *login* and *hostname* from Workbench:
-notextile. <pre><code>$ <span class="userinput">ssh -o "ProxyCommand ssh -a -x -p2222 turnout@switchyard.{{ site.arvados_api_host }} shell" -A -x <b>you@shell</b></span></code></pre>
+notextile. <pre><code>$ <span class="userinput">ssh -o "ProxyCommand ssh -a -x -p2222 turnout@switchyard.{{ site.arvados_api_host }} <b>shell</b>" -A -x <b>you@shell</b></span></code></pre>
-There are several things going on here:
+This command does several things at once. You usually cannot log in directly to virtual machines over the public Internet. Instead, you log into a "switchyard" server and then tell the switchyard which virtual machine you want to connect to.
-The VMs typically have addresses that are not globally routable, so you cannot log in directly. Instead, you log into a "switchyard" server and then tell the switchyard which VM you want to connect to.
-
-* @-o "ProxyCommand ..."@ option instructs ssh to run the specified command and then tunnel your ssh connection over the proxy.
-* @-a@ tells ssh not to forward your ssh-agent credentials to the switchyard
-* @-x@ tells ssh not to forward your X session to the switchyard
-* @-p2222@ specifies that the switchyard is running on non-standard port 2222
-* <code>turnout@switchyard.{{ site.arvados_api_host }}</code> specifies the user (@turnout@) and hostname (@switchyard.{{ site.arvados_api_host }}@) of the switchboard server that will proxy our connection to the VM.
-* @shell@ is the name of the VM that we want to connect to. This is sent to the switchyard server as if it were an ssh command, and the switchyard server connects to the VM on our behalf.
-* After the ProxyCommand section, the @-x@ must be repeated because it applies to the connection to VM instead of the switchyard.
+* @-o "ProxyCommand ..."@ configures ssh to run the specified command to create a proxy and route your connection through it.
+* @-a@ tells ssh not to forward your ssh-agent credentials to the switchyard.
+* @-x@ tells ssh not to forward your X session to the switchyard.
+* @-p2222@ specifies that the switchyard is running on non-standard port 2222.
+* <code>turnout@switchyard.{{ site.arvados_api_host }}</code> specifies the user (@turnout@) and hostname (@switchyard.{{ site.arvados_api_host }}@) of the switchyard server that will proxy our connection to the VM.
+* *@shell@* is the name of the VM that we want to connect to. This is sent to the switchyard server as if it were an ssh command, and the switchyard server connects to the VM on our behalf.
+* After the ProxyCommand section, we repeat @-x@ to disable X session forwarding to the virtual machine.
* @-A@ specifies that we want to forward access to @ssh-agent@ to the VM.
-* Finally, *<code>you@shell</code>* specifies your username and repeats the hostname of the VM. The username can be found in the *logins* column in the VMs Workbench page, discussed above.
+* Finally, *<code>you@shell</code>* specifies your login name and repeats the hostname of the VM. The username can be found in the *logins* column in the VMs Workbench page, discussed in the previous section.
You should now be able to log into the Arvados VM and "check your environment.":check-environment.html
h3. Configuration (recommended)
-Since the above command line is cumbersome, it can be greatly simplfied by adding the following section your @~/.ssh/config@ file:
+The command line above is cumbersome, but you can configure ssh to remember many of these settings. Add this text to the file @.ssh/config@ in your home directory (create a new file if @.ssh/config@ doesn't exist):
<notextile>
<pre><code class="userinput">Host *.arvados
# Open PuTTY from the Start Menu.
# On the Session screen set the Host Name (or IP address) to “shell”.
# On the Session screen set the Port to “22”.
-# On the Connection %(rarr)→% Data screen set the Auto-login username to the username listed in the *logins* column on the Arvados Workbench _Access %(rarr)→% VMs_ page.
+# On the Connection %(rarr)→% Data screen set the Auto-login username to the username listed in the *logins* column on the Arvados Workbench page _Compute %(rarr)→% Virtual machines_.
# On the Connection %(rarr)→% Proxy screen set the Proxy Type to “Local”.
# On the Connection %(rarr)→% Proxy screen in the “Telnet command, or local proxy command” box enter:
<code>plink -P 2222 turnout@switchyard.qr1hi.arvadosapi.com %host</code>
Make sure there is no newline at the end of the text entry.
-# Return to the Session screen. In the Saved Sessions box, enter a name for this configuration and hit Save.
+# Return to the Session screen. In the Saved Sessions box, enter a name for this configuration and click Save.
+
+_Note: We recommend you do not delete the “Default” Saved Session._
h3. Connecting to the VM
-# Open PuTTY
+# Open PuTTY from the Start Menu.
# Click on the Saved Session name you created in the previous section.
# Click Load to load those saved session settings.
-# Click Open and that will open the SSH window at the command prompt. You will now be logged in to your virtual machine.
+# Click Open to open the SSH window at the command prompt. You will now be logged into your virtual machine.
You should now be able to log into the Arvados VM and "check your environment.":check-environment.html
This user guide introduces how to use the major components of Arvados. These are:
* Keep: Content-addressable cluster file system designed for robust storage of very large files, such as whole genome sequences running in the hundreds of gigabytes
-* Crunch: Cluster compute engine designed for genomic analysis, e.g. alignment, variant calls
-* Metadata Database: Information about the genomic data stored in Keep, such as genomic traits, human subjects
-* Workbench: Web interface to Arvados components
+* Crunch: Cluster compute engine designed for genomic analysis, such as alignment and variant calls
+* Metadata Database: Information about the genomic data stored in Keep, such as genomic traits and human subjects
+* Workbench: Arvados' Web interface
h2. Prerequisites
To get the most value out of this guide, you should be comfortable with the following:
-# Using a secure shell client such as @ssh@ or @putty@ to log on to a remote server
-# Using the unix command line shell @bash@
+# Using a secure shell client such as @ssh@ or @putty@ to log on to a remote server
+# Using the Unix command line shell @bash@
# Viewing and editing files using a unix text editor such as @vi@, @emacs@, or @nano@
# Programming in @python@
# Revision control using @git@
We also recommend you read the "Arvados Platform Overview":https://arvados.org/projects/arvados/wiki#Platform-Overview for an introduction and background information about Arvados.
-The examples in this guide uses the Arvados instance located at "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/ . If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
+The examples in this guide use the Arvados instance located at "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/. If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
-The Arvados public beta instance is located at "https://workbench.qr1hi.arvadosapi.com/":https://workbench.qr1hi.arvadosapi.com/ . You must have an account in order to use this service. If you would like to request an account, please send an email to "arvados@curoverse.com":mailto:arvados@curoverse.com .
+The Arvados public beta instance is located at "https://workbench.qr1hi.arvadosapi.com/":https://workbench.qr1hi.arvadosapi.com/. You must have an account in order to use this service. If you would like to request an account, please send an email to "arvados@curoverse.com":mailto:arvados@curoverse.com.
h2. Typographic conventions
<notextile>
<ul>
-<li>Code blocks which are set aside from the text indicate user input to the system. Commands that should be entered into a Unix shell are indicated by the directory where you should enter the command ('~' indicates your home directory) followed by '$', followed by the highlighted <span class="userinput">command to enter</span> (do not enter the '$'), and possibly followed by example command output in black. For example, the following block indicates that you should type "ls foo.*" while in your home directory and the expected output will be "foo.input" and "foo.output".
-<pre><code>~$ <span class="userinput">ls foo</span>
-foo
+<li>Code blocks which are set aside from the text indicate user input to the system. Commands that should be entered into a Unix shell are indicated by the directory where you should enter the command ('~' indicates your home directory) followed by '$', followed by the highlighted <span class="userinput">command to enter</span> (do not enter the '$'), and possibly followed by example command output in black. For example, the following block indicates that you should type <code>ls foo.*</code> while in your home directory and the expected output will be "foo.input" and "foo.output".
+<pre><code>~$ <span class="userinput">ls foo.*</span>
+foo.input foo.output
</code></pre>
</li>
<li>Code blocks inline with text emphasize specific <code>programs</code>, <code>files</code>, or <code>options</code> that are being discussed.</li>
-<li>Bold text emphasizes <b>specific items</b> to look when discussing Arvados Workbench pages.</li>
-<li>A sequence of steps separated by right arrows (<span class="rarr">→</span>) indicate a path the user should follow through the Arvados Workbench to access some piece of information under discussion. The steps indicate a menu, hyperlink, column name, field name, or other label on the page that guide the user where to look or click.
+<li>Bold text emphasizes <b>specific items</b> to review on Arvados Workbench pages.</li>
+<li>A sequence of steps separated by right arrows (<span class="rarr">→</span>) indicate a path the user should follow through the Arvados Workbench. The steps indicate a menu, hyperlink, column name, field name, or other label on the page that guide the user where to look or click.
</li>
</ul>
</notextile>
The Arvados API token is a secret key that enables the @arv@ command line client to access Arvados with the proper permissions.
-Access the Arvados workbench using this link: "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/
+Access the Arvados Workbench using this link: "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/ (Replace @{{ site.arvados_api_host }}@ with the hostname of your local Arvados instance if necessary.)
-(Replace @{{ site.arvados_api_host }}@ with the hostname of your local Arvados instance if necessary.)
+Open a shell on the system where you want to use the Arvados client. This may be your local workstation, or "an Arvados virtual machine accessed with ssh":{{site.baseurl}}/user/getting_started/ssh-access.html.
-First, open a shell on the system on which you intend to use the Arvados client (this may be your local workstation, or an Arvados VM, refer to "Accessing Arvados over ssh":{{site.baseurl}}/user/getting_started/ssh-access.html ) .
-
-Click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item _Manage API token_ to go to the "api client authorizations" page.
+Click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu. Click on the menu item *Manage API tokens* to go to the "Api client authorizations" page.
h2. The easy way
-For your convenience, the "api client authorizations" page on Workbench provides a "Help" tab that provides a command you may copy and paste directly into the shell. It will look something like this:
+For your convenience, the "Api client authorizations" page on Workbench provides a *Help* tab that includes a command you may copy and paste directly into the shell. It will look something like this:
bc. ### Pasting the following lines at a shell prompt will allow Arvados SDKs
-### to authenticate to your account, youraddress@example.com
+### to authenticate to your account, you@example.com
read ARVADOS_API_TOKEN <<EOF
2jv9346o396exampledonotuseexampledonotuseexes7j1ld
EOF
export ARVADOS_API_TOKEN ARVADOS_API_HOST={{ site.arvados_api_host }}
-* The @read@ command takes the contents of stdin and puts it into the shell variable named on the command line.
-* The @<<EOF@ notation means read each line on stdin and pipe it to the command, terminating on reading the line @EOF@.
-* The @export@ command puts a local shell variable into the environment that will be inherited by child processes (e.g. the @arv@ client).
+* The @read@ command reads text input until @EOF@ (designated by @<<EOF@) and stores it in the @ARVADOS_API_TOKEN@ environment variable.
+* The @export@ command puts a local shell variable into the environment that will be inherited by child processes such as the @arv@ client.
h2. Setting the environment manually
</code></pre>
</notextile>
-* @ARVADOS_API_HOST@ tells @arv@ which host to connect to
-* @ARVADOS_API_TOKEN@ is the secret key used by the Arvados API server to authenticate access.
+* @ARVADOS_API_HOST@ tells @arv@ which host to connect to.
+* @ARVADOS_API_TOKEN@ is the secret key used by the Arvados API server to authenticate access. Its value is the text you copied from the *api_token* column on the Workbench.
If you are connecting to a development instance with a unverified/self-signed SSL certificate, set this variable to skip SSL validation:
h2. settings.conf
-Arvados tools will also look for the authentication information in @~/.config/arvados/settings.conf@. If you have already put the variables into the environment with instructions above, you can use these commands to create an Arvados configuration file:
+Arvados tools will also look for the authentication information in @~/.config/arvados/settings.conf@. If you have already put the variables into the environment following the instructions above, you can use these commands to create an Arvados configuration file:
<notextile>
<pre><code>$ <span class="userinput">echo "ARVADOS_API_HOST=$ARVADOS_API_HOST" > ~/.config/arvados/settings.conf</span>
h2. .bashrc
-Alternately, you may add the declarations of @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ to the @~/.bashrc@ file on the system on which you intend to use the Arvados client. If you have already put the variables into the environment with instructions above, you can use these commands to append the environment variables to your @~/.bashrc@:
+Alternately, you may add the declarations of @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ to the @~/.bashrc@ file on the system on which you intend to use the Arvados client. If you have already put the variables into the environment following the instructions above, you can use these commands to append the environment variables to your @~/.bashrc@:
<notextile>
<pre><code>$ <span class="userinput">echo "export ARVADOS_API_HOST=$ARVADOS_API_HOST" >> ~/.bashrc</span>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Job and Pipeline Reference"
+...
+
+h2. Submitting jobs
+
+table(table table-bordered table-condensed).
+|_. Attribute |_. Type|_. Accepted values |_. Required|_. Description|
+|script |string |filename |yes |The actual script that will be run by crunch. Must be the name of an executable file in the crunch_scripts/ directory at the git revision specified by script_version.|
+|script_version |string |git branch, tag, or version hash |yes |The code version to run, which is available in the specified repository. May be a git hash or tag to specify an exact version, or a branch. If it is a branch, use the branch head.|
+|repository |string |name of git repository hosted by Arvados |yes |The repository to search for script_version.|
+|script_parameters |object |any JSON object |yes |The input parameters for the job, with the parameter names as keys mapping to parameter values.|
+|minimum_script_version |string |git branch, tag, or version hash |no |The minimum acceptable script version when deciding whether to re-use a past job.|
+|exclude_script_versions|array of strings|git branch, tag, or version hash|no |Script versions to exclude when deciding whether to re-use a past job.|
+|nondeterministic |boolean | |no |If true, never re-use a past job, and flag this job so it will never be considered for re-use.|
+|no_reuse |boolean | |no |If true, do not re-use a past job, but this job may be re-used.|
+
+When a job is executed, the 'script_version' field is resolved to an exact git revision and the git hash for that revision is recorded in 'script_version'. If 'script_version' can't be resolved, the job submission will be rejected.
+
+h3. Reusing jobs
+
+Because Arvados records the exact version of the script, input parameters, and runtime environment [1] that was used to run the job, if the script is deterministic (meaning that the same code version is guaranteed to produce the same outputs from the same inputs) then it is possible to re-use the results of past jobs, and avoid re-running the computation to save time. Arvados uses the following algorithm to determine if a past job can be re-used:
+
+notextile. <div class="spaced-out">
+
+# If 'nondeterministic' or 'no_reuse' are true, always create a new job.
+# Find a list of acceptable values for 'script_version'. If 'minimum_script_version' is specified, this is the set of all revisions in the git commit graph between 'minimum_script_version' and 'script_version' (inclusive) [2]. If 'minimum_script_version' is not specified, only 'script_version' is added to the list. If 'exclude_script_versions' is specified, the listed versions are excluded from the list.
+# Select jobs have the same 'script' and 'script_parameters' attributes, and where the 'script_version' attribute is in the list of acceptable versions. Exclude jobs that failed or set 'nondeterministic' to true.
+# If there is more than one candidate job, check that all selected past jobs actually did produce the same output.
+# If everything passed, re-use one of the selected past jobs (if there is more than one match, which job will be returned is undefined). Otherwise create a new job.
+
+fn1. As of this writing, versioning the runtime environment is still under development.
+
+fn2. This may include parallel branches if there is more than one path between 'minimum_script_version' and 'script_version' in the git commit graph. Use 'exclude_script_versions' to blacklist specific versions.
+
+</div>
+
+h3. Examples
+
+Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" branch head. Arvados is allowed to re-use a previous job if the script_version of the past job is the same as the "master" branch head (i.e., there have not been any subsequent commits to "master").
+
+<notextile><pre>
+{
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+Run using exactly the version "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5". Arvados is allowed to re-use a previous job if the "script_version" of that job is also "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5".
+
+<notextile><pre>
+{
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "script_version": "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5",
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+Arvados is allowed to re-use a previous job if the "script_version" of the past job is between "earlier_version_tag" and the head of the "master" branch (inclusive), but not "blacklisted_version_tag". If there are no previous jobs, run the job using the head of the "master" branch as specified in "script_version".
+
+<notextile><pre>
+{
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "minimum_script_version": "earlier_version_tag",
+ "script_version": "master",
+ "exclude_script_versions": ["blacklisted_version_tag"],
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+Run the script "crunch_scripts/monte-carlo.py" in the repository "you" using the "master" branch head. Because it is marked as "nondeterministic", never re-use previous jobs, and never re-use this job.
+
+<notextile><pre>
+{
+ "script": "monte-carlo.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "nondeterministic": true,
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+h2. Pipelines
+
+Pipelines consist of a set of "components". Each component is an Arvados job submission, so when a component job is submitted, Arvados may re-use past jobs based on the rules described above.
+
+table(table table-bordered table-condensed).
+|_. Attribute |_. Type |_. Accepted values |_. Required|_. Description|
+|name |string |any |yes |The human-readable name of the pipeline template.|
+|components |object |JSON object containing job submission objects|yes |The component jobs that make up the pipeline, with the component name as the key. |
+
+h3. Script parameters
+
+When used in a pipeline, each parameter in the 'script_parameters' attribute of a component job can specify that the input parameter must be supplied by the user, or the input parameter should be linked to the output of another component. To do this, the value of the parameter should be JSON object containing one of the following attributes:
+
+table(table table-bordered table-condensed).
+|_. Attribute |_. Type |_. Accepted values |_. Description|
+|default |any |any |The default value for this parameter.|
+|required |boolean |true or false |Specifies whether the parameter is required to have a value or not.|
+|dataclass |string |One of 'Collection', 'File' [3], 'number', or 'text' |Data type of this parameter.|
+|output_of |string |the name of another component in the pipeline |Specifies that the value of this parameter should be set to the 'output' attribute of the job that corresponds to the specified component.|
+
+The 'output_of' parameter is especially important, as this is how components are actually linked together to form a pipeline. Component jobs that depend on the output of other components do not run until the parent job completes and has produced output. If the parent job fails, the entire pipeline fails.
+
+fn3. The 'File' type refers to a specific file within a Keep collection in the form 'collection_hash/filename', for example '887cd41e9c613463eab2f0d885c6dd96+83/bob.txt'.
+
+h3. Examples
+
+This is a pipeline named "Filter md5 hash values" with two components, "do_hash" and "filter". The "input" script parameter of the "do_hash" component is required to be filled in by the user, and the expected data type is "Collection". This also specifies that the "input" script parameter of the "filter" component is the output of "do_hash", so "filter" will not run until "do_hash" completes successfully. When the pipeline runs, past jobs that meet the criteria described above may be substituted for either or both components to avoid redundant computation.
+
+<notextile><pre>
+{
+ "name": "Filter md5 hash values",
+ "components": {
+ "do_hash": {
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "required": true,
+ "dataclass": "Collection"
+ }
+ },
+ },
+ "filter": {
+ "script": "0-filter.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "output_of": "do_hash"
+ }
+ },
+ }
+ }
+}
+</pre></notextile>
+
+This pipeline consists of three components. The components "thing1" and "thing2" both depend on "cat_in_the_hat". Once the "cat_in_the_hat" job is complete, both "thing1" and "thing2" can run in parallel, because they do not depend on each other.
+
+<notextile><pre>
+{
+ "name": "Wreck the house",
+ "components": {
+ "cat_in_the_hat": {
+ "script": "cat.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": { }
+ },
+ "thing1": {
+ "script": "thing1.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "output_of": "cat_in_the_hat"
+ }
+ },
+ },
+ "thing2": {
+ "script": "thing2.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "output_of": "cat_in_the_hat"
+ }
+ },
+ },
+ }
+}
+</pre></notextile>
+
+This pipeline consists of three components. The component "cleanup" depends on "thing1" and "thing2". Both "thing1" and "thing2" are started immediately and can run in parallel, because they do not depend on each other, but "cleanup" cannot begin until both "thing1" and "thing2" have completed.
+
+<notextile><pre>
+{
+ "name": "Clean the house",
+ "components": {
+ "thing1": {
+ "script": "thing1.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": { }
+ },
+ "thing2": {
+ "script": "thing2.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": { }
+ },
+ "cleanup": {
+ "script": "cleanup.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "mess1": {
+ "output_of": "thing1"
+ },
+ "mess2": {
+ "output_of": "thing2"
+ }
+ }
+ }
+ }
+}
+</pre></notextile>
In order to reassemble the file, Keep stores a *collection* data block which lists in sequence the data blocks that make up the original file. A collection data block may store the information for multiple files, including a directory structure.
-In this example we will use @c1bad4b39ca5a924e481008009d94e32+210@ which we added to Keep in "the first Keep tutorial":{{ site.baseurl }}/users/tutorial/tutorial-keep.html. First let us examine the contents of this collection using @arv keep get@:
+In this example we will use @c1bad4b39ca5a924e481008009d94e32+210@ which we added to Keep in "the first Keep tutorial":{{ site.baseurl }}/user/tutorials/tutorial-keep.html. First let us examine the contents of this collection using @arv keep get@:
<notextile>
<pre><code>~$ <span class="userinput">arv keep get c1bad4b39ca5a924e481008009d94e32+210</span>
</code></pre>
</notextile>
-The command @arv keep get@ fetches the contents of the locator @c1bad4b39ca5a924e481008009d94e32+210@. This is a locator for a collection data block, so it fetches the contents of the collection. In this example, this collection consists of a single file @var-GS000016015-ASM.tsv.bz2@ which is 227212247 bytes long, and is stored using four sequential data blocks, <code>204e43b8a1185621ca55a94839582e6f+67108864</code>, <code>b9677abbac956bd3e86b1deb28dfac03+67108864</code>, <code>fc15aff2a762b13f521baf042140acec+67108864</code>, <code>323d2a3ce20370c4ca1d3462a344f8fd+25885655</code>.
+The command @arv keep get@ fetches the contents of the collection @c1bad4b39ca5a924e481008009d94e32+210@. In this example, this collection includes a single file @var-GS000016015-ASM.tsv.bz2@ which is 227212247 bytes long, and is stored using four sequential data blocks, @204e43b8a1185621ca55a94839582e6f+67108864@, @b9677abbac956bd3e86b1deb28dfac03+67108864@, @fc15aff2a762b13f521baf042140acec+67108864@, and @323d2a3ce20370c4ca1d3462a344f8fd+25885655@.
-Let's use @arv keep get@ to download the first datablock:
+Let's use @arv keep get@ to download the first data block:
notextile. <pre><code>~$ <span class="userinput">cd /scratch/<b>you</b></span>
/scratch/<b>you</b>$ <span class="userinput">arv keep get 204e43b8a1185621ca55a94839582e6f+67108864 > block1</span></code></pre>
</notextile>
Notice that the block identifer <code>204e43b8a1185621ca55a94839582e6f+67108864</code> consists of:
-* the md5 hash @204e43b8a1185621ca55a94839582e6f@ which matches the md5 hash of @block1@
-* a size hint @67108864@ which matches the size of @block1@
+* the md5 hash of @block1@, @204e43b8a1185621ca55a94839582e6f@, plus
+* the size of @block1@, @67108864@.
"script_parameters":{
"input": "887cd41e9c613463eab2f0d885c6dd96+83"
},
- "script_version":"<b>you</b>:master"
+ "repository":"$USER",
+ "script_version":"master"
},
"filter":{
"script":"0-filter.py",
"output_of":"do_hash"
}
},
- "script_version":"<b>you</b>:master"
+ "repository":"$USER",
+ "script_version":"master"
}
}
}
~$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span></code></pre>
</notextile>
+(Your shell should automatically fill in @$USER@ with your login name. The JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
You can run this pipeline from the command line using @arv pipeline run@, filling in the UUID that you received from @arv pipeline_template create@:
<notextile>
-<pre><code>~$ <span class="userinput">arv pipeline run --template qr1hi-p5p6p-xxxxxxxxxxxxxxx</span>
+<pre><code>~$ <span class="userinput">arv pipeline run --run-here --template qr1hi-p5p6p-xxxxxxxxxxxxxxx</span>
2013-12-16 14:08:40 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 queued 2013-12-16T14:08:40Z
filter - -
2013-12-16 14:08:51 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 8e1b6acdd3f2f1da722538127c5c6202+56
+do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 1ed9ed18ef31ad21bcabcfeff7777bae+162
filter qr1hi-8i9sb-w5k40fztqgg9i2x queued 2013-12-16T14:08:50Z
2013-12-16 14:09:01 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 8e1b6acdd3f2f1da722538127c5c6202+56
-filter qr1hi-8i9sb-w5k40fztqgg9i2x 735ac35adf430126cf836547731f3af6+56
+do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 1ed9ed18ef31ad21bcabcfeff7777bae+162
+filter qr1hi-8i9sb-w5k40fztqgg9i2x d3bcc2ee0f0ea31049000c721c0f3a2a+56
</code></pre>
</notextile>
-This instantiates your pipeline and displays a live feed of its status. The new pipeline instance will also show up on the Workbench %(rarr)→% Compute %(rarr)→% Pipeline instances page.
+This instantiates your pipeline and displays a live feed of its status. The new pipeline instance will also show up on Workbench *Activity* %(rarr)→% *Recent pipeline instances* page.
Arvados adds each pipeline component to the job queue as its dependencies are satisfied (or immediately if it has no dependencies) and finishes when all components are completed or failed and there is no more work left to do.
-The Keep locators of the output of each of @"do_hash"@ and @"filter"@ component are available from the output log shown above. The output is also available on the Workbench by navigating to %(rarr)→% Compute %(rarr)→% Pipeline instances %(rarr)→% pipeline uuid under the *id* column %(rarr)→% components.
+The Keep locators of the output of each of @"do_hash"@ and @"filter"@ component are available from the output log shown above. The output is also available on the Workbench by navigating to *Activity* %(rarr)→% *Recent pipeline instances* %(rarr)→% pipeline UUID under the *Instance* column %(rarr)→% *output* column.
<notextile>
-<pre><code>~$ <span class="userinput">arv keep get 8e1b6acdd3f2f1da722538127c5c6202+56/md5sum.txt</span>
-0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
-504938460ef369cd275e4ef58994cffe bob.txt
-8f3b36aff310e06f3c5b9e95678ff77a carol.txt
-~$ <span class="userinput">arv keep get 735ac35adf430126cf836547731f3af6+56/0-filter.txt</span>
-0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
+<pre><code>~$ <span class="userinput">arv keep get 1ed9ed18ef31ad21bcabcfeff7777bae+162/md5sum.txt</span>
+0f1d6bcf55c34bed7f92a805d2d89bbf 887cd41e9c613463eab2f0d885c6dd96+83/./alice.txt
+504938460ef369cd275e4ef58994cffe 887cd41e9c613463eab2f0d885c6dd96+83/./bob.txt
+8f3b36aff310e06f3c5b9e95678ff77a 887cd41e9c613463eab2f0d885c6dd96+83/./carol.txt
+~$ <span class="userinput">arv keep get d3bcc2ee0f0ea31049000c721c0f3a2a+56/0-filter.txt</span>
+0f1d6bcf55c34bed7f92a805d2d89bbf 887cd41e9c613463eab2f0d885c6dd96+83/./alice.txt
</code></pre>
</notextile>
You can specify values for pipeline component script_parameters like this:
<notextile>
-<pre><code>~$ <span class="userinput">arv pipeline run --template qr1hi-p5p6p-xxxxxxxxxxxxxxx do_hash::input=c1bad4b39ca5a924e481008009d94e32+210</span>
+<pre><code>~$ <span class="userinput">arv pipeline run --run-here --template qr1hi-p5p6p-xxxxxxxxxxxxxxx do_hash::input=c1bad4b39ca5a924e481008009d94e32+210</span>
2013-12-17 20:31:24 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
do_hash qr1hi-8i9sb-rffhuay4jryl2n2 queued 2013-12-17T20:31:24Z
filter - -
filter - -
2013-12-17 20:31:55 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 880b55fb4470b148a447ff38cacdd952+54
+do_hash qr1hi-8i9sb-rffhuay4jryl2n2 50cafdb29cc21dd6eaec85ba9e0c6134+56
filter qr1hi-8i9sb-j347g1sqovdh0op queued 2013-12-17T20:31:55Z
2013-12-17 20:32:05 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 880b55fb4470b148a447ff38cacdd952+54
+do_hash qr1hi-8i9sb-rffhuay4jryl2n2 50cafdb29cc21dd6eaec85ba9e0c6134+56
filter qr1hi-8i9sb-j347g1sqovdh0op 490cd451c8108824b8a17e3723e1f236+19
</code></pre>
</notextile>
Now check the output:
<notextile>
-<pre><code>~$ <span class="userinput">arv keep get 880b55fb4470b148a447ff38cacdd952+54/md5sum.txt</span>
-44b8ae3fde7a8a88d2f7ebd237625b4f var-GS000016015-ASM.tsv.bz2
+<pre><code>~$ <span class="userinput">arv keep get 50cafdb29cc21dd6eaec85ba9e0c6134+56/md5sum.txt</span>
+44b8ae3fde7a8a88d2f7ebd237625b4f c1bad4b39ca5a924e481008009d94e32+210/./var-GS000016015-ASM.tsv.bz2
~$ <span class="userinput">arv keep get 490cd451c8108824b8a17e3723e1f236+19/0-filter.txt</span>
-~$
</code></pre>
</notextile>
-Since none of the files in the collection have hash code that start with 0, output of the filter component is empty.
+Since none of the files in the collection have hash code that start with 0, the output of the filter component is empty.
~$ <span class="userinput">cat >the_job <<EOF
{
"script":"GATK2-VariantFiltration",
+ "repository":"arvados",
"script_version":"$src_version",
"script_parameters":
{
h2. Create a new script
-Change to your git directory and create a new script in "crunch_scripts/".
+Change to your git directory and create a new script in @crunch_scripts/@.
<notextile>
<pre><code>~$ <span class="userinput">cd <b>you</b>/crunch_scripts</span>
h2. Using arv-crunch-job to run the job in your VM
-Instead of a git commit hash, we provide the path to the directory in the "script_version" parameter. The script specified in "script" will actually be searched for in the "crunch_scripts/" subdirectory of the directory specified "script_version". Although we are running the script locally, the script still requires access to the Arvados API server and Keep storage service. The job will be recorded in the Arvados job history, and visible in Workbench.
+Instead of a git commit hash, we provide the path to the directory in the "script_version" parameter. The script specified in "script" will actually be searched for in the @crunch_scripts/@ subdirectory of the directory specified "script_version". Although we are running the script locally, the script still requires access to the Arvados API server and Keep storage service. The job will be recorded in the Arvados job history, and visible in Workbench.
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
+ "repository":"",
"script":"hello-world.py",
- "script_version":"/home/<b>you</b>/<b>you</b>",
+ "script_version":"$HOME/$USER",
"script_parameters":{}
}
EOF</span>
-~/<b>you</b>/crunch_scripts</span>$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
+</code></pre>
+</notextile>
+
+Your shell should fill in values for @$HOME@ and @$USER@ so that the saved JSON points "script_version" at the directory with your checkout. Now you can run that job:
+
+<notextile>
+<pre><code>~/<b>you</b>/crunch_scripts</span>$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 check slurm allocation
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 node localhost - 1 slots
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 start
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 stderr hello world
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 child 29834 on localhost.1 exit 0 signal 0 success=
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 failure (#1, permanent) after 0 seconds
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 output
+2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 output
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 Every node has failed -- giving up on this round
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 wait for last 0 children to finish
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 status: 0 done, 0 running, 0 todo
~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x hello-world-fixed.py</span>
~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
+ "repository":"",
"script":"hello-world-fixed.py",
- "script_version":"/home/<b>you</b>/<b>you</b>",
+ "script_version":"$HOME/$USER",
"script_parameters":{}
}
EOF</span>
2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 check slurm allocation
2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 node localhost - 1 slots
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 start
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world.py
+2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world-fixed.py
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_version /home/<b>you</b>/<b>you</b>
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_parameters {}
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 runtime_constraints {"max_tasks_per_node":0}
2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 Freeze not implemented
2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 collate
2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 output 576c44d762ba241b0a674aa43152b52a+53
+WARNING:root:API lookup failed for collection 576c44d762ba241b0a674aa43152b52a+53 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://qr1hi.arvadosapi.com/arvados/v1/collections/576c44d762ba241b0a674aa43152b52a%2B53?alt=json returned "Not Found">)
2013-12-12_21:57:03 qr1hi-8i9sb-79260ykfew5trzl 31578 finish
-2013-12-12_21:57:04 qr1hi-8i9sb-79260ykfew5trzl 31578 meta key is 9f937693334d0c9234ccc1f808ee7117+1761
</code></pre>
</notextile>
+(The WARNING issued near the end of the script may be safely ignored here; it is the Arvados SDK letting you know that it could not find a collection named @576c44d762ba241b0a674aa43152b52a+53@ and that it is going to try looking up a block by that name instead.)
+
The job succeeded, with output in Keep object @576c44d762ba241b0a674aa43152b52a+53@. Let's look at our output:
<notextile>
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
-You will create a job to run the "hash" crunch script. The "hash" script computes the md5 hash of each file in a collection.
+You will create a job to run the "hash" Crunch script. The "hash" script computes the md5 hash of each file in a collection.
h2. Jobs
-Crunch pipelines consist of one or more jobs. A "job" is a single run of a specific version of a crunch script with a specific input. You an also run jobs individually.
+Crunch pipelines consist of one or more jobs. A "job" is a single run of a specific version of a Crunch script with a specific input. You can also run jobs individually.
-A request to run a crunch job are is described using a JSON object. For example:
+A request to run a Crunch job are is described using a JSON object. For example:
<notextile>
-<pre><code>~$ <span class="userinput">cat >the_job <<EOF
+<pre><code>~$ <span class="userinput">cat >~/the_job <<EOF
{
"script": "hash",
- "script_version": "arvados:master",
- "script_parameters":
- {
+ "repository": "arvados",
+ "script_version": "master",
+ "script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
- }
+ },
+ "no_reuse": "true"
}
EOF
</code></pre>
</notextile>
-* @cat@ is a standard Unix utility that simply copies standard input to standard output
-* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@
-* @>the_job@ redirects standard output to a file called @the_job@
-* @"script"@ specifies the name of the script to run. The script is searched for in the "crunch_scripts/" subdirectory of the @git@ checkout specified by @"script_version"@.
-* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, or in the form "repository:branch" (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run. You can access a list of available @git@ repositories on the Arvados workbench under "Compute %(rarr)→% Code repositories":http://{{site.arvados_workbench_host}}/repositories .
-* @"script_parameters"@ are provided to the script. In this case, the input is the locator for the collection that we inspected in the previous section.
+* @cat@ is a standard Unix utility that writes a sequence of input to standard output.
+* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@.
+* @>~/the_job@ redirects standard output to a file called @~/the_job@.
+* @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench under "*Compute* %(rarr)→% *Code repositories*":https://{{site.arvados_workbench_host}}/repositories.
+* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit git revision hash, a tag, or a branch (in which case it will use the most recent commit on the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
+* @"script"@ specifies the name of the script to run. The script is searched for in the @crunch_scripts/@ subdirectory of the git repository.
+* @"script_parameters"@ are provided to the script. In this case, the input is the PGP data Collection that we "put in Keep earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
+* Setting the @"no_reuse"@ flag tells Crunch not to reuse work from past jobs. This helps ensure that you can watch a new Job process for the rest of this tutorial, without reusing output from a past run that you made, or somebody else marked as public. (If you want to experiment, after the first run below finishes, feel free to edit this job to remove the @"no_reuse"@ line and resubmit it. See what happens!)
Use @arv job create@ to actually submit the job. It should print out a JSON object which describes the newly created job:
<notextile>
-<pre><code>~$ <span class="userinput">arv job create --job "$(cat the_job)"</span>
+<pre><code>~$ <span class="userinput">arv job create --job "$(cat ~/the_job)"</span>
{
"href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
"kind":"arvados#job",
The job is now queued and will start running as soon as it reaches the front of the queue. Fields to pay attention to include:
- * @"uuid"@ is the unique identifier for this specific job
+ * @"uuid"@ is the unique identifier for this specific job.
* @"script_version"@ is the actual revision of the script used. This is useful if the version was described using the "repository:branch" format.
h2. Monitor job progress
-Go to the "Workbench dashboard":http://{{site.arvados_workbench_host}}. Your job should be at the top of the "Recent jobs" table. This table refreshes automatically. When the job has completed successfully, it will show <span class="label label-success">finished</span> in the *Status* column.
+Go to the "Workbench dashboard":https://{{site.arvados_workbench_host}} and visit *Activity* %(rarr)→% *Recent jobs*. Your job should be near the top of the table. This table refreshes automatically. When the job has completed successfully, it will show <span class="label label-success">finished</span> in the *Status* column.
On the command line, you can access log messages while the job runs using @arv job log_tail_follow@:
h2. Inspect the job output
-On the "Workbench dashboard":http://{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table. Click on the link under *Output* for your job to go to the files page with the job output. The files page lists all the files that were output by the job. Click on the link under the *files* column to view a file, or click on the download icon <span class="glyphicon glyphicon-download-alt"></span> to download the output file.
+On the "Workbench dashboard":https://{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table. Click on the link under *Output* for your job to go to the files page with the job output. The files page lists all the files that were output by the job. Click on the link under the *file* column to view a file, or click on the download icon <span class="glyphicon glyphicon-download-alt"></span> to download the output file.
On the command line, you can use @arv job get@ to access a JSON object describing the output:
<notextile>
<pre><code>~$ <span class="userinput">arv keep ls dd755dbc8d49a67f4fe7dc843e4f10a6+54</span>
-md5sum.txt
+./md5sum.txt
</code></pre>
</notextile>
</code></pre>
</notextile>
-This md5 hash matches the md5 hash which we computed earlier.
+This md5 hash matches the md5 hash which we "computed earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
h2. The job log
-When the job completes, you can access the job log. On the workbench dashboard, this is the link under the *Log* column of the *Recent jobs* table.
+When the job completes, you can access the job log. On the Workbench, visit *Activity* %(rarr)→% *Recent jobs* %(rarr)→% your job's UUID under the *uuid* column %(rarr)→% the collection link on the *log* row.
-On the command line, the keep identifier listed in the @"log"@ field from @arv job get@ specifies a collection. You can list the files in the collection:
+On the command line, the Keep identifier listed in the @"log"@ field from @arv job get@ specifies a collection. You can list the files in the collection:
<notextile>
<pre><code>~$ <span class="userinput">arv keep ls xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91</span>
-qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
+./qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
</code></pre>
</notextile>
-The log collection consists of one log file named with the job id. You can access it using @arv keep get@:
+The log collection consists of one log file named with the job's UUID. You can access it using @arv keep get@:
<notextile>
<pre><code>~$ <span class="userinput">arv keep get xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91/qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt</span>
-2013-12-16_20:44:35 qr1hi-8i9sb-1pm1t02dezhupss 7575 check slurm allocation
-2013-12-16_20:44:35 qr1hi-8i9sb-1pm1t02dezhupss 7575 node compute13 - 8 slots
-2013-12-16_20:44:36 qr1hi-8i9sb-1pm1t02dezhupss 7575 start
-2013-12-16_20:44:36 qr1hi-8i9sb-1pm1t02dezhupss 7575 Install revision d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 Clean-work-dir exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 Install exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 script hash
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 script_version d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 runtime_constraints {"max_tasks_per_node":0}
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 start level 0
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 0 done, 0 running, 1 todo
-2013-12-16_20:44:38 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 job_task qr1hi-ot0gb-23c1k3kwrf8da62
-2013-12-16_20:44:38 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 child 7681 started on compute13.1
-
-2013-12-16_20:44:38 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 0 done, 1 running, 0 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 child 7681 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 success in 1 seconds
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 output
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 wait for last 0 children to finish
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 start level 1
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 job_task qr1hi-ot0gb-iwr0o3unqothg28
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 child 7716 started on compute13.1
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 1 done, 1 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 child 7716 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 success in 13 seconds
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 output dd755dbc8d49a67f4fe7dc843e4f10a6+54
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 wait for last 0 children to finish
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 2 done, 0 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 release job allocation
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 Freeze not implemented
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 collate
-2013-12-16_20:44:53 qr1hi-8i9sb-1pm1t02dezhupss 7575 output dd755dbc8d49a67f4fe7dc843e4f10a6+54+K@qr1hi
-2013-12-16_20:44:53 qr1hi-8i9sb-1pm1t02dezhupss 7575 finish
+2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 check slurm allocation
+2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 node compute13 - 8 slots
+2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 start
+2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Install revision d9cd657b733d578ac0d2167dd75967aa4f22e0ac
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Clean-work-dir exited 0
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Install exited 0
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 script hash
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 script_version d9cd657b733d578ac0d2167dd75967aa4f22e0ac
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 runtime_constraints {"max_tasks_per_node":0}
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 start level 0
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 0 done, 0 running, 1 todo
+2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 job_task qr1hi-ot0gb-23c1k3kwrf8da62
+2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 started on compute13.1
+2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 0 done, 1 running, 0 todo
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 on compute13.1 exit 0 signal 0 success=true
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 success in 1 seconds
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 output
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 wait for last 0 children to finish
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 1 done, 0 running, 1 todo
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 start level 1
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 1 done, 0 running, 1 todo
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 job_task qr1hi-ot0gb-iwr0o3unqothg28
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 started on compute13.1
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 1 done, 1 running, 0 todo
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 on compute13.1 exit 0 signal 0 success=true
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 success in 13 seconds
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 output dd755dbc8d49a67f4fe7dc843e4f10a6+54
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 wait for last 0 children to finish
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 2 done, 0 running, 0 todo
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 release job allocation
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Freeze not implemented
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 collate
+2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 output dd755dbc8d49a67f4fe7dc843e4f10a6+54+K@qr1hi
+2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 finish
</code></pre>
</notextile>
notextile. <pre>~/<b>you</b>/crunch_scripts$ <code class="userinput">nano parallel-hash.py</code></pre>
-Add the following code to compute the md5 hash of each file in a
+Add the following code to compute the md5 hash of each file in a collection:
<notextile> {% code 'parallel_hash_script_py' as python %} </notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
"script": "parallel-hash.py",
- "script_version": "<b>you</b>:master",
+ "repository": "$USER",
+ "script_version": "master",
"script_parameters":
{
"input": "887cd41e9c613463eab2f0d885c6dd96+83"
</code></pre>
</notextile>
+(Your shell should automatically fill in @$USER@ with your login name. The job JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
Because the job ran in parallel, each instance of parallel-hash creates a separate @md5sum.txt@ as output. Arvados automatically collates theses files into a single collection, which is the output of the job:
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">arv keep ls e2ccd204bca37c77c0ba59fc470cd0f7+162</span>
-md5sum.txt
-md5sum.txt
-md5sum.txt
+./md5sum.txt
~/<b>you</b>/crunch_scripts$ <span class="userinput">arv keep get e2ccd204bca37c77c0ba59fc470cd0f7+162/md5sum.txt</span>
0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
504938460ef369cd275e4ef58994cffe bob.txt
</code></pre>
</notextile>
-Next, using @nano@ or your favorite Unix text editor, create a new file called @run-md5sum.py@ in the @crunch_scripts@ directory.
+Next, using @nano@ or your favorite Unix text editor, create a new file called @run-md5sum.py@ in the @crunch_scripts@ directory.
notextile. <pre>~/<b>you</b>/crunch_scripts$ <code class="userinput">nano run-md5sum.py</code></pre>
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x run-md5sum.py</span></code></pre>
-Next, add the file to @git@ staging, commit and push:
+Next, use @git@ to stage the file, commit, and push:
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git add run-md5sum.py</span>
</code></pre>
</notextile>
-You should now be able to run your new script using Crunch, with "script" referring to our new "run-md5sum.py" script.
+You should now be able to run your new script using Crunch, with @"script"@ referring to our new @run-md5sum.py@ script.
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_pipeline <<EOF
"dataclass": "Collection"
}
},
- "script_version":"<b>you</b>:master"
+ "repository":"$USER",
+ "script_version":"master"
}
}
}
</code></pre>
</notextile>
-Your new pipeline template will appear on the "Workbench %(rarr)→% Compute %(rarr)→% Pipeline templates":http://{{ site.arvados_workbench_host }}/pipeline_instances page. You can run the "pipeline using workbench":tutorial-pipeline-workbench.html
+(Your shell should automatically fill in @$USER@ with your login name. The JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
+Your new pipeline template will appear on the Workbench "Compute %(rarr)→% Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_instances page. You can run the "pipeline using Workbench":tutorial-pipeline-workbench.html.
h2. Setting up Git
-As discussed in the previous tutorial, all Crunch scripts are managed through the @git@ revision control system.
-
-First, you should do some basic configuration for git (you only need to do this the first time):
+All Crunch scripts are managed through the @git@ revision control system. Before you start using git, you should do some basic configuration (you only need to do this the first time):
<notextile>
<pre><code>~$ <span class="userinput">git config --global user.name "Your Name"</span>
~$ <span class="userinput">git config --global user.email <b>you</b>@example.com</span></code></pre>
</notextile>
-On the Arvados Workbench, navigate to "Compute %(rarr)→% Code repositories":http://{{site.arvados_workbench_host}}/repositories . You should see a repository with your user name listed in the *name* column. Next to *name* is the column *push_url*. Copy the *push_url* value associated with your repository. This should look like <notextile><code>git@git.{{ site.arvados_api_host }}:<b>you</b>.git</code></notextile>.
+On the Arvados Workbench, navigate to "Compute %(rarr)→% Code repositories":https://{{site.arvados_workbench_host}}/repositories. You should see a repository with your user name listed in the *name* column. Next to *name* is the column *push_url*. Copy the *push_url* value associated with your repository. This should look like <notextile><code>git@git.{{ site.arvados_api_host }}:<b>you</b>.git</code></notextile>.
Next, on the Arvados virtual machine, clone your git repository:
notextile. <pre><code>$ <span class="userinput">man gittutorial</span></code></pre>
-or <b>"click here to search Google for git tutorials":http://google.com/#q=git+tutorial</b>
+or *"search Google for git tutorials":http://google.com/#q=git+tutorial*.
{% include 'notebox_end' %}
h2. Creating a Crunch script
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x hash.py</span></code></pre>
{% include 'notebox_begin' %}
-The steps below describe how to execute the script after committing changes to git. To run a script locally for testing, please see "debugging a crunch script":{{site.baseurl}}/user/topics/tutorial-job-debug.html .
+The steps below describe how to execute the script after committing changes to git. To run a script locally for testing, please see "debugging a crunch script":{{site.baseurl}}/user/topics/tutorial-job-debug.html.
{% include 'notebox_end' %}
-Next, add the file to @git@ staging. This tells @git@ that the file should be included on the next commit.
+Next, add the file to git staging. This tells @git@ that the file should be included on the next commit.
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git add hash.py</span></code></pre>
-Next, commit your changes to git. All staged changes are recorded into the local @git@ repository:
+Next, commit your changes to git. All staged changes are recorded into the local git repository:
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git commit -m"my first script"</span>
"dataclass": "Collection"
}
},
- "script_version":"<b>you</b>:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":true
}
}
}
</span></code></pre>
</notextile>
-* @cat@ is a standard Unix utility that simply copies standard input to standard output
-* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@
-* @>the_pipeline@ redirects standard output to a file called @the_pipeline@
-* @"name"@ is a human-readable name for the pipeline
-* @"components"@ is a set of scripts that make up the pipeline
-* The component is listed with a human-readable name (@"do_hash"@ in this example)
-* @"script"@ specifies the name of the script to run. The script is searched for in the "crunch_scripts/" subdirectory of the @git@ checkout specified by @"script_version"@.
-* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, or in the form "repository:branch" (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run. You can access a list of available @git@ repositories on the Arvados workbench under "Compute %(rarr)→% Code repositories":http://{{site.arvados_workbench_host}}//repositories .
+* @cat@ is a standard Unix utility that writes a sequence of input to standard output.
+* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@.
+* @>the_pipeline@ redirects standard output to a file called @the_pipeline@.
+* @"name"@ is a human-readable name for the pipeline.
+* @"components"@ is a set of scripts that make up the pipeline.
+* The component is listed with a human-readable name (@"do_hash"@ in this example).
+* @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench under "Compute %(rarr)→% Code repositories":https://{{site.arvados_workbench_host}}/repositories. Your shell should automatically fill in @$USER@ with your login name, so that the final JSON has @"repository"@ pointed at your personal git repository.
+* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit git revision hash, a tag, or a branch (in which case it will use the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
+* @"script"@ specifies the filename of the script to run. Crunch expects to find this in the @crunch_scripts/@ subdirectory of the git repository.
* @"script_parameters"@ describes the parameters for the script. In this example, there is one parameter called @input@ which is @required@ and is a @Collection@.
+* @"output_is_persistent"@ indicates whether the output of the job is considered valuable. If this value is false (or not given), the output will be treated as intermediate data and eventually deleted to reclaim disk space.
-Now, use @arv pipeline_template create@ tell Arvados about your pipeline template:
+Now, use @arv pipeline_template create@ to register your pipeline template in Arvados:
<notextile>
<pre><code>~$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span>
</code></pre>
</notextile>
-Your new pipeline template will appear on the "Workbench %(rarr)→% Compute %(rarr)→% Pipeline templates":http://{{ site.arvados_workbench_host }}/pipeline_instances page. You can run the "pipeline using workbench":tutorial-pipeline-workbench.html
+Your new pipeline template will appear on the Workbench "Compute %(rarr)→% Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_instances page. You can run the "pipeline using Workbench":tutorial-pipeline-workbench.html.
The Arvados distributed file system is called *Keep*. Keep is a content-addressable file system. This means that files are managed using special unique identifiers derived from the _contents_ of the file, rather than human-assigned file names (specifically, the md5 hash). This has a number of advantages:
* Files can be stored and replicated across a cluster of servers without requiring a central name server.
-* Systematic validation of data integrity by both server and client because the checksum is built into the identifier.
-* Minimizes data duplication (two files with the same contents will result in the same identifier, and will not be stored twice.)
-* Avoids data race conditions (an identifier always points to the same data.)
+* Both the server and client systematically validate data integrity because the checksum is built into the identifier.
+* Data duplication is minimized—two files with the same contents will have in the same identifier, and will not be stored twice.
+* It avoids data race conditions, since an identifier always points to the same data.
h1. Putting Data into Keep
-We will start with downloading a freely available VCF file from the "Personal Genome Project (PGP)":http://www.personalgenomes.org subject "hu599905":https://my.personalgenomes.org/profile/hu599905 to a staging directory on the VM, and then add it to Keep.
+We will start by downloading a freely available VCF file from "Personal Genome Project (PGP)":http://www.personalgenomes.org subject "hu599905":https://my.personalgenomes.org/profile/hu599905 to a staging directory on the VM, and adding it to Keep. In the following commands, replace *@you@* with your login name.
-In the following tutorials, replace <b><code>you</code></b> with your user id.
-
-First, log into the Arvados VM instance and set up the staging area:
+First, log into your Arvados VM and set up the staging area:
notextile. <pre><code>~$ <span class="userinput">mkdir /scratch/<b>you</b></span></code></pre>
/scratch/<b>you</b>$ <span class="userinput">echo "hello bob" > tmp/bob.txt</span>
/scratch/<b>you</b>$ <span class="userinput">echo "hello carol" > tmp/carol.txt</span>
/scratch/<b>you</b>$ <span class="userinput">arv keep put tmp</span>
-0M / 0M 100.0%
+0M / 0M 100.0%
887cd41e9c613463eab2f0d885c6dd96+83
</code></pre>
</notextile>
h2. Using Workbench
-You may access collections through the "Collections section of Arvados Workbench":https://{{ site.arvados_workbench_host }}/collections located at "https://{{ site.arvados_workbench_host }}/collections":https://{{ site.arvados_workbench_host }}/collections . You can also access individual collections and individual files within a collection. Some examples:
+You may access collections through the "Collections section of Arvados Workbench":https://{{ site.arvados_workbench_host }}/collections at *Data* %(rarr)→% *Collections (data files)*. You can also access individual files within a collection. Some examples:
* "https://{{ site.arvados_workbench_host }}/collections/c1bad4b39ca5a924e481008009d94e32+210":https://{{ site.arvados_workbench_host }}/collections/c1bad4b39ca5a924e481008009d94e32+210
* "https://{{ site.arvados_workbench_host }}/collections/887cd41e9c613463eab2f0d885c6dd96+83/alice.txt":https://{{ site.arvados_workbench_host }}/collections/887cd41e9c613463eab2f0d885c6dd96+83/alice.txt
-h2(#arv-get). Using arv-get
+h2(#arv-get). Using the command line
You can view the contents of a collection using @arv keep ls@:
<notextile>
<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep get c1bad4b39ca5a924e481008009d94e32+210/ .</span>
+/scratch/<b>you</b>$ <span class="userinput">ls var-GS000016015-ASM.tsv.bz2</span>
+var-GS000016015-ASM.tsv.bz2
</code></pre>
</notextile>
h2. Using arv-mount
-Use @arv-mount@ to take advantage of the "File System in User Space / FUSE":http://fuse.sourceforge.net/ feature of the Linux kernel to mount a Keep collection as if it were a regular directory tree.
+Use @arv-mount@ to mount a Keep collection and access it using traditional filesystem tools.
<notextile>
-<pre><code>/scratch/<b>you</b>$ <span class="userinput">mkdir mnt</span>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">mkdir -p mnt</span>
/scratch/<b>you</b>$ <span class="userinput">arv-mount --collection c1bad4b39ca5a924e481008009d94e32+210 mnt &</span>
/scratch/<b>you</b>$ <span class="userinput">cd mnt</span>
/scratch/<b>you</b>/mnt$ <span class="userinput">ls</span>
You can also mount the entire Keep namespace in "magic directory" mode:
<notextile>
-<pre><code>/scratch/<b>you</b>$ <span class="userinput">mkdir mnt</span>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">mkdir -p mnt</span>
/scratch/<b>you</b>$ <span class="userinput">arv-mount mnt &</span>
/scratch/<b>you</b>$ <span class="userinput">cd mnt/c1bad4b39ca5a924e481008009d94e32+210</span>
/scratch/<b>you</b>/mnt/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">ls</span>
</code></pre>
</notextile>
-Using @arv-mount@ has several significant benefits:
+@arv-mount@ provides several features:
* You can browse, open and read Keep entries as if they are regular files.
* It is easy for existing tools to access files in Keep.
-* Data is downloaded on demand, it is not necessary to download an entire file or collection to start processing
+* Data is downloaded on demand. It is not necessary to download an entire file or collection to start processing.
"dataclass": "Collection"
}
},
- "script_version":"<b>you</b>:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":false
},
- "filter":{
+ "do_filter":{
"script":"0-filter.py",
"script_parameters":{
"input":{
"output_of":"do_hash"
}
},
- "script_version":"<b>you</b>:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":true
}
}
}
</span></code></pre>
</notextile>
-* @"output_of"@ indicates that the @input@ of the @do_hash@ component is connected to the @output@ of @filter@. This is a _dependency_. Arvados uses the dependencies between jobs to automatically determine the correct order to run the jobs.
+* @"output_of"@ indicates that the @output@ of the @do_hash@ component should be used as the @"input"@ of @do_filter@. Arvados uses these dependencies between jobs to automatically determine the correct order to run them.
-Now, use @arv pipeline_template create@ tell Arvados about your pipeline template:
+(Your shell should automatically fill in @$USER@ with your login name. The JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
+Now, use @arv pipeline_template create@ to register your pipeline template in Arvados:
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat ~/the_pipeline)"</span>
</code></pre>
</notextile>
-Your new pipeline template will appear on the "Workbench %(rarr)→% Compute %(rarr)→% Pipeline templates":http://{{ site.arvados_workbench_host }}/pipeline_instances page.
-
+Your new pipeline template will appear on the Workbench "Compute %(rarr)→% Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_instances page.
notextile. <div class="spaced-out">
-# Go to "Collections":http://{{ site.arvados_workbench_host }}/collections .
-# On the collections page, go to the search box <span class="glyphicon glyphicon-search"></span> and search for "tutorial".
-# This should yield a collection with the contents "var-GS000016015-ASM.tsv.bz2"
-# Click on the check box to the left of "var-GS000016015-ASM.tsv.bz2". This puts the collection in your persistent selection list. Click on the paperclip <span class="glyphicon glyphicon-paperclip"></span> in the upper right to get a dropdown menu listing your current selections.
-# Go to "Pipeline templates":http://{{ site.arvados_workbench_host }}/pipeline_templates .
-# Look for a pipeline named "Tutorial pipeline".
-# Click on the play button <span class="glyphicon glyphicon-play"></span> to the left of "Tutorial pipeline". This will take you to a new page to configure the pipeline.
-# Under *parameter* look for "input". Set the value of "input" by clicking on on "none" to get a editing popup. At the top of the selection list in the editing popup will be the collection that you selected in step 4.
-# You can now click on "Run pipeline" in the upper right to start the pipeline.
-# This will reload the page with the pipeline queued to run.
+# Go to "Collections":https://{{ site.arvados_workbench_host }}/collections (*Data* %(rarr)→% *Collections (data files)*).
+# On the Collections page, go to the search box <span class="glyphicon glyphicon-search"></span> and search for "tutorial".
+# The results should include a collection with the contents *var-GS000016015-ASM.tsv.bz2*.
+# Click on the check box to the left of *var-GS000016015-ASM.tsv.bz2*. This puts the collection in your persistent selection list. You can click on the paperclip <span class="glyphicon glyphicon-paperclip"></span> in the upper right to review your current selections.
+# Go to "Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_templates (*Compute* %(rarr)→% *Pipeline templates*).
+# Look for a pipeline named *Tutorial pipeline*.
+# Click on the play button <span class="glyphicon glyphicon-play"></span> to the left of *Tutorial pipeline*. This will take you to a new page to configure the pipeline.
+# Under the *parameter* column, look for *input*. Set the value of *input* by clicking on *none* to get a selection popup. The collection that you selected in step 4 will be at the top of that pulldown menu. Select that collection in the pulldown menu.
+# You can now click on the *Run pipeline* button in the upper right to start the pipeline. A new page shows the pipeline status, queued to run.
# The page refreshes automatically every 15 seconds. You should see the pipeline running, and then finish successfully.
-# Once it is finished, click on the link under the *output* column. This will take you to the collection page for the output of this pipeline.
-# Click on "md5sum.txt" to see the actual file that is the output of this pipeline.
-# On the collection page, click on the "Provenance graph" tab to see a graphical representation of the data elements and pipelines that were involved in generating this file.
+# Once the pipeline is finished, click on the link under the *output* column. This will take you to the collection page for the output of this pipeline.
+# Click on *md5sum.txt* to see the actual file that is the output of this pipeline.
+# Go back to the collection page for the result. Click on the *Provenance graph* tab to see a graph illustrating the collections and scripts that were used to generate this file.
notextile. </div>
RUN apt-get update && \
apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 \
supervisor && \
- git clone git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
+ git clone --bare git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
RUN /bin/mkdir -p /usr/src/arvados/services
ADD generated/api.tar.gz /usr/src/arvados/services/
ENV RAILS_ENV production
ADD generated/config_databases.sh /tmp/config_databases.sh
ADD generated/superuser_token /tmp/superuser_token
-RUN sh /tmp/config_databases.sh && \
+RUN bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
+ sh /tmp/config_databases.sh && \
rm /tmp/config_databases.sh && \
/etc/init.d/postgresql start && \
cd /usr/src/arvados/services/api && \
./script/create_superuser_token.rb $(cat /tmp/superuser_token) && \
chown www-data:www-data config.ru && \
chown www-data:www-data log -R && \
+ mkdir -p tmp && \
chown www-data:www-data tmp -R
# Configure Apache and Passenger.
# config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
require 'net/http'
config.compute_node_nameservers = [ '@@ARVADOS_DNS_SERVER@@' ]
-
+ config.compute_node_domain = false
config.uuid_prefix = '@@API_HOSTNAME@@'
# Authentication stub: hard code pre-approved API tokens.
fi
if [[ "$2" != '' ]]; then
local name="$2"
- args="$args -name $name"
+ args="$args --name $name"
fi
if [[ "$3" != '' ]]; then
local volume="$3"
fi
if [[ "$4" != '' ]]; then
local link="$4"
- args="$args -link $link"
+ args="$args --link $link"
fi
local image=$5
# Install prerequisite packages for Arvados
# * git, curl, rvm
-# * Arvados source code in /usr/src/arvados-upstream, for preseeding gem installation
+# * Arvados source code in /usr/src/arvados, for preseeding gem installation
RUN apt-get update && \
- apt-get -q -y install -q -y openssh-server apt-utils git curl locales postgresql-server-dev-9.1 && \
+ apt-get -q -y install -q -y openssh-server apt-utils git curl \
+ libcurl3 libcurl3-gnutls libcurl4-openssl-dev locales \
+ postgresql-server-dev-9.1 && \
/bin/mkdir -p /root/.ssh && \
/bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
/usr/sbin/locale-gen && \
- curl -L https://get.rvm.io | bash -s stable --ruby=2.1.0 && \
- git clone https://github.com/curoverse/arvados.git /usr/src/arvados-upstream
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1.0 && \
+ /bin/mkdir -p /usr/src/arvados
+
+ADD generated/arvados.tar.gz /usr/src/arvados/
# Set up RVM environment. These are just the env variables created by
# /usr/local/rvm/scripts/rvm, which can't be run from a non-login shell.
# https://github.com/rubygems/rubygems.org/issues/613.
RUN gem update --system && \
gem install bundler && \
- bundle install --gemfile=/usr/src/arvados-upstream/apps/workbench/Gemfile && \
- bundle install --gemfile=/usr/src/arvados-upstream/services/api/Gemfile && \
- bundle install --gemfile=/usr/src/arvados-upstream/doc/Gemfile
+ bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
+ bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
+ bundle install --gemfile=/usr/src/arvados/doc/Gemfile
ADD generated/id_rsa.pub /root/.ssh/authorized_keys
RUN chown root:root /root/.ssh/authorized_keys
#! /bin/bash
-build_ok=true
-
-# Check that:
-# * IP forwarding is enabled in the kernel.
-
-if [ "$(/sbin/sysctl --values net.ipv4.ip_forward)" != "1" ]
-then
- echo >&2 "WARNING: IP forwarding must be enabled in the kernel."
- echo >&2 "Try: sudo sysctl net.ipv4.ip_forward=1"
- build_ok=false
-fi
-
-# * Docker can be found in the user's path
-# * The user is in the docker group
-# * cgroup is mounted
-# * the docker daemon is running
-
-if ! docker images > /dev/null 2>&1
+# make sure Ruby 1.9.3 is installed before proceeding
+if ! ruby -e 'exit RUBY_VERSION >= "1.9.3"' 2>/dev/null
then
- echo >&2 "WARNING: docker could not be run."
- echo >&2 "Please make sure that:"
- echo >&2 " * You have permission to read and write /var/run/docker.sock"
- echo >&2 " * a 'cgroup' volume is mounted on your machine"
- echo >&2 " * the docker daemon is running"
- build_ok=false
-fi
+ echo "Installing Arvados requires at least Ruby 1.9.3."
+ echo "You may need to enter your password."
+ read -p "Press Ctrl-C to abort, or else press ENTER to install ruby1.9.3 and continue. " unused
-# * config.yml exists
-if [ '!' -f config.yml ]
-then
- echo >&2 "WARNING: no config.yml found in the current directory"
- echo >&2 "Copy config.yml.example to config.yml and update it with settings for your site."
- build_ok=false
+ sudo apt-get update
+ sudo apt-get -y install ruby1.9.3
fi
-# If ok to build, then go ahead and run make
-if $build_ok
-then
- make $*
-fi
+build_tools/build.rb $*
# `make clean' removes the files generated in the build directory
# but does not remove any docker images generated in previous builds
clean:
+ -rm -rf build
-rm *-image */generated/*
-@rmdir */generated
# Dependencies for */generated files which are prerequisites
# for building docker images.
+CONFIG_RB = build_tools/config.rb
+
+BUILD = build/.buildstamp
+
BASE_DEPS = base/Dockerfile $(BASE_GENERATED)
API_DEPS = api/Dockerfile $(API_GENERATED)
SSO_DEPS = sso/passenger.conf $(SSO_GENERATED)
-BASE_GENERATED = base/generated
+BASE_GENERATED = base/generated/arvados.tar.gz
API_GENERATED = \
api/generated/apache2_vhost \
sso/seeds.rb.in \
sso/secret_token.rb.in
-$(BASE_GENERATED): config.yml
- ./config.rb
+$(BUILD):
+ mkdir -p build
+ rsync -rlp --exclude=docker/ --exclude='**/log/*' --exclude='**/tmp/*' \
+ --chmod=Da+rx,Fa+rX ../ build/
+ touch build/.buildstamp
+
+$(BASE_GENERATED): config.yml $(BUILD)
+ $(CONFIG_RB)
+ mkdir -p base/generated
+ tar -czf base/generated/arvados.tar.gz -C build .
$(API_GENERATED): config.yml $(API_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(WORKBENCH_GENERATED): config.yml $(WORKBENCH_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(WAREHOUSE_GENERATED): config.yml $(WAREHOUSE_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(SSO_GENERATED): config.yml $(SSO_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
# The docker build -q option suppresses verbose build output.
# Necessary to prevent failure on building warehouse; see
# ============================================================
# The main Arvados servers: api, doc, workbench, warehouse
-api-image: passenger-image $(API_DEPS)
+api-image: passenger-image $(BUILD) $(API_DEPS)
mkdir -p api/generated
- tar -c -z -f api/generated/api.tar.gz -C ../services api
+ tar -czf api/generated/api.tar.gz -C build/services api
$(DOCKER_BUILD) -t arvados/api api
- echo -n "Built at $(date)" > api-image
+ date >api-image
-doc-image: base-image $(DOC_DEPS)
+doc-image: base-image $(BUILD) $(DOC_DEPS)
mkdir -p doc/generated
- tar -c -z -f doc/generated/doc.tar.gz -C .. doc
+ tar -czf doc/generated/doc.tar.gz -C build doc
$(DOCKER_BUILD) -t arvados/doc doc
- echo -n "Built at $(date)" > doc-image
+ date >doc-image
-workbench-image: passenger-image $(WORKBENCH_DEPS)
+workbench-image: passenger-image $(BUILD) $(WORKBENCH_DEPS)
mkdir -p workbench/generated
- tar -c -z -f workbench/generated/workbench.tar.gz -C ../apps workbench
+ tar -czf workbench/generated/workbench.tar.gz -C build/apps workbench
$(DOCKER_BUILD) -t arvados/workbench workbench
- echo -n "Built at $(date)" > workbench-image
+ date >workbench-image
warehouse-image: base-image $(WAREHOUSE_DEPS)
$(DOCKER_BUILD) -t arvados/warehouse warehouse
- echo -n "Built at $(date)" > warehouse-image
+ date >warehouse-image
sso-image: passenger-image $(SSO_DEPS)
$(DOCKER_BUILD) -t arvados/sso sso
- echo -n "Built at $(date)" > sso-image
+ date >sso-image
# ============================================================
# The arvados/base image is the base Debian image plus packages
passenger-image: base-image
$(DOCKER_BUILD) -t arvados/passenger passenger
- echo -n "Built at $(date)" > passenger-image
+ date >passenger-image
base-image: debian-image $(BASE_DEPS)
$(DOCKER_BUILD) -t arvados/base base
- echo -n "Built at $(date)" > base-image
+ date >base-image
debian-image:
./mkimage-debootstrap.sh arvados/debian wheezy ftp://ftp.us.debian.org/debian/
- echo -n "Built at $(date)" > debian-image
-
+ date >debian-image
--- /dev/null
+#! /usr/bin/env ruby
+
+require 'optparse'
+require 'tempfile'
+require 'yaml'
+
+def main options
+ if not ip_forwarding_enabled?
+ warn "NOTE: IP forwarding must be enabled in the kernel."
+ warn "Turning IP forwarding on now."
+ sudo %w(/sbin/sysctl net.ipv4.ip_forward=1)
+ end
+
+ # Check that:
+ # * Docker is installed and can be found in the user's path
+ # * Docker can be run as a non-root user
+ # - TODO: put the user is in the docker group if necessary
+ # - TODO: mount cgroup automatically
+ # - TODO: start the docker service if not started
+
+ docker_path = %x(which docker).chomp
+ if docker_path.empty?
+ warn "Docker not found."
+ warn ""
+ warn "Please make sure that Docker has been installed and"
+ warn "can be found in your PATH."
+ warn ""
+ warn "Installation instructions for a variety of platforms can be found at"
+ warn "http://docs.docker.io/en/latest/installation/"
+ exit
+ elsif not docker_ok?
+ warn "WARNING: docker could not be run."
+ warn "Please make sure that:"
+ warn " * You have permission to read and write /var/run/docker.sock"
+ warn " * a 'cgroup' volume is mounted on your machine"
+ warn " * the docker daemon is running"
+ exit
+ end
+
+ # Check that debootstrap is installed.
+ if not debootstrap_ok?
+ warn "Installing debootstrap."
+ sudo '/usr/bin/apt-get', 'install', 'debootstrap'
+ end
+
+ # Generate a config.yml if it does not exist or is empty
+ if not File.size? 'config.yml'
+ print "Generating config.yml.\n"
+ print "Arvados needs to know the email address of the administrative user,\n"
+ print "so that when that user logs in they are automatically made an admin.\n"
+ print "This should be the email address you use to log in to Google.\n"
+ print "\n"
+ admin_email_address = ""
+ until is_valid_email? admin_email_address
+ print "Enter your Google ID email address here: "
+ admin_email_address = gets.strip
+ if not is_valid_email? admin_email_address
+ print "That doesn't look like a valid email address. Please try again.\n"
+ end
+ end
+
+ File.open 'config.yml', 'w' do |config_out|
+ config = YAML.load_file 'config.yml.example'
+ config['API_AUTO_ADMIN_USER'] = admin_email_address
+ config['API_HOSTNAME'] = generate_api_hostname
+ config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
+ config.each_key do |var|
+ if var.end_with?('_PW') or var.end_with?('_SECRET')
+ config[var] = rand(2**256).to_s(36)
+ end
+ config_out.write "#{var}: #{config[var]}\n"
+ end
+ end
+ end
+
+ # If all prerequisites are met, go ahead and build.
+ if ip_forwarding_enabled? and
+ docker_ok? and
+ debootstrap_ok? and
+ File.exists? 'config.yml'
+ warn "Building Arvados."
+ system '/usr/bin/make', '-f', options[:makefile], *ARGV
+ end
+end
+
+# sudo
+# Execute the arg list 'cmd' under sudo.
+# cmd can be passed either as a series of arguments or as a
+# single argument consisting of a list, e.g.:
+# sudo 'apt-get', 'update'
+# sudo(['/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'])
+# sudo %w(/usr/bin/apt-get install lxc-docker)
+#
+def sudo(*cmd)
+ # user can pass a single list in as an argument
+ # to allow usage like: sudo %w(apt-get install foo)
+ warn "You may need to enter your password here."
+ if cmd.length == 1 and cmd[0].class == Array
+ cmd = cmd[0]
+ end
+ system '/usr/bin/sudo', *cmd
+end
+
+# is_valid_email?
+# Returns true if its arg looks like a valid email address.
+# This is a very very loose sanity check.
+#
+def is_valid_email? str
+ str.match /^\S+@\S+\.\S+$/
+end
+
+# generate_api_hostname
+# Generates a 5-character randomly chosen API hostname.
+#
+def generate_api_hostname
+ rand(2**256).to_s(36)[0...5]
+end
+
+# ip_forwarding_enabled?
+# Returns 'true' if IP forwarding is enabled in the kernel
+#
+def ip_forwarding_enabled?
+ %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n"
+end
+
+# debootstrap_ok?
+# Returns 'true' if debootstrap is installed and working.
+#
+def debootstrap_ok?
+ return system '/usr/sbin/debootstrap --version > /dev/null 2>&1'
+end
+
+# docker_ok?
+# Returns 'true' if docker can be run as the current user.
+#
+def docker_ok?
+ return system 'docker images > /dev/null 2>&1'
+end
+
+# find_or_create_ssh_key arvados_name
+# Returns the SSH public key appropriate for this Arvados instance,
+# generating one if necessary.
+#
+def find_or_create_ssh_key arvados_name
+ ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
+ unless File.exists? ssh_key_file
+ system 'ssh-keygen',
+ '-f', ssh_key_file,
+ '-C', "arvados@#{arvados_name}",
+ '-P', ''
+ end
+
+ return "#{ssh_key_file}.pub"
+end
+
+# install_docker
+# Determines which Docker package is suitable for this Linux distro
+# and installs it, resolving any dependencies.
+# NOTE: not in use yet.
+
+def install_docker
+ linux_distro = %x(lsb_release --id).split.last
+ linux_release = %x(lsb_release --release).split.last
+ linux_version = linux_distro + " " + linux_release
+ kernel_release = `uname -r`
+
+ case linux_distro
+ when 'Ubuntu'
+ if not linux_release.match '^1[234]\.'
+ warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)."
+ warn "Your system is Ubuntu #{linux_release}."
+ exit
+ end
+ if linux_release.match '^12' and kernel_release.start_with? '3.2'
+ # Ubuntu Precise ships with a 3.2 kernel and must be upgraded.
+ warn "Your kernel #{kernel_release} must be upgraded to run Docker."
+ warn "To do this:"
+ warn " sudo apt-get update"
+ warn " sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
+ warn " sudo reboot"
+ exit
+ else
+ # install AUFS
+ sudo 'apt-get', 'update'
+ sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}"
+ end
+
+ # add Docker repository
+ sudo %w(/usr/bin/apt-key adv
+ --keyserver keyserver.ubuntu.com
+ --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9)
+ source_file = Tempfile.new('arv')
+ source_file.write("deb http://get.docker.io/ubuntu docker main\n")
+ source_file.close
+ sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list'
+ sudo %w(/usr/bin/apt-get update)
+ sudo %w(/usr/bin/apt-get install lxc-docker)
+
+ # Set up for non-root access
+ sudo %w(/usr/sbin/groupadd docker)
+ sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'
+ sudo %w(/usr/sbin/service docker restart)
+ when 'Debian'
+ else
+ warn "Must be running a Debian or Ubuntu release in order to run Docker."
+ exit
+ end
+end
+
+
+if __FILE__ == $PROGRAM_NAME
+ options = { :makefile => File.join(File.dirname(__FILE__), 'Makefile') }
+ OptionParser.new do |opts|
+ opts.on('-m', '--makefile MAKEFILE-PATH',
+ 'Path to the Makefile used to build Arvados Docker images') do |mk|
+ options[:makefile] = mk
+ end
+ end
+
+ main options
+end
# For each *.in file in the docker directories, substitute any
# @@variables@@ found in the file with the appropriate config
# variable. Support up to 10 levels of nesting.
-#
+#
# TODO(twp): add the *.in files directory to the source tree, and
# when expanding them, add them to the "generated" directory with
# the same tree structure as in the original source. Then all
File.delete(stale_file)
end
+File.umask(022)
Dir.glob('*/*.in') do |template_file|
generated_dir = File.join(File.dirname(template_file), 'generated')
Dir.mkdir(generated_dir) unless Dir.exists? generated_dir
output_path = File.join(generated_dir, File.basename(template_file, '.in'))
- output = File.open(output_path, "w")
- File.open(template_file) do |input|
- input.each_line do |line|
+ File.open(output_path, "w") do |output|
+ File.open(template_file) do |input|
+ input.each_line do |line|
- @count = 0
- while @count < 10
- @out = line.gsub!(/@@(.*?)@@/) do |var|
- if config.key?(Regexp.last_match[1])
- config[Regexp.last_match[1]]
- else
- var.gsub!(/@@/, '@_NOT_FOUND_@')
+ # This count is used to short-circuit potential
+ # infinite loops of variable substitution.
+ @count = 0
+ while @count < 10
+ @out = line.gsub!(/@@(.*?)@@/) do |var|
+ if config.key?(Regexp.last_match[1])
+ config[Regexp.last_match[1]]
+ else
+ var.gsub!(/@@/, '@_NOT_FOUND_@')
+ end
end
+ break if @out.nil?
+ @count += 1
end
- break if @out.nil?
- @count += 1
- end
- output.write(line)
+ output.write(line)
+ end
end
end
- output.close
end
# Copy the ssh public key file to base/generated (if a path is given)
generated_dir = File.join('base/generated')
Dir.mkdir(generated_dir) unless Dir.exists? generated_dir
-if config.key?('PUBLIC_KEY_PATH') &&
- ! (config['PUBLIC_KEY_PATH'] == '') &&
- File.readable?(config['PUBLIC_KEY_PATH'])
+if (!config['PUBLIC_KEY_PATH'].nil? and
+ File.readable? config['PUBLIC_KEY_PATH'])
FileUtils.cp(config['PUBLIC_KEY_PATH'],
File.join(generated_dir, 'id_rsa.pub'))
end
ADD generated/doc.tar.gz /usr/src/arvados/
# Build static site
-RUN /bin/sed -ri 's/^baseurl: .*$/baseurl: /' /usr/src/arvados/doc/_config.yml && \
+RUN bundle install --gemfile=/usr/src/arvados/doc/Gemfile && \
+ /bin/sed -ri 's/^baseurl: .*$/baseurl: /' /usr/src/arvados/doc/_config.yml && \
cd /usr/src/arvados/doc && \
LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8" rake
+++ /dev/null
-#! /bin/bash
-
-# Wrapper script for `docker build'.
-# This is a workaround for https://github.com/dotcloud/docker/issues/1875.
-
-tmpfile=$(mktemp)
-trap "rm $tmpfile; exit 1" SIGHUP SIGINT SIGTERM
-
-docker build $* | tee ${tmpfile}
-if $(grep -q 'Error build' ${tmpfile})
-then
- result=1
-else
- result=0
-fi
-
-rm $tmpfile
-exit $result
--- /dev/null
+#! /bin/sh
+
+# Install prerequisites.
+sudo apt-get install curl libcurl3 libcurl3-gnutls libcurl4-openssl-dev python-pip
+
+# Install RVM.
+curl -sSL https://get.rvm.io | bash -s stable
+source ~/.rvm/scripts/rvm
+rvm install 2.1.0
+
+# Install arvados-cli.
+gem install arvados-cli
+sudo pip install --upgrade httplib2
# Update Arvados source
RUN /bin/mkdir -p /usr/src/arvados/apps
ADD generated/workbench.tar.gz /usr/src/arvados/apps/
+ADD generated/secret_token.rb /usr/src/arvados/apps/workbench/config/initializers/secret_token.rb
+ADD generated/production.rb /usr/src/arvados/apps/workbench/config/environments/production.rb
+ADD passenger.conf /etc/apache2/conf.d/passenger
+
-RUN touch /usr/src/arvados/apps/workbench/log/production.log && \
+RUN bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
+ touch /usr/src/arvados/apps/workbench/log/production.log && \
chmod 666 /usr/src/arvados/apps/workbench/log/production.log && \
touch /usr/src/arvados/apps/workbench/db/production.sqlite3 && \
bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
cd /usr/src/arvados/apps/workbench && \
- rake assets:precompile
+ rake assets:precompile && \
+ chown -R www-data:www-data /usr/src/arvados/apps/workbench
# Configure Apache
ADD generated/apache2_vhost /etc/apache2/sites-available/workbench
a2ensite workbench && \
a2enmod rewrite
-# Set up the production environment
-ADD generated/secret_token.rb /usr/src/arvados/apps/workbench/config/initializers/secret_token.rb
-ADD generated/production.rb /usr/src/arvados/apps/workbench/config/environments/production.rb
-ADD passenger.conf /etc/apache2/conf.d/passenger
-
ADD apache2_foreground.sh /etc/apache2/foreground.sh
# Start Apache
s.executables << "arv-run-pipeline-instance"
s.executables << "arv-crunch-job"
s.executables << "arv-tag"
+ s.add_runtime_dependency 'arvados', '~> 0.1.0'
s.add_runtime_dependency 'google-api-client', '~> 0.6.3'
s.add_runtime_dependency 'activesupport', '~> 3.2', '>= 3.2.13'
s.add_runtime_dependency 'json', '~> 1.7', '>= 1.7.7'
rescue LoadError
abort <<-EOS
-Please install all required gems:
+Please install all required gems:
gem install activesupport andand curb google-api-client json oj trollop
end
end
-client = ArvadosClient.new(:host => ENV['ARVADOS_API_HOST'], :application_name => 'arvados-cli', :application_version => '1.0')
-arvados = client.discovered_api('arvados', ENV['ARVADOS_API_VERSION'])
+begin
+ client = ArvadosClient.new(:host => ENV['ARVADOS_API_HOST'], :application_name => 'arvados-cli', :application_version => '1.0')
+ arvados = client.discovered_api('arvados', ENV['ARVADOS_API_VERSION'])
+rescue Exception => e
+ puts "Failed to connect to Arvados API server: #{e}"
+ exit 1
+end
def to_boolean(s)
!!(s =~ /^(true|t|yes|y|1)$/i)
end
banner += "\n"
STDERR.puts banner
-
- if not method.nil? and method != '--help' then
+
+ if not method.nil? and method != '--help' then
Trollop::die ("Unknown method #{method.inspect} " +
"for resource #{resource.inspect}")
end
banner += "\n"
STDERR.puts banner
- if not resource.nil? and resource != '--help' then
+ if not resource.nil? and resource != '--help' then
Trollop::die "Unknown resource type #{resource.inspect}"
end
exit 255
curl.headers['Accept'] = 'text/plain'
curl.headers['Authorization'] = "OAuth2 #{ENV['ARVADOS_API_TOKEN']}"
if ENV['ARVADOS_API_HOST_INSECURE']
- curl.ssl_verify_peer = false
+ curl.ssl_verify_peer = false
curl.ssl_verify_host = false
end
if global_opts[:verbose]
puts results['uuid']
end
end
-
-
# [--no-wait] Make only as much progress as possible without entering
# a sleep/poll loop.
#
-# [--no-reuse-finished] Do not reuse existing outputs to satisfy
-# pipeline components. Always submit a new job
-# or use an existing job which has not yet
-# finished.
-#
# [--no-reuse] Do not reuse existing jobs to satisfy pipeline
# components. Submit a new job for every component.
#
abort "#{$0}: fatal: ARVADOS_API_TOKEN environment variable not set."
begin
+ require 'arvados'
require 'rubygems'
require 'json'
require 'pp'
abort <<-EOS
#{$0}: fatal: #{l.message}
Some runtime dependencies may be missing.
-Try: gem install pp google-api-client json trollop
+Try: gem install arvados pp google-api-client json trollop
EOS
end
"Do not wait for jobs to finish. Just look up status, submit new jobs if needed, and exit.",
:short => :none,
:type => :boolean)
- opt(:no_reuse_finished,
- "Do not reuse existing outputs to satisfy pipeline components. Always submit a new job or use an existing job which has not yet finished.",
- :short => :none,
- :type => :boolean)
opt(:no_reuse,
"Do not reuse existing jobs to satisfy pipeline components. Submit a new job for every component.",
:short => :none,
:application_name => File.split($0).last,
:application_version => $application_version.to_s)
$arvados = $client.discovered_api('arvados', $arvados_api_version)
+$arv = Arvados.new api_version: 'v1'
class PipelineInstance
if j.is_a? Hash and j[:uuid]
@cache[j[:uuid]] = j
else
- debuglog "create job: #{j[:errors] rescue nil}", 0
+ debuglog "create job: #{j[:errors] rescue nil} with attribute #{attributes}", 0
nil
end
end
moretodo = false
@components.each do |cname, c|
job = nil
+ # Is the job satisfying this component already known to be
+ # finished? (Already meaning "before we query API server about
+ # the job's current state")
+ c_already_finished = (c[:job] &&
+ c[:job][:uuid] &&
+ !c[:job][:success].nil?)
if !c[:job] and
- c[:script_parameters].select { |pname, p| p.is_a? Hash }.empty?
- # Job is fully specified (all parameter values are present) but
- # no particular job has been found.
-
- debuglog "component #{cname} ready to satisfy."
-
- c.delete :wait
- second_place_job = nil # satisfies component, but not finished yet
-
- (@options[:no_reuse] ? [] : JobCache.
- where(script: c[:script],
- script_parameters: c[:script_parameters],
- script_version_descends_from: c[:script_version])
- ).each do |candidate_job|
- candidate_params_downcase = Hash[candidate_job[:script_parameters].
- map { |k,v| [k.downcase,v] }]
- c_params_downcase = Hash[c[:script_parameters].
- map { |k,v| [k.downcase,v] }]
-
- debuglog "component #{cname} considering job #{candidate_job[:uuid]} version #{candidate_job[:script_version]} parameters #{candidate_params_downcase.inspect}", 3
-
- unless candidate_params_downcase == c_params_downcase
- next
- end
-
- if c[:script_version] !=
- candidate_job[:script_version][0,c[:script_version].length]
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
- next
- end
-
- unless candidate_job[:success] || candidate_job[:running] ||
- (!candidate_job[:started_at] && !candidate_job[:cancelled_at])
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2
- next
- end
-
- if candidate_job[:success]
- unless @options[:no_reuse_finished]
- job = candidate_job
- $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}"
- c[:job] = job
- end
- else
- second_place_job ||= candidate_job
- end
- break
- end
- if not c[:job] and second_place_job
- job = second_place_job
- $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
+ c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
+ # No job yet associated with this component and is component inputs
+ # are fully specified (any output_of script_parameters are resolved
+ # to real value)
+ job = JobCache.create({:script => c[:script],
+ :script_parameters => c[:script_parameters],
+ :script_version => c[:script_version],
+ :repository => c[:repository],
+ :minimum_script_version => c[:minimum_script_version],
+ :exclude_script_versions => c[:exclude_minimum_script_versions],
+ :nondeterministic => c[:nondeterministic],
+ :no_reuse => @options[:no_reuse],
+ :output_is_persistent => c[:output_is_persistent] || false})
+ if job
+ debuglog "component #{cname} new job #{job[:uuid]}"
c[:job] = job
+ else
+ debuglog "component #{cname} new job failed"
end
- if not c[:job]
- debuglog "component #{cname} not satisfied by any existing job."
- if !@options[:dry_run]
- debuglog "component #{cname} new job."
- job = JobCache.create(:script => c[:script],
- :script_parameters => c[:script_parameters],
- :runtime_constraints => c[:runtime_constraints] || {},
- :script_version => c[:script_version] || 'master')
- if job
- debuglog "component #{cname} new job #{job[:uuid]}"
- c[:job] = job
- else
- debuglog "component #{cname} new job failed"
- end
- end
- end
- else
- c[:wait] = true
end
+
if c[:job] and c[:job][:uuid]
if (c[:job][:running] or
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
+ # Job is running so update copy of job record
c[:job] = JobCache.get(c[:job][:uuid])
end
+
if c[:job][:success]
# Populate script_parameters of other components waiting for
# this job
end
end
end
+ unless c_already_finished
+ # This is my first time discovering that the job
+ # succeeded. (At the top of this loop, I was still
+ # waiting for it to finish.)
+ if c[:output_is_persistent]
+ # I need to make sure a resources/wants link is in
+ # place to protect the output from garbage
+ # collection. (Normally Crunch does this for me, but
+ # here I might be reusing the output of someone else's
+ # job and I need to make sure it's understood that the
+ # output is valuable to me, too.)
+ wanted = c[:job][:output]
+ debuglog "checking for existing persistence link for #{wanted}"
+ @my_user_uuid ||= $arv.user.current[:uuid]
+ links = $arv.link.list(limit: 1,
+ filters:
+ [%w(link_class = resources),
+ %w(name = wants),
+ %w(tail_uuid =) + [@my_user_uuid],
+ %w(head_uuid =) + [wanted]
+ ])[:items]
+ if links.any?
+ debuglog "link already exists, uuid #{links.first[:uuid]}"
+ else
+ newlink = $arv.link.create link: \
+ {
+ link_class: 'resources',
+ name: 'wants',
+ tail_kind: 'arvados#user',
+ tail_uuid: @my_user_uuid,
+ head_kind: 'arvados#collection',
+ head_uuid: wanted
+ }
+ debuglog "added link, uuid #{newlink[:uuid]}"
+ end
+ end
+ end
elsif c[:job][:running] ||
(!c[:job][:started_at] && !c[:job][:cancelled_at])
+ # Job is still running
moretodo = true
elsif c[:job][:cancelled_at]
debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."
end
end
end
-
+
if ended == @components.length or failed > 0
@instance[:active] = false
@instance[:success] = (succeeded == @components.length)
#!/usr/bin/perl
-# -*- mode: perl; perl-indent-level: 2; -*-
+# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
=head1 NAME
use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
use Arvados;
use Getopt::Long;
-use Warehouse;
-use Warehouse::Stream;
-use IPC::System::Simple qw(capturex);
+use IPC::Open2;
+use IO::Select;
+use File::Temp;
use Fcntl ':flock';
$ENV{"TMPDIR"} ||= "/tmp";
}
$job_id = $Job->{'uuid'};
-$metastream = Warehouse::Stream->new(whc => new Warehouse);
-$metastream->clear;
-$metastream->name('.');
-$metastream->write_start($job_id . '.log.txt');
-
+my $keep_logfile = $job_id . '.log.txt';
+my $local_logfile = File::Temp->new();
$Job->{'runtime_constraints'} ||= {};
$Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0;
or croak ("git clone $repo failed: exit ".($?>>8));
system("cd $ENV{CRUNCH_SRC} && git config clean.requireForce false");
}
- `cd $ENV{CRUNCH_SRC} && git remote set-url origin \"\$CRUNCH_SRC_URL\" && git fetch -q origin`;
+ `cd $ENV{CRUNCH_SRC} && git remote set-url origin \"\$CRUNCH_SRC_URL\" && git fetch -q --tags origin`;
# If this looks like a subversion r#, look for it in git-svn commit messages
}
$ENV{"TASK_SLOT_NODE"} = $slot[$childslot]->{node}->{name};
$ENV{"TASK_SLOT_NUMBER"} = $slot[$childslot]->{cpu};
- $ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/".$slot[$childslot]->{cpu};
- $ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}."/keep";
+ $ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/$id.$$";
+ $ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}.".keep";
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
$ENV{"CRUNCH_NODE_SLOTS"} = $slot[$childslot]->{node}->{ncpus};
$ENV{"PATH"} = $ENV{"CRUNCH_INSTALL"} . "/bin:" . $ENV{"PATH"};
if ($Job->{'output'})
{
eval {
- my $manifest_text = capturex("whget", $Job->{'output'});
+ my $manifest_text = `arv keep get \Q$Job->{'output'}\E`;
$arv->{'collections'}->{'create'}->execute('collection' => {
'uuid' => $Job->{'output'},
'manifest_text' => $manifest_text,
});
+ if ($Job->{'output_is_persistent'}) {
+ $arv->{'links'}->{'create'}->execute('link' => {
+ 'tail_kind' => 'arvados#user',
+ 'tail_uuid' => $User->{'uuid'},
+ 'head_kind' => 'arvados#collection',
+ 'head_uuid' => $Job->{'output'},
+ 'link_class' => 'resources',
+ 'name' => 'wants',
+ });
+ }
};
if ($@) {
Log (undef, "Failed to register output manifest: $@");
} split ("\n", $jobstep[$job]->{stderr});
}
+sub fetch_block
+{
+ my $hash = shift;
+ my ($keep, $child_out, $output_block);
+
+ my $cmd = "arv keep get \Q$hash\E";
+ open($keep, '-|', $cmd) or die "fetch_block: $cmd: $!";
+ sysread($keep, $output_block, 64 * 1024 * 1024);
+ close $keep;
+ return $output_block;
+}
sub collate_output
{
- my $whc = Warehouse->new;
Log (undef, "collate");
- $whc->write_start (1);
+
+ my ($child_out, $child_in);
+ my $pid = open2($child_out, $child_in, 'arv', 'keep', 'put', '--raw');
my $joboutput;
for (@jobstep)
{
if ($output !~ /^[0-9a-f]{32}(\+\S+)*$/)
{
$output_in_keep ||= $output =~ / [0-9a-f]{32}\S*\+K/;
- $whc->write_data ($output);
+ print $child_in $output;
}
elsif (@jobstep == 1)
{
$joboutput = $output;
- $whc->write_finish;
+ last;
}
- elsif (defined (my $outblock = $whc->fetch_block ($output)))
+ elsif (defined (my $outblock = fetch_block ($output)))
{
$output_in_keep ||= $outblock =~ / [0-9a-f]{32}\S*\+K/;
- $whc->write_data ($outblock);
+ print $child_in $outblock;
}
else
{
- my $errstr = $whc->errstr;
- $whc->write_data ("XXX fetch_block($output) failed: $errstr XXX\n");
+ Log (undef, "XXX fetch_block($output) failed XXX");
$main::success = 0;
}
}
- $joboutput = $whc->write_finish if !defined $joboutput;
+ $child_in->close;
+
+ if (!defined $joboutput) {
+ my $s = IO::Select->new($child_out);
+ if ($s->can_read(120)) {
+ sysread($child_out, $joboutput, 64 * 1024 * 1024);
+ chomp($joboutput);
+ } else {
+ Log (undef, "timed out reading from 'arv keep put'");
+ }
+ }
+ waitpid($pid, 0);
+
if ($joboutput)
{
Log (undef, "output $joboutput");
}
print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message);
- return if !$metastream;
- $metastream->write_data ($datetime . " " . $message);
+ if ($metastream) {
+ print $metastream $datetime . " " . $message;
+ }
}
sub save_meta
{
my $justcheckpoint = shift; # false if this will be the last meta saved
- my $m = $metastream;
- $m = $m->copy if $justcheckpoint;
- $m->write_finish;
- my $whc = Warehouse->new;
- my $loglocator = $whc->store_block ($m->as_string);
- $arv->{'collections'}->{'create'}->execute('collection' => {
- 'uuid' => $loglocator,
- 'manifest_text' => $m->as_string,
- });
- undef $metastream if !$justcheckpoint; # otherwise Log() will try to use it
+ return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
+
+ $local_logfile->flush;
+ my $cmd = "arv keep put --filename \Q$keep_logfile\E "
+ . quotemeta($local_logfile->filename);
+ my $loglocator = `$cmd`;
+ die "system $cmd failed: $?" if $?;
+
+ $local_logfile = undef; # the temp file is automatically deleted
Log (undef, "log manifest is $loglocator");
$Job->{'log'} = $loglocator;
$Job->update_attributes('log', $loglocator) if $job_has_uuid;
sub thaw
{
croak ("Thaw not implemented");
-
- my $whc;
- my $key = shift;
- Log (undef, "thaw from $key");
-
- @jobstep = ();
- @jobstep_done = ();
- @jobstep_todo = ();
- @jobstep_tomerge = ();
- $jobstep_tomerge_level = 0;
- my $frozenjob = {};
-
- my $stream = new Warehouse::Stream ( whc => $whc,
- hash => [split (",", $key)] );
- $stream->rewind;
- while (my $dataref = $stream->read_until (undef, "\n\n"))
- {
- if ($$dataref =~ /^job /)
- {
- foreach (split ("\n", $$dataref))
- {
- my ($k, $v) = split ("=", $_, 2);
- $frozenjob->{$k} = freezeunquote ($v);
- }
- next;
- }
-
- if ($$dataref =~ /^merge (\d+) (.*)/)
- {
- $jobstep_tomerge_level = $1;
- @jobstep_tomerge
- = map { freezeunquote ($_) } split ("\n", freezeunquote($2));
- next;
- }
-
- my $Jobstep = { };
- foreach (split ("\n", $$dataref))
- {
- my ($k, $v) = split ("=", $_, 2);
- $Jobstep->{$k} = freezeunquote ($v) if $k;
- }
- $Jobstep->{'failures'} = 0;
- push @jobstep, $Jobstep;
-
- if ($Jobstep->{exitcode} eq "0")
- {
- push @jobstep_done, $#jobstep;
- }
- else
- {
- push @jobstep_todo, $#jobstep;
- }
- }
-
- foreach (qw (script script_version script_parameters))
- {
- $Job->{$_} = $frozenjob->{$_};
- }
- $Job->save if $job_has_uuid;
}
--- /dev/null
+#! /usr/bin/perl
+
+use strict;
+
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+ NAME => 'Arvados',
+ VERSION_FROM => 'lib/Arvados.pm'
+);
gem 'google-api-client', '~> 0.6.3'
gem 'trollop'
-gem 'arvados-cli', '>= 0.1.20140311162926'
+gem 'arvados-cli', '>= 0.1.20140328152103'
addressable (2.3.5)
andand (1.3.3)
arel (3.0.2)
- arvados-cli (0.1.20140311162926)
+ arvados (0.1.20140328152103)
+ activesupport (>= 3.2.13)
+ andand
+ google-api-client (~> 0.6.3)
+ json (>= 1.7.7)
+ arvados-cli (0.1.20140328152103)
activesupport (~> 3.2, >= 3.2.13)
andand (~> 1.3, >= 1.3.3)
+ arvados (~> 0.1.0)
curb (~> 0.8)
google-api-client (~> 0.6.3)
json (~> 1.7, >= 1.7.7)
DEPENDENCIES
acts_as_api
andand
- arvados-cli (>= 0.1.20140311162926)
+ arvados-cli (>= 0.1.20140328152103)
coffee-rails (~> 3.2.0)
google-api-client (~> 0.6.3)
jquery-rails
def load_filters_param
if params[:filters].is_a? Array
@filters = params[:filters]
- elsif params[:filters].is_a? String
+ elsif params[:filters].is_a? String and !params[:filters].empty?
begin
@filters = Oj.load params[:filters]
raise unless @filters.is_a? Array
cond_out = []
param_out = []
@filters.each do |attr, operator, operand|
- if !model_class.searchable_columns.index attr.to_s
+ if !model_class.searchable_columns(operator).index attr.to_s
raise ArgumentError.new("Invalid attribute '#{attr}' in condition")
end
case operator.downcase
if @where.is_a? Hash and @where.any?
conditions = ['1=1']
@where.each do |attr,value|
- if attr == :any
+ if attr.to_s == 'any'
if value.is_a?(Array) and
value.length == 2 and
- value[0] == 'contains' and
- model_class.columns.collect(&:name).index('name') then
+ value[0] == 'contains' then
ilikes = []
- model_class.searchable_columns.each do |column|
+ model_class.searchable_columns('ilike').each do |column|
ilikes << "#{table_name}.#{column} ilike ?"
conditions << "%#{value[1]}%"
end
:items => @objects.as_api_response(nil)
}
if @objects.respond_to? :except
- @object_list[:items_available] = @objects.except(:limit).except(:offset).count
+ @object_list[:items_available] = @objects.
+ except(:limit).except(:offset).
+ count(:id, distinct: true)
end
render json: @object_list
end
skip_before_filter :find_object_by_uuid, :only => :queue
skip_before_filter :render_404_if_no_object, :only => :queue
- def index
- return super unless @where.is_a? Hash
- want_ancestor = @where[:script_version_descends_from]
- if want_ancestor
- # Check for missing commit_ancestor rows, and create them if
- # possible.
- @objects.
- dup.
- includes(:commit_ancestors). # I wish Rails would let me
- # specify here which
- # commit_ancestors I am
- # interested in.
- each do |o|
- if o.commit_ancestors.
- select { |ca| ca.ancestor == want_ancestor }.
- empty? and !o.script_version.nil?
- begin
- o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
- rescue
+ def create
+ [:repository, :script, :script_version, :script_parameters].each do |r|
+ if !resource_attrs[r]
+ return render json: {
+ :error => "#{r} attribute must be specified"
+ }, status: :unprocessable_entity
+ end
+ end
+
+ r = Commit.find_commit_range(current_user,
+ resource_attrs[:repository],
+ resource_attrs[:minimum_script_version],
+ resource_attrs[:script_version],
+ resource_attrs[:exclude_script_versions])
+ if !resource_attrs[:nondeterministic] and !resource_attrs[:no_reuse]
+ # Search for jobs where the script_version is in the list of commits
+ # returned by find_commit_range
+ @object = nil
+ Job.readable_by(current_user).where(script: resource_attrs[:script],
+ script_version: r).
+ each do |j|
+ if j.nondeterministic != true and
+ j.success != false and
+ j.script_parameters == resource_attrs[:script_parameters]
+ # Record the first job in the list
+ if !@object
+ @object = j
+ end
+ # Ensure that all candidate jobs actually did produce the same output
+ if @object.output != j.output
+ @object = nil
+ break
end
end
- o.commit_ancestors.
- select { |ca| ca.ancestor == want_ancestor }.
- select(&:is).
- first
+ if @object
+ return show
+ end
end
- # Now it is safe to do an .includes().where() because we are no
- # longer interested in jobs that have other ancestors but not
- # want_ancestor.
- @objects = @objects.
- includes(:commit_ancestors).
- where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
- want_ancestor, true)
end
+ if r
+ resource_attrs[:script_version] = r[0]
+ end
+
+ # Don't pass these on to activerecord
+ resource_attrs.delete(:minimum_script_version)
+ resource_attrs.delete(:exclude_script_versions)
+ resource_attrs.delete(:no_reuse)
super
end
class Arvados::V1::SchemaController < ApplicationController
+ skip_before_filter :find_objects_for_index
skip_before_filter :find_object_by_uuid
skip_before_filter :render_404_if_no_object
skip_before_filter :require_auth_scope_all
- def show
- classes = Rails.cache.fetch 'arvados_v1_schema' do
- Rails.application.eager_load!
- classes = {}
- ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
- classes[k] = k.columns.collect do |col|
- if k.serialized_attributes.has_key? col.name
- { name: col.name,
- type: k.serialized_attributes[col.name].object_class.to_s }
- else
- { name: col.name,
- type: col.type }
- end
- end
- end
- classes
- end
- render json: classes
- end
-
- def discovery_rest_description
+ def index
expires_in 24.hours, public: true
discovery = Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
generatedAt: Time.now.iso8601,
title: "Arvados API",
description: "The API to interact with Arvados.",
- documentationLink: "https://redmine.clinicalfuture.com/projects/arvados/",
+ documentationLink: "http://doc.arvados.org/api/index.html",
protocol: "rest",
baseUrl: root_url + "/arvados/v1/",
basePath: "/arvados/v1/",
class Arvados::V1::UsersController < ApplicationController
skip_before_filter :find_object_by_uuid, only:
- [:activate, :event_stream, :current, :system]
+ [:activate, :event_stream, :current, :system, :setup]
skip_before_filter :render_404_if_no_object, only:
- [:activate, :event_stream, :current, :system]
+ [:activate, :event_stream, :current, :system, :setup]
+ before_filter :admin_required, only: [:setup, :unsetup]
def current
@object = current_user
end
end
end
-
+
def event_stream
channel = current_user.andand.uuid
if current_user.andand.is_admin
end
show
end
+
+ # create user object and all the needed links
+ def setup
+ @object = nil
+ if params[:uuid]
+ @object = User.find_by_uuid params[:uuid]
+ if !@object
+ return render_404_if_no_object
+ end
+ object_found = true
+ else
+ if !params[:user]
+ raise ArgumentError.new "Required uuid or user"
+ else
+ if params[:user]['uuid']
+ @object = User.find_by_uuid params[:user]['uuid']
+ if @object
+ object_found = true
+ end
+ end
+
+ if !@object
+ if !params[:user]['email']
+ raise ArgumentError.new "Require user email"
+ end
+
+ if !params[:openid_prefix]
+ raise ArgumentError.new "Required openid_prefix parameter is missing."
+ end
+
+ @object = model_class.create! resource_attrs
+ end
+ end
+ end
+
+ if object_found
+ @response = @object.setup_repo_vm_links params[:repo_name],
+ params[:vm_uuid], params[:openid_prefix]
+ else
+ @response = User.setup @object, params[:openid_prefix],
+ params[:repo_name], params[:vm_uuid]
+ end
+
+ render json: { kind: "arvados#HashList", items: @response }
+ end
+
+ # delete user agreements, vm, repository, login links; set state to inactive
+ def unsetup
+ reload_object_before_update
+ @object.unsetup
+ show
+ end
+
end
"#{current_api_base}/#{self.class.to_s.pluralize.underscore}/#{self.uuid}"
end
- def self.searchable_columns
+ def self.searchable_columns operator
+ textonly_operator = !operator.match(/[<=>]/)
self.columns.collect do |col|
- if [:string, :text, :datetime, :integer].index(col.type) && col.name != 'owner_uuid'
+ if col.name == 'owner_uuid'
+ nil
+ elsif [:string, :text].index(col.type)
+ col.name
+ elsif !textonly_operator and [:datetime, :integer].index(col.type)
col.name
end
end.compact
class Commit < ActiveRecord::Base
require 'shellwords'
- # Make sure the specified commit really exists, and return the full
- # sha1 commit hash.
- #
- # Accepts anything "git rev-list" accepts, optionally (and
- # preferably) preceded by "repo_name:".
- #
- # Examples: "1234567", "master", "apps:1234567", "apps:master",
- # "apps:HEAD"
-
- def self.find_by_commit_ish(commit_ish)
- want_repo = nil
- if commit_ish.index(':')
- want_repo, commit_ish = commit_ish.split(':',2)
+ def self.git_check_ref_format(e)
+ if !e or e.empty? or e[0] == '-' or e[0] == '$'
+ # definitely not valid
+ false
+ else
+ `git check-ref-format --allow-onelevel #{e.shellescape}`
+ $?.success?
end
- repositories.each do |repo_name, repo|
- next if want_repo and want_repo != repo_name
- ENV['GIT_DIR'] = repo[:git_dir]
- IO.foreach("|git rev-list --max-count=1 --format=oneline 'origin/'#{commit_ish.shellescape} 2>/dev/null || git rev-list --max-count=1 --format=oneline ''#{commit_ish.shellescape}") do |line|
- sha1, message = line.strip.split " ", 2
- next if sha1.length != 40
- begin
- Commit.find_or_create_by_repository_name_and_sha1_and_message(repo_name, sha1, message[0..254])
- rescue
- logger.warn "find_or_create failed: repo_name #{repo_name} sha1 #{sha1} message #{message[0..254]}"
- # Ignore cache failure. Commit is real. We should proceed.
+ end
+
+ def self.find_commit_range(current_user, repository, minimum, maximum, exclude)
+ if (minimum and !git_check_ref_format(minimum)) or !git_check_ref_format(maximum)
+ logger.warn "find_commit_range called with invalid minimum or maximum: '#{minimum}', '#{maximum}'"
+ return nil
+ end
+
+ if minimum and minimum.empty?
+ minimum = nil
+ end
+
+ if !maximum
+ maximum = "HEAD"
+ end
+
+ # Get list of actual repository directories under management
+ on_disk_repos = repositories
+
+ # Get list of repository objects readable by user
+ readable = Repository.readable_by(current_user)
+
+ # filter repository objects on requested repository name
+ if repository
+ readable = readable.where(name: repository)
+ end
+
+ commits = []
+ readable.each do |r|
+ if on_disk_repos[r.name]
+ ENV['GIT_DIR'] = on_disk_repos[r.name][:git_dir]
+
+ # We've filtered for invalid characters, so we can pass the contents of
+ # minimum and maximum safely on the command line
+
+ # Get the commit hash for the upper bound
+ max_hash = nil
+ IO.foreach("|git rev-list --max-count=1 #{maximum.shellescape}") do |line|
+ max_hash = line.strip
+ end
+
+ # If not found or string is invalid, nothing else to do
+ next if !max_hash or !git_check_ref_format(max_hash)
+
+ resolved_exclude = nil
+ if exclude
+ resolved_exclude = []
+ exclude.each do |e|
+ if git_check_ref_format(e)
+ IO.foreach("|git rev-list --max-count=1 #{e.shellescape}") do |line|
+ resolved_exclude.push(line.strip)
+ end
+ else
+ logger.warn "find_commit_range called with invalid exclude invalid characters: '#{exclude}'"
+ return nil
+ end
+ end
+ end
+
+ if minimum
+ # Get the commit hash for the lower bound
+ min_hash = nil
+ IO.foreach("|git rev-list --max-count=1 #{minimum.shellescape}") do |line|
+ min_hash = line.strip
+ end
+
+ # If not found or string is invalid, nothing else to do
+ next if !min_hash or !git_check_ref_format(min_hash)
+
+ # Now find all commits between them
+ IO.foreach("|git rev-list #{min_hash.shellescape}..#{max_hash.shellescape}") do |line|
+ hash = line.strip
+ commits.push(hash) if !resolved_exclude or !resolved_exclude.include? hash
+ end
+
+ commits.push(min_hash) if !resolved_exclude or !resolved_exclude.include? min_hash
+ else
+ commits.push(max_hash) if !resolved_exclude or !resolved_exclude.include? max_hash
end
- return sha1
end
end
- nil
+
+ if !commits or commits.empty?
+ nil
+ else
+ commits
+ end
end
# Import all commits from configured git directory into the commits
end
end
+ def self.refresh_repositories
+ @repositories = nil
+ end
+
protected
def self.repositories
next if repo.match /^\./
git_dir = File.join(@gitdirbase,
repo.match(/\.git$/) ? repo : File.join(repo, '.git'))
+ next if git_dir == Rails.configuration.git_internal_dir
repo_name = repo.sub(/\.git$/, '')
@repositories[repo_name] = {git_dir: git_dir}
end
t.add :started_at
t.add :finished_at
t.add :output
+ t.add :output_is_persistent
t.add :success
t.add :running
t.add :is_locked_by_uuid
t.add :dependencies
t.add :log_stream_href
t.add :log_buffer
+ t.add :nondeterministic
+ t.add :repository
end
def assert_finished
order('priority desc, created_at')
end
+ def self.running
+ self.where('running = ?', true).
+ order('priority desc, created_at')
+ end
+
protected
def foreign_key_attributes
return true
end
if new_record? or script_version_changed?
- sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
+ sha1 = Commit.find_commit_range(current_user, nil, nil, self.script_version, nil)[0] rescue nil
if sha1
self.script_version = sha1
else
before_update :prevent_privilege_escalation
before_update :prevent_inactive_admin
before_create :check_auto_admin
+ after_create :add_system_group_permission_link
after_create AdminNotifier
has_many :authorized_keys, :foreign_key => :authorized_user_uuid, :primary_key => :uuid
Group.where('owner_uuid in (?)', lookup_uuids).each do |group|
newgroups << [group.owner_uuid, group.uuid, 'can_manage']
end
- Link.where('tail_uuid in (?) and link_class = ? and head_uuid like ?',
+ Link.where('tail_uuid in (?) and link_class = ? and (head_uuid like ? or head_uuid like ?)',
lookup_uuids,
'permission',
- Group.uuid_like_pattern).each do |link|
+ Group.uuid_like_pattern,
+ User.uuid_like_pattern).each do |link|
newgroups << [link.tail_uuid, link.head_uuid, link.name]
end
newgroups.each do |tail_uuid, head_uuid, perm_name|
end
end
+ def self.setup(user, openid_prefix, repo_name=nil, vm_uuid=nil)
+ return user.setup_repo_vm_links(repo_name, vm_uuid, openid_prefix)
+ end
+
+ # create links
+ def setup_repo_vm_links(repo_name, vm_uuid, openid_prefix)
+ oid_login_perm = create_oid_login_perm openid_prefix
+ repo_perm = create_user_repo_link repo_name
+ vm_login_perm = create_vm_login_permission_link vm_uuid, repo_name
+ group_perm = create_user_group_link
+
+ return [oid_login_perm, repo_perm, vm_login_perm, group_perm, self].compact
+ end
+
+ # delete user signatures, login, repo, and vm perms, and mark as inactive
+ def unsetup
+ # delete oid_login_perms for this user
+ oid_login_perms = Link.where(tail_uuid: self.email,
+ head_kind: 'arvados#user',
+ link_class: 'permission',
+ name: 'can_login')
+ oid_login_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete repo_perms for this user
+ repo_perms = Link.where(tail_uuid: self.uuid,
+ head_kind: 'arvados#repository',
+ link_class: 'permission',
+ name: 'can_write')
+ repo_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete vm_login_perms for this user
+ vm_login_perms = Link.where(tail_uuid: self.uuid,
+ head_kind: 'arvados#virtualMachine',
+ link_class: 'permission',
+ name: 'can_login')
+ vm_login_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete "All users' group read permissions for this user
+ group = Group.where(name: 'All users').select do |g|
+ g[:uuid].match /-f+$/
+ end.first
+ group_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: group[:uuid],
+ head_kind: 'arvados#group',
+ link_class: 'permission',
+ name: 'can_read')
+ group_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete any signatures by this user
+ signed_uuids = Link.where(link_class: 'signature',
+ tail_kind: 'arvados#user',
+ tail_uuid: self.uuid)
+ signed_uuids.each do |sign|
+ Link.delete sign
+ end
+
+ # mark the user as inactive
+ self.is_active = false
+ self.save!
+ end
+
protected
def permission_to_update
upstream_path.delete start
merged
end
+
+ def create_oid_login_perm (openid_prefix)
+ login_perm_props = {identity_url_prefix: openid_prefix}
+
+ # Check oid_login_perm
+ oid_login_perms = Link.where(tail_uuid: self.email,
+ head_kind: 'arvados#user',
+ link_class: 'permission',
+ name: 'can_login')
+
+ if !oid_login_perms.any?
+ # create openid login permission
+ oid_login_perm = Link.create(link_class: 'permission',
+ name: 'can_login',
+ tail_kind: 'email',
+ tail_uuid: self.email,
+ head_kind: 'arvados#user',
+ head_uuid: self.uuid,
+ properties: login_perm_props
+ )
+ logger.info { "openid login permission: " + oid_login_perm[:uuid] }
+ else
+ oid_login_perm = oid_login_perms.first
+ end
+
+ return oid_login_perm
+ end
+
+ def create_user_repo_link(repo_name)
+ # repo_name is optional
+ if not repo_name
+ logger.warn ("Repository name not given for #{self.uuid}.")
+ return
+ end
+
+ # Check for an existing repository with the same name we're about to use.
+ repo = Repository.where(name: repo_name).first
+
+ if repo
+ logger.warn "Repository exists for #{repo_name}: #{repo[:uuid]}."
+
+ # Look for existing repository access for this repo
+ repo_perms = Link.where(tail_uuid: self.uuid,
+ head_kind: 'arvados#repository',
+ head_uuid: repo[:uuid],
+ link_class: 'permission',
+ name: 'can_write')
+ if repo_perms.any?
+ logger.warn "User already has repository access " +
+ repo_perms.collect { |p| p[:uuid] }.inspect
+ return repo_perms.first
+ end
+ end
+
+ # create repo, if does not already exist
+ repo ||= Repository.create(name: repo_name)
+ logger.info { "repo uuid: " + repo[:uuid] }
+
+ repo_perm = Link.create(tail_kind: 'arvados#user',
+ tail_uuid: self.uuid,
+ head_kind: 'arvados#repository',
+ head_uuid: repo[:uuid],
+ link_class: 'permission',
+ name: 'can_write')
+ logger.info { "repo permission: " + repo_perm[:uuid] }
+ return repo_perm
+ end
+
+ # create login permission for the given vm_uuid, if it does not already exist
+ def create_vm_login_permission_link(vm_uuid, repo_name)
+ begin
+
+ # vm uuid is optional
+ if vm_uuid
+ vm = VirtualMachine.where(uuid: vm_uuid).first
+
+ if not vm
+ logger.warn "Could not find virtual machine for #{vm_uuid.inspect}"
+ raise "No vm found for #{vm_uuid}"
+ end
+ else
+ return
+ end
+
+ logger.info { "vm uuid: " + vm[:uuid] }
+
+ login_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: vm[:uuid],
+ head_kind: 'arvados#virtualMachine',
+ link_class: 'permission',
+ name: 'can_login')
+
+ perm_exists = false
+ login_perms.each do |perm|
+ if perm.properties[:username] == repo_name
+ perm_exists = true
+ break
+ end
+ end
+
+ if !perm_exists
+ login_perm = Link.create(tail_kind: 'arvados#user',
+ tail_uuid: self.uuid,
+ head_kind: 'arvados#virtualMachine',
+ head_uuid: vm[:uuid],
+ link_class: 'permission',
+ name: 'can_login',
+ properties: {username: repo_name})
+ logger.info { "login permission: " + login_perm[:uuid] }
+ else
+ login_perm = login_perms.first
+ end
+
+ return login_perm
+ end
+ end
+
+ # add the user to the 'All users' group
+ def create_user_group_link
+ # Look up the "All users" group (we expect uuid *-*-fffffffffffffff).
+ group = Group.where(name: 'All users').select do |g|
+ g[:uuid].match /-f+$/
+ end.first
+
+ if not group
+ logger.warn "No 'All users' group with uuid '*-*-fffffffffffffff'."
+ raise "No 'All users' group with uuid '*-*-fffffffffffffff' is found"
+ else
+ logger.info { "\"All users\" group uuid: " + group[:uuid] }
+
+ group_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: group[:uuid],
+ head_kind: 'arvados#group',
+ link_class: 'permission',
+ name: 'can_read')
+
+ if !group_perms.any?
+ group_perm = Link.create(tail_kind: 'arvados#user',
+ tail_uuid: self.uuid,
+ head_kind: 'arvados#group',
+ head_uuid: group[:uuid],
+ link_class: 'permission',
+ name: 'can_read')
+ logger.info { "group permission: " + group_perm[:uuid] }
+ else
+ group_perm = group_perms.first
+ end
+
+ return group_perm
+ end
+ end
+
+ # Give the special "System group" permission to manage this user and
+ # all of this user's stuff.
+ #
+ def add_system_group_permission_link
+ act_as_system_user do
+ Link.create(link_class: 'permission',
+ name: 'can_manage',
+ tail_kind: 'arvados#group',
+ tail_uuid: system_group_uuid,
+ head_kind: 'arvados#user',
+ head_uuid: self.uuid)
+ end
+ end
end
secret_token: ~
uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
- git_repositories_dir: /var/cache/git
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ git_repositories_dir: /var/lib/arvados/git
+
+ # This is a (bare) repository that stores commits used in jobs. When a job
+ # runs, the source commits are first fetched into this repository, then this
+ # repository is used to deploy to compute nodes. This should NOT be a
+ # subdirectory of {git_repositiories_dir}.
+ git_internal_dir: /var/lib/arvados/internal.git
# :none or :slurm_immediate
crunch_job_wrapper: :none
secret_token: <%= rand(2**512).to_s(36) %>
common:
-
- # Git repositories must be readable by api server, or you won't be
- # able to submit crunch jobs. To pass the test suites, put a clone
- # of the arvados tree in {git_repositories_dir}/arvados.git or
- # {git_repositories_dir}/arvados/.git
- #
#git_repositories_dir: /var/cache/git
+ #git_internal_dir: /var/cache/arvados/internal.git
encoding: utf8
database: arvados_test
username: arvados
- password: ********
+ password: xxxxxxxx
host: localhost
production:
encoding: utf8
database: arvados_production
username: arvados
- password: ********
+ password: xxxxxxxx
host: localhost
# Initialize the rails application
Server::Application.initialize!
+begin
+ Rails.cache.clear
+rescue Errno::ENOENT => e
+ # Cache directory does not exist? Then cache is clear, proceed.
+ Rails.logger.warn "In Rails.cache.clear, ignoring #{e.inspect}"
+end
config.force_ssl = false
- config.git_repositories_dir = '/var/cache/git'
-
- config.crunch_job_wrapper = :none
- config.crunch_job_user = 'crunch' # if false, do not set uid when running jobs
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- config.crunch_refresh_trigger = '/tmp/crunch_refresh_trigger'
-
- # config.dnsmasq_conf_dir = '/etc/dnsmasq.d'
-
- # config.compute_node_ami = 'ami-cbca41a2'
- # config.compute_node_ec2run_args = '-g arvados-compute'
- # config.compute_node_spot_bid = 0.11
-
- # config.compute_node_domain = `hostname --domain`.strip
-
- # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
- config.compute_node_nameservers = ['192.168.201.3']
-
- config.uuid_prefix('development@' + `hostname`.strip)
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.new_users_are_active = false
- config.admin_notifier_email_from = 'arvados@example.com'
- config.email_subject_prefix = '[ARVADOS] '
-
- # Visitors to the API server will be redirected to the workbench
- config.workbench_address = "http://localhost:3000/"
-
- # The e-mail address of the user you would like to become marked as an admin
- # user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses openid against Google's servers, so in that case this
- # should be an address associated with a Google account.
- config.auto_admin_user = ''
end
# Send deprecation notices to registered listeners
config.active_support.deprecation = :notify
- config.git_repositories_dir = '/var/cache/git'
-
- config.crunch_job_wrapper = :slurm_immediate
- config.crunch_job_user = 'crunch' # if false, do not set uid when running jobs
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- config.crunch_refresh_trigger = '/tmp/crunch_refresh_trigger'
-
- # config.dnsmasq_conf_dir = '/etc/dnsmasq.d'
-
- # config.compute_node_ami = 'ami-cbca41a2'
- # config.compute_node_ec2run_args = '-g arvados-compute'
- # config.compute_node_spot_bid = 0.11
-
- # config.compute_node_domain = `hostname --domain`.strip
-
- # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
- require 'net/http'
- config.compute_node_nameservers = ['local', 'public'].collect do |iface|
- Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- end << '172.16.0.23'
-
- config.uuid_prefix = Digest::MD5.hexdigest('cfi-aws-0').to_i(16).to_s(36)[0..4] # '9ujm1'
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.new_users_are_active = false
- config.admin_notifier_email_from = 'arvados@example.com'
- config.email_subject_prefix = '[ARVADOS] '
-
- # Visitors to the API server will be redirected to the workbench
- config.workbench_address = "http://workbench." + `hostname`
-
- # The e-mail address of the user you would like to become marked as an admin
- # user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses openid against Google's servers, so in that case this
- # should be an address associated with a Google account.
- config.auto_admin_user = ''
end
# Raise exception on mass assignment protection for Active Record models
config.active_record.mass_assignment_sanitizer = :strict
- config.git_repositories_dir = '/var/cache/git'
-
- config.crunch_job_wrapper = :slurm_immediate
- config.crunch_job_user = 'crunch' # if false, do not set uid when running jobs
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- config.crunch_refresh_trigger = '/tmp/crunch_refresh_trigger_test'
-
- # config.dnsmasq_conf_dir = '/etc/dnsmasq.d'
-
- # config.compute_node_ami = 'ami-cbca41a2'
- # config.compute_node_ec2run_args = '-g arvados-compute'
- # config.compute_node_spot_bid = 0.11
- config.compute_node_ec2_tag_enable = false
-
- # config.compute_node_domain = `hostname --domain`.strip
-
# No need for SSL while testing
config.force_ssl = false
- # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
- config.compute_node_nameservers = [ "172.16.0.23" ]
-
- config.uuid_prefix = 'zzzzz'
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.new_users_are_active = false
- config.admin_notifier_email_from = 'arvados@example.com'
- config.email_subject_prefix = '[ARVADOS] '
-
- # Visitors to the API server will be redirected to the workbench
- config.workbench_address = "http://localhost:3000/"
-
- # The e-mail address of the user you would like to become marked as an admin
- # user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses openid against Google's servers, so in that case this
- # should be an address associated with a Google account.
- config.auto_admin_user = ''
end
namespace :arvados do
namespace :v1 do
- match '/schema' => 'schema#show'
match '/nodes/:uuid/ping' => 'nodes#ping', :as => :ping_node
match '/keep_disks/ping' => 'keep_disks#ping', :as => :ping_keep_disk
match '/links/from/:tail_uuid' => 'links#index', :as => :arvados_v1_links_from
post '/jobs/:uuid/cancel' => 'jobs#cancel'
match '/users/:uuid/event_stream' => 'users#event_stream'
post '/users/:uuid/activate' => 'users#activate'
+ post '/users/setup' => 'users#setup'
+ post '/users/:uuid/unsetup' => 'users#unsetup'
match '/virtual_machines/get_all_logins' => 'virtual_machines#get_all_logins'
match '/virtual_machines/:uuid/logins' => 'virtual_machines#logins'
post '/api_client_authorizations/create_system_auth' => 'api_client_authorizations#create_system_auth'
match '/login', :to => 'user_sessions#login'
match '/logout', :to => 'user_sessions#logout'
- match '/discovery/v1/apis/arvados/v1/rest', :to => 'arvados/v1/schema#discovery_rest_description'
+ match '/discovery/v1/apis/arvados/v1/rest', :to => 'arvados/v1/schema#index'
match '/static/login_failure', :to => 'static#login_failure', :as => :login_failure
--- /dev/null
+class AddNondeterministicColumnToJob < ActiveRecord::Migration
+ def up
+ add_column :jobs, :nondeterministic, :boolean
+ end
+
+ def down
+ remove_column :jobs, :nondeterministic
+ end
+end
--- /dev/null
+class SeparateRepositoryFromScriptVersion < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def fixup pt
+ c = pt.components
+ c.each do |k, v|
+ commit_ish = v["script_version"]
+ if commit_ish.andand.index(':')
+ want_repo, commit_ish = commit_ish.split(':',2)
+ v[:repository] = want_repo
+ v[:script_version] = commit_ish
+ end
+ end
+ pt.save!
+ end
+
+ def up
+ act_as_system_user do
+ PipelineTemplate.all.each do |pt|
+ fixup pt
+ end
+ PipelineInstance.all.each do |pt|
+ fixup pt
+ end
+ end
+ end
+
+ def down
+ raise ActiveRecord::IrreversibleMigration
+ end
+end
--- /dev/null
+class AddRepositoryColumnToJob < ActiveRecord::Migration
+ def up
+ add_column :jobs, :repository, :string
+ end
+
+ def down
+ remove_column :jobs, :repository
+ end
+end
--- /dev/null
+class AddOutputIsPersistentToJob < ActiveRecord::Migration
+ def change
+ add_column :jobs, :output_is_persistent, :boolean, null: false, default: false
+ end
+end
--- /dev/null
+class AddSystemGroup < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def up
+ # Make sure the system group exists.
+ system_group
+ end
+
+ def down
+ act_as_system_user do
+ system_group.destroy
+
+ # Destroy the automatically generated links giving system_group
+ # permission on all users.
+ Link.destroy_all(tail_uuid: system_group_uuid, head_kind: 'arvados#user')
+ end
+ end
+end
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140325175653) do
+ActiveRecord::Schema.define(:version => 20140402001908) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
t.boolean "running"
t.boolean "success"
t.string "output"
- t.datetime "created_at"
- t.datetime "updated_at"
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
t.string "priority"
t.string "is_locked_by_uuid"
t.string "log"
t.text "runtime_constraints"
t.boolean "nondeterministic"
t.string "repository"
+ t.boolean "output_is_persistent", :default => false, :null => false
end
add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"
-# This file should contain all the record creation needed to seed the database with its default values.
-# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+# This file seeds the database with initial/default values.
#
-# Examples:
-#
-# cities = City.create([{ :name => 'Chicago' }, { :name => 'Copenhagen' }])
-# Mayor.create(:name => 'Emanuel', :city => cities.first)
+# It is invoked by `rake db:seed` and `rake db:setup`.
+
+# These two methods would create the system user and group objects on
+# demand later anyway, but it's better form to create them up front.
+include CurrentApiClient
+system_user
+system_group
'000000000000000'].join('-')
end
+ def system_group_uuid
+ [Server::Application.config.uuid_prefix,
+ Group.uuid_prefix,
+ '000000000000000'].join('-')
+ end
+
def system_user
if not $system_user
real_current_user = Thread.current[:user]
$system_user
end
+ def system_group
+ if not $system_group
+ act_as_system_user do
+ ActiveRecord::Base.transaction do
+ $system_group = Group.
+ where(uuid: system_group_uuid).first_or_create do |g|
+ g.update_attributes(name: "System group",
+ description: "System group")
+ User.all.collect(&:uuid).each do |user_uuid|
+ Link.create(link_class: 'permission',
+ name: 'can_manage',
+ tail_kind: 'arvados#group',
+ tail_uuid: system_group_uuid,
+ head_kind: 'arvados#user',
+ head_uuid: user_uuid)
+ end
+ end
+ end
+ end
+ end
+ $system_group
+ end
+
def act_as_system_user
if block_given?
user_was = Thread.current[:user]
desc 'Ensure site configuration has all required settings'
task check: :environment do
$application_config.sort.each do |k, v|
- $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ if ENV.has_key?('QUIET') then
+ # Make sure we still check for the variable to exist
+ eval("Rails.configuration.#{k}")
+ else
+ if /(password|secret)/.match(k) then
+ # Make sure we still check for the variable to exist, but don't print the value
+ eval("Rails.configuration.#{k}")
+ $stderr.puts "%-32s %s" % [k, '*********']
+ else
+ $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ end
+ end
end
end
end
raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
end
+ require 'shellwords'
+
+ arvados_internal = Rails.configuration.git_internal_dir
+ if not File.exists? arvados_internal
+ $stderr.puts `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init --bare`
+ end
+
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository + '.git')
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository, '.git') unless File.exists? src_repo
+
+ unless src_repo
+ $stderr.puts "dispatch: #{File.join Rails.configuration.git_repositories_dir, job.repository} doesn't exist"
+ sleep 1
+ untake(job)
+ next
+ end
+
+ $stderr.puts `cd #{arvados_internal.shellescape} && git fetch --no-tags #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
cmd_args << job_auth.api_token
cmd_args << '--job'
cmd_args << job.uuid
-
- commit = Commit.where(sha1: job.script_version).first
- if commit
- cmd_args << '--git-dir'
- if File.exists?(File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git'))
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git')
- else
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name, '.git')
- end
- end
+ cmd_args << '--git-dir'
+ cmd_args << arvados_internal
$stderr.puts "dispatch: #{cmd_args.join ' '}"
end
def update_pipelines
+ expire_tokens = @pipe_auth_tokens.dup
@todo_pipelines.each do |p|
- pipe_auth = ApiClientAuthorization.
- new(user: User.where('uuid=?', p.modified_by_user_uuid).first,
- api_client_id: 0)
- pipe_auth.save
-
+ pipe_auth = (@pipe_auth_tokens[p.uuid] ||= ApiClientAuthorization.
+ create(user: User.where('uuid=?', p.modified_by_user_uuid).first,
+ api_client_id: 0))
puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
+ expire_tokens.delete p.uuid
+ end
+
+ expire_tokens.each do |k, v|
+ v.update_attributes expires_at: Time.now
+ @pipe_auth_tokens.delete k
end
end
def run
act_as_system_user
@running ||= {}
+ @pipe_auth_tokens ||= { }
$stderr.puts "dispatch: ready"
while !$signal[:term] or @running.size > 0
read_pipes
unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
start_jobs
end
- unless @todo_pipelines.empty? or did_recently(:update_pipelines, 5.0)
+ unless (@todo_pipelines.empty? and @pipe_auth_tokens.empty?) or did_recently(:update_pipelines, 5.0)
update_pipelines
end
end
:config => File.expand_path("config.ru"),
:SSLEnable => true,
:SSLVerifyClient => OpenSSL::SSL::VERIFY_NONE,
- :SSLCertName => [["CN", WEBrick::Utils::getservername]]
+ :SSLCertName => [["CN", "#{WEBrick::Utils::getservername} #{Time.now().to_s}"]]
})
end
end
-end
+end
######### /SSL
--- /dev/null
+#!/usr/bin/env ruby
+
+abort 'Error: Ruby >= 1.9.3 required.' if RUBY_VERSION < '1.9.3'
+
+require 'logger'
+require 'trollop'
+
+log = Logger.new STDERR
+log.progname = $0.split('/').last
+
+opts = Trollop::options do
+ banner ''
+ banner "Usage: #{log.progname} " +
+ "{user_uuid_or_email} {user_and_repo_name} {vm_uuid}"
+ banner ''
+ opt :debug, <<-eos
+Show debug messages.
+ eos
+ opt :openid_prefix, <<-eos, default: 'https://www.google.com/accounts/o8/id'
+If creating a new user record, require authentication from an OpenID \
+with this OpenID prefix *and* a matching email address in order to \
+claim the account.
+ eos
+end
+
+log.level = (ENV['DEBUG'] || opts.debug) ? Logger::DEBUG : Logger::WARN
+
+if ARGV.count != 3
+ Trollop::die "required arguments are missing"
+end
+
+user_arg, user_repo_name, vm_uuid = ARGV
+
+require 'arvados'
+arv = Arvados.new(api_version: 'v1')
+
+# Look up the given user by uuid or, failing that, email address.
+begin
+ found_user = arv.user.get(uuid: user_arg)
+rescue Arvados::TransactionFailedError
+ found = arv.user.list(where: {email: user_arg})[:items]
+
+ if found.count == 0
+ if !user_arg.match(/\w\@\w+\.\w+/)
+ abort "About to create new user, but #{user_arg.inspect} " +
+ "does not look like an email address. Stop."
+ end
+ elsif found.count != 1
+ abort "Found #{found.count} users with email. Stop."
+ else
+ found_user = found.first
+ end
+end
+
+# Invoke user setup method
+if (found_user)
+ user = arv.user.setup uuid: found_user[:uuid], repo_name: user_repo_name,
+ vm_uuid: vm_uuid, openid_prefix: opts.openid_prefix
+else
+ user = arv.user.setup user: {email: user_arg}, repo_name: user_repo_name,
+ vm_uuid: vm_uuid, openid_prefix: opts.openid_prefix
+end
+
+log.info {"user uuid: " + user[:uuid]}
+
+puts user.inspect
api_token: 1a9ffdcga2o7cw8q12dndskomgs1ygli3ns9k2o9hgzgmktc78
expires_at: 2038-01-01 00:00:00
+miniadmin:
+ api_client: untrusted
+ user: miniadmin
+ api_token: 2zb2y9pw3e70270te7oe3ewaantea3adyxjascvkz0zob7q7xb
+ expires_at: 2038-01-01 00:00:00
+
+rominiadmin:
+ api_client: untrusted
+ user: rominiadmin
+ api_token: 5tsb2pc3zlatn1ortl98s2tqsehpby88wmmnzmpsjmzwa6payh
+ expires_at: 2038-01-01 00:00:00
+
active:
api_client: untrusted
user: active
user_agreement:
- uuid: b519d9cb706a29fc7ea24dbea2f05851+93
+ uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
owner_uuid: qr1hi-tpzed-tpj2ff66551eyym
created_at: 2013-12-26T19:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
name: Private
description: Private Group
+private_and_can_read_foofile:
+ uuid: zzzzz-j7d0g-22xp1wpjul508rk
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: Private and Can Read Foofile
+ description: Another Private Group
+
system_owned_group:
uuid: zzzzz-j7d0g-8ulrifv67tve5sx
owner_uuid: zzzzz-tpzed-000000000000000
uuid: zzzzz-j7d0g-fffffffffffffff
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
name: All users
+
+testusergroup_admins:
+ uuid: zzzzz-j7d0g-48foin4vonvc2at
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: Administrators of a subset of users
cancelled_by_client_uuid: ~
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
output: ~
cancelled_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
output: ~
running: 0
done: 1
runtime_constraints: {}
+
+previous_job_run:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ script: hash
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "1"
+ success: true
+
+nondeterminisic_job_run:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykyyy
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ script: hash2
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "1"
+ success: true
+ nondeterministic: true
\ No newline at end of file
tail_uuid: zzzzz-tpzed-000000000000000
link_class: signature
name: require
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851+93
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
user_agreement_readable:
tail_uuid: zzzzz-j7d0g-fffffffffffffff
link_class: permission
name: can_read
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851+93
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
active_user_member_of_all_users_group:
tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
link_class: signature
name: click
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851+93
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
user_agreement_signed_by_inactive:
tail_uuid: zzzzz-tpzed-7sg468ezxwnodxs
link_class: signature
name: click
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851+93
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
spectator_user_member_of_all_users_group:
head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
properties: {}
+foo_file_readable_by_active_duplicate_permission:
+ uuid: zzzzz-o0j2j-2qlmhgothiur55r
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-000000000000000
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ properties: {}
+
+foo_file_readable_by_active_redundant_permission_via_private_group:
+ uuid: zzzzz-o0j2j-5s8ry7sn6bwxb7w
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-000000000000000
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#group
+ tail_uuid: zzzzz-j7d0g-22xp1wpjul508rk
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ properties: {}
+
bar_file_readable_by_active:
uuid: zzzzz-o0j2j-8hppiuduf8eqdng
owner_uuid: zzzzz-tpzed-000000000000000
head_uuid: zzzzz-8i9sb-cjs4pklxxjykyuq
properties: {}
+foo_repository_readable_by_spectator:
+ uuid: zzzzz-o0j2j-cpy7p41hpk5xxx
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ link_class: permission
+ name: can_read
+ head_kind: arvados#repository
+ head_uuid: zzzzz-2x53u-382brsig8rp3666
+ properties: {}
+
+miniadmin_user_is_a_testusergroup_admin:
+ uuid: zzzzz-o0j2j-38vvkciz7qc12j9
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-04-01 13:53:33 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-04-01 13:53:33 -0400
+ updated_at: 2014-04-01 13:53:33 -0400
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-2bg9x0oeydcw5hm
+ link_class: permission
+ name: can_manage
+ head_kind: arvados#group
+ head_uuid: zzzzz-j7d0g-48foin4vonvc2at
+ properties: {}
+
+rominiadmin_user_is_a_testusergroup_admin:
+ uuid: zzzzz-o0j2j-6b0hz5hr107mc90
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-04-01 13:53:33 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-04-01 13:53:33 -0400
+ updated_at: 2014-04-01 13:53:33 -0400
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-4hvxm4n25emegis
+ link_class: permission
+ name: can_read
+ head_kind: arvados#group
+ head_uuid: zzzzz-j7d0g-48foin4vonvc2at
+ properties: {}
+
+testusergroup_can_manage_active_user:
+ uuid: zzzzz-o0j2j-2vaqhxz6hsf4k1d
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-04-01 13:56:10 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-04-01 13:56:10 -0400
+ updated_at: 2014-04-01 13:56:10 -0400
+ tail_kind: arvados#group
+ tail_uuid: zzzzz-j7d0g-48foin4vonvc2at
+ link_class: permission
+ name: can_manage
+ head_kind: arvados#user
+ head_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ properties: {}
--- /dev/null
+foo:
+ uuid: zzzzz-2x53u-382brsig8rp3666
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: foo
--- /dev/null
+owned_by_active_user:
+ uuid: zzzzz-2x53u-3zx463qyo0k4xrn
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+
+owned_by_private_group:
+ uuid: zzzzz-2x53u-5m3qwg45g3nlpu6
+ owner_uuid: zzzzz-j7d0g-rew6elm53kancon
+
+owned_by_spectator:
+ uuid: zzzzz-2x53u-3b0xxwzlbzxq5yr
+ owner_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
is_admin: true
prefs: {}
+miniadmin:
+ uuid: zzzzz-tpzed-2bg9x0oeydcw5hm
+ email: miniadmin@arvados.local
+ first_name: TestCase
+ last_name: User Group Administrator
+ identity_url: https://miniadmin.openid.local
+ is_active: true
+ is_admin: false
+ prefs: {}
+
+rominiadmin:
+ uuid: zzzzz-tpzed-4hvxm4n25emegis
+ email: rominiadmin@arvados.local
+ first_name: TestCase
+ last_name: Read-Only User Group Administrator
+ identity_url: https://rominiadmin.openid.local
+ is_active: true
+ is_admin: false
+ prefs: {}
+
active:
uuid: zzzzz-tpzed-xurymjxw79nv3jz
email: active-user@arvados.local
assert_not_nil assigns(:objects)
end
+ [0,1,2].each do |limit|
+ test "get index with limit=#{limit}" do
+ authorize_with :active
+ get :index, limit: limit
+ assert_response :success
+ assert_equal limit, assigns(:objects).count
+ resp = JSON.parse(@response.body)
+ assert_equal limit, resp['limit']
+ end
+ end
+
+ test "items.count == items_available" do
+ authorize_with :active
+ get :index, limit: 100000
+ assert_response :success
+ resp = JSON.parse(@response.body)
+ assert_equal resp['items_available'], assigns(:objects).length
+ assert_equal resp['items_available'], resp['items'].count
+ unique_uuids = resp['items'].collect { |i| i['uuid'] }.compact.uniq
+ assert_equal unique_uuids.count, resp['items'].count
+ end
+
+ test "get index with limit=2 offset=99999" do
+ # Assume there are not that many test fixtures.
+ authorize_with :active
+ get :index, limit: 2, offset: 99999
+ assert_response :success
+ assert_equal 0, assigns(:objects).count
+ resp = JSON.parse(@response.body)
+ assert_equal 2, resp['limit']
+ assert_equal 99999, resp['offset']
+ end
+
test "should create" do
authorize_with :active
test_collection = {
assert_nil resp['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'] # foo
end
+ test "search collections with 'any' operator" do
+ authorize_with :active
+ get :index, {
+ where: { any: ['contains', '7f9102c395f4ffc5e3'] }
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal 1, found.count
+ assert_equal true, !!found.index('1f4b0bc7583c2a7f9102c395f4ffc5e3+45')
+ end
+
end
--- /dev/null
+require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
+
+class Arvados::V1::CommitsControllerTest < ActionController::TestCase
+ fixtures :repositories, :users
+
+ # See git_setup.rb for the commit log for test.git.tar
+ include GitSetup
+
+ test "test_find_commit_range" do
+ authorize_with :active
+
+ # single
+ a = Commit.find_commit_range(users(:active), nil, nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ assert_equal ['31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ #test "test_branch1" do
+ a = Commit.find_commit_range(users(:active), nil, nil, 'master', nil)
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57'], a
+
+ #test "test_branch2" do
+ a = Commit.find_commit_range(users(:active), 'foo', nil, 'b1', nil)
+ assert_equal ['1de84a854e2b440dc53bf42f8548afa4c17da332'], a
+
+ #test "test_branch3" do
+ a = Commit.find_commit_range(users(:active), 'foo', nil, 'HEAD', nil)
+ assert_equal ['1de84a854e2b440dc53bf42f8548afa4c17da332'], a
+
+ #test "test_single_revision_repo" do
+ a = Commit.find_commit_range(users(:active), "foo", nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ assert_equal ['31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+ a = Commit.find_commit_range(users(:active), "bar", nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ assert_equal nil, a
+
+ #test "test_multi_revision" do
+ a = Commit.find_commit_range(users(:active), nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', nil)
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '4fe459abe02d9b365932b8f5dc419439ab4e2577', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ #test "test_tag" do
+ a = Commit.find_commit_range(users(:active), nil, 'tag1', 'master', nil)
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '4fe459abe02d9b365932b8f5dc419439ab4e2577'], a
+
+ #test "test_multi_revision_exclude" do
+ a = Commit.find_commit_range(users(:active), nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['4fe459abe02d9b365932b8f5dc419439ab4e2577'])
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ #test "test_multi_revision_tagged_exclude" do
+ a = Commit.find_commit_range(users(:active), nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['tag1'])
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ Dir.mktmpdir do |touchdir|
+ # invalid input to maximum
+ a = Commit.find_commit_range(users(:active), nil, nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to maximum
+ a = Commit.find_commit_range(users(:active), nil, nil, "$(uname>#{touchdir}/uh_oh)", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to minimum
+ a = Commit.find_commit_range(users(:active), nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to minimum
+ a = Commit.find_commit_range(users(:active), nil, "$(uname>#{touchdir}/uh_oh)", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to 'excludes'
+ a = Commit.find_commit_range(users(:active), nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["4fe459abe02d9b365932b8f5dc419439ab4e2577 ; touch #{touchdir}/uh_oh"])
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to 'excludes'
+ a = Commit.find_commit_range(users(:active), nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["$(uname>#{touchdir}/uh_oh)"])
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ end
+
+ end
+
+end
--- /dev/null
+require 'fileutils'
+require 'tmpdir'
+
+# Commit log for test.git.tar
+# master is the main branch
+# b1 is a branch off of master
+# tag1 is a tag
+#
+# 1de84a8 * b1
+# 077ba2a * master
+# 4fe459a * tag1
+# 31ce37f * foo
+
+module GitSetup
+ def setup
+ @tmpdir = Dir.mktmpdir()
+ #puts "setup #{@tmpdir}"
+ `cp test/test.git.tar #{@tmpdir} && cd #{@tmpdir} && tar xf test.git.tar`
+ Rails.configuration.git_repositories_dir = "#{@tmpdir}/test"
+ Commit.refresh_repositories
+ end
+
+ def teardown
+ #puts "teardown #{@tmpdir}"
+ FileUtils.remove_entry @tmpdir, true
+ end
+end
--- /dev/null
+require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
+
+class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
+ fixtures :repositories, :users, :jobs, :links
+
+ # See git_setup.rb for the commit log for test.git.tar
+ include GitSetup
+
+ test "test_reuse_job" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_reuse_job_range" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ minimum_script_version: "tag1",
+ script_version: "master",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_different_input" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '2'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_different_version" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "master",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '2'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_submitted_nondeterministic" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ },
+ nondeterministic: true
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_past_nondeterministic" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash2",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykyyy', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_no_permission" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :spectator
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_excluded" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ minimum_script_version: "31ce37fe365b3dc204300a3e4c396ad333ed0556",
+ script_version: "master",
+ repository: "foo",
+ exclude_script_versions: ["tag1"],
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ end
+
+
+end
require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
class Arvados::V1::JobsControllerTest < ActionController::TestCase
+ include GitSetup
+
test "submit a job" do
authorize_with :active
post :create, job: {
script: "hash",
script_version: "master",
+ repository: "foo",
script_parameters: {}
}
assert_response :success
new_job = JSON.parse(@response.body)
assert_not_nil new_job['uuid']
assert_not_nil new_job['script_version'].match(/^[0-9a-f]{40}$/)
+ # Default: not persistent
+ assert_equal false, new_job['output_is_persistent']
end
test "normalize output and log uuids when creating job" do
script: "hash",
script_version: "master",
script_parameters: {},
+ repository: "foo",
started_at: Time.now,
finished_at: Time.now,
running: false,
}
assert_response :success
found = assigns(:objects).collect(&:uuid)
- assert_equal true, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ assert_equal 0, found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ assert_equal 1, found.count
end
test "search jobs by nonexistent column with < query" do
}
assert_response 422
end
+
+
end
test "should get fresh discovery document" do
MAX_SCHEMA_AGE = 60
- get :discovery_rest_description
+ get :index
assert_response :success
discovery_doc = JSON.parse(@response.body)
assert_equal 'discovery#restDescription', discovery_doc['kind']
require 'test_helper'
class Arvados::V1::UsersControllerTest < ActionController::TestCase
+ include CurrentApiClient
+
+ setup do
+ @all_links_at_start = Link.all
+ @vm_uuid = virtual_machines(:testvm).uuid
+ end
test "activate a user after signing UA" do
authorize_with :inactive_but_signed_user_agreement
assert_equal true, me['is_active']
end
+ test "create new user with user as input" do
+ authorize_with :admin
+ post :create, user: {
+ first_name: "test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ assert_response :success
+ created = JSON.parse(@response.body)
+ assert_equal 'test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected uuid for the newly created user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+ end
+
+ test "create user with user, vm and repo as input" do
+ authorize_with :admin
+ repo_name = 'test_repo'
+
+ post :setup, {
+ repo_name: repo_name,
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ user: {
+ uuid: "this_is_agreeable",
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected non-null uuid for the new user'
+ assert_equal 'this_is_agreeable', created['uuid']
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # arvados#user, repo link and link add user to 'All users' group
+ verify_num_links @all_links_at_start, 4
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ repo_name, created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ verify_system_group_permission_link_for created['uuid']
+
+ # invoke setup again with the same data
+ post :setup, {
+ repo_name: repo_name,
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ user: {
+ uuid: "this_is_agreeable",
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+
+ response_items = JSON.parse(@response.body)['items']
+
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected non-null uuid for the new user'
+ assert_equal 'this_is_agreeable', created['uuid']
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # arvados#user, repo link and link add user to 'All users' group
+ verify_num_links @all_links_at_start, 5
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ repo_name, created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ verify_system_group_permission_link_for created['uuid']
+ end
+
+ test "setup user with bogus uuid and expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ uuid: 'bogus_uuid',
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'Path not found'), 'Expected 404'
+ end
+
+ test "setup user with bogus uuid in user and expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {uuid: 'bogus_uuid'},
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'ArgumentError: Require user email'),
+ 'Expected RuntimeError'
+ end
+
+ test "setup user with no uuid and user, expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'Required uuid or user'),
+ 'Expected ArgumentError'
+ end
+
+ test "setup user with no uuid and email, expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {},
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? '<ArgumentError: Require user email'),
+ 'Expected ArgumentError'
+ end
+
+ test "invoke setup with existing uuid, vm and repo and verify links" do
+ authorize_with :inactive
+ get :current
+ assert_response :success
+ inactive_user = JSON.parse(@response.body)
+
+ authorize_with :admin
+
+ post :setup, {
+ uuid: inactive_user['uuid'],
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ resp_obj = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
+ assert_equal inactive_user['uuid'], resp_obj['uuid']
+ assert_equal inactive_user['email'], resp_obj['email'],
+ 'expecting inactive user email'
+
+ # expect repo and vm links
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', resp_obj['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, resp_obj['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "invoke setup with existing uuid in user, verify response" do
+ authorize_with :inactive
+ get :current
+ assert_response :success
+ inactive_user = JSON.parse(@response.body)
+
+ authorize_with :admin
+
+ post :setup, {
+ user: {uuid: inactive_user['uuid']},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ resp_obj = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
+ assert_equal inactive_user['uuid'], resp_obj['uuid']
+ assert_equal inactive_user['email'], resp_obj['email'],
+ 'expecting inactive user email'
+ end
+
+ test "invoke setup with existing uuid but different email, expect original email" do
+ authorize_with :inactive
+ get :current
+ assert_response :success
+ inactive_user = JSON.parse(@response.body)
+
+ authorize_with :admin
+
+ post :setup, {
+ uuid: inactive_user['uuid'],
+ user: {email: 'junk_email'}
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ resp_obj = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
+ assert_equal inactive_user['uuid'], resp_obj['uuid']
+ assert_equal inactive_user['email'], resp_obj['email'],
+ 'expecting inactive user email'
+ end
+
+ test "setup user with valid email and repo as input" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for the new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # four extra links; system_group, login, group and repo perms
+ verify_num_links @all_links_at_start, 4
+ end
+
+ test "setup user with fake vm and expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ vm_uuid: 'no_such_vm',
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? "No vm found for no_such_vm"),
+ 'Expected RuntimeError: No vm found for no_such_vm'
+ end
+
+ test "setup user with valid email, repo and real vm as input" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ vm_uuid: @vm_uuid,
+ user: {email: 'foo@example.com'}
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for the new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # five extra links; system_group, login, group, vm, repo
+ verify_num_links @all_links_at_start, 5
+ end
+
+ test "setup user with valid email, no vm and repo as input" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # three extra links; system_group, login, and group
+ verify_num_links @all_links_at_start, 3
+ end
+
+ test "setup user with email, first name, repo name and vm uuid" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ user: {
+ first_name: 'test_first_name',
+ email: 'foo@example.com'
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+ assert_equal 'test_first_name', response_object['first_name'],
+ 'expecting first name'
+
+ # five extra links; system_group, login, group, repo and vm
+ verify_num_links @all_links_at_start, 5
+ end
+
+ test "setup user twice with email and check two different objects created" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ repo_name: 'test_repo',
+ user: {
+ email: 'foo@example.com'
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+ # system_group, openid, group, and repo. No vm link.
+ verify_num_links @all_links_at_start, 4
+
+ # create again
+ post :setup, {
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object2 = find_obj_in_resp response_items, 'User', nil
+ assert_not_equal response_object['uuid'], response_object2['uuid'],
+ 'expected same uuid as first create operation'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # +1 extra login link +1 extra system_group link pointing to the new User
+ verify_num_links @all_links_at_start, 6
+ end
+
+ test "setup user with openid prefix" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ openid_prefix: 'http://www.example.com/account',
+ user: {
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected uuid for new user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # verify links
+ # four new links: system_group, arvados#user, repo, and 'All users' group.
+ verify_num_links @all_links_at_start, 4
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "invoke setup with no openid prefix, expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ user: {
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'openid_prefix parameter is missing'),
+ 'Expected ArgumentError'
+ end
+
+ test "setup user with user, vm and repo and verify links" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ },
+ vm_uuid: @vm_uuid,
+ repo_name: 'test_repo',
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected uuid for new user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # five new links: system_group, arvados#user, repo, vm and 'All
+ # users' group link
+ verify_num_links @all_links_at_start, 5
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "create user as non admin user and expect error" do
+ authorize_with :active
+
+ post :create, {
+ user: {email: 'foo@example.com'}
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'PermissionDenied'),
+ 'Expected PermissionDeniedError'
+ end
+
+ test "setup user as non admin user and expect error" do
+ authorize_with :active
+
+ post :setup, {
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ user: {email: 'foo@example.com'}
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'Forbidden'),
+ 'Expected Forbidden error'
+ end
+
+ test "setup user in multiple steps and verify response" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'http://www.example.com/account',
+ user: {
+ email: "foo@example.com"
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil created['uuid'], 'expected uuid for new user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_equal created['email'], 'foo@example.com', 'expected input email'
+
+ # three new links: system_group, arvados#user, and 'All users' group.
+ verify_num_links @all_links_at_start, 3
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#repository', false, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ # invoke setup with a repository
+ post :setup, {
+ openid_prefix: 'http://www.example.com/account',
+ repo_name: 'new_repo',
+ uuid: created['uuid']
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'foo@example.com', created['email'], 'expected input email'
+
+ # verify links
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'new_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ # invoke setup with a vm_uuid
+ post :setup, {
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'http://www.example.com/account',
+ user: {
+ email: 'junk_email'
+ },
+ uuid: created['uuid']
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal created['email'], 'foo@example.com', 'expected original email'
+
+ # verify links
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ # since no repo name in input, we won't get any; even though user has one
+ verify_link response_items, 'arvados#repository', false, 'permission', 'can_write',
+ 'new_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "setup and unsetup user" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil created['uuid'], 'expected uuid for the new user'
+ assert_equal created['email'], 'foo@example.com', 'expected given email'
+
+ # five extra links: system_group, login, group, repo and vm
+ verify_num_links @all_links_at_start, 5
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ verify_link_existence created['uuid'], created['email'], true, true, true, true, false
+
+ # now unsetup this user
+ post :unsetup, uuid: created['uuid']
+ assert_response :success
+
+ created2 = JSON.parse(@response.body)
+ assert_not_nil created2['uuid'], 'expected uuid for the newly created user'
+ assert_equal created['uuid'], created2['uuid'], 'expected uuid not found'
+
+ verify_link_existence created['uuid'], created['email'], false, false, false, false, false
+ end
+
+ test "unsetup active user" do
+ authorize_with :active
+ get :current
+ assert_response :success
+ active_user = JSON.parse(@response.body)
+ assert_not_nil active_user['uuid'], 'expected uuid for the active user'
+ assert active_user['is_active'], 'expected is_active for active user'
+ assert active_user['is_invited'], 'expected is_invited for active user'
+
+ verify_link_existence active_user['uuid'], active_user['email'],
+ false, false, false, true, true
+
+ authorize_with :admin
+
+ # now unsetup this user
+ post :unsetup, uuid: active_user['uuid']
+ assert_response :success
+
+ response_user = JSON.parse(@response.body)
+ assert_not_nil response_user['uuid'], 'expected uuid for the upsetup user'
+ assert_equal active_user['uuid'], response_user['uuid'], 'expected uuid not found'
+ assert !response_user['is_active'], 'expected user to be inactive'
+ assert !response_user['is_invited'], 'expected user to be uninvited'
+
+ verify_link_existence response_user['uuid'], response_user['email'],
+ false, false, false, false, false
+ end
+
+ def verify_num_links (original_links, expected_additional_links)
+ links_now = Link.all
+ assert_equal expected_additional_links, Link.all.size-original_links.size,
+ "Expected #{expected_additional_links.inspect} more links"
+ end
+
+ def find_obj_in_resp (response_items, object_type, head_kind=nil)
+ return_obj = nil
+ response_items.each { |x|
+ if !x
+ next
+ end
+
+ if object_type == 'User'
+ if !x['head_kind']
+ return_obj = x
+ break
+ end
+ else # looking for a link
+ if x['head_kind'] == head_kind
+ return_obj = x
+ break
+ end
+ end
+ }
+ return return_obj
+ end
+
+ def verify_link(response_items, link_object_name, expect_link, link_class,
+ link_name, head_uuid, tail_uuid, head_kind, fetch_object, class_name)
+
+ link = find_obj_in_resp response_items, 'Link', link_object_name
+
+ if !expect_link
+ assert_nil link, "Expected no link for #{link_object_name}"
+ return
+ end
+
+ assert_not_nil link, "Expected link for #{link_object_name}"
+
+ if fetch_object
+ object = Object.const_get(class_name).where(name: head_uuid)
+ assert [] != object, "expected #{class_name} with name #{head_uuid}"
+ head_uuid = object.first[:uuid]
+ end
+ assert_equal link['link_class'], link_class,
+ "did not find expected link_class for #{link_object_name}"
+
+ assert_equal link['name'], link_name,
+ "did not find expected link_name for #{link_object_name}"
+
+ assert_equal link['tail_uuid'], tail_uuid,
+ "did not find expected tail_uuid for #{link_object_name}"
+
+ assert_equal link['head_kind'], head_kind,
+ "did not find expected head_kind for #{link_object_name}"
+
+ assert_equal link['head_uuid'], head_uuid,
+ "did not find expected head_uuid for #{link_object_name}"
+ end
+
+ def verify_link_existence uuid, email, expect_oid_login_perms,
+ expect_repo_perms, expect_vm_perms, expect_group_perms, expect_signatures
+ # verify that all links are deleted for the user
+ oid_login_perms = Link.where(tail_uuid: email,
+ head_kind: 'arvados#user',
+ link_class: 'permission',
+ name: 'can_login')
+ if expect_oid_login_perms
+ assert oid_login_perms.any?, "expected oid_login_perms"
+ else
+ assert !oid_login_perms.any?, "expected all oid_login_perms deleted"
+ end
+
+ repo_perms = Link.where(tail_uuid: uuid,
+ head_kind: 'arvados#repository',
+ link_class: 'permission',
+ name: 'can_write')
+ if expect_repo_perms
+ assert repo_perms.any?, "expected repo_perms"
+ else
+ assert !repo_perms.any?, "expected all repo_perms deleted"
+ end
+
+ vm_login_perms = Link.where(tail_uuid: uuid,
+ head_kind: 'arvados#virtualMachine',
+ link_class: 'permission',
+ name: 'can_login')
+ if expect_vm_perms
+ assert vm_login_perms.any?, "expected vm_login_perms"
+ else
+ assert !vm_login_perms.any?, "expected all vm_login_perms deleted"
+ end
+
+ group = Group.where(name: 'All users').select do |g|
+ g[:uuid].match /-f+$/
+ end.first
+ group_read_perms = Link.where(tail_uuid: uuid,
+ head_uuid: group[:uuid],
+ head_kind: 'arvados#group',
+ link_class: 'permission',
+ name: 'can_read')
+ if expect_group_perms
+ assert group_read_perms.any?, "expected all users group read perms"
+ else
+ assert !group_read_perms.any?, "expected all users group perm deleted"
+ end
+
+ signed_uuids = Link.where(link_class: 'signature',
+ tail_kind: 'arvados#user',
+ tail_uuid: uuid)
+
+ if expect_signatures
+ assert signed_uuids.any?, "expected signatures"
+ else
+ assert !signed_uuids.any?, "expected all signatures deleted"
+ end
+
+ end
+
+ def verify_system_group_permission_link_for user_uuid
+ assert_equal 1, Link.where(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: system_group_uuid,
+ head_uuid: user_uuid).count
+ end
end
assert_equal "arvados#collectionList", jresponse['kind']
end
+ test "get index with filters= (empty string)" do
+ get "/arvados/v1/collections", {:format => :json, :filters => ''}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ assert_response :success
+ assert_equal "arvados#collectionList", jresponse['kind']
+ end
+
+ test "get index with where= (empty string)" do
+ get "/arvados/v1/collections", {:format => :json, :where => ''}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ assert_response :success
+ assert_equal "arvados#collectionList", jresponse['kind']
+ end
+
test "controller 404 response is json" do
get "/arvados/v1/thingsthatdonotexist", {:format => :xml}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
assert_response 404
class PermissionsTest < ActionDispatch::IntegrationTest
fixtures :users, :groups, :api_client_authorizations, :collections
- test "adding and removing direct can_read links" do
- auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:spectator).api_token}"}
- admin_auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
+ def auth auth_fixture
+ {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(auth_fixture).api_token}"}
+ end
+ test "adding and removing direct can_read links" do
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
# try to add permission as spectator
head_uuid: collections(:foo_file).uuid,
properties: {}
}
- }, auth
+ }, auth(:spectator)
assert_response 422
# add permission as admin
head_uuid: collections(:foo_file).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
u = jresponse['uuid']
assert_response :success
# read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response :success
# try to delete permission as spectator
- delete "/arvados/v1/links/#{u}", {:format => :json}, auth
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:spectator)
assert_response 403
# delete permission as admin
- delete "/arvados/v1/links/#{u}", {:format => :json}, admin_auth
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
end
test "adding can_read links from user to group, group to collection" do
- auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:spectator).api_token}"}
- admin_auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
-
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
# add permission for spectator to read group
head_uuid: groups(:private).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
# add permission for group to read collection
head_uuid: collections(:foo_file).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
u = jresponse['uuid']
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response :success
# delete permission for group to read collection
- delete "/arvados/v1/links/#{u}", {:format => :json}, admin_auth
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
end
test "adding can_read links from group to collection, user to group" do
- auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:spectator).api_token}"}
- admin_auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
-
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
# add permission for group to read collection
head_uuid: collections(:foo_file).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
# add permission for spectator to read group
head_uuid: groups(:private).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
u = jresponse['uuid']
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response :success
# delete permission for spectator to read group
- delete "/arvados/v1/links/#{u}", {:format => :json}, admin_auth
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
end
test "adding can_read links from user to group, group to group, group to collection" do
- auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:spectator).api_token}"}
- admin_auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
-
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response 404
# add permission for user to read group
head_uuid: groups(:private).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
assert_response :success
# add permission for group to read group
head_uuid: groups(:empty_lonely_group).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
assert_response :success
# add permission for group to read collection
head_uuid: collections(:foo_file).uuid,
properties: {}
}
- }, admin_auth
+ }, auth(:admin)
u = jresponse['uuid']
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
assert_response :success
# delete permission for group to read collection
- delete "/arvados/v1/links/#{u}", {:format => :json}, admin_auth
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
assert_response :success
# try to read collection as spectator
- get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+ end
+
+ test "read-only group-admin sees correct subset of user list" do
+ get "/arvados/v1/users", {:format => :json}, auth(:rominiadmin)
+ assert_response :success
+ resp_uuids = jresponse['items'].collect { |i| i['uuid'] }
+ [[true, users(:rominiadmin).uuid],
+ [true, users(:active).uuid],
+ [false, users(:miniadmin).uuid],
+ [false, users(:spectator).uuid]].each do |should_find, uuid|
+ assert_equal should_find, !resp_uuids.index(uuid).nil?, "rominiadmin should #{'not ' if !should_find}see #{uuid} in user list"
+ end
+ end
+
+ test "read-only group-admin cannot modify administered user" do
+ put "/arvados/v1/users/#{users(:active).uuid}", {
+ :user => {
+ first_name: 'KilroyWasHere'
+ },
+ :format => :json
+ }, auth(:rominiadmin)
+ assert_response 403
+ end
+
+ test "read-only group-admin cannot read or update non-administered user" do
+ get "/arvados/v1/users/#{users(:spectator).uuid}", {
+ :format => :json
+ }, auth(:rominiadmin)
+ assert_response 404
+
+ put "/arvados/v1/users/#{users(:spectator).uuid}", {
+ :user => {
+ first_name: 'KilroyWasHere'
+ },
+ :format => :json
+ }, auth(:rominiadmin)
assert_response 404
end
+
+ test "RO group-admin finds user's specimens, RW group-admin can update" do
+ [[:rominiadmin, false],
+ [:miniadmin, true]].each do |which_user, update_should_succeed|
+ get "/arvados/v1/specimens", {:format => :json}, auth(which_user)
+ assert_response :success
+ resp_uuids = jresponse['items'].collect { |i| i['uuid'] }
+ [[true, specimens(:owned_by_active_user).uuid],
+ [true, specimens(:owned_by_private_group).uuid],
+ [false, specimens(:owned_by_spectator).uuid],
+ ].each do |should_find, uuid|
+ assert_equal(should_find, !resp_uuids.index(uuid).nil?,
+ "%s should%s see %s in specimen list" %
+ [which_user.to_s,
+ should_find ? '' : 'not ',
+ uuid])
+ put "/arvados/v1/specimens/#{uuid}", {
+ :specimen => {
+ properties: {
+ miniadmin_was_here: true
+ }
+ },
+ :format => :json
+ }, auth(which_user)
+ if !should_find
+ assert_response 404
+ elsif !update_should_succeed
+ assert_response 403
+ else
+ assert_response :success
+ end
+ end
+ end
+ end
+
end
require 'test_helper'
class UserTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ include CurrentApiClient
+
+ # The fixture services/api/test/fixtures/users.yml serves as the input for this test case
+ setup do
+ # Make sure system_user exists before making "pre-test users" list
+ system_user
+
+ @all_users = User.find(:all)
+
+ @all_users.each do |user|
+ if user.uuid == system_user_uuid
+ @system_user = user
+ elsif user.is_admin && user.is_active
+ @admin_user = user
+ elsif user.is_active && !user.is_admin
+ @active_user = user
+ elsif !user.is_active && !user.is_invited
+ @uninvited_user = user
+ end
+ end
+ end
+
+ test "check non-admin active user properties" do
+ assert !@active_user.is_admin, 'is_admin should not be set for a non-admin user'
+ assert @active_user.is_active, 'user should be active'
+ assert @active_user.is_invited, 'is_invited should be set'
+ assert_not_nil @active_user.prefs, "user's preferences should be non-null, but may be size zero"
+ assert (@active_user.can? :read=>"#{@active_user.uuid}"), "user should be able to read own object"
+ assert (@active_user.can? :write=>"#{@active_user.uuid}"), "user should be able to write own object"
+ assert (@active_user.can? :manage=>"#{@active_user.uuid}"), "user should be able to manage own object"
+
+ assert @active_user.groups_i_can(:read).size > 0, "active user should be able read at least one group"
+
+ # non-admin user cannot manage or write other user objects
+ assert !(@active_user.can? :read=>"#{@uninvited_user.uuid}")
+ assert !(@active_user.can? :write=>"#{@uninvited_user.uuid}")
+ assert !(@active_user.can? :manage=>"#{@uninvited_user.uuid}")
+ end
+
+ test "check admin user properties" do
+ assert @admin_user.is_admin, 'is_admin should be set for admin user'
+ assert @admin_user.is_active, 'admin user cannot be inactive'
+ assert @admin_user.is_invited, 'is_invited should be set'
+ assert_not_nil @admin_user.uuid.size, "user's uuid should be non-null"
+ assert_not_nil @admin_user.prefs, "user's preferences should be non-null, but may be size zero"
+ assert @admin_user.identity_url.size > 0, "user's identity url is expected"
+ assert @admin_user.can? :read=>"#{@admin_user.uuid}"
+ assert @admin_user.can? :write=>"#{@admin_user.uuid}"
+ assert @admin_user.can? :manage=>"#{@admin_user.uuid}"
+
+ assert @admin_user.groups_i_can(:read).size > 0, "admin active user should be able read at least one group"
+ assert @admin_user.groups_i_can(:write).size > 0, "admin active user should be able write to at least one group"
+ assert @admin_user.groups_i_can(:manage).size > 0, "admin active user should be able manage at least one group"
+
+ # admin user can also write or manage other users
+ assert @admin_user.can? :read=>"#{@uninvited_user.uuid}"
+ assert @admin_user.can? :write=>"#{@uninvited_user.uuid}"
+ assert @admin_user.can? :manage=>"#{@uninvited_user.uuid}"
+ end
+
+ test "check inactive and uninvited user properties" do
+ assert !@uninvited_user.is_admin, 'is_admin should not be set for a non-admin user'
+ assert !@uninvited_user.is_active, 'user should be inactive'
+ assert !@uninvited_user.is_invited, 'is_invited should not be set'
+ assert @uninvited_user.can? :read=>"#{@uninvited_user.uuid}"
+ assert @uninvited_user.can? :write=>"#{@uninvited_user.uuid}"
+ assert @uninvited_user.can? :manage=>"#{@uninvited_user.uuid}"
+
+ assert @uninvited_user.groups_i_can(:read).size == 0, "inactive and uninvited user should not be able read any groups"
+ assert @uninvited_user.groups_i_can(:write).size == 0, "inactive and uninvited user should not be able write to any groups"
+ assert @uninvited_user.groups_i_can(:manage).size == 0, "inactive and uninvited user should not be able manage any groups"
+ end
+
+ test "find user method checks" do
+ User.find(:all).each do |user|
+ assert_not_nil user.uuid, "non-null uuid expected for " + user.full_name
+ end
+
+ user = users(:active) # get the active user
+
+ found_user = User.find(user.id) # find a user by the row id
+
+ assert_equal found_user.full_name, user.first_name + ' ' + user.last_name
+ assert_equal found_user.identity_url, user.identity_url
+ end
+
+ test "create new user" do
+ Thread.current[:user] = @admin_user # set admin user as the current user
+
+ user = User.new
+ user.first_name = "first_name_for_newly_created_user"
+ user.save
+
+ # verify there is one extra user in the db now
+ assert_equal @all_users.size+1, User.find(:all).size
+
+ user = User.find(user.id) # get the user back
+ assert_equal(user.first_name, 'first_name_for_newly_created_user')
+ assert_not_nil user.uuid, 'uuid should be set for newly created user'
+ assert_nil user.email, 'email should be null for newly created user, because it was not passed in'
+ assert_nil user.identity_url, 'identity_url should be null for newly created user, because it was not passed in'
+
+ user.first_name = 'first_name_for_newly_created_user_updated'
+ user.save
+ user = User.find(user.id) # get the user back
+ assert_equal(user.first_name, 'first_name_for_newly_created_user_updated')
+ end
+
+ test "update existing user" do
+ Thread.current[:user] = @active_user # set active user as current user
+ @active_user.first_name = "first_name_changed"
+ @active_user.save
+
+ @active_user = User.find(@active_user.id) # get the user back
+ assert_equal(@active_user.first_name, 'first_name_changed')
+
+ # admin user also should be able to update the "active" user info
+ Thread.current[:user] = @admin_user # set admin user as current user
+ @active_user.first_name = "first_name_changed_by_admin_for_active_user"
+ @active_user.save
+
+ @active_user = User.find(@active_user.id) # get the user back
+ assert_equal(@active_user.first_name, 'first_name_changed_by_admin_for_active_user')
+ end
+
+ test "delete a user and verify" do
+ active_user_uuid = @active_user.uuid
+
+ Thread.current[:user] = @admin_user
+ @active_user.delete
+
+ found_deleted_user = false
+ User.find(:all).each do |user|
+ if user.uuid == active_user_uuid
+ found_deleted_user = true
+ break
+ end
+ end
+ assert !found_deleted_user, "found deleted user: "+active_user_uuid
+
+ end
+
+ test "create new user as non-admin user" do
+ Thread.current[:user] = @active_user
+
+ begin
+ user = User.new
+ user.save
+ rescue ArvadosModel::PermissionDeniedError => e
+ end
+ assert (e.message.include? 'PermissionDeniedError'),
+ 'Expected PermissionDeniedError'
+ end
+
+ test "setup new user" do
+ Thread.current[:user] = @admin_user
+
+ email = 'foo@example.com'
+ openid_prefix = 'http://openid/prefix'
+
+ user = User.new
+ user.email = email
+ user.uuid = 'abcdefghijklmnop'
+
+ vm = VirtualMachine.create
+
+ response = User.setup user, openid_prefix, 'test_repo', vm.uuid
+
+ resp_user = find_obj_in_resp response, 'User'
+ verify_user resp_user, email
+
+ oid_login_perm = find_obj_in_resp response, 'Link', 'arvados#user'
+ verify_link oid_login_perm, 'permission', 'can_login', resp_user[:email],
+ resp_user[:uuid]
+ assert_equal openid_prefix, oid_login_perm[:properties][:identity_url_prefix],
+ 'expected identity_url_prefix not found for oid_login_perm'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ vm_perm = find_obj_in_resp response, 'Link', 'arvados#virtualMachine'
+ verify_link vm_perm, 'permission', 'can_login', resp_user[:uuid], vm.uuid
+ end
+
+ test "setup new user in multiple steps" do
+ Thread.current[:user] = @admin_user
+
+ email = 'foo@example.com'
+ openid_prefix = 'http://openid/prefix'
+
+ user = User.new
+ user.email = email
+ user.uuid = 'abcdefghijklmnop'
+
+ response = User.setup user, openid_prefix
+
+ resp_user = find_obj_in_resp response, 'User'
+ verify_user resp_user, email
+
+ oid_login_perm = find_obj_in_resp response, 'Link', 'arvados#user'
+ verify_link oid_login_perm, 'permission', 'can_login', resp_user[:email],
+ resp_user[:uuid]
+ assert_equal openid_prefix, oid_login_perm[:properties][:identity_url_prefix],
+ 'expected identity_url_prefix not found for oid_login_perm'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ # invoke setup again with repo_name
+ response = User.setup user, openid_prefix, 'test_repo'
+ resp_user = find_obj_in_resp response, 'User', nil
+ verify_user resp_user, email
+ assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ # invoke setup again with a vm_uuid
+ vm = VirtualMachine.create
+
+ response = User.setup user, openid_prefix, 'test_repo', vm.uuid
+
+ resp_user = find_obj_in_resp response, 'User', nil
+ verify_user resp_user, email
+ assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ vm_perm = find_obj_in_resp response, 'Link', 'arvados#virtualMachine'
+ verify_link vm_perm, 'permission', 'can_login', resp_user[:uuid], vm.uuid
+ end
+
+ def find_obj_in_resp (response, object_type, head_kind=nil)
+ return_obj = nil
+ response.each { |x|
+ if x.class.name == object_type
+ if head_kind
+ if x.head_kind == head_kind
+ return_obj = x
+ break
+ end
+ else
+ return_obj = x
+ break
+ end
+ end
+ }
+ return return_obj
+ end
+
+ def verify_user (resp_user, email)
+ assert_not_nil resp_user, 'expected user object'
+ assert_not_nil resp_user['uuid'], 'expected user object'
+ assert_equal email, resp_user['email'], 'expected email not found'
+
+ end
+
+ def verify_link (link_object, link_class, link_name, tail_uuid, head_uuid)
+ assert_not_nil link_object, 'expected link for #{link_class} #{link_name}'
+ assert_not_nil link_object[:uuid],
+ 'expected non-nil uuid for link for #{link_class} #{link_name}'
+ assert_equal link_class, link_object[:link_class],
+ 'expected link_class not found for #{link_class} #{link_name}'
+ assert_equal link_name, link_object[:name],
+ 'expected link_name not found for #{link_class} #{link_name}'
+ assert_equal tail_uuid, link_object[:tail_uuid],
+ 'expected tail_uuid not found for #{link_class} #{link_name}'
+ if head_uuid
+ assert_equal head_uuid, link_object[:head_uuid],
+ 'expected head_uuid not found for #{link_class} #{link_name}'
+ end
+ end
+
end
--- /dev/null
+package main
+
+import (
+ "bufio"
+ "crypto/md5"
+ "errors"
+ "fmt"
+ "github.com/gorilla/mux"
+ "log"
+ "net/http"
+ "os"
+ "strings"
+)
+
+const DEFAULT_PORT = 25107
+const BLOCKSIZE = 64 * 1024 * 1024
+
+var PROC_MOUNTS = "/proc/mounts"
+
+var KeepVolumes []string
+
+func main() {
+ // Look for local keep volumes.
+ KeepVolumes = FindKeepVolumes()
+ if len(KeepVolumes) == 0 {
+ log.Fatal("could not find any keep volumes")
+ }
+ for _, v := range KeepVolumes {
+ log.Println("keep volume:", v)
+ }
+
+ // Set up REST handlers.
+ //
+ // Start with a router that will route each URL path to an
+ // appropriate handler.
+ //
+ rest := mux.NewRouter()
+ rest.HandleFunc("/{hash:[0-9a-f]{32}}", GetBlockHandler).Methods("GET")
+
+ // Tell the built-in HTTP server to direct all requests to the REST
+ // router.
+ http.Handle("/", rest)
+
+ // Start listening for requests.
+ port := fmt.Sprintf(":%d", DEFAULT_PORT)
+ http.ListenAndServe(port, nil)
+}
+
+// FindKeepVolumes
+// Returns a list of Keep volumes mounted on this system.
+//
+// A Keep volume is a normal or tmpfs volume with a /keep
+// directory at the top level of the mount point.
+//
+func FindKeepVolumes() []string {
+ vols := make([]string, 0)
+
+ if f, err := os.Open(PROC_MOUNTS); err != nil {
+ log.Fatalf("opening %s: %s\n", PROC_MOUNTS, err)
+ } else {
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ args := strings.Fields(scanner.Text())
+ dev, mount := args[0], args[1]
+ if (dev == "tmpfs" || strings.HasPrefix(dev, "/dev/")) && mount != "/" {
+ keep := mount + "/keep"
+ if st, err := os.Stat(keep); err == nil && st.IsDir() {
+ vols = append(vols, keep)
+ }
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ log.Fatal(err)
+ }
+ }
+ return vols
+}
+
+func GetBlockHandler(w http.ResponseWriter, req *http.Request) {
+ hash := mux.Vars(req)["hash"]
+
+ block, err := GetBlock(hash)
+ if err != nil {
+ http.Error(w, err.Error(), 404)
+ return
+ }
+
+ _, err = w.Write(block)
+ if err != nil {
+ log.Printf("GetBlockHandler: writing response: %s", err)
+ }
+
+ return
+}
+
+func GetBlock(hash string) ([]byte, error) {
+ var buf = make([]byte, BLOCKSIZE)
+
+ // Attempt to read the requested hash from a keep volume.
+ for _, vol := range KeepVolumes {
+ var f *os.File
+ var err error
+ var nread int
+
+ path := fmt.Sprintf("%s/%s/%s", vol, hash[0:3], hash)
+
+ f, err = os.Open(path)
+ if err != nil {
+ log.Printf("%s: opening %s: %s\n", vol, path, err)
+ continue
+ }
+
+ nread, err = f.Read(buf)
+ if err != nil {
+ log.Printf("%s: reading %s: %s\n", vol, path, err)
+ continue
+ }
+
+ // Double check the file checksum.
+ //
+ filehash := fmt.Sprintf("%x", md5.Sum(buf[:nread]))
+ if filehash != hash {
+ // TODO(twp): this condition probably represents a bad disk and
+ // should raise major alarm bells for an administrator: e.g.
+ // they should be sent directly to an event manager at high
+ // priority or logged as urgent problems.
+ //
+ log.Printf("%s: checksum mismatch: %s (actual hash %s)\n",
+ vol, path, filehash)
+ continue
+ }
+
+ // Success!
+ return buf[:nread], nil
+ }
+
+ log.Printf("%s: not found on any volumes, giving up\n", hash)
+ return buf, errors.New("not found: " + hash)
+}
--- /dev/null
+package main
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path"
+ "testing"
+)
+
+var TEST_BLOCK = []byte("The quick brown fox jumps over the lazy dog.")
+var TEST_HASH = "e4d909c290d0fb1ca068ffaddf22cbd0"
+var BAD_BLOCK = []byte("The magic words are squeamish ossifrage.")
+
+// Test simple block reads.
+func TestGetBlockOK(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes and store a block in each of them.
+ KeepVolumes = setup(t, 2)
+ fmt.Println("KeepVolumes = ", KeepVolumes)
+
+ for _, vol := range KeepVolumes {
+ store(t, vol, TEST_HASH, TEST_BLOCK)
+ }
+
+ // Check that GetBlock returns success.
+ result, err := GetBlock(TEST_HASH)
+ if err != nil {
+ t.Errorf("GetBlock error: %s", err)
+ }
+ if fmt.Sprint(result) != fmt.Sprint(TEST_BLOCK) {
+ t.Errorf("expected %s, got %s", TEST_BLOCK, result)
+ }
+}
+
+// Test block reads when one Keep volume is missing.
+func TestGetBlockOneKeepOK(t *testing.T) {
+ defer teardown()
+
+ // Two test Keep volumes, only the second has a block.
+ KeepVolumes = setup(t, 2)
+ store(t, KeepVolumes[1], TEST_HASH, TEST_BLOCK)
+
+ // Check that GetBlock returns success.
+ result, err := GetBlock(TEST_HASH)
+ if err != nil {
+ t.Errorf("GetBlock error: %s", err)
+ }
+ if fmt.Sprint(result) != fmt.Sprint(TEST_BLOCK) {
+ t.Errorf("expected %s, got %s", TEST_BLOCK, result)
+ }
+}
+
+// Test block read failure.
+func TestGetBlockFail(t *testing.T) {
+ defer teardown()
+
+ // Create two empty test Keep volumes.
+ KeepVolumes = setup(t, 2)
+
+ // Check that GetBlock returns failure.
+ result, err := GetBlock(TEST_HASH)
+ if err == nil {
+ t.Errorf("GetBlock incorrectly returned success: ", result)
+ }
+}
+
+// Test reading a corrupt block.
+func TestGetBlockCorrupt(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes and store a block in each of them,
+ // but the hash of the block does not match the filename.
+ KeepVolumes = setup(t, 2)
+ for _, vol := range KeepVolumes {
+ store(t, vol, TEST_HASH, BAD_BLOCK)
+ }
+
+ // Check that GetBlock returns failure.
+ result, err := GetBlock(TEST_HASH)
+ if err == nil {
+ t.Errorf("GetBlock incorrectly returned success: %s", result)
+ }
+}
+
+// Test finding Keep volumes.
+func TestFindKeepVolumes(t *testing.T) {
+ defer teardown()
+
+ // Initialize two keep volumes.
+ var tempVols []string = setup(t, 2)
+
+ // Set up a bogus PROC_MOUNTS file.
+ if f, err := ioutil.TempFile("", "keeptest"); err == nil {
+ for _, vol := range tempVols {
+ fmt.Fprintf(f, "tmpfs %s tmpfs opts\n", path.Dir(vol))
+ }
+ f.Close()
+ PROC_MOUNTS = f.Name()
+
+ // Check that FindKeepVolumes finds the temp volumes.
+ resultVols := FindKeepVolumes()
+ if len(tempVols) != len(resultVols) {
+ t.Fatalf("set up %d volumes, FindKeepVolumes found %d\n",
+ len(tempVols), len(resultVols))
+ }
+ for i := range tempVols {
+ if tempVols[i] != resultVols[i] {
+ t.Errorf("FindKeepVolumes returned %s, expected %s\n",
+ resultVols[i], tempVols[i])
+ }
+ }
+
+ os.Remove(f.Name())
+ }
+}
+
+// Test that FindKeepVolumes returns an empty slice when no Keep volumes
+// are present.
+func TestFindKeepVolumesFail(t *testing.T) {
+ defer teardown()
+
+ // Set up a bogus PROC_MOUNTS file with no Keep vols.
+ if f, err := ioutil.TempFile("", "keeptest"); err == nil {
+ fmt.Fprintln(f, "rootfs / rootfs opts 0 0")
+ fmt.Fprintln(f, "sysfs /sys sysfs opts 0 0")
+ fmt.Fprintln(f, "proc /proc proc opts 0 0")
+ fmt.Fprintln(f, "udev /dev devtmpfs opts 0 0")
+ fmt.Fprintln(f, "devpts /dev/pts devpts opts 0 0")
+ f.Close()
+ PROC_MOUNTS = f.Name()
+
+ // Check that FindKeepVolumes returns an empty array.
+ resultVols := FindKeepVolumes()
+ if len(resultVols) != 0 {
+ t.Fatalf("FindKeepVolumes returned %v", resultVols)
+ }
+
+ os.Remove(PROC_MOUNTS)
+ }
+}
+
+// setup
+// Create KeepVolumes for testing.
+// Returns a slice of pathnames to temporary Keep volumes.
+//
+func setup(t *testing.T, num_volumes int) []string {
+ vols := make([]string, num_volumes)
+ for i := range vols {
+ if dir, err := ioutil.TempDir(os.TempDir(), "keeptest"); err == nil {
+ vols[i] = dir + "/keep"
+ os.Mkdir(vols[i], 0755)
+ } else {
+ t.Fatal(err)
+ }
+ }
+ return vols
+}
+
+// teardown
+// Cleanup to perform after each test.
+//
+func teardown() {
+ for _, vol := range KeepVolumes {
+ os.RemoveAll(path.Dir(vol))
+ }
+}
+
+// store
+//
+func store(t *testing.T, keepdir string, filename string, block []byte) error {
+ blockdir := fmt.Sprintf("%s/%s", keepdir, filename[:3])
+ if err := os.MkdirAll(blockdir, 0755); err != nil {
+ t.Fatal(err)
+ }
+
+ blockpath := fmt.Sprintf("%s/%s", blockdir, filename)
+ if f, err := os.Create(blockpath); err == nil {
+ f.Write(block)
+ f.Close()
+ } else {
+ t.Fatal(err)
+ }
+
+ return nil
+}
projects / arvados.git / commitdiff
