$scroller = $(window);
$scroller.
addClass('infinite-scroller').
- on('scroll', { container: this }, maybe_load_more_content);
+ on('scroll resize', { container: this }, maybe_load_more_content).
+ trigger('scroll');
});
});
width: 100%;
border: none;
}
+
+div.rounded {
+ border-radius: 3px;
+}
around_filter :require_thread_api_token, except: ERROR_ACTIONS
before_filter :accept_uuid_as_id_param, except: ERROR_ACTIONS
before_filter :check_user_agreements, except: ERROR_ACTIONS
+ before_filter :check_user_profile, except: [:update_profile] + ERROR_ACTIONS
before_filter :check_user_notifications, except: ERROR_ACTIONS
before_filter :load_filters_and_paging_params, except: ERROR_ACTIONS
before_filter :find_object_by_uuid, except: [:index, :choose] + ERROR_ACTIONS
end
def load_filters_and_paging_params
+ @order = params[:order] || 'created_at desc'
+ @order = [@order] unless @order.is_a? Array
+
@limit ||= 200
if params[:limit]
@limit = params[:limit].to_i
respond_to do |f|
f.json { render json: @objects }
f.html {
- if params['tab_pane']
- comparable = self.respond_to? :compare
- render(partial: 'show_' + params['tab_pane'].downcase,
- locals: { comparable: comparable, objects: @objects })
+ if params[:tab_pane]
+ render_pane params[:tab_pane]
else
render
end
end
end
+ helper_method :render_pane
+ def render_pane tab_pane, opts={}
+ render_opts = {
+ partial: 'show_' + tab_pane.downcase,
+ locals: {
+ comparable: self.respond_to?(:compare),
+ objects: @objects,
+ tab_pane: tab_pane
+ }.merge(opts[:locals] || {})
+ }
+ if opts[:to_string]
+ render_to_string render_opts
+ else
+ render render_opts
+ end
+ end
+
def index
find_objects_for_index if !@objects
render_index
f.json { render json: @object.attributes.merge(href: url_for(@object)) }
f.html {
if params['tab_pane']
- comparable = self.respond_to? :compare
- render(partial: 'show_' + params['tab_pane'].downcase,
- locals: { comparable: comparable, objects: @objects })
+ render_pane params['tab_pane']
elsif request.method.in? ['GET', 'HEAD']
render
else
Thread.current[:arvados_api_token] = new_token
if new_token.nil?
Thread.current[:user] = nil
- elsif (new_token == session[:arvados_api_token]) and
- session[:user].andand[:is_active]
- Thread.current[:user] = User.new(session[:user])
else
Thread.current[:user] = User.current
end
false # We may redirect to login, or not, based on the current action.
else
session[:arvados_api_token] = params[:api_token]
- session[:user] = {
- uuid: user.uuid,
- email: user.email,
- first_name: user.first_name,
- last_name: user.last_name,
- is_active: user.is_active,
- is_admin: user.is_admin,
- prefs: user.prefs
- }
+
if !request.format.json? and request.method.in? ['GET', 'HEAD']
# Repeat this request with api_token in the (new) session
# cookie instead of the query string. This prevents API
true
end
+ def check_user_profile
+ if request.method.downcase != 'get' || params[:partial] ||
+ params[:tab_pane] || params[:action_method] ||
+ params[:action] == 'setup_popup'
+ return true
+ end
+
+ if missing_required_profile?
+ render 'users/profile'
+ end
+ true
+ end
+
+ helper_method :missing_required_profile?
+ def missing_required_profile?
+ missing_required = false
+
+ profile_config = Rails.configuration.user_profile_form_fields
+ if current_user && profile_config
+ current_user_profile = current_user.prefs[:profile]
+ profile_config.kind_of?(Array) && profile_config.andand.each do |entry|
+ if entry['required']
+ if !current_user_profile ||
+ !current_user_profile[entry['key'].to_sym] ||
+ current_user_profile[entry['key'].to_sym].empty?
+ missing_required = true
+ break
+ end
+ end
+ end
+ end
+
+ missing_required
+ end
+
def select_theme
return Rails.configuration.arvados_theme
end
super
end
+ def load_contents_objects kinds=[]
+ kind_filters = @filters.select do |attr,op,val|
+ op == 'is_a' and val.is_a? Array and val.count > 1
+ end
+ if /^created_at\b/ =~ @order[0] and kind_filters.count == 1
+ # If filtering on multiple types and sorting by date: Get the
+ # first page of each type, sort the entire set, truncate to one
+ # page, and use the last item on this page as a filter for
+ # retrieving the next page. Ideally the API would do this for
+ # us, but it doesn't (yet).
+ nextpage_operator = /\bdesc$/i =~ @order[0] ? '<' : '>'
+ @objects = []
+ @name_link_for = {}
+ kind_filters.each do |attr,op,val|
+ (val.is_a?(Array) ? val : [val]).each do |type|
+ objects = @object.contents(order: @order,
+ limit: @limit,
+ include_linked: true,
+ filters: (@filters - kind_filters + [['uuid', 'is_a', type]]),
+ offset: @offset)
+ objects.each do |object|
+ @name_link_for[object.andand.uuid] = objects.links_for(object, 'name').first
+ end
+ @objects += objects
+ end
+ end
+ @objects = @objects.to_a.sort_by(&:created_at)
+ @objects.reverse! if nextpage_operator == '<'
+ @objects = @objects[0..@limit-1]
+ @next_page_filters = @filters.reject do |attr,op,val|
+ attr == 'created_at' and op == nextpage_operator
+ end
+ if @objects.any?
+ @next_page_filters += [['created_at',
+ nextpage_operator,
+ @objects.last.created_at]]
+ @next_page_href = url_for(partial: :contents_rows,
+ filters: @next_page_filters.to_json)
+ else
+ @next_page_href = nil
+ end
+ else
+ @objects = @object.contents(order: @order,
+ limit: @limit,
+ include_linked: true,
+ filters: @filters,
+ offset: @offset)
+ @next_page_href = next_page_href(partial: :contents_rows)
+ end
+ end
+
def show
if !@object
return render_not_found("object not found")
end
- @objects = @object.contents(limit: 50,
- include_linked: true,
- filters: params[:filters],
- offset: params[:offset] || 0)
- @logs = Log.limit(10).filter([['object_uuid', '=', @object.uuid]])
- @users = User.limit(10000).
- select(["uuid", "is_active", "first_name", "last_name"]).
- filter([['is_active', '=', 'true']])
- @groups = Group.limit(10000).
- select(["uuid", "name", "description"])
@user_is_manager = false
@share_links = []
end
end
- @objects_and_names = get_objects_and_names @objects
-
if params[:partial]
+ load_contents_objects
respond_to do |f|
f.json {
render json: {
content: render_to_string(partial: 'show_contents_rows.html',
- formats: [:html],
- locals: {
- objects_and_names: @objects_and_names,
- project: @object
- }),
- next_page_href: (next_page_offset and
- url_for(offset: next_page_offset, filters: params[:filters], partial: true))
+ formats: [:html]),
+ next_page_href: @next_page_href
}
}
end
else
+ @objects = []
super
end
end
end
helper_method :get_objects_and_names
- def get_objects_and_names(objects)
+ def get_objects_and_names(objects=nil)
+ objects = @objects if objects.nil?
objects_and_names = []
objects.each do |object|
- if !(name_links = objects.links_for(object, 'name')).empty?
+ if objects.respond_to? :links_for and
+ !(name_links = objects.links_for(object, 'name')).empty?
name_links.each do |name_link|
objects_and_names << [object, name_link]
end
+ elsif @name_link_for.andand[object.uuid]
+ objects_and_names << [object, @name_link_for[object.uuid]]
elsif object.respond_to? :name
objects_and_names << [object, object]
else
class UserAgreementsController < ApplicationController
skip_before_filter :check_user_agreements
skip_before_filter :find_object_by_uuid
+ skip_before_filter :check_user_profile
def model_class
Collection
end
modal_path = choose_collections_path \
({ title: chooser_title,
- filters: [['tail_uuid', '=', object.owner_uuid]].to_json,
+ filters: [['owner_uuid', '=', object.owner_uuid]].to_json,
action_name: 'OK',
action_href: pipeline_instance_path(id: object.uuid),
action_method: 'patch',
arvados_api_client.api(self, "/setup", params)
end
+ def update_profile params
+ self.private_reload(arvados_api_client.api(self.class,
+ "/#{self.uuid}/profile",
+ params))
+ end
+
end
<div class="modal-body">
<div class="input-group">
<% if params[:by_project].to_s != "false" %>
+ <% selected_project_name = 'All projects'
+ @filters.andand.each do |attr, op, val|
+ if attr == 'owner_uuid' and op == '='
+ if val == current_user.uuid
+ selected_project_name = "Home"
+ else
+ selected_project_name = Group.find(val).name rescue val
+ end
+ end
+ end
+ %>
<div class="input-group-btn" data-filterable-target=".modal.arv-choose .selectable-container">
<button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">
- All projects <span class="caret"></span>
+ <%= selected_project_name %> <span class="caret"></span>
</button>
<ul class="dropdown-menu" role="menu">
<li>
<div id="<%= pane %>-scroll" style="margin-top:0.5em;">
<div class="pane-content">
<% if i == 0 %>
- <%= render(partial: 'show_' + pane.downcase,
- locals: { comparable: comparable, objects: @objects }) %>
+ <%= render_pane pane, to_string: true %>
<% else %>
<div class="spinner spinner-32px spinner-h-center"></div>
<% end %>
<%= content_for :content_top %>
- <% if @object and @object.is_a?(Group) and @object.group_class == 'project' %>
- <div class="pull-right">
- <%= content_for :tab_line_buttons %>
- </div>
- <br clear="all" />
-<% else %>
- <br clear="all" />
- <div class="pull-right">
- <%= content_for :tab_line_buttons %>
- </div>
-<% end %>
+<div class="pull-right">
+ <%= content_for :tab_line_buttons %>
+</div>
+<br clear="all" />
<%= content_for :tab_panes %>
<% end %>
<%= render partial: 'content', layout: 'content_layout', locals: {pane_list: controller.show_pane_list }%>
-
-<% content_for :tab_line_buttons do %>
-<span style="padding-left: 1em">Collection storage status:</span>
-<%= render partial: 'toggle_persist', locals: { uuid: @object.uuid, current_state: (@is_persistent ? 'persistent' : 'cache') } %>
-<% end %>
-
<% file_tree = @object.andand.files_tree %>
<% if file_tree.nil? or file_tree.empty? %>
<p>This collection is empty.</p>
-<% content_for :tab_line_buttons do %>
- <%= form_tag collections_path, method: 'get', remote: true, class: 'form-search' do %>
- <div class="input-group">
- <%= text_field_tag :search, params[:search], class: 'form-control', placeholder: 'Search collections' %>
- <span class="input-group-btn">
- <%= button_tag(class: 'btn btn-info') do %>
- <span class="glyphicon glyphicon-search"></span>
- <% end %>
- </span>
- </div>
- <% end %>
-<% end %>
-
<%= render partial: "paging", locals: {results: @collections, object: @object} %>
<div style="padding-right: 1em">
--- /dev/null
+<% content_for :tab_line_buttons do %>
+ <%= form_tag collections_path, method: 'get', remote: true, class: 'form-search' do %>
+ <div class="input-group">
+ <%= text_field_tag :search, params[:search], class: 'form-control', placeholder: 'Search collections' %>
+ <span class="input-group-btn">
+ <%= button_tag(class: 'btn btn-info') do %>
+ <span class="glyphicon glyphicon-search"></span>
+ <% end %>
+ </span>
+ </div>
+ <% end %>
+<% end %>
+
+<%= render file: 'application/index.html.erb', locals: local_assigns %>
</div>
</div>
-<%= render file: 'application/show.html.erb' %>
+<% content_for :tab_line_buttons do %>
+ <span style="padding-left: 1em">Collection storage status:</span>
+ <%= render partial: 'toggle_persist', locals: { uuid: @object.uuid, current_state: (@is_persistent ? 'persistent' : 'cache') } %>
+<% end %>
+
+<%= render file: 'application/show.html.erb', locals: local_assigns %>
<% end %>
<% end %>
<%= content_for :content_top %>
-<%= content_for :tab_line_buttons %>
+<div class="pull-right">
+ <%= content_for :tab_line_buttons %>
+</div>
<%= content_for :tab_panes %>
<%= current_user.email %>
</a>
<ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header">
+ My account
+ </li>
<% if current_user.is_active %>
<li role="presentation"><a href="/manage_account" role="menuitem"><i class="fa fa-key fa-fw"></i> Manage account</a></li>
+ <% if Rails.configuration.user_profile_form_fields %>
+ <li role="presentation"><a href="/users/<%=current_user.uuid%>/profile" role="menuitem"><i class="fa fa-key fa-fw"></i> Manage profile</a></li>
+ <% end %>
<li role="presentation" class="divider"></li>
<% end %>
<li role="presentation"><a href="<%= logout_path %>" role="menuitem"><i class="fa fa-sign-out fa-fw"></i> Log out</a></li>
-<%= content_for :tab_line_buttons do %>
-<%= form_tag({action: 'compare', controller: params[:controller], method: 'get'}, {method: 'get', id: 'compare', class: 'pull-right small-form-margin'}) do |f| %>
- <%= submit_tag 'Compare 2 or 3 selected', {class: 'btn btn-primary', disabled: true, style: 'display: none'} %>
-
-<% end rescue nil %>
-<% end %>
-
<%= render partial: "paging", locals: {results: @objects, object: @object} %>
<%= form_tag do |f| %>
--- /dev/null
+<% content_for :tab_line_buttons do %>
+<%= form_tag({action: 'compare', controller: params[:controller], method: 'get'}, {method: 'get', id: 'compare', class: 'pull-right small-form-margin'}) do |f| %>
+ <%= submit_tag 'Compare 2 or 3 selected', {class: 'btn btn-primary', disabled: true, style: 'display: none'} %>
+
+<% end rescue nil %>
+<% end %>
+
+<%= render file: 'application/index.html.erb', locals: local_assigns %>
-<% content_for :tab_line_buttons do %>
- <%= button_to(choose_projects_path(id: "run-pipeline-button",
- title: 'Choose project',
- editable: true,
- action_name: 'Choose',
- action_href: pipeline_instances_path,
- action_method: 'post',
- action_data: {selection_param: 'pipeline_instance[owner_uuid]',
- 'pipeline_instance[pipeline_template_uuid]' => @object.uuid,
- 'success' => 'redirect-to-created-object'
- }.to_json),
- { class: "btn btn-primary btn-sm", remote: true, method: 'get' }
- ) do %>
- Run this pipeline
- <% end %>
-<% end %>
-
<%= render_pipeline_components("editable", :json, editable: false) %>
--- /dev/null
+<% content_for :tab_line_buttons do %>
+ <%= button_to(choose_projects_path(id: "run-pipeline-button",
+ title: 'Choose project',
+ editable: true,
+ action_name: 'Choose',
+ action_href: pipeline_instances_path,
+ action_method: 'post',
+ action_data: {selection_param: 'pipeline_instance[owner_uuid]',
+ 'pipeline_instance[pipeline_template_uuid]' => @object.uuid,
+ 'success' => 'redirect-to-created-object'
+ }.to_json),
+ { class: "btn btn-primary btn-sm", remote: true, method: 'get' }
+ ) do %>
+ Run this pipeline
+ <% end %>
+<% end %>
+
+<%= render file: 'application/show.html.erb', locals: local_assigns %>
-<% objects_and_names.each do |object, name_link| %>
+<% get_objects_and_names.each do |object, name_link| %>
<% name_object = (object.respond_to?(:name) || !name_link) ? object : name_link %>
<tr class="filterable"
data-object-uuid="<%= name_object.uuid %>"
data-kind="<%= object.kind %>"
+ data-object-created-at="<%= object.created_at %>"
>
<td>
- <%= render partial: 'selection_checkbox', locals: {object: name_object, friendly_name: ((name_object.name rescue '') || '')} %>
+ <div style="width:1em; display:inline-block;">
+ <%= render partial: 'selection_checkbox', locals: {object: name_object, friendly_name: ((name_object.name rescue '') || '')} %>
+ </div>
- <% if project.editable? %>
- <%= link_to({action: 'remove_item', id: project.uuid, item_uuid: ((name_link && name_link.uuid) || object.uuid)}, method: :delete, remote: true, data: {confirm: "Remove #{object.class_for_display.downcase} #{name_object.name rescue object.uuid} from this project?", toggle: 'tooltip', placement: 'top'}, class: 'btn btn-sm btn-default btn-nodecorate', title: 'remove') do %>
+ <% if @object.editable? %>
+ <%= link_to({action: 'remove_item', id: @object.uuid, item_uuid: ((name_link && name_link.uuid) || object.uuid)}, method: :delete, remote: true, data: {confirm: "Remove #{object.class_for_display.downcase} #{name_object.name rescue object.uuid} from this project?", toggle: 'tooltip', placement: 'top'}, class: 'btn btn-sm btn-default btn-nodecorate', title: 'remove') do %>
<i class="fa fa-fw fa-trash-o"></i>
<% end %>
<% else %>
-<% if @object.uuid != current_user.uuid # Not the "Home" project %>
-<% content_for :content_top do %>
-
-<h2>
- <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => "New project" } %>
-</h2>
-
-<div class="arv-description-as-subtitle">
- <%= render_editable_attribute @object, 'description', nil, { 'data-emptytext' => "(No description provided)", 'data-toggle' => 'manual' } %>
-</div>
-
-<% end %>
-<% end %>
-
-<% content_for :tab_line_buttons do %>
- <% if @object.editable? %>
- <%= link_to(
- choose_collections_path(
- title: 'Add data to project:',
- multiple: true,
- action_name: 'Add',
- action_href: actions_path(id: @object.uuid),
- action_method: 'post',
- action_data: {selection_param: 'selection[]', copy_selections_into_project: @object.uuid, success: 'page-refresh'}.to_json),
- { class: "btn btn-primary btn-sm", remote: true, method: 'get', data: {'event-after-select' => 'page-refresh'} }) do %>
- <i class="fa fa-fw fa-plus"></i> Add data...
- <% end %>
- <%= link_to(
- choose_pipeline_templates_path(
- title: 'Choose a pipeline to run:',
- action_name: 'Next: choose inputs <i class="fa fa-fw fa-arrow-circle-right"></i>',
- action_href: pipeline_instances_path,
- action_method: 'post',
- action_data: {'selection_param' => 'pipeline_instance[pipeline_template_uuid]', 'pipeline_instance[owner_uuid]' => @object.uuid, 'success' => 'redirect-to-created-object'}.to_json),
- { class: "btn btn-primary btn-sm", remote: true, method: 'get' }) do %>
- <i class="fa fa-fw fa-gear"></i> Run a pipeline...
- <% end %>
- <%= link_to projects_path('project[owner_uuid]' => @object.uuid), method: 'post', class: 'btn btn-sm btn-primary' do %>
- <i class="fa fa-fw fa-plus"></i>
- Add a subproject
- <% end %>
- <% if @object.uuid != current_user.uuid # Not the "Home" project %>
- <%= link_to(
- choose_projects_path(
- title: 'Move this project to...',
- editable: true,
- my_root_selectable: true,
- action_name: 'Move',
- action_href: project_path(@object.uuid),
- action_method: 'put',
- action_data: {selection_param: 'project[owner_uuid]', success: 'page-refresh'}.to_json),
- { class: "btn btn-sm btn-primary arv-move-to-project", remote: true, method: 'get' }) do %>
- <i class="fa fa-fw fa-truck"></i> Move project...
- <% end %>
- <%= link_to(project_path(id: @object.uuid), method: 'delete', class: 'btn btn-sm btn-primary', data: {confirm: "Really delete project '#{@object.name}'?"}) do %>
- <i class="fa fa-fw fa-trash-o"></i> Delete project
- <% end %>
- <% end %>
- <% end %>
-<% end %>
-
-<%
- filters = [['uuid', 'is_a', "arvados#collection"]]
- @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
- objects_and_names = get_objects_and_names @objects
- page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Data_collections'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+ filters: [['uuid', 'is_a', "arvados#collection"]]
+ }.merge(local_assigns) %>
-<%
- filters = [['uuid', 'is_a', ["arvados#pipelineInstance","arvados#job"]]]
- @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
- objects_and_names = get_objects_and_names @objects
- page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Jobs_and_pipelines'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+ filters: [['uuid', 'is_a', ["arvados#job", "arvados#pipelineInstance"]]]
+ }.merge(local_assigns) %>
-<%
- filters = [['uuid', 'is_a', ["arvados#human","arvados#specimen","arvados#trait"]]]
- @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
- objects_and_names = get_objects_and_names @objects
- page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Other_objects'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+ filters: [['uuid', 'is_a', ["arvados#human", "arvados#specimen", "arvados#trait"]]]
+ }.merge(local_assigns) %>
-<%
- filters = [['uuid', 'is_a', "arvados#pipelineTemplate"]]
- @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
- objects_and_names = get_objects_and_names @objects
- page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Pipeline_templates'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+ filters: [['uuid', 'is_a', ["arvados#pipelineTemplate"]]]
+ }.merge(local_assigns) %>
<%
uuid_map = {}
- [@users, @groups].each do |obj_list|
- obj_list.each { |o| uuid_map[o.uuid] = o }
+ if @share_links
+ [User, Group].each do |type|
+ type.limit(10000)
+ .filter([['uuid','in',@share_links.collect(&:tail_uuid)]])
+ .each do |o|
+ uuid_map[o.uuid] = o
+ end
+ end
end
perm_name_desc_map = {}
perm_desc_name_map = {}
-<%
- filters = [['uuid', 'is_a', "arvados#group"]]
- @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
- objects_and_names = get_objects_and_names @objects
- page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Subprojects'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+ filters: [['uuid', 'is_a', ["arvados#group"]]]
+ }.merge(local_assigns) %>
<div class="row">
<div class="col-sm-5">
<div class="btn-group btn-group-sm">
- <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">Selection... <i class="fa fa-fw fa-long-arrow-down "></i></button>
+ <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">Selection... <span class="caret"></span></button>
<ul class="dropdown-menu" role="menu">
<li><%= link_to "Compare selected", '#',
'data-href' => compare_pipeline_instances_path,
</div>
</div>
<div class="col-sm-4 pull-right">
- <input type="text" class="form-control filterable-control" placeholder="Search project contents" data-filterable-target="table.arv-index.arv-project-<%= tab_name %> tbody"/>
+ <input type="text" class="form-control filterable-control" placeholder="Search project contents" data-filterable-target="table.arv-index.arv-project-<%= tab_pane %> tbody"/>
</div>
</div>
- <table class="table table-condensed table-fixedlayout arv-index arv-project-<%= tab_name %>" style="overflow-x: hidden">
+ <table class="table table-condensed table-fixedlayout arv-index arv-project-<%= tab_pane %>" style="overflow-x: hidden">
<colgroup>
<col width="40%" />
<col width="60%" />
</colgroup>
- <tbody data-infinite-scroller="#<%= tab_name %>-scroll" data-infinite-content-href="<%= url_for(format: :json, partial: :contents_rows, offset: page_offset, filters: "#{filters}") if page_offset %>">
- <%= render partial: 'show_contents_rows', locals: {project: @object, objects_and_names: objects_and_names} %>
+ <tbody data-infinite-scroller="#<%= tab_pane %>-scroll" data-infinite-content-href="<%= url_for partial: :contents_rows, filters: filters.to_json %>">
</tbody>
<thead>
<tr>
</tr>
</thead>
</table>
-
</div>
--- /dev/null
+<% if @object.uuid != current_user.uuid # Not the "Home" project %>
+<% content_for :content_top do %>
+
+<h2>
+ <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => "New project" } %>
+</h2>
+
+<div class="arv-description-as-subtitle">
+ <%= render_editable_attribute @object, 'description', nil, { 'data-emptytext' => "(No description provided)", 'data-toggle' => 'manual' } %>
+</div>
+
+<% end %>
+<% end %>
+
+<% content_for :tab_line_buttons do %>
+ <% if @object.editable? %>
+ <%= link_to(
+ choose_collections_path(
+ title: 'Add data to project:',
+ multiple: true,
+ action_name: 'Add',
+ action_href: actions_path(id: @object.uuid),
+ action_method: 'post',
+ action_data: {selection_param: 'selection[]', copy_selections_into_project: @object.uuid, success: 'page-refresh'}.to_json),
+ { class: "btn btn-primary btn-sm", remote: true, method: 'get', data: {'event-after-select' => 'page-refresh'} }) do %>
+ <i class="fa fa-fw fa-plus"></i> Add data...
+ <% end %>
+ <%= link_to(
+ choose_pipeline_templates_path(
+ title: 'Choose a pipeline to run:',
+ action_name: 'Next: choose inputs <i class="fa fa-fw fa-arrow-circle-right"></i>',
+ action_href: pipeline_instances_path,
+ action_method: 'post',
+ action_data: {'selection_param' => 'pipeline_instance[pipeline_template_uuid]', 'pipeline_instance[owner_uuid]' => @object.uuid, 'success' => 'redirect-to-created-object'}.to_json),
+ { class: "btn btn-primary btn-sm", remote: true, method: 'get' }) do %>
+ <i class="fa fa-fw fa-gear"></i> Run a pipeline...
+ <% end %>
+ <%= link_to projects_path('project[owner_uuid]' => @object.uuid), method: 'post', class: 'btn btn-sm btn-primary' do %>
+ <i class="fa fa-fw fa-plus"></i>
+ Add a subproject
+ <% end %>
+ <% if @object.uuid != current_user.uuid # Not the "Home" project %>
+ <%= link_to(
+ choose_projects_path(
+ title: 'Move this project to...',
+ editable: true,
+ my_root_selectable: true,
+ action_name: 'Move',
+ action_href: project_path(@object.uuid),
+ action_method: 'put',
+ action_data: {selection_param: 'project[owner_uuid]', success: 'page-refresh'}.to_json),
+ { class: "btn btn-sm btn-primary arv-move-to-project", remote: true, method: 'get' }) do %>
+ <i class="fa fa-fw fa-truck"></i> Move project...
+ <% end %>
+ <%= link_to(project_path(id: @object.uuid), method: 'delete', class: 'btn btn-sm btn-primary', data: {confirm: "Really delete project '#{@object.name}'?"}) do %>
+ <i class="fa fa-fw fa-trash-o"></i> Delete project
+ <% end %>
+ <% end %>
+ <% end %>
+<% end %>
+
+<%= render file: 'application/show.html.erb', locals: local_assigns %>
--- /dev/null
+<%
+ profile_config = Rails.configuration.user_profile_form_fields
+ current_user_profile = current_user.prefs[:profile]
+ show_save_button = false
+
+ profile_message = Rails.configuration.user_profile_form_message ? Rails.configuration.user_profile_form_message : 'You can manage your profile using this page. Any feilds in red are required and missing. Please fill in those fields before you can accesse Arvados Workbench.'
+
+ missing_required = missing_required_profile?
+
+ profile_url = '/users/'+current_user.uuid+'/profile'
+ target = request.url.partition('?target=')[-1]
+ target = request.url if target.empty?
+ return_to_url = (request.url.ends_with? profile_url) ? profile_url : profile_url+'?target='+target
+%>
+
+<div>
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h4 class="panel-title">
+ Profile
+ </h4>
+ </div>
+ <div class="panel-body">
+ <% if !missing_required && params.andand.keys.include?('target') %>
+ <div class="rounded" style="border-width: 1px; border-style: dotted; border-color: lightgray;">
+ <p style="margin: 8px;">Thank you for filling in your profile. If you are done updating your profile,
+ you can now access Arvados Workbench by clicking on this button.
+ <form action="<%=target%>">
+ <input style="margin-left: 8px;" class="btn btn-primary" type="submit" value="Access Arvados Workbench">
+ </form>
+ </p>
+ </div>
+ <% else %>
+ <div class="rounded" style="border-width: 1px; border-style: dotted; border-color: lightgray;">
+ <p style="margin: 8px;"> <%=raw(profile_message)%> </p>
+ </div>
+ <% end %>
+
+ <div class="rounded" style="border-width: 1px; border-style: dotted; border-color: lightgray;">
+ <%= form_tag "/users/#{current_user.uuid}", {method: 'patch', id: 'save_profile_form', name: 'save_profile_form', class: 'form-horizontal'} do %>
+ <%= hidden_field_tag :return_to, return_to_url %>
+ <div class="form-group">
+ <label for="email" class="col-sm-3 control-label"> Email </label>
+ <div class="col-sm-8">
+ <p class="form-control-static" id="email" name="email"><%=current_user.email%></p>
+ </div>
+ </div>
+ <div class="form-group">
+ <label for="first_name" class="col-sm-3 control-label"> First name </label>
+ <div class="col-sm-8">
+ <p class="form-control-static" id="first_name" name="first_name"><%=current_user.first_name%></p>
+ </div>
+ </div>
+ <div class="form-group">
+ <label for="last_name" class="col-sm-3 control-label"> Last name </label>
+ <div class="col-sm-8">
+ <p class="form-control-static" id="last_name" name="last_name"><%=current_user.last_name%></p>
+ </div>
+ </div>
+ <div class="form-group">
+ <label for="identity_url" class="col-sm-3 control-label"> Identity URL </label>
+ <div class="col-sm-8">
+ <p class="form-control-static" id="identity_url" name="identity_url"><%=current_user.andand.identity_url%></p>
+ </div>
+ </div>
+
+ <% profile_config.kind_of?(Array) && profile_config.andand.each do |entry| %>
+ <% if entry['key'] %>
+ <%
+ show_save_button = true
+ label = entry['required'] ? '* ' : ''
+ label += entry['form_field_title']
+ value = current_user_profile[entry['key'].to_sym] if current_user_profile
+ %>
+ <div class="form-group">
+ <label for="<%=entry['key']%>"
+ class="col-sm-3 control-label"
+ style=<%="color:red" if entry['required']&&(!value||value.empty?)%>> <%=label%>
+ </label>
+ <% if entry['type'] == 'select' %>
+ <div class="col-sm-8">
+ <select class="form-control" name="user[prefs][:profile][:<%=entry['key']%>]">
+ <% entry['options'].each do |option| %>
+ <option value="<%=option%>" <%='selected' if option==value%>><%=option%></option>
+ <% end %>
+ </select>
+ </div>
+ <% else %>
+ <div class="col-sm-8">
+ <input type="text" class="form-control" name="user[prefs][:profile][:<%=entry['key']%>]" placeholder="<%=entry['form_field_description']%>" value="<%=value%>" ></input>
+ </div>
+ <% end %>
+ </div>
+ <% end %>
+ <% end %>
+
+ <% if show_save_button %>
+ <div class="form-group">
+ <div class="col-sm-offset-3 col-sm-8">
+ <button type="submit" class="btn btn-primary">Save profile</button>
+ </div>
+ </div>
+ <% end %>
+ <% end %>
+ </div>
+ </div>
+ </div>
+</div>
site_name: Workbench:test
+ # Enable user profile with one required field
+ user_profile_form_fields:
+ - key: organization
+ type: text
+ form_field_title: Institution
+ form_field_description: Your organization
+ required: true
+ - key: role
+ type: select
+ form_field_title: Your role
+ form_field_description: Choose the category that best describes your role in your organization.
+ options:
+ - Bio-informatician
+ - Computational biologist
+ - Biologist or geneticist
+ - Software developer
+ - IT
+ - Other
+
common:
assets.js_compressor: false
assets.css_compressor: false
secret_key_base: false
default_openid_prefix: https://www.google.com/accounts/o8/id
send_user_setup_notification_email: true
+
+ # Set user_profile_form_fields to enable and configure the user profile page.
+ # Default is set to false. A commented setting with full description is provided below.
+ user_profile_form_fields: false
+
+ # Below is a sample setting of user_profile_form_fields config parameter.
+ # This configuration parameter should be set to either false (to disable) or
+ # to an array as shown below.
+ # Configure the list of input fields to be displayed in the profile page
+ # using the attribute "key" for each of the input fields.
+ # This sample shows configuration with one required and one optional form fields.
+ # For each of these input fields:
+ # You can specify "type" as "text" or "select".
+ # List the "options" to be displayed for each of the "select" menu.
+ # Set "required" as "true" for any of these fields to make them required.
+ # If any of the required fields are missing in the user's profile, the user will be
+ # redirected to the profile page before they can access any Workbench features.
+ #user_profile_form_fields:
+ # - key: organization
+ # type: text
+ # form_field_title: Institution/Company
+ # form_field_description: Your organization
+ # required: true
+ # - key: role
+ # type: select
+ # form_field_title: Your role
+ # form_field_description: Choose the category that best describes your role in your organization.
+ # options:
+ # - Bio-informatician
+ # - Computational biologist
+ # - Biologist or geneticist
+ # - Software developer
+ # - IT
+ # - Other
+
+ # Use "user_profile_form_message" to configure the message you want to display in
+ # the profile page. If this is not provided, a default message will be displayed.
+ user_profile_form_message: Welcome to Arvados. Please fill in all required fields before you can access Arvados Workbench. Missing required fields are in <span style="color:red">red</span>.
post 'sudo', :on => :member
post 'unsetup', :on => :member
get 'setup_popup', :on => :member
+ get 'profile', :on => :member
end
get '/manage_account' => 'users#manage_account'
get "/add_ssh_key_popup" => 'users#add_ssh_key_popup', :as => :add_ssh_key_popup
format: "json"},
session_for(:active))
assert_response :success
- json_response = Oj.load(@response.body)
assert_equal(uuid_list, json_response["success"])
end
format: "json"},
session_for(:project_viewer))
assert_response 422
- json_response = Oj.load(@response.body)
assert(json_response["errors"].andand.
any? { |msg| msg.start_with?("#{share_uuid}: ") },
"JSON response missing properly formatted sharing error")
test "viewer can't manage asubproject" do
refute user_can_manage(:project_viewer, "asubproject")
end
+
+ test 'projects#show tab infinite scroll partial obeys limit' do
+ get_contents_rows(limit: 1, filters: [['uuid','is_a',['arvados#job']]])
+ assert_response :success
+ assert_equal(1, json_response['content'].scan('<tr').count,
+ "Did not get exactly one row")
+ end
+
+ ['', ' asc', ' desc'].each do |direction|
+ test "projects#show tab partial orders correctly by #{direction}" do
+ _test_tab_content_order direction
+ end
+ end
+
+ def _test_tab_content_order direction
+ get_contents_rows(limit: 100,
+ order: "created_at#{direction}",
+ filters: [['uuid','is_a',['arvados#job',
+ 'arvados#pipelineInstance']]])
+ assert_response :success
+ not_grouped_by_kind = nil
+ last_timestamp = nil
+ last_kind = nil
+ found_kind = {}
+ json_response['content'].scan /<tr[^>]+>/ do |tr_tag|
+ found_timestamps = 0
+ tr_tag.scan(/\ data-object-created-at=\"(.*?)\"/).each do |t,|
+ if last_timestamp
+ correct_operator = / desc$/ =~ direction ? :>= : :<=
+ assert_operator(last_timestamp, correct_operator, t,
+ "Rows are not sorted by created_at#{direction}")
+ end
+ last_timestamp = t
+ found_timestamps += 1
+ end
+ assert_equal(1, found_timestamps,
+ "Content row did not have exactly one timestamp")
+
+ # Confirm that the test for timestamp ordering couldn't have
+ # passed merely because the test fixtures have convenient
+ # timestamps (e.g., there is only one pipeline and one job in
+ # the project being tested, or there are no pipelines at all in
+ # the project being tested):
+ tr_tag.scan /\ data-kind=\"(.*?)\"/ do |kind|
+ if last_kind and last_kind != kind and found_kind[kind]
+ # We saw this kind before, then a different kind, then
+ # this kind again. That means objects are not grouped by
+ # kind.
+ not_grouped_by_kind = true
+ end
+ found_kind[kind] ||= 0
+ found_kind[kind] += 1
+ last_kind = kind
+ end
+ end
+ assert_equal(true, not_grouped_by_kind,
+ "Could not confirm that results are not grouped by kind")
+ end
+
+ def get_contents_rows params
+ params = {
+ id: api_fixture('users')['active']['uuid'],
+ partial: :contents_rows,
+ format: :json,
+ }.merge(params)
+ encoded_params = Hash[params.map { |k,v|
+ [k, (v.is_a?(Array) || v.is_a?(Hash)) ? v.to_json : v]
+ }]
+ get :show, encoded_params, session_for(:active)
+ end
end
--- /dev/null
+require 'integration_helper'
+require 'selenium-webdriver'
+require 'headless'
+
+class ApplicationLayoutTest < ActionDispatch::IntegrationTest
+ setup do
+ headless = Headless.new
+ headless.start
+ Capybara.current_driver = :selenium
+
+ @user_profile_form_fields = Rails.configuration.user_profile_form_fields
+ end
+
+ teardown do
+ Rails.configuration.user_profile_form_fields = @user_profile_form_fields
+ end
+
+ def verify_homepage_with_profile user, invited, has_profile
+ profile_config = Rails.configuration.user_profile_form_fields
+
+ if !user
+ assert page.has_text? 'Please log in'
+ assert page.has_text? 'The "Log in" button below will show you a Google sign-in page'
+ assert page.has_no_text? 'My projects'
+ assert page.has_link? "Log in to #{Rails.configuration.site_name}"
+ elsif profile_config && !has_profile && user['is_active']
+ add_profile user
+ elsif user['is_active']
+ assert page.has_text? 'My projects'
+ assert page.has_text? 'Projects shared with me'
+ assert page.has_no_text? 'Save profile'
+ elsif invited
+ assert page.has_text? 'Please check the box below to indicate that you have read and accepted the user agreement'
+ assert page.has_no_text? 'Save profile'
+ else
+ assert page.has_text? 'Your account is inactive'
+ assert page.has_no_text? 'Save profile'
+ end
+
+ within('.navbar-fixed-top') do
+ if !user
+ assert page.has_link? 'Log in'
+ else
+ # my account menu
+ assert page.has_link? "#{user['email']}"
+ find('a', text: "#{user['email']}").click
+ within('.dropdown-menu') do
+ if user['is_active']
+ assert page.has_no_link? ('Not active')
+ assert page.has_no_link? ('Sign agreements')
+
+ assert page.has_link? ('Manage account')
+
+ if profile_config
+ assert page.has_link? ('Manage profile')
+ else
+ assert page.has_no_link? ('Manage profile')
+ end
+ end
+ assert page.has_link? ('Log out')
+ end
+ end
+ end
+ end
+
+ # test the help menu
+ def check_help_menu
+ within('.navbar-fixed-top') do
+ page.find("#arv-help").click
+ within('.dropdown-menu') do
+ assert page.has_link? 'Tutorials and User guide'
+ assert page.has_link? 'API Reference'
+ assert page.has_link? 'SDK Reference'
+ end
+ end
+ end
+
+ def verify_system_menu user
+ if user && user['is_active']
+ look_for_add_new = nil
+ within('.navbar-fixed-top') do
+ page.find("#system-menu").click
+ if user['is_admin']
+ within('.dropdown-menu') do
+ assert page.has_text? 'Groups'
+ assert page.has_link? 'Repositories'
+ assert page.has_link? 'Virtual machines'
+ assert page.has_link? 'SSH keys'
+ assert page.has_link? 'API tokens'
+ find('a', text: 'Users').click
+ look_for_add_new = 'Add a new user'
+ end
+ else
+ within('.dropdown-menu') do
+ assert page.has_no_text? 'Users'
+ assert page.has_no_link? 'Repositories'
+ assert page.has_no_link? 'Virtual machines'
+ assert page.has_no_link? 'SSH keys'
+ assert page.has_no_link? 'API tokens'
+
+ find('a', text: 'Groups').click
+ look_for_add_new = 'Add a new group'
+ end
+ end
+ end
+ if look_for_add_new
+ assert page.has_text? look_for_add_new
+ end
+ else
+ assert page.has_no_link? '#system-menu'
+ end
+ end
+
+ # test manage_account page
+ def verify_manage_account user
+ if user && user['is_active']
+ within('.navbar-fixed-top') do
+ find('a', text: "#{user['email']}").click
+ within('.dropdown-menu') do
+ find('a', text: 'Manage account').click
+ end
+ end
+
+ # now in manage account page
+ assert page.has_text? 'Virtual Machines'
+ assert page.has_text? 'Repositories'
+ assert page.has_text? 'SSH Keys'
+ assert page.has_text? 'Current Token'
+
+ assert page.has_text? 'The Arvados API token is a secret key that enables the Arvados SDKs to access Arvados'
+
+ click_link 'Add new SSH key'
+
+ within '.modal-content' do
+ assert page.has_text? 'Public Key'
+ assert page.has_button? 'Cancel'
+ assert page.has_button? 'Submit'
+
+ page.find_field('public_key').set 'first test with an incorrect ssh key value'
+ click_button 'Submit'
+ assert page.has_text? 'Public key does not appear to be a valid ssh-rsa or dsa public key'
+
+ public_key_str = api_fixture('authorized_keys')['active']['public_key']
+ page.find_field('public_key').set public_key_str
+ page.find_field('name').set 'added_in_test'
+ click_button 'Submit'
+ assert page.has_text? 'Public key already exists in the database, use a different key.'
+
+ new_key = SSHKey.generate
+ page.find_field('public_key').set new_key.ssh_public_key
+ page.find_field('name').set 'added_in_test'
+ click_button 'Submit'
+ end
+
+ # key must be added. look for it in the refreshed page
+ assert page.has_text? 'added_in_test'
+ end
+ end
+
+ # Check manage profile page and add missing profile to the user
+ def add_profile user
+ assert page.has_no_text? 'My projects'
+ assert page.has_no_text? 'Projects shared with me'
+
+ assert page.has_text? 'Profile'
+ assert page.has_text? 'First name'
+ assert page.has_text? 'Last name'
+ assert page.has_text? 'Identity URL'
+ assert page.has_text? 'Email'
+ assert page.has_text? user['email']
+
+ # Using the default profile which has message and one required field
+
+ # Save profile without filling in the required field. Expect to be back in this profile page again
+ click_button "Save profile"
+ assert page.has_text? 'Profile'
+ assert page.has_text? 'First name'
+ assert page.has_text? 'Last name'
+ assert page.has_text? 'Save profile'
+
+ # This time fill in required field and then save. Expect to go to requested page after that.
+ profile_message = Rails.configuration.user_profile_form_message
+ required_field_title = ''
+ required_field_key = ''
+ profile_config = Rails.configuration.user_profile_form_fields
+ profile_config.andand.each do |entry|
+ if entry['required']
+ required_field_key = entry['key']
+ required_field_title = entry['form_field_title']
+ end
+ end
+
+ assert page.has_text? profile_message[0,25]
+ assert page.has_text? required_field_title
+ page.find_field('user[prefs][:profile][:'+required_field_key+']').set 'value to fill required field'
+
+ click_button "Save profile"
+ # profile saved and in profile page now with success
+ assert page.has_text? 'Thank you for filling in your profile'
+ click_button 'Access Arvados Workbench'
+
+ # profile saved and in home page now
+ assert page.has_text? 'My projects'
+ assert page.has_text? 'Projects shared with me'
+ end
+
+ # test the search box
+ def verify_search_box user
+ if user && user['is_active']
+ # let's search for a valid uuid
+ within('.navbar-fixed-top') do
+ page.find_field('search').set user['uuid']
+ page.find('.glyphicon-search').click
+ end
+
+ # we should now be in the user's page as a result of search
+ assert page.has_text? user['first_name']
+
+ # let's search again for an invalid valid uuid
+ within('.navbar-fixed-top') do
+ search_for = String.new user['uuid']
+ search_for[0]='1'
+ page.find_field('search').set search_for
+ page.find('.glyphicon-search').click
+ end
+
+ # we should see 'not found' error page
+ assert page.has_text? 'Not Found'
+
+ # let's search for the anonymously accessible project
+ publicly_accessible_project = api_fixture('groups')['anonymously_accessible_project']
+
+ within('.navbar-fixed-top') do
+ # search again for the anonymously accessible project
+ page.find_field('search').set publicly_accessible_project['name'][0,10]
+ page.find('.glyphicon-search').click
+ end
+
+ within '.modal-content' do
+ assert page.has_text? 'All projects'
+ assert page.has_text? 'Search'
+ assert page.has_text? 'Cancel'
+ assert_selector('div', text: publicly_accessible_project['name'])
+ find(:xpath, '//div[./span[contains(.,publicly_accessible_project["uuid"])]]').click
+
+ click_button 'Show'
+ end
+
+ # seeing "Unrestricted public data" now
+ assert page.has_text? publicly_accessible_project['name']
+ assert page.has_text? publicly_accessible_project['description']
+ end
+ end
+
+ [
+ [nil, nil, false, false],
+ ['inactive', api_fixture('users')['inactive'], true, false],
+ ['inactive_uninvited', api_fixture('users')['inactive_uninvited'], false, false],
+ ['active', api_fixture('users')['active'], true, true],
+ ['admin', api_fixture('users')['admin'], true, true],
+ ['active_no_prefs', api_fixture('users')['active_no_prefs'], true, false],
+ ['active_no_prefs_profile', api_fixture('users')['active_no_prefs_profile'], true, false],
+ ].each do |token, user, invited, has_profile|
+ test "visit home page when profile is configured for user #{token}" do
+ # Our test config enabled profile by default. So, no need to update config
+ if !token
+ visit ('/')
+ else
+ visit page_with_token(token)
+ end
+
+ verify_homepage_with_profile user, invited, has_profile
+ end
+
+ test "visit home page when profile not configured for user #{token}" do
+ Rails.configuration.user_profile_form_fields = false
+
+ if !token
+ visit ('/')
+ else
+ visit page_with_token(token)
+ end
+
+ verify_homepage_with_profile user, invited, has_profile
+ end
+
+ test "check help for user #{token}" do
+ if !token
+ visit ('/')
+ else
+ visit page_with_token(token)
+ end
+
+ check_help_menu
+ end
+ end
+
+ [
+ ['active', api_fixture('users')['active'], true, true],
+ ['admin', api_fixture('users')['admin'], true, true],
+ ].each do |token, user|
+ test "test system menu for user #{token}" do
+ visit page_with_token(token)
+ verify_system_menu user
+ end
+
+ test "test manage account for user #{token}" do
+ visit page_with_token(token)
+ verify_manage_account user
+ end
+
+ test "test search for user #{token}" do
+ visit page_with_token(token)
+ verify_search_box user
+ end
+ end
+end
click
within('.modal-dialog') do
+ assert_selector 'button.dropdown-toggle', text: 'A Project'
first('span', text: 'foo_tag').click
find('button', text: 'OK').click
end
specimen_uuid = api_fixture('specimens')['owned_by_aproject_with_no_name_link']['uuid']
visit page_with_token 'active', '/projects/' + project_uuid
click_link 'Other objects'
- within(".selection-action-container") do
- within (first('tr', text: 'Specimen')) do
+ within '.selection-action-container' do
+ # Wait for the tab to load:
+ assert_selector 'tr[data-kind="arvados#specimen"]'
+ within first('tr', text: 'Specimen') do
find(".fa-pencil").click
find('.editable-input input').set('Now I have a name.')
find('.glyphicon-ok').click
- find('.editable', text: 'Now I have a name.').click
+ assert_selector '.editable', text: 'Now I have a name.'
find(".fa-pencil").click
find('.editable-input input').set('Now I have a new name.')
find('.glyphicon-ok').click
- end
+ end
wait_for_ajax
- find('.editable', text: 'Now I have a new name.')
+ assert_selector '.editable', text: 'Now I have a new name.'
end
visit current_path
click_link 'Other objects'
arvados_api_token: api_fixture('api_client_authorizations')[api_client_auth_name.to_s]['api_token']
}
end
+ def json_response
+ Oj.load(@response.body)
+ end
end
class ApiServerForTests
#!/usr/bin/env python
+# collection-merge
+#
+# Merge two or more collections together. Can also be used to extract specific
+# files from a collection to produce a new collection.
+#
+# input:
+# An array of collections or collection/file paths in script_parameter["input"]
+#
+# output:
+# A manifest with the collections merged. Duplicate file names will
+# have their contents concatenated in the order that they appear in the input
+# array.
+
import arvados
import md5
import subst
if fn in s.files():
merged += s.files()[fn].as_manifest()
-crm = arvados.CollectionReader(merged)
-
-combined = crm.manifest_text(strip=True)
-
-m = hashlib.new('md5')
-m.update(combined)
-
-uuid = "{}+{}".format(m.hexdigest(), len(combined))
-
-collection = arvados.api().collections().create(
- body={
- 'uuid': uuid,
- 'manifest_text': crm.manifest_text(),
- }).execute()
-
-for s in src:
- l = arvados.api().links().create(body={
- "link": {
- "tail_uuid": s,
- "head_uuid": uuid,
- "link_class": "provenance",
- "name": "provided"
- }}).execute()
-
-arvados.current_task().set_output(uuid)
+arvados.current_task().set_output(merged)
--- /dev/null
+#!/usr/bin/env python
+
+#
+# decompress-all.py
+#
+# Decompress all compressed files in the collection using the "dtrx" tool and
+# produce a new collection with the contents. Uncompressed files
+# are passed through.
+#
+# input:
+# A collection at script_parameters["input"]
+#
+# output:
+# A manifest of the uncompressed contents of the input collection.
+
+import arvados
+import re
+import subprocess
+import os
+import sys
+
+arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True,
+ input_as_path=True)
+
+task = arvados.current_task()
+
+input_file = task['parameters']['input']
+
+infile_parts = re.match(r"(^[a-f0-9]{32}\+\d+)(\+\S+)*(/.*)?(/[^/]+)$", input_file)
+
+outdir = os.path.join(task.tmpdir, "output")
+os.makedirs(outdir)
+os.chdir(outdir)
+
+if infile_parts == None:
+ print >>sys.stderr, "Failed to parse input filename '%s' as a Keep file\n" % input_file
+ sys.exit(1)
+
+cr = arvados.CollectionReader(infile_parts.group(1))
+streamname = infile_parts.group(3)[1:]
+filename = infile_parts.group(4)[1:]
+
+if streamname != None:
+ subprocess.call(["mkdir", "-p", streamname])
+ os.chdir(streamname)
+else:
+ streamname = '.'
+
+m = re.match(r'.*\.(gz|Z|bz2|tgz|tbz|zip|rar|7z|cab|deb|rpm|cpio|gem)$', arvados.get_task_param_mount('input'), re.IGNORECASE)
+
+if m != None:
+ rc = subprocess.call(["dtrx", "-r", "-n", "-q", arvados.get_task_param_mount('input')])
+ if rc == 0:
+ out = arvados.CollectionWriter()
+ out.write_directory_tree(outdir, max_manifest_depth=0)
+ task.set_output(out.finish())
+ else:
+ sys.exit(rc)
+else:
+ streamreader = filter(lambda s: s.name() == streamname, cr.all_streams())[0]
+ filereader = streamreader.files()[filename]
+ task.set_output(streamname + filereader.as_manifest()[1:])
import time
import arvados.commands.put as put
import signal
+import stat
+import copy
+import traceback
+import pprint
+import multiprocessing
+import logging
os.umask(0077)
+logging.basicConfig(format="run-command: %(message)s")
t = arvados.current_task().tmpdir
os.chdir("output")
+outdir = os.getcwd()
+
+taskp = None
+jobp = arvados.current_job()['script_parameters']
if len(arvados.current_task()['parameters']) > 0:
- p = arvados.current_task()['parameters']
-else:
- p = arvados.current_job()['script_parameters']
+ taskp = arvados.current_task()['parameters']
links = []
def sub_link(v):
- r = os.path.basename(v)
- os.symlink(os.path.join(os.environ['TASK_KEEPMOUNT'], v) , r)
+ r = os.path.join(outdir, os.path.basename(v))
+ os.symlink(v, r)
links.append(r)
return r
def sub_tmpdir(v):
return os.path.join(arvados.current_task().tmpdir, 'tmpdir')
+def sub_outdir(v):
+ return outdir
+
def sub_cores(v):
- return os.environ['CRUNCH_NODE_SLOTS']
+ return str(multiprocessing.cpu_count())
+
+def sub_jobid(v):
+ return os.environ['JOB_UUID']
+
+def sub_taskid(v):
+ return os.environ['TASK_UUID']
+
+def sub_jobsrc(v):
+ return os.environ['CRUNCH_SRC']
subst.default_subs["link "] = sub_link
-subst.default_subs["tmpdir"] = sub_tmpdir
+subst.default_subs["task.tmpdir"] = sub_tmpdir
+subst.default_subs["task.outdir"] = sub_outdir
+subst.default_subs["job.srcdir"] = sub_jobsrc
subst.default_subs["node.cores"] = sub_cores
-
-rcode = 1
+subst.default_subs["job.uuid"] = sub_jobid
+subst.default_subs["task.uuid"] = sub_taskid
def machine_progress(bytes_written, bytes_expected):
return "run-command: wrote {} total {}\n".format(
sp.send_signal(signum)
self.sig = signum
+def expand_item(p, c):
+ if isinstance(c, dict):
+ if "foreach" in c and "command" in c:
+ var = c["foreach"]
+ items = get_items(p, p[var])
+ r = []
+ for i in items:
+ params = copy.copy(p)
+ params[var] = i
+ r.extend(expand_list(params, c["command"]))
+ return r
+ elif isinstance(c, list):
+ return expand_list(p, c)
+ elif isinstance(c, str) or isinstance(c, unicode):
+ return [subst.do_substitution(p, c)]
+
+ return []
+
+def expand_list(p, l):
+ return [exp for arg in l for exp in expand_item(p, arg)]
+
+def get_items(p, value):
+ if isinstance(value, list):
+ return expand_list(p, value)
+
+ fn = subst.do_substitution(p, value)
+ mode = os.stat(fn).st_mode
+ prefix = fn[len(os.environ['TASK_KEEPMOUNT'])+1:]
+ if mode != None:
+ if stat.S_ISDIR(mode):
+ items = ["$(dir %s/%s/)" % (prefix, l) for l in os.listdir(fn)]
+ elif stat.S_ISREG(mode):
+ with open(fn) as f:
+ items = [line for line in f]
+ return items
+ else:
+ return None
+
+stdoutname = None
+stdoutfile = None
+rcode = 1
+
try:
- cmd = []
- for c in p["command"]:
- cmd.append(subst.do_substitution(p, c))
-
- stdoutname = None
- stdoutfile = None
- if "stdout" in p:
- stdoutname = subst.do_substitution(p, p["stdout"])
+ if "task.foreach" in jobp:
+ if arvados.current_task()['sequence'] == 0:
+ var = jobp["task.foreach"]
+ items = get_items(jobp, jobp[var])
+ logging.info("parallelizing on %s with items %s" % (var, items))
+ if items != None:
+ for i in items:
+ params = copy.copy(jobp)
+ params[var] = i
+ arvados.api().job_tasks().create(body={
+ 'job_uuid': arvados.current_job()['uuid'],
+ 'created_by_job_task_uuid': arvados.current_task()['uuid'],
+ 'sequence': 1,
+ 'parameters': params
+ }
+ ).execute()
+ arvados.current_task().set_output(None)
+ sys.exit(0)
+ else:
+ sys.exit(1)
+ else:
+ taskp = jobp
+
+ cmd = expand_list(taskp, taskp["command"])
+
+ if "save.stdout" in taskp:
+ stdoutname = subst.do_substitution(taskp, taskp["save.stdout"])
stdoutfile = open(stdoutname, "wb")
- print("run-command: {}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+ logging.info("{}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+
+except Exception as e:
+ logging.exception("caught exception")
+ logging.error("task parameters was:")
+ logging.error(pprint.pformat(taskp))
+ sys.exit(1)
+try:
sp = subprocess.Popen(cmd, shell=False, stdout=stdoutfile)
sig = SigHandler()
rcode = sp.wait()
if sig.sig != None:
- print("run-command: terminating on signal %s" % sig.sig)
+ logging.critical("terminating on signal %s" % sig.sig)
sys.exit(2)
else:
- print("run-command: completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
+ logging.info("completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
except Exception as e:
- print("run-command: caught exception: {}".format(e))
+ logging.exception("caught exception")
# restore default signal handlers.
signal.signal(signal.SIGINT, signal.SIG_DFL)
for l in links:
os.unlink(l)
-print("run-command: the follow output files will be saved to keep:")
+logging.info("the following output files will be saved to keep:")
-subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"])
+subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"], stdout=sys.stderr)
-print("run-command: start writing output to keep")
+logging.info("start writing output to keep")
done = False
resume_cache = put.ResumeCache(os.path.join(arvados.current_task().tmpdir, "upload-output-checkpoint"))
}).execute()
done = True
except KeyboardInterrupt:
- print("run-command: terminating on signal 2")
+ logging.critical("terminating on signal 2")
sys.exit(2)
except Exception as e:
- print("run-command: caught exception: {}".format(e))
+ logging.exception("caught exception:")
time.sleep(5)
sys.exit(rcode)
--- /dev/null
+#!/usr/bin/python
+
+import arvados
+import re
+import hashlib
+import string
+
+api = arvados.api('v1')
+
+piece = 0
+manifest_text = ""
+
+# Look for paired reads
+
+inp = arvados.CollectionReader(arvados.getjobparam('reads'))
+
+manifest_list = []
+
+chunking = False #arvados.getjobparam('chunking')
+
+def nextline(reader, start):
+ n = -1
+ while True:
+ r = reader.readfrom(start, 128)
+ if r == '':
+ break
+ n = string.find(r, "\n")
+ if n > -1:
+ break
+ else:
+ start += 128
+ return n
+
+# Chunk a fastq into approximately 64 MiB chunks. Requires that the input data
+# be decompressed ahead of time, such as using decompress-all.py. Generates a
+# new manifest, but doesn't actually move any data around. Handles paired
+# reads by ensuring that each chunk of a pair gets the same number of records.
+#
+# This works, but in practice is so slow that potential gains in alignment
+# performance are lost in the prep time, which is why it is currently disabled.
+#
+# A better algorithm would seek to a file position a bit less than the desired
+# chunk size and then scan ahead for the next record, making sure that record
+# was matched by the read pair.
+def splitfastq(p):
+ for i in xrange(0, len(p)):
+ p[i]["start"] = 0
+ p[i]["end"] = 0
+
+ count = 0
+ recordsize = [0, 0]
+
+ global piece
+ finish = False
+ while not finish:
+ for i in xrange(0, len(p)):
+ recordsize[i] = 0
+
+ # read next 4 lines
+ for i in xrange(0, len(p)):
+ for ln in xrange(0, 4):
+ r = nextline(p[i]["reader"], p[i]["end"]+recordsize[i])
+ if r == -1:
+ finish = True
+ break
+ recordsize[i] += (r+1)
+
+ splitnow = finish
+ for i in xrange(0, len(p)):
+ if ((p[i]["end"] - p[i]["start"]) + recordsize[i]) >= (64*1024*1024):
+ splitnow = True
+
+ if splitnow:
+ for i in xrange(0, len(p)):
+ global manifest_list
+ print >>sys.stderr, "Finish piece ./_%s/%s (%s %s)" % (piece, p[i]["reader"].name(), p[i]["start"], p[i]["end"])
+ manifest = []
+ manifest.extend(["./_" + str(piece)])
+ manifest.extend([d[arvados.LOCATOR] for d in p[i]["reader"]._stream._data_locators])
+ manifest.extend(["{}:{}:{}".format(seg[arvados.LOCATOR]+seg[arvados.OFFSET], seg[arvados.SEGMENTSIZE], p[i]["reader"].name().replace(' ', '\\040')) for seg in arvados.locators_and_ranges(p[i]["reader"].segments, p[i]["start"], p[i]["end"] - p[i]["start"])])
+ manifest_list.append(manifest)
+ p[i]["start"] = p[i]["end"]
+ piece += 1
+ else:
+ for i in xrange(0, len(p)):
+ p[i]["end"] += recordsize[i]
+ count += 1
+ if count % 10000 == 0:
+ print >>sys.stderr, "Record %s at %s" % (count, p[i]["end"])
+
+prog = re.compile(r'(.*?)(_[12])?\.fastq(\.gz)?$')
+
+# Look for fastq files
+for s in inp.all_streams():
+ for f in s.all_files():
+ name_pieces = prog.match(f.name())
+ if name_pieces != None:
+ if s.name() != ".":
+ # The downstream tool (run-command) only iterates over the top
+ # level of directories so if there are fastq files in
+ # directories in the input, the choice is either to forget
+ # there are directories (which might lead to name conflicts) or
+ # just fail.
+ print >>sys.stderr, "fastq must be at the root of the collection"
+ sys.exit(1)
+
+ p = None
+ if name_pieces.group(2) != None:
+ if name_pieces.group(2) == "_1":
+ p = [{}, {}]
+ p[0]["reader"] = s.files()[name_pieces.group(0)]
+ p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + (name_pieces.group(3) if name_pieces.group(3) else '')]
+ else:
+ p = [{}]
+ p[0]["reader"] = s.files()[name_pieces.group(0)]
+
+ if p != None:
+ if chunking:
+ splitfastq(p)
+ else:
+ for i in xrange(0, len(p)):
+ m = p[i]["reader"].as_manifest()[1:]
+ manifest_list.append(["./_" + str(piece), m[:-1]])
+ piece += 1
+
+manifest_text = "\n".join(" ".join(m) for m in manifest_list)
+
+arvados.current_task().set_output(manifest_text)
return os.path.splitext(os.path.basename(v))[0]
def sub_glob(v):
- return glob.glob(v)[0]
+ l = glob.glob(v)
+ if len(l) == 0:
+ raise Exception("$(glob): No match on '%s'" % v)
+ else:
+ return l[0]
default_subs = {"file ": sub_file,
"dir ": sub_dir,
- user/tutorials/intro-crunch.html.textile.liquid
- user/tutorials/running-external-program.html.textile.liquid
- user/tutorials/tutorial-firstscript.html.textile.liquid
- - user/topics/tutorial-job-debug.html.textile.liquid
+ - user/tutorials/tutorial-submit-job.html.textile.liquid
- user/topics/tutorial-parallel.html.textile.liquid
- user/examples/crunch-examples.html.textile.liquid
- Query the metadata database:
|limit|integer (default 100)|Maximum number of collections to return.|query||
|order|string|Order in which to return matching collections.|query||
|filters|array|Conditions for filtering collections.|query||
+|select|array|Data fields to return in the result list.|query|@["uuid", "manifest_text"]@|
+
+N.B.: Because adding access tokens to manifests can be computationally expensive, the @manifest_text@ field is not included in results by default. If you need it, pass a @select@ parameter that includes @manifest_text@.
h2. update
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: "Debugging a Crunch script"
-...
-
-To test changes to a script by running a job, the change must be pushed to your hosted repository, and the job might have to wait in the queue before it runs. This cycle can be an inefficient way to develop and debug scripts. This tutorial demonstrates an alternative: using @arv-crunch-job@ to run your job in your local VM. This avoids the job queue and allows you to execute the script directly from your git working tree without committing or pushing.
-
-{% include 'tutorial_expectations' %}
-
-This tutorial uses @$USER@ to denote your username. Replace @$USER@ with your user name in all the following examples.
-
-h2. Create a new script
-
-Change to your Git working directory and create a new script in @crunch_scripts/@.
-
-<notextile>
-<pre><code>~$ <span class="userinput">cd $USER/crunch_scripts</span>
-~/$USER/crunch_scripts$ <span class="userinput">cat >hello-world.py <<EOF
-#!/usr/bin/env python
-
-print "hello world"
-print "this script will fail, and that is expected!"
-EOF</span>
-~/$USER/crunch_scripts$ <span class="userinput">chmod +x hello-world.py</span>
-</code></pre>
-</notextile>
-
-h2. Using arv-crunch-job to run the job in your VM
-
-Instead of a Git commit hash, we provide the path to the directory in the "script_version" parameter. The script specified in "script" is expected to be in the @crunch_scripts/@ subdirectory of the directory specified "script_version". Although we are running the script locally, the script still requires access to the Arvados API server and Keep storage service. The job will be recorded in the Arvados job history, and visible in Workbench.
-
-<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
-{
- "repository":"",
- "script":"hello-world.py",
- "script_version":"$HOME/$USER",
- "script_parameters":{}
-}
-EOF</span>
-</code></pre>
-</notextile>
-
-Your shell should fill in values for @$HOME@ and @$USER@ so that the saved JSON points "script_version" at the directory with your checkout. Now you can run that job:
-
-<notextile>
-<pre><code>~/$USER/crunch_scripts</span>$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 check slurm allocation
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 node localhost - 1 slots
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 start
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script hello-world.py
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script_version /home/$USER/$USER
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script_parameters {}
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 runtime_constraints {"max_tasks_per_node":0}
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 start level 0
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 status: 0 done, 0 running, 1 todo
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 job_task qr1hi-ot0gb-4zdajby8cjmlguh
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 child 29834 started on localhost.1
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 status: 0 done, 1 running, 0 todo
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 stderr hello world
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 stderr this script will fail, and that is expected!
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 child 29834 on localhost.1 exit 0 signal 0 success=
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 failure (#1, permanent) after 0 seconds
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 output
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 Every node has failed -- giving up on this round
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 wait for last 0 children to finish
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 status: 0 done, 0 running, 0 todo
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 Freeze not implemented
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 collate
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 output d41d8cd98f00b204e9800998ecf8427e+0
-2013-12-12_21:36:44 qr1hi-8i9sb-okzukfzkpbrnhst 29827 meta key is c00bfbd58e6f58ce3bebdd47f745a70f+1857
-</code></pre>
-</notextile>
-
-These are the lines of interest:
-
-bc. 2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 stderr hello world
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 stderr this script will fail, and that is expected!
-
-The script's output is captured in the log, which is useful for print statement debugging. However, although this script returned a status code of 0 (success), the job failed. Why? For a job to complete successfully scripts must explicitly add their output to Keep, and then tell Arvados about it. Here is a second try:
-
-<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">cat >hello-world-fixed.py <<EOF
-#!/usr/bin/env python
-
-import arvados
-
-# Create a new collection
-out = arvados.CollectionWriter()
-
-# Set the name of the file in the collection to write to
-out.set_current_file_name('hello.txt')
-
-# Actually output our text
-out.write('hello world')
-
-# Commit the collection to Keep
-out_collection = out.finish()
-
-# Tell Arvados which Keep object is our output
-arvados.current_task().set_output(out_collection)
-
-# Done!
-EOF</span>
-~/$USER/crunch_scripts$ <span class="userinput">chmod +x hello-world-fixed.py</span>
-~/$USER/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
-{
- "repository":"",
- "script":"hello-world-fixed.py",
- "script_version":"$HOME/$USER",
- "script_parameters":{}
-}
-EOF</span>
-~/$USER/crunch_scripts$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
-2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 check slurm allocation
-2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 node localhost - 1 slots
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 start
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world-fixed.py
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_version /home/$USER/$USER
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_parameters {}
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 runtime_constraints {"max_tasks_per_node":0}
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 start level 0
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 status: 0 done, 0 running, 1 todo
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 0 job_task qr1hi-ot0gb-u8g594ct0wt7f3f
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 0 child 31585 started on localhost.1
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 status: 0 done, 1 running, 0 todo
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 0 child 31585 on localhost.1 exit 0 signal 0 success=true
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 0 success in 1 seconds
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 0 output 576c44d762ba241b0a674aa43152b52a+53
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 wait for last 0 children to finish
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 status: 1 done, 0 running, 0 todo
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 Freeze not implemented
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 collate
-2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 output 576c44d762ba241b0a674aa43152b52a+53
-WARNING:root:API lookup failed for collection 576c44d762ba241b0a674aa43152b52a+53 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://qr1hi.arvadosapi.com/arvados/v1/collections/576c44d762ba241b0a674aa43152b52a%2B53?alt=json returned "Not Found">)
-2013-12-12_21:57:03 qr1hi-8i9sb-79260ykfew5trzl 31578 finish
-</code></pre>
-</notextile>
-
-(The WARNING issued near the end of the script may be safely ignored here; it is the Arvados SDK letting you know that it could not find a collection named @576c44d762ba241b0a674aa43152b52a+53@ and that it is going to try looking up a block by that name instead.)
-
-The job succeeded, with output in Keep object @576c44d762ba241b0a674aa43152b52a+53@. Let's look at our output:
-
-<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">arv keep get 576c44d762ba241b0a674aa43152b52a+53/hello.txt</span>
-hello world
-</code></pre>
-</notextile>
-
-h3. Location of temporary files
-
-Crunch job tasks are supplied with @TASK_WORK@ and @JOB_WORK@ environment variables, to be used as scratch space. When running in local development mode using @arv-crunch-job@, Crunch sets these variables to point to directory called @crunch-job-{USERID}@ in @TMPDIR@ (or @/tmp@ if @TMPDIR@ is not set).
-
-* Set @TMPDIR@ to @/scratch@ to make Crunch use a directory like @/scratch/crunch-job-{USERID}/@ for temporary space.
-
-* Set @CRUNCH_TMP@ to @/scratch/foo@ to make Crunch use @/scratch/foo/@ for temporary space (omitting the default @crunch-job-{USERID}@ leaf name)
-
-h3. Testing job scripts without SDKs and Keep access
-
-Read and write data to @/tmp/@ instead of Keep. This only works with the Python SDK.
-
-notextile. <pre><code>~$ <span class="userinput">export KEEP_LOCAL_STORE=/tmp</span></code></pre>
---
layout: default
navsection: userguide
-title: "Parallel Crunch tasks"
+title: "Concurrent Crunch tasks"
...
-In the previous tutorials, we used @arvados.job_setup.one_task_per_input_file()@ to automatically parallelize our jobs by creating a separate task per file. For some types of jobs, you may need to split the work up differently, for example creating tasks to process different segments of a single large file. In this this tutorial will demonstrate how to create Crunch tasks directly.
+In the previous tutorials, we used @arvados.job_setup.one_task_per_input_file()@ to automatically create concurrent jobs by creating a separate task per file. For some types of jobs, you may need to split the work up differently, for example creating tasks to process different segments of a single large file. In this this tutorial will demonstrate how to create Crunch tasks directly.
Start by entering the @crunch_scripts@ directory of your Git repository:
</code></pre>
</notextile>
-Next, using @nano@ or your favorite Unix text editor, create a new file called @parallel-hash.py@ in the @crunch_scripts@ directory.
+Next, using @nano@ or your favorite Unix text editor, create a new file called @concurrent-hash.py@ in the @crunch_scripts@ directory.
-notextile. <pre>~/$USER/crunch_scripts$ <code class="userinput">nano parallel-hash.py</code></pre>
+notextile. <pre>~/$USER/crunch_scripts$ <code class="userinput">nano concurrent-hash.py</code></pre>
Add the following code to compute the MD5 hash of each file in a collection:
-<notextile> {% code 'parallel_hash_script_py' as python %} </notextile>
+<notextile> {% code 'concurrent_hash_script_py' as python %} </notextile>
Make the file executable:
-notextile. <pre><code>~/$USER/crunch_scripts$ <span class="userinput">chmod +x parallel-hash.py</span></code></pre>
+notextile. <pre><code>~/$USER/crunch_scripts$ <span class="userinput">chmod +x concurrent-hash.py</span></code></pre>
Add the file to the Git staging area, commit, and push:
<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">git add parallel-hash.py</span>
-~/$USER/crunch_scripts$ <span class="userinput">git commit -m"parallel hash"</span>
+<pre><code>~/$USER/crunch_scripts$ <span class="userinput">git add concurrent-hash.py</span>
+~/$USER/crunch_scripts$ <span class="userinput">git commit -m"concurrent hash"</span>
~/$USER/crunch_scripts$ <span class="userinput">git push origin master</span>
</code></pre>
</notextile>
-You should now be able to run your new script using Crunch, with "script" referring to our new "parallel-hash.py" script. We will use a different input from our previous examples. We will use @887cd41e9c613463eab2f0d885c6dd96+83@ which consists of three files, "alice.txt", "bob.txt" and "carol.txt" (the example collection used previously in "fetching data from Arvados using Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html#dir).
+You should now be able to run your new script using Crunch, with "script" referring to our new "concurrent-hash.py" script. We will use a different input from our previous examples. We will use @887cd41e9c613463eab2f0d885c6dd96+83@ which consists of three files, "alice.txt", "bob.txt" and "carol.txt" (the example collection used previously in "fetching data from Arvados using Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html#dir).
<notextile>
<pre><code>~/$USER/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
- "script": "parallel-hash.py",
+ "script": "concurrent-hash.py",
"repository": "$USER",
"script_version": "master",
"script_parameters":
(Your shell should automatically fill in @$USER@ with your login name. The job JSON that gets saved should have @"repository"@ pointed at your personal Git repository.)
-Because the job ran in parallel, each instance of parallel-hash creates a separate @md5sum.txt@ as output. Arvados automatically collates theses files into a single collection, which is the output of the job:
+Because the job ran in concurrent, each instance of concurrent-hash creates a separate @md5sum.txt@ as output. Arvados automatically collates theses files into a single collection, which is the output of the job:
<notextile>
<pre><code>~/$USER/crunch_scripts$ <span class="userinput">arv keep ls e2ccd204bca37c77c0ba59fc470cd0f7+162</span>
These PGP IDs let us find public profiles, for example:
-* "https://my.personalgenomes.org/profile/huE2E371":https://my.personalgenomes.org/profile/huE2E371
-* "https://my.personalgenomes.org/profile/huDF04CC":https://my.personalgenomes.org/profile/huDF04CC
+* "https://my.pgp-hms.org/profile/huE2E371":https://my.pgp-hms.org/profile/huE2E371
+* "https://my.pgp-hms.org/profile/huDF04CC":https://my.pgp-hms.org/profile/huDF04CC
* ...
h2. Find genomic data from specific humans
-Now we want to find collections in Keep that were provided by these humans. We search the "links" resource for "provenance" links that point to subjects in list of humans with the non-melanoma skin cancer trait:
+Now we want to find collections in Keep that were provided by these humans. We search the "links" resource for "provenance" links that point to entries in the list of humans with the non-melanoma skin cancer trait:
<notextile>
<pre><code>>>> <span class="userinput">provenance_links = arvados.api().links().list(limit=1000, filters=[
# print PGP public profile links with file locators
for c in collections['items']:
for f in c['files']:
- print "https://my.personalgenomes.org/profile/%s %s %s%s" % (pgpid[c['uuid']], c['uuid'], ('' if f[0] == '.' else f[0]+'/'), f[1])
+ print "https://my.pgp-hms.org/profile/%s %s %s%s" % (pgpid[c['uuid']], c['uuid'], ('' if f[0] == '.' else f[0]+'/'), f[1])
</span>
-https://my.personalgenomes.org/profile/hu43860C a58dca7609fa84c8c38a7e926a97b2fc var-GS00253-DNA_A01_200_37-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/huB1FD55 ea30eb9e46eedf7f05ed6e348c2baf5d var-GS000010320-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/huDF04CC 4ab0df8f22f595d1747a22c476c05873 var-GS000010427-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu7A2F1D 756d0ada29b376140f64e7abfe6aa0e7 var-GS000014566-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu553620 7ed4e425bb1c7cc18387cbd9388181df var-GS000015272-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/huD09534 542112e210daff30dd3cfea4801a9f2f var-GS000016374-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu599905 33a9f3842b01ea3fdf27cc582f5ea2af var-GS000016015-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu43860C a58dca7609fa84c8c38a7e926a97b2fc+302 var-GS00253-DNA_A01_200_37-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/huB1FD55 ea30eb9e46eedf7f05ed6e348c2baf5d+291 var-GS000010320-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/huDF04CC 4ab0df8f22f595d1747a22c476c05873+242 var-GS000010427-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu7A2F1D 756d0ada29b376140f64e7abfe6aa0e7+242 var-GS000014566-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu553620 7ed4e425bb1c7cc18387cbd9388181df+242 var-GS000015272-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/huD09534 542112e210daff30dd3cfea4801a9f2f+242 var-GS000016374-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu599905 33a9f3842b01ea3fdf27cc582f5ea2af+242 var-GS000016015-ASM.tsv.bz2
-https://my.personalgenomes.org/profile/hu599905 d6e2e57cd60ba5979006d0b03e45e726+81 Witch_results.zip
-https://my.personalgenomes.org/profile/hu553620 ea4f2d325592a1272f989d141a917fdd+85 Devenwood_results.zip
-https://my.personalgenomes.org/profile/hu7A2F1D 4580f6620bb15b25b18373766e14e4a7+85 Innkeeper_results.zip
-https://my.personalgenomes.org/profile/huD09534 fee37be9440b912eb90f5e779f272416+82 Hallet_results.zip
+https://my.pgp-hms.org/profile/hu43860C a58dca7609fa84c8c38a7e926a97b2fc var-GS00253-DNA_A01_200_37-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/huB1FD55 ea30eb9e46eedf7f05ed6e348c2baf5d var-GS000010320-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/huDF04CC 4ab0df8f22f595d1747a22c476c05873 var-GS000010427-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu7A2F1D 756d0ada29b376140f64e7abfe6aa0e7 var-GS000014566-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu553620 7ed4e425bb1c7cc18387cbd9388181df var-GS000015272-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/huD09534 542112e210daff30dd3cfea4801a9f2f var-GS000016374-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu599905 33a9f3842b01ea3fdf27cc582f5ea2af var-GS000016015-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu43860C a58dca7609fa84c8c38a7e926a97b2fc+302 var-GS00253-DNA_A01_200_37-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/huB1FD55 ea30eb9e46eedf7f05ed6e348c2baf5d+291 var-GS000010320-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/huDF04CC 4ab0df8f22f595d1747a22c476c05873+242 var-GS000010427-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu7A2F1D 756d0ada29b376140f64e7abfe6aa0e7+242 var-GS000014566-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu553620 7ed4e425bb1c7cc18387cbd9388181df+242 var-GS000015272-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/huD09534 542112e210daff30dd3cfea4801a9f2f+242 var-GS000016374-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu599905 33a9f3842b01ea3fdf27cc582f5ea2af+242 var-GS000016015-ASM.tsv.bz2
+https://my.pgp-hms.org/profile/hu599905 d6e2e57cd60ba5979006d0b03e45e726+81 Witch_results.zip
+https://my.pgp-hms.org/profile/hu553620 ea4f2d325592a1272f989d141a917fdd+85 Devenwood_results.zip
+https://my.pgp-hms.org/profile/hu7A2F1D 4580f6620bb15b25b18373766e14e4a7+85 Innkeeper_results.zip
+https://my.pgp-hms.org/profile/huD09534 fee37be9440b912eb90f5e779f272416+82 Hallet_results.zip
</code></pre>
</notextile>
title: "Writing a Crunch script"
...
-This tutorial demonstrates how to create a new Arvados pipeline using the Arvados Python SDK. The Arvados SDK supports access to advanced features not available using the @run-command@ wrapper, such as scheduling parallel tasks across nodes.
+This tutorial demonstrates how to write a script using Arvados Python SDK. The Arvados SDK supports access to advanced features not available using the @run-command@ wrapper, such as scheduling concurrent tasks across nodes.
{% include 'tutorial_expectations' %}
This tutorial uses @$USER@ to denote your username. Replace @$USER@ with your user name in all the following examples.
-h2. Setting up Git
-
-All Crunch scripts are managed through the Git revision control system. Before you start using Git, you should do some basic configuration (you only need to do this the first time):
+Start by creating a directory called @$USER@ . Next, create a subdirectory called @crunch_scripts@ and change to that directory:
<notextile>
-<pre><code>~$ <span class="userinput">git config --global user.name "Your Name"</span>
-~$ <span class="userinput">git config --global user.email $USER@example.com</span></code></pre>
-</notextile>
-
-On the Arvados Workbench, navigate to "Code repositories":https://{{site.arvados_workbench_host}}/repositories. You should see a repository with your user name listed in the *name* column. Next to *name* is the column *push_url*. Copy the *push_url* value associated with your repository. This should look like <notextile><code>git@git.{{ site.arvados_api_host }}:$USER.git</code></notextile>.
-
-Next, on the Arvados virtual machine, clone your Git repository:
-
-<notextile>
-<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
-~$ <span class="userinput">git clone git@git.{{ site.arvados_api_host }}:$USER.git</span>
-Cloning into '$USER'...</code></pre>
-</notextile>
-
-This will create a Git repository in the directory called @$USER@ in your home directory. Say yes when prompted to continue with connection.
-Ignore any warning that you are cloning an empty repository.
-
-{% include 'notebox_begin' %}
-For more information about using Git, try
-
-notextile. <pre><code>$ <span class="userinput">man gittutorial</span></code></pre>
-
-or *"search Google for Git tutorials":http://google.com/#q=git+tutorial*.
-{% include 'notebox_end' %}
-
-h2. Creating a Crunch script
-
-Start by entering the @$USER@ directory created by @git clone@. Next create a subdirectory called @crunch_scripts@ and change to that directory:
-
-<notextile>
-<pre><code>~$ <span class="userinput">cd $USER</span>
-~/$USER$ <span class="userinput">mkdir crunch_scripts</span>
-~/$USER$ <span class="userinput">cd crunch_scripts</span></code></pre>
+<pre><code>~$ <span class="userinput">mkdir -p tutorial/crunch_scripts</span>
+~$ <span class="userinput">cd tutorial/crunch_scripts</span></code></pre>
</notextile>
Next, using @nano@ or your favorite Unix text editor, create a new file called @hash.py@ in the @crunch_scripts@ directory.
-notextile. <pre>~/$USER/crunch_scripts$ <code class="userinput">nano hash.py</code></pre>
+notextile. <pre>~/tutorial/crunch_scripts$ <code class="userinput">nano hash.py</code></pre>
Add the following code to compute the MD5 hash of each file in a collection:
Make the file executable:
-notextile. <pre><code>~/$USER/crunch_scripts$ <span class="userinput">chmod +x hash.py</span></code></pre>
-
-{% include 'notebox_begin' %}
-The steps below describe how to execute the script after committing changes to Git. To run a single script locally for testing (bypassing the job queue) please see "debugging a crunch script":{{site.baseurl}}/user/topics/tutorial-job-debug.html.
-
-{% include 'notebox_end' %}
+notextile. <pre><code>~/tutorial/crunch_scripts$ <span class="userinput">chmod +x hash.py</span></code></pre>
-Next, add the file to the staging area. This tells @git@ that the file should be included on the next commit.
-
-notextile. <pre><code>~/$USER/crunch_scripts$ <span class="userinput">git add hash.py</span></code></pre>
-
-Next, commit your changes. All staged changes are recorded into the local git repository:
-
-<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">git commit -m"my first script"</span>
-[master (root-commit) 27fd88b] my first script
- 1 file changed, 45 insertions(+)
- create mode 100755 crunch_scripts/hash.py</code></pre>
-</notextile>
-
-Finally, upload your changes to the Arvados server:
-
-<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">git push origin master</span>
-Counting objects: 4, done.
-Compressing objects: 100% (2/2), done.
-Writing objects: 100% (4/4), 682 bytes, done.
-Total 4 (delta 0), reused 0 (delta 0)
-To git@git.qr1hi.arvadosapi.com:$USER.git
- * [new branch] master -> master</code></pre>
-</notextile>
-
-h2. Create a pipeline template
-
-Next, create a file that contains the pipeline definition:
+Next, create a submission job record. This describes a specific invocation of your script:
<notextile>
-<pre><code>~/$USER/crunch_scripts$ <span class="userinput">cd ~</span>
-~$ <span class="userinput">cat >the_pipeline <<EOF
+<pre><code>~/tutorial/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
- "name":"My md5 pipeline",
- "components":{
- "do_hash":{
- "script":"hash.py",
- "script_parameters":{
- "input":{
- "required": true,
- "dataclass": "Collection"
- }
- },
- "repository":"$USER",
- "script_version":"master",
- "output_is_persistent":true,
- "runtime_constraints":{
- "docker_image":"arvados/jobs-java-bwa-samtools"
- }
- }
- }
+ "repository":"",
+ "script":"hash.py",
+ "script_version":"$HOME/tutorial",
+ "script_parameters":{
+ "input":"c1bad4b39ca5a924e481008009d94e32+210"
+ }
}
-EOF
-</span></code></pre>
+EOF</span>
+</code></pre>
</notextile>
-* @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench under "Code repositories":https://{{site.arvados_workbench_host}}/repositories.
-* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit Git revision hash, a tag, or a branch (in which case it will use the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
-* @"script"@ specifies the filename of the script to run. Crunch expects to find this in the @crunch_scripts/@ subdirectory of the Git repository.
-
-Now, use @arv pipeline_template create@ to register your pipeline template in Arvados:
+You can now run your script on your local workstation or VM using @arv-crunch-job@:
<notextile>
-<pre><code>~$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span>
+<pre><code>~/tutorial/crunch_scripts</span>$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
+2014-08-06_15:16:22 qr1hi-8i9sb-qyrat80ef927lam 14473 check slurm allocation
+2014-08-06_15:16:22 qr1hi-8i9sb-qyrat80ef927lam 14473 node localhost - 1 slots
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 start
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 script hash.py
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 script_version /home/peter/peter
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 runtime_constraints {"max_tasks_per_node":0}
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 start level 0
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 status: 0 done, 0 running, 1 todo
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 0 job_task qr1hi-ot0gb-lptn85mwkrn9pqo
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 0 child 14478 started on localhost.1
+2014-08-06_15:16:23 qr1hi-8i9sb-qyrat80ef927lam 14473 status: 0 done, 1 running, 0 todo
+2014-08-06_15:16:24 qr1hi-8i9sb-qyrat80ef927lam 14473 0 stderr crunchstat: Running [stdbuf --output=0 --error=0 /home/$USER/tutorial/crunch_scripts/hash.py]
+2014-08-06_15:16:24 qr1hi-8i9sb-qyrat80ef927lam 14473 0 child 14478 on localhost.1 exit 0 signal 0 success=true
+2014-08-06_15:16:24 qr1hi-8i9sb-qyrat80ef927lam 14473 0 success in 1 seconds
+2014-08-06_15:16:24 qr1hi-8i9sb-qyrat80ef927lam 14473 0 output
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 wait for last 0 children to finish
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 status: 1 done, 0 running, 1 todo
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 start level 1
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 status: 1 done, 0 running, 1 todo
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 1 job_task qr1hi-ot0gb-e3obm0lv6k6p56a
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 1 child 14504 started on localhost.1
+2014-08-06_15:16:25 qr1hi-8i9sb-qyrat80ef927lam 14473 status: 1 done, 1 running, 0 todo
+2014-08-06_15:16:26 qr1hi-8i9sb-qyrat80ef927lam 14473 1 stderr crunchstat: Running [stdbuf --output=0 --error=0 /home/$USER/tutorial/crunch_scripts/hash.py]
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 1 child 14504 on localhost.1 exit 0 signal 0 success=true
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 1 success in 10 seconds
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 1 output 50cafdb29cc21dd6eaec85ba9e0c6134+56+Aef0f991b80fa0b75f802e58e70b207aa184d24ff@53f4bbd3
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 wait for last 0 children to finish
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 status: 2 done, 0 running, 0 todo
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 Freeze not implemented
+2014-08-06_15:16:35 qr1hi-8i9sb-qyrat80ef927lam 14473 collate
+2014-08-06_15:16:36 qr1hi-8i9sb-qyrat80ef927lam 14473 output d6338df28d6b8e5d14929833b417e20e+107+Adf1ce81222b6992ce5d33d8bfb28a6b5a1497898@53f4bbd4
+2014-08-06_15:16:37 qr1hi-8i9sb-qyrat80ef927lam 14473 finish
+2014-08-06_15:16:38 qr1hi-8i9sb-qyrat80ef927lam 14473 log manifest is 7fe8cf1d45d438a3ca3ac4a184b7aff4+83
</code></pre>
</notextile>
-h2. Running your pipeline
+Although the job runs locally, the output of the job has been saved to Keep, the Arvados file store. The "output" line (third from the bottom) provides the "Keep locator":{{site.baseurl}}/user/tutorials/tutorial-keep-get.html to which the script's output has been saved. Copy the output identifier and use @arv-ls@ to list the contents of your output collection, and @arv-get@ to download it to the current directory:
-Your new pipeline template should appear at the top of the Workbench "pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_templates page. You can run your pipeline "using Workbench":tutorial-pipeline-workbench.html or the "command line.":{{site.baseurl}}/user/topics/running-pipeline-command-line.html
+<notextile>
+<pre><code>~/tutorial/crunch_scripts$ <span class="userinput">arv-ls d6338df28d6b8e5d14929833b417e20e+107+Adf1ce81222b6992ce5d33d8bfb28a6b5a1497898@53f4bbd4</span>
+./md5sum.txt
+~/tutorial/crunch_scripts$ <span class="userinput">arv-get d6338df28d6b8e5d14929833b417e20e+107+Adf1ce81222b6992ce5d33d8bfb28a6b5a1497898@53f4bbd4/ .</span>
+~/tutorial/crunch_scripts$ <span class="userinput">cat md5sum.txt</span>
+44b8ae3fde7a8a88d2f7ebd237625b4f c1bad4b39ca5a924e481008009d94e32+210/./var-GS000016015-ASM.tsv.bz2
+</code></pre>
+</notextile>
-For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/schema/PipelineTemplate.html
+Running locally is convenient for development and debugging, as it permits a fast iterative development cycle. Your job run is also recorded by Arvados, and will show up in the "Recent jobs and pipelines" panel on the "Workbench dashboard":https://{{site.arvados_workbench_host}}. This provides limited provenance, by recording the input parameters, the execution log, and the output. However, running locally does not allow you to scale out to multiple nodes, and does not store the complete system snapshot required to achieve reproducibility; to that you need to "submit a job to the Arvados cluster":{{site.baseurl}}/user/tutorials/tutorial-submit-job.html
title: "Uploading data"
...
-This tutorial describes how to to upload new Arvados data collections using the command line tool @arv-put@. This example uses a freely available TSV file containing variant annotations from "Personal Genome Project (PGP)":http://www.personalgenomes.org subject "hu599905.":https://my.personalgenomes.org/profile/hu599905
+This tutorial describes how to to upload new Arvados data collections using the command line tool @arv-put@. This example uses a freely available TSV file containing variant annotations from "Personal Genome Project (PGP)":http://www.pgp-hms.org participant "hu599905.":https://my.pgp-hms.org/profile/hu599905
notextile. <div class="spaced-out">
# On system from which you will upload data, configure the environment with the Arvados instance host name and authentication token as decribed in "Getting an API token.":{{site.baseurl}}/user/reference/api-tokens.html (If you are logged into an Arvados VM, you can skip this step.)
# Download the following example file. (If you are uploading your own data, you can skip this step.)
<notextile>
-<pre><code>~$ <span class="userinput">curl -o var-GS000016015-ASM.tsv.bz2 'https://warehouse.personalgenomes.org/warehouse/f815ec01d5d2f11cb12874ab2ed50daa+234+K@ant/var-GS000016015-ASM.tsv.bz2'</span>
+<pre><code>~$ <span class="userinput">curl -o var-GS000016015-ASM.tsv.bz2 'https://warehouse.pgp-hms.org/warehouse/f815ec01d5d2f11cb12874ab2ed50daa+234+K@ant/var-GS000016015-ASM.tsv.bz2'</span>
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 216M 100 216M 0 0 10.0M 0 0:00:21 0:00:21 --:--:-- 9361k
--- /dev/null
+---
+layout: default
+navsection: userguide
+navmenu: Tutorials
+title: "Running on an Arvados cluster"
+...
+
+This tutorial demonstrates how to create a pipeline to run your crunch script on an Arvados cluster. Cluster jobs can scale out to multiple nodes, and use @git@ and @docker@ to store the complete system snapshot required to achieve reproducibilty.
+
+{% include 'tutorial_expectations' %}
+
+This tutorial uses @$USER@ to denote your username. Replace @$USER@ with your user name in all the following examples.
+
+h2. Setting up Git
+
+All Crunch scripts are managed through the Git revision control system. Before you start using Git, you should do some basic configuration (you only need to do this the first time):
+
+<notextile>
+<pre><code>~$ <span class="userinput">git config --global user.name "Your Name"</span>
+~$ <span class="userinput">git config --global user.email $USER@example.com</span></code></pre>
+</notextile>
+
+On the Arvados Workbench, navigate to "Code repositories":https://{{site.arvados_workbench_host}}/repositories. You should see a repository with your user name listed in the *name* column. Next to *name* is the column *push_url*. Copy the *push_url* value associated with your repository. This should look like <notextile><code>git@git.{{ site.arvados_api_host }}:$USER.git</code></notextile>.
+
+Next, on the Arvados virtual machine, clone your Git repository:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
+~$ <span class="userinput">git clone git@git.{{ site.arvados_api_host }}:$USER.git</span>
+Cloning into '$USER'...</code></pre>
+</notextile>
+
+This will create a Git repository in the directory called @$USER@ in your home directory. Say yes when prompted to continue with connection.
+Ignore any warning that you are cloning an empty repository.
+
+{% include 'notebox_begin' %}
+For more information about using Git, try
+
+notextile. <pre><code>$ <span class="userinput">man gittutorial</span></code></pre>
+
+or *"search Google for Git tutorials":http://google.com/#q=git+tutorial*.
+{% include 'notebox_end' %}
+
+h2. Creating a Crunch script
+
+Start by entering the @$USER@ directory created by @git clone@. Next create a subdirectory called @crunch_scripts@ and change to that directory:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd $USER</span>
+~/$USER$ <span class="userinput">mkdir crunch_scripts</span>
+~/$USER$ <span class="userinput">cd crunch_scripts</span></code></pre>
+</notextile>
+
+Next, using @nano@ or your favorite Unix text editor, create a new file called @hash.py@ in the @crunch_scripts@ directory.
+
+notextile. <pre>~/$USER/crunch_scripts$ <code class="userinput">nano hash.py</code></pre>
+
+Add the following code to compute the MD5 hash of each file in a collection (if you already completed "Writing a Crunch script":tutorial-firstscript.html you can just copy the @hash.py@ file you created previously.)
+
+<notextile> {% code 'tutorial_hash_script_py' as python %} </notextile>
+
+Make the file executable:
+
+notextile. <pre><code>~/$USER/crunch_scripts$ <span class="userinput">chmod +x hash.py</span></code></pre>
+
+Next, add the file to the staging area. This tells @git@ that the file should be included on the next commit.
+
+notextile. <pre><code>~/$USER/crunch_scripts$ <span class="userinput">git add hash.py</span></code></pre>
+
+Next, commit your changes. All staged changes are recorded into the local git repository:
+
+<notextile>
+<pre><code>~/$USER/crunch_scripts$ <span class="userinput">git commit -m"my first script"</span>
+[master (root-commit) 27fd88b] my first script
+ 1 file changed, 45 insertions(+)
+ create mode 100755 crunch_scripts/hash.py</code></pre>
+</notextile>
+
+Finally, upload your changes to the Arvados server:
+
+<notextile>
+<pre><code>~/$USER/crunch_scripts$ <span class="userinput">git push origin master</span>
+Counting objects: 4, done.
+Compressing objects: 100% (2/2), done.
+Writing objects: 100% (4/4), 682 bytes, done.
+Total 4 (delta 0), reused 0 (delta 0)
+To git@git.qr1hi.arvadosapi.com:$USER.git
+ * [new branch] master -> master</code></pre>
+</notextile>
+
+h2. Create a pipeline template
+
+Next, create a file that contains the pipeline definition:
+
+<notextile>
+<pre><code>~/$USER/crunch_scripts$ <span class="userinput">cd ~</span>
+~$ <span class="userinput">cat >the_pipeline <<EOF
+{
+ "name":"My md5 pipeline",
+ "components":{
+ "do_hash":{
+ "script":"hash.py",
+ "script_parameters":{
+ "input":{
+ "required": true,
+ "dataclass": "Collection"
+ }
+ },
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":true,
+ "runtime_constraints":{
+ "docker_image":"arvados/jobs-java-bwa-samtools"
+ }
+ }
+ }
+}
+EOF
+</span></code></pre>
+</notextile>
+
+* @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench under "Code repositories":https://{{site.arvados_workbench_host}}/repositories.
+* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit Git revision hash, a tag, or a branch (in which case it will use the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
+* @"script"@ specifies the filename of the script to run. Crunch expects to find this in the @crunch_scripts/@ subdirectory of the Git repository.
+
+Now, use @arv pipeline_template create@ to register your pipeline template in Arvados:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span>
+</code></pre>
+</notextile>
+
+h2. Running your pipeline
+
+Your new pipeline template should appear at the top of the Workbench "pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_templates page. You can run your pipeline "using Workbench":tutorial-pipeline-workbench.html or the "command line.":{{site.baseurl}}/user/topics/running-pipeline-command-line.html
+
+For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/schema/PipelineTemplate.html
# Install dependencies and set up system.
# The FUSE packages help ensure that we can install the Python SDK (arv-mount).
RUN /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
- libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl \
+ libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl dtrx \
fuse libattr1-dev libfuse-dev && \
/usr/sbin/adduser --disabled-password \
--gecos 'Crunch execution user' crunch && \
# Install Arvados packages.
RUN (find /usr/src/arvados/sdk -name '*.gem' -print0 | \
- xargs -0rn 1 gem install) && \
+ xargs -0rn 1 /usr/local/rvm/bin/rvm-exec default gem install) && \
cd /usr/src/arvados/services/fuse && \
python setup.py install && \
cd /usr/src/arvados/sdk/python && \
class Google::APIClient
def discovery_document(api, version)
api = api.to_s
- return @discovery_documents["#{api}:#{version}"] ||=
+ discovery_uri = self.discovery_uri(api, version)
+ discovery_uri_hash = Digest::MD5.hexdigest(discovery_uri)
+ return @discovery_documents[discovery_uri_hash] ||=
begin
# fetch new API discovery doc if stale
- cached_doc = File.expand_path '~/.cache/arvados/discovery_uri.json' rescue nil
+ cached_doc = File.expand_path "~/.cache/arvados/discovery-#{discovery_uri_hash}.json" rescue nil
if cached_doc.nil? or not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400
response = self.execute!(:http_method => :get,
- :uri => self.discovery_uri(api, version),
+ :uri => discovery_uri,
:authenticated => false)
begin
end
def self.create(attributes)
result = $client.execute(:api_method => $arvados.pipeline_instances.create,
- :body => {
- :pipeline_instance => attributes.to_json
+ :body_object => {
+ :pipeline_instance => attributes
},
:authenticated => false,
:headers => {
:parameters => {
:uuid => @pi[:uuid]
},
- :body => {
- :pipeline_instance => @attributes_to_update.to_json
+ :body_object => {
+ :pipeline_instance => @attributes_to_update
},
:authenticated => false,
:headers => {
body = {job: no_nil_values(job)}.merge(no_nil_values(create_params))
result = $client.execute(:api_method => $arvados.jobs.create,
- :body => body,
+ :body_object => body,
:authenticated => false,
:headers => {
authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
msg += "Job submission was: #{body.to_json}"
$client.execute(:api_method => $arvados.logs.create,
- :body => {
+ :body_object => {
:log => {
:object_uuid => pipeline[:uuid],
:event_type => 'stderr',
end
def setup_instance
- if $options[:submit]
- @instance ||= PipelineInstance.
- create(:components => @components,
- :pipeline_template_uuid => @template[:uuid],
- :state => 'New')
+ if @instance
+ @instance[:properties][:run_options] ||= {}
+ if @options[:no_reuse]
+ # override properties of existing instance
+ @instance[:properties][:run_options][:enable_job_reuse] = false
+ else
+ # Default to "enable reuse" if not specified. (This code path
+ # can go away when old clients go away.)
+ if @instance[:properties][:run_options][:enable_job_reuse].nil?
+ @instance[:properties][:run_options][:enable_job_reuse] = true
+ end
+ end
else
- @instance ||= PipelineInstance.
- create(:components => @components,
- :pipeline_template_uuid => @template[:uuid],
- :state => 'RunningOnClient')
+ @instance = PipelineInstance.
+ create(components: @components,
+ properties: {
+ run_options: {
+ enable_job_reuse: !@options[:no_reuse]
+ }
+ },
+ pipeline_template_uuid: @template[:uuid],
+ state: ($options[:submit] ? 'RunningOnServer' : 'RunningOnClient'))
end
self
end
# dealing with new API servers.
:minimum_script_version => c[:minimum_script_version],
:exclude_script_versions => c[:exclude_minimum_script_versions],
- :find_or_create => !(@options[:no_reuse] || c[:nondeterministic]),
+ :find_or_create => (@instance[:properties][:run_options].andand[:enable_job_reuse] &&
+ !c[:nondeterministic]),
:filters => c[:filters]
})
if job
use Fcntl ':flock';
use File::Path qw( make_path );
+use constant EX_TEMPFAIL => 75;
+
$ENV{"TMPDIR"} ||= "/tmp";
unless (defined $ENV{"CRUNCH_TMP"}) {
$ENV{"CRUNCH_TMP"} = $ENV{"TMPDIR"} . "/crunch-job";
{
$Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
if (!$force_unlock) {
+ # If some other crunch-job process has grabbed this job (or we see
+ # other evidence that the job is already underway) we exit
+ # EX_TEMPFAIL so crunch-dispatch (our parent process) doesn't
+ # mark the job as failed.
if ($Job->{'is_locked_by_uuid'}) {
- croak("Job is locked: " . $Job->{'is_locked_by_uuid'});
+ Log(undef, "Job is locked by " . $Job->{'is_locked_by_uuid'});
+ exit EX_TEMPFAIL;
}
if ($Job->{'success'} ne undef) {
- croak("Job 'success' flag (" . $Job->{'success'} . ") is not null");
+ Log(undef, "Job 'success' flag (" . $Job->{'success'} . ") is not null");
+ exit EX_TEMPFAIL;
}
if ($Job->{'running'}) {
- croak("Job 'running' flag is already set");
+ Log(undef, "Job 'running' flag is already set");
+ exit EX_TEMPFAIL;
}
if ($Job->{'started_at'}) {
- croak("Job 'started_at' time is already set (" . $Job->{'started_at'} . ")");
+ Log(undef, "Job 'started_at' time is already set (" . $Job->{'started_at'} . ")");
+ exit EX_TEMPFAIL;
}
}
}
# Claim this job, and make sure nobody else does
unless ($Job->update_attributes('is_locked_by_uuid' => $User->{'uuid'}) &&
$Job->{'is_locked_by_uuid'} == $User->{'uuid'}) {
- croak("Error while updating / locking job");
+ Log(undef, "Error while updating / locking job, exiting ".EX_TEMPFAIL);
+ exit EX_TEMPFAIL;
}
$Job->update_attributes('started_at' => scalar gmtime,
'running' => 1,
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
- exit (111);
+ # exec() failed, we assume nothing happened.
+ Log(undef, "srun() failed on build script");
+ die;
}
close("writer");
if (!defined $childpid)
Log (undef, "finish");
save_meta();
-exit 0;
+exit ($Job->{'success'} ? 1 : 0);
import errors
import util
+# Set up Arvados logging based on the user's configuration.
+# All Arvados code should log under the arvados hierarchy.
+log_handler = logging.StreamHandler()
+log_handler.setFormatter(logging.Formatter(
+ '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
+ '%Y-%m-%d %H:%M:%S'))
+logger = logging.getLogger('arvados')
+logger.addHandler(log_handler)
+logger.setLevel(logging.DEBUG if config.get('ARVADOS_DEBUG')
+ else logging.WARNING)
+
def task_set_output(self,s):
api('v1').job_tasks().update(uuid=self['uuid'],
body={
import errors
import util
+_logger = logging.getLogger('arvados.api')
services = {}
class CredentialsFromEnv(object):
`apiclient.discovery.build`. If the `discoveryServiceUrl` or `http`
keyword arguments are missing, this function will set default values for
them, based on the current Arvados configuration settings."""
- if config.get('ARVADOS_DEBUG'):
- logging.basicConfig(level=logging.DEBUG)
-
if not cache or not services.get(version):
if not version:
version = 'v1'
- logging.info("Using default API version. " +
- "Call arvados.api('%s') instead." %
+ _logger.info("Using default API version. " +
+ "Call arvados.api('%s') instead.",
version)
if 'discoveryServiceUrl' not in kwargs:
import errors
import util
+_logger = logging.getLogger('arvados.collection')
+
def normalize_stream(s, stream):
stream_tokens = [s]
sortedfiles = list(stream.keys())
class CollectionReader(object):
def __init__(self, manifest_locator_or_text):
- if re.search(r'^[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text):
+ if re.match(r'[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text):
self._manifest_locator = manifest_locator_or_text
self._manifest_text = None
- elif re.search(r'^\S+( [a-f0-9]{32,}(\+\S+)*)*( \d+:\d+:\S+)+\n', manifest_locator_or_text):
+ elif re.match(r'(\S+)( [a-f0-9]{32}(\+\d+)(\+\S+)*)+( \d+:\d+:\S+)+', manifest_locator_or_text):
self._manifest_text = manifest_locator_or_text
self._manifest_locator = None
else:
uuid=self._manifest_locator).execute()
self._manifest_text = c['manifest_text']
except Exception as e:
- logging.warning("API lookup failed for collection %s (%s: %s)" %
- (self._manifest_locator, type(e), str(e)))
+ _logger.warning("API lookup failed for collection %s (%s: %s)",
+ self._manifest_locator, type(e), str(e))
self._manifest_text = Keep.get(self._manifest_locator)
self._streams = []
for stream_line in self._manifest_text.split("\n"):
import config
import logging
+_logger = logging.getLogger('arvados.events')
+
class EventClient(WebSocketClient):
def __init__(self, url, filters, on_event):
ssl_options = None
ws = EventClient(url, filters, on_event)
ws.connect()
return ws
- except:
- logging.exception('')
+ except Exception:
+ _logger.exception('')
if (ws):
- ws.close_connection()
+ ws.close_connection()
raise
import datetime
import ssl
+_logger = logging.getLogger('arvados.keep')
global_client_object = None
from api import *
self.run_with_limiter(limiter)
def run_with_limiter(self, limiter):
- logging.debug("KeepWriterThread %s proceeding %s %s" %
- (str(threading.current_thread()),
- self.args['data_hash'],
- self.args['service_root']))
+ _logger.debug("KeepWriterThread %s proceeding %s %s",
+ str(threading.current_thread()),
+ self.args['data_hash'],
+ self.args['service_root'])
h = httplib2.Http(timeout=self.args.get('timeout', None))
url = self.args['service_root'] + self.args['data_hash']
api_token = config.get('ARVADOS_API_TOKEN')
headers['X-Keep-Desired-Replication'] = str(self.args['want_copies'])
try:
- logging.debug("Uploading to {}".format(url))
+ _logger.debug("Uploading to {}".format(url))
resp, content = h.request(url.encode('utf-8'), 'PUT',
headers=headers,
body=self.args['data'])
body=body)
if re.match(r'^2\d\d$', resp['status']):
self._success = True
- logging.debug("KeepWriterThread %s succeeded %s %s" %
- (str(threading.current_thread()),
- self.args['data_hash'],
- self.args['service_root']))
+ _logger.debug("KeepWriterThread %s succeeded %s %s",
+ str(threading.current_thread()),
+ self.args['data_hash'],
+ self.args['service_root'])
replicas_stored = 1
if 'x-keep-replicas-stored' in resp:
# Tick the 'done' counter for the number of replica
pass
limiter.save_response(content.strip(), replicas_stored)
else:
- logging.warning("Request fail: PUT %s => %s %s" %
- (url, resp['status'], content))
+ _logger.warning("Request fail: PUT %s => %s %s",
+ url, resp['status'], content)
except (httplib2.HttpLib2Error,
httplib.HTTPException,
ssl.SSLError) as e:
# When using https, timeouts look like ssl.SSLError from here.
# "SSLError: The write operation timed out"
- logging.warning("Request fail: PUT %s => %s: %s" %
- (url, type(e), str(e)))
+ _logger.warning("Request fail: PUT %s => %s: %s",
+ url, type(e), str(e))
def __init__(self, **kwargs):
self.lock = threading.Lock()
f['service_port']))
for f in keep_services)
self.service_roots = sorted(set(roots))
- logging.debug(str(self.service_roots))
+ _logger.debug(str(self.service_roots))
finally:
self.lock.release()
# Remove the digits just used from the seed
seed = seed[8:]
- logging.debug(str(pseq))
+ _logger.debug(str(pseq))
return pseq
class CacheSlot(object):
self._cache_lock.release()
def get(self, locator):
- #logging.debug("Keep.get %s" % (locator))
-
if re.search(r',', locator):
return ''.join(self.get(x) for x in locator.split(','))
if 'KEEP_LOCAL_STORE' in os.environ:
expect_hash = re.sub(r'\+.*', '', locator)
slot, first = self.reserve_cache(expect_hash)
- #logging.debug("%s %s %s" % (slot, first, expect_hash))
if not first:
v = slot.get()
def get_url(self, url, headers, expect_hash):
h = httplib2.Http()
try:
- logging.info("Request: GET %s" % (url))
+ _logger.info("Request: GET %s", url)
with timer.Timer() as t:
resp, content = h.request(url.encode('utf-8'), 'GET',
headers=headers)
- logging.info("Received %s bytes in %s msec (%s MiB/sec)" % (len(content),
- t.msecs,
- (len(content)/(1024*1024))/t.secs))
+ _logger.info("Received %s bytes in %s msec (%s MiB/sec)",
+ len(content), t.msecs,
+ (len(content)/(1024*1024))/t.secs)
if re.match(r'^2\d\d$', resp['status']):
m = hashlib.new('md5')
m.update(content)
md5 = m.hexdigest()
if md5 == expect_hash:
return content
- logging.warning("Checksum fail: md5(%s) = %s" % (url, md5))
+ _logger.warning("Checksum fail: md5(%s) = %s", url, md5)
except Exception as e:
- logging.info("Request fail: GET %s => %s: %s" %
- (url, type(e), str(e)))
+ _logger.info("Request fail: GET %s => %s: %s",
+ url, type(e), str(e))
return None
def put(self, data, **kwargs):
threads_retry = []
for t in threads:
if not t.success():
- logging.warning("Retrying: PUT %s %s" % (
- t.args['service_root'],
- t.args['data_hash']))
+ _logger.warning("Retrying: PUT %s %s",
+ t.args['service_root'],
+ t.args['data_hash'])
retry_with_args = t.args.copy()
t_retry = KeepClient.KeepWriterThread(**retry_with_args)
t_retry.start()
import gflags
import httplib
import httplib2
-import logging
import os
import pprint
import sys
import sys
import logging
-logger = logging.getLogger(os.path.basename(sys.argv[0]))
+import arvados
+
+logger = logging.getLogger('arvados.arv-get')
+
+def abort(msg, code=1):
+ print >>sys.stderr, "arv-get:", msg
+ exit(code)
parser = argparse.ArgumentParser(
description='Copy data from Keep to a local file or pipe.')
args.destination[-1] == os.path.sep):
args.destination = os.path.join(args.destination,
os.path.basename(args.locator))
- logger.debug("Appended source file name to destination directory: %s" %
+ logger.debug("Appended source file name to destination directory: %s",
args.destination)
# Turn on --progress by default if stderr is a tty and stdout isn't.
args.destination = args.destination.rstrip(os.sep)
-import arvados
-
r = re.search(r'^(.*?)(/.*)?$', args.locator)
collection = r.group(1)
get_prefix = r.group(2)
try:
if not args.n:
if not args.f and os.path.exists(args.destination):
- logger.error('Local file %s already exists' % args.destination)
- sys.exit(1)
+ abort('Local file %s already exists.' % (args.destination,))
with open(args.destination, 'wb') as f:
try:
c = arvados.api('v1').collections().get(
uuid=collection).execute()
manifest = c['manifest_text']
except Exception as e:
- logging.warning(
- "API lookup failed for collection %s (%s: %s)" %
- (collection, type(e), str(e)))
+ logger.warning(
+ "Collection %s not found. " +
+ "Trying to fetch directly from Keep (deprecated).",
+ collection)
manifest = arvados.Keep.get(collection)
f.write(manifest)
sys.exit(0)
except arvados.errors.NotFoundError as e:
- logger.error(e)
- sys.exit(1)
+ abort(e)
reader = arvados.CollectionReader(collection)
os.path.join(s.name(), f.name())[len(get_prefix)+1:])
if (not (args.n or args.f or args.skip_existing) and
os.path.exists(dest_path)):
- logger.error('Local file %s already exists' % dest_path)
- sys.exit(1)
+ abort('Local file %s already exists.' % (dest_path,))
else:
if os.path.join(s.name(), f.name()) != '.' + get_prefix:
continue
todo += [(s, f, dest_path)]
todo_bytes += f.size()
except arvados.errors.NotFoundError as e:
- logger.error(e)
- sys.exit(1)
+ abort(e)
# Read data, and (if not -n) write to local file(s) or pipe.
digestor = None
if not args.n:
if args.skip_existing and os.path.exists(outfilename):
- logger.debug('Local file %s exists. Skipping.' % outfilename)
+ logger.debug('Local file %s exists. Skipping.', outfilename)
continue
elif not args.f and (os.path.isfile(outfilename) or
os.path.isdir(outfilename)):
# Good thing we looked again: apparently this file wasn't
# here yet when we checked earlier.
- logger.error('Local file %s already exists' % outfilename)
- sys.exit(1)
+ abort('Local file %s already exists.' % (outfilename,))
if args.r:
arvados.util.mkdir_dash_p(os.path.dirname(outfilename))
try:
outfile = open(outfilename, 'wb')
except Exception as e:
- logger.error('Open(%s) failed: %s' % (outfilename, e))
- sys.exit(1)
+ abort('Open(%s) failed: %s' % (outfilename, e))
if args.hash:
digestor = hashlib.new(args.hash)
try:
import re
import string
import sys
-import logging
-
-logger = logging.getLogger(os.path.basename(sys.argv[0]))
parser = argparse.ArgumentParser(
description='List contents of a manifest')
import re
import string
import sys
-import logging
-
-logger = logging.getLogger(os.path.basename(sys.argv[0]))
parser = argparse.ArgumentParser(
description='Read manifest on standard input and put normalized manifest on standard output.')
import arvados
from arvados.events import subscribe
+logger = logging.getLogger('arvados.arv-ws')
+
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--uuid', type=str, default="")
args = parser.parse_args()
try:
ws = subscribe(api, filters, lambda ev: on_message(ev))
ws.run_forever()
-except KeyboardInterrupt:
- print '' # don't log it
-except:
- logging.exception('')
+except Exception:
+ logger.exception('')
if (ws):
ws.close_connection()
class Google::APIClient
def discovery_document(api, version)
api = api.to_s
- return @discovery_documents["#{api}:#{version}"] ||=
+ discovery_uri = self.discovery_uri(api, version)
+ discovery_uri_hash = Digest::MD5.hexdigest(discovery_uri)
+ return @discovery_documents[discovery_uri_hash] ||=
begin
# fetch new API discovery doc if stale
- cached_doc = File.expand_path '~/.cache/arvados/discovery_uri.json'
- if not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400
+ cached_doc = File.expand_path "~/.cache/arvados/discovery-#{discovery_uri_hash}.json" rescue nil
+ if cached_doc.nil? or not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400
response = self.execute!(:http_method => :get,
- :uri => self.discovery_uri(api, version),
+ :uri => discovery_uri,
:authenticated => false)
- FileUtils.makedirs(File.dirname cached_doc)
- File.open(cached_doc, 'w') do |f|
- f.puts response.body
+ begin
+ FileUtils.makedirs(File.dirname cached_doc)
+ File.open(cached_doc, 'w') do |f|
+ f.puts response.body
+ end
+ rescue
+ return JSON.load response.body
end
end
addressable (2.3.6)
andand (1.3.3)
arel (3.0.3)
- arvados (0.1.20140708213257)
+ arvados (0.1.20140812162850)
activesupport (>= 3.2.13)
andand
google-api-client (~> 0.6.3)
json (>= 1.7.7)
jwt (>= 0.1.5, < 1.0.0)
- arvados-cli (0.1.20140708213257)
+ arvados-cli (0.1.20140812162850)
activesupport (~> 3.2, >= 3.2.13)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1.0)
coffee-script-source
execjs
coffee-script-source (1.7.0)
- curb (0.8.5)
+ curb (0.8.6)
daemon_controller (1.2.0)
database_cleaner (1.2.0)
erubis (2.7.0)
highline (1.6.21)
hike (1.2.3)
httpauth (0.2.1)
- i18n (0.6.9)
+ i18n (0.6.11)
journey (1.0.4)
jquery-rails (3.1.0)
railties (>= 3.0, < 5.0)
mime-types (~> 1.16)
treetop (~> 1.4.8)
mime-types (1.25.1)
- multi_json (1.10.0)
+ multi_json (1.10.1)
multipart-post (1.2.0)
net-scp (1.2.0)
net-ssh (>= 2.6.5)
jwt (~> 0.1.4)
multi_json (~> 1.0)
rack (~> 1.2)
- oj (2.9.0)
+ oj (2.10.0)
omniauth (1.1.1)
hashie (~> 1.2)
rack
uglifier (2.5.0)
execjs (>= 0.3.0)
json (>= 1.8.0)
- uuidtools (2.1.4)
+ uuidtools (2.1.5)
websocket-driver (0.3.2)
PLATFORMS
end
end
- @objects = @objects.select(@select.map { |s| "#{table_name}.#{ActiveRecord::Base.connection.quote_column_name s.to_s}" }.join ", ") if @select
+ if @select
+ # Map attribute names in @select to real column names, resolve
+ # those to fully-qualified SQL column names, and pass the
+ # resulting string to the select method.
+ api_column_map = model_class.attributes_required_columns
+ columns_list = @select.
+ flat_map { |attr| api_column_map[attr] }.
+ uniq.
+ map { |s| "#{table_name}.#{ActiveRecord::Base.connection.quote_column_name s}" }
+ @objects = @objects.select(columns_list.join(", "))
+ end
@objects = @objects.order(@orders.join ", ") if @orders.any?
@objects = @objects.limit(@limit)
@objects = @objects.offset(@offset)
end
# Remove any permission signatures from the manifest.
- resource_attrs[:manifest_text]
- .gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) { |word|
- word.strip!
- loc = Locator.parse(word)
- if loc
- " " + loc.without_signature.to_s
- else
- " " + word
- end
- }
+ munge_manifest_locators(resource_attrs[:manifest_text]) do |loc|
+ loc.without_signature.to_s
+ end
# Save the collection with the stripped manifest.
act_as_system_user do
end
def show
- if current_api_client_authorization
- signing_opts = {
- key: Rails.configuration.blob_signing_key,
- api_token: current_api_client_authorization.api_token,
- ttl: Rails.configuration.blob_signing_ttl,
- }
- @object[:manifest_text]
- .gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) { |word|
- word.strip!
- loc = Locator.parse(word)
- if loc
- " " + Blob.sign_locator(word, signing_opts)
- else
- " " + word
- end
- }
- end
- render json: @object.as_api_response(:with_data)
+ sign_manifests(@object[:manifest_text])
+ super
+ end
+
+ def index
+ sign_manifests(*@objects.map { |c| c[:manifest_text] })
+ super
end
def collection_uuid(uuid)
logger.debug "visiting #{uuid}"
- if m
+ if m
# uuid is a collection
Collection.readable_by(current_user).where(uuid: uuid).each do |c|
visited[uuid] = c.as_api_response
Job.readable_by(current_user).where(log: uuid).each do |job|
generate_provenance_edges(visited, job.uuid)
end
-
+
else
# uuid is something else
rsc = ArvadosModel::resource_class_for_uuid uuid
logger.debug "visiting #{uuid}"
- if m
+ if m
# uuid is a collection
Collection.readable_by(current_user).where(uuid: uuid).each do |c|
visited[uuid] = c.as_api_response
Job.readable_by(current_user).where(["jobs.script_parameters like ?", "%#{uuid}%"]).each do |job|
generate_used_by_edges(visited, job.uuid)
end
-
+
else
# uuid is something else
rsc = ArvadosModel::resource_class_for_uuid uuid
render json: visited
end
+ def self.munge_manifest_locators(manifest)
+ # Given a manifest text and a block, yield each locator,
+ # and replace it with whatever the block returns.
+ manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) do |word|
+ if loc = Locator.parse(word.strip)
+ " " + yield(loc)
+ else
+ " " + word
+ end
+ end
+ end
+
protected
+
+ def find_objects_for_index
+ # Omit manifest_text from index results unless expressly selected.
+ @select ||= model_class.api_accessible_attributes(:user).
+ map { |attr_spec| attr_spec.first.to_s } - ["manifest_text"]
+ super
+ end
+
def find_object_by_uuid
super
if !@object and !params[:uuid].match(/^[0-9a-f]+\+\d+$/)
end
end
end
+
+ def munge_manifest_locators(manifest, &block)
+ self.class.munge_manifest_locators(manifest, &block)
+ end
+
+ def sign_manifests(*manifests)
+ if current_api_client_authorization
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: current_api_client_authorization.api_token,
+ ttl: Rails.configuration.blob_signing_ttl,
+ }
+ manifests.each do |text|
+ munge_manifest_locators(text) do |loc|
+ Blob.sign_locator(loc.to_s, signing_opts)
+ end
+ end
+ end
+ end
end
index
end
+ def self._create_requires_parameters
+ (super rescue {}).
+ merge({
+ find_or_create: {
+ type: 'boolean', required: false, default: false
+ },
+ filters: {
+ type: 'array', required: false
+ },
+ minimum_script_version: {
+ type: 'string', required: false
+ },
+ exclude_script_versions: {
+ type: 'array', required: false
+ },
+ })
+ end
+
def self._queue_requires_parameters
self._index_requires_parameters
end
if ArvadosModel::resource_class_for_uuid(perm.tail_uuid) == Group
@users.each do |user_uuid, user|
user.group_permissions.each do |group_uuid, perm_mask|
- if perm_mask[:write]
+ if perm_mask[:manage]
+ perms << {name: 'can_manage', user_uuid: user_uuid}
+ elsif perm_mask[:write]
perms << {name: 'can_write', user_uuid: user_uuid}
elsif perm_mask[:read]
perms << {name: 'can_read', user_uuid: user_uuid}
end
@repo_info.values.each do |repo_users|
repo_users[:user_permissions].each do |user_uuid,perms|
- if perms['can_write']
+ if perms['can_manage']
+ perms[:gitolite_permissions] = 'RW'
+ perms['can_write'] = true
+ perms['can_read'] = true
+ elsif perms['can_write']
perms[:gitolite_permissions] = 'RW'
perms['can_read'] = true
elsif perms['can_read']
}.compact.first
if httpMethod and
route.defaults[:controller] == 'arvados/v1/' + k.to_s.underscore.pluralize and
- !d_methods[action.to_sym] and
- ctl_class.action_methods.include? action and
- ![:show, :index, :destroy].include?(action.to_sym)
- method = {
- id: "arvados.#{k.to_s.underscore.pluralize}.#{action}",
- path: route.path.spec.to_s.sub('/arvados/v1/','').sub('(.:format)','').sub(/:(uu)?id/,'{uuid}'),
- httpMethod: httpMethod,
- description: "#{route.defaults[:action]} #{k.to_s.underscore.pluralize}",
- parameters: {},
- response: {
- "$ref" => (action == 'index' ? "#{k.to_s}List" : k.to_s)
- },
- scopes: [
- "https://api.clinicalfuture.com/auth/arvados"
- ]
- }
- route.segment_keys.each do |key|
- if key != :format
- key = :uuid if key == :id
- method[:parameters][key] = {
- type: "string",
- description: "",
- required: true,
- location: "path"
- }
+ ctl_class.action_methods.include? action
+ if !d_methods[action.to_sym]
+ method = {
+ id: "arvados.#{k.to_s.underscore.pluralize}.#{action}",
+ path: route.path.spec.to_s.sub('/arvados/v1/','').sub('(.:format)','').sub(/:(uu)?id/,'{uuid}'),
+ httpMethod: httpMethod,
+ description: "#{action} #{k.to_s.underscore.pluralize}",
+ parameters: {},
+ response: {
+ "$ref" => (action == 'index' ? "#{k.to_s}List" : k.to_s)
+ },
+ scopes: [
+ "https://api.clinicalfuture.com/auth/arvados"
+ ]
+ }
+ route.segment_keys.each do |key|
+ if key != :format
+ key = :uuid if key == :id
+ method[:parameters][key] = {
+ type: "string",
+ description: "",
+ required: true,
+ location: "path"
+ }
+ end
end
+ else
+ # We already built a generic method description, but we
+ # might find some more required parameters through
+ # introspection.
+ method = d_methods[action.to_sym]
end
if ctl_class.respond_to? "_#{action}_requires_parameters".to_sym
ctl_class.send("_#{action}_requires_parameters".to_sym).each do |k, v|
end
end
end
- d_methods[route.defaults[:action].to_sym] = method
+ d_methods[action.to_sym] = method
end
end
end
def self._setup_requires_parameters
{
- send_notification_email: { type: 'boolean', required: true },
+ user: {
+ type: 'object', required: false
+ },
+ openid_prefix: {
+ type: 'string', required: false
+ },
+ repo_name: {
+ type: 'string', required: false
+ },
+ vm_uuid: {
+ type: 'string', required: false
+ },
+ send_notification_email: {
+ type: 'boolean', required: false, default: false
+ },
}
end
--- /dev/null
+class ProfileNotifier < ActionMailer::Base
+ default from: Rails.configuration.admin_notifier_email_from
+
+ def profile_created(user, address)
+ @user = user
+ mail(to: address, subject: "Profile created by #{@user.email}")
+ end
+end
self.columns.select { |col| col.name == attr.to_s }.first
end
+ def self.attributes_required_columns
+ # This method returns a hash. Each key is the name of an API attribute,
+ # and it's mapped to a list of database columns that must be fetched
+ # to generate that attribute.
+ # This implementation generates a simple map of attributes to
+ # matching column names. Subclasses can override this method
+ # to specify that method-backed API attributes need to fetch
+ # specific columns from the database.
+ all_columns = columns.map(&:name)
+ api_column_map = Hash.new { |hash, key| hash[key] = [] }
+ methods.grep(/^api_accessible_\w+$/).each do |method_name|
+ next if method_name == :api_accessible_attributes
+ send(method_name).each_pair do |api_attr_name, col_name|
+ col_name = col_name.to_s
+ if all_columns.include?(col_name)
+ api_column_map[api_attr_name.to_s] |= [col_name]
+ end
+ end
+ end
+ api_column_map
+ end
+
# Return nil if current user is not allowed to see the list of
# writers. Otherwise, return a list of user_ and group_uuids with
# write permission. (If not returning nil, current_user is always in
api_accessible :user, extend: :common do |t|
t.add :data_size
t.add :files
+ t.add :manifest_text
end
- api_accessible :with_data, extend: :user do |t|
- t.add :manifest_text
+ def self.attributes_required_columns
+ super.merge({ "data_size" => ["manifest_text"],
+ "files" => ["manifest_text"],
+ })
end
def redundancy_status
if self.cancelled_at and not self.cancelled_at_was
self.cancelled_at = Time.now
self.cancelled_by_user_uuid = current_user.uuid
- self.cancelled_by_client_uuid = current_api_client.uuid
+ self.cancelled_by_client_uuid = current_api_client.andand.uuid
@need_crunch_dispatch_trigger = true
else
self.cancelled_at = self.cancelled_at_was
before_create :check_auto_admin
after_create :add_system_group_permission_link
after_create :send_admin_notifications
+ after_update :send_profile_created_notification
+
has_many :authorized_keys, :foreign_key => :authorized_user_uuid, :primary_key => :uuid
AdminNotifier.new_inactive_user(self).deliver
end
end
+
+ # Send notification if the user saved profile for the first time
+ def send_profile_created_notification
+ if self.prefs_changed?
+ if self.prefs_was.andand.empty? || !self.prefs_was.andand['profile']
+ profile_notification_address = Rails.configuration.user_profile_notification_address
+ ProfileNotifier.profile_created(self, profile_notification_address).deliver if profile_notification_address
+ end
+ end
+ end
+
end
--- /dev/null
+Profile created by user <%=@user.full_name%> <%=@user.email%>
+User's profile: <%=@user.prefs['profile']%>
secret_token: <%= rand(2**512).to_s(36) %>
blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
+ # email address to which mail should be sent when the user creates profile for the first time
+ user_profile_notification_address: arvados@example.com
+
common:
uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
# to sign session tokens. IMPORTANT: This is a site secret. It
# should be at least 50 characters.
secret_token: ~
+
+ # email address to which mail should be sent when the user creates profile for the first time
+ user_profile_notification_address: false
$stderr.puts j_done[:stderr_buf] + "\n"
end
- # Wait the thread
- j_done[:wait_thr].value
+ # Wait the thread (returns a Process::Status)
+ exit_status = j_done[:wait_thr].value
jobrecord = Job.find_by_uuid(job_done.uuid)
- if jobrecord.started_at
+ if exit_status.to_i != 75 and jobrecord.started_at
# Clean up state fields in case crunch-job exited without
# putting the job in a suitable "finished" state.
jobrecord.running = false
# Don't fail the job if crunch-job didn't even get as far as
# starting it. If the job failed to run due to an infrastructure
# issue with crunch-job or slurm, we want the job to stay in the
- # queue.
+ # queue. If crunch-job exited after losing a race to another
+ # crunch-job process, it exits 75 and we should leave the job
+ # record alone so the winner of the race do its thing.
+ #
+ # There is still an unhandled race condition: If our crunch-job
+ # process is about to lose a race with another crunch-job
+ # process, but crashes before getting to its "exit 75" (for
+ # example, "cannot fork" or "cannot reach API server") then we
+ # will assume incorrectly that it's our process's fault
+ # jobrecord.started_at is non-nil, and mark the job as failed
+ # even though the winner of the race is probably still doing
+ # fine.
end
# Invalidate the per-job auth token
user: job_reader
api_token: e99512cdc0f3415c2428b9758f33bdfb07bc3561b00e86e7e6
expires_at: 2038-01-01 00:00:00
+
+active_no_prefs:
+ api_client: untrusted
+ user: active_no_prefs
+ api_token: 3kg612cdc0f3415c2428b9758f33bdfb07bc3561b00e86qdmi
+ expires_at: 2038-01-01 00:00:00
+
+active_no_prefs_profile:
+ api_client: untrusted
+ user: active_no_prefs_profile
+ api_token: 3kg612cdc0f3415c242856758f33bdfb07bc3561b00e86qdmi
+ expires_at: 2038-01-01 00:00:00
cancelled_at: ~
cancelled_by_user_uuid: ~
cancelled_by_client_uuid: ~
+ created_at: <%= 3.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
cancelled_at: <%= 1.minute.ago.to_s(:db) %>
cancelled_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
cancelled_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ created_at: <%= 4.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
cancelled_by_user_uuid: ~
cancelled_by_client_uuid: ~
script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
+ created_at: <%= 5.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: <%= 2.minute.ago.to_s(:db) %>
running: false
script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
script_parameters:
input: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ created_at: <%= 4.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: <%= 2.minute.ago.to_s(:db) %>
running: false
script_parameters:
input: fa7aeb5140e2848d39b416daeef4ffc5+45
an_integer: 1
+ created_at: <%= 4.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: <%= 2.minute.ago.to_s(:db) %>
running: false
previous_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ created_at: <%= 14.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
repository: foo
script: hash
previous_docker_job_run:
uuid: zzzzz-8i9sb-k6emstgk4kw4yhi
+ created_at: <%= 14.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
repository: foo
script: hash
previous_job_run_no_output:
uuid: zzzzz-8i9sb-cjs4pklxxjykppp
+ created_at: <%= 14.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
repository: foo
script: hash
nondeterminisic_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykyyy
+ created_at: <%= 14.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
repository: foo
script: hash2
nearly_finished_job:
uuid: zzzzz-8i9sb-2gx6rz0pjl033w3
+ created_at: <%= 14.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
repository: arvados
script: doesnotexist
queued:
uuid: zzzzz-8i9sb-grx15v5mjnsyxk7
+ created_at: <%= 1.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
cancelled_at: ~
cancelled_by_user_uuid: ~
state: New
uuid: zzzzz-d1hrv-f4gneyn6br1xize
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: <%= 1.minute.ago.to_s(:db) %>
has_component_with_no_script_parameters:
state: Ready
uuid: zzzzz-d1hrv-1xfj6xkicf2muk2
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: <%= 10.minute.ago.to_s(:db) %>
components:
foo:
script: foo
state: Ready
uuid: zzzzz-d1hrv-jq16l10gcsnyumo
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: <%= 3.minute.ago.to_s(:db) %>
components:
foo:
script: foo
state: Ready
uuid: zzzzz-d1hrv-1yfj6xkidf2muk3
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: <%= 3.1.minute.ago.to_s(:db) %>
components:
foo:
script: foo
# Helps test that clients cope with funny-shaped components.
# For an example, see #3321.
uuid: zzzzz-d1hrv-jobspeccomponts
+ created_at: <%= 30.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-04-14 12:35:04 -0400
updated_at: 2014-04-14 12:35:04 -0400
identity_url: https://admin.openid.local
is_active: true
is_admin: true
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: IT
miniadmin:
owner_uuid: zzzzz-tpzed-000000000000000
identity_url: https://miniadmin.openid.local
is_active: true
is_admin: false
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: IT
rominiadmin:
owner_uuid: zzzzz-tpzed-000000000000000
identity_url: https://rominiadmin.openid.local
is_active: true
is_admin: false
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: IT
active:
owner_uuid: zzzzz-tpzed-000000000000000
identity_url: https://active-user.openid.local
is_active: true
is_admin: false
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
project_viewer:
owner_uuid: zzzzz-tpzed-000000000000000
identity_url: https://project-viewer.openid.local
is_active: true
is_admin: false
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
future_project_user:
# Workbench tests give this user permission on aproject.
identity_url: https://future-project-user.openid.local
is_active: true
is_admin: false
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
spectator:
owner_uuid: zzzzz-tpzed-000000000000000
identity_url: https://spectator.openid.local
is_active: true
is_admin: false
- prefs: {}
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
inactive_uninvited:
owner_uuid: zzzzz-tpzed-000000000000000
identity_url: https://spectator.openid.local
is_active: true
is_admin: false
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
+
+active_no_prefs:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zzzzz-tpzed-a46c42d1td4aoj4
+ email: active_no_prefs@arvados.local
+ first_name: NoPrefs
+ last_name: NoProfile
+ identity_url: https://active_no_prefs.openid.local
+ is_active: true
+ is_admin: false
prefs: {}
+
+active_no_prefs_profile:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zzzzz-tpzed-a46c98d1td4aoj4
+ email: active_no_prefs_profile@arvados.local
+ first_name: HasPrefs
+ last_name: NoProfile
+ identity_url: https://active_no_prefs_profile.openid.local
+ is_active: true
+ is_admin: false
+ prefs:
+ test: abc
authorize_with :active
get :index
assert_response :success
- assert_not_nil assigns(:objects)
+ assert(assigns(:objects).andand.any?, "no Collections returned in index")
+ refute(json_response["items"].any? { |c| c.has_key?("manifest_text") },
+ "basic Collections index included manifest_text")
+ end
+
+ test "can get non-database fields via index select" do
+ authorize_with :active
+ get(:index, filters: [["uuid", "=", collections(:foo_file).uuid]],
+ select: %w(uuid owner_uuid files))
+ assert_response :success
+ assert_equal(1, json_response["items"].andand.size,
+ "wrong number of items returned for index")
+ assert_equal([[".", "foo", 3]], json_response["items"].first["files"],
+ "wrong file list in index result")
+ end
+
+ test "can select only non-database fields for index" do
+ authorize_with :active
+ get(:index, select: %w(data_size files))
+ assert_response :success
+ assert(json_response["items"].andand.any?, "no items found in index")
+ json_response["items"].each do |coll|
+ assert_equal(coll["data_size"],
+ coll["files"].inject(0) { |size, fspec| size + fspec.last },
+ "mismatch between data size and file list")
+ end
+ end
+
+ test "index with manifest_text selected returns signed locators" do
+ columns = %w(uuid owner_uuid data_size files manifest_text)
+ authorize_with :active
+ get :index, select: columns
+ assert_response :success
+ assert(assigns(:objects).andand.any?,
+ "no Collections returned for index with columns selected")
+ json_response["items"].each do |coll|
+ assert_equal(columns, columns & coll.keys,
+ "Collections index did not respect selected columns")
+ loc_regexp = / [[:xdigit:]]{32}\+\d+\S+/
+ pos = 0
+ while match = loc_regexp.match(coll["manifest_text"], pos)
+ assert_match(/\+A[[:xdigit:]]+@[[:xdigit:]]{8}\b/, match.to_s,
+ "Locator in manifest_text was not signed")
+ pos = match.end(0)
+ end
+ end
end
[0,1,2].each do |limit|
check_active_users_index
end
+ test "update active_no_prefs user profile and expect notification email" do
+ authorize_with :admin
+
+ put :update, {
+ id: users(:active_no_prefs).uuid,
+ user: {
+ prefs: {:profile => {'organization' => 'example.com'}}
+ }
+ }
+ assert_response :success
+
+ found_email = false
+ ActionMailer::Base.deliveries.andand.each do |email|
+ if email.subject == "Profile created by #{users(:active_no_prefs).email}"
+ found_email = true
+ break
+ end
+ end
+ assert_equal true, found_email, 'Expected email after creating profile'
+ end
+
+ test "update active_no_prefs_profile user profile and expect notification email" do
+ authorize_with :admin
+
+ user = {}
+ user[:prefs] = users(:active_no_prefs_profile).prefs
+ user[:prefs][:profile] = {:profile => {'organization' => 'example.com'}}
+ put :update, {
+ id: users(:active_no_prefs_profile).uuid,
+ user: user
+ }
+ assert_response :success
+
+ found_email = false
+ ActionMailer::Base.deliveries.andand.each do |email|
+ if email.subject == "Profile created by #{users(:active_no_prefs_profile).email}"
+ found_email = true
+ break
+ end
+ end
+ assert_equal true, found_email, 'Expected email after creating profile'
+ end
+
+ test "update active user profile and expect no notification email" do
+ authorize_with :admin
+
+ put :update, {
+ id: users(:active).uuid,
+ user: {
+ prefs: {:profile => {'organization' => 'anotherexample.com'}}
+ }
+ }
+ assert_response :success
+
+ found_email = false
+ ActionMailer::Base.deliveries.andand.each do |email|
+ if email.subject == "Profile created by #{users(:active).email}"
+ found_email = true
+ break
+ end
+ end
+ assert_equal false, found_email, 'Expected no email after updating profile'
+ end
+
+
NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
"last_name"].sort
import json
import logging
+_logger = logging.getLogger('arvados.arvados_fuse')
+
from time import time
from llfuse import FUSEError
try:
self.update()
except apiclient.errors.HttpError as e:
- logging.debug(e)
+ _logger.debug(e)
def __getitem__(self, item):
self.checkupdate()
self.fresh()
return True
except Exception as detail:
- logging.debug("arv-mount %s: error: %s" % (self.collection_locator,detail))
+ _logger.debug("arv-mount %s: error: %s",
+ self.collection_locator, detail)
return False
class MagicDirectory(Directory):
else:
return False
except Exception as e:
- logging.debug('arv-mount exception keep %s', e)
+ _logger.debug('arv-mount exception keep %s', e)
return False
def __getitem__(self, item):
return entry
def lookup(self, parent_inode, name):
- logging.debug("arv-mount lookup: parent_inode %i name %s", parent_inode, name)
+ _logger.debug("arv-mount lookup: parent_inode %i name %s",
+ parent_inode, name)
inode = None
if name == '.':
return fh
def read(self, fh, off, size):
- logging.debug("arv-mount read %i %i %i", fh, off, size)
+ _logger.debug("arv-mount read %i %i %i", fh, off, size)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
del self._filehandles[fh]
def opendir(self, inode):
- logging.debug("arv-mount opendir: inode %i", inode)
+ _logger.debug("arv-mount opendir: inode %i", inode)
if inode in self.inodes:
p = self.inodes[inode]
return fh
def readdir(self, fh, off):
- logging.debug("arv-mount readdir: fh %i off %i", fh, off)
+ _logger.debug("arv-mount readdir: fh %i off %i", fh, off)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
raise llfuse.FUSEError(errno.EBADF)
- logging.debug("arv-mount handle.entry %s", handle.entry)
+ _logger.debug("arv-mount handle.entry %s", handle.entry)
e = off
while e < len(handle.entry):
import os
import signal
import subprocess
-import traceback
from arvados_fuse import *
+logger = logging.getLogger('arvados.arv-mount')
+
if __name__ == '__main__':
# Handle command line parameters
parser = argparse.ArgumentParser(
else:
daemon_ctx = None
- # Set up logging.
- # If we're daemonized without a logfile, there's nowhere to log, so don't.
- if args.logfile or (daemon_ctx is None):
- log_conf = {}
- if args.debug:
- log_conf['level'] = logging.DEBUG
- arvados.config.settings()['ARVADOS_DEBUG'] = 'true'
- if args.logfile:
- log_conf['filename'] = args.logfile
- logging.basicConfig(**log_conf)
- logging.debug("arv-mount debugging enabled")
+ # Configure a logger based on command-line switches.
+ # If we're using a contemporary Python SDK (mid-August 2014),
+ # configure the arvados hierarchy logger.
+ # Otherwise, configure the program root logger.
+ base_logger = getattr(arvados, 'logger', None)
+
+ if args.logfile:
+ log_handler = logging.FileHandler(args.logfile)
+ elif daemon_ctx:
+ log_handler = logging.NullHandler()
+ elif base_logger:
+ log_handler = arvados.log_handler
+ else:
+ log_handler = logging.StreamHandler()
+
+ if base_logger is None:
+ base_logger = logging.getLogger()
+ else:
+ base_logger.removeHandler(arvados.log_handler)
+ base_logger.addHandler(log_handler)
+
+ if args.debug:
+ base_logger.setLevel(logging.DEBUG)
+ logger.debug("arv-mount debugging enabled")
try:
# Create the request handler
else:
# Set up the request handler with the 'magic directory' at the root
operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
- except Exception as ex:
- logging.error("arv-mount: exception during API setup")
- logging.error(traceback.format_exc())
+ except Exception:
+ logger.exception("arv-mount: exception during API setup")
exit(1)
# FUSE options, see mount.fuse(8)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
except Exception as e:
- logging.error('arv-mount: exception during exec %s' % (args.exec_args,))
- logging.error(traceback.format_exc())
+ logger.exception('arv-mount: exception during exec %s',
+ args.exec_args)
try:
rc = e.errno
except AttributeError:
llfuse.init(operations, args.mountpoint, opts)
llfuse.main()
except Exception as e:
- logging.error('arv-mount: exception during mount')
- logging.error(traceback.format_exc())
+ logger.exception('arv-mount: exception during mount')
exit(getattr(e, 'errno', 1))
projects / arvados.git / commitdiff