Merge branch 'master' into 3296-user-profile
authorradhika <radhika@curoverse.com>
Tue, 12 Aug 2014 19:10:46 +0000 (15:10 -0400)
committerradhika <radhika@curoverse.com>
Tue, 12 Aug 2014 19:10:46 +0000 (15:10 -0400)
39 files changed:
apps/workbench/app/assets/javascripts/infinite_scroll.js
apps/workbench/app/controllers/application_controller.rb
apps/workbench/app/controllers/projects_controller.rb
apps/workbench/app/views/application/_content.html.erb
apps/workbench/app/views/application/_content_layout.html.erb
apps/workbench/app/views/application/_tab_line_buttons.html.erb [new file with mode: 0644]
apps/workbench/app/views/application/show.html.erb
apps/workbench/app/views/collections/_show_files.html.erb
apps/workbench/app/views/collections/_show_recent.html.erb
apps/workbench/app/views/collections/index.html.erb [new file with mode: 0644]
apps/workbench/app/views/collections/show.html.erb
apps/workbench/app/views/keep_disks/_content_layout.html.erb
apps/workbench/app/views/pipeline_instances/_show_recent.html.erb
apps/workbench/app/views/pipeline_instances/index.html.erb [new file with mode: 0644]
apps/workbench/app/views/pipeline_templates/_show_components.html.erb
apps/workbench/app/views/pipeline_templates/show.html.erb [new file with mode: 0644]
apps/workbench/app/views/projects/_show_contents_rows.html.erb
apps/workbench/app/views/projects/_show_data_collections.html.erb
apps/workbench/app/views/projects/_show_jobs_and_pipelines.html.erb
apps/workbench/app/views/projects/_show_other_objects.html.erb
apps/workbench/app/views/projects/_show_pipeline_templates.html.erb
apps/workbench/app/views/projects/_show_sharing.html.erb
apps/workbench/app/views/projects/_show_subprojects.html.erb
apps/workbench/app/views/projects/_show_tab_contents.html.erb
apps/workbench/app/views/projects/show.html.erb [new file with mode: 0644]
apps/workbench/test/functional/projects_controller_test.rb
apps/workbench/test/integration/projects_test.rb
apps/workbench/test/test_helper.rb
crunch_scripts/collection-merge
crunch_scripts/decompress-all.py [new file with mode: 0755]
crunch_scripts/run-command
crunch_scripts/split-fastq.py [new file with mode: 0755]
crunch_scripts/subst.py
docker/jobs/Dockerfile
sdk/cli/bin/arv-run-pipeline-instance
services/api/app/controllers/arvados/v1/repositories_controller.rb
services/api/app/models/job.rb
services/api/test/fixtures/jobs.yml
services/api/test/fixtures/pipeline_instances.yml

index 2cb9748dadc09e71596b9cd6e4264d11e16389c0..e70ca259e4766c0ca00de11b5dcb9d3dc13200ab 100644 (file)
@@ -111,6 +111,7 @@ $(document).
                 $scroller = $(window);
             $scroller.
                 addClass('infinite-scroller').
-                on('scroll', { container: this }, maybe_load_more_content);
+                on('scroll resize', { container: this }, maybe_load_more_content).
+                trigger('scroll');
         });
     });
index 0802113911084e5911b4d2f08382cee8bdfa0565..222888085d9e3bb95973f2a2865c3efbb569b7f5 100644 (file)
@@ -90,6 +90,9 @@ class ApplicationController < ActionController::Base
   end
 
   def load_filters_and_paging_params
+    @order = params[:order] || 'created_at desc'
+    @order = [@order] unless @order.is_a? Array
+
     @limit ||= 200
     if params[:limit]
       @limit = params[:limit].to_i
@@ -129,10 +132,8 @@ class ApplicationController < ActionController::Base
     respond_to do |f|
       f.json { render json: @objects }
       f.html {
-        if params['tab_pane']
-          comparable = self.respond_to? :compare
-          render(partial: 'show_' + params['tab_pane'].downcase,
-                 locals: { comparable: comparable, objects: @objects })
+        if params[:tab_pane]
+          render_pane params[:tab_pane]
         else
           render
         end
@@ -141,6 +142,23 @@ class ApplicationController < ActionController::Base
     end
   end
 
+  helper_method :render_pane
+  def render_pane tab_pane, opts={}
+    render_opts = {
+      partial: 'show_' + tab_pane.downcase,
+      locals: {
+        comparable: self.respond_to?(:compare),
+        objects: @objects,
+        tab_pane: tab_pane
+      }.merge(opts[:locals] || {})
+    }
+    if opts[:to_string]
+      render_to_string render_opts
+    else
+      render render_opts
+    end
+  end
+
   def index
     find_objects_for_index if !@objects
     render_index
@@ -178,9 +196,7 @@ class ApplicationController < ActionController::Base
       f.json { render json: @object.attributes.merge(href: url_for(@object)) }
       f.html {
         if params['tab_pane']
-          comparable = self.respond_to? :compare
-          render(partial: 'show_' + params['tab_pane'].downcase,
-                 locals: { comparable: comparable, objects: @objects })
+          render_pane params['tab_pane']
         elsif request.method.in? ['GET', 'HEAD']
           render
         else
index 9f4c58877581dfdba308d9167c8610b2f7b4ac31..6287793f765bf82ddd0e15e3037f7a11c7d3c05a 100644 (file)
@@ -128,20 +128,61 @@ class ProjectsController < ApplicationController
     super
   end
 
+  def load_contents_objects kinds=[]
+    kind_filters = @filters.select do |attr,op,val|
+      op == 'is_a' and val.is_a? Array and val.count > 1
+    end
+    if /^created_at\b/ =~ @order[0] and kind_filters.count == 1
+      # If filtering on multiple types and sorting by date: Get the
+      # first page of each type, sort the entire set, truncate to one
+      # page, and use the last item on this page as a filter for
+      # retrieving the next page. Ideally the API would do this for
+      # us, but it doesn't (yet).
+      nextpage_operator = /\bdesc$/i =~ @order[0] ? '<' : '>'
+      @objects = []
+      @name_link_for = {}
+      kind_filters.each do |attr,op,val|
+        (val.is_a?(Array) ? val : [val]).each do |type|
+          objects = @object.contents(order: @order,
+                                     limit: @limit,
+                                     include_linked: true,
+                                     filters: (@filters - kind_filters + [['uuid', 'is_a', type]]),
+                                     offset: @offset)
+          objects.each do |object|
+            @name_link_for[object.andand.uuid] = objects.links_for(object, 'name').first
+          end
+          @objects += objects
+        end
+      end
+      @objects = @objects.to_a.sort_by(&:created_at)
+      @objects.reverse! if nextpage_operator == '<'
+      @objects = @objects[0..@limit-1]
+      @next_page_filters = @filters.reject do |attr,op,val|
+        attr == 'created_at' and op == nextpage_operator
+      end
+      if @objects.any?
+        @next_page_filters += [['created_at',
+                                nextpage_operator,
+                                @objects.last.created_at]]
+        @next_page_href = url_for(partial: :contents_rows,
+                                  filters: @next_page_filters.to_json)
+      else
+        @next_page_href = nil
+      end
+    else
+      @objects = @object.contents(order: @order,
+                                  limit: @limit,
+                                  include_linked: true,
+                                  filters: @filters,
+                                  offset: @offset)
+      @next_page_href = next_page_href(partial: :contents_rows)
+    end
+  end
+
   def show
     if !@object
       return render_not_found("object not found")
     end
-    @objects = @object.contents(limit: 50,
-                                include_linked: true,
-                                filters: params[:filters],
-                                offset: params[:offset] || 0)
-    @logs = Log.limit(10).filter([['object_uuid', '=', @object.uuid]])
-    @users = User.limit(10000).
-      select(["uuid", "is_active", "first_name", "last_name"]).
-      filter([['is_active', '=', 'true']])
-    @groups = Group.limit(10000).
-      select(["uuid", "name", "description"])
 
     @user_is_manager = false
     @share_links = []
@@ -154,24 +195,19 @@ class ProjectsController < ApplicationController
       end
     end
 
-    @objects_and_names = get_objects_and_names @objects
-
     if params[:partial]
+      load_contents_objects
       respond_to do |f|
         f.json {
           render json: {
             content: render_to_string(partial: 'show_contents_rows.html',
-                                      formats: [:html],
-                                      locals: {
-                                        objects_and_names: @objects_and_names,
-                                        project: @object
-                                      }),
-            next_page_href: (next_page_offset and
-                             url_for(offset: next_page_offset, filters: params[:filters], partial: true))
+                                      formats: [:html]),
+            next_page_href: @next_page_href
           }
         }
       end
     else
+      @objects = []
       super
     end
   end
@@ -188,13 +224,17 @@ class ProjectsController < ApplicationController
   end
 
   helper_method :get_objects_and_names
-  def get_objects_and_names(objects)
+  def get_objects_and_names(objects=nil)
+    objects = @objects if objects.nil?
     objects_and_names = []
     objects.each do |object|
-      if !(name_links = objects.links_for(object, 'name')).empty?
+      if objects.respond_to? :links_for and
+          !(name_links = objects.links_for(object, 'name')).empty?
         name_links.each do |name_link|
           objects_and_names << [object, name_link]
         end
+      elsif @name_link_for.andand[object.uuid]
+        objects_and_names << [object, @name_link_for[object.uuid]]
       elsif object.respond_to? :name
         objects_and_names << [object, object]
       else
index 776787b416e97e4490dcc7171f135ba3cd9b6acd..f7ae90912f51f9facc73b8566d75063eaf3814ee 100644 (file)
@@ -21,8 +21,7 @@
     <div id="<%= pane %>-scroll" style="margin-top:0.5em;">
       <div class="pane-content">
         <% if i == 0 %>
-          <%= render(partial: 'show_' + pane.downcase,
-                     locals: { comparable: comparable, objects: @objects }) %>
+          <%= render_pane pane, to_string: true %>
         <% else %>
           <div class="spinner spinner-32px spinner-h-center"></div>
         <% end %>
index 519e8853f3bd77a10b6e28bc8cb0cafcb3d6bb24..3e50b6eb8c3a97e19fb485909a2dbaf2fcd819a9 100644 (file)
@@ -1,13 +1,6 @@
 <%= content_for :content_top %>
-  <% if @object and @object.is_a?(Group) and @object.group_class == 'project' %>
-  <div class="pull-right">
-    <%= content_for :tab_line_buttons %>
-  </div>
-  <br clear="all" />
-<% else %>
-  <br clear="all" />
-  <div class="pull-right">
-    <%= content_for :tab_line_buttons %>
-  </div>
-<% end %>
+<div class="pull-right">
+  <%= content_for :tab_line_buttons %>
+</div>
+<br clear="all" />
 <%= content_for :tab_panes %>
diff --git a/apps/workbench/app/views/application/_tab_line_buttons.html.erb b/apps/workbench/app/views/application/_tab_line_buttons.html.erb
new file mode 100644 (file)
index 0000000..e69de29
index d6eca3a2155d2e7c50b58e030884fc5c8f323dcc..105e1c356d69f140cc7bcc6cbf0a4069a9be2e61 100644 (file)
@@ -27,4 +27,3 @@
 <% end %>
 
 <%= render partial: 'content', layout: 'content_layout', locals: {pane_list: controller.show_pane_list }%>
-
index 8576d4044069266e971023eb635f333482de14b8..ea55577e7077a28d464975dc49b60a8d50b54900 100644 (file)
@@ -1,8 +1,3 @@
-<% content_for :tab_line_buttons do %>
-<span style="padding-left: 1em">Collection storage status:</span>
-<%= render partial: 'toggle_persist', locals: { uuid: @object.uuid, current_state: (@is_persistent ? 'persistent' : 'cache') } %>
-<% end %>
-
 <% file_tree = @object.andand.files_tree %>
 <% if file_tree.nil? or file_tree.empty? %>
   <p>This collection is empty.</p>
index 2f81073305ad4f7598d61a133ddf8802a0f621e1..d624749a4a1e4a9645ee678e61b3594e0db3f7e5 100644 (file)
@@ -1,16 +1,3 @@
-<% content_for :tab_line_buttons do %>
- <%= form_tag collections_path, method: 'get', remote: true, class: 'form-search' do %>
- <div class="input-group">
-   <%= text_field_tag :search, params[:search], class: 'form-control', placeholder: 'Search collections' %>
-   <span class="input-group-btn">
-     <%= button_tag(class: 'btn btn-info') do %>
-     <span class="glyphicon glyphicon-search"></span>
-     <% end %>
-   </span>
- </div>
- <% end %>
-<% end %>
-
 <%= render partial: "paging", locals: {results: @collections, object: @object} %>
 
 <div style="padding-right: 1em">
diff --git a/apps/workbench/app/views/collections/index.html.erb b/apps/workbench/app/views/collections/index.html.erb
new file mode 100644 (file)
index 0000000..061b05b
--- /dev/null
@@ -0,0 +1,14 @@
+<% content_for :tab_line_buttons do %>
+ <%= form_tag collections_path, method: 'get', remote: true, class: 'form-search' do %>
+ <div class="input-group">
+   <%= text_field_tag :search, params[:search], class: 'form-control', placeholder: 'Search collections' %>
+   <span class="input-group-btn">
+     <%= button_tag(class: 'btn btn-info') do %>
+     <span class="glyphicon glyphicon-search"></span>
+     <% end %>
+   </span>
+ </div>
+ <% end %>
+<% end %>
+
+<%= render file: 'application/index.html.erb', locals: local_assigns %>
index a51c450ea0824e99f30e51322c12cff1710e4427..83dcb4511bfbc36aaae6910492c48e777bcd9cf0 100644 (file)
@@ -90,4 +90,9 @@
   </div>
 </div>
 
-<%= render file: 'application/show.html.erb' %>
+<% content_for :tab_line_buttons do %>
+  <span style="padding-left: 1em">Collection storage status:</span>
+  <%= render partial: 'toggle_persist', locals: { uuid: @object.uuid, current_state: (@is_persistent ? 'persistent' : 'cache') } %>
+<% end %>
+
+<%= render file: 'application/show.html.erb', locals: local_assigns %>
index 0f5cd7aeceaa209f3be7a64ceb981cf13167498d..56177ae6e57cef33c7c9b959a8287eb0f96a5008 100644 (file)
@@ -17,5 +17,7 @@
   <% end %>
 <% end %>
 <%= content_for :content_top %>
-<%= content_for :tab_line_buttons %>
+<div class="pull-right">
+  <%= content_for :tab_line_buttons %>
+</div>
 <%= content_for :tab_panes %>
index 0b78e07d65e875facfee37ee65fba34ad76847e1..08b24f13cb03faa1dcb17c1c1830f90aa5706c64 100644 (file)
@@ -1,10 +1,3 @@
-<%= content_for :tab_line_buttons do %>
-<%= form_tag({action: 'compare', controller: params[:controller], method: 'get'}, {method: 'get', id: 'compare', class: 'pull-right small-form-margin'}) do |f| %>
-  <%= submit_tag 'Compare 2 or 3 selected', {class: 'btn btn-primary', disabled: true, style: 'display: none'} %>
-  &nbsp;
-<% end rescue nil %>
-<% end %>
-
 <%= render partial: "paging", locals: {results: @objects, object: @object} %>
 
 <%= form_tag do |f| %>
diff --git a/apps/workbench/app/views/pipeline_instances/index.html.erb b/apps/workbench/app/views/pipeline_instances/index.html.erb
new file mode 100644 (file)
index 0000000..4b73bd4
--- /dev/null
@@ -0,0 +1,8 @@
+<% content_for :tab_line_buttons do %>
+<%= form_tag({action: 'compare', controller: params[:controller], method: 'get'}, {method: 'get', id: 'compare', class: 'pull-right small-form-margin'}) do |f| %>
+  <%= submit_tag 'Compare 2 or 3 selected', {class: 'btn btn-primary', disabled: true, style: 'display: none'} %>
+  &nbsp;
+<% end rescue nil %>
+<% end %>
+
+<%= render file: 'application/index.html.erb', locals: local_assigns %>
index 1f2c1baaf4930dda3da6a179652d9fe43fdf7fe8..cd03a5c790ef6bbde11590dc18d4acc4f621ef41 100644 (file)
@@ -1,18 +1 @@
-<% content_for :tab_line_buttons do %>
-  <%= button_to(choose_projects_path(id: "run-pipeline-button",
-                                     title: 'Choose project',
-                                     editable: true,
-                                     action_name: 'Choose',
-                                     action_href: pipeline_instances_path,
-                                     action_method: 'post',
-                                     action_data: {selection_param: 'pipeline_instance[owner_uuid]',
-                                                   'pipeline_instance[pipeline_template_uuid]' => @object.uuid,
-                                                   'success' => 'redirect-to-created-object'
-                                                  }.to_json),
-                { class: "btn btn-primary btn-sm", remote: true, method: 'get' }
-               ) do %>
-                   Run this pipeline
-                 <% end %>
-<% end %>
-
 <%= render_pipeline_components("editable", :json, editable: false) %>
diff --git a/apps/workbench/app/views/pipeline_templates/show.html.erb b/apps/workbench/app/views/pipeline_templates/show.html.erb
new file mode 100644 (file)
index 0000000..725d63d
--- /dev/null
@@ -0,0 +1,18 @@
+<% content_for :tab_line_buttons do %>
+  <%= button_to(choose_projects_path(id: "run-pipeline-button",
+                                     title: 'Choose project',
+                                     editable: true,
+                                     action_name: 'Choose',
+                                     action_href: pipeline_instances_path,
+                                     action_method: 'post',
+                                     action_data: {selection_param: 'pipeline_instance[owner_uuid]',
+                                                   'pipeline_instance[pipeline_template_uuid]' => @object.uuid,
+                                                   'success' => 'redirect-to-created-object'
+                                                  }.to_json),
+                { class: "btn btn-primary btn-sm", remote: true, method: 'get' }
+               ) do %>
+                   Run this pipeline
+                 <% end %>
+<% end %>
+
+<%= render file: 'application/show.html.erb', locals: local_assigns %>
index b690a1bf8f621b53556bf5480f3c536eba6e55be..d3a2e98cc313e7564a6e6486de86dd9c7e96c705 100644 (file)
@@ -1,14 +1,17 @@
-<% objects_and_names.each do |object, name_link| %>
+<% get_objects_and_names.each do |object, name_link| %>
   <% name_object = (object.respond_to?(:name) || !name_link) ? object : name_link %>
   <tr class="filterable"
       data-object-uuid="<%= name_object.uuid %>"
       data-kind="<%= object.kind %>"
+      data-object-created-at="<%= object.created_at %>"
       >
     <td>
-      <%= render partial: 'selection_checkbox', locals: {object: name_object, friendly_name: ((name_object.name rescue '') || '')} %>
+      <div style="width:1em; display:inline-block;">
+        <%= render partial: 'selection_checkbox', locals: {object: name_object, friendly_name: ((name_object.name rescue '') || '')} %>
+      </div>
 
-      <% if project.editable? %>
-        <%= link_to({action: 'remove_item', id: project.uuid, item_uuid: ((name_link && name_link.uuid) || object.uuid)}, method: :delete, remote: true, data: {confirm: "Remove #{object.class_for_display.downcase} #{name_object.name rescue object.uuid} from this project?", toggle: 'tooltip', placement: 'top'}, class: 'btn btn-sm btn-default btn-nodecorate', title: 'remove') do %>
+      <% if @object.editable? %>
+        <%= link_to({action: 'remove_item', id: @object.uuid, item_uuid: ((name_link && name_link.uuid) || object.uuid)}, method: :delete, remote: true, data: {confirm: "Remove #{object.class_for_display.downcase} #{name_object.name rescue object.uuid} from this project?", toggle: 'tooltip', placement: 'top'}, class: 'btn btn-sm btn-default btn-nodecorate', title: 'remove') do %>
           <i class="fa fa-fw fa-trash-o"></i>
         <% end %>
       <% else %>
index 337629a4c812a1084d29dad136a35b84fb5ef194..e56321dde8a55b4137e082882b8afe77244d7e80 100644 (file)
@@ -1,69 +1,3 @@
-<% if @object.uuid != current_user.uuid # Not the "Home" project %>
-<% content_for :content_top do %>
-
-<h2>
-  <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => "New project" } %>
-</h2>
-
-<div class="arv-description-as-subtitle">
-  <%= render_editable_attribute @object, 'description', nil, { 'data-emptytext' => "(No description provided)", 'data-toggle' => 'manual' } %>
-</div>
-
-<% end %>
-<% end %>
-
-<% content_for :tab_line_buttons do %>
-  <% if @object.editable? %>
-    <%= link_to(
-         choose_collections_path(
-           title: 'Add data to project:',
-           multiple: true,
-           action_name: 'Add',
-           action_href: actions_path(id: @object.uuid),
-           action_method: 'post',
-           action_data: {selection_param: 'selection[]', copy_selections_into_project: @object.uuid, success: 'page-refresh'}.to_json),
-         { class: "btn btn-primary btn-sm", remote: true, method: 'get', data: {'event-after-select' => 'page-refresh'} }) do %>
-      <i class="fa fa-fw fa-plus"></i> Add data...
-    <% end %>
-    <%= link_to(
-         choose_pipeline_templates_path(
-           title: 'Choose a pipeline to run:',
-           action_name: 'Next: choose inputs <i class="fa fa-fw fa-arrow-circle-right"></i>',
-           action_href: pipeline_instances_path,
-           action_method: 'post',
-           action_data: {'selection_param' => 'pipeline_instance[pipeline_template_uuid]', 'pipeline_instance[owner_uuid]' => @object.uuid, 'success' => 'redirect-to-created-object'}.to_json),
-         { class: "btn btn-primary btn-sm", remote: true, method: 'get' }) do %>
-      <i class="fa fa-fw fa-gear"></i> Run a pipeline...
-    <% end %>
-    <%= link_to projects_path('project[owner_uuid]' => @object.uuid), method: 'post', class: 'btn btn-sm btn-primary' do %>
-      <i class="fa fa-fw fa-plus"></i>
-      Add a subproject
-    <% end %>
-    <% if @object.uuid != current_user.uuid # Not the "Home" project %>
-      <%= link_to(
-          choose_projects_path(
-           title: 'Move this project to...',
-           editable: true,
-           my_root_selectable: true,
-           action_name: 'Move',
-           action_href: project_path(@object.uuid),
-           action_method: 'put',
-           action_data: {selection_param: 'project[owner_uuid]', success: 'page-refresh'}.to_json),
-          { class: "btn btn-sm btn-primary arv-move-to-project", remote: true, method: 'get' }) do %>
-        <i class="fa fa-fw fa-truck"></i> Move project...
-      <% end %>
-      <%= link_to(project_path(id: @object.uuid), method: 'delete', class: 'btn btn-sm btn-primary', data: {confirm: "Really delete project '#{@object.name}'?"}) do %>
-        <i class="fa fa-fw fa-trash-o"></i> Delete project
-      <% end %>
-    <% end %>
-  <% end %>
-<% end %>
-
-<% 
-  filters = [['uuid', 'is_a', "arvados#collection"]]
-  @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
-  objects_and_names = get_objects_and_names @objects
-  page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Data_collections'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+    filters: [['uuid', 'is_a', "arvados#collection"]]
+    }.merge(local_assigns) %>
index f9e6306a9d44f01130b473442698563f35f34b56..df31fec8ee0bf19df7058fa358d0330636d06798 100644 (file)
@@ -1,8 +1,3 @@
-<%
-  filters = [['uuid', 'is_a', ["arvados#pipelineInstance","arvados#job"]]]
-  @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
-  objects_and_names = get_objects_and_names @objects
-  page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Jobs_and_pipelines'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+    filters: [['uuid', 'is_a', ["arvados#job", "arvados#pipelineInstance"]]]
+    }.merge(local_assigns) %>
index 308034ed5429559fdab0bc73b0745ca4e5c23e45..af6fbd1a92ab9b5049e64d635741f4295e7a46a5 100644 (file)
@@ -1,8 +1,3 @@
-<%
-  filters = [['uuid', 'is_a', ["arvados#human","arvados#specimen","arvados#trait"]]]
-  @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
-  objects_and_names = get_objects_and_names @objects
-  page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Other_objects'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+    filters: [['uuid', 'is_a', ["arvados#human", "arvados#specimen", "arvados#trait"]]]
+    }.merge(local_assigns) %>
index c54e28dfebdb9170da31e6b5913ad1c86f49d7ad..b875b086ec3e1528a1a29fb35df40ba1b239dd7c 100644 (file)
@@ -1,8 +1,3 @@
-<%
-  filters = [['uuid', 'is_a', "arvados#pipelineTemplate"]]
-  @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
-  objects_and_names = get_objects_and_names @objects
-  page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Pipeline_templates'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+    filters: [['uuid', 'is_a', ["arvados#pipelineTemplate"]]]
+    }.merge(local_assigns) %>
index ad23065876cfea66acc6caa31eb05b46770ab92b..ff0062c24bb15b44c72e13628d44f659f2e3dff2 100644 (file)
@@ -1,7 +1,13 @@
 <%
    uuid_map = {}
-   [@users, @groups].each do |obj_list|
-     obj_list.each { |o| uuid_map[o.uuid] = o }
+   if @share_links
+     [User, Group].each do |type|
+       type.limit(10000)
+         .filter([['uuid','in',@share_links.collect(&:tail_uuid)]])
+         .each do |o|
+         uuid_map[o.uuid] = o
+       end
+     end
    end
    perm_name_desc_map = {}
    perm_desc_name_map = {}
index 4497ca4ea739a9dfc180f3cc2f37697b4313a20f..2c0ba6017831b5cab5c2361a98aa5a49ed9f1e1a 100644 (file)
@@ -1,8 +1,3 @@
-<%
-  filters = [['uuid', 'is_a', "arvados#group"]]
-  @objects = @object.contents({limit: 50, include_linked: true, :filters => filters})
-  objects_and_names = get_objects_and_names @objects
-  page_offset = next_page_offset @objects
-%>
-
-<%= render partial: 'show_tab_contents', locals: {project: @object, objects_and_names: objects_and_names, filters: filters, page_offset: page_offset, tab_name: 'Subprojects'} %>
+<%= render_pane 'tab_contents', to_string: true, locals: {
+    filters: [['uuid', 'is_a', ["arvados#group"]]]
+    }.merge(local_assigns) %>
index df37d3e1c7051ad6b1da1fcb0af7fb77d6aecc91..c9c4dc06d1e41062571de3e31629e02ad23203a8 100644 (file)
@@ -2,7 +2,7 @@
   <div class="row">
     <div class="col-sm-5">
       <div class="btn-group btn-group-sm">
-        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">Selection... <i class="fa fa-fw fa-long-arrow-down "></i></button>
+        <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown">Selection... <span class="caret"></span></button>
         <ul class="dropdown-menu" role="menu">
           <li><%= link_to "Compare selected", '#',
                   'data-href' => compare_pipeline_instances_path,
       </div>
     </div>
     <div class="col-sm-4 pull-right">
-      <input type="text" class="form-control filterable-control" placeholder="Search project contents" data-filterable-target="table.arv-index.arv-project-<%= tab_name %> tbody"/>
+      <input type="text" class="form-control filterable-control" placeholder="Search project contents" data-filterable-target="table.arv-index.arv-project-<%= tab_pane %> tbody"/>
     </div>
   </div>
 
-  <table class="table table-condensed table-fixedlayout arv-index arv-project-<%= tab_name %>" style="overflow-x: hidden">
+  <table class="table table-condensed table-fixedlayout arv-index arv-project-<%= tab_pane %>" style="overflow-x: hidden">
     <colgroup>
       <col width="40%" />
       <col width="60%" />
     </colgroup>
-    <tbody data-infinite-scroller="#<%= tab_name %>-scroll" data-infinite-content-href="<%= url_for(format: :json, partial: :contents_rows, offset: page_offset, filters: "#{filters}") if page_offset %>">
-      <%= render partial: 'show_contents_rows', locals: {project: @object, objects_and_names: objects_and_names} %>
+    <tbody data-infinite-scroller="#<%= tab_pane %>-scroll" data-infinite-content-href="<%= url_for partial: :contents_rows, filters: filters.to_json %>">
     </tbody>
     <thead>
       <tr>
@@ -55,5 +54,4 @@
       </tr>
     </thead>
   </table>
-
 </div>
diff --git a/apps/workbench/app/views/projects/show.html.erb b/apps/workbench/app/views/projects/show.html.erb
new file mode 100644 (file)
index 0000000..c221ca1
--- /dev/null
@@ -0,0 +1,62 @@
+<% if @object.uuid != current_user.uuid # Not the "Home" project %>
+<% content_for :content_top do %>
+
+<h2>
+  <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => "New project" } %>
+</h2>
+
+<div class="arv-description-as-subtitle">
+  <%= render_editable_attribute @object, 'description', nil, { 'data-emptytext' => "(No description provided)", 'data-toggle' => 'manual' } %>
+</div>
+
+<% end %>
+<% end %>
+
+<% content_for :tab_line_buttons do %>
+  <% if @object.editable? %>
+    <%= link_to(
+         choose_collections_path(
+           title: 'Add data to project:',
+           multiple: true,
+           action_name: 'Add',
+           action_href: actions_path(id: @object.uuid),
+           action_method: 'post',
+           action_data: {selection_param: 'selection[]', copy_selections_into_project: @object.uuid, success: 'page-refresh'}.to_json),
+         { class: "btn btn-primary btn-sm", remote: true, method: 'get', data: {'event-after-select' => 'page-refresh'} }) do %>
+      <i class="fa fa-fw fa-plus"></i> Add data...
+    <% end %>
+    <%= link_to(
+         choose_pipeline_templates_path(
+           title: 'Choose a pipeline to run:',
+           action_name: 'Next: choose inputs <i class="fa fa-fw fa-arrow-circle-right"></i>',
+           action_href: pipeline_instances_path,
+           action_method: 'post',
+           action_data: {'selection_param' => 'pipeline_instance[pipeline_template_uuid]', 'pipeline_instance[owner_uuid]' => @object.uuid, 'success' => 'redirect-to-created-object'}.to_json),
+         { class: "btn btn-primary btn-sm", remote: true, method: 'get' }) do %>
+      <i class="fa fa-fw fa-gear"></i> Run a pipeline...
+    <% end %>
+    <%= link_to projects_path('project[owner_uuid]' => @object.uuid), method: 'post', class: 'btn btn-sm btn-primary' do %>
+      <i class="fa fa-fw fa-plus"></i>
+      Add a subproject
+    <% end %>
+    <% if @object.uuid != current_user.uuid # Not the "Home" project %>
+      <%= link_to(
+          choose_projects_path(
+           title: 'Move this project to...',
+           editable: true,
+           my_root_selectable: true,
+           action_name: 'Move',
+           action_href: project_path(@object.uuid),
+           action_method: 'put',
+           action_data: {selection_param: 'project[owner_uuid]', success: 'page-refresh'}.to_json),
+          { class: "btn btn-sm btn-primary arv-move-to-project", remote: true, method: 'get' }) do %>
+        <i class="fa fa-fw fa-truck"></i> Move project...
+      <% end %>
+      <%= link_to(project_path(id: @object.uuid), method: 'delete', class: 'btn btn-sm btn-primary', data: {confirm: "Really delete project '#{@object.name}'?"}) do %>
+        <i class="fa fa-fw fa-trash-o"></i> Delete project
+      <% end %>
+    <% end %>
+  <% end %>
+<% end %>
+
+<%= render file: 'application/show.html.erb', locals: local_assigns %>
index 39c32ac4ac3d959bc31dc3b91bdffd87af1b440f..ae0cad09e3917213d5f5a266370aed68666f95b0 100644 (file)
@@ -38,7 +38,6 @@ class ProjectsControllerTest < ActionController::TestCase
            format: "json"},
          session_for(:active))
     assert_response :success
-    json_response = Oj.load(@response.body)
     assert_equal(uuid_list, json_response["success"])
   end
 
@@ -50,7 +49,6 @@ class ProjectsControllerTest < ActionController::TestCase
            format: "json"},
          session_for(:project_viewer))
     assert_response 422
-    json_response = Oj.load(@response.body)
     assert(json_response["errors"].andand.
              any? { |msg| msg.start_with?("#{share_uuid}: ") },
            "JSON response missing properly formatted sharing error")
@@ -87,4 +85,74 @@ class ProjectsControllerTest < ActionController::TestCase
   test "viewer can't manage asubproject" do
     refute user_can_manage(:project_viewer, "asubproject")
   end
+
+  test 'projects#show tab infinite scroll partial obeys limit' do
+    get_contents_rows(limit: 1, filters: [['uuid','is_a',['arvados#job']]])
+    assert_response :success
+    assert_equal(1, json_response['content'].scan('<tr').count,
+                 "Did not get exactly one row")
+  end
+
+  ['', ' asc', ' desc'].each do |direction|
+    test "projects#show tab partial orders correctly by #{direction}" do
+      _test_tab_content_order direction
+    end
+  end
+
+  def _test_tab_content_order direction
+    get_contents_rows(limit: 100,
+                      order: "created_at#{direction}",
+                      filters: [['uuid','is_a',['arvados#job',
+                                                'arvados#pipelineInstance']]])
+    assert_response :success
+    not_grouped_by_kind = nil
+    last_timestamp = nil
+    last_kind = nil
+    found_kind = {}
+    json_response['content'].scan /<tr[^>]+>/ do |tr_tag|
+      found_timestamps = 0
+      tr_tag.scan(/\ data-object-created-at=\"(.*?)\"/).each do |t,|
+        if last_timestamp
+          correct_operator = / desc$/ =~ direction ? :>= : :<=
+          assert_operator(last_timestamp, correct_operator, t,
+                          "Rows are not sorted by created_at#{direction}")
+        end
+        last_timestamp = t
+        found_timestamps += 1
+      end
+      assert_equal(1, found_timestamps,
+                   "Content row did not have exactly one timestamp")
+
+      # Confirm that the test for timestamp ordering couldn't have
+      # passed merely because the test fixtures have convenient
+      # timestamps (e.g., there is only one pipeline and one job in
+      # the project being tested, or there are no pipelines at all in
+      # the project being tested):
+      tr_tag.scan /\ data-kind=\"(.*?)\"/ do |kind|
+        if last_kind and last_kind != kind and found_kind[kind]
+          # We saw this kind before, then a different kind, then
+          # this kind again. That means objects are not grouped by
+          # kind.
+          not_grouped_by_kind = true
+        end
+        found_kind[kind] ||= 0
+        found_kind[kind] += 1
+        last_kind = kind
+      end
+    end
+    assert_equal(true, not_grouped_by_kind,
+                 "Could not confirm that results are not grouped by kind")
+  end
+
+  def get_contents_rows params
+    params = {
+      id: api_fixture('users')['active']['uuid'],
+      partial: :contents_rows,
+      format: :json,
+    }.merge(params)
+    encoded_params = Hash[params.map { |k,v|
+                            [k, (v.is_a?(Array) || v.is_a?(Hash)) ? v.to_json : v]
+                          }]
+    get :show, encoded_params, session_for(:active)
+  end
 end
index 38d4063ce2a422695265ad1721b7683a6e2d493e..b2c9d772d14b652688273023eeeb0d7ee9111017 100644 (file)
@@ -30,18 +30,20 @@ class ProjectsTest < ActionDispatch::IntegrationTest
     specimen_uuid = api_fixture('specimens')['owned_by_aproject_with_no_name_link']['uuid']
     visit page_with_token 'active', '/projects/' + project_uuid
     click_link 'Other objects'
-    within(".selection-action-container") do
-      within (first('tr', text: 'Specimen')) do
+    within '.selection-action-container' do
+      # Wait for the tab to load:
+      assert_selector 'tr[data-kind="arvados#specimen"]'
+      within first('tr', text: 'Specimen') do
         find(".fa-pencil").click
         find('.editable-input input').set('Now I have a name.')
         find('.glyphicon-ok').click
-        find('.editable', text: 'Now I have a name.').click
+        assert_selector '.editable', text: 'Now I have a name.'
         find(".fa-pencil").click
         find('.editable-input input').set('Now I have a new name.')
         find('.glyphicon-ok').click
-        end
+      end
       wait_for_ajax
-      find('.editable', text: 'Now I have a new name.')
+      assert_selector '.editable', text: 'Now I have a new name.'
     end
     visit current_path
     click_link 'Other objects'
index e833f970c02d211905cd504a4d7a25a13a25a016..260bff043c8cfc10cfc08f3d272835c0be214d51 100644 (file)
@@ -70,6 +70,9 @@ class ActiveSupport::TestCase
       arvados_api_token: api_fixture('api_client_authorizations')[api_client_auth_name.to_s]['api_token']
     }
   end
+  def json_response
+    Oj.load(@response.body)
+  end
 end
 
 class ApiServerForTests
index f16d62466a1e1853b56266ed58ef9935fd3f974b..63b63fa95152375946a50ed4d4e1fd482e380c7a 100755 (executable)
@@ -1,5 +1,18 @@
 #!/usr/bin/env python
 
+# collection-merge
+#
+# Merge two or more collections together.  Can also be used to extract specific
+# files from a collection to produce a new collection.
+#
+# input:
+# An array of collections or collection/file paths in script_parameter["input"]
+#
+# output:
+# A manifest with the collections merged.  Duplicate file names will
+# have their contents concatenated in the order that they appear in the input
+# array.
+
 import arvados
 import md5
 import subst
@@ -30,28 +43,4 @@ for c in p["input"]:
                 if fn in s.files():
                     merged += s.files()[fn].as_manifest()
 
-crm = arvados.CollectionReader(merged)
-
-combined = crm.manifest_text(strip=True)
-
-m = hashlib.new('md5')
-m.update(combined)
-
-uuid = "{}+{}".format(m.hexdigest(), len(combined))
-
-collection = arvados.api().collections().create(
-    body={
-        'uuid': uuid,
-        'manifest_text': crm.manifest_text(),
-    }).execute()
-
-for s in src:
-    l = arvados.api().links().create(body={
-        "link": {
-            "tail_uuid": s,
-            "head_uuid": uuid,
-            "link_class": "provenance",
-            "name": "provided"
-        }}).execute()
-
-arvados.current_task().set_output(uuid)
+arvados.current_task().set_output(merged)
diff --git a/crunch_scripts/decompress-all.py b/crunch_scripts/decompress-all.py
new file mode 100755 (executable)
index 0000000..c1e1e82
--- /dev/null
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+#
+# decompress-all.py
+#
+# Decompress all compressed files in the collection using the "dtrx" tool and
+# produce a new collection with the contents.  Uncompressed files
+# are passed through.
+#
+# input:
+# A collection at script_parameters["input"]
+#
+# output:
+# A manifest of the uncompressed contents of the input collection.
+
+import arvados
+import re
+import subprocess
+import os
+import sys
+
+arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True,
+                                          input_as_path=True)
+
+task = arvados.current_task()
+
+input_file = task['parameters']['input']
+
+infile_parts = re.match(r"(^[a-f0-9]{32}\+\d+)(\+\S+)*(/.*)?(/[^/]+)$", input_file)
+
+outdir = os.path.join(task.tmpdir, "output")
+os.makedirs(outdir)
+os.chdir(outdir)
+
+if infile_parts == None:
+    print >>sys.stderr, "Failed to parse input filename '%s' as a Keep file\n" % input_file
+    sys.exit(1)
+
+cr = arvados.CollectionReader(infile_parts.group(1))
+streamname = infile_parts.group(3)[1:]
+filename = infile_parts.group(4)[1:]
+
+if streamname != None:
+    subprocess.call(["mkdir", "-p", streamname])
+    os.chdir(streamname)
+else:
+    streamname = '.'
+
+m = re.match(r'.*\.(gz|Z|bz2|tgz|tbz|zip|rar|7z|cab|deb|rpm|cpio|gem)$', arvados.get_task_param_mount('input'), re.IGNORECASE)
+
+if m != None:
+    rc = subprocess.call(["dtrx", "-r", "-n", "-q", arvados.get_task_param_mount('input')])
+    if rc == 0:
+        out = arvados.CollectionWriter()
+        out.write_directory_tree(outdir, max_manifest_depth=0)
+        task.set_output(out.finish())
+    else:
+        sys.exit(rc)
+else:
+    streamreader = filter(lambda s: s.name() == streamname, cr.all_streams())[0]
+    filereader = streamreader.files()[filename]
+    task.set_output(streamname + filereader.as_manifest()[1:])
index e6ec889bbcbdbd731b7e6930379b1957062b4178..7d77248674465d253fbd340330c226c3d76480fa 100755 (executable)
@@ -10,8 +10,15 @@ import subst
 import time
 import arvados.commands.put as put
 import signal
+import stat
+import copy
+import traceback
+import pprint
+import multiprocessing
+import logging
 
 os.umask(0077)
+logging.basicConfig(format="run-command: %(message)s")
 
 t = arvados.current_task().tmpdir
 
@@ -23,30 +30,46 @@ os.mkdir("output")
 
 os.chdir("output")
 
+outdir = os.getcwd()
+
+taskp = None
+jobp = arvados.current_job()['script_parameters']
 if len(arvados.current_task()['parameters']) > 0:
-    p = arvados.current_task()['parameters']
-else:
-    p = arvados.current_job()['script_parameters']
+    taskp = arvados.current_task()['parameters']
 
 links = []
 
 def sub_link(v):
-    r = os.path.basename(v)
-    os.symlink(os.path.join(os.environ['TASK_KEEPMOUNT'], v) , r)
+    r = os.path.join(outdir, os.path.basename(v))
+    os.symlink(v, r)
     links.append(r)
     return r
 
 def sub_tmpdir(v):
     return os.path.join(arvados.current_task().tmpdir, 'tmpdir')
 
+def sub_outdir(v):
+    return outdir
+
 def sub_cores(v):
-     return os.environ['CRUNCH_NODE_SLOTS']
+     return str(multiprocessing.cpu_count())
+
+def sub_jobid(v):
+     return os.environ['JOB_UUID']
+
+def sub_taskid(v):
+     return os.environ['TASK_UUID']
+
+def sub_jobsrc(v):
+     return os.environ['CRUNCH_SRC']
 
 subst.default_subs["link "] = sub_link
-subst.default_subs["tmpdir"] = sub_tmpdir
+subst.default_subs["task.tmpdir"] = sub_tmpdir
+subst.default_subs["task.outdir"] = sub_outdir
+subst.default_subs["job.srcdir"] = sub_jobsrc
 subst.default_subs["node.cores"] = sub_cores
-
-rcode = 1
+subst.default_subs["job.uuid"] = sub_jobid
+subst.default_subs["task.uuid"] = sub_taskid
 
 def machine_progress(bytes_written, bytes_expected):
     return "run-command: wrote {} total {}\n".format(
@@ -60,19 +83,87 @@ class SigHandler(object):
         sp.send_signal(signum)
         self.sig = signum
 
+def expand_item(p, c):
+    if isinstance(c, dict):
+        if "foreach" in c and "command" in c:
+            var = c["foreach"]
+            items = get_items(p, p[var])
+            r = []
+            for i in items:
+                params = copy.copy(p)
+                params[var] = i
+                r.extend(expand_list(params, c["command"]))
+            return r
+    elif isinstance(c, list):
+        return expand_list(p, c)
+    elif isinstance(c, str) or isinstance(c, unicode):
+        return [subst.do_substitution(p, c)]
+
+    return []
+
+def expand_list(p, l):
+    return [exp for arg in l for exp in expand_item(p, arg)]
+
+def get_items(p, value):
+    if isinstance(value, list):
+        return expand_list(p, value)
+
+    fn = subst.do_substitution(p, value)
+    mode = os.stat(fn).st_mode
+    prefix = fn[len(os.environ['TASK_KEEPMOUNT'])+1:]
+    if mode != None:
+        if stat.S_ISDIR(mode):
+            items = ["$(dir %s/%s/)" % (prefix, l) for l in os.listdir(fn)]
+        elif stat.S_ISREG(mode):
+            with open(fn) as f:
+                items = [line for line in f]
+        return items
+    else:
+        return None
+
+stdoutname = None
+stdoutfile = None
+rcode = 1
+
 try:
-    cmd = []
-    for c in p["command"]:
-        cmd.append(subst.do_substitution(p, c))
-
-    stdoutname = None
-    stdoutfile = None
-    if "stdout" in p:
-        stdoutname = subst.do_substitution(p, p["stdout"])
+    if "task.foreach" in jobp:
+        if arvados.current_task()['sequence'] == 0:
+            var = jobp["task.foreach"]
+            items = get_items(jobp, jobp[var])
+            logging.info("parallelizing on %s with items %s" % (var, items))
+            if items != None:
+                for i in items:
+                    params = copy.copy(jobp)
+                    params[var] = i
+                    arvados.api().job_tasks().create(body={
+                        'job_uuid': arvados.current_job()['uuid'],
+                        'created_by_job_task_uuid': arvados.current_task()['uuid'],
+                        'sequence': 1,
+                        'parameters': params
+                        }
+                    ).execute()
+                arvados.current_task().set_output(None)
+                sys.exit(0)
+            else:
+                sys.exit(1)
+    else:
+        taskp = jobp
+
+    cmd = expand_list(taskp, taskp["command"])
+
+    if "save.stdout" in taskp:
+        stdoutname = subst.do_substitution(taskp, taskp["save.stdout"])
         stdoutfile = open(stdoutname, "wb")
 
-    print("run-command: {}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+    logging.info("{}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+
+except Exception as e:
+    logging.exception("caught exception")
+    logging.error("task parameters was:")
+    logging.error(pprint.pformat(taskp))
+    sys.exit(1)
 
+try:
     sp = subprocess.Popen(cmd, shell=False, stdout=stdoutfile)
     sig = SigHandler()
 
@@ -85,13 +176,13 @@ try:
     rcode = sp.wait()
 
     if sig.sig != None:
-        print("run-command: terminating on signal %s" % sig.sig)
+        logging.critical("terminating on signal %s" % sig.sig)
         sys.exit(2)
     else:
-        print("run-command: completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
+        logging.info("completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
 
 except Exception as e:
-    print("run-command: caught exception: {}".format(e))
+    logging.exception("caught exception")
 
 # restore default signal handlers.
 signal.signal(signal.SIGINT, signal.SIG_DFL)
@@ -101,11 +192,11 @@ signal.signal(signal.SIGQUIT, signal.SIG_DFL)
 for l in links:
     os.unlink(l)
 
-print("run-command: the follow output files will be saved to keep:")
+logging.info("the following output files will be saved to keep:")
 
-subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"])
+subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"], stdout=sys.stderr)
 
-print("run-command: start writing output to keep")
+logging.info("start writing output to keep")
 
 done = False
 resume_cache = put.ResumeCache(os.path.join(arvados.current_task().tmpdir, "upload-output-checkpoint"))
@@ -125,10 +216,10 @@ while not done:
                                              }).execute()
         done = True
     except KeyboardInterrupt:
-        print("run-command: terminating on signal 2")
+        logging.critical("terminating on signal 2")
         sys.exit(2)
     except Exception as e:
-        print("run-command: caught exception: {}".format(e))
+        logging.exception("caught exception:")
         time.sleep(5)
 
 sys.exit(rcode)
diff --git a/crunch_scripts/split-fastq.py b/crunch_scripts/split-fastq.py
new file mode 100755 (executable)
index 0000000..be37f04
--- /dev/null
@@ -0,0 +1,128 @@
+#!/usr/bin/python
+
+import arvados
+import re
+import hashlib
+import string
+
+api = arvados.api('v1')
+
+piece = 0
+manifest_text = ""
+
+# Look for paired reads
+
+inp = arvados.CollectionReader(arvados.getjobparam('reads'))
+
+manifest_list = []
+
+chunking = False #arvados.getjobparam('chunking')
+
+def nextline(reader, start):
+    n = -1
+    while True:
+        r = reader.readfrom(start, 128)
+        if r == '':
+            break
+        n = string.find(r, "\n")
+        if n > -1:
+            break
+        else:
+            start += 128
+    return n
+
+# Chunk a fastq into approximately 64 MiB chunks.  Requires that the input data
+# be decompressed ahead of time, such as using decompress-all.py.  Generates a
+# new manifest, but doesn't actually move any data around.  Handles paired
+# reads by ensuring that each chunk of a pair gets the same number of records.
+#
+# This works, but in practice is so slow that potential gains in alignment
+# performance are lost in the prep time, which is why it is currently disabled.
+#
+# A better algorithm would seek to a file position a bit less than the desired
+# chunk size and then scan ahead for the next record, making sure that record
+# was matched by the read pair.
+def splitfastq(p):
+    for i in xrange(0, len(p)):
+        p[i]["start"] = 0
+        p[i]["end"] = 0
+
+    count = 0
+    recordsize = [0, 0]
+
+    global piece
+    finish = False
+    while not finish:
+        for i in xrange(0, len(p)):
+            recordsize[i] = 0
+
+        # read next 4 lines
+        for i in xrange(0, len(p)):
+            for ln in xrange(0, 4):
+                r = nextline(p[i]["reader"], p[i]["end"]+recordsize[i])
+                if r == -1:
+                    finish = True
+                    break
+                recordsize[i] += (r+1)
+
+        splitnow = finish
+        for i in xrange(0, len(p)):
+            if ((p[i]["end"] - p[i]["start"]) + recordsize[i]) >= (64*1024*1024):
+                splitnow = True
+
+        if splitnow:
+            for i in xrange(0, len(p)):
+                global manifest_list
+                print >>sys.stderr, "Finish piece ./_%s/%s (%s %s)" % (piece, p[i]["reader"].name(), p[i]["start"], p[i]["end"])
+                manifest = []
+                manifest.extend(["./_" + str(piece)])
+                manifest.extend([d[arvados.LOCATOR] for d in p[i]["reader"]._stream._data_locators])
+                manifest.extend(["{}:{}:{}".format(seg[arvados.LOCATOR]+seg[arvados.OFFSET], seg[arvados.SEGMENTSIZE], p[i]["reader"].name().replace(' ', '\\040')) for seg in arvados.locators_and_ranges(p[i]["reader"].segments, p[i]["start"], p[i]["end"] - p[i]["start"])])
+                manifest_list.append(manifest)
+                p[i]["start"] = p[i]["end"]
+            piece += 1
+        else:
+            for i in xrange(0, len(p)):
+                p[i]["end"] += recordsize[i]
+            count += 1
+            if count % 10000 == 0:
+                print >>sys.stderr, "Record %s at %s" % (count, p[i]["end"])
+
+prog = re.compile(r'(.*?)(_[12])?\.fastq(\.gz)?$')
+
+# Look for fastq files
+for s in inp.all_streams():
+    for f in s.all_files():
+        name_pieces = prog.match(f.name())
+        if name_pieces != None:
+            if s.name() != ".":
+                # The downstream tool (run-command) only iterates over the top
+                # level of directories so if there are fastq files in
+                # directories in the input, the choice is either to forget
+                # there are directories (which might lead to name conflicts) or
+                # just fail.
+                print >>sys.stderr, "fastq must be at the root of the collection"
+                sys.exit(1)
+
+            p = None
+            if name_pieces.group(2) != None:
+                if name_pieces.group(2) == "_1":
+                    p = [{}, {}]
+                    p[0]["reader"] = s.files()[name_pieces.group(0)]
+                    p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + (name_pieces.group(3) if name_pieces.group(3) else '')]
+            else:
+                p = [{}]
+                p[0]["reader"] = s.files()[name_pieces.group(0)]
+
+            if p != None:
+                if chunking:
+                    splitfastq(p)
+                else:
+                    for i in xrange(0, len(p)):
+                        m = p[i]["reader"].as_manifest()[1:]
+                        manifest_list.append(["./_" + str(piece), m[:-1]])
+                    piece += 1
+
+manifest_text = "\n".join(" ".join(m) for m in manifest_list)
+
+arvados.current_task().set_output(manifest_text)
index 2598e1cc944397324c9bb02b931a8bc7c20ed5a8..8154d0ed0c6dc7ee16c24e9de80a2ed031770687 100644 (file)
@@ -44,7 +44,11 @@ def sub_basename(v):
     return os.path.splitext(os.path.basename(v))[0]
 
 def sub_glob(v):
-    return glob.glob(v)[0]
+    l = glob.glob(v)
+    if len(l) == 0:
+        raise Exception("$(glob): No match on '%s'" % v)
+    else:
+        return l[0]
 
 default_subs = {"file ": sub_file,
                 "dir ": sub_dir,
index 2cad65c52746ecfdd40cb9236440cfbb9c714e11..1b493e3a30a88c68395dee7f52413be3f9f7b6db 100644 (file)
@@ -4,7 +4,7 @@ MAINTAINER Brett Smith <brett@curoverse.com>
 # Install dependencies and set up system.
 # The FUSE packages help ensure that we can install the Python SDK (arv-mount).
 RUN /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
-      libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl \
+      libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl dtrx \
       fuse libattr1-dev libfuse-dev && \
     /usr/sbin/adduser --disabled-password \
       --gecos 'Crunch execution user' crunch && \
@@ -13,7 +13,7 @@ RUN /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
 
 # Install Arvados packages.
 RUN (find /usr/src/arvados/sdk -name '*.gem' -print0 | \
-      xargs -0rn 1 gem install) && \
+      xargs -0rn 1 /usr/local/rvm/bin/rvm-exec default gem install) && \
     cd /usr/src/arvados/services/fuse && \
     python setup.py install && \
     cd /usr/src/arvados/sdk/python && \
index e37f2a9ff61ce725f52f8582bdcb9167af966008..f20acebbd0057fe8131848299e1cda39c45501bf 100755 (executable)
@@ -243,8 +243,8 @@ class PipelineInstance
   end
   def self.create(attributes)
     result = $client.execute(:api_method => $arvados.pipeline_instances.create,
-                             :body => {
-                               :pipeline_instance => attributes.to_json
+                             :body_object => {
+                               :pipeline_instance => attributes
                              },
                              :authenticated => false,
                              :headers => {
@@ -262,8 +262,8 @@ class PipelineInstance
                              :parameters => {
                                :uuid => @pi[:uuid]
                              },
-                             :body => {
-                               :pipeline_instance => @attributes_to_update.to_json
+                             :body_object => {
+                               :pipeline_instance => @attributes_to_update
                              },
                              :authenticated => false,
                              :headers => {
@@ -328,7 +328,7 @@ class JobCache
     body = {job: no_nil_values(job)}.merge(no_nil_values(create_params))
 
     result = $client.execute(:api_method => $arvados.jobs.create,
-                             :body => body,
+                             :body_object => body,
                              :authenticated => false,
                              :headers => {
                                authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
@@ -346,7 +346,7 @@ class JobCache
       msg += "Job submission was: #{body.to_json}"
 
       $client.execute(:api_method => $arvados.logs.create,
-                      :body => {
+                      :body_object => {
                         :log => {
                           :object_uuid => pipeline[:uuid],
                           :event_type => 'stderr',
index 94c172da326ae3e4936b01aceea3f62a35ffee22..0452c523bf3a21348264350c8f9deba96bfefedd 100644 (file)
@@ -18,7 +18,9 @@ class Arvados::V1::RepositoriesController < ApplicationController
         if ArvadosModel::resource_class_for_uuid(perm.tail_uuid) == Group
           @users.each do |user_uuid, user|
             user.group_permissions.each do |group_uuid, perm_mask|
-              if perm_mask[:write]
+              if perm_mask[:manage]
+                perms << {name: 'can_manage', user_uuid: user_uuid}
+              elsif perm_mask[:write]
                 perms << {name: 'can_write', user_uuid: user_uuid}
               elsif perm_mask[:read]
                 perms << {name: 'can_read', user_uuid: user_uuid}
@@ -57,7 +59,11 @@ class Arvados::V1::RepositoriesController < ApplicationController
     end
     @repo_info.values.each do |repo_users|
       repo_users[:user_permissions].each do |user_uuid,perms|
-        if perms['can_write']
+        if perms['can_manage']
+          perms[:gitolite_permissions] = 'RW'
+          perms['can_write'] = true
+          perms['can_read'] = true
+        elsif perms['can_write']
           perms[:gitolite_permissions] = 'RW'
           perms['can_read'] = true
         elsif perms['can_read']
index fc445ae24edb5977a2092f7d92af86c300eb21ad..1ef0b797c044574a2b9eba72e1fcbd0aeba77e16 100644 (file)
@@ -191,7 +191,7 @@ class Job < ArvadosModel
       if self.cancelled_at and not self.cancelled_at_was
         self.cancelled_at = Time.now
         self.cancelled_by_user_uuid = current_user.uuid
-        self.cancelled_by_client_uuid = current_api_client.uuid
+        self.cancelled_by_client_uuid = current_api_client.andand.uuid
         @need_crunch_dispatch_trigger = true
       else
         self.cancelled_at = self.cancelled_at_was
index e142faeb5bde344a8385f9b50b30f96a797049c6..3274381b60b020dfac475af00f30950d887d0163 100644 (file)
@@ -4,6 +4,7 @@ running:
   cancelled_at: ~
   cancelled_by_user_uuid: ~
   cancelled_by_client_uuid: ~
+  created_at: <%= 3.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: ~
   script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
@@ -26,6 +27,7 @@ running_cancelled:
   cancelled_at: <%= 1.minute.ago.to_s(:db) %>
   cancelled_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   cancelled_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+  created_at: <%= 4.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: ~
   script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
@@ -49,6 +51,7 @@ uses_nonexistent_script_version:
   cancelled_by_user_uuid: ~
   cancelled_by_client_uuid: ~
   script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
+  created_at: <%= 5.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: <%= 2.minute.ago.to_s(:db) %>
   running: false
@@ -73,6 +76,7 @@ foobar:
   script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
   script_parameters:
     input: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+  created_at: <%= 4.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: <%= 2.minute.ago.to_s(:db) %>
   running: false
@@ -98,6 +102,7 @@ barbaz:
   script_parameters:
     input: fa7aeb5140e2848d39b416daeef4ffc5+45
     an_integer: 1
+  created_at: <%= 4.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: <%= 2.minute.ago.to_s(:db) %>
   running: false
@@ -116,6 +121,7 @@ barbaz:
 
 previous_job_run:
   uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+  created_at: <%= 14.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   repository: foo
   script: hash
@@ -128,6 +134,7 @@ previous_job_run:
 
 previous_docker_job_run:
   uuid: zzzzz-8i9sb-k6emstgk4kw4yhi
+  created_at: <%= 14.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   repository: foo
   script: hash
@@ -141,6 +148,7 @@ previous_docker_job_run:
 
 previous_job_run_no_output:
   uuid: zzzzz-8i9sb-cjs4pklxxjykppp
+  created_at: <%= 14.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   repository: foo
   script: hash
@@ -153,6 +161,7 @@ previous_job_run_no_output:
 
 nondeterminisic_job_run:
   uuid: zzzzz-8i9sb-cjs4pklxxjykyyy
+  created_at: <%= 14.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   repository: foo
   script: hash2
@@ -165,6 +174,7 @@ nondeterminisic_job_run:
 
 nearly_finished_job:
   uuid: zzzzz-8i9sb-2gx6rz0pjl033w3
+  created_at: <%= 14.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   repository: arvados
   script: doesnotexist
@@ -184,6 +194,7 @@ nearly_finished_job:
 
 queued:
   uuid: zzzzz-8i9sb-grx15v5mjnsyxk7
+  created_at: <%= 1.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   cancelled_at: ~
   cancelled_by_user_uuid: ~
index f73558df401a3fd85fdced9cdb944c970dc26d08..7bfc2e1b3fe65d4882e2528eb6efaba07f858d6f 100644 (file)
@@ -2,11 +2,13 @@ new_pipeline:
   state: New
   uuid: zzzzz-d1hrv-f4gneyn6br1xize
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: <%= 1.minute.ago.to_s(:db) %>
 
 has_component_with_no_script_parameters:
   state: Ready
   uuid: zzzzz-d1hrv-1xfj6xkicf2muk2
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: <%= 10.minute.ago.to_s(:db) %>
   components:
    foo:
     script: foo
@@ -17,6 +19,7 @@ has_component_with_empty_script_parameters:
   state: Ready
   uuid: zzzzz-d1hrv-jq16l10gcsnyumo
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: <%= 3.minute.ago.to_s(:db) %>
   components:
    foo:
     script: foo
@@ -27,6 +30,7 @@ has_job:
   state: Ready
   uuid: zzzzz-d1hrv-1yfj6xkidf2muk3
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  created_at: <%= 3.1.minute.ago.to_s(:db) %>
   components:
    foo:
     script: foo
@@ -41,6 +45,7 @@ components_is_jobspec:
   # Helps test that clients cope with funny-shaped components.
   # For an example, see #3321.
   uuid: zzzzz-d1hrv-jobspeccomponts
+  created_at: <%= 30.minute.ago.to_s(:db) %>
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   created_at: 2014-04-14 12:35:04 -0400
   updated_at: 2014-04-14 12:35:04 -0400