--- /dev/null
+#!/usr/bin/env ruby
+
+# usage: list-inactive-users.rb [n-days-old-to-ignore]
+#
+# (default = 7)
+
+abort 'Error: Ruby >= 1.9.3 required.' if RUBY_VERSION < '1.9.3'
+
+threshold = ARGV.shift.to_i rescue 7
+
+require 'arvados'
+arv = Arvados.new(api_version: 'v1')
+
+saidheader = false
+arv.user.list(where: {is_active: false})[:items].each do |user|
+ if Time.now - Time.parse(user[:created_at]) < threshold*86400
+ if !saidheader
+ saidheader = true
+ puts "Inactive users who first logged in <#{threshold} days ago:"
+ puts ""
+ end
+ puts "#{user[:modified_at]} #{user[:uuid]} #{user[:full_name]} <#{user[:email]}>"
+ end
+end
login_perm = arv.link.create(link: {
tail_kind: 'arvados#user',
tail_uuid: user[:uuid],
- head_kind: 'arvados#virtual_machine',
+ head_kind: 'arvados#virtualMachine',
head_uuid: vm[:uuid],
link_class: 'permission',
name: 'can_login',
end
gem 'jquery-rails'
-gem 'twitter-bootstrap-rails'
-gem 'anjlab-bootstrap-rails', '~> 2.3', :require => 'bootstrap-rails'
-gem 'bootstrap-editable-rails'
+gem 'bootstrap-sass', '~> 3.1.0'
+gem 'bootstrap-x-editable-rails'
+
gem 'less'
gem 'less-rails'
i18n (~> 0.6, >= 0.6.4)
multi_json (~> 1.0)
andand (1.3.3)
- anjlab-bootstrap-rails (2.3.1.2)
- railties (>= 3.0)
- sass (>= 3.2)
arel (3.0.2)
- bootstrap-editable-rails (0.0.5)
- railties (>= 3.1)
+ bootstrap-sass (3.1.0.1)
+ sass (~> 3.2)
+ bootstrap-x-editable-rails (1.5.1.1)
+ railties (>= 3.0)
builder (3.0.4)
capistrano (2.15.5)
highline
treetop (1.4.15)
polyglot
polyglot (>= 0.3.1)
- twitter-bootstrap-rails (2.2.8)
- actionpack (>= 3.1)
- execjs
- rails (>= 3.1)
- railties (>= 3.1)
tzinfo (0.3.38)
uglifier (2.3.1)
execjs (>= 0.3.0)
DEPENDENCIES
RedCloth
andand
- anjlab-bootstrap-rails (~> 2.3)
- bootstrap-editable-rails
+ bootstrap-sass (~> 3.1.0)
+ bootstrap-x-editable-rails
coffee-rails (~> 3.2.0)
httpclient
jquery-rails
sqlite3
themes_for_rails
therubyracer
- twitter-bootstrap-rails
uglifier (>= 1.0.3)
//
//= require jquery
//= require jquery_ujs
-//= require twitter/bootstrap
-//= require bootstrap-editable
-//= require bootstrap-editable-rails
+//= require bootstrap
+//= require bootstrap/dropdown
+//= require bootstrap/tab
+//= require bootstrap/tooltip
+//= require bootstrap/popover
+//= require bootstrap3-editable/bootstrap-editable
//= require_tree .
jQuery(function($){
}
targets.fadeToggle(200);
});
+ $(document).
+ on('ajax:send', function(e, xhr) {
+ $('.loading').show();
+ }).
+ on('ajax:complete', function(e, status) {
+ $('.loading').hide();
+ });
})(jQuery);
+
+
+
--- /dev/null
+$.fn.editable.defaults.ajaxOptions = {type: 'put', dataType: 'json'};
+$.fn.editable.defaults.send = 'always';
+$.fn.editable.defaults.params = function (params) {
+ var a = {};
+ var key = params.pk.key;
+ a.id = params.pk.id;
+ a[key] = {};
+ a[key][params.name] = params.value;
+ return a;
+};
\ No newline at end of file
--- /dev/null
+# Place all the behaviors and hooks related to the matching controller here.
+# All this logic will automatically be available in application.js.
+# You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/
--- /dev/null
+function graph_zoom(divId, svgId, scale) {
+ var pg = document.getElementById(divId);
+ vcenter = (pg.scrollTop + (pg.scrollHeight - pg.scrollTopMax)/2.0) / pg.scrollHeight;
+ hcenter = (pg.scrollLeft + (pg.scrollWidth - pg.scrollLeftMax)/2.0) / pg.scrollWidth;
+ var g = document.getElementById(svgId);
+ g.setAttribute("height", parseFloat(g.getAttribute("height")) * scale);
+ g.setAttribute("width", parseFloat(g.getAttribute("width")) * scale);
+ pg.scrollTop = (vcenter * pg.scrollHeight) - (pg.scrollHeight - pg.scrollTopMax)/2.0;
+ pg.scrollLeft = (hcenter * pg.scrollWidth) - (pg.scrollWidth - pg.scrollLeftMax)/2.0;
+ smart_scroll_fixup();
+}
+
+function smart_scroll_fixup(s) {
+ console.log(s);
+ if (s != null && s.type == 'shown.bs.tab') {
+ s = [s.target];
+ }
+ else {
+ s = $(".smart-scroll");
+ }
+ console.log(s);
+ for (var i = 0; i < s.length; i++) {
+ a = s[i];
+ var h = window.innerHeight - a.getBoundingClientRect().top - 20;
+ height = String(h) + "px";
+ a.style.height = height;
+ }
+}
+
+$(window).on('load resize scroll', smart_scroll_fixup);
* compiled file, but it's generally better to create a new file per style scope.
*
*= require_self
- *= require bootstrap_and_overrides
- *= require bootstrap-editable
+ *= require bootstrap
+ *= require bootstrap3-editable/bootstrap-editable
*= require_tree .
*/
form.small-form-margin {
margin-bottom: 2px;
}
+.nowrap {
+ white-space: nowrap;
+}
+
+.navbar .nav li.nav-separator > span {
+ display: block;
+ float: none;
+ color: #bbbbbb;
+ padding: 10px 0 10px;
+ text-decoration: none;
+ text-shadow: 0 1px 0 #ffffff;
+}
+/*.navbar .nav .dropdown .dropdown-menu li a {
+ padding: 2px 20px;
+}*/
+
+ul.arvados-nav {
+ list-style: none;
+ padding-left: 0em;
+ margin-left: 0em;
+}
+
+ul.arvados-nav li ul {
+ list-style: none;
+ padding-left: 0;
+}
+
+ul.arvados-nav li ul li {
+ list-style: none;
+ padding-left: 1em;
+}
+
+.dax {
+ max-width: 10%;
+ margin-right: 1em;
+ float: left
+}
+
+.smart-scroll {
+ overflow: auto;
+}
+
+.inline-progress-container div.progress {
+ margin-bottom: 0;
+}
+
+.inline-progress-container {
+ width: 100px;
+ display:inline-block;
+}
--- /dev/null
+/* Colors
+ * Contextual variations of badges
+ * Bootstrap 3.0 removed contexts for badges, we re-introduce them, based on what is done for labels
+ */
+
+.badge.badge-error {
+ background-color: #b94a48;
+}
+
+.badge.badge-warning {
+ background-color: #f89406;
+}
+
+.badge.badge-success {
+ background-color: #468847;
+}
+
+.badge.badge-info {
+ background-color: #3a87ad;
+}
+
+.badge.badge-inverse {
+ background-color: #333333;
+}
+
+.badge.badge-alert {
+ background: red;
+}
+++ /dev/null
-@import "twitter/bootstrap/bootstrap";
-@import "twitter/bootstrap/responsive";
-
-// Set the correct sprite paths
-@iconSpritePath: asset-path("twitter/bootstrap/glyphicons-halflings");
-@iconWhiteSpritePath: asset-path("twitter/bootstrap/glyphicons-halflings-white");
-
-// Set the Font Awesome (Font Awesome is default. You can disable by commenting below lines)
-@fontAwesomeEotPath: asset-url("fontawesome-webfont.eot");
-@fontAwesomeEotPath_iefix: asset-url("fontawesome-webfont.eot#iefix");
-@fontAwesomeWoffPath: asset-url("fontawesome-webfont.woff");
-@fontAwesomeTtfPath: asset-url("fontawesome-webfont.ttf");
-@fontAwesomeSvgPath: asset-url("fontawesome-webfont.svg#fontawesomeregular");
-
-// Font Awesome
-@import "fontawesome/font-awesome";
-
-// Glyphicons
-//@import "twitter/bootstrap/sprites.less";
-
-// Your custom LESS stylesheets goes here
-//
-// Since bootstrap was imported above you have access to its mixins which
-// you may use and inherit here
-//
-// If you'd like to override bootstrap's own variables, you can do so here as well
-// See http://twitter.github.com/bootstrap/customize.html#variables for their names and documentation
-//
-// Example:
-// @linkColor: #ff0000;
--- /dev/null
+// Place all the styles related to the KeepDisks controller here.
+// They will automatically be included in application.css.
+// You can use Sass (SCSS) here: http://sass-lang.com/
--- /dev/null
+/* http://codepen.io/alucard11/pen/IxLDJ */
+
+.loading {
+ background: #1b1b1b;
+}
+
+.loading .socket{
+ width: 200px;
+ height: 200px;
+ position: absolute;
+ left: 50%;
+ margin-left: -100px;
+ top: 50%;
+ margin-top: -100px;
+}
+
+.loading .hex-brick{
+ background: #ABF8FF;
+ width: 30px;
+ height: 17px;
+ position: absolute;
+ top: 5px;
+ animation-name: fade;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ -webkit-animation-name: fade;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+}
+
+.loading .h2{
+ transform: rotate(60deg);
+ -webkit-transform: rotate(60deg);
+}
+
+.loading .h3{
+ transform: rotate(-60deg);
+ -webkit-transform: rotate(-60deg);
+}
+
+.loading .gel{
+ height: 30px;
+ width: 30px;
+ transition: all .3s;
+ -webkit-transition: all .3s;
+ position: absolute;
+ top: 50%;
+ left: 50%;
+}
+
+.loading .center-gel{
+ margin-left: -15px;
+ margin-top: -15px;
+
+ animation-name: pulse;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ -webkit-animation-name: pulse;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+}
+
+.loading .c1{
+ margin-left: -47px;
+ margin-top: -15px;
+}
+
+.loading .c2{
+ margin-left: -31px;
+ margin-top: -43px;
+}
+
+.loading .c3{
+ margin-left: 1px;
+ margin-top: -43px;
+}
+
+.loading .c4{
+ margin-left: 17px;
+ margin-top: -15px;
+}
+.loading .c5{
+ margin-left: -31px;
+ margin-top: 13px;
+}
+
+.loading .c6{
+ margin-left: 1px;
+ margin-top: 13px;
+}
+
+.loading .c7{
+ margin-left: -63px;
+ margin-top: -43px;
+}
+
+.loading .c8{
+ margin-left: 33px;
+ margin-top: -43px;
+}
+
+.loading .c9{
+ margin-left: -15px;
+ margin-top: 41px;
+}
+
+.loading .c10{
+ margin-left: -63px;
+ margin-top: 13px;
+}
+
+.loading .c11{
+ margin-left: 33px;
+ margin-top: 13px;
+}
+
+.loading .c12{
+ margin-left: -15px;
+ margin-top: -71px;
+}
+
+.loading .c13{
+ margin-left: -47px;
+ margin-top: -71px;
+}
+
+.loading .c14{
+ margin-left: 17px;
+ margin-top: -71px;
+}
+
+.loading .c15{
+ margin-left: -47px;
+ margin-top: 41px;
+}
+
+.loading .c16{
+ margin-left: 17px;
+ margin-top: 41px;
+}
+
+.c17{
+ margin-left: -79px;
+ margin-top: -15px;
+}
+
+.loading .c18{
+ margin-left: 49px;
+ margin-top: -15px;
+}
+
+.loading .c19{
+ margin-left: -63px;
+ margin-top: -99px;
+}
+
+.loading .c20{
+ margin-left: 33px;
+ margin-top: -99px;
+}
+
+.loading .c21{
+ margin-left: 1px;
+ margin-top: -99px;
+}
+
+.loading .c22{
+ margin-left: -31px;
+ margin-top: -99px;
+}
+
+.loading .c23{
+ margin-left: -63px;
+ margin-top: 69px;
+}
+
+.loading .c24{
+ margin-left: 33px;
+ margin-top: 69px;
+}
+
+.loading .c25{
+ margin-left: 1px;
+ margin-top: 69px;
+}
+
+.loading .c26{
+ margin-left: -31px;
+ margin-top: 69px;
+}
+
+.loading .c27{
+ margin-left: -79px;
+ margin-top: -15px;
+}
+
+.loading .c28{
+ margin-left: -95px;
+ margin-top: -43px;
+}
+
+.loading .c29{
+ margin-left: -95px;
+ margin-top: 13px;
+}
+
+.loading .c30{
+ margin-left: 49px;
+ margin-top: 41px;
+}
+
+.loading .c31{
+ margin-left: -79px;
+ margin-top: -71px;
+}
+
+.loading .c32{
+ margin-left: -111px;
+ margin-top: -15px;
+}
+
+.loading .c33{
+ margin-left: 65px;
+ margin-top: -43px;
+}
+
+.loading .c34{
+ margin-left: 65px;
+ margin-top: 13px;
+}
+
+.loading .c35{
+ margin-left: -79px;
+ margin-top: 41px;
+}
+
+.loading .c36{
+ margin-left: 49px;
+ margin-top: -71px;
+}
+
+.loading .c37{
+ margin-left: 81px;
+ margin-top: -15px;
+}
+
+.loading .r1{
+ animation-name: pulse;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ animation-delay: .2s;
+ -webkit-animation-name: pulse;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+ -webkit-animation-delay: .2s;
+}
+
+.loading .r2{
+ animation-name: pulse;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ animation-delay: .4s;
+ -webkit-animation-name: pulse;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+ -webkit-animation-delay: .4s;
+}
+
+.loading .r3{
+ animation-name: pulse;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ animation-delay: .6s;
+ -webkit-animation-name: pulse;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+ -webkit-animation-delay: .6s;
+}
+
+.loading .r1 > .hex-brick{
+ animation-name: fade;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ animation-delay: .2s;
+ -webkit-animation-name: fade;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+ -webkit-animation-delay: .2s;
+}
+
+.loading .r2 > .hex-brick{
+ animation-name: fade;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ animation-delay: .4s;
+ -webkit-animation-name: fade;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+ -webkit-animation-delay: .4s;
+}
+
+.loading .r3 > .hex-brick{
+ animation-name: fade;
+ animation-duration: 2s;
+ animation-iteration-count: infinite;
+ animation-delay: .6s;
+ -webkit-animation-name: fade;
+ -webkit-animation-duration: 2s;
+ -webkit-animation-iteration-count: infinite;
+ -webkit-animation-delay: .6s;
+}
+
+
+@keyframes pulse{
+ 0%{
+ -webkit-transform: scale(1);
+ transform: scale(1);
+ }
+
+ 50%{
+ -webkit-transform: scale(0.01);
+ transform: scale(0.01);
+ }
+
+ 100%{
+ -webkit-transform: scale(1);
+ transform: scale(1);
+ }
+}
+
+@keyframes fade{
+ 0%{
+ background: #ABF8FF;
+ }
+
+ 50%{
+ background: #90BBBF;
+ }
+
+ 100%{
+ background: #ABF8FF;
+ }
+}
+
+@-webkit-keyframes pulse{
+ 0%{
+ -webkit-transform: scale(1);
+ transform: scale(1);
+ }
+
+ 50%{
+ -webkit-transform: scale(0.01);
+ transform: scale(0.01);
+ }
+
+ 100%{
+ -webkit-transform: scale(1);
+ transform: scale(1);
+ }
+}
+
+@-webkit-keyframes fade{
+ 0%{
+ background: #ABF8FF;
+ }
+
+ 50%{
+ background: #389CA6;
+ }
+
+ 100%{
+ background: #ABF8FF;
+ }
+}
end
super
end
+
+ def index_pane_list
+ %w(Recent Help)
+ end
+
end
class ApplicationController < ActionController::Base
+ respond_to :html, :json, :js
protect_from_forgery
around_filter :thread_clear
- around_filter :thread_with_api_token, :except => [:render_exception, :render_not_found]
+ around_filter :thread_with_mandatory_api_token, :except => [:render_exception, :render_not_found]
+ around_filter :thread_with_optional_api_token
before_filter :find_object_by_uuid, :except => [:index, :render_exception, :render_not_found]
before_filter :check_user_agreements, :except => [:render_exception, :render_not_found]
+ before_filter :check_user_notifications, :except => [:render_exception, :render_not_found]
theme :select_theme
begin
self.render_error status: 404
end
-
def index
@objects ||= model_class.limit(1000).all
respond_to do |f|
f.json { render json: @objects }
f.html { render }
+ f.js { render }
end
end
redirect_to params[:return_to] || @object
end
}
+ f.js { render }
end
end
def destroy
if @object.destroy
- redirect_to(params[:return_to] || :back)
+ respond_to do |f|
+ f.html {
+ redirect_to(params[:return_to] || :back)
+ }
+ f.js { render }
+ end
else
self.render_error status: 422
end
controller_name.classify.constantize
end
+ def breadcrumb_page_name
+ (@breadcrumb_page_name ||
+ (@object.friendly_link_name if @object.respond_to? :friendly_link_name))
+ end
+
+ def index_pane_list
+ %w(Recent)
+ end
+
+ def show_pane_list
+ %w(Attributes Metadata JSON API)
+ end
+
protected
def find_object_by_uuid
def thread_clear
Thread.current[:arvados_api_token] = nil
Thread.current[:user] = nil
+ Rails.cache.delete_matched(/^request_#{Thread.current.object_id}_/)
yield
+ Rails.cache.delete_matched(/^request_#{Thread.current.object_id}_/)
end
def thread_with_api_token(login_optional = false)
end
end
- def thread_with_optional_api_token
- thread_with_api_token(true) do
+ def thread_with_mandatory_api_token
+ thread_with_api_token do
+ yield
+ end
+ end
+
+ # This runs after thread_with_mandatory_api_token in the filter chain.
+ def thread_with_optional_api_token
+ if Thread.current[:arvados_api_token]
+ # We are already inside thread_with_mandatory_api_token.
yield
+ else
+ # We skipped thread_with_mandatory_api_token. Use the optional version.
+ thread_with_api_token(true) do
+ yield
+ end
end
end
def select_theme
return Rails.configuration.arvados_theme
end
+
+ @@notification_tests = []
+
+ @@notification_tests.push lambda { |controller, current_user|
+ AuthorizedKey.limit(1).where(authorized_user_uuid: current_user.uuid).each do
+ return nil
+ end
+ return lambda { |view|
+ view.render partial: 'notifications/ssh_key_notification'
+ }
+ }
+
+ @@notification_tests.push lambda { |controller, current_user|
+ AuthorizedKey.limit(1).where(authorized_user_uuid: current_user.uuid).each do
+ return nil
+ end
+ return lambda { |view|
+ view.render partial: 'notifications/jobs_notification'
+ }
+ }
+
+ @@notification_tests.push lambda { |controller, current_user|
+ Job.limit(1).where(created_by: current_user.uuid).each do
+ return nil
+ end
+ return lambda { |view|
+ view.render partial: 'notifications/jobs_notification'
+ }
+ }
+
+ @@notification_tests.push lambda { |controller, current_user|
+ Collection.limit(1).where(created_by: current_user.uuid).each do
+ return nil
+ end
+ return lambda { |view|
+ view.render partial: 'notifications/collections_notification'
+ }
+ }
+
+ @@notification_tests.push lambda { |controller, current_user|
+ PipelineInstance.limit(1).where(created_by: current_user.uuid).each do
+ return nil
+ end
+ return lambda { |view|
+ view.render partial: 'notifications/pipelines_notification'
+ }
+ }
+
+ def check_user_notifications
+ @notification_count = 0
+ @notifications = []
+
+ if current_user
+ @showallalerts = false
+ @@notification_tests.each do |t|
+ a = t.call(self, current_user)
+ if a
+ @notification_count += 1
+ @notifications.push a
+ end
+ end
+ end
+
+ if @notification_count == 0
+ @notification_count = ''
+ end
+ end
end
class AuthorizedKeysController < ApplicationController
+ def index_pane_list
+ %w(Recent Help)
+ end
+
def new
super
@object.authorized_user_uuid = current_user.uuid if current_user
class CollectionsController < ApplicationController
- skip_before_filter :find_object_by_uuid, :only => [:graph]
+ skip_before_filter :find_object_by_uuid, :only => [:provenance]
skip_before_filter :check_user_agreements, :only => [:show_file]
- def graph
- index
+ def show_pane_list
+ %w(Files Attributes Metadata Provenance_graph Used_by JSON API)
end
-
def index
if params[:search].andand.length.andand > 0
tags = Link.where(any: ['contains', params[:search]])
self.response_body = FileStreamer.new opts
end
+
def show
return super if !@object
@provenance = []
@sourcedata[collection.uuid][:collection] = collection
end
end
+
+ Collection.where(uuid: @object.uuid).each do |u|
+ @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, "provenance_svg", {:direction => :bottom_up, :combine_jobs => :script_only} rescue nil
+ @used_by_svg = ProvenanceHelper::create_provenance_graph u.used_by, "used_by_svg", {:direction => :top_down, :combine_jobs => :script_only, :pdata_only => true} rescue nil
+ end
end
protected
class JobsController < ApplicationController
+
+ def generate_provenance(jobs)
+ nodes = []
+ collections = []
+ jobs.each do |j|
+ nodes << j
+ collections << j[:output]
+ collections.concat(ProvenanceHelper::find_collections(j[:script_parameters]))
+ nodes << {:uuid => j[:script_version]}
+ end
+
+ Collection.where(uuid: collections).each do |c|
+ nodes << c
+ end
+
+ @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {:all_script_parameters => true, :script_version_nodes => true}
+ end
+
def index
- @jobs = Job.all
+ @svg = ""
+ if params[:uuid]
+ @jobs = Job.where(uuid: params[:uuid])
+ generate_provenance(@jobs)
+ else
+ @jobs = Job.all
+ end
+ end
+
+ def show
+ generate_provenance([@object])
+ end
+
+ def index_pane_list
+ if params[:uuid]
+ %w(Recent Provenance)
+ else
+ %w(Recent)
+ end
+ end
+
+ def show_pane_list
+ %w(Attributes Provenance Metadata JSON API)
end
end
--- /dev/null
+class KeepDisksController < ApplicationController
+end
class PipelineInstancesController < ApplicationController
+ skip_before_filter :find_object_by_uuid, only: :compare
+ before_filter :find_objects_by_uuid, only: :compare
+ include PipelineInstancesHelper
+
+ def graph(pipelines)
+ count = {}
+ provenance = {}
+ pips = {}
+ n = 1
+
+ pipelines.each do |p|
+ collections = []
+
+ p.components.each do |k, v|
+ j = v[:job] || next
+
+ uuid = j[:uuid].intern
+ provenance[uuid] = j
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+
+ collections << j[:output]
+ ProvenanceHelper::find_collections(j[:script_parameters]).each do |k|
+ collections << k
+ end
+
+ uuid = j[:script_version].intern
+ provenance[uuid] = {:uuid => uuid}
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+ end
+
+ Collection.where(uuid: collections.compact).each do |c|
+ uuid = c.uuid.intern
+ provenance[uuid] = c
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+ end
+
+ n = n << 1
+ end
+
+ return provenance, pips
+ end
+
+ def show
+ @pipelines = [@object]
+
+ if params[:compare]
+ PipelineInstance.where(uuid: params[:compare]).each do |p|
+ @pipelines << p
+ end
+ end
+
+ provenance, pips = graph(@pipelines)
+
+ @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :all_script_parameters => true,
+ :combine_jobs => :script_and_version,
+ :script_version_nodes => true,
+ :pips => pips }
+ super
+ end
+
+ def compare
+ @breadcrumb_page_name = 'compare'
+
+ @rows = [] # each is {name: S, components: [...]}
+
+ # Build a table: x=pipeline y=component
+ @objects.each_with_index do |pi, pi_index|
+ pipeline_jobs(pi).each do |component|
+ # Find a cell with the same name as this component but no
+ # entry for this pipeline
+ target_row = nil
+ @rows.each_with_index do |row, row_index|
+ if row[:name] == component[:name] and !row[:components][pi_index]
+ target_row = row
+ end
+ end
+ if !target_row
+ target_row = {name: component[:name], components: []}
+ @rows << target_row
+ end
+ target_row[:components][pi_index] = component
+ end
+ end
+
+ @rows.each do |row|
+ # Build a "normal" pseudo-component for this row by picking the
+ # most common value for each attribute. If all values are
+ # equally common, there is no "normal".
+ normal = {} # attr => most common value
+ highscore = {} # attr => how common "normal" is
+ score = {} # attr => { value => how common }
+ row[:components].each do |pj|
+ pj.each do |k,v|
+ vstr = for_comparison v
+ score[k] ||= {}
+ score[k][vstr] = (score[k][vstr] || 0) + 1
+ highscore[k] ||= 0
+ if score[k][vstr] == highscore[k]
+ # tie for first place = no "normal"
+ normal.delete k
+ elsif score[k][vstr] == highscore[k] + 1
+ # more pipelines have v than anything else
+ highscore[k] = score[k][vstr]
+ normal[k] = vstr
+ end
+ end
+ end
+
+ # Add a hash in component[:is_normal]: { attr => is_the_value_normal? }
+ row[:components].each do |pj|
+ pj[:is_normal] = {}
+ pj.each do |k,v|
+ pj[:is_normal][k] = (normal.has_key?(k) && normal[k] == for_comparison(v))
+ end
+ end
+ end
+
+ provenance, pips = graph(@objects)
+
+ @pipelines = @objects
+
+ @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :all_script_parameters => true,
+ :combine_jobs => :script_and_version,
+ :script_version_nodes => true,
+ :pips => pips }
+ end
+
+ def show_pane_list
+ %w(Components Graph Attributes Metadata JSON API)
+ end
+
+ def compare_pane_list
+ %w(Compare Graph)
+ end
+
+ protected
+ def for_comparison v
+ if v.is_a? Hash or v.is_a? Array
+ v.to_json
+ else
+ v.to_s
+ end
+ end
+
+ def find_objects_by_uuid
+ @objects = model_class.where(uuid: params[:uuids])
+ end
+
end
class RepositoriesController < ApplicationController
+ def index_pane_list
+ %w(recent help)
+ end
end
class SessionsController < ApplicationController
- skip_around_filter :thread_with_api_token, :only => [:destroy, :index]
+ skip_around_filter :thread_with_mandatory_api_token, :only => [:destroy, :index]
+ skip_around_filter :thread_with_optional_api_token, :only => [:destroy, :index]
skip_before_filter :find_object_by_uuid, :only => [:destroy, :index]
def destroy
session.clear
class UsersController < ApplicationController
skip_before_filter :find_object_by_uuid, :only => :welcome
- skip_around_filter :thread_with_api_token, :only => :welcome
- around_filter :thread_with_optional_api_token, :only => :welcome
+ skip_around_filter :thread_with_mandatory_api_token, :only => :welcome
def welcome
if current_user
- redirect_to home_user_path(current_user.uuid)
+ params[:action] = 'home'
+ home
end
end
@tutorial_complete = {
'Run a job' => @my_last_job
}
+ respond_to do |f|
+ f.js { render template: 'users/home.js' }
+ f.html { render template: 'users/home' }
+ end
end
end
class VirtualMachinesController < ApplicationController
+ def index_pane_list
+ %w(recent help)
+ end
def index
@objects ||= model_class.all
@vm_logins = {}
if opts[:friendly_name]
begin
- friendly_name = resource_class.find(link_uuid).friendly_link_name
- if friendly_name and not friendly_name.empty?
- link_name = friendly_name
- end
+ link_name = resource_class.find(link_uuid).friendly_link_name
rescue RuntimeError
# If that lookup failed, the link will too. So don't make one.
return attrvalue
link_name = "#{resource_class.to_s}: #{link_name}"
end
end
+ style_opts[:class] = (style_opts[:class] || '') + ' nowrap'
link_to link_name, { controller: resource_class.to_s.underscore.pluralize, action: 'show', id: link_uuid }, style_opts
else
attrvalue
"data-emptytext" => "none",
"data-placement" => "bottom",
"data-type" => input_type,
- "data-resource" => object.class.to_s.underscore,
- "data-name" => attr,
"data-url" => url_for(action: "update", id: object.uuid, controller: object.class.to_s.pluralize.underscore),
- "data-original-title" => "Update #{attr.gsub '_', ' '}",
+ "data-title" => "Update #{attr.gsub '_', ' '}",
+ "data-name" => attr,
+ "data-pk" => "{id: \"#{object.uuid}\", key: \"#{object.class.to_s.underscore}\"}",
:class => "editable"
}.merge(htmloptions)
end
--- /dev/null
+module KeepDisksHelper
+end
module PipelineInstancesHelper
- def pipeline_jobs
- if @object.components[:steps].is_a? Array
- pipeline_jobs_oldschool
- elsif @object.components.is_a? Hash
- pipeline_jobs_newschool
+ def pipeline_summary object=nil
+ object ||= @object
+ ret = {todo:0, running:0, queued:0, done:0, failed:0, total:0}
+ object.components.values.each do |c|
+ ret[:total] += 1
+ case
+ when !c[:job]
+ ret[:todo] += 1
+ when c[:job][:success]
+ ret[:done] += 1
+ when c[:job][:failed]
+ ret[:failed] += 1
+ when c[:job][:finished_at]
+ ret[:running] += 1 # XXX finished but !success and !failed??
+ when c[:job][:started_at]
+ ret[:running] += 1
+ else
+ ret[:queued] += 1
+ end
+ end
+ ret.merge! Hash[ret.collect do |k,v|
+ [('percent_' + k.to_s).to_sym,
+ ret[:total]<1 ? 0 : (100.0*v/ret[:total]).floor]
+ end]
+ ret
+ end
+
+ def pipeline_jobs object=nil
+ object ||= @object
+ if object.components[:steps].is_a? Array
+ pipeline_jobs_oldschool object
+ elsif object.components.is_a? Hash
+ pipeline_jobs_newschool object
+ end
+ end
+
+ def render_pipeline_jobs
+ pipeline_jobs.collect do |pj|
+ render_pipeline_job pj
+ end
+ end
+
+ def render_pipeline_job pj
+ if pj[:percent_done]
+ pj[:progress_bar] = raw("<div class=\"progress\" style=\"width:100px\"><span class=\"progress-bar progress-bar-success\" style=\"width:#{pj[:percent_done]}%\"></span><span class=\"progress-bar\" style=\"width:#{pj[:percent_running]}%\"></span></div>")
+ elsif pj[:progress]
+ raw("<div class=\"progress\" style=\"width:100px\"><span class=\"progress-bar\" style=\"width:#{pj[:progress]*100}%\"></span></div>")
end
+ pj[:output_link] = link_to_if_arvados_object pj[:output]
+ pj[:job_link] = link_to_if_arvados_object pj[:job][:uuid]
+ pj
end
protected
- def pipeline_jobs_newschool
+ def pipeline_jobs_newschool object
ret = []
i = -1
- @object.components.each do |cname, c|
+ object.components.each do |cname, c|
i += 1
pj = {index: i, name: cname}
pj[:job] = c[:job].is_a?(Hash) ? c[:job] : {}
pj[:progress] = 0.0
end
end
- if pj[:job]
- if pj[:job][:success]
- pj[:result] = 'complete'
- pj[:complete] = true
- pj[:progress] = 1.0
- elsif pj[:job][:finished_at]
- pj[:result] = 'failed'
- pj[:failed] = true
- elsif pj[:job][:started_at]
- pj[:result] = 'running'
- else
- pj[:result] = 'queued'
- end
+ if pj[:job][:success]
+ pj[:result] = 'complete'
+ pj[:complete] = true
+ pj[:progress] = 1.0
+ elsif pj[:job][:finished_at]
+ pj[:result] = 'failed'
+ pj[:failed] = true
+ elsif pj[:job][:started_at]
+ pj[:result] = 'running'
+ elsif pj[:job][:uuid]
+ pj[:result] = 'queued'
+ else
+ pj[:result] = 'none'
end
pj[:job_id] = pj[:job][:uuid]
- pj[:job_link] = link_to_if_arvados_object pj[:job][:uuid]
- pj[:script_version] = pj[:job][:script_version]
+ pj[:script] = pj[:job][:script] || c[:script]
+ pj[:script_parameters] = pj[:job][:script_parameters] || c[:script_parameters]
+ pj[:script_version] = pj[:job][:script_version] || c[:script_version]
pj[:output] = pj[:job][:output]
pj[:finished_at] = (Time.parse(pj[:job][:finished_at]) rescue nil)
- pj[:progress_bar] = raw("<div class=\"progress\" style=\"width:100px\"><div class=\"bar bar-success\" style=\"width:#{pj[:percent_done]}%\"></div><div class=\"bar\" style=\"width:#{pj[:percent_running]}%\"></div></div>")
- pj[:output_link] = link_to_if_arvados_object pj[:output]
ret << pj
end
ret
end
- def pipeline_jobs_oldschool
+ def pipeline_jobs_oldschool object
ret = []
- @object.components[:steps].each_with_index do |step, i|
+ object.components[:steps].each_with_index do |step, i|
pj = {index: i, name: step[:name]}
if step[:complete] and step[:complete] != 0
if step[:output_data_locator]
pj[:script_version] = (step[:warehousejob][:revision] rescue nil)
pj[:output] = step[:output_data_locator]
pj[:finished_at] = (Time.parse(step[:warehousejob][:finishtime]) rescue nil)
- pj[:progress_bar] = raw("<div class=\"progress\" style=\"width:100px\"><div class=\"bar\" style=\"width:#{pj[:progress]*100}%\"></div></div>")
- pj[:output_link] = link_to_if_arvados_object pj[:output]
ret << pj
end
ret
--- /dev/null
+module ProvenanceHelper
+
+ class GenerateGraph
+ def initialize(pdata, opts)
+ @pdata = pdata
+ @opts = opts
+ @visited = {}
+ @jobs = {}
+ end
+
+ def self.collection_uuid(uuid)
+ m = /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+ if m
+ #if m[2]
+ return m[1]
+ #else
+ # Collection.where(uuid: ['contains', m[1]]).each do |u|
+ # puts "fixup #{uuid} to #{u.uuid}"
+ # return u.uuid
+ # end
+ #end
+ else
+ nil
+ end
+ end
+
+ def determine_fillcolor(n)
+ bgcolor = ""
+ case n
+ when 1
+ bgcolor = "style=filled,fillcolor=\"#88ff88\""
+ when 2
+ bgcolor = "style=filled,fillcolor=\"#8888ff\""
+ when 3
+ bgcolor = "style=filled,fillcolor=\"#88ffff\""
+ end
+ bgcolor
+ end
+
+ def describe_node(uuid)
+ bgcolor = determine_fillcolor @opts[:pips][uuid] if @opts[:pips]
+
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+ if rsc
+ href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+
+ #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
+ if rsc == Collection
+ #puts uuid
+ if uuid == :"d41d8cd98f00b204e9800998ecf8427e+0"
+ # special case
+ #puts "empty!"
+ return "\"#{uuid}\" [label=\"(empty collection)\"];\n"
+ end
+ if @pdata[uuid]
+ #puts @pdata[uuid]
+ if @pdata[uuid][:name]
+ return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
+ else
+ files = nil
+ if @pdata[uuid].respond_to? :files
+ files = @pdata[uuid].files
+ elsif @pdata[uuid][:files]
+ files = @pdata[uuid][:files]
+ end
+
+ if files
+ i = 0
+ label = ""
+ while i < 3 and i < files.length
+ label += "\\n" unless label == ""
+ label += files[i][1]
+ i += 1
+ end
+ if i < files.length
+ label += "\\n⋮"
+ end
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
+ end
+ end
+ end
+ return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\",#{bgcolor}];\n"
+ end
+ end
+ "\"#{uuid}\" [#{bgcolor}];\n"
+ end
+
+ def job_uuid(job)
+ if @opts[:combine_jobs] == :script_only
+ uuid = "#{job[:script]}"
+ elsif @opts[:combine_jobs] == :script_and_version
+ uuid = "#{job[:script]}_#{job[:script_version]}"
+ else
+ uuid = "#{job[:uuid]}"
+ end
+
+ @jobs[uuid] = [] unless @jobs[uuid]
+ @jobs[uuid] << job unless @jobs[uuid].include? job
+
+ uuid
+ end
+
+ def edge(tail, head, extra)
+ if @opts[:direction] == :bottom_up
+ gr = "\"#{tail}\" -> \"#{head}\""
+ else
+ gr = "\"#{head}\" -> \"#{tail}\""
+ end
+ if extra.length > 0
+ gr += "["
+ extra.each do |k, v|
+ gr += "#{k}=\"#{v}\","
+ end
+ gr += "]"
+ end
+ gr += ";\n"
+ gr
+ end
+
+ def script_param_edges(job, prefix, sp)
+ gr = ""
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ if prefix.size > 0
+ k = prefix + "::" + k.to_s
+ end
+ gr += script_param_edges(job, k.to_s, v)
+ end
+ when Array
+ i = 0
+ node = ""
+ sp.each do |v|
+ if GenerateGraph::collection_uuid(v)
+ gr += script_param_edges(job, "#{prefix}[#{i}]", v)
+ elsif @opts[:all_script_parameters]
+ node += "', '" unless node == ""
+ node = "['" if node == ""
+ node += "#{v}"
+ end
+ i += 1
+ end
+ unless node == ""
+ node += "']"
+ #puts node
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{node}\" [label=\"#{node}\"];\n"
+ gr += edge(job_uuid(job), node, {:label => prefix})
+ end
+ else
+ m = GenerateGraph::collection_uuid(sp)
+ #puts "#{m} pdata is #{@pdata[m.intern]}"
+ if m and (@pdata[m.intern] or (not @opts[:pdata_only]))
+ gr += edge(job_uuid(job), m, {:label => prefix})
+ gr += generate_provenance_edges(m)
+ elsif @opts[:all_script_parameters]
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
+ gr += edge(job_uuid(job), sp, {:label => prefix})
+ end
+ end
+ end
+ gr
+ end
+
+ def generate_provenance_edges(uuid)
+ gr = ""
+ m = GenerateGraph::collection_uuid(uuid)
+ uuid = m if m
+
+ uuid = uuid.intern if uuid
+
+ if (not uuid) or uuid.empty? or @visited[uuid]
+
+ #puts "already @visited #{uuid}"
+ return ""
+ end
+
+ if not @pdata[uuid] then
+ return describe_node(uuid)
+ else
+ @visited[uuid] = true
+ end
+
+ #puts "visiting #{uuid}"
+
+ if m
+ # uuid is a collection
+ gr += describe_node(uuid)
+
+ if m == :"d41d8cd98f00b204e9800998ecf8427e+0"
+ # empty collection, don't follow any further
+ return gr
+ end
+
+ @pdata.each do |k, job|
+ if job[:output] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "output"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ if job[:log] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "log"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ end
+ else
+ # uuid is something else
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+
+ if rsc == Job
+ job = @pdata[uuid]
+ if job
+ gr += script_param_edges(job, "", job[:script_parameters])
+
+ if @opts[:script_version_nodes]
+ gr += describe_node(job[:script_version])
+ gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
+ end
+ end
+ else
+ gr += describe_node(uuid)
+ end
+ end
+
+ @pdata.each do |k, link|
+ if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ gr += describe_node(link[:tail_uuid])
+ gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"})
+ gr += generate_provenance_edges(link[:tail_uuid])
+ end
+ end
+
+ #puts "finished #{uuid}"
+
+ gr
+ end
+
+ def describe_jobs
+ gr = ""
+ @jobs.each do |k, v|
+ gr += "\"#{k}\" [href=\"/jobs?"
+
+ n = 0
+ v.each do |u|
+ gr += "uuid%5b%5d=#{u[:uuid]}&"
+ n |= @opts[:pips][u[:uuid].intern] if @opts[:pips] and @opts[:pips][u[:uuid].intern]
+ end
+
+ gr += "\",label=\""
+
+ if @opts[:combine_jobs] == :script_only
+ gr += uuid = "#{v[0][:script]}"
+ elsif @opts[:combine_jobs] == :script_and_version
+ gr += uuid = "#{v[0][:script]}"
+ else
+ gr += uuid = "#{v[0][:script]}\\n#{v[0][:finished_at]}"
+ end
+ gr += "\",#{determine_fillcolor n}];\n"
+ end
+ gr
+ end
+
+ end
+
+ def self.create_provenance_graph(pdata, svgId, opts={})
+ if pdata.is_a? Array or pdata.is_a? ArvadosResourceList
+ p2 = {}
+ pdata.each do |k|
+ p2[k[:uuid].intern] = k if k[:uuid]
+ end
+ pdata = p2
+ end
+
+ unless pdata.is_a? Hash
+ raise "create_provenance_graph accepts Array or Hash for pdata only, pdata is #{pdata.class}"
+ end
+
+ gr = """strict digraph {
+node [fontsize=10,shape=box];
+edge [fontsize=10];
+"""
+
+ if opts[:direction] == :bottom_up
+ gr += "edge [dir=back];"
+ end
+
+ #puts "@pdata is #{pdata}"
+
+ g = GenerateGraph.new(pdata, opts)
+
+ pdata.each do |k, v|
+ gr += g.generate_provenance_edges(k)
+ end
+
+ gr += g.describe_jobs
+
+ gr += "}"
+ svg = ""
+
+ #puts gr
+
+ require 'open3'
+
+ Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
+ stdin.print(gr)
+ stdin.close
+ svg = stdout.read()
+ wait_thr.value
+ stdout.close()
+ end
+
+ svg = svg.sub(/<\?xml.*?\?>/m, "")
+ svg = svg.sub(/<!DOCTYPE.*?>/m, "")
+ svg = svg.sub(/<svg /, "<svg id=\"#{svgId}\" ")
+ end
+
+ def self.find_collections(sp)
+ c = []
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ c.concat(find_collections(v))
+ end
+ when Array
+ sp.each do |v|
+ c.concat(find_collections(v))
+ end
+ else
+ m = GenerateGraph::collection_uuid(sp)
+ if m
+ c << m
+ end
+ end
+ end
+ c
+ end
+end
resources_kind = class_kind(resources_kind).pluralize if resources_kind.is_a? Class
url = "#{self.arvados_v1_base}/#{resources_kind}#{action}"
+ # Clean up /arvados/v1/../../discovery/v1 to /discovery/v1
+ url.sub! '/arvados/v1/../../', '/'
+
query = {"api_token" => api_token}
if !data.nil?
data.each do |k,v|
@arvados_schema ||= api 'schema', ''
end
+ def discovery
+ @discovery ||= api '../../discovery/v1/apis/arvados/v1/rest', ''
+ end
+
def kind_class(kind)
kind.match(/^arvados\#(.+?)(_list|List)?$/)[1].pluralize.classify.constantize rescue nil
end
attr_accessor :attribute_sortkey
def self.uuid_infix_object_kind
- @@uuid_infix_object_kind ||= {
- '4zz18' => 'arvados#collection',
- 'tpzed' => 'arvados#user',
- 'ozdt8' => 'arvados#api_client',
- '8i9sb' => 'arvados#job',
- 'o0j2j' => 'arvados#link',
- '57u5n' => 'arvados#log',
- 'j58dm' => 'arvados#specimen',
- 'p5p6p' => 'arvados#pipeline_template',
- 'mxsvm' => 'arvados#pipeline_template', # legacy Pipeline objects
- 'd1hrv' => 'arvados#pipeline_instance',
- 'uo14g' => 'arvados#pipeline_instance', # legacy PipelineInstance objects
- 'j7d0g' => 'arvados#group',
- 'ldvyl' => 'arvados#group' # only needed for legacy Project objects
- }
+ @@uuid_infix_object_kind ||=
+ begin
+ infix_kind = {}
+ $arvados_api_client.discovery[:schemas].each do |name, schema|
+ if schema[:uuidPrefix]
+ infix_kind[schema[:uuidPrefix]] =
+ 'arvados#' + name.to_s.camelcase(:lower)
+ end
+ end
+
+ # Recognize obsolete types.
+ infix_kind.
+ merge('mxsvm' => 'arvados#pipelineTemplate', # Pipeline
+ 'uo14g' => 'arvados#pipelineInstance', # PipelineInvocation
+ 'ldvyl' => 'arvados#group') # Project
+ end
end
def initialize(*args)
self.columns
@attribute_info
end
- def self.find(uuid)
+ def self.find(uuid, opts={})
if uuid.class != String or uuid.length < 27 then
raise 'argument to find() must be a uuid string. Acceptable formats: warehouse locator or string with format xxxxx-xxxxx-xxxxxxxxxxxxxxx'
end
- new.private_reload(uuid)
+
+ # Only do one lookup on the API side per {class, uuid, workbench
+ # request} unless {cache: false} is given via opts.
+ cache_key = "request_#{Thread.current.object_id}_#{self.to_s}_#{uuid}"
+ if opts[:cache] == false
+ Rails.cache.write cache_key, $arvados_api_client.api(self, '/' + uuid)
+ end
+ hash = Rails.cache.fetch cache_key do
+ $arvados_api_client.api(self, '/' + uuid)
+ end
+ new.private_reload(hash)
end
def self.order(*args)
ArvadosResourceList.new(self).order(*args)
end
def friendly_link_name
- if self.class.column_names.include? 'name'
- self.name
- end
+ (name if self.respond_to? :name) || uuid
end
protected
def attribute_editable?(attr)
false
end
+
+ def self.creatable?
+ false
+ end
+
+ def provenance
+ $arvados_api_client.api "collections/#{self.uuid}/", "provenance"
+ end
+
+ def used_by
+ $arvados_api_client.api "collections/#{self.uuid}/", "used_by"
+ end
end
--- /dev/null
+class KeepDisk < ArvadosBase
+ def self.creatable?
+ current_user and current_user.is_admin
+ end
+end
class Node < ArvadosBase
- attr_accessor :object
+ def self.creatable?
+ current_user and current_user.is_admin
+ end
def friendly_link_name
- self.hostname
+ (hostname && !hostname.empty?) ? hostname : uuid
end
end
def attribute_editable?(attr)
attr == 'name'
end
+
+ def attributes_for_display
+ super.reject { |k,v| k == 'components' }
+ end
end
{}))
end
- def attribute_editable?(attr)
+ def attributes_for_display
+ super.reject { |k,v| %w(owner_uuid default_owner_uuid identity_url prefs).index k }
+ end
+
+ def attribute_editable?(attr)
(not (self.uuid.andand.match(/000000000000000$/) and self.is_admin)) and super(attr)
end
super]
end
def friendly_link_name
- self.hostname
+ (hostname && !hostname.empty?) ? hostname : uuid
end
end
unset ARVADOS_API_HOST_INSECURE
<% end %>
</pre>
-
-<%= render partial: 'index' %>
<% if obj.attribute_editable?(attr) %>
<%= render_editable_attribute obj, attr %>
<% if resource_class_for_uuid(attrvalue, {referring_object: obj, referring_attr: attr}) %>
- (<%= link_to_if_arvados_object attrvalue, {referring_attr: attr, referring_object: obj, with_class_name: true, friendly_name: true} %>)
+ <br />
+ (<%= link_to_if_arvados_object attrvalue, {referring_attr: attr, referring_object: obj, with_class_name: true, friendly_name: true} %>)
<% end %>
<% elsif attr == 'uuid' %>
<%= link_to_if_arvados_object attrvalue, {referring_attr: attr, referring_object: obj, with_class_name: false, friendly_name: false} %>
<% end %>
<!--
<% if resource_class_for_uuid(attrvalue, {referring_object: obj, referring_attr: attr}) %>
- <%= link_to_if_arvados_object(attrvalue, { referring_object: obj, link_text: raw('<span class="icon-hand-right"></span>'), referring_attr: attr }) %>
+ <%= link_to_if_arvados_object(attrvalue, { referring_object: obj, link_text: raw('<span class="glyphicon glyphicon-hand-right"></span>'), referring_attr: attr }) %>
<% end %>
-->
<% end %>
<% content_for :arvados_object_table do %>
-<h2><%= @object.class %> <%= @object.uuid %></h2>
-<%= form_for @object do |f| %>
-<table class="table topalign">
- <thead>
- </thead>
- <tbody>
- <% @object.attributes_for_display.each do |attr, attrvalue| %>
- <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
- <% end %>
- </tbody>
-</table>
-<% incoming = Link.where(tail_uuid: @object.uuid) %>
-<% if incoming.items_available > 0 %>
-<h3>Incoming Links</h3>
-<table class="table topalign">
- <thead>
- </thead>
- <tbody>
- <% incoming.each do |link| %>
- <tr>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "uuid", attrvalue: link.uuid } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "link_class", attrvalue: link.link_class } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "name", attrvalue: link.name } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "head_uuid", attrvalue: link.head_uuid } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "properties", attrvalue: link.properties } %></td>
- </tr>
- <% end %>
- </tbody>
-</table>
-<% end %>
-
-<% outgoing = Link.where(head_uuid: @object.uuid) %>
-<% if outgoing.items_available > 0 %>
-<h3>Outgoing Links</h3>
-<table class="table topalign">
- <thead>
- </thead>
- <tbody>
- <% outgoing.each do |link| %>
- <tr>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "uuid", attrvalue: link.uuid } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "link_class", attrvalue: link.link_class } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "name", attrvalue: link.name } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "tail_uuid", attrvalue: link.tail_uuid } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "properties", attrvalue: link.properties } %></td>
- </tr>
- <% end %>
- </tbody>
-</table>
-<% end %>
-
-<% end %>
<% end %>
<% if content_for? :page_content %>
</div>
<% end %>
<div id="arvados-object-json" class="tab-pane fade in active">
- <pre>
-<%= JSON.pretty_generate(@object.attributes.reject { |k,v| k == 'id' }) rescue nil %>
- </pre>
- </div>
- <% if @object.andand.uuid %>
-
- <div id="arvados-object-curl" class="tab-pane fade">
- <pre>
-curl -X PUT \
- -H "Authorization: OAuth2 $ARVADOS_API_TOKEN" \
- --data-urlencode <%= @object.class.to_s.underscore %>@/dev/stdin \
- https://$ARVADOS_API_HOST/arvados/v1/<%= @object.class.to_s.pluralize.underscore %>/<%= @object.uuid %> \
- <<EOF
-<%= JSON.pretty_generate({@object.attributes.keys[-3] => @object.attributes.values[-3]}) %>
-EOF
- </pre>
- </div>
-
- <div id="arvados-object-arv" class="tab-pane fade">
- <pre>
-arv --pretty <%= @object.class.to_s.underscore %> get \
- --uuid <%= @object.uuid %>
-
-arv <%= @object.class.to_s.underscore %> update \
- --uuid <%= @object.uuid %> \
- --<%= @object.class.to_s.underscore.gsub '_', '-' %> '<%= JSON.generate({@object.attributes.keys[-3] => @object.attributes.values[-3]}).gsub("'","'\''") %>'
- </pre>
</div>
- <div id="arvados-object-python" class="tab-pane fade">
- <pre>
-import arvados
-
-x = arvados.api().<%= @object.class.to_s.pluralize.underscore %>().get(uuid='<%= @object.uuid %>').execute()
- </pre>
- </div>
-
- <% end %>
</div>
</div>
+<% object ||= @object %>
<% if attrvalue.is_a? Hash then attrvalue.each do |infokey, infocontent| %>
<tr class="info">
<td><%= attr %>[<%= infokey %>]</td>
<td>
- <%= render partial: 'application/arvados_attr_value', locals: { obj: @object, attr: nil, attrvalue: infocontent } %>
+ <%= render partial: 'application/arvados_attr_value', locals: { obj: object, attr: nil, attrvalue: infocontent } %>
</td>
</tr>
<% end %>
<tr class="<%= 'info' if %w(uuid owner_uuid created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at).index(attr.to_s).nil? %>">
<td><%= attr %></td>
<td>
- <%= render partial: 'application/arvados_attr_value', locals: { obj: @object, attr: attr, attrvalue: attrvalue } %>
+ <%= render partial: 'application/arvados_attr_value', locals: { obj: object, attr: attr, attrvalue: attrvalue } %>
</td>
</tr>
<% end %>
--- /dev/null
+<% content_for :tab_panes do %>
+
+<% comparable = controller.respond_to? :compare %>
+<% pane_list ||= %w(recent) %>
+<% panes = Hash[pane_list.map { |pane|
+ [pane, render(partial: 'show_' + pane.downcase,
+ locals: { comparable: comparable })]
+ }.compact] %>
+
+<ul class="nav nav-tabs">
+ <% panes.each_with_index do |(pane, content), i| %>
+ <li class="<%= 'active' if i==0 %>"><a href="#<%= pane %>" data-toggle="tab" id="<%= pane %>-tab"> <%= pane.gsub('_', ' ') %></a></li>
+ <% end %>
+</ul>
+<div class="tab-content">
+<% panes.each_with_index do |(pane, content), i| %>
+ <div id="<%= pane %>" class="tab-pane fade <%= 'in active' if i==0 %>">
+ <div class="smart-scroll" style="margin-top:0.5em;">
+ <%= content %>
+ </div>
+ </div>
+<% end %>
+</div>
+
+<% end %>
+
+<% content_for :js do %>
+ $(window).on('load', function() {
+ $('ul.nav-tabs > li > a').on('shown.bs.tab', smart_scroll_fixup);
+ });
+<% end %>
--- /dev/null
+<%= content_for :content_top %>
+<%= content_for :tab_line_buttons %>
+<%= content_for :tab_panes %>
-
-<h2 class="pull-left"><%= controller.model_class.to_s.pluralize.underscore.capitalize.gsub '_', ' ' %></h2>
-<br/>
-<% if controller.model_class.creatable? %>
-<%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
- { action: 'create', return_to: request.url },
- { class: 'btn btn-primary pull-right' } %>
-<% end %>
-
-
-<% if @objects.empty? %>
-<br/>
-<p style="text-align: center">
- No <%= controller.model_class.to_s.pluralize.underscore.gsub '_', ' ' %> to display.
-</p>
-
-<% else %>
-
-<% attr_blacklist = ' created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at' %>
-
-<table class="table arv-index">
- <thead>
- <tr>
- <% @objects.first.attributes_for_display.each do |attr, attrvalue| %>
- <% next if attr_blacklist.index(" "+attr) %>
- <th class="arv-attr-<%= attr %>">
- <%= controller.model_class.attribute_info[attr.to_sym].andand[:column_heading] or attr.sub /_uuid/, '' %>
- </th>
- <% end %>
- <th>
- <!-- a column for delete buttons -->
- </th>
- </tr>
- </thead>
-
- <tbody>
- <% @objects.each do |object| %>
- <tr>
- <% object.attributes_for_display.each do |attr, attrvalue| %>
- <% next if attr_blacklist.index(" "+attr) %>
- <td class="arv-object-<%= object.class.to_s %> arv-attr-<%= attr %>">
- <% if attr == 'uuid' %>
- <%= link_to_if_arvados_object object %>
- <%= link_to_if_arvados_object(object, { link_text: raw('<i class="icon-hand-right"></i>') }) %>
- <% else %>
- <% if object.attribute_editable? attr %>
- <%= render_editable_attribute object, attr %>
- <% else %>
- <%= resource_class_for_uuid(attrvalue, referring_attr: attr, referring_object: @object).to_s %>
- <%= attrvalue %>
- <% end %>
- <%= link_to_if_arvados_object(attrvalue, { referring_object: @object, link_text: raw('<i class="icon-hand-right"></i>') }) if resource_class_for_uuid(attrvalue, {referring_object: @object}) %>
- <% end %>
- </td>
- <% end %>
- <td>
- <% if object.editable? %>
- <%= link_to({action: 'destroy', id: object.uuid}, method: :delete, data: {confirm: "You are about to delete #{controller.model_class} #{object.uuid}.\n\nAre you sure?"}) do %>
- <i class="icon-trash"></i>
- <!-- <%= object.inspect %> -->
- <% end %>
- <% end %>
- </td>
- </tr>
- <% end %>
- </tbody>
-
- <tfoot>
- </tfoot>
-</table>
-
-<% end %>
-
--- /dev/null
+<div class="socket">
+ <div class="gel center-gel">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c1 r1">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c2 r1">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c3 r1">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c4 r1">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c5 r1">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c6 r1">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+
+ <div class="gel c7 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+
+ <div class="gel c8 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c9 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c10 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c11 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c12 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c13 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c14 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c15 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c16 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c17 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c18 r2">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c19 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c20 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c21 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c22 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c23 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c24 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c25 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c26 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c28 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c29 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c30 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c31 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c32 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c33 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c34 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c35 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c36 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+ <div class="gel c37 r3">
+ <div class="hex-brick h1"></div>
+ <div class="hex-brick h2"></div>
+ <div class="hex-brick h3"></div>
+ </div>
+
+</div>
--- /dev/null
+<% if @object.andand.uuid %>
+
+<div class="panel panel-default">
+ <div class="panel-heading">curl</div>
+ <div class="panel-body">
+ <pre>
+curl -X PUT \
+ -H "Authorization: OAuth2 $ARVADOS_API_TOKEN" \
+ --data-urlencode <%= @object.class.to_s.underscore %>@/dev/stdin \
+ https://$ARVADOS_API_HOST/arvados/v1/<%= @object.class.to_s.pluralize.underscore %>/<%= @object.uuid %> \
+ <<EOF
+<%= JSON.pretty_generate({@object.attributes.keys[-3] => @object.attributes.values[-3]}) %>
+EOF
+ </pre>
+ </div>
+</div>
+
+<div class="panel panel-default">
+ <div class="panel-heading"><b>arv</b> command line tool</div>
+ <div class="panel-body">
+ <pre>
+arv --pretty <%= @object.class.to_s.underscore %> get \
+ --uuid <%= @object.uuid %>
+
+arv <%= @object.class.to_s.underscore %> update \
+ --uuid <%= @object.uuid %> \
+ --<%= @object.class.to_s.underscore.gsub '_', '-' %> '<%= JSON.generate({@object.attributes.keys[-3] => @object.attributes.values[-3]}).gsub("'","'\''") %>'
+ </pre>
+ </div>
+</div>
+
+<div class="panel panel-default">
+ <div class="panel-heading"><b>Python</b> SDK</div>
+ <div class="panel-body">
+ <pre>
+import arvados
+
+x = arvados.api().<%= @object.class.to_s.pluralize.underscore %>().get(uuid='<%= @object.uuid %>').execute()
+ </pre>
+<% end %>
+ </div>
+</div>
--- /dev/null
+<%= form_for @object do |f| %>
+<table class="table topalign">
+ <thead>
+ </thead>
+ <tbody>
+ <% @object.attributes_for_display.each do |attr, attrvalue| %>
+ <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
+ <% end %>
+ </tbody>
+</table>
+
+<% end %>
+
--- /dev/null
+<pre>
+<%= JSON.pretty_generate(@object.attributes.reject { |k,v| k == 'id' }) rescue nil %>
+</pre>
--- /dev/null
+<% outgoing = Link.where(tail_uuid: @object.uuid) %>
+<% incoming = Link.where(head_uuid: @object.uuid) %>
+
+<h3>Metadata about this object</h3>
+<% if outgoing.items_available > 0 %>
+<table class="table topalign">
+ <thead>
+ <tr>
+ <th>metadata uuid</th>
+ <th>class</th>
+ <th>name</th>
+ <th>properties</th>
+ <th>object</th>
+ </tr>
+ </thead>
+ <tbody>
+ <% outgoing.each do |link| %>
+ <tr>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "uuid", attrvalue: link.uuid } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "link_class", attrvalue: link.link_class } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "name", attrvalue: link.name } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "properties", attrvalue: link.properties } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "head_uuid", attrvalue: link.head_uuid } %></td>
+ </tr>
+ <% end %>
+ </tbody>
+</table>
+<% else %>
+No metadata.
+<% end %>
+
+<h3>Metadata that refers to this object</h3>
+<% if outgoing.items_available > 0 %>
+<table class="table topalign">
+ <thead>
+ <tr>
+ <th>metadata uuid</th>
+ <th>subject</th>
+ <th>class</th>
+ <th>name</th>
+ <th>properties</th>
+ </tr>
+ </thead>
+ <tbody>
+ <% incoming.each do |link| %>
+ <tr>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "uuid", attrvalue: link.uuid } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "tail_uuid", attrvalue: link.tail_uuid } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "link_class", attrvalue: link.link_class } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "name", attrvalue: link.name } %></td>
+ <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "properties", attrvalue: link.properties } %></td>
+ </tr>
+ <% end %>
+ </tbody>
+</table>
+<% else %>
+No metadata.
+<% end %>
--- /dev/null
+<% if @objects.empty? %>
+<br/>
+<p style="text-align: center">
+ No <%= controller.model_class.to_s.pluralize.underscore.gsub '_', ' ' %> to display.
+</p>
+
+<% else %>
+
+<% attr_blacklist = ' created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at' %>
+
+<table class="table table-condensed arv-index">
+ <thead>
+ <tr>
+ <% @objects.first.attributes_for_display.each do |attr, attrvalue| %>
+ <% next if attr_blacklist.index(" "+attr) %>
+ <th class="arv-attr-<%= attr %>">
+ <%= controller.model_class.attribute_info[attr.to_sym].andand[:column_heading] or attr.sub /_uuid/, '' %>
+ </th>
+ <% end %>
+ <th>
+ <!-- a column for delete buttons -->
+ </th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <% @objects.each do |object| %>
+ <tr data-object-uuid="<%= object.uuid %>">
+ <% object.attributes_for_display.each do |attr, attrvalue| %>
+ <% next if attr_blacklist.index(" "+attr) %>
+ <td class="arv-object-<%= object.class.to_s %> arv-attr-<%= attr %>">
+ <% if attr == 'uuid' %>
+ <%= link_to_if_arvados_object object %>
+ <%= link_to_if_arvados_object(object, { link_text: raw('<i class="icon-hand-right"></i>') }) %>
+ <% else %>
+ <% if object.attribute_editable? attr %>
+ <%= render_editable_attribute object, attr %>
+ <% else %>
+ <%= resource_class_for_uuid(attrvalue, referring_attr: attr, referring_object: @object).to_s %>
+ <%= attrvalue %>
+ <% end %>
+ <%= link_to_if_arvados_object(attrvalue, { referring_object: @object, link_text: raw('<i class="icon-hand-right"></i>') }) if resource_class_for_uuid(attrvalue, {referring_object: @object}) %>
+ <% end %>
+ </td>
+ <% end %>
+ <td>
+ <% if object.editable? %>
+ <%= link_to({action: 'destroy', id: object.uuid}, method: :delete, remote: true, data: {confirm: "You are about to delete #{controller.model_class} #{object.uuid}.\n\nAre you sure?"}) do %>
+ <i class="glyphicon glyphicon-trash"></i>
+ <% end %>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+
+ <tfoot>
+ </tfoot>
+</table>
+
+<% end %>
--- /dev/null
+<%= content_for :css do %>
+/* Need separate style for each instance of svg div because javascript will manipulate the properties. */
+#<%= divId %> {
+ padding-left: 3px;
+ overflow: auto;
+ border: solid;
+ border-width: 1px;
+ border-color: gray;
+ position: absolute;
+ left: 1px;
+ right: 1px;
+}
+path:hover {
+stroke-width: 5;
+}
+path {
+stroke-linecap: round;
+}
+<% end %>
+
+<%= content_for :js do %>
+ $(window).on('load', function() {
+ $(window).on('load resize scroll', function () { graph_zoom("<%= divId %>","<%=svgId %>", 1) } );
+ });
+<% end %>
+
+<div id="_<%= divId %>_container" style="padding-top: 41px; margin-top: -41px">
+ <div style="text-align: right">
+ <a style="cursor: pointer"><span class="glyphicon glyphicon-zoom-out" onclick="graph_zoom('<%= divId %>', '<%= svgId %>', .9)"></span></a>
+ <a style="cursor: pointer"><span class="glyphicon glyphicon-zoom-in" onclick="graph_zoom('<%= divId %>', '<%= svgId %>', 1./.9)"></span></a>
+ </div>
+
+ <div id="<%= divId %>" class="smart-scroll">
+ <span id="_<%= divId %>_center" style="padding-left: 0px"></span>
+ <%= raw(svg) %>
+ </div>
+ <div id="_<%= divId %>_padding" style="padding-bottom: 1em"></div>
+</div>
--- /dev/null
+$('[data-object-uuid=<%= @object.uuid %>]').hide('slow', function() {
+ $(this).remove();
+});
-<%= render partial: 'index' %>
+<% content_for :page_title do %>
+<%= controller.model_class.to_s.pluralize.underscore.capitalize.gsub('_', ' ') %>
+<% end %>
+
+<% content_for :tab_line_buttons do %>
+
+<% if controller.model_class.creatable? %>
+<%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
+ { action: 'create', return_to: request.url },
+ { class: 'btn btn-primary pull-right' } %>
+<% end %>
+
+<% end %>
+
+<%= render partial: 'content', layout: 'content_layout', locals: {pane_list: controller.index_pane_list }%>
-<% if @object.respond_to? :properties %>
-
<% content_for :page_title do %>
-<%= @object.properties[:page_title] || @object.uuid %>
+ <%= (@object.respond_to?(:properties) ? @object.properties[:page_title] : nil) ||
+ @object.friendly_link_name %>
<% end %>
-<% if @object.properties[:page_content] %>
-<% content_for :page_content do %>
-<h1>
-<%= render_content_from_database(@object.properties[:page_title] || @object.uuid) %>
-</h1>
+<% content_for :content_top do %>
-<% if @object.properties[:page_subtitle] %>
-<h4>
-<%= render_content_from_database @object.properties[:page_subtitle] %>
-</h4>
-<% end %>
+<% if @object.respond_to? :properties %>
+ <% if @object.properties[:page_content] %>
+ <% content_for :page_content do %>
+ <h1>
+ <%= render_content_from_database(@object.properties[:page_title] || @object.uuid) %>
+ </h1>
+
+ <% if @object.properties[:page_subtitle] %>
+ <h4>
+ <%= render_content_from_database @object.properties[:page_subtitle] %>
+ </h4>
+ <% end %>
-<%= render_content_from_database @object.properties[:page_content] %>
-<% end %>
+ <%= render_content_from_database @object.properties[:page_content] %>
+ <% end %>
+ <% end %>
<% end %>
+
<% end %>
+<%= render partial: 'content', layout: 'content_layout', locals: {pane_list: controller.show_pane_list }%>
-<%= render :partial => 'application/arvados_object' %>
--- /dev/null
+<p>
+ More information about how to log in to VMs:
+</p>
+<ul>
+ <li>
+ <%= link_to raw('Arvados Docs → User Guide → SSH access'),
+ "#{Rails.configuration.arvados_docsite}/user/getting_started/ssh-access.html",
+ target: "_blank"%>.
+ </li>
+</ul>
+++ /dev/null
-<%= render partial: 'application/index' %>
-
-<hr />
-
-<p>
- See also:
- <%= link_to raw('Arvados Docs → User Guide → SSH access'),
- "#{Rails.configuration.arvados_docsite}/user/getting_started/ssh-access.html",
- target: "_blank"%>.
-</p>
<tr class="collection">
<td>
<%= link_to_if_arvados_object c.uuid %>
- </td><td>
- <% c.files.each do |file| %>
- <%= file[0] == '.' ? file[1] : "#{file[0]}/#{file[1]}" %>
+ </td>
+ <td>
+ <% i = 0 %>
+ <% while i < 3 and i < c.files.length %>
+ <% file = c.files[i] %>
+ <% file_path = "#{file[0]}/#{file[1]}" %>
+ <%= link_to file[1], {controller: 'collections', action: 'show_file', uuid: c.uuid, file: file_path, size: file[2], disposition: 'inline'}, {title: 'View in browser'} %><br />
+ <% i += 1 %>
+ <% end %>
+ <% if i < c.files.length %>
+ ⋮
<% end %>
- </td><td>
+ </td>
+ <td><%= link_to_if_arvados_object c.owner_uuid, friendly_name: true %></td>
+ <td>
<%= raw(distance_of_time_in_words(c.created_at, Time.now).sub('about ','~').sub(' ',' ')) if c.created_at %>
- </td><td>
+ </td>
+ <td>
<% if @collection_info[c.uuid] %>
<%= @collection_info[c.uuid][:tags].uniq.join ', ' %>
<% end %>
- </td><td>
+ </td>
+ <td>
<% if @collection_info[c.uuid][:wanted_by_me] %>
<span class="label label-info">2×</span>
<% elsif @collection_info[c.uuid][:wanted] %>
+++ /dev/null
-<ul class="nav nav-pills">
- <% [['Table', collections_path],
- ['Graph', collections_graph_path],
- ['Inspect', @object ? collection_path(@object.uuid) : '#']].
- each do |name, path| %>
- <li class="<%= 'active' if request.fullpath == path %> <%= 'disabled' if path == '#' %>"><%= link_to name, path %></li>
- <% end %>
-</ul>
--- /dev/null
+<table class="table table-condensed table-fixedlayout">
+ <colgroup>
+ <col width="35%" />
+ <col width="40%" />
+ <col width="15%" />
+ <col width="10%" />
+ </colgroup>
+ <thead>
+ <tr>
+ <th>path</th>
+ <th>file</th>
+ <th style="text-align:right">size</th>
+ <th>d/l</th>
+ </tr>
+ </thead><tbody>
+ <% if @object then @object.files.sort_by{|f|f[1]}.each do |file| %>
+ <% file_path = "#{file[0]}/#{file[1]}" %>
+ <tr>
+ <td>
+ <%= file[0] %>
+ </td>
+
+ <td>
+ <%= link_to file[1], {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'inline'}, {title: 'View in browser'} %>
+ </td>
+
+ <td style="text-align:right">
+ <%= raw(human_readable_bytes_html(file[2])) %>
+ </td>
+
+ <td>
+ <div style="display:inline-block">
+ <%= link_to raw('<i class="glyphicon glyphicon-download-alt"></i>'), {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'attachment'}, {class: 'btn btn-info btn-sm', title: 'Download'} %>
+ </div>
+ </td>
+ </tr>
+ <% end; end %>
+ </tbody>
+</table>
--- /dev/null
+<table class="topalign table table-bordered">
+ <thead>
+ <tr class="contain-align-left">
+ <th>
+ job
+ </th><th>
+ version
+ </th><th>
+ status
+ </th><th>
+ start
+ </th><th>
+ finish
+ </th><th>
+ clock time
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+
+ <% @provenance.reverse.each do |p| %>
+ <% j = p[:job] %>
+
+ <% if j %>
+
+ <tr class="job">
+ <td>
+ <tt><%= j.uuid %></tt>
+ <br />
+ <tt class="deemphasize"><%= j.submit_id %></tt>
+ </td><td>
+ <%= j.script_version %>
+ </td><td>
+ <span class="label <%= if j.success then 'label-success'; elsif j.running then 'label-primary'; else 'label-warning'; end %>">
+ <%= j.success || j.running ? 'ok' : 'failed' %>
+ </span>
+ </td><td>
+ <%= j.started_at %>
+ </td><td>
+ <%= j.finished_at %>
+ </td><td>
+ <% if j.started_at and j.finished_at %>
+ <%= raw(distance_of_time_in_words(j.started_at, j.finished_at).sub('about ','~').sub(' ',' ')) %>
+ <% elsif j.started_at and j.running %>
+ <%= raw(distance_of_time_in_words(j.started_at, Time.now).sub('about ','~').sub(' ',' ')) %> (running)
+ <% end %>
+ </td>
+ </tr>
+
+ <% else %>
+ <tr>
+ <td>
+ <span class="label label-danger">lookup fail</span>
+ <br />
+ <tt class="deemphasize"><%= p[:target] %></tt>
+ </td><td colspan="4">
+ </td>
+ </tr>
+ <% end %>
+
+ <% end %>
+
+ </tbody>
+</table>
--- /dev/null
+<%= content_for :css do %>
+<%# https://github.com/mbostock/d3/wiki/Ordinal-Scales %>
+<% n=-1; %w(#1f77b4 #ff7f0e #2ca02c #d62728 #9467bd #8c564b #e377c2 #7f7f7f #bcbd22 #17becf).each do |color| %>
+.colorseries-10-<%= n += 1 %>, .btn.colorseries-10-<%= n %>:hover, .label.colorseries-10-<%= n %>:hover {
+ *background-color: <%= color %>;
+ background-color: <%= color %>;
+ background-image: none;
+}
+<% end %>
+.colorseries-nil { }
+.label a {
+ color: inherit;
+}
+<% end %>
+
+<table class="topalign table table-bordered">
+ <thead>
+ </thead>
+ <tbody>
+
+ <% @provenance.reverse.each do |p| %>
+ <% j = p[:job] %>
+
+ <% if j %>
+
+ <tr class="job">
+ <td style="padding-bottom: 3em">
+ <table class="table" style="margin-bottom: 0; background: #f0f0ff">
+ <% j.script_parameters.each do |k,v| %>
+ <tr>
+ <td style="width: 20%">
+ <%= k.to_s %>
+ </td><td style="width: 60%">
+ <% if v and @output2job.has_key? v %>
+ <tt class="label colorseries-10-<%= @output2colorindex[v] %>"><%= link_to_if_arvados_object v %></tt>
+ <% else %>
+ <span class="deemphasize"><%= link_to_if_arvados_object v %></span>
+ <% end %>
+ </td><td style="text-align: center; width: 20%">
+ <% if v
+ if @protected[v]
+ labelclass = 'success'
+ labeltext = 'keep'
+ else
+ labelclass = @output2job.has_key?(v) ? 'warning' : 'danger'
+ labeltext = 'cache'
+ end %>
+
+ <tt class="label label-<%= labelclass %>"><%= labeltext %></tt>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </table>
+ <div style="text-align: center">
+ ↓
+ <br />
+ <span class="label"><%= j.script %><br /><tt><%= link_to_if j.script_version.match(/[0-9a-f]{40}/), j.script_version, "https://arvados.org/projects/arvados/repository/revisions/#{j.script_version}/entry/crunch_scripts/#{j.script}" if j.script_version %></tt></span>
+ <br />
+ ↓
+ <br />
+ <tt class="label colorseries-10-<%= @output2colorindex[p[:output]] %>"><%= link_to_if_arvados_object p[:output] %></tt>
+ </div>
+ </td>
+ <td>
+ <tt><span class="deemphasize">job:</span><br /><%= link_to_if_arvados_object j %><br /><span class="deemphasize"><%= j.submit_id %></span></tt>
+ </td>
+ </tr>
+
+ <% else %>
+ <tr>
+ <td>
+ <span class="label label-danger">lookup fail</span>
+ <br />
+ <tt class="deemphasize"><%= p[:target] %></tt>
+ </td><td colspan="5">
+ </td>
+ </tr>
+ <% end %>
+
+ <% end %>
+
+ </tbody>
+</table>
--- /dev/null
+<%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph_div",
+ svgId: "provenance_svg",
+ svg: @prov_svg } %>
--- /dev/null
+<% content_for :tab_line_buttons do %>
+<div class="pull-right" style="width: 30%">
+ <%= form_tag collections_path, method: 'get', remote: true, class: 'form-search' do %>
+ <div class="input-group">
+ <%= text_field_tag :search, params[:search], class: 'form-control', placeholder: 'Search collections' %>
+ <span class="input-group-btn">
+ <%= button_tag(class: 'btn btn-info') do %>
+ <span class="glyphicon glyphicon-search"></span>
+ <% end %>
+ </span>
+ </div>
+ <% end %>
+</div>
+<% end %>
+
+<div style="padding-right: 1em">
+
+<table id="collections-index" class="topalign table table-condensed table-fixedlayout">
+ <colgroup>
+ <col width="10%" />
+ <col width="36%" />
+ <col width="22%" />
+ <col width="8%" />
+ <col width="16%" />
+ <col width="8%" />
+ </colgroup>
+ <thead>
+ <tr class="contain-align-left">
+ <th>uuid</th>
+ <th>contents</th>
+ <th>owner</th>
+ <th>age</th>
+ <th>tags</th>
+ <th>storage</th>
+ </tr>
+ </thead>
+ <tbody>
+ <%= render partial: 'index_tbody' %>
+ </tbody>
+</table>
+</div>
+
+<% content_for :js do %>
+$(document).on('click', 'form[data-remote] input[type=submit]', function() {
+ $('table#collections-index tbody').fadeTo(200, 0.3);
+ return true;
+});
+<% end %>
--- /dev/null
+<table class="table table-bordered table-striped">
+ <thead>
+ <tr class="contain-align-left">
+ <th>
+ collection
+ </th><th class="data-size">
+ data size
+ </th><th>
+ storage
+ </th><th>
+ origin
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+
+ <% @sourcedata.values.each do |sourcedata| %>
+
+ <tr class="collection">
+ <td>
+ <tt class="label"><%= sourcedata[:uuid] %></tt>
+ </td><td class="data-size">
+ <%= raw(human_readable_bytes_html(sourcedata[:collection].data_size)) if sourcedata[:collection] and sourcedata[:collection].data_size %>
+ </td><td>
+ <% if @protected[sourcedata[:uuid]] %>
+ <span class="label label-success">keep</span>
+ <% else %>
+ <span class="label label-danger">cache</span>
+ <% end %>
+ </td><td>
+ <% if sourcedata[:data_origins] %>
+ <% sourcedata[:data_origins].each do |data_origin| %>
+ <span class="deemphasize"><%= data_origin[0] %></span>
+ <%= data_origin[2] %>
+ <br />
+ <% end %>
+ <% end %>
+ </td>
+ </tr>
+
+ <% end %>
+
+ </tbody>
+</table>
--- /dev/null
+<%= render partial: 'application/svg_div', locals: {
+ divId: "used_by_graph",
+ svgId: "used_by_svg",
+ svg: @used_by_svg } %>
+
+++ /dev/null
-<%#= render :partial => 'nav' %>
-
-<div class="pull-right">
- <%= form_tag collections_path, method: 'get', remote: true, class: 'form-search' do %>
- <div class="input-append">
- <%= text_field_tag :search, params[:search], class: 'search-query' %>
- <%= submit_tag "Search", name: nil, class: 'btn btn-info' %>
- </div>
- <% end %>
-</div>
-
-<table id="collections-index" class="topalign table table-bordered table-condensed table-fixedlayout table-smallcontent">
- <colgroup>
- <col width="10%" />
- <col width="50%" />
- <col width="16%" />
- <col width="16%" />
- <col width="8%" />
- </colgroup>
- <thead>
- <tr class="contain-align-left">
- <th>
- uuid
- </th><th>
- contents
- </th><th>
- age
- </th><th>
- tags
- </th><th>
- storage
- </th>
- </tr>
- </thead>
- <tbody>
- <%= render partial: 'index_tbody' %>
- </tbody>
-</table>
-
-<% content_for :js do %>
-$(document).on('click', 'form[data-remote] input[type=submit]', function() {
- $('table#collections-index tbody').fadeTo(200, 0.3);
- return true;
-});
-<% end %>
+++ /dev/null
-<%= content_for :head do %>
-<style>
-<%# https://github.com/mbostock/d3/wiki/Ordinal-Scales %>
-<% n=-1; %w(#1f77b4 #ff7f0e #2ca02c #d62728 #9467bd #8c564b #e377c2 #7f7f7f #bcbd22 #17becf).each do |color| %>
-.colorseries-10-<%= n += 1 %>, .btn.colorseries-10-<%= n %>:hover, .label.colorseries-10-<%= n %>:hover {
- *background-color: <%= color %>;
- background-color: <%= color %>;
- background-image: none;
-}
-<% end %>
-.colorseries-nil { }
-.label a {
- color: inherit;
-}
-</style>
-<% end %>
-
-<%#= render :partial => 'nav' %>
-
-<ul class="nav nav-tabs">
- <li class="active"><a href="#files" data-toggle="tab">Files (<%= @object.files ? @object.files.size : 0 %>)</a></li>
- <li><a href="#provenance" data-toggle="tab">Provenance (<%= @provenance.size %>)</a></li>
- <li><a href="#jobs" data-toggle="tab">Jobs (<%= @provenance.size %>)</a></li>
- <li><a href="#sourcedata" data-toggle="tab">Source data (<%= @sourcedata.size %>)</a></li>
- <li><a href="#owner-groups-resources" data-toggle="tab">Owner, groups, resources</a></li>
-</ul>
-
-<div class="tab-content">
- <div id="files" class="tab-pane fade in active">
- <table class="table table-bordered" style="table-layout: fixed">
- <thead>
- <tr>
- <th>path</th>
- <th>file</th>
- <th style="width:1.5em">d/l</th>
- <th style="width: 7em; text-align:right">size</th>
- </tr>
- </thead><tbody>
- <% if @object then @object.files.sort_by{|f|f[1]}.each do |file| %>
- <% file_path = "#{file[0]}/#{file[1]}" %>
- <tr>
- <td>
- <%= file[0] %>
- </td>
-
- <td>
- <%= link_to file[1], {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'inline'}, {title: 'View in browser'} %>
- </td>
-
- <td>
- <div style="display:inline-block">
- <%= link_to raw('<i class="icon-download"></i>'), {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'attachment'}, {class: 'label label-info', title: 'Download'} %>
- </div>
- </td>
-
- <td style="text-align:right">
- <%= raw(human_readable_bytes_html(file[2])) %>
- </td>
-
- </tr>
- <% end; end %>
- </tbody>
- </table>
- </div>
- <div id="provenance" class="tab-pane fade">
- <table class="topalign table table-bordered">
- <thead>
- </thead>
- <tbody>
-
- <% @provenance.reverse.each do |p| %>
- <% j = p[:job] %>
-
- <% if j %>
-
- <tr class="job">
- <td style="padding-bottom: 3em">
- <table class="table" style="margin-bottom: 0; background: #f0f0ff">
- <% j.script_parameters.each do |k,v| %>
- <tr>
- <td style="width: 20%">
- <%= k.to_s %>
- </td><td style="width: 60%">
- <% if v and @output2job.has_key? v %>
- <tt class="label colorseries-10-<%= @output2colorindex[v] %>"><%= link_to_if_arvados_object v %></tt>
- <% else %>
- <span class="deemphasize"><%= link_to_if_arvados_object v %></span>
- <% end %>
- </td><td style="text-align: center; width: 20%">
- <% if v
- if @protected[v]
- labelclass = 'success'
- labeltext = 'keep'
- else
- labelclass = @output2job.has_key?(v) ? 'warning' : 'danger'
- labeltext = 'cache'
- end %>
-
- <tt class="label label-<%= labelclass %>"><%= labeltext %></tt>
- <% end %>
- </td>
- </tr>
- <% end %>
- </table>
- <div style="text-align: center">
- ↓
- <br />
- <span class="label"><%= j.script %><br /><tt><%= link_to_if j.script_version.match(/[0-9a-f]{40}/), j.script_version, "https://arvados.org/projects/arvados/repository/revisions/#{j.script_version}/entry/crunch_scripts/#{j.script}" if j.script_version %></tt></span>
- <br />
- ↓
- <br />
- <tt class="label colorseries-10-<%= @output2colorindex[p[:output]] %>"><%= link_to_if_arvados_object p[:output] %></tt>
- </div>
- </td>
- <td>
- <tt><span class="deemphasize">job:</span><br /><%= link_to_if_arvados_object j %><br /><span class="deemphasize"><%= j.submit_id %></span></tt>
- </td>
- </tr>
-
- <% else %>
- <tr>
- <td>
- <span class="label label-danger">lookup fail</span>
- <br />
- <tt class="deemphasize"><%= p[:target] %></tt>
- </td><td colspan="5">
- </td>
- </tr>
- <% end %>
-
- <% end %>
-
- </tbody>
- </table>
- </div>
- <div id="jobs" class="tab-pane fade">
- <table class="topalign table table-bordered">
- <thead>
- <tr class="contain-align-left">
- <th>
- job
- </th><th>
- version
- </th><th>
- status
- </th><th>
- start
- </th><th>
- finish
- </th><th>
- clock time
- </th>
- </tr>
- </thead>
- <tbody>
-
- <% @provenance.reverse.each do |p| %>
- <% j = p[:job] %>
-
- <% if j %>
-
- <tr class="job">
- <td>
- <tt><%= j.uuid %></tt>
- <br />
- <tt class="deemphasize"><%= j.submit_id %></tt>
- </td><td>
- <%= j.script_version %>
- </td><td>
- <span class="label <%= if j.success then 'label-success'; elsif j.active then 'label-primary'; else 'label-warning'; end %>">
- <%= j.success || j.active ? 'ok' : 'failed' %>
- </span>
- </td><td>
- <%= j.started_at %>
- </td><td>
- <%= j.finished_at %>
- </td><td>
- <% if j.started_at and j.finished_at %>
- <%= raw(distance_of_time_in_words(j.started_at, j.finished_at).sub('about ','~').sub(' ',' ')) %>
- <% elsif j.started_at and j.running %>
- <%= raw(distance_of_time_in_words(j.started_at, Time.now).sub('about ','~').sub(' ',' ')) %> (running)
- <% end %>
- </td>
- </tr>
-
- <% else %>
- <tr>
- <td>
- <span class="label label-danger">lookup fail</span>
- <br />
- <tt class="deemphasize"><%= p[:target] %></tt>
- </td><td colspan="4">
- </td>
- </tr>
- <% end %>
-
- <% end %>
-
- </tbody>
- </table>
- </div>
- <div id="sourcedata" class="tab-pane fade">
- <table class="table table-bordered table-striped">
- <thead>
- <tr class="contain-align-left">
- <th>
- collection
- </th><th class="data-size">
- data size
- </th><th>
- storage
- </th><th>
- origin
- </th>
- </tr>
- </thead>
- <tbody>
-
- <% @sourcedata.values.each do |sourcedata| %>
-
- <tr class="collection">
- <td>
- <tt class="label"><%= sourcedata[:uuid] %></tt>
- </td><td class="data-size">
- <%= raw(human_readable_bytes_html(sourcedata[:collection].data_size)) if sourcedata[:collection] and sourcedata[:collection].data_size %>
- </td><td>
- <% if @protected[sourcedata[:uuid]] %>
- <span class="label label-success">keep</span>
- <% else %>
- <span class="label label-danger">cache</span>
- <% end %>
- </td><td>
- <% if sourcedata[:data_origins] %>
- <% sourcedata[:data_origins].each do |data_origin| %>
- <span class="deemphasize"><%= data_origin[0] %></span>
- <%= data_origin[2] %>
- <br />
- <% end %>
- <% end %>
- </td>
- </tr>
-
- <% end %>
-
- </tbody>
- </table>
- </div>
- <div id="owner-groups-resources" class="tab-pane fade">
- <%= render :partial => 'application/arvados_object' %>
- </div>
-</div>
--- /dev/null
+<%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @svg } %>
-<% content_for :head do %>
-<style>
+<% content_for :css do %>
table.topalign>tbody>tr>td {
vertical-align: top;
}
table.topalign>thead>tr>td {
vertical-align: bottom;
}
-</style>
<% end %>
<table class="topalign table">
</tbody>
</table>
+
+++ /dev/null
-<%= render :partial => 'application/arvados_object' %>
<meta charset="utf-8">
<title>
<% if content_for? :page_title %>
- <%= yield :page_title %>
+ <%= yield :page_title %> / <%= Rails.configuration.site_name %>
<% else %>
<%= Rails.configuration.site_name %>
<% end %>
<%= yield :js %>
<% end %>
<style>
- .container {
- padding-top: 60px; /* 60px to make the container go all the way to the
- bottom of the topbar */
+ <%= yield :css %>
+ body {
+ min-height: 100%;
+ height: 100%;
+ }
+
+ body > div.container-fluid {
+ padding-top: 70px; /* 70px to make the container go all the way to the bottom of the navbar */
}
+
+ body > div.container-fluid > div.col-sm-9.col-sm-offset-3 {
+ overflow: auto;
+ }
+
@media (max-width: 979px) { body { padding-top: 0; } }
- <%= yield :css %>
+
+ .navbar .nav li.nav-separator > span.glyphicon.glyphicon-arrow-right {
+ padding-top: 1.25em;
+ }
+
+ /* Setting the height needs to be fixed with javascript. */
+ .dropdown-menu {
+ padding-right: 20px;
+ max-height: 440px;
+ width: 400px;
+ overflow-y: auto;
+ }
+
+ .arvados-nav-container {
+ position: fixed;
+ top: 70px;
+ height: calc(100% - 70px);
+ overflow: auto;
+ z-index: 2;
+ }
+ .arvados-nav-active {
+ background: rgb(66, 139, 202);
+ }
+ .arvados-nav-active a {
+ color: white;
+ }
</style>
</head>
<body>
- <div class="navbar navbar-inverse navbar-fixed-top">
- <div class="navbar-inner">
- <a class="brand" style="margin-left: 1px" href="/"><%= Rails.configuration.site_name rescue Rails.application.class.parent_name %></a>
+ <div class="navbar navbar-default navbar-fixed-top">
+ <div class="container-fluid">
+ <ul class="nav navbar-nav navbar-left">
+ <li><a class="navbar-brand" href="/"><%= Rails.configuration.site_name rescue Rails.application.class.parent_name %></a></li>
+ <% if current_user %>
+ <% if content_for?(:breadcrumbs) %>
+ <%= yield(:breadcrumbs) %>
+ <% else %>
+ <li class="nav-separator"><span class="glyphicon glyphicon-arrow-right"></span></li>
+ <li>
+ <%= link_to(
+ controller.model_class.to_s.pluralize.underscore.gsub('_', ' '),
+ url_for({controller: params[:controller]})) %>
+ </li>
+ <% if params[:action] != 'index' %>
+ <li class="nav-separator">
+ <span class="glyphicon glyphicon-arrow-right"></span>
+ </li>
+ <li>
+<%= link_to controller.breadcrumb_page_name, request.fullpath %>
+ </li>
+ <% end %>
+ <% end %>
+ <% end %>
+ </ul>
+
+ <ul class="nav navbar-nav navbar-right">
+
+ <% if current_user %>
+ <li>
+ <div class="loading" style="transform: translate(-20px,20px) scale(0.1,0.1); -ms-transform: translate(-20px,20px) scale(0.1,0.1); -webkit-transform: translate(-20px,20px) scale(0.1,0.1); display: none">
+ <%= render partial: 'loading' %>
+ </div>
+ </li>
+
+ <!-- XXX placeholder for this when search is implemented
+ <li>
+ <form class="navbar-form" role="search">
+ <div class="input-group" style="width: 220px">
+ <input type="text" class="form-control" placeholder="search">
+ <span class="input-group-addon"><span class="glyphicon glyphicon-search"></span></span>
+ </div>
+ </form>
+ </li>
+ -->
+
+ <!-- XXX placeholder for this when persistent selection is implemented
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <span class="glyphicon glyphicon-paperclip"></span>
+ <span class="badge badge-alert"><%= @selection_count %></span>
+ <span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu" role="menu">
+ <li style="padding: 10px">No selections.</li>
+ </ul>
+ </li>
+ -->
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <span class="glyphicon glyphicon-envelope"></span>
+ <span class="badge badge-alert"><%= @notification_count %></span>
+ <span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu" role="menu">
+ <% if @notifications.length > 0 %>
+ <% @notifications.each_with_index do |n, i| %>
+ <% if i > 0 %><li class="divider"></li><% end %>
+ <li style="padding: 10px"><%= n.call(self) %></li>
+ <% end %>
+ <% else %>
+ <li style="padding: 10px">No notifications.</li>
+ <% end %>
+ </ul>
+ </li>
- <ul class="nav pull-right">
- <% if current_user -%>
- <li><span class="badge badge-info" style="margin: 10px auto 10px; padding-top: 4px; padding-bottom: 4px"><%= current_user.email %></span></li>
- <li><a href="<%= logout_path %>">Log out</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <span class="glyphicon glyphicon-user"></span><span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header"><%= current_user.email %></li>
+ <li role="presentation" class="divider"></li>
+ <li role="presentation"><a href="/authorized_keys" role="menuitem">Manage ssh keys</a></li>
+ <li role="presentation"><a href="/api_client_authorizations" role="menuitem">Manage API tokens</a></li>
+ <li role="presentation" class="divider"></li>
+ <li role="presentation"><a href="<%= logout_path %>" role="menuitem">Log out</a></li>
+ </ul>
+ </li>
<% else -%>
<li><a href="<%= $arvados_api_client.arvados_login_url(return_to: root_url) %>">Log in</a></li>
<% end -%>
</ul>
+ </div>
+ </div>
- <% if current_user.andand.is_active %>
- <ul class="nav">
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
- Access <b class="caret"></b>
- </a>
- <ul class="dropdown-menu">
- <li><%= link_to 'Keys', authorized_keys_path %></li>
- <li><%= link_to 'VMs', virtual_machines_path %></li>
- <li><%= link_to 'Repositories', repositories_path %></li>
- <li><%= link_to 'API Tokens', api_client_authorizations_path %></li>
- </ul>
- </li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
- Compute <b class="caret"></b>
- </a>
- <ul class="dropdown-menu">
- <li><%= link_to 'Jobs', jobs_path %></li>
- <li><%= link_to 'Pipeline instances', pipeline_instances_path %></li>
- <li><%= link_to 'Pipeline templates', pipeline_templates_path %></li>
- </ul>
- </li>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
- Data <b class="caret"></b>
- </a>
- <ul class="dropdown-menu">
- <li><%= link_to 'Collections', collections_path %></li>
- <li><%= link_to 'Links', links_path %></li>
- <li><%= link_to 'Humans', humans_path %></li>
- <li><%= link_to 'Traits', traits_path %></li>
- </ul>
+ <div class="container-fluid">
+ <div class="col-sm-3">
+ <div class="left-nav arvados-nav-container">
+ <% if current_user %>
+ <div class="well">
+ <ul class="arvados-nav">
+ <li class="<%= 'arvados-nav-active' if params[:action] == 'home' %>">
+ <a href="/">Dashboard</a>
</li>
- <% if current_user.is_admin %>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
- Admin <b class="caret"></b>
- </a>
- <ul class="dropdown-menu">
- <li><%= link_to 'Users', users_path %></li>
- <li><%= link_to 'Groups', groups_path %></li>
- <li><%= link_to 'Nodes', nodes_path %></li>
+
+ <% [['Data', [['humans'],
+ ['traits'],
+ ['specimens'],
+ ['collections', 'Files'],
+ ['links', 'Metadata']]],
+ ['Compute', [['pipeline_templates', 'Pipelines'],
+ ['repositories', 'Code repositories'],
+ ['virtual_machines']]],
+ ['Activity', [['pipeline_instances', 'Recent pipelines'],
+ ['jobs', 'Recent jobs']]],
+ ['System', [['users'],
+ ['groups'],
+ ['nodes', 'Compute nodes'],
+ ['keep_disks']]]].each do |j| %>
+ <li><%= j[0] %>
+ <ul>
+ <% j[1].each do |k| %>
+ <% unless k[0] == 'users' and !current_user.andand.is_admin %>
+ <li class="<%= 'arvados-nav-active' if (params[:controller] == k[0] && params[:action] != 'home') %>">
+ <a href="/<%= k[0] %>">
+ <%= if k[1] then k[1] else k[0].capitalize.gsub('_', ' ') end %>
+ </a>
+ </li>
+ <% end %>
+ <% end %>
</ul>
</li>
<% end %>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
- Docs <b class="caret"></b>
- </a>
- <ul class="dropdown-menu">
+
+ <li>Help
+ <ul>
<li><%= link_to 'Tutorials and User guide', "#{Rails.configuration.arvados_docsite}/user", target: "_blank" %></li>
<li><%= link_to 'API Reference', "#{Rails.configuration.arvados_docsite}/api", target: "_blank" %></li>
+ <li><%= link_to 'SDK Reference', "#{Rails.configuration.arvados_docsite}/sdk", target: "_blank" %></li>
<li><%= link_to 'Admin guide', "#{Rails.configuration.arvados_docsite}/admin", target: "_blank" %></li>
- <li><%= link_to 'Install guide', "#{Rails.configuration.arvados_docsite}/install", target: "_blank" %></li>
</ul>
</li>
-
- </ul>
- <% end %>
- </div>
+ </ul>
+ </div>
+ <% end %>
+ </div>
+ </div>
+ <div class="col-sm-9 col-sm-offset-3">
+ <div id="content">
+ <%= yield %>
+ </div>
+ </div>
</div>
- <div class="container">
-
- <%= yield %>
-
- </div> <!-- /container -->
-
<%= piwik_tracking_tag %>
<%= javascript_tag do %>
<%= yield :footer_js %>
--- /dev/null
+<% if @object %>
+(<%= @object.link_class %>)
+<%= @object.name %>:
+<%= @object.tail_kind.andand.sub 'arvados#', '' %>
+→
+<%= @object.head_kind.andand.sub 'arvados#', '' %>
+<% end %>
+
<td>
<% if current_user and (current_user.is_admin or current_user.uuid == link.owner_uuid) %>
- <%= link_to raw('<i class="icon-trash"></i>'), { action: 'destroy', id: link.uuid }, { confirm: 'Delete this link?', method: 'delete' } %>
+ <%= link_to raw('<i class="glyphicon glyphicon-trash"></i>'), { action: 'destroy', id: link.uuid }, { confirm: 'Delete this link?', method: 'delete' } %>
<% end %>
</td>
--- /dev/null
+ <%= image_tag "dax.png", class: "dax" %>
+ <p>
+ Hi, I noticed you haven't uploaded a new collection yet.
+ <%= link_to "Click here to learn how to upload data to Arvados Keep.",
+ "#{Rails.configuration.arvados_docsite}/user/tutorials/tutorial-keep.html",
+ style: "font-weight: bold", target: "_blank" %>
+ </p>
--- /dev/null
+ <p><%= image_tag "dax.png", class: "dax" %>
+ Hi, I noticed you haven't run a job yet.
+ <%= link_to "Click here to learn how to run an Arvados Crunch job.",
+ "#{Rails.configuration.arvados_docsite}/user/tutorials/tutorial-job1.html",
+ style: "font-weight: bold",
+ target: "_blank" %>
+ </p>
+
--- /dev/null
+ <p><%= image_tag "dax.png", class: "dax" %>
+ Hi, I noticed you haven't run a pipeline yet.
+ <%= link_to "Click here to learn how to run an Arvados Crunch pipeline.",
+ "#{Rails.configuration.arvados_docsite}/user/tutorials/tutorial-new-pipeline.html",
+ style: "font-weight: bold",
+ target: "_blank" %>
+ </p>
--- /dev/null
+ <%= image_tag "dax.png", class: "dax" %>
+ <div>
+ Hi, I noticed that you have not yet set up an SSH public key for use with Arvados.
+ <%= link_to "Click here to learn about SSH keys in Arvados.",
+ "#{Rails.configuration.arvados_docsite}/user/getting_started/ssh-access.html",
+ style: "font-weight: bold",
+ target: "_blank" %>
+ When you have an SSH key you would like to use, paste the SSH public key
+ in the text box.
+ </div>
+ <%= form_for AuthorizedKey.new, remote: true do |f| %>
+ <div class="row-fluid">
+ <%= hidden_field_tag :return_to, request.original_url %>
+ <%= hidden_field_tag :disable_element, 'input[type=submit]' %>
+ <%= f.text_area :public_key, rows: 4, placeholder: "Paste your public key here", style: "width: 100%" %>
+ </div>
+ <div class="row-fluid" style="padding-top: 0; padding-bottom: 15px">
+ <%= f.submit :Save, value: raw("✓"), class: "btn btn-primary pull-right" %>
+ </div>
+<% end %>
--- /dev/null
+<% content_for :css do %>
+ .pipeline_color_legend {
+ padding-left: 1em;
+ padding-right: 1em;
+ }
+table.pipeline-components-table thead th {
+ text-align: bottom;
+}
+table.pipeline-components-table div.progress {
+ margin-bottom: 0;
+}
+<% end %>
+<br />
+
+<table class="table pipeline-components-table">
+ <colgroup>
+ <col width="15%" />
+ <col width="15%" />
+ <col width="35%" />
+ <col width="35%" />
+ </colgroup>
+ <thead>
+ <tr>
+ <th>
+ component
+ </th><th>
+ progress
+ <%= link_to '(refresh)', request.fullpath, class: 'refresh', remote: true, method: 'get' %>
+ </th><th>
+ script, version
+ </th><th>
+ output
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ <% render_pipeline_jobs.each do |pj| %>
+ <tr>
+ <td>
+ <% if pj[:job].andand[:uuid] %>
+ <%= link_to pj[:name], job_url(id: pj[:job][:uuid]) %>
+ <% else %>
+ <%= pj[:name] %>
+ <% end %>
+ </td><td>
+ <%= pj[:progress_bar] %>
+ <% if pj[:job].andand[:cancelled_at] %>
+ <span class="pull-right label label-warning">cancelled</span>
+ <% elsif pj[:failed] %>
+ <span class="pull-right label label-warning">failed</span>
+ <% elsif pj[:result] == 'queued' %>
+ <span class="pull-right label">queued</span>
+ <% end %>
+ </td><td>
+ <%= pj[:script] %>
+ <br /><span class="deemphasize"><%= pj[:script_version] %></span>
+ </td><td>
+ <%= link_to_if_arvados_object pj[:output] %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ <tfoot>
+ <tr><td colspan="4"></td></tr>
+ </tfoot>
+</table>
+
+<% if @object.active %>
+<% content_for :js do %>
+setInterval(function(){$('a.refresh').click()}, 30000);
+<% end %>
+<% end %>
+
+<pre><%= JSON.pretty_generate @object.attributes %></pre>
--- /dev/null
+<% content_for :css do %>
+ .pipeline_color_legend {
+ padding-left: 1em;
+ padding-right: 1em;
+ }
+<% end %>
+
+<% if @pipelines.count > 1 %>
+ <div style="text-align: center">
+ <span class="pipeline_color_legend" style="background: #88ff88">This pipeline</span>
+ <span class="pipeline_color_legend" style="background: #8888ff">Comparison pipeline</span>
+ <span class="pipeline_color_legend" style="background: #88ffff">Shared by both pipelines</span>
+ </div>
+<% end %>
+
+<%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @prov_svg } %>
--- /dev/null
+var new_content = "<%= escape_javascript(render template: 'pipeline_instances/show.html') %>";
+if ($('div.body-content').html() != new_content)
+ $('div.body-content').html(new_content);
--- /dev/null
+<%= content_for :tab_line_buttons do %>
+<%= form_tag({action: 'compare', controller: params[:controller], method: 'get'}, {method: 'get', id: 'compare', class: 'pull-right small-form-margin'}) do |f| %>
+ <%= submit_tag 'Compare selected', {class: 'btn btn-primary', disabled: true, style: 'display: none'} %>
+
+<% end rescue nil %>
+<% end %>
+
+<%= form_tag do |f| %>
+
+<table class="table table-condensed table-fixedlayout">
+ <colgroup>
+ <col width="5%" />
+ <col width="10%" />
+ <col width="20%" />
+ <col width="10%" />
+ <col width="30%" />
+ <col width="15%" />
+ <col width="10%" />
+ </colgroup>
+ <thead>
+ <tr class="contain-align-left">
+ <th>
+ </th><th>
+ Status
+ </th><th>
+ Instance
+ </th><th colspan="2">
+ Template
+ </th><th>
+ Owner
+ </th><th>
+ Age
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+
+ <% @objects.sort_by { |ob| ob.created_at }.reverse.each do |ob| %>
+
+ <tr data-object-uuid="<%= ob.uuid %>">
+ <td>
+ <%= check_box_tag 'uuids[]', ob.uuid, false %>
+ </td><td>
+ <% if ob.success %>
+ <span class="label label-success">success</span>
+ <% elsif ob.active %>
+ <span class="label label-info">active</span>
+ <% end %>
+ </td><td colspan="2">
+ <%= link_to_if_arvados_object ob, friendly_name: true %>
+ </td><td>
+ <%= link_to_if_arvados_object ob.pipeline_template_uuid, friendly_name: true %>
+ </td><td>
+ <%= link_to_if_arvados_object ob.owner_uuid, friendly_name: true %>
+ </td><td>
+ <%= distance_of_time_in_words(ob.created_at, Time.now) %>
+ </td>
+ </tr>
+ <tr>
+ <td style="border-top: 0;" colspan="3">
+ </td>
+ <td style="border-top: 0; opacity: 0.5;" colspan="4">
+ <% ob.components.each do |cname, c| %>
+ <% status = if !(c.is_a?(Hash) && c[:job].is_a?(Hash))
+ nil
+ elsif c[:job][:success]
+ 'success'
+ elsif c[:job][:running]
+ 'info'
+ else
+ 'warning'
+ end %>
+ <span class="label label-<%= status || 'default' %>"><%= cname.to_s %></span>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+
+ </tbody>
+</table>
+
+<% end %>
+
+<% content_for :footer_js do %>
+var showhide_compare = function() {
+ var form = $('form#compare')[0];
+ $('input[type=hidden][name="uuids[]"]', form).remove();
+ $('input[type=submit]', form).prop('disabled',true);
+ $('input[name="uuids[]"]').each(function(){
+ if(this.checked) {
+ $('input[type=submit]', form).prop('disabled',false).show();
+ $(form).append($('<input type="hidden" name="uuids[]"/>').val(this.value));
+ }
+ });
+};
+$('form input[name="uuids[]"]').on('click', showhide_compare);
+showhide_compare();
+<% end %>
--- /dev/null
+<% content_for :css do %>
+.notnormal {
+ background: #ffffaa;
+}
+.headrow div {
+ padding-top: .5em;
+ padding-bottom: .5em;
+}
+.headrow:first-child {
+ border-bottom: 1px solid black;
+}
+<% end %>
+
+<% pi_span = [(10.0/[@objects.count,1].max).floor,1].max %>
+
+<div class="headrow">
+ <div class="row">
+ <div class="col-sm-2">
+ <%# label %>
+ </div>
+ <% @objects.each do |object| %>
+ <div class="col-sm-<%= pi_span %>" style="overflow-x: hidden; text-overflow: ellipsis;">
+ <%= link_to_if_arvados_object object, friendly_name: true %>
+ <br />
+ Template: <%= link_to_if_arvados_object object.pipeline_template_uuid, friendly_name: true %>
+ </div>
+ <% end %>
+ </div>
+</div>
+
+<% @rows.each do |row| %>
+<div class="row">
+ <div class="col-sm-2">
+ <%= row[:name] %>
+ </div>
+ <% @objects.each_with_index do |_, x| %>
+ <div class="col-sm-<%= pi_span %>">
+ <div class="row">
+ <div class="col-sm-12">
+
+ <% if row[:components][x] %>
+ <% pj = render_pipeline_job row[:components][x] %>
+
+ <%= link_to_if_arvados_object pj[:job_id], {friendly_name: true, with_class_name: true}, {class: 'deemphasize'} %>
+ <br />
+
+ <% %w(script script_version script_parameters output).each do |key| %>
+ <% unless key=='output' and pj[:result] != 'complete' %>
+ <% val = pj[key.to_sym] || pj[:job].andand[key.to_sym] %>
+ <% link_name = case
+ when !val
+ val = ''
+ when key == 'script_version' && val.match(/^[0-9a-f]{7,}$/)
+ val = val[0..7] # TODO: leave val alone, make link_to handle git commits
+ when key == 'output'
+ val.sub! /\+K.*$/, ''
+ val[0..12]
+ when key == 'script_parameters'
+ val = val.keys.sort.join(', ')
+ end
+ %>
+ <span class="deemphasize"><%= key %>:</span> <span class="<%= 'notnormal' if !pj[:is_normal][key.to_sym] %>"><%= link_to_if_arvados_object val, {friendly_name: true, link_text: link_name} %></span>
+ <% end %>
+ <br />
+ <% end %>
+ <% else %>
+ None
+ <% end %>
+ </div>
+ </div>
+ </div>
+ <% end %>
+</div>
+<div class="row" style="padding: .5em">
+</div>
+<% end %>
+
+
--- /dev/null
+<%= render partial: 'content', layout: 'content_layout', locals: {pane_list: controller.compare_pane_list } %>
+++ /dev/null
-<table class="table table-hover">
- <thead>
- <tr class="contain-align-left">
- <th>
- status
- </th><th>
- id
- </th><th>
- name
- </th><th>
- template
- </th><th>
- owner
- </th><th>
- components
- </th><th>
- dependencies
- </th><th>
- created
- </th>
- </tr>
- </thead>
- <tbody>
-
- <% @objects.sort_by { |ob| ob[:created_at] }.reverse.each do |ob| %>
-
- <tr>
- <td>
- <% if ob.success %>
- <span class="label label-success">success</span>
- <% elsif ob.active %>
- <span class="label label-info">active</span>
- <% end %>
- </td><td>
- <%= link_to_if_arvados_object ob %>
- </td><td>
- <%= ob.name %>
- </td><td>
- <%= link_to_if_arvados_object ob.pipeline_template_uuid %>
- </td><td>
- <%= link_to_if_arvados_object ob.owner_uuid %>
- </td><td>
- <% ob.components.each do |cname, c| %>
- <% status = if !(c.is_a?(Hash) && c[:job].is_a?(Hash)) then nil elsif c[:job][:success] then 'success' elsif c[:job][:running] then 'info' else 'warning' end %>
- <span class="label <%= "label-#{status}" if status %>"><%= cname.to_s %></span>
- <% end %>
- </td><td>
- <small>
- <% ob.dependencies.each do |d| %>
- <%= d %><br />
- <% end %>
- </small>
- </td><td>
- <%= ob.created_at %>
- </td>
- </tr>
-
- <% end %>
-
- </tbody>
-</table>
+++ /dev/null
-<table class="table table-condensed table-hover topalign">
- <thead>
- </thead>
- <tbody>
- <% @object.attributes_for_display.each do |attr, attrvalue| %>
- <% if attr == 'components' and attrvalue.is_a? Hash %>
-
- <tr class="info"><td><%= attr %></td><td>
- <table class="table">
- <% pipeline_jobs.each do |pj| %>
- <tr><% %w(index name result job_link script script_version progress_detail progress_bar output_link).each do |key| %>
- <td>
- <% if key == 'script_version' %>
- <%= pj[key.to_sym][0..6] rescue '' %>
- <% else %>
- <%= pj[key.to_sym] %>
- <% end %>
- </td>
- <% end %>
- </tr>
- <% end %>
- </table>
- </td></tr>
-
- <% else %>
- <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
- <% end %>
- <% end %>
- </tbody>
-</table>
-<pre>
-<%= JSON.pretty_generate(@object.attributes) rescue nil %>
-</pre>
<% end %>
</tbody>
</table>
-<pre>
-<%= JSON.pretty_generate(@object.attributes) rescue nil %>
-</pre>
-<%= render partial: 'application/index' %>
-
<% if (example = @objects.select(&:push_url).first) %>
<p>
+<% content_for :breadcrumbs do '' end %>
+
<% n_files = @required_user_agreements.collect(&:files).flatten(1).count %>
<% content_for :page_title do %>
<% if n_files == 1 %>
+<% content_for :breadcrumbs do raw '<!-- -->' end %>
<% content_for :css do %>
.dash-list {
padding: 9px 0;
<div class="container-fluid">
- <div class="span3 pull-right">
- <br/>
-
- <div class="well">
-<% if current_user.andand.is_active %>
- <p>Your account status:<br/>
- <strong>Active</strong></p>
-<% elsif current_user %>
- <p>Your account status:<br/>
- <strong>New / inactive</strong></p>
- <p>
- Your account must be activated by an Arvados administrator. If this
- is your first time accessing Arvados and would like to request
- access, or you believe you are seeing the page in error, please
- <%= link_to "contact us", Rails.configuration.activation_contact_link %>.
- You should receive an email at the address you used to log in when
- your account is activated.
- </p>
- <p>
- <%= link_to raw('Contact us ✉'),
- Rails.configuration.activation_contact_link, class: "btn btn-primary" %></p>
-<% end %>
- </div>
-
- <%= render :partial => 'notifications' %>
- </div>
-
- <div class="span8">
<%= render :partial => 'tables' %>
- </div>
</div>
<% if @my_pipelines.count == 0 || @showallalerts %>
<div class="alert alert-info daxalert">
<button type="button" class="close" data-dismiss="alert">×</button>
- <p><%= image_tag "dax.png", class: "dax" %>
- Hi, I noticed you haven't run a pipeline yet.
- <%= link_to "Click here to learn how to run an Arvados Crunch pipeline.",
- "#{Rails.configuration.arvados_docsite}/user/tutorials/tutorial-new-pipeline.html",
- style: "font-weight: bold",
- target: "_blank" %>
- </p>
+
</div>
<% end %>
<% end %>
-<div class="well">
- <p><strong>Useful links</strong></p>
- <p><ul>
- <li><%= link_to "Arvados project page", "http://arvados.org", target: "_blank" %></li>
- <li><%= link_to "Tutorials and user guide",
- "#{Rails.configuration.arvados_docsite}/user/", target: "_blank" %></li>
- </ul>
- </p>
-</div>
-
+<!--
<% if current_user.andand.is_active %>
<div class="well">
<p><strong>System status</strong></p>
</table>
</div>
<% end %>
+-->
<% if current_user.andand.is_active %>
<div>
<strong>Recent jobs</strong>
+ <%= link_to '(refresh)', request.fullpath, class: 'refresh', remote: true, method: 'get' %>
<%= link_to raw("Show all jobs →"), jobs_path, class: 'pull-right' %>
<% if not current_user.andand.is_active or @my_jobs.empty? %>
<p>(None)</p>
<% else %>
<table class="table table-bordered table-condensed table-fixedlayout">
- <colgroup>
- <col width="28%" />
- <col width="38%" />
- <col width="7%" />
- <col width="15%" />
- <col width="12%" />
- </colgroup>
+ <colgroup>
+ <col width="20%" />
+ <col width="20%" />
+ <col width="20%" />
+ <col width="13%" />
+ <col width="27%" />
+ </colgroup>
<tr>
<th>Script</th>
<span class="label label-success">finished</span>
<% elsif j.success == false %>
<span class="label label-danger">failed</span>
- <% elsif j.running and j.started_at and not j.finished_at %>
- <% percent_total_tasks = 100 / (j.tasks_summary[:running] + j.tasks_summary[:done] + j.tasks_summary[:todo]) rescue 0 %>
- <div class="progress" style="margin-bottom: 0">
- <div class="bar bar-success" style="width: <%= j.tasks_summary[:done] * percent_total_tasks rescue 0 %>%;"></div>
- <div class="bar" style="width: <%= j.tasks_summary[:running] * percent_total_tasks rescue 0 %>%; opacity: 0.3"></div>
- </div>
+ <% elsif j.finished_at %>
+ <span class="label">finished?</span>
+ <% elsif j.started_at %>
+ <span class="label label-success">running</span>
<% else %>
<span class="label">queued</span>
<% end %>
+ <% percent_total_tasks = 100 / (j.tasks_summary[:running] + j.tasks_summary[:done] + j.tasks_summary[:todo]) rescue 0 %>
+ <div class="inline-progress-container pull-right">
+ <div class="progress">
+ <span class="progress-bar progress-bar-success" style="width: <%= j.tasks_summary[:done] * percent_total_tasks rescue 0 %>%;">
+ </span>
+ <span class="progress-bar" style="width: <%= j.tasks_summary[:running] * percent_total_tasks rescue 0 %>%;">
+ </span>
+ <% if j.success == false %>
+ <span class="progress-bar progress-bar-danger" style="width: <%= tasks_summary[:failed] * percent_total_tasks rescue 0 %>%;">
+ </span>
+ <% end %>
+ </div>
+ </div>
</td>
</tr>
<div>
<strong>Recent pipeline instances</strong>
- <%= link_to raw("Show all pipeline instances →"), jobs_path, class: 'pull-right' %>
+ <%= link_to '(refresh)', request.fullpath, class: 'refresh', remote: true, method: 'get' %>
+ <%= link_to raw("Show all pipeline instances →"), pipeline_instances_path, class: 'pull-right' %>
<% if not current_user.andand.is_active or @my_pipelines.empty? %>
<p>(None)</p>
<% else %>
<table class="table table-bordered table-condensed table-fixedlayout">
<colgroup>
- <col width="73%" />
- <col width="15%" />
- <col width="12%" />
+ <col width="30%" />
+ <col width="30%" />
+ <col width="13%" />
+ <col width="27%" />
</colgroup>
<tr>
- <th>Pipeline template</th>
+ <th>Instance</th>
+ <th>Template</th>
<th>Age</th>
<th>Status</th>
</tr>
<tr>
<td>
<small>
- <% PipelineTemplate.limit(1).where(uuid: p.pipeline_template_uuid).each do |i| %>
- <%= link_to i.name, pipeline_instance_path(p.uuid) %>
- <% end %>
+ <%= link_to_if_arvados_object p.uuid, friendly_name: true %>
+ </small>
+ </td>
+
+ <td>
+ <small>
+ <%= link_to_if_arvados_object p.pipeline_template_uuid, friendly_name: true %>
</small>
</td>
<span class="label label-success">finished</span>
<% elsif p.success == false %>
<span class="label label-danger">failed</span>
+ <% elsif p.active and p.modified_at < 30.minutes.ago %>
+ <span class="label label-info">stopped</span>
<% elsif p.active %>
<span class="label label-info">running</span>
<% else %>
<span class="label">queued</span>
<% end %>
+
+ <% summary = pipeline_summary p %>
+ <div class="inline-progress-container pull-right">
+ <div class="progress">
+ <span class="progress-bar progress-bar-success" style="width: <%= summary[:percent_done] %>%;">
+ </span>
+ <% if p.success == false %>
+ <span class="progress-bar progress-bar-danger" style="width: <%= 100.0 - summary[:percent_done] %>%;">
+ </span>
+ <% else %>
+ <span class="progress-bar" style="width: <%= summary[:percent_running] %>%;">
+ </span>
+ <span class="progress-bar progress-bar-info" style="width: <%= summary[:percent_queued] %>%;">
+ </span>
+ <span class="progress-bar progress-bar-danger" style="width: <%= summary[:percent_failed] %>%;">
+ </span>
+ <% end %>
+ </div>
+ </div>
</td>
</tr>
<div>
<strong>Recent collections</strong>
+ <%= link_to '(refresh)', request.fullpath, class: 'refresh', remote: true, method: 'get' %>
<%= link_to raw("Show all collections →"), collections_path, class: 'pull-right' %>
- <div class="pull-right" style="padding-right: 1em">
+ <div class="pull-right" style="padding-right: 1em; width: 30%;">
<%= form_tag collections_path,
method: 'get',
class: 'form-search small-form-margin' do %>
- <div class="input-append">
- <%= text_field_tag :search, params[:search], class: 'search-query search-mini' %>
- <%= submit_tag "Search", name: nil, class: 'btn btn-mini btn-info' %>
+ <div class="input-group input-group-sm">
+ <%= text_field_tag :search, params[:search], class: 'form-control', placeholder: 'Search' %>
+ <span class="input-group-btn">
+ <%= button_tag(class: 'btn btn-info') do %>
+ <span class="glyphicon glyphicon-search"></span>
+ <% end %>
+ </span>
</div>
<% end %>
</div>
<% else %>
<%= image_tag "dax.png", style: "max-width=40%" %>
<% end %>
+
+<% content_for :js do %>
+setInterval(function(){$('a.refresh:eq(0)').click()}, 60000);
+<% end %>
--- /dev/null
+var new_content = "<%= escape_javascript(render partial: 'tables') %>";
+if ($('div#home-tables').html() != new_content)
+ $('div#home-tables').html(new_content);
+$('.loading').hide();
+++ /dev/null
-<table class="table">
- <thead>
- <tr class="contain-align-left">
- <th>
- id
- </th><th>
- name
- </th><th>
- email
- </th><th>
- active?
- </th><th>
- admin?
- </th><th>
- owner
- </th><th>
- default group
- </th><th>
- </th>
- </tr>
- </thead>
- <tbody>
-
- <% @objects.sort_by { |u| u[:created_at] }.each do |u| %>
-
- <tr>
- <td>
- <%= link_to_if_arvados_object u %>
- </td><td>
- <%= render_editable_attribute u, 'first_name' %>
- <%= render_editable_attribute u, 'last_name' %>
- </td><td>
- <%= render_editable_attribute u, 'email' %>
- </td><td>
- <%= render_editable_attribute u, 'is_active', u.is_active ? 'Active' : 'No', "data-type" => "select", "data-source" => '[{value:1,text:"Active"},{value:0,text:"No"}]', "data-value" => u.is_active ? "1" : "0" %>
- </td><td>
- <%= render_editable_attribute u, 'is_admin', u.is_admin ? 'Admin' : 'No', "data-type" => "select", "data-source" => '[{value:1,text:"admin"},{value:0,text:"No"}]', "data-value" => u.is_admin ? "1" : "0" %>
- </td><td>
- <%= render_editable_attribute u, 'owner_uuid' %>
- </td><td>
- <%= render_editable_attribute u, 'default_owner_uuid' %>
- </td>
-
- <td>
- <% if current_user and current_user.is_admin %>
- <%= link_to raw('<i class="icon-trash"></i>'), { action: 'destroy', id: u.uuid }, { confirm: 'Delete this user?', method: 'delete' } %>
- <% end %>
- </td>
-
- </tr>
-
- <% end %>
- <% if @objects.count == 0 %>
- <tr>
- <td colspan="7">
- (no users)
- </td>
- </tr>
- <% end %>
-
- </tbody>
-</table>
+<% content_for :breadcrumbs do raw '<!-- -->' end %>
+
<%= image_tag "dax.png", style: "float: left; max-width: 25%; margin-right: 2em" %>
<h1>Hi there! Please log in to use <%= Rails.configuration.site_name %>.</h1>
<div class="row-fluid">
-<%= render partial: 'application/index' %>
-
-<hr />
-
<p>
Sample <code>~/.ssh/config</code> section:
</p>
ArvadosWorkbench::Application.routes.draw do
themes_for_rails
+ resources :keep_disks
resources :user_agreements
post '/user_agreements/sign' => 'user_agreements#sign'
get '/user_agreements/signatures' => 'user_agreements#signatures'
resources :groups
resources :specimens
resources :pipeline_templates
- resources :pipeline_instances
+ resources :pipeline_instances do
+ get 'compare', on: :collection
+ end
resources :links
match '/collections/graph' => 'collections#graph'
resources :collections
--- /dev/null
+require 'test_helper'
+
+class KeepDisksControllerTest < ActionController::TestCase
+end
--- /dev/null
+require 'test_helper'
+
+class KeepDisksHelperTest < ActionView::TestCase
+end
--- /dev/null
+require 'test_helper'
+
+class KeepDiskTest < ActiveSupport::TestCase
+end
{% if page.navsection == 'userguide' or page.navsection == 'api' or page.navsection == 'sdk' %}
<ol class="nav nav-list">
{% for menu_item in site.navbar[page.navsection] %}
- <li>{{ menu_item }}
+ <li><span class="nav-header">{{ menu_item }}</span>
<ol class="nav nav-list">
{% for navorder in (0..99) %}
{% for p in site.pages %}
files=[input_file],
decompress=False)
-# Run the 'md5sum' command on the input file, with the current working
+# Run the external 'md5sum' program on the input file, with the current working
# directory set to the location the input file was extracted to.
stdoutdata, stderrdata = arvados.util.run_command(
['md5sum', input_file],
{
"tail_kind":"arvados#user",
"tail_uuid":"$user_uuid",
-"head_kind":"arvados#virtual_machine",
+"head_kind":"arvados#virtualMachine",
"head_uuid":"$vm_uuid",
"link_class":"permission",
"name":"can_login",
table(table table-bordered table-condensed).
|*Attribute*|*Type*|*Description*|*Example*|
-|kind|string|@arvados#{resource_type}_list@|@arvados#project_list@|
+|kind|string|@arvados#{resource_type}List@|@arvados#projectList@|
|etag|string|The ETag[1] of the resource list|@cd3o1wi9sf934saajykawrz2e@|
|self_link|string|||
|next_page_token|string|||
**Links** describe relationships between Arvados objects, and from objects to primitives.
-Links are directional: each metadata object has a tail (subject), class, name, properties, and head (object or value). A Link may describe a relationship between two objects in an Arvados database: e.g. a _permission_ link between a User and a Group defines the permissions that User has to read or modify the Group. Other Links simply represent metadata for a single object, e.g. the _identifier_ Link, in which the _name_ property represents a human-readable identifier for the object at the link's head.
+Links are directional: each metadata object has a tail (the "subject" being described), class, name, properties, and head (the "object" that describes the "subject"). A Link may describe a relationship between two objects in an Arvados database: e.g. a _permission_ link between a User and a Group defines the permissions that User has to read or modify the Group. Other Links simply represent metadata for a single object, e.g. the _identifier_ Link, in which the _name_ property represents a human-readable identifier for the object at the link's head.
For links that don't make sense to share between API clients, a _link_class_ that begins with @client@ (like @client.my_app_id@ or @client.my_app_id.anythinghere@) should be used.
# and will generate Textile documentation files in the current
# directory.
-import requests
+import argparse
+import pprint
import re
+import requests
import os
-import pprint
+import sys #debugging
+
+p = argparse.ArgumentParser(description='Generate Arvados API method documentation.')
+
+p.add_argument('--host',
+ type=str,
+ default='localhost',
+ help="The hostname or IP address of the API server")
+
+p.add_argument('--port',
+ type=int,
+ default=9900,
+ help="The port of the API server")
+
+p.add_argument('--output-dir',
+ type=str,
+ default='.',
+ help="Directory in which to write output files.")
+
+args = p.parse_args()
+
+api_url = 'https://{host}:{port}/discovery/v1/apis/arvados/v1/rest'.format(**vars(args))
-r = requests.get('https://localhost:9900/discovery/v1/apis/arvados/v1/rest',
- verify=False)
+r = requests.get(api_url, verify=False)
if r.status_code != 200:
raise Exception('Bad status code %d: %s' % (r.status_code, r.text))
raise Exception('Unexpected content type: %s: %s' %
(r.headers.get('content-type', ''), r.text))
-api = r.json
+api = r.json()
resource_num = 0
for resource in sorted(api[u'resources']):
resource_num = resource_num + 1
- out_fname = resource + '.textile'
+ out_fname = os.path.join(args.output_dir, resource + '.textile')
if os.path.exists(out_fname):
- print "PATH EXISTS ", out_fname
- next
+ backup_name = out_fname + '.old'
+ try:
+ os.rename(out_fname, backup_name)
+ except OSError as e:
+ print "WARNING: could not back up {1} as {2}: {3}".format(
+ out_fname, backup_name, e)
outf = open(out_fname, 'w')
outf.write(
"""---
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+In this tutorial, you will use the external program @md5sum@ to compute hashes instead of the built-in Python library used in earlier tutorials.
+
Start by entering the @crunch_scripts@ directory of your git repository:
<notextile>
<pre><code>$ <span class="userinput">cd you/crunch_scripts</span>
</code></pre>
</notextile>
-
-Next, using your favorite text editor, create a new file called @run-md5sum.py@ in the @crunch_scripts@ directory. Add the following code to compute the md5 hash of each file in a collection:
+
+Next, using your favorite text editor, create a new file called @run-md5sum.py@ in the @crunch_scripts@ directory. Add the following code to use the @md5sum@ program to compute the hash of each file in a collection:
<pre><code class="userinput">{% include run-md5sum.py %}</code></pre>
Next, on the Arvados virtual machine, clone your git repository:
<notextile>
-<pre><code>$ <span class="userinput">git clone git://git.{{ site.arvados_api_host }}:you.git</span>
+<pre><code>$ <span class="userinput">git clone git@git.{{ site.arvados_api_host }}:you.git</span>
Cloning into 'you'...</code></pre>
</notextile>
--- /dev/null
+#! /bin/bash
+
+build_ok=true
+
+# Check that:
+# * IP forwarding is enabled in the kernel.
+
+if [ "$(/sbin/sysctl --values net.ipv4.ip_forward)" != "1" ]
+then
+ echo >&2 "WARNING: IP forwarding must be enabled in the kernel."
+ echo >&2 "Try: sudo sysctl net.ipv4.ip_forward=1"
+ build_ok=false
+fi
+
+# * Docker can be found in the user's path
+# * The user is in the docker group
+# * cgroup is mounted
+# * the docker daemon is running
+
+if ! docker images > /dev/null 2>&1
+then
+ echo >&2 "WARNING: docker could not be run."
+ echo >&2 "Please make sure that:"
+ echo >&2 " * You have permission to read and write /var/run/docker.sock"
+ echo >&2 " * a 'cgroup' volume is mounted on your machine"
+ echo >&2 " * the docker daemon is running"
+ build_ok=false
+fi
+
+# * config.yml exists
+if [ '!' -f config.yml ]
+then
+ echo >&2 "WARNING: no config.yml found in the current directory"
+ echo >&2 "Copy config.yml.example to config.yml and update it with settings for your site."
+ build_ok=false
+fi
+
+# If ok to build, then go ahead and run make
+if $build_ok
+then
+ make $*
+fi
"arvados/warehouse"
fi
- ARVADOS_API_HOST=$(ip_address "api_server")
- ARVADOS_API_HOST_INSECURE=yes
- ARVADOS_API_TOKEN=$(cat api/generated/superuser_token)
+ if [ -d $HOME/.config/arvados ] || mkdir -p $HOME/.config/arvados
+ then
+ cat >$HOME/.config/arvados/settings.conf <<EOF
+ARVADOS_API_HOST=$(ip_address "api_server")
+ARVADOS_API_HOST_INSECURE=yes
+ARVADOS_API_TOKEN=$(cat api/generated/superuser_token)
+EOF
- echo "To run a test suite:"
- echo "export ARVADOS_API_HOST=$ARVADOS_API_HOST"
- echo "export ARVADOS_API_HOST_INSECURE=$ARVADOS_API_HOST_INSECURE"
- echo "export ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN"
- echo "python -m unittest discover ../sdk/python"
+ echo "To run a test suite:"
+ echo "python -m unittest discover ../sdk/python"
+ fi
}
function do_stop {
source 'https://rubygems.org'
+gemspec
gem 'minitest', '>= 5.0.0'
gem 'rake'
-gem 'google-api-client', '~> 0.6.3'
-gem 'activesupport', '>= 3.2.13'
-gem 'json', '>= 1.7.7'
-gem 'trollop', '>= 2.0'
-gem 'andand', '>= 1.3.3'
-gem 'oj', '>= 2.0.3'
-gem 'curb', '~> 0.8'
+PATH
+ remote: .
+ specs:
+ arvados-cli (0.1.20140205194548)
+ activesupport (~> 3.2, >= 3.2.13)
+ andand (~> 1.3, >= 1.3.3)
+ curb (~> 0.8)
+ google-api-client (~> 0.6, >= 0.6.3)
+ json (~> 1.7, >= 1.7.7)
+ oj (~> 2.0, >= 2.0.3)
+ trollop (~> 2.0)
+
GEM
remote: https://rubygems.org/
specs:
multi_json (>= 1.5)
launchy (2.4.2)
addressable (~> 2.3)
- minitest (5.2.1)
+ minitest (5.2.2)
multi_json (1.8.4)
multipart-post (1.2.0)
oj (2.5.4)
ruby
DEPENDENCIES
- activesupport (>= 3.2.13)
- andand (>= 1.3.3)
- curb (~> 0.8)
- google-api-client (~> 0.6.3)
- json (>= 1.7.7)
+ arvados-cli!
minitest (>= 5.0.0)
- oj (>= 2.0.3)
rake
- trollop (>= 2.0)
s.executables << "arv-run-pipeline-instance"
s.executables << "arv-crunch-job"
s.executables << "arv-tag"
- s.add_dependency('google-api-client', '>= 0.6.3')
- s.add_dependency('activesupport', '>= 3.2.13')
- s.add_dependency('json', '>= 1.7.7')
- s.add_dependency('trollop', '>= 2.0')
- s.add_dependency('andand', '>= 1.3.3')
- s.add_dependency('oj', '>= 2.0.3')
- s.add_dependency('curb', '~> 0.8')
+ s.add_runtime_dependency 'google-api-client', '~> 0.6', '>= 0.6.3'
+ s.add_runtime_dependency 'activesupport', '~> 3.2', '>= 3.2.13'
+ s.add_runtime_dependency 'json', '~> 1.7', '>= 1.7.7'
+ s.add_runtime_dependency 'trollop', '~> 2.0'
+ s.add_runtime_dependency 'andand', '~> 1.3', '>= 1.3.3'
+ s.add_runtime_dependency 'oj', '~> 2.0', '>= 2.0.3'
+ s.add_runtime_dependency 'curb', '~> 0.8'
s.homepage =
'http://arvados.org'
end
#
# Ward Vandewege <ward@clinicalfuture.com>
+require 'fileutils'
+
if RUBY_VERSION < '1.9.3' then
abort <<-EOS
#{$0.gsub(/^\.\//,'')} requires Ruby version 1.9.3 or higher.
# read authentication data from ~/.config/arvados if present
lineno = 0
-config_file = File.expand_path('~/.config/arvados')
+config_file = File.expand_path('~/.config/arvados/settings.conf')
if File.exist? config_file then
File.open(config_file, 'r').each do |line|
lineno = lineno + 1
class Google::APIClient
def discovery_document(api, version)
- api = api.to_s
- return @discovery_documents["#{api}:#{version}"] ||= (begin
- response = self.execute!(
- :http_method => :get,
- :uri => self.discovery_uri(api, version),
- :authenticated => false
- )
- response.body.class == String ? JSON.parse(response.body) : response.body
- end)
+ api = api.to_s
+ return @discovery_documents["#{api}:#{version}"] ||=
+ begin
+ # fetch new API discovery doc if stale
+ cached_doc = File.expand_path '~/.cache/arvados/discovery_uri.json'
+ if not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400
+ response = self.execute!(:http_method => :get,
+ :uri => self.discovery_uri(api, version),
+ :authenticated => false)
+ FileUtils.makedirs(File.dirname cached_doc)
+ File.open(cached_doc, 'w') do |f|
+ f.puts response.body
+ end
+ end
+
+ File.open(cached_doc) { |f| JSON.load f }
+ end
end
end
discovery_document["resources"][resource.pluralize]["methods"].
each do |k,v|
description = ''
- description = ' ' + v["description"] if v.include?("description")
+ if v.include? "description"
+ # add only the first line of the discovery doc description
+ description = ' ' + v["description"].split("\n").first.chomp
+ end
banner += " #{sprintf("%20s",k)}#{description}\n"
end
banner += "\n"
banner += "\n\n"
discovery_document["resources"].each do |k,v|
description = ''
- if discovery_document["schemas"].include?(k.singularize.capitalize) and
- discovery_document["schemas"][k.singularize.capitalize].include?('description') then
- description = ' ' + discovery_document["schemas"][k.singularize.capitalize]["description"]
+ resource_info = discovery_document["schemas"][k.singularize.capitalize]
+ if resource_info and resource_info.include?('description')
+ # add only the first line of the discovery doc description
+ description = ' ' + resource_info["description"].split("\n").first.chomp
end
banner += " #{sprintf("%30s",k.singularize)}#{description}\n"
end
end
global_opts = Trollop::options do
+ version __FILE__
banner "arv: the Arvados CLI tool"
opt :dry_run, "Don't actually do anything", :short => "-n"
opt :verbose, "Print some things on stderr"
required: is_required,
type: :string
}
- discovered_params[resource.to_sym] = body_object
end
end
--- /dev/null
+require 'minitest/autorun'
+require 'digest/md5'
+
+class TestCollectionCreate < Minitest::Test
+ def setup
+ end
+
+ def test_small_collection
+ uuid = Digest::MD5.hexdigest(foo_manifest) + '+' + foo_manifest.size.to_s
+ out, err = capture_subprocess_io do
+ assert_arv('--format', 'uuid', 'collection', 'create', '--collection', {
+ uuid: uuid,
+ manifest_text: foo_manifest
+ }.to_json)
+ end
+ assert_equal uuid+"\n", out
+ assert_equal '', err
+ $stderr.puts err
+ end
+
+ protected
+ def assert_arv(*args)
+ expect = case args.first
+ when true, false
+ args.shift
+ else
+ true
+ end
+ assert_equal(expect,
+ system(['./bin/arv', 'arv'], *args),
+ "`arv #{args.join ' '}` " +
+ "should exit #{if expect then 0 else 'non-zero' end}")
+ end
+
+ def foo_manifest
+ ". #{Digest::MD5.hexdigest('foo')}+3 0:3:foo\n"
+ end
+end
def test_single_tag_single_obj
# Add a single tag.
tag_uuid, err = capture_subprocess_io do
- assert arv_tag 'add', 'test_tag1', '--object', 'uuid1'
+ assert arv_tag '--short', 'add', 'test_tag1', '--object', 'uuid1'
end
assert_empty err
out, err = capture_subprocess_io do
- assert arv '-h', 'link', 'show', '--uuid', tag_uuid.rstrip
+ assert arv 'link', 'show', '--uuid', tag_uuid.rstrip
end
assert_empty err
# Remove the tag.
out, err = capture_subprocess_io do
- assert arv_tag '-h', 'remove', 'test_tag1', '--object', 'uuid1'
+ assert arv_tag 'remove', 'test_tag1', '--object', 'uuid1'
end
assert_empty err
# Verify that the link no longer exists.
out, err = capture_subprocess_io do
- assert_equal false, arv('-h', 'link', 'show', '--uuid', links[0]['uuid'])
+ assert_equal false, arv('link', 'show', '--uuid', links[0]['uuid'])
end
assert_equal "Error: Path not found\n", err
assert_empty err
out, err = capture_subprocess_io do
- assert arv '-h', 'link', 'list', '--where', '{"link_class":"tag","name":"test_tag1"}'
+ assert arv 'link', 'list', '--where', '{"link_class":"tag","name":"test_tag1"}'
end
assert_empty err
sub build
{
my $self = shift;
- $self->{'authToken'} ||= $ENV{'ARVADOS_API_TOKEN'};
- $self->{'apiHost'} ||= $ENV{'ARVADOS_API_HOST'};
- $self->{'apiProtocolScheme'} ||= $ENV{'ARVADOS_API_PROTOCOL_SCHEME'};
+
+ $config = load_config_file("$ENV{HOME}/.config/arvados/settings.conf");
+
+ $self->{'authToken'} ||=
+ $ENV{ARVADOS_API_TOKEN} || $config->{ARVADOS_API_TOKEN};
+
+ $self->{'apiHost'} ||=
+ $ENV{ARVADOS_API_HOST} || $config->{ARVADOS_API_HOST};
+
+ $self->{'apiProtocolScheme'} ||=
+ $ENV{ARVADOS_API_PROTOCOL_SCHEME} ||
+ $config->{ARVADOS_API_PROTOCOL_SCHEME};
$self->{'ua'} = new Arvados::Request;
Arvados::Request->new();
}
+sub load_config_file ($)
+{
+ my $config_file = shift;
+ my %config;
+
+ if (open (CONF, $config_file)) {
+ while (<CONF>) {
+ next if /^\s*#/ || /^\s*$/; # skip comments and blank lines
+ chomp;
+ my ($key, $val) = split /\s*=\s*/, $_, 2;
+ $config{$key} = $val;
+ }
+ }
+ close CONF;
+ return \%config;
+}
+
1;
/build/
/dist/
/*.egg-info
+/tmp
import time
import threading
-import apiclient
-import apiclient.discovery
-
-config = None
-EMPTY_BLOCK_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
-services = {}
-
-from stream import *
+from api import *
from collection import *
from keep import *
-
-
-# Arvados configuration settings are taken from $HOME/.config/arvados.
-# Environment variables override settings in the config file.
-#
-class ArvadosConfig(dict):
- def __init__(self, config_file):
- dict.__init__(self)
- if os.path.exists(config_file):
- with open(config_file, "r") as f:
- for config_line in f:
- var, val = config_line.rstrip().split('=', 2)
- self[var] = val
- for var in os.environ:
- if var.startswith('ARVADOS_'):
- self[var] = os.environ[var]
-
-class errors:
- class SyntaxError(Exception):
- pass
- class AssertionError(Exception):
- pass
- class NotFoundError(Exception):
- pass
- class CommandFailedError(Exception):
- pass
- class KeepWriteError(Exception):
- pass
- class NotImplementedError(Exception):
- pass
-
-class CredentialsFromEnv(object):
- @staticmethod
- def http_request(self, uri, **kwargs):
- global config
- from httplib import BadStatusLine
- if 'headers' not in kwargs:
- kwargs['headers'] = {}
- kwargs['headers']['Authorization'] = 'OAuth2 %s' % config.get('ARVADOS_API_TOKEN', 'ARVADOS_API_TOKEN_not_set')
- try:
- return self.orig_http_request(uri, **kwargs)
- except BadStatusLine:
- # This is how httplib tells us that it tried to reuse an
- # existing connection but it was already closed by the
- # server. In that case, yes, we would like to retry.
- # Unfortunately, we are not absolutely certain that the
- # previous call did not succeed, so this is slightly
- # risky.
- return self.orig_http_request(uri, **kwargs)
- def authorize(self, http):
- http.orig_http_request = http.request
- http.request = types.MethodType(self.http_request, http)
- return http
+from stream import *
+import errors
+import util
def task_set_output(self,s):
api('v1').job_tasks().update(uuid=self['uuid'],
def getjobparam(*args):
return current_job()['script_parameters'].get(*args)
-# Monkey patch discovery._cast() so objects and arrays get serialized
-# with json.dumps() instead of str().
-_cast_orig = apiclient.discovery._cast
-def _cast_objects_too(value, schema_type):
- global _cast_orig
- if (type(value) != type('') and
- (schema_type == 'object' or schema_type == 'array')):
- return json.dumps(value)
- else:
- return _cast_orig(value, schema_type)
-apiclient.discovery._cast = _cast_objects_too
-
-def api(version=None):
- global services, config
-
- if not config:
- config = ArvadosConfig(os.environ['HOME'] + '/.config/arvados')
- if 'ARVADOS_DEBUG' in config:
- logging.basicConfig(level=logging.DEBUG)
-
- if not services.get(version):
- apiVersion = version
- if not version:
- apiVersion = 'v1'
- logging.info("Using default API version. " +
- "Call arvados.api('%s') instead." %
- apiVersion)
- if 'ARVADOS_API_HOST' not in config:
- raise Exception("ARVADOS_API_HOST is not set. Aborting.")
- url = ('https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' %
- config['ARVADOS_API_HOST'])
- credentials = CredentialsFromEnv()
-
- # Use system's CA certificates (if we find them) instead of httplib2's
- ca_certs = '/etc/ssl/certs/ca-certificates.crt'
- if not os.path.exists(ca_certs):
- ca_certs = None # use httplib2 default
-
- http = httplib2.Http(ca_certs=ca_certs)
- http = credentials.authorize(http)
- if re.match(r'(?i)^(true|1|yes)$',
- config.get('ARVADOS_API_HOST_INSECURE', 'no')):
- http.disable_ssl_certificate_validation=True
- services[version] = apiclient.discovery.build(
- 'arvados', apiVersion, http=http, discoveryServiceUrl=url)
- return services[version]
-
class JobTask(object):
def __init__(self, parameters=dict(), runtime_constraints=dict()):
print "init jobtask %s %s" % (parameters, runtime_constraints)
).execute()
exit(0)
-class util:
- @staticmethod
- def clear_tmpdir(path=None):
- """
- Ensure the given directory (or TASK_TMPDIR if none given)
- exists and is empty.
- """
- if path == None:
- path = current_task().tmpdir
- if os.path.exists(path):
- p = subprocess.Popen(['rm', '-rf', path])
- stdout, stderr = p.communicate(None)
- if p.returncode != 0:
- raise Exception('rm -rf %s: %s' % (path, stderr))
- os.mkdir(path)
-
- @staticmethod
- def run_command(execargs, **kwargs):
- kwargs.setdefault('stdin', subprocess.PIPE)
- kwargs.setdefault('stdout', subprocess.PIPE)
- kwargs.setdefault('stderr', sys.stderr)
- kwargs.setdefault('close_fds', True)
- kwargs.setdefault('shell', False)
- p = subprocess.Popen(execargs, **kwargs)
- stdoutdata, stderrdata = p.communicate(None)
- if p.returncode != 0:
- raise errors.CommandFailedError(
- "run_command %s exit %d:\n%s" %
- (execargs, p.returncode, stderrdata))
- return stdoutdata, stderrdata
-
- @staticmethod
- def git_checkout(url, version, path):
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- if not os.path.exists(path):
- util.run_command(["git", "clone", url, path],
- cwd=os.path.dirname(path))
- util.run_command(["git", "checkout", version],
- cwd=path)
- return path
-
- @staticmethod
- def tar_extractor(path, decompress_flag):
- return subprocess.Popen(["tar",
- "-C", path,
- ("-x%sf" % decompress_flag),
- "-"],
- stdout=None,
- stdin=subprocess.PIPE, stderr=sys.stderr,
- shell=False, close_fds=True)
-
- @staticmethod
- def tarball_extract(tarball, path):
- """Retrieve a tarball from Keep and extract it to a local
- directory. Return the absolute path where the tarball was
- extracted. If the top level of the tarball contained just one
- file or directory, return the absolute path of that single
- item.
-
- tarball -- collection locator
- path -- where to extract the tarball: absolute, or relative to job tmp
- """
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
- already_have_it = False
- try:
- if os.readlink(os.path.join(path, '.locator')) == tarball:
- already_have_it = True
- except OSError:
- pass
- if not already_have_it:
-
- # emulate "rm -f" (i.e., if the file does not exist, we win)
- try:
- os.unlink(os.path.join(path, '.locator'))
- except OSError:
- if os.path.exists(os.path.join(path, '.locator')):
- os.unlink(os.path.join(path, '.locator'))
-
- for f in CollectionReader(tarball).all_files():
- if re.search('\.(tbz|tar.bz2)$', f.name()):
- p = util.tar_extractor(path, 'j')
- elif re.search('\.(tgz|tar.gz)$', f.name()):
- p = util.tar_extractor(path, 'z')
- elif re.search('\.tar$', f.name()):
- p = util.tar_extractor(path, '')
- else:
- raise errors.AssertionError(
- "tarball_extract cannot handle filename %s" % f.name())
- while True:
- buf = f.read(2**20)
- if len(buf) == 0:
- break
- p.stdin.write(buf)
- p.stdin.close()
- p.wait()
- if p.returncode != 0:
- lockfile.close()
- raise errors.CommandFailedError(
- "tar exited %d" % p.returncode)
- os.symlink(tarball, os.path.join(path, '.locator'))
- tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
- lockfile.close()
- if len(tld_extracts) == 1:
- return os.path.join(path, tld_extracts[0])
- return path
-
- @staticmethod
- def zipball_extract(zipball, path):
- """Retrieve a zip archive from Keep and extract it to a local
- directory. Return the absolute path where the archive was
- extracted. If the top level of the archive contained just one
- file or directory, return the absolute path of that single
- item.
-
- zipball -- collection locator
- path -- where to extract the archive: absolute, or relative to job tmp
- """
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
- already_have_it = False
- try:
- if os.readlink(os.path.join(path, '.locator')) == zipball:
- already_have_it = True
- except OSError:
- pass
- if not already_have_it:
-
- # emulate "rm -f" (i.e., if the file does not exist, we win)
- try:
- os.unlink(os.path.join(path, '.locator'))
- except OSError:
- if os.path.exists(os.path.join(path, '.locator')):
- os.unlink(os.path.join(path, '.locator'))
-
- for f in CollectionReader(zipball).all_files():
- if not re.search('\.zip$', f.name()):
- raise errors.NotImplementedError(
- "zipball_extract cannot handle filename %s" % f.name())
- zip_filename = os.path.join(path, os.path.basename(f.name()))
- zip_file = open(zip_filename, 'wb')
- while True:
- buf = f.read(2**20)
- if len(buf) == 0:
- break
- zip_file.write(buf)
- zip_file.close()
-
- p = subprocess.Popen(["unzip",
- "-q", "-o",
- "-d", path,
- zip_filename],
- stdout=None,
- stdin=None, stderr=sys.stderr,
- shell=False, close_fds=True)
- p.wait()
- if p.returncode != 0:
- lockfile.close()
- raise errors.CommandFailedError(
- "unzip exited %d" % p.returncode)
- os.unlink(zip_filename)
- os.symlink(zipball, os.path.join(path, '.locator'))
- tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
- lockfile.close()
- if len(tld_extracts) == 1:
- return os.path.join(path, tld_extracts[0])
- return path
-
- @staticmethod
- def collection_extract(collection, path, files=[], decompress=True):
- """Retrieve a collection from Keep and extract it to a local
- directory. Return the absolute path where the collection was
- extracted.
-
- collection -- collection locator
- path -- where to extract: absolute, or relative to job tmp
- """
- matches = re.search(r'^([0-9a-f]+)(\+[\w@]+)*$', collection)
- if matches:
- collection_hash = matches.group(1)
- else:
- collection_hash = hashlib.md5(collection).hexdigest()
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
- already_have_it = False
- try:
- if os.readlink(os.path.join(path, '.locator')) == collection_hash:
- already_have_it = True
- except OSError:
- pass
-
- # emulate "rm -f" (i.e., if the file does not exist, we win)
- try:
- os.unlink(os.path.join(path, '.locator'))
- except OSError:
- if os.path.exists(os.path.join(path, '.locator')):
- os.unlink(os.path.join(path, '.locator'))
-
- files_got = []
- for s in CollectionReader(collection).all_streams():
- stream_name = s.name()
- for f in s.all_files():
- if (files == [] or
- ((f.name() not in files_got) and
- (f.name() in files or
- (decompress and f.decompressed_name() in files)))):
- outname = f.decompressed_name() if decompress else f.name()
- files_got += [outname]
- if os.path.exists(os.path.join(path, stream_name, outname)):
- continue
- util.mkdir_dash_p(os.path.dirname(os.path.join(path, stream_name, outname)))
- outfile = open(os.path.join(path, stream_name, outname), 'wb')
- for buf in (f.readall_decompressed() if decompress
- else f.readall()):
- outfile.write(buf)
- outfile.close()
- if len(files_got) < len(files):
- raise errors.AssertionError(
- "Wanted files %s but only got %s from %s" %
- (files, files_got,
- [z.name() for z in CollectionReader(collection).all_files()]))
- os.symlink(collection_hash, os.path.join(path, '.locator'))
-
- lockfile.close()
- return path
-
- @staticmethod
- def mkdir_dash_p(path):
- if not os.path.exists(path):
- util.mkdir_dash_p(os.path.dirname(path))
- try:
- os.mkdir(path)
- except OSError:
- if not os.path.exists(path):
- os.mkdir(path)
-
- @staticmethod
- def stream_extract(stream, path, files=[], decompress=True):
- """Retrieve a stream from Keep and extract it to a local
- directory. Return the absolute path where the stream was
- extracted.
-
- stream -- StreamReader object
- path -- where to extract: absolute, or relative to job tmp
- """
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
-
- files_got = []
- for f in stream.all_files():
- if (files == [] or
- ((f.name() not in files_got) and
- (f.name() in files or
- (decompress and f.decompressed_name() in files)))):
- outname = f.decompressed_name() if decompress else f.name()
- files_got += [outname]
- if os.path.exists(os.path.join(path, outname)):
- os.unlink(os.path.join(path, outname))
- util.mkdir_dash_p(os.path.dirname(os.path.join(path, outname)))
- outfile = open(os.path.join(path, outname), 'wb')
- for buf in (f.readall_decompressed() if decompress
- else f.readall()):
- outfile.write(buf)
- outfile.close()
- if len(files_got) < len(files):
- raise errors.AssertionError(
- "Wanted files %s but only got %s from %s" %
- (files, files_got, [z.name() for z in stream.all_files()]))
- lockfile.close()
- return path
-
- @staticmethod
- def listdir_recursive(dirname, base=None):
- allfiles = []
- for ent in sorted(os.listdir(dirname)):
- ent_path = os.path.join(dirname, ent)
- ent_base = os.path.join(base, ent) if base else ent
- if os.path.isdir(ent_path):
- allfiles += util.listdir_recursive(ent_path, ent_base)
- else:
- allfiles += [ent_base]
- return allfiles
--- /dev/null
+import httplib2
+import json
+import logging
+import os
+import re
+import types
+
+import apiclient
+import apiclient.discovery
+import config
+import errors
+import util
+
+services = {}
+
+class CredentialsFromEnv(object):
+ @staticmethod
+ def http_request(self, uri, **kwargs):
+ from httplib import BadStatusLine
+ if 'headers' not in kwargs:
+ kwargs['headers'] = {}
+ kwargs['headers']['Authorization'] = 'OAuth2 %s' % config.get('ARVADOS_API_TOKEN', 'ARVADOS_API_TOKEN_not_set')
+ try:
+ return self.orig_http_request(uri, **kwargs)
+ except BadStatusLine:
+ # This is how httplib tells us that it tried to reuse an
+ # existing connection but it was already closed by the
+ # server. In that case, yes, we would like to retry.
+ # Unfortunately, we are not absolutely certain that the
+ # previous call did not succeed, so this is slightly
+ # risky.
+ return self.orig_http_request(uri, **kwargs)
+ def authorize(self, http):
+ http.orig_http_request = http.request
+ http.request = types.MethodType(self.http_request, http)
+ return http
+
+# Monkey patch discovery._cast() so objects and arrays get serialized
+# with json.dumps() instead of str().
+_cast_orig = apiclient.discovery._cast
+def _cast_objects_too(value, schema_type):
+ global _cast_orig
+ if (type(value) != type('') and
+ (schema_type == 'object' or schema_type == 'array')):
+ return json.dumps(value)
+ else:
+ return _cast_orig(value, schema_type)
+apiclient.discovery._cast = _cast_objects_too
+
+def http_cache(data_type):
+ path = os.environ['HOME'] + '/.cache/arvados/' + data_type
+ try:
+ util.mkdir_dash_p(path)
+ except OSError:
+ path = None
+ return path
+
+def api(version=None):
+ global services
+
+ if 'ARVADOS_DEBUG' in config.settings():
+ logging.basicConfig(level=logging.DEBUG)
+
+ if not services.get(version):
+ apiVersion = version
+ if not version:
+ apiVersion = 'v1'
+ logging.info("Using default API version. " +
+ "Call arvados.api('%s') instead." %
+ apiVersion)
+ if 'ARVADOS_API_HOST' not in config.settings():
+ raise Exception("ARVADOS_API_HOST is not set. Aborting.")
+ url = ('https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' %
+ config.get('ARVADOS_API_HOST'))
+ credentials = CredentialsFromEnv()
+
+ # Use system's CA certificates (if we find them) instead of httplib2's
+ ca_certs = '/etc/ssl/certs/ca-certificates.crt'
+ if not os.path.exists(ca_certs):
+ ca_certs = None # use httplib2 default
+
+ http = httplib2.Http(ca_certs=ca_certs,
+ cache=http_cache('discovery'))
+ http = credentials.authorize(http)
+ if re.match(r'(?i)^(true|1|yes)$',
+ config.get('ARVADOS_API_HOST_INSECURE', 'no')):
+ http.disable_ssl_certificate_validation=True
+ services[version] = apiclient.discovery.build(
+ 'arvados', apiVersion, http=http, discoveryServiceUrl=url)
+ return services[version]
+
import time
import threading
-from stream import *
from keep import *
+from stream import *
+import config
+import errors
class CollectionReader(object):
def __init__(self, manifest_locator_or_text):
(self._current_stream_length, len(self._current_stream_files)))
else:
if len(self._current_stream_locators) == 0:
- self._current_stream_locators += [EMPTY_BLOCK_LOCATOR]
+ self._current_stream_locators += [config.EMPTY_BLOCK_LOCATOR]
self._finished_streams += [[self._current_stream_name,
self._current_stream_locators,
self._current_stream_files]]
--- /dev/null
+# config.py - configuration settings and global variables for Arvados clients
+#
+# Arvados configuration settings are taken from $HOME/.config/arvados.
+# Environment variables override settings in the config file.
+
+import os
+import re
+
+_settings = None
+default_config_file = os.environ['HOME'] + '/.config/arvados/settings.conf'
+
+EMPTY_BLOCK_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
+
+def initialize(config_file=default_config_file):
+ global _settings
+ _settings = {}
+ if os.path.exists(config_file):
+ with open(config_file, "r") as f:
+ for config_line in f:
+ if re.match('^\s*#', config_line):
+ continue
+ var, val = config_line.rstrip().split('=', 2)
+ _settings[var] = val
+ for var in os.environ:
+ if var.startswith('ARVADOS_'):
+ _settings[var] = os.environ[var]
+
+def get(key, default_val=None):
+ return settings().get(key, default_val)
+
+def settings():
+ if _settings is None:
+ initialize()
+ return _settings
--- /dev/null
+# errors.py - Arvados-specific exceptions.
+
+class SyntaxError(Exception):
+ pass
+class AssertionError(Exception):
+ pass
+class NotFoundError(Exception):
+ pass
+class CommandFailedError(Exception):
+ pass
+class KeepWriteError(Exception):
+ pass
+class NotImplementedError(Exception):
+ pass
global_client_object = None
-from arvados import *
+from api import *
+import config
+import arvados.errors
class Keep:
@staticmethod
self.args = kwargs
def run(self):
- global config
with self.args['thread_limiter'] as limiter:
if not limiter.shall_i_proceed():
# My turn arrived, but the job has been done without
self.args['service_root']))
h = httplib2.Http()
url = self.args['service_root'] + self.args['data_hash']
- api_token = config['ARVADOS_API_TOKEN']
+ api_token = config.get('ARVADOS_API_TOKEN')
headers = {'Authorization': "OAuth2 %s" % api_token}
try:
resp, content = h.request(url.encode('utf-8'), 'PUT',
return pseq
def get(self, locator):
- global config
if re.search(r',', locator):
return ''.join(self.get(x) for x in locator.split(','))
if 'KEEP_LOCAL_STORE' in os.environ:
for service_root in self.shuffled_service_roots(expect_hash):
h = httplib2.Http()
url = service_root + expect_hash
- api_token = config['ARVADOS_API_TOKEN']
+ api_token = config.get('ARVADOS_API_TOKEN')
headers = {'Authorization': "OAuth2 %s" % api_token,
'Accept': 'application/octet-stream'}
try:
except (httplib2.HttpLib2Error, httplib.ResponseNotReady) as e:
logging.info("Request fail: GET %s => %s: %s" %
(url, type(e), str(e)))
- raise errors.NotFoundError("Block not found: %s" % expect_hash)
+ raise arvados.errors.NotFoundError("Block not found: %s" % expect_hash)
def put(self, data, **kwargs):
if 'KEEP_LOCAL_STORE' in os.environ:
have_copies = thread_limiter.done()
if have_copies == want_copies:
return (data_hash + '+' + str(len(data)))
- raise errors.KeepWriteError(
+ raise arvados.errors.KeepWriteError(
"Write fail for %s: wanted %d but wrote %d" %
(data_hash, want_copies, have_copies))
def local_store_get(locator):
r = re.search('^([0-9a-f]{32,})', locator)
if not r:
- raise errors.NotFoundError(
+ raise arvados.errors.NotFoundError(
"Invalid data locator: '%s'" % locator)
- if r.group(0) == EMPTY_BLOCK_LOCATOR.split('+')[0]:
+ if r.group(0) == config.EMPTY_BLOCK_LOCATOR.split('+')[0]:
return ''
with open(os.path.join(os.environ['KEEP_LOCAL_STORE'], r.group(0)), 'r') as f:
return f.read()
import threading
from keep import *
+import config
+import errors
class StreamFileReader(object):
def __init__(self, stream, pos, size, name):
break
yield data
+ def seek(self, pos):
+ self._filepos = pos
+
def bunzip2(self, size):
decompressor = bz2.BZ2Decompressor()
for chunk in self.readall(size):
def as_manifest(self):
if self.size() == 0:
return ("%s %s 0:0:%s\n"
- % (self._stream.name(), EMPTY_BLOCK_LOCATOR, self.name()))
+ % (self._stream.name(), config.EMPTY_BLOCK_LOCATOR, self.name()))
return string.join(self._stream.tokens_for_range(self._pos, self._size),
" ") + "\n"
--- /dev/null
+import fcntl
+import hashlib
+import os
+import re
+import subprocess
+import errno
+
+def clear_tmpdir(path=None):
+ """
+ Ensure the given directory (or TASK_TMPDIR if none given)
+ exists and is empty.
+ """
+ if path == None:
+ path = current_task().tmpdir
+ if os.path.exists(path):
+ p = subprocess.Popen(['rm', '-rf', path])
+ stdout, stderr = p.communicate(None)
+ if p.returncode != 0:
+ raise Exception('rm -rf %s: %s' % (path, stderr))
+ os.mkdir(path)
+
+def run_command(execargs, **kwargs):
+ kwargs.setdefault('stdin', subprocess.PIPE)
+ kwargs.setdefault('stdout', subprocess.PIPE)
+ kwargs.setdefault('stderr', sys.stderr)
+ kwargs.setdefault('close_fds', True)
+ kwargs.setdefault('shell', False)
+ p = subprocess.Popen(execargs, **kwargs)
+ stdoutdata, stderrdata = p.communicate(None)
+ if p.returncode != 0:
+ raise errors.CommandFailedError(
+ "run_command %s exit %d:\n%s" %
+ (execargs, p.returncode, stderrdata))
+ return stdoutdata, stderrdata
+
+def git_checkout(url, version, path):
+ if not re.search('^/', path):
+ path = os.path.join(current_job().tmpdir, path)
+ if not os.path.exists(path):
+ util.run_command(["git", "clone", url, path],
+ cwd=os.path.dirname(path))
+ util.run_command(["git", "checkout", version],
+ cwd=path)
+ return path
+
+def tar_extractor(path, decompress_flag):
+ return subprocess.Popen(["tar",
+ "-C", path,
+ ("-x%sf" % decompress_flag),
+ "-"],
+ stdout=None,
+ stdin=subprocess.PIPE, stderr=sys.stderr,
+ shell=False, close_fds=True)
+
+def tarball_extract(tarball, path):
+ """Retrieve a tarball from Keep and extract it to a local
+ directory. Return the absolute path where the tarball was
+ extracted. If the top level of the tarball contained just one
+ file or directory, return the absolute path of that single
+ item.
+
+ tarball -- collection locator
+ path -- where to extract the tarball: absolute, or relative to job tmp
+ """
+ if not re.search('^/', path):
+ path = os.path.join(current_job().tmpdir, path)
+ lockfile = open(path + '.lock', 'w')
+ fcntl.flock(lockfile, fcntl.LOCK_EX)
+ try:
+ os.stat(path)
+ except OSError:
+ os.mkdir(path)
+ already_have_it = False
+ try:
+ if os.readlink(os.path.join(path, '.locator')) == tarball:
+ already_have_it = True
+ except OSError:
+ pass
+ if not already_have_it:
+
+ # emulate "rm -f" (i.e., if the file does not exist, we win)
+ try:
+ os.unlink(os.path.join(path, '.locator'))
+ except OSError:
+ if os.path.exists(os.path.join(path, '.locator')):
+ os.unlink(os.path.join(path, '.locator'))
+
+ for f in CollectionReader(tarball).all_files():
+ if re.search('\.(tbz|tar.bz2)$', f.name()):
+ p = util.tar_extractor(path, 'j')
+ elif re.search('\.(tgz|tar.gz)$', f.name()):
+ p = util.tar_extractor(path, 'z')
+ elif re.search('\.tar$', f.name()):
+ p = util.tar_extractor(path, '')
+ else:
+ raise errors.AssertionError(
+ "tarball_extract cannot handle filename %s" % f.name())
+ while True:
+ buf = f.read(2**20)
+ if len(buf) == 0:
+ break
+ p.stdin.write(buf)
+ p.stdin.close()
+ p.wait()
+ if p.returncode != 0:
+ lockfile.close()
+ raise errors.CommandFailedError(
+ "tar exited %d" % p.returncode)
+ os.symlink(tarball, os.path.join(path, '.locator'))
+ tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+ lockfile.close()
+ if len(tld_extracts) == 1:
+ return os.path.join(path, tld_extracts[0])
+ return path
+
+def zipball_extract(zipball, path):
+ """Retrieve a zip archive from Keep and extract it to a local
+ directory. Return the absolute path where the archive was
+ extracted. If the top level of the archive contained just one
+ file or directory, return the absolute path of that single
+ item.
+
+ zipball -- collection locator
+ path -- where to extract the archive: absolute, or relative to job tmp
+ """
+ if not re.search('^/', path):
+ path = os.path.join(current_job().tmpdir, path)
+ lockfile = open(path + '.lock', 'w')
+ fcntl.flock(lockfile, fcntl.LOCK_EX)
+ try:
+ os.stat(path)
+ except OSError:
+ os.mkdir(path)
+ already_have_it = False
+ try:
+ if os.readlink(os.path.join(path, '.locator')) == zipball:
+ already_have_it = True
+ except OSError:
+ pass
+ if not already_have_it:
+
+ # emulate "rm -f" (i.e., if the file does not exist, we win)
+ try:
+ os.unlink(os.path.join(path, '.locator'))
+ except OSError:
+ if os.path.exists(os.path.join(path, '.locator')):
+ os.unlink(os.path.join(path, '.locator'))
+
+ for f in CollectionReader(zipball).all_files():
+ if not re.search('\.zip$', f.name()):
+ raise errors.NotImplementedError(
+ "zipball_extract cannot handle filename %s" % f.name())
+ zip_filename = os.path.join(path, os.path.basename(f.name()))
+ zip_file = open(zip_filename, 'wb')
+ while True:
+ buf = f.read(2**20)
+ if len(buf) == 0:
+ break
+ zip_file.write(buf)
+ zip_file.close()
+
+ p = subprocess.Popen(["unzip",
+ "-q", "-o",
+ "-d", path,
+ zip_filename],
+ stdout=None,
+ stdin=None, stderr=sys.stderr,
+ shell=False, close_fds=True)
+ p.wait()
+ if p.returncode != 0:
+ lockfile.close()
+ raise errors.CommandFailedError(
+ "unzip exited %d" % p.returncode)
+ os.unlink(zip_filename)
+ os.symlink(zipball, os.path.join(path, '.locator'))
+ tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+ lockfile.close()
+ if len(tld_extracts) == 1:
+ return os.path.join(path, tld_extracts[0])
+ return path
+
+def collection_extract(collection, path, files=[], decompress=True):
+ """Retrieve a collection from Keep and extract it to a local
+ directory. Return the absolute path where the collection was
+ extracted.
+
+ collection -- collection locator
+ path -- where to extract: absolute, or relative to job tmp
+ """
+ matches = re.search(r'^([0-9a-f]+)(\+[\w@]+)*$', collection)
+ if matches:
+ collection_hash = matches.group(1)
+ else:
+ collection_hash = hashlib.md5(collection).hexdigest()
+ if not re.search('^/', path):
+ path = os.path.join(current_job().tmpdir, path)
+ lockfile = open(path + '.lock', 'w')
+ fcntl.flock(lockfile, fcntl.LOCK_EX)
+ try:
+ os.stat(path)
+ except OSError:
+ os.mkdir(path)
+ already_have_it = False
+ try:
+ if os.readlink(os.path.join(path, '.locator')) == collection_hash:
+ already_have_it = True
+ except OSError:
+ pass
+
+ # emulate "rm -f" (i.e., if the file does not exist, we win)
+ try:
+ os.unlink(os.path.join(path, '.locator'))
+ except OSError:
+ if os.path.exists(os.path.join(path, '.locator')):
+ os.unlink(os.path.join(path, '.locator'))
+
+ files_got = []
+ for s in CollectionReader(collection).all_streams():
+ stream_name = s.name()
+ for f in s.all_files():
+ if (files == [] or
+ ((f.name() not in files_got) and
+ (f.name() in files or
+ (decompress and f.decompressed_name() in files)))):
+ outname = f.decompressed_name() if decompress else f.name()
+ files_got += [outname]
+ if os.path.exists(os.path.join(path, stream_name, outname)):
+ continue
+ mkdir_dash_p(os.path.dirname(os.path.join(path, stream_name, outname)))
+ outfile = open(os.path.join(path, stream_name, outname), 'wb')
+ for buf in (f.readall_decompressed() if decompress
+ else f.readall()):
+ outfile.write(buf)
+ outfile.close()
+ if len(files_got) < len(files):
+ raise errors.AssertionError(
+ "Wanted files %s but only got %s from %s" %
+ (files, files_got,
+ [z.name() for z in CollectionReader(collection).all_files()]))
+ os.symlink(collection_hash, os.path.join(path, '.locator'))
+
+ lockfile.close()
+ return path
+
+def mkdir_dash_p(path):
+ if not os.path.isdir(path):
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno == errno.EEXIST and os.path.isdir(path):
+ # It is not an error if someone else creates the
+ # directory between our exists() and makedirs() calls.
+ pass
+ else:
+ raise
+
+def stream_extract(stream, path, files=[], decompress=True):
+ """Retrieve a stream from Keep and extract it to a local
+ directory. Return the absolute path where the stream was
+ extracted.
+
+ stream -- StreamReader object
+ path -- where to extract: absolute, or relative to job tmp
+ """
+ if not re.search('^/', path):
+ path = os.path.join(current_job().tmpdir, path)
+ lockfile = open(path + '.lock', 'w')
+ fcntl.flock(lockfile, fcntl.LOCK_EX)
+ try:
+ os.stat(path)
+ except OSError:
+ os.mkdir(path)
+
+ files_got = []
+ for f in stream.all_files():
+ if (files == [] or
+ ((f.name() not in files_got) and
+ (f.name() in files or
+ (decompress and f.decompressed_name() in files)))):
+ outname = f.decompressed_name() if decompress else f.name()
+ files_got += [outname]
+ if os.path.exists(os.path.join(path, outname)):
+ os.unlink(os.path.join(path, outname))
+ util.mkdir_dash_p(os.path.dirname(os.path.join(path, outname)))
+ outfile = open(os.path.join(path, outname), 'wb')
+ for buf in (f.readall_decompressed() if decompress
+ else f.readall()):
+ outfile.write(buf)
+ outfile.close()
+ if len(files_got) < len(files):
+ raise errors.AssertionError(
+ "Wanted files %s but only got %s from %s" %
+ (files, files_got, [z.name() for z in stream.all_files()]))
+ lockfile.close()
+ return path
+
+def listdir_recursive(dirname, base=None):
+ allfiles = []
+ for ent in sorted(os.listdir(dirname)):
+ ent_path = os.path.join(dirname, ent)
+ ent_base = os.path.join(base, ent) if base else ent
+ if os.path.isdir(ent_path):
+ allfiles += util.listdir_recursive(ent_path, ent_base)
+ else:
+ allfiles += [ent_base]
+ return allfiles
--- /dev/null
+#!/usr/bin/env python
+
+import argparse
+import hashlib
+import os
+import re
+import string
+import sys
+import logging
+import fuse
+import errno
+import stat
+import arvados
+import time
+
+class KeepMount(fuse.LoggingMixIn, fuse.Operations):
+ 'Read-only Keep mount.'
+
+ def __init__(self):
+ self.arv = arvados.api('v1')
+ self.reader = None
+ self.collections = {}
+ self.audited = dict(read={})
+
+ def load_collection(self, uuid):
+ if uuid in self.collections:
+ return
+ now = time.time()
+ reader = arvados.CollectionReader(uuid)
+ files = {}
+ files[''] = dict(
+ stat=dict(
+ st_mode=(stat.S_IFDIR | 0755), st_ctime=now,
+ st_mtime=now, st_atime=now, st_nlink=2))
+ try:
+ for s in reader.all_streams():
+ for f in s.all_files():
+ path = re.sub(r'^\./', '', os.path.join(s.name(), f.name()))
+ files[path] = dict(
+ stat=dict(
+ st_mode=(stat.S_IFREG | 0444),
+ st_size=f.size(), st_nlink=1,
+ st_ctime=now, st_mtime=now, st_atime=now),
+ arv_file=f)
+ logger.debug("collection.load: %s: %s" % (uuid, path))
+ except:
+ # TODO: propagate real error, don't assume ENOENT
+ raise fuse.FuseOSError(errno.ENOENT)
+ self.collections[uuid] = dict(reader=reader, files=files)
+ logger.info("collection.load %s" % uuid)
+
+ def setup_reader(self, path):
+ logger.debug("%s", path.split('/'))
+ return True
+
+ def set_args(self, args):
+ self.args = args
+
+ def parse_and_load(self, path):
+ parts = path.split(os.path.sep, 2)
+ while len(parts) < 3:
+ parts += ['']
+ if not re.match(r'[0-9a-f]{32,}(\+\S+?)*', parts[1]):
+ raise fuse.FuseOSError(errno.ENOENT)
+ if self.args.collection != []:
+ if parts[1] not in self.args.collection:
+ raise fuse.FuseOSError(errno.EPERM)
+ self.load_collection(parts[1])
+ return parts[0:3]
+
+ def audit_read(self, uuid):
+ if self.args.audit and uuid not in self.audited['read']:
+ self.audited['read'][uuid] = True
+ logger.info("collection.read %s" % uuid)
+
+ def read(self, path, size, offset, fh):
+ _, uuid, target = self.parse_and_load(path)
+ if (uuid not in self.collections or
+ target not in self.collections[uuid]['files']):
+ raise fuse.FuseOSError(errno.ENOENT)
+ self.audit_read(uuid)
+ f = self.collections[uuid]['files'][target]['arv_file']
+ f.seek(offset)
+ return f.read(size)
+
+ def readdir(self, path, fh):
+ if path == '/':
+ raise fuse.FuseOSError(errno.EPERM)
+ _, uuid, target = self.parse_and_load(path)
+ if uuid not in self.collections:
+ raise fuse.FuseOSError(errno.ENOENT)
+ if target != '' and target[-1] != os.path.sep:
+ target += os.path.sep
+ dirs = {}
+ for filepath in self.collections[uuid]['files']:
+ if filepath != '':
+ logger.debug(filepath)
+ if target == '' or 0 == string.find(filepath, target):
+ dirs[filepath[len(target):].split(os.path.sep)[0]] = True
+ return ['.', '..'] + dirs.keys()
+
+ def getattr(self, path, fh=None):
+ if path == '/':
+ now = time.time()
+ return dict(st_mode=(stat.S_IFDIR | 0111), st_ctime=now,
+ st_mtime=now, st_atime=now, st_nlink=2)
+ _, uuid, target = self.parse_and_load(path)
+ if uuid not in self.collections:
+ raise fuse.FuseOSError(errno.ENOENT)
+ if target in self.collections[uuid]['files']:
+ return self.collections[uuid]['files'][target]['stat']
+ for filepath in self.collections[uuid]['files']:
+ if filepath != '':
+ if target == '' or 0 == string.find(filepath, target + '/'):
+ return self.collections[uuid]['files']['']['stat']
+ raise fuse.FuseOSError(errno.ENOENT)
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description='Mount Keep data under the local filesystem.')
+ parser.add_argument('mountpoint', type=str,
+ help="""
+Mount point.
+""")
+ parser.add_argument('--collection', type=str, action='append', default=[],
+ help="""
+Collection locator. If none supplied, provide access to all readable
+manifests.
+""")
+ parser.add_argument('--audit', action='store_true',
+ help="""
+Print the collection uuid on stderr the first time a given collection
+is read.
+""")
+ parser.add_argument('--debug', action='store_true',
+ help="""
+Print debug messages.
+""")
+ parser.add_argument('--foreground', action='store_true',
+ help="""
+Run in foreground, instead of detaching and running as a daemon.
+""")
+ args = parser.parse_args()
+ return args
+
+if __name__ == '__main__':
+ args = parse_args()
+ logger = logging.getLogger(os.path.basename(sys.argv[0]))
+ if args.audit:
+ logging.basicConfig(level=logging.INFO)
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ mounter = KeepMount()
+ mounter.set_args(args)
+ fuse = fuse.FUSE(mounter,
+ args.mountpoint,
+ foreground=args.foreground,
+ fsname='arv-mount')
httplib2==0.8
python-gflags==2.0
urllib3==1.7.1
+fusepy==2.0.2
scripts=[
'bin/arv-get',
'bin/arv-put',
+ 'bin/arv-mount',
],
install_requires=[
'python-gflags',
'google-api-python-client',
+ 'httplib2',
+ 'urllib3',
+ 'fusepy',
],
zip_safe=False)
--- /dev/null
+import unittest
+import os
+import arvados.util
+
+class MkdirDashPTest(unittest.TestCase):
+ def setUp(self):
+ try:
+ os.path.mkdir('./tmp')
+ except:
+ pass
+ def tearDown(self):
+ try:
+ os.unlink('./tmp/bar')
+ os.rmdir('./tmp/foo')
+ os.rmdir('./tmp')
+ except:
+ pass
+ def runTest(self):
+ arvados.util.mkdir_dash_p('./tmp/foo')
+ with open('./tmp/bar', 'wb') as f:
+ f.write('bar')
+ self.assertRaises(OSError, arvados.util.mkdir_dash_p, './tmp/bar')
--- /dev/null
+source 'https://rubygems.org'
+gemspec
+gem 'rake'
+gem 'minitest', '>= 5.0.0'
--- /dev/null
+PATH
+ remote: .
+ specs:
+ arvados (0.1.20140127093947)
+ activesupport (>= 3.2.13)
+ andand
+ google-api-client (~> 0.6.3)
+ json (>= 1.7.7)
+ minitest (>= 5.0.0)
+ rake
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (3.2.16)
+ i18n (~> 0.6, >= 0.6.4)
+ multi_json (~> 1.0)
+ addressable (2.3.5)
+ andand (1.3.3)
+ autoparse (0.3.3)
+ addressable (>= 2.3.1)
+ extlib (>= 0.9.15)
+ multi_json (>= 1.0.0)
+ extlib (0.9.16)
+ faraday (0.8.9)
+ multipart-post (~> 1.2.0)
+ google-api-client (0.6.4)
+ addressable (>= 2.3.2)
+ autoparse (>= 0.3.3)
+ extlib (>= 0.9.15)
+ faraday (~> 0.8.4)
+ jwt (>= 0.1.5)
+ launchy (>= 2.1.1)
+ multi_json (>= 1.0.0)
+ signet (~> 0.4.5)
+ uuidtools (>= 2.1.0)
+ i18n (0.6.9)
+ json (1.8.1)
+ jwt (0.1.11)
+ multi_json (>= 1.5)
+ launchy (2.4.2)
+ addressable (~> 2.3)
+ minitest (5.2.2)
+ multi_json (1.8.4)
+ multipart-post (1.2.0)
+ rake (10.1.1)
+ signet (0.4.5)
+ addressable (>= 2.2.3)
+ faraday (~> 0.8.1)
+ jwt (>= 0.1.5)
+ multi_json (>= 1.0.0)
+ uuidtools (2.1.4)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ arvados!
--- /dev/null
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+ t.libs << 'test'
+end
+
+desc 'Run tests'
+task default: :test
s.email = 'gem-dev@curoverse.com'
s.licenses = ['Apache License, Version 2.0']
s.files = ["lib/arvados.rb"]
- s.add_dependency('google-api-client', '>= 0.6.3')
+ s.add_dependency('google-api-client', '~> 0.6.3')
s.add_dependency('activesupport', '>= 3.2.13')
s.add_dependency('json', '>= 1.7.7')
+ s.add_dependency('andand')
s.homepage =
'http://arvados.org'
end
require 'google/api_client'
require 'active_support/inflector'
require 'json'
+require 'fileutils'
+require 'andand'
ActiveSupport::Inflector.inflections do |inflect|
inflect.irregular 'specimen', 'specimens'
class TransactionFailedError < StandardError
end
+ @@config = nil
@@debuglevel = 0
class << self
attr_accessor :debuglevel
@application_name ||= File.split($0).last
@arvados_api_version = opts[:api_version] ||
- ENV['ARVADOS_API_VERSION'] ||
+ config['ARVADOS_API_VERSION'] ||
'v1'
@arvados_api_host = opts[:api_host] ||
- ENV['ARVADOS_API_HOST'] or
+ config['ARVADOS_API_HOST'] or
raise "#{$0}: no :api_host or ENV[ARVADOS_API_HOST] provided."
@arvados_api_token = opts[:api_token] ||
- ENV['ARVADOS_API_TOKEN'] or
+ config['ARVADOS_API_TOKEN'] or
raise "#{$0}: no :api_token or ENV[ARVADOS_API_TOKEN] provided."
- if (opts[:api_host] ? opts[:suppress_ssl_warnings] :
- ENV['ARVADOS_API_HOST_INSECURE'])
+ if (opts[:suppress_ssl_warnings] or
+ config['ARVADOS_API_HOST_INSECURE'])
suppress_warnings do
OpenSSL::SSL.const_set 'VERIFY_PEER', OpenSSL::SSL::VERIFY_NONE
end
each do |method|
class << klass; self; end.class_eval do
define_method method.name do |*params|
- self.api_exec(method.name.to_sym, *params)
+ self.api_exec method, *params
end
end
end
# Give the new class access to the API
klass.instance_eval do
@arvados = _arvados
- # These should be pulled from the discovery document instead:
+ # TODO: Pull these from the discovery document instead.
@api_models_sym = classname.underscore.split('/').last.pluralize.to_sym
@api_model_sym = classname.underscore.split('/').last.to_sym
end
api = api.to_s
return @discovery_documents["#{api}:#{version}"] ||=
begin
- response = self.execute!(
- :http_method => :get,
- :uri => self.discovery_uri(api, version),
- :authenticated => false
- )
- response.body.class == String ? JSON.parse(response.body) : response.body
+ # fetch new API discovery doc if stale
+ cached_doc = File.expand_path '~/.cache/arvados/discovery_uri.json'
+ if not File.exist?(cached_doc) or (Time.now - File.mtime(cached_doc)) > 86400
+ response = self.execute!(:http_method => :get,
+ :uri => self.discovery_uri(api, version),
+ :authenticated => false)
+ FileUtils.makedirs(File.dirname cached_doc)
+ File.open(cached_doc, 'w') do |f|
+ f.puts response.body
+ end
+ end
+
+ File.open(cached_doc) { |f| JSON.load f }
end
end
end
$stderr.puts "#{File.split($0).last} #{$$}: #{message}" if @@debuglevel >= verbosity
end
+ def config(config_file_path="~/.config/arvados/settings.conf")
+ return @@config if @@config
+
+ # Initialize config settings with environment variables.
+ config = {}
+ config['ARVADOS_API_HOST'] = ENV['ARVADOS_API_HOST']
+ config['ARVADOS_API_TOKEN'] = ENV['ARVADOS_API_TOKEN']
+ config['ARVADOS_API_HOST_INSECURE'] = ENV['ARVADOS_API_HOST_INSECURE']
+ config['ARVADOS_API_VERSION'] = ENV['ARVADOS_API_VERSION']
+
+ expanded_path = File.expand_path config_file_path
+ if File.exist? expanded_path
+ # Load settings from the config file.
+ lineno = 0
+ File.open(expanded_path).each do |line|
+ lineno = lineno + 1
+ # skip comments and blank lines
+ next if line.match('^\s*#') or not line.match('\S')
+ var, val = line.chomp.split('=', 2)
+ # allow environment settings to override config files.
+ if var and val
+ config[var] ||= val
+ else
+ warn "#{expanded_path}: #{lineno}: could not parse `#{line}'"
+ end
+ end
+ end
+
+ @@config = config
+ end
+
class Model
def self.arvados_api
arvados.arvados_api
self.class.arvados.class.debuglog *args
end
def self.api_exec(method, parameters={})
+ api_method = arvados_api.send(api_models_sym).send(method.name.to_sym)
parameters = parameters.
- merge(:api_token => ENV['ARVADOS_API_TOKEN'])
+ merge(:api_token => arvados.config['ARVADOS_API_TOKEN'])
parameters.each do |k,v|
parameters[k] = v.to_json if v.is_a? Array or v.is_a? Hash
end
+ # Look for objects expected by request.properties.(key).$ref and
+ # move them from parameters (query string) to request body.
+ body = nil
+ method.discovery_document['request'].
+ andand['properties'].
+ andand.each do |k,v|
+ if v.is_a? Hash and v['$ref']
+ body ||= {}
+ body[k] = parameters.delete k.to_sym
+ end
+ end
result = client.
- execute(:api_method => arvados_api.send(api_models_sym).send(method),
+ execute(:api_method => api_method,
:authenticated => false,
- :parameters => parameters)
+ :parameters => parameters,
+ :body => body)
resp = JSON.parse result.body, :symbolize_names => true
if resp[:errors]
raise Arvados::TransactionFailedError.new(resp[:errors])
--- /dev/null
+require 'minitest/autorun'
+require 'arvados'
+require 'digest/md5'
+
+class TestBigRequest < Minitest::Test
+ def setup
+ begin
+ Dir.mkdir './tmp'
+ rescue Errno::EEXIST
+ end
+ @@arv = Arvados.new
+ end
+
+ def boring_manifest nblocks
+ x = '.'
+ (0..nblocks).each do |z|
+ x += ' d41d8cd98f00b204e9800998ecf8427e+0'
+ end
+ x += "0:0:foo.txt\n"
+ x
+ end
+
+ def test_create_manifest nblocks=1
+ manifest_text = boring_manifest nblocks
+ uuid = Digest::MD5.hexdigest(manifest_text) + '+' + manifest_text.size.to_s
+ c = @@arv.collection.create(collection: {
+ uuid: uuid,
+ manifest_text: manifest_text
+ })
+ assert_equal uuid, c[:uuid]
+ end
+
+ def test_create_big_manifest
+ # This ensures that manifest_text is passed in the request body:
+ # it's too large to fit in the query string.
+ test_create_manifest 9999
+ end
+end
class ApplicationController < ActionController::Base
include CurrentApiClient
+ respond_to :json
protect_from_forgery
around_filter :thread_with_auth_info, :except => [:render_error, :render_not_found]
before_filter :load_where_param, :only => :index
before_filter :find_objects_for_index, :only => :index
- before_filter :find_object_by_uuid, :except => [:index, :create]
+ before_filter :find_object_by_uuid, :except => [:index, :create,
+ :render_error,
+ :render_not_found]
before_filter :reload_object_before_update, :only => :update
+ before_filter :render_404_if_no_object, except: [:index, :create,
+ :render_error,
+ :render_not_found]
attr_accessor :resource_attrs
end
def show
- if @object
- render json: @object.as_api_response
- else
- render_not_found("object not found")
- end
+ render json: @object.as_api_response
end
def create
end
def update
- if !@object
- return render_not_found("object not found")
- end
attrs_to_update = resource_attrs.reject { |k,v|
[:kind, :etag, :href].index k
}
:with => :render_error
end
+ def render_404_if_no_object
+ render_not_found "Object not found" if !@object
+ end
+
def render_error(e)
logger.error e.inspect
logger.error e.backtrace.collect { |x| x + "\n" }.join('') if e.backtrace
end
def find_objects_for_index
- uuid_list = [current_user.uuid, *current_user.groups_i_can(:read)]
- sanitized_uuid_list = uuid_list.
- collect { |uuid| model_class.sanitize(uuid) }.join(', ')
- or_references_me = ''
- if model_class == Link and current_user
- or_references_me = "OR (#{table_name}.link_class in (#{model_class.sanitize 'permission'}, #{model_class.sanitize 'resources'}) AND #{model_class.sanitize current_user.uuid} IN (#{table_name}.head_uuid, #{table_name}.tail_uuid))"
- end
- @objects ||= model_class.
- joins("LEFT JOIN links permissions ON permissions.head_uuid in (#{table_name}.owner_uuid, #{table_name}.uuid) AND permissions.tail_uuid in (#{sanitized_uuid_list}) AND permissions.link_class='permission'").
- where("?=? OR #{table_name}.owner_uuid in (?) OR #{table_name}.uuid=? OR permissions.head_uuid IS NOT NULL #{or_references_me}",
- true, current_user.is_admin,
- uuid_list,
- current_user.uuid)
+ @objects ||= model_class.readable_by(current_user)
if !@where.empty?
conditions = ['1=1']
@where.each do |attr,value|
if attr == :any
if value.is_a?(Array) and
+ value.length == 2 and
value[0] == 'contains' and
model_class.columns.collect(&:name).index('name') then
ilikes = []
conditions[0] << " and #{table_name}.#{attr} is ?"
conditions << nil
elsif value.is_a? Array
- conditions[0] << " and #{table_name}.#{attr} in (?)"
- conditions << value
+ if value[0] == 'contains' and value.length == 2
+ conditions[0] << " and #{table_name}.#{attr} like ?"
+ conditions << "%#{value[1]}%"
+ else
+ conditions[0] << " and #{table_name}.#{attr} in (?)"
+ conditions << value
+ end
elsif value.is_a? String or value.is_a? Fixnum or value == true or value == false
conditions[0] << " and #{table_name}.#{attr}=?"
conditions << value
%w(created_at modified_by_client_uuid modified_by_user_uuid modified_at).each do |x|
@attrs.delete x.to_sym
end
+ @attrs = @attrs.symbolize_keys if @attrs.is_a? HashWithIndifferentAccess
@attrs
end
def self.accept_attribute_as_json(attr, force_class=nil)
before_filter lambda { accept_attribute_as_json attr, force_class }
end
+ accept_attribute_as_json :properties, Hash
+ accept_attribute_as_json :info, Hash
def accept_attribute_as_json(attr, force_class)
- if params[resource_name].is_a? Hash
- if params[resource_name][attr].is_a? String
- params[resource_name][attr] = Oj.load(params[resource_name][attr],
- symbol_keys: true)
- if force_class and !params[resource_name][attr].is_a? force_class
+ if params[resource_name] and resource_attrs.is_a? Hash
+ if resource_attrs[attr].is_a? String
+ resource_attrs[attr] = Oj.load(resource_attrs[attr],
+ symbol_keys: false)
+ if force_class and !resource_attrs[attr].is_a? force_class
raise TypeError.new("#{resource_name}[#{attr.to_s}] must be a #{force_class.to_s}")
end
+ elsif resource_attrs[attr].is_a? Hash
+ # Convert symbol keys to strings (in hashes provided by
+ # resource_attrs)
+ resource_attrs[attr] = resource_attrs[attr].
+ with_indifferent_access.to_hash
end
end
end
'arvados#group'
end
unless current_user.can? write: owner_uuid
+ logger.warn "User #{current_user.andand.uuid} tried to set collection owner_uuid to #{owner_uuid}"
raise ArvadosModel::PermissionDeniedError
end
act_as_system_user do
end
show
end
+
+ def collection_uuid(uuid)
+ m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid)
+ if m
+ m[1]
+ else
+ nil
+ end
+ end
+
+ def script_param_edges(visited, sp)
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ script_param_edges(visited, v)
+ end
+ when Array
+ sp.each do |v|
+ script_param_edges(visited, v)
+ end
+ else
+ m = collection_uuid(sp)
+ if m
+ generate_provenance_edges(visited, m)
+ end
+ end
+ end
+ end
+
+ def generate_provenance_edges(visited, uuid)
+ m = collection_uuid(uuid)
+ uuid = m if m
+
+ if not uuid or uuid.empty? or visited[uuid]
+ return ""
+ end
+
+ logger.debug "visiting #{uuid}"
+
+ if m
+ # uuid is a collection
+ Collection.readable_by(current_user).where(uuid: uuid).each do |c|
+ visited[uuid] = c.as_api_response
+ visited[uuid][:files] = []
+ c.files.each do |f|
+ visited[uuid][:files] << f
+ end
+ end
+
+ Job.readable_by(current_user).where(output: uuid).each do |job|
+ generate_provenance_edges(visited, job.uuid)
+ end
+
+ Job.readable_by(current_user).where(log: uuid).each do |job|
+ generate_provenance_edges(visited, job.uuid)
+ end
+
+ else
+ # uuid is something else
+ rsc = ArvadosModel::resource_class_for_uuid uuid
+ if rsc == Job
+ Job.readable_by(current_user).where(uuid: uuid).each do |job|
+ visited[uuid] = job.as_api_response
+ script_param_edges(visited, job.script_parameters)
+ end
+ elsif rsc != nil
+ rsc.where(uuid: uuid).each do |r|
+ visited[uuid] = r.as_api_response
+ end
+ end
+ end
+
+ Link.readable_by(current_user).
+ where(head_uuid: uuid, link_class: "provenance").
+ each do |link|
+ visited[link.uuid] = link.as_api_response
+ generate_provenance_edges(visited, link.tail_uuid)
+ end
+
+ #puts "finished #{uuid}"
+ end
+
+ def provenance
+ visited = {}
+ generate_provenance_edges(visited, @object[:uuid])
+ render json: visited
+ end
+
+ def generate_used_by_edges(visited, uuid)
+ m = collection_uuid(uuid)
+ uuid = m if m
+
+ if not uuid or uuid.empty? or visited[uuid]
+ return ""
+ end
+
+ logger.debug "visiting #{uuid}"
+
+ if m
+ # uuid is a collection
+ Collection.readable_by(current_user).where(uuid: uuid).each do |c|
+ visited[uuid] = c.as_api_response
+ visited[uuid][:files] = []
+ c.files.each do |f|
+ visited[uuid][:files] << f
+ end
+ end
+
+ if uuid == "d41d8cd98f00b204e9800998ecf8427e+0"
+ # special case for empty collection
+ return
+ end
+
+ Job.readable_by(current_user).where(["jobs.script_parameters like ?", "%#{uuid}%"]).each do |job|
+ generate_used_by_edges(visited, job.uuid)
+ end
+
+ else
+ # uuid is something else
+ rsc = ArvadosModel::resource_class_for_uuid uuid
+ if rsc == Job
+ Job.readable_by(current_user).where(uuid: uuid).each do |job|
+ visited[uuid] = job.as_api_response
+ generate_used_by_edges(visited, job.output)
+ end
+ elsif rsc != nil
+ rsc.where(uuid: uuid).each do |r|
+ visited[uuid] = r.as_api_response
+ end
+ end
+ end
+
+ Link.readable_by(current_user).
+ where(tail_uuid: uuid, link_class: "provenance").
+ each do |link|
+ visited[link.uuid] = link.as_api_response
+ generate_used_by_edges(visited, link.head_uuid)
+ end
+
+ #puts "finished #{uuid}"
+ end
+
+ def used_by
+ visited = {}
+ generate_used_by_edges(visited, @object[:uuid])
+ render json: visited
+ end
+
+ protected
+ def find_object_by_uuid
+ super
+ if !@object and !params[:uuid].match(/^[0-9a-f]+\+\d+$/)
+ # Normalize the given uuid and search again.
+ hash_part = params[:uuid].match(/^([0-9a-f]*)/)[1]
+ collection = Collection.where('uuid like ?', hash_part + '+%').first
+ if collection
+ # We know the collection exists, and what its real uuid is in
+ # the database. Now, throw out @objects and repeat the usual
+ # lookup procedure. (Returning the collection at this point
+ # would bypass permission checks.)
+ @objects = nil
+ @where = { uuid: collection.uuid }
+ find_objects_for_index
+ @object = @objects.first
+ end
+ end
+ end
+
end
accept_attribute_as_json :runtime_constraints, Hash
accept_attribute_as_json :tasks_summary, Hash
skip_before_filter :find_object_by_uuid, :only => :queue
+ skip_before_filter :render_404_if_no_object, :only => :queue
def index
want_ancestor = @where[:script_version_descends_from]
}
end
def ping
- if !@object
- if current_user.andand.is_admin
- @object = KeepDisk.new(filesystem_uuid: params[:filesystem_uuid])
- @object.save!
-
- # In the first ping from this new filesystem_uuid, we can't
- # expect the keep node to know the ping_secret so we made sure
- # we got an admin token. Here we add ping_secret to params so
- # KeepNode.ping() understands this update is properly
- # authenticated.
- params[:ping_secret] = @object.ping_secret
- else
- return render_not_found "object not found"
- end
- end
-
params[:service_host] ||= request.env['REMOTE_ADDR']
if not @object.ping params
return render_not_found "object not found"
end
+ # Render the :superuser view (i.e., include the ping_secret) even
+ # if !current_user.is_admin. This is safe because @object.ping's
+ # success implies the ping_secret was already known by the client.
render json: @object.as_api_response(:superuser)
end
@objects = model_class.where('1=1')
super
end
+
+ def find_object_by_uuid
+ @object = KeepDisk.where(uuid: (params[:id] || params[:uuid])).first
+ if !@object && current_user.andand.is_admin
+ # Create a new KeepDisk and ping it.
+ @object = KeepDisk.new(filesystem_uuid: params[:filesystem_uuid])
+ @object.save!
+
+ # In the first ping from this new filesystem_uuid, we can't
+ # expect the keep node to know the ping_secret so we made sure
+ # we got an admin token. Here we add ping_secret to params so
+ # KeepNode.ping() understands this update is properly
+ # authenticated.
+ params[:ping_secret] = @object.ping_secret
+ end
+ end
end
class Arvados::V1::NodesController < ApplicationController
skip_before_filter :require_auth_scope_all, :only => :ping
skip_before_filter :find_object_by_uuid, :only => :ping
+ skip_before_filter :render_404_if_no_object, :only => :ping
def create
@object = Node.new
class Arvados::V1::RepositoriesController < ApplicationController
+ skip_before_filter :find_object_by_uuid, :only => :get_all_permissions
+ skip_before_filter :render_404_if_no_object, :only => :get_all_permissions
before_filter :admin_required, :only => :get_all_permissions
def get_all_permissions
@users = {}
class Arvados::V1::SchemaController < ApplicationController
skip_before_filter :find_object_by_uuid
+ skip_before_filter :render_404_if_no_object
skip_before_filter :require_auth_scope_all
def show
end
def discovery_rest_description
+ expires_in 24.hours, public: true
discovery = Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
discovery = {
id: k.to_s,
description: k.to_s,
type: "object",
+ uuidPrefix: (k.respond_to?(:uuid_prefix) ? k.uuid_prefix : nil),
properties: {
uuid: {
type: "string",
id: "arvados.#{k.to_s.underscore.pluralize}.list",
path: k.to_s.underscore.pluralize,
httpMethod: "GET",
- description: "List #{k.to_s.underscore.pluralize}.",
+ description:
+ %|List #{k.to_s.pluralize}.
+
+ The <code>list</code> method returns a
+ <a href="/api/resources.html">resource list</a> of
+ matching #{k.to_s.pluralize}. For example:
+
+ <pre>
+ {
+ "kind":"arvados##{k.to_s.camelcase(:lower)}List",
+ "etag":"",
+ "self_link":"",
+ "next_page_token":"",
+ "next_link":"",
+ "items":[
+ ...
+ ],
+ "items_available":745,
+ "_profile":{
+ "request_time":0.157236317
+ }
+ </pre>|,
parameters: {
limit: {
type: "integer",
default: 100,
format: "int32",
minimum: 0,
- location: "query"
- },
- pageToken: {
- type: "string",
- description: "Page token.",
- location: "query"
- },
- q: {
- type: "string",
- description: "Query string for searching #{k.to_s.underscore.pluralize}.",
- location: "query"
+ location: "query",
},
where: {
type: "object",
path: "#{k.to_s.underscore.pluralize}",
httpMethod: "POST",
description: "Create a new #{k.to_s}.",
- parameters: {
- k.to_s.underscore => {
- type: "object",
- required: false,
- location: "query",
- properties: object_properties
- }
- },
+ parameters: {},
request: {
- required: false,
+ required: true,
properties: {
k.to_s.underscore => {
"$ref" => k.to_s
description: "The UUID of the #{k.to_s} in question.",
required: true,
location: "path"
- },
- k.to_s.underscore => {
- type: "object",
- required: false,
- location: "query",
- properties: object_properties
}
},
request: {
- required: false,
+ required: true,
properties: {
k.to_s.underscore => {
"$ref" => k.to_s
class Arvados::V1::UserAgreementsController < ApplicationController
before_filter :admin_required, except: [:index, :sign, :signatures]
- skip_before_filter :find_object, only: [:sign, :signatures]
+ skip_before_filter :find_object_by_uuid, only: [:sign, :signatures]
+ skip_before_filter :render_404_if_no_object, only: [:sign, :signatures]
def model_class
Link
class Arvados::V1::UsersController < ApplicationController
+ skip_before_filter :find_object_by_uuid, only:
+ [:activate, :event_stream, :current, :system]
+ skip_before_filter :render_404_if_no_object, only:
+ [:activate, :event_stream, :current, :system]
+
def current
@object = current_user
show
else
logger.warn "User #{@object.uuid} called users.activate " +
"before signing agreements #{todo_uuids.inspect}"
- raise ArgumentError.new \
+ raise ArvadosModel::PermissionDeniedError.new \
"Cannot activate without user agreements #{todo_uuids.inspect}."
end
end
class Arvados::V1::VirtualMachinesController < ApplicationController
skip_before_filter :find_object_by_uuid, :only => :get_all_logins
+ skip_before_filter :render_404_if_no_object, :only => :get_all_logins
skip_before_filter(:require_auth_scope_all,
:only => [:logins, :get_all_logins])
before_filter(:admin_required,
class StaticController < ApplicationController
+ respond_to :json, :html
skip_before_filter :find_object_by_uuid
+ skip_before_filter :render_404_if_no_object
skip_before_filter :require_auth_scope_all, :only => [ :home, :login_failure ]
def home
before_filter :require_auth_scope_all, :only => [ :destroy ]
skip_before_filter :find_object_by_uuid
+ skip_before_filter :render_404_if_no_object
respond_to :html
+require 'assign_uuid'
class ArvadosModel < ActiveRecord::Base
self.abstract_class = true
end
end
+ def self.readable_by user
+ uuid_list = [user.uuid, *user.groups_i_can(:read)]
+ sanitized_uuid_list = uuid_list.
+ collect { |uuid| sanitize(uuid) }.join(', ')
+ or_references_me = ''
+ if self == Link and user
+ or_references_me = "OR (#{table_name}.link_class in (#{sanitize 'permission'}, #{sanitize 'resources'}) AND #{sanitize user.uuid} IN (#{table_name}.head_uuid, #{table_name}.tail_uuid))"
+ end
+ joins("LEFT JOIN links permissions ON permissions.head_uuid in (#{table_name}.owner_uuid, #{table_name}.uuid) AND permissions.tail_uuid in (#{sanitized_uuid_list}) AND permissions.link_class='permission'").
+ where("?=? OR #{table_name}.owner_uuid in (?) OR #{table_name}.uuid=? OR permissions.head_uuid IS NOT NULL #{or_references_me}",
+ true, user.is_admin,
+ uuid_list,
+ user.uuid)
+ end
+
protected
def ensure_permission_to_create
end
end
end
+
+ def self.resource_class_for_uuid(uuid)
+ if uuid.is_a? ArvadosModel
+ return uuid.class
+ end
+ unless uuid.is_a? String
+ return nil
+ end
+ if uuid.match /^[0-9a-f]{32}(\+[^,]+)*(,[0-9a-f]{32}(\+[^,]+)*)*$/
+ return Collection
+ end
+ resource_class = nil
+
+ Rails.application.eager_load!
+ uuid.match /^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/ do |re|
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
+ if k.respond_to?(:uuid_prefix)
+ if k.uuid_prefix == re[1]
+ return k
+ end
+ end
+ end
+ end
+ nil
+ end
+
end
if self.manifest_text.nil? and self.uuid.nil?
super
elsif self.manifest_text and self.uuid
- if self.uuid.gsub(/\+[^,]+/,'') == Digest::MD5.hexdigest(self.manifest_text)
+ self.uuid.gsub! /\+.*/, ''
+ if self.uuid == Digest::MD5.hexdigest(self.manifest_text)
+ self.uuid.gsub! /$/, '+' + self.manifest_text.length.to_s
true
else
errors.add :uuid, 'uuid does not match checksum of manifest_text'
# instead of a commit-ish.
return true
end
- sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
- if sha1
- self.script_version = sha1
- else
- raise ArgumentError.new("Specified script_version does not resolve to a commit")
+ if new_record? or script_version_changed?
+ sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
+ if sha1
+ self.script_version = sha1
+ else
+ raise ArgumentError.new("Specified script_version does not resolve to a commit")
+ end
end
end
script_parameters_changed? or
script_version_changed? or
(!cancelled_at_was.nil? and
- (cancelled_by_client_changed? or
- cancelled_by_user_changed? or
+ (cancelled_by_client_uuid_changed? or
+ cancelled_by_user_uuid_changed? or
cancelled_at_changed?)) or
started_at_changed? or
finished_at_changed? or
if o[:ec2_instance_id]
if !self.info[:ec2_instance_id]
self.info[:ec2_instance_id] = o[:ec2_instance_id]
- tag_cmd = ("ec2-create-tags #{o[:ec2_instance_id]} " +
- "--tag 'Name=#{self.uuid}'")
- `#{tag_cmd}`
+ if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
+ tag_cmd = ("ec2-create-tags #{o[:ec2_instance_id]} " +
+ "--tag 'Name=#{self.uuid}'")
+ `#{tag_cmd}`
+ end
elsif self.info[:ec2_instance_id] != o[:ec2_instance_id]
logger.debug "Multiple nodes have credentials for #{self.uuid}"
raise "#{self.uuid} is already running at #{self.info[:ec2_instance_id]} so rejecting ping from #{o[:ec2_instance_id]}"
end while true
self.hostname = self.class.hostname_for_slot(self.slot_number)
if info[:ec2_instance_id]
- `ec2-create-tags #{self.info[:ec2_instance_id]} --tag 'hostname=#{self.hostname}'`
+ if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
+ `ec2-create-tags #{self.info[:ec2_instance_id]} --tag 'hostname=#{self.hostname}'`
+ end
end
end
result.match(/INSTANCE\s*(i-[0-9a-f]+)/) do |m|
instance_id = m[1]
self.info[:ec2_instance_id] = instance_id
- `ec2-create-tags #{instance_id} --tag 'Name=#{self.uuid}'`
+ if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
+ `ec2-create-tags #{instance_id} --tag 'Name=#{self.uuid}'`
+ end
end
result.match(/SPOTINSTANCEREQUEST\s*(sir-[0-9a-f]+)/) do |m|
sir_id = m[1]
self.info[:ec2_sir_id] = sir_id
- `ec2-create-tags #{sir_id} --tag 'Name=#{self.uuid}'`
+ if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
+ `ec2-create-tags #{sir_id} --tag 'Name=#{self.uuid}'`
+ end
end
self.save!
end
end
def can?(actions)
+ return true if is_admin
actions.each do |action, target|
target_uuid = target
if target.respond_to? :uuid
lookup_uuids = todo.keys
lookup_uuids.each do |uuid| done[uuid] = true end
todo = {}
+ newgroups = []
+ Group.where('owner_uuid in (?)', lookup_uuids).each do |group|
+ newgroups << [group.owner_uuid, group.uuid, 'can_manage']
+ end
Link.where('tail_uuid in (?) and link_class = ? and head_kind = ?',
lookup_uuids,
'permission',
'arvados#group').each do |link|
- unless done.has_key? link.head_uuid
- todo[link.head_uuid] = true
+ newgroups << [link.tail_uuid, link.head_uuid, link.name]
+ end
+ newgroups.each do |tail_uuid, head_uuid, perm_name|
+ unless done.has_key? head_uuid
+ todo[head_uuid] = true
end
link_permissions = {}
- case link.name
+ case perm_name
when 'can_read'
link_permissions = {read:true}
when 'can_write'
when 'can_manage'
link_permissions = ALL_PERMISSIONS
end
- permissions_from[link.tail_uuid] ||= {}
- permissions_from[link.tail_uuid][link.head_uuid] ||= {}
+ permissions_from[tail_uuid] ||= {}
+ permissions_from[tail_uuid][head_uuid] ||= {}
link_permissions.each do |k,v|
- permissions_from[link.tail_uuid][link.head_uuid][k] ||= v
+ permissions_from[tail_uuid][head_uuid][k] ||= v
end
end
end
# config.compute_node_ami = 'ami-cbca41a2'
# config.compute_node_ec2run_args = '-g arvados-compute'
# config.compute_node_spot_bid = 0.11
+ config.compute_node_ec2_tag_enable = false
# config.compute_node_domain = `hostname --domain`.strip
# config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
config.compute_node_nameservers = [ "172.16.0.23" ]
- config.uuid_prefix('test@' + `hostname`.strip)
+ config.uuid_prefix = 'zzzzz'
# Authentication stub: hard code pre-approved API tokens.
# config.accept_api_token = { rand(2**256).to_s(36) => true }
match '/repositories/get_all_permissions' => 'repositories#get_all_permissions'
get '/user_agreements/signatures' => 'user_agreements#signatures'
post '/user_agreements/sign' => 'user_agreements#sign'
+ get '/collections/:uuid/provenance' => 'collections#provenance'
+ get '/collections/:uuid/used_by' => 'collections#used_by'
resources :collections
resources :links
resources :nodes
--- /dev/null
+class NormalizeCollectionUuid < ActiveRecord::Migration
+ def count_orphans
+ %w(head tail).each do |ht|
+ results = ActiveRecord::Base.connection.execute(<<-EOS)
+SELECT COUNT(links.*)
+ FROM links
+ LEFT JOIN collections c
+ ON links.#{ht}_uuid = c.uuid
+ WHERE (#{ht}_kind='arvados#collection' or #{ht}_uuid ~ '^[0-9a-f]{32,}')
+ AND #{ht}_uuid IS NOT NULL
+ AND #{ht}_uuid NOT IN (SELECT uuid FROM collections)
+EOS
+ puts "#{results.first['count'].to_i} links with #{ht}_uuid pointing nowhere."
+ end
+ end
+
+ def up
+ # Normalize uuids in the collections table to
+ # {hash}+{size}. Existing uuids might be {hash},
+ # {hash}+{size}+K@{instance-name}, {hash}+K@{instance-name}, etc.
+
+ count_orphans
+ puts "Normalizing collection UUIDs."
+
+ update_sql <<-EOS
+UPDATE collections
+ SET uuid = regexp_replace(uuid,'\\+.*','') || '+' || length(manifest_text)
+ WHERE uuid !~ '^[0-9a-f]{32,}\\+[0-9]+$'
+ AND (regexp_replace(uuid,'\\+.*','') || '+' || length(manifest_text))
+ NOT IN (SELECT uuid FROM collections)
+EOS
+
+ count_orphans
+ puts "Updating links by stripping +K@.* from *_uuid attributes."
+
+ update_sql <<-EOS
+UPDATE links
+ SET head_uuid = regexp_replace(head_uuid,'\\+K@.*','')
+ WHERE head_uuid like '%+K@%'
+EOS
+ update_sql <<-EOS
+UPDATE links
+ SET tail_uuid = regexp_replace(tail_uuid,'\\+K@.*','')
+ WHERE tail_uuid like '%+K@%'
+EOS
+
+ count_orphans
+ puts "Updating links by searching bare collection hashes using regexp."
+
+ # Next, update {hash} (and any other non-normalized forms) to
+ # {hash}+{size}. This can only work where the corresponding
+ # collection is found in the collections table (otherwise we can't
+ # know the size).
+ %w(head tail).each do |ht|
+ update_sql <<-EOS
+UPDATE links
+ SET #{ht}_uuid = c.uuid
+ FROM collections c
+ WHERE #{ht}_uuid IS NOT NULL
+ AND (#{ht}_kind='arvados#collection' or #{ht}_uuid ~ '^[0-9a-f]{32,}')
+ AND #{ht}_uuid NOT IN (SELECT uuid FROM collections)
+ AND regexp_replace(#{ht}_uuid,'\\+.*','') = regexp_replace(c.uuid,'\\+.*','')
+ AND c.uuid ~ '^[0-9a-f]{32,}\\+[0-9]+$'
+EOS
+ end
+
+ count_orphans
+ puts "Stripping \"+K@.*\" from jobs.output, jobs.log, job_tasks.output."
+
+ update_sql <<-EOS
+UPDATE jobs
+ SET output = regexp_replace(output,'\\+K@.*','')
+ WHERE output ~ '^[0-9a-f]{32,}\\+[0-9]+\\+K@\\w+$'
+EOS
+ update_sql <<-EOS
+UPDATE jobs
+ SET log = regexp_replace(log,'\\+K@.*','')
+ WHERE log ~ '^[0-9a-f]{32,}\\+[0-9]+\\+K@\\w+$'
+EOS
+ update_sql <<-EOS
+UPDATE job_tasks
+ SET output = regexp_replace(output,'\\+K@.*','')
+ WHERE output ~ '^[0-9a-f]{32,}\\+[0-9]+\\+K@\\w+$'
+EOS
+
+ puts "Done."
+ end
+
+ def down
+ end
+end
--- /dev/null
+class FixLinkKindUnderscores < ActiveRecord::Migration
+ def up
+ update_sql <<-EOS
+UPDATE links
+ SET head_kind = 'arvados#virtualMachine'
+ WHERE head_kind = 'arvados#virtual_machine'
+EOS
+ end
+
+ def down
+ update_sql <<-EOS
+UPDATE links
+ SET head_kind = 'arvados#virtual_machine'
+ WHERE head_kind = 'arvados#virtualMachine'
+EOS
+ end
+end
--- /dev/null
+class NormalizeCollectionUuidsInScriptParameters < ActiveRecord::Migration
+ include CurrentApiClient
+ def up
+ act_as_system_user do
+ PipelineInstance.all.each do |pi|
+ pi.save! if fix_values_recursively(pi.components)
+ end
+ Job.all.each do |j|
+ changed = false
+ j.script_parameters.each do |p, v|
+ if v.is_a? String and v.match /\+K/
+ v.gsub! /\+K\@\w+/, ''
+ changed = true
+ end
+ end
+ j.save! if changed
+ end
+ end
+ end
+
+ def down
+ end
+
+ protected
+ def fix_values_recursively fixme
+ changed = false
+ if fixme.is_a? String
+ if fixme.match /\+K/
+ fixme.gsub! /\+K\@\w+/, ''
+ return true
+ else
+ return false
+ end
+ elsif fixme.is_a? Array
+ fixme.each do |v|
+ changed = fix_values_recursively(v) || changed
+ end
+ elsif fixme.is_a? Hash
+ fixme.each do |p, v|
+ changed = fix_values_recursively(v) || changed
+ end
+ end
+ changed
+ end
+end
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20131007180607) do
+ActiveRecord::Schema.define(:version => 20140129184311) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
end
def kind
- 'arvados#' + self.class.to_s.underscore
+ 'arvados#' + self.class.to_s.camelcase(:lower)
end
def etag
api_token: 27bnddk6x2nmq00a1e3gq43n9tsl5v87a3faqar2ijj8tud5en
expires_at: 2038-01-01 00:00:00
+spectator:
+ api_client: untrusted
+ user: spectator
+ api_token: zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu
+ expires_at: 2038-01-01 00:00:00
+
inactive:
api_client: untrusted
user: inactive
modified_at: 2013-12-26T19:22:54Z
updated_at: 2013-12-26T19:22:54Z
manifest_text: ". 6a4ff0499484c6c79c95cd8c566bd25f+249025 0:249025:GNU_General_Public_License,_version_3.pdf\n"
+
+foo_file:
+ uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ owner_uuid: qr1hi-tpzed-000000000000000
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n"
+
+bar_file:
+ uuid: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: qr1hi-tpzed-000000000000000
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+
+baz_file:
+ uuid: ea10d51bcf88862dbcc36eb292017dfd+45
+ owner_uuid: qr1hi-tpzed-000000000000000
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz\n"
name: Public
description: Public Group
+private:
+ uuid: zzzzz-j7d0g-rew6elm53kancon
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: Private
+ description: Private Group
+
+system_owned_group:
+ uuid: zzzzz-j7d0g-8ulrifv67tve5sx
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: System Private
+ description: System-owned Group
+
+empty_lonely_group:
+ uuid: zzzzz-j7d0g-jtp06ulmvsezgyu
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: Empty
+ description: Empty Group
+
all_users:
uuid: zzzzz-j7d0g-fffffffffffffff
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
running: 1
done: 1
runtime_constraints: {}
+
+uses_nonexistent_script_version:
+ uuid: zzzzz-8i9sb-7m339pu0x9mla88
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ cancelled_at: ~
+ cancelled_by_user_uuid: ~
+ cancelled_by_client_uuid: ~
+ script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
+ started_at: <%= 3.minute.ago.to_s(:db) %>
+ finished_at: <%= 2.minute.ago.to_s(:db) %>
+ running: false
+ success: true
+ output: d41d8cd98f00b204e9800998ecf8427e+0
+ priority: ~
+ log: d41d8cd98f00b204e9800998ecf8427e+0
+ is_locked_by_uuid: ~
+ tasks_summary:
+ failed: 0
+ todo: 0
+ running: 0
+ done: 1
+ runtime_constraints: {}
+
+foobar:
+ uuid: zzzzz-8i9sb-aceg2bnq7jt7kon
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ cancelled_at: ~
+ cancelled_by_user_uuid: ~
+ cancelled_by_client_uuid: ~
+ script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
+ script_parameters:
+ input: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ started_at: <%= 3.minute.ago.to_s(:db) %>
+ finished_at: <%= 2.minute.ago.to_s(:db) %>
+ running: false
+ success: true
+ output: fa7aeb5140e2848d39b416daeef4ffc5+45
+ priority: ~
+ log: d41d8cd98f00b204e9800998ecf8427e+0
+ is_locked_by_uuid: ~
+ tasks_summary:
+ failed: 0
+ todo: 0
+ running: 0
+ done: 1
+ runtime_constraints: {}
+
+barbaz:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykyuq
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ cancelled_at: ~
+ cancelled_by_user_uuid: ~
+ cancelled_by_client_uuid: ~
+ script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ started_at: <%= 3.minute.ago.to_s(:db) %>
+ finished_at: <%= 2.minute.ago.to_s(:db) %>
+ running: false
+ success: true
+ output: ea10d51bcf88862dbcc36eb292017dfd+45
+ priority: ~
+ log: d41d8cd98f00b204e9800998ecf8427e+0
+ is_locked_by_uuid: ~
+ tasks_summary:
+ failed: 0
+ todo: 0
+ running: 0
+ done: 1
+ runtime_constraints: {}
head_uuid: b519d9cb706a29fc7ea24dbea2f05851
properties: {}
+user_agreement_readable:
+ uuid: zzzzz-o0j2j-qpf60gg4fwjlmex
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#group
+ tail_uuid: zzzzz-j7d0g-fffffffffffffff
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851
+ properties: {}
+
+active_user_member_of_all_users_group:
+ uuid: zzzzz-o0j2j-ctbysaduejxfrs5
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_kind: arvados#group
+ head_uuid: zzzzz-j7d0g-fffffffffffffff
+ properties: {}
+
+active_user_can_manage_system_owned_group:
+ uuid: zzzzz-o0j2j-3sa30nd3bqn1msh
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-02-03 15:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-02-03 15:42:26 -0800
+ updated_at: 2014-02-03 15:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_manage
+ head_kind: arvados#group
+ head_uuid: zzzzz-j7d0g-8ulrifv67tve5sx
+ properties: {}
+
user_agreement_signed_by_active:
uuid: zzzzz-o0j2j-4x85a69tqlrud1z
owner_uuid: zzzzz-tpzed-000000000000000
head_uuid: b519d9cb706a29fc7ea24dbea2f05851
properties: {}
+spectator_user_member_of_all_users_group:
+ uuid: zzzzz-o0j2j-0s8ql1redzf8kvn
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ link_class: permission
+ name: can_read
+ head_kind: arvados#group
+ head_uuid: zzzzz-j7d0g-fffffffffffffff
+ properties: {}
+
inactive_user_member_of_all_users_group:
uuid: zzzzz-o0j2j-osckxpy5hl5fjk5
owner_uuid: zzzzz-tpzed-000000000000000
head_kind: arvados#group
head_uuid: zzzzz-j7d0g-fffffffffffffff
properties: {}
+
+foo_file_readable_by_active:
+ uuid: zzzzz-o0j2j-dp1d8395ldqw22r
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ properties: {}
+
+bar_file_readable_by_active:
+ uuid: zzzzz-o0j2j-8hppiuduf8eqdng
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: fa7aeb5140e2848d39b416daeef4ffc5+45
+ properties: {}
+
+bar_file_readable_by_spectator:
+ uuid: zzzzz-o0j2j-0mhldkqozsltcli
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: fa7aeb5140e2848d39b416daeef4ffc5+45
+ properties: {}
+
+baz_file_publicly_readable:
+ uuid: zzzzz-o0j2j-132ne3lk954vtoc
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#group
+ tail_uuid: zzzzz-j7d0g-fffffffffffffff
+ link_class: permission
+ name: can_read
+ head_kind: arvados#collection
+ head_uuid: ea10d51bcf88862dbcc36eb292017dfd+45
+ properties: {}
+
+barbaz_job_readable_by_spectator:
+ uuid: zzzzz-o0j2j-cpy7p41hpk531e1
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_kind: arvados#user
+ tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ link_class: permission
+ name: can_read
+ head_kind: arvados#job
+ head_uuid: zzzzz-8i9sb-cjs4pklxxjykyuq
+ properties: {}
+
is_admin: false
prefs: {}
+spectator:
+ uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ email: spectator@arvados.local
+ first_name: Spect
+ last_name: Ator
+ identity_url: https://spectator.openid.local
+ is_active: true
+ is_admin: false
+ prefs: {}
+
inactive_uninvited:
uuid: zzzzz-tpzed-rf2ec3ryh4vb5ma
email: inactive-uninvited-user@arvados.local
--- /dev/null
+testvm:
+ uuid: zzzzz-2x53u-382brsig8rp3064
+ owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ hostname: testvm.shell
assert_nil assigns(:objects)
end
+ test "create with owner_uuid set to owned group" do
+ authorize_with :active
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
+ post :create, {
+ collection: {
+ owner_uuid: 'zzzzz-j7d0g-rew6elm53kancon',
+ manifest_text: manifest_text,
+ uuid: "d30fe8ae534397864cb96c544f4cf102"
+ }
+ }
+ assert_response :success
+ resp = JSON.parse(@response.body)
+ assert_equal 'zzzzz-tpzed-000000000000000', resp['owner_uuid']
+ end
+
+ test "create with owner_uuid set to group i can_manage" do
+ authorize_with :active
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
+ post :create, {
+ collection: {
+ owner_uuid: 'zzzzz-j7d0g-8ulrifv67tve5sx',
+ manifest_text: manifest_text,
+ uuid: "d30fe8ae534397864cb96c544f4cf102"
+ }
+ }
+ assert_response :success
+ resp = JSON.parse(@response.body)
+ assert_equal 'zzzzz-tpzed-000000000000000', resp['owner_uuid']
+ end
+
+ test "create with owner_uuid set to group with no can_manage permission" do
+ authorize_with :active
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
+ post :create, {
+ collection: {
+ owner_uuid: 'zzzzz-j7d0g-it30l961gq3t0oi',
+ manifest_text: manifest_text,
+ uuid: "d30fe8ae534397864cb96c544f4cf102"
+ }
+ }
+ assert_response 403
+ end
+
+ test "admin create with owner_uuid set to group with no permission" do
+ authorize_with :admin
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e 0:0:foo.txt\n"
+ post :create, {
+ collection: {
+ owner_uuid: 'zzzzz-j7d0g-it30l961gq3t0oi',
+ manifest_text: manifest_text,
+ uuid: "d30fe8ae534397864cb96c544f4cf102"
+ }
+ }
+ assert_response :success
+ end
+
test "should create with collection passed as json" do
authorize_with :active
post :create, {
assert_response 422
end
+ test "get full provenance for baz file" do
+ authorize_with :active
+ get :provenance, uuid: 'ea10d51bcf88862dbcc36eb292017dfd+45'
+ assert_response :success
+ resp = JSON.parse(@response.body)
+ assert_not_nil resp['ea10d51bcf88862dbcc36eb292017dfd+45'] # baz
+ assert_not_nil resp['fa7aeb5140e2848d39b416daeef4ffc5+45'] # bar
+ assert_not_nil resp['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'] # foo
+ assert_not_nil resp['zzzzz-8i9sb-cjs4pklxxjykyuq'] # bar->baz
+ assert_not_nil resp['zzzzz-8i9sb-aceg2bnq7jt7kon'] # foo->bar
+ end
+
+ test "get no provenance for foo file" do
+ # spectator user cannot even see baz collection
+ authorize_with :spectator
+ get :provenance, uuid: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
+ assert_response 404
+ end
+
+ test "get partial provenance for baz file" do
+ # spectator user can see bar->baz job, but not foo->bar job
+ authorize_with :spectator
+ get :provenance, uuid: 'ea10d51bcf88862dbcc36eb292017dfd+45'
+ assert_response :success
+ resp = JSON.parse(@response.body)
+ assert_not_nil resp['ea10d51bcf88862dbcc36eb292017dfd+45'] # baz
+ assert_not_nil resp['fa7aeb5140e2848d39b416daeef4ffc5+45'] # bar
+ assert_not_nil resp['zzzzz-8i9sb-cjs4pklxxjykyuq'] # bar->baz
+ assert_nil resp['zzzzz-8i9sb-aceg2bnq7jt7kon'] # foo->bar
+ assert_nil resp['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'] # foo
+ end
+
end
class Arvados::V1::GroupsControllerTest < ActionController::TestCase
+ test "attempt to delete group without read or write access" do
+ authorize_with :active
+ post :destroy, id: groups(:empty_lonely_group).uuid
+ assert_response 404
+ end
+
test "attempt to delete group without write access" do
authorize_with :active
- post :destroy, id: groups(:public).uuid
+ post :destroy, id: groups(:all_users).uuid
assert_response 403
end
assert_not_nil job['cancelled_at'], 'un-cancelled job stays cancelled'
end
+ test "update a job without failing script_version check" do
+ authorize_with :admin
+ put :update, {
+ id: jobs(:uses_nonexistent_script_version).uuid,
+ job: {
+ owner_uuid: users(:admin).uuid
+ }
+ }
+ assert_response :success
+ put :update, {
+ id: jobs(:uses_nonexistent_script_version).uuid,
+ job: {
+ owner_uuid: users(:active).uuid
+ }
+ }
+ assert_response :success
+ end
end
class Arvados::V1::KeepDisksControllerTest < ActionController::TestCase
- test "add keep node with admin token" do
+ test "add keep disk with admin token" do
authorize_with :admin
post :ping, {
ping_secret: '', # required by discovery doc, but ignored
}
assert_response :success
assert_not_nil assigns(:object)
- new_keep_node = JSON.parse(@response.body)
- assert_not_nil new_keep_node['uuid']
- assert_not_nil new_keep_node['ping_secret']
- assert_not_equal '', new_keep_node['ping_secret']
+ new_keep_disk = JSON.parse(@response.body)
+ assert_not_nil new_keep_disk['uuid']
+ assert_not_nil new_keep_disk['ping_secret']
+ assert_not_equal '', new_keep_disk['ping_secret']
end
- test "add keep node with no filesystem_uuid" do
+ test "add keep disk with no filesystem_uuid" do
authorize_with :admin
opts = {
ping_secret: '',
assert_not_nil JSON.parse(@response.body)['uuid']
end
- test "refuse to add keep node without admin token" do
+ test "refuse to add keep disk without admin token" do
post :ping, {
ping_secret: '',
service_host: '::1',
assert_response 404
end
- test "ping from keep node" do
+ test "ping keep disk" do
post :ping, {
uuid: keep_disks(:nonfull).uuid,
ping_secret: keep_disks(:nonfull).ping_secret,
}
assert_response :success
assert_not_nil assigns(:object)
- keep_node = JSON.parse(@response.body)
- assert_not_nil keep_node['uuid']
- assert_not_nil keep_node['ping_secret']
+ keep_disk = JSON.parse(@response.body)
+ assert_not_nil keep_disk['uuid']
+ assert_not_nil keep_disk['ping_secret']
end
- test "should get index with ping_secret" do
+ test "admin should get index with ping_secret" do
authorize_with :admin
get :index
assert_response :success
assert_not_nil items[0]['ping_secret']
end
- # inactive user does not see any keep disks
- test "inactive user should get empty index" do
+ # inactive user sees keep disks
+ test "inactive user should get index" do
authorize_with :inactive
get :index
assert_response :success
items = JSON.parse(@response.body)['items']
- assert_equal 0, items.size
+ assert_not_equal 0, items.size
end
# active user sees non-secret attributes of keep disks
require 'test_helper'
class Arvados::V1::LinksControllerTest < ActionController::TestCase
+
+ test "no symbol keys in serialized hash" do
+ link = {
+ properties: {username: 'testusername'},
+ link_class: 'test',
+ name: 'encoding',
+ tail_kind: 'arvados#user',
+ tail_uuid: users(:admin).uuid,
+ head_kind: 'arvados#virtualMachine',
+ head_uuid: virtual_machines(:testvm).uuid
+ }
+ authorize_with :admin
+ [link, link.to_json].each do |formatted_link|
+ post :create, link: formatted_link
+ assert_response :success
+ assert_not_nil assigns(:object)
+ assert_equal 'testusername', assigns(:object).properties['username']
+ assert_equal false, assigns(:object).properties.has_key?(:username)
+ end
+ end
+
end
require 'test_helper'
class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase
+ test "should get_all_logins with admin token" do
+ authorize_with :admin
+ get :get_all_permissions
+ assert_response :success
+ end
+
+ test "should get_all_logins with non-admin token" do
+ authorize_with :active
+ get :get_all_permissions
+ assert_response 403
+ end
end
assert_response :success
me = JSON.parse(@response.body)
post :activate, uuid: me['uuid']
- assert_response 422
+ assert_response 403
get :current
assert_response :success
me = JSON.parse(@response.body)
class CollectionsApiTest < ActionDispatch::IntegrationTest
fixtures :all
+ def jresponse
+ @jresponse ||= ActiveSupport::JSON.decode @response.body
+ end
+
test "should get index" do
get "/arvados/v1/collections", {:format => :json}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
- @json_response ||= ActiveSupport::JSON.decode @response.body
assert_response :success
- assert_equal "arvados#collectionList", @json_response['kind']
+ assert_equal "arvados#collectionList", jresponse['kind']
+ end
+
+ test "controller 404 response is json" do
+ get "/arvados/v1/thingsthatdonotexist", {:format => :xml}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ assert_response 404
+ assert_equal 1, jresponse['errors'].length
+ assert_equal true, jresponse['errors'][0].is_a?(String)
+ end
+
+ test "object 404 response is json" do
+ get "/arvados/v1/groups/zzzzz-j7d0g-o5ba971173cup4f", {}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ assert_response 404
+ assert_equal 1, jresponse['errors'].length
+ assert_equal true, jresponse['errors'][0].is_a?(String)
end
end
+++ /dev/null
-Install dependencies
-
- rvm use 1.9.3
- gem install sinatra
- gem install thin
-
-Set up Keep backing store directories
-
- mount /dev/some-disk /mnt/point
- mkdir -p /mnt/point/keep
-
-Start server
-
- RUBYLIB=../../sdk/ruby RACK_ENV=production IP=0.0.0.0 PORT=25107 ./keep.rb
-
-Start server With SSL support
-
- export SSL_CERT=/etc/ssl/certs/keep.crt
- export SSL_KEY=/etc/ssl/private/keep.pem
- RUBYLIB=... RACK_ENV=... IP=... PORT=... ./keep.rb
+++ /dev/null
-#!/usr/bin/env ruby
-
-require 'sinatra/base'
-require 'digest/md5'
-require 'digest/sha1'
-require 'arvados'
-
-class Keep < Sinatra::Base
- @@ssl_flag = false
- def self.ssl_flag
- @@ssl_flag
- end
-
- configure do
- mime_type :binary, 'application/octet-stream'
- enable :logging
- set :port, (ENV['PORT'] || '25107').to_i
- set :bind, (ENV['IP'] || '0.0.0.0')
- end
-
- def verify_hash(data, hash)
- if hash.length == 32
- Digest::MD5.hexdigest(data) == hash && hash
- elsif hash.length == 40
- Digest::SHA1.hexdigest(data) == hash && hash
- else
- false
- end
- end
-
- def self.debuglevel
- if ENV['DEBUG'] and ENV['DEBUG'].match /^-?\d+/
- ENV['DEBUG'].to_i
- else
- 0
- end
- end
-
- def self.debuglog(loglevel, msg)
- if debuglevel >= loglevel
- $stderr.puts "[keepd/#{$$} #{Time.now}] #{msg}"
- end
- end
- def debuglog(*args)
- self.class.debuglog *args
- end
-
- def self.keepdirs
- return @@keepdirs if defined? @@keepdirs
- # Configure backing store directories
- @@keepdirs = []
- rootdir = (ENV['KEEP_ROOT'] || '/').sub /\/$/, ''
- `mount`.split("\n").each do |mountline|
- dev, on_txt, mountpoint, type_txt, fstype, opts = mountline.split
- if on_txt == 'on' and type_txt == 'type'
- debuglog 2, "dir #{mountpoint} is mounted"
- if mountpoint[0..(rootdir.length)] == rootdir + '/'
- debuglog 2, "dir #{mountpoint} is in #{rootdir}/"
- keepdir = "#{mountpoint.sub /\/$/, ''}/keep"
- if File.exists? "#{keepdir}/."
- kd = {
- :root => keepdir,
- :arvados => {},
- :arvados_file => File.join(keepdir, 'arvados_keep_disk.json'),
- :readonly => false,
- :device => dev,
- :device_inode => File.stat(dev).ino
- }
- if opts.gsub(/[\(\)]/, '').split(',').index('ro')
- kd[:readonly] = true
- end
- debuglog 2, "keepdir #{kd.inspect}"
- begin
- kd[:arvados] = JSON.parse(File.read(kd[:arvados_file]), symbolize_names: true)
- rescue
- debuglog 0, "keepdir #{kd.inspect} is new (no #{kd[:arvados_file]})"
- end
- @@keepdirs << kd
- end
- end
- end
- end
- Dir.open('/dev/disk/by-uuid/').each do |fs_uuid|
- next if fs_uuid.match /^\./
- fs_root_inode = File.stat("/dev/disk/by-uuid/#{fs_uuid}").ino
- @@keepdirs.each do |kd|
- if kd[:device_inode] == fs_root_inode
- kd[:filesystem_uuid] = fs_uuid
- debuglog 0, "keepdir #{kd.reject { |k,v| k==:arvados }.inspect}"
- end
- end
- end
- @@keepdirs
- end
- self.keepdirs
-
- def find_backfile(hash, opts)
- subdir = hash[0..2]
- @@keepdirs.each do |keepdir|
- backfile = "#{keepdir[:root]}/#{subdir}/#{hash}"
- if File.exists? backfile
- data = nil
- File.open("#{keepdir[:root]}/lock", "a+") do |f|
- if f.flock File::LOCK_EX
- data = File.read backfile
- end
- end
- if data and (!opts[:verify_hash] or verify_hash data, hash)
- return [backfile, data]
- end
- end
- end
- nil
- end
-
- get '/:locator' do |locator|
- regs = locator.match /^([0-9a-f]{32,})/
- if regs
- hash = regs[1]
- backfile, data = find_backfile hash, :verify_hash => false
- if data
- content_type :binary
- body data
- else
- status 404
- body 'not found'
- end
- else
- pass
- end
- self.class.ping_arvados
- end
-
- put '/:locator' do |locator|
- data = request.body.read
- hash = verify_hash(data, locator)
- if not hash
- status 422
- body "Checksum mismatch"
- return
- end
- backfile, havedata = find_backfile hash, :verify_hash => true
- if havedata
- status 200
- body 'OK'
- else
- wrote = nil
- subdir = hash[0..2]
- @@keepdirs.each do |keepdir|
- next if keepdir[:readonly]
- backdir = "#{keepdir[:root]}/#{subdir}"
- if !File.exists? backdir
- begin
- Dir.mkdir backdir
- rescue
- end
- end
- backfile = "#{keepdir[:root]}/#{subdir}/#{hash}"
- File.open("#{keepdir[:root]}/lock", "a+") do |lf|
- if lf.flock File::LOCK_EX
- File.open(backfile + ".tmp", "a+") do |wf|
- if wf.flock File::LOCK_EX
- wf.seek 0, File::SEEK_SET
- wf.truncate 0
- wrote = wf.write data
- end
- if wrote == data.length
- File.rename backfile+".tmp", backfile
- break
- else
- File.unlink backfile+".tmp"
- end
- end
- end
- end
- end
- if wrote == data.length
- status 200
- body 'OK'
- else
- status 500
- body 'Fail'
- end
- end
- self.class.ping_arvados
- end
-
- protected
-
- def self.ping_arvados
- return if defined? @@last_ping_at and @@last_ping_at > Time.now - 300
- @@last_ping_at = Time.now
- begin
- @@arvados ||= Arvados.new(api_version: 'v1', api_token: '')
- @@keepdirs.each do |kd|
- ack = @@arvados.keep_disk.ping(uuid: kd[:arvados][:uuid],
- service_port: settings.port,
- service_ssl_flag: Keep.ssl_flag,
- ping_secret: kd[:arvados][:ping_secret],
- is_readable: true,
- is_writable: !kd[:readonly],
- filesystem_uuid: kd[:filesystem_uuid])
- if ack and ack[:last_ping_at]
- debuglog 0, "device #{kd[:device]} uuid #{ack[:uuid]} last_ping_at #{ack[:last_ping_at]}"
- if kd[:arvados].empty?
- File.open(kd[:arvados_file]+'.tmp', 'a+', 0600) do end
- File.open(kd[:arvados_file]+'.tmp', 'r+', 0600) do |f|
- if f.flock File::LOCK_EX
- f.seek 0, File::SEEK_SET
- f.truncate 0
- f.write ack.to_json
- File.rename kd[:arvados_file]+'.tmp', kd[:arvados_file]
- kd[:arvados] = ack
- end
- end
- end
- else
- debuglog 0, "device #{kd[:device]} ping fail"
- end
- end
- rescue Exception => e
- debuglog 0, "ping_arvados: #{e.inspect}"
- end
- end
- self.ping_arvados
-
- if app_file == $0
- run! do |server|
- if ENV['SSL_CERT'] and ENV['SSL_KEY']
- ssl_options = {
- :cert_chain_file => ENV['SSL_CERT'],
- :private_key_file => ENV['SSL_KEY'],
- :verify_peer => false
- }
- @@ssl_flag = true
- server.ssl = true
- server.ssl_options = ssl_options
- end
- end
- end
-end
projects / arvados.git / commitdiff