services/keep/bin
services/keep/pkg
services/keep/src/github.com
+sdk/java/target
+*.class
# This can be a symlink to ../../../doc/.site in dev setups
/public/doc
+
+# SimpleCov reports
+/coverage
+
+# Dev/test SSL certificates
+/self-signed.key
+/self-signed.pem
gem 'capybara'
gem 'poltergeist'
gem 'headless'
+ # Note: "require: false" here tells bunder not to automatically
+ # 'require' the packages during application startup. Installation is
+ # still mandatory.
+ gem 'simplecov', '~> 0.7.1', require: false
+ gem 'simplecov-rcov', require: false
end
gem 'jquery-rails'
multi_json (~> 1.0)
rubyzip (~> 1.0)
websocket (~> 1.0.4)
+ simplecov (0.7.1)
+ multi_json (~> 1.0)
+ simplecov-html (~> 0.7.1)
+ simplecov-html (0.7.1)
+ simplecov-rcov (0.2.3)
+ simplecov (>= 0.4.1)
sprockets (2.2.2)
hike (~> 1.2)
multi_json (~> 1.0)
sass
sass-rails (~> 3.2.0)
selenium-webdriver
+ simplecov (~> 0.7.1)
+ simplecov-rcov
sqlite3
themes_for_rails
therubyracer
'X-CSRF-Token': $('meta[name="csrf-token"]').attr('content')
}
});
- $('.editable').editable();
$('[data-toggle=tooltip]').tooltip();
$('.expand-collapse-row').on('click', function(event) {
-$.fn.editable.defaults.ajaxOptions = {type: 'put', dataType: 'json'};
+$.fn.editable.defaults.ajaxOptions = {type: 'post', dataType: 'json'};
$.fn.editable.defaults.send = 'always';
// Default for editing is popup. I experimented with inline which is a little
$.fn.editable.defaults.params = function (params) {
var a = {};
var key = params.pk.key;
- a.id = params.pk.id;
- a[key] = {};
+ a.id = $(this).attr('data-object-uuid') || params.pk.id;
+ a[key] = params.pk.defaults || {};
+ // Remove null values. Otherwise they get transmitted as empty
+ // strings in request params.
+ for (i in a[key]) {
+ if (a[key][i] == null)
+ delete a[key][i];
+ }
a[key][params.name] = params.value;
+ if (!a.id) {
+ a['_method'] = 'post';
+ } else {
+ a['_method'] = 'put';
+ }
return a;
};
}
}
+$(document).
+ on('ready ajax:complete', function() {
+ $('#editable-submit').click(function() {
+ console.log($(this));
+ });
+ $('.editable').
+ editable({
+ success: function(response, newValue) {
+ // If we just created a new object, stash its UUID
+ // so we edit it next time instead of creating
+ // another new object.
+ if (!$(this).attr('data-object-uuid') && response.uuid) {
+ $(this).attr('data-object-uuid', response.uuid);
+ }
+ if (response.href) {
+ $(this).editable('option', 'url', response.href);
+ }
+ return;
+ }
+ }).
+ on('hidden', function(e, reason) {
+ // After saving a new attribute, update the same
+ // information if it appears elsewhere on the page.
+ if (reason != 'save') return;
+ var html = $(this).html();
+ var uuid = $(this).attr('data-object-uuid');
+ var attr = $(this).attr('data-name');
+ var edited = this;
+ if (uuid && attr) {
+ $("[data-object-uuid='" + uuid + "']" +
+ "[data-name='" + attr + "']").each(function() {
+ if (this != edited)
+ $(this).html(html);
+ });
+ }
+ });
+ });
+
$.fn.editabletypes.text.defaults.tpl = '<input type="text" name="editable-text">'
$.fn.editableform.buttons = '\
--- /dev/null
+$(document).
+ on('ready ajax:complete', function() {
+ $("[data-toggle='x-editable']").click(function(e) {
+ e.stopPropagation();
+ $($(this).attr('data-toggle-selector')).editable('toggle');
+ });
+ }).on('paste keyup change', 'input.search-folder-contents', function() {
+ var q = new RegExp($(this).val(), 'i');
+ $(this).closest('div.panel').find('tbody tr').each(function() {
+ $(this).toggle(!!$(this).text().match(q));
+ });
+ });
--- /dev/null
+# Place all the behaviors and hooks related to the matching controller here.
+# All this logic will automatically be available in application.js.
+# You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/
# Place all the behaviors and hooks related to the matching controller here.
# All this logic will automatically be available in application.js.
# You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/
+
+cache_age_in_days = (milliseconds_age) ->
+ ONE_DAY = 1000 * 60 * 60 * 24
+ milliseconds_age / ONE_DAY
+
+cache_age_hover = (milliseconds_age) ->
+ 'Cache age ' + cache_age_in_days(milliseconds_age).toFixed(1) + ' days.'
+
+cache_age_axis_label = (milliseconds_age) ->
+ cache_age_in_days(milliseconds_age).toFixed(0) + ' days'
+
+float_as_percentage = (proportion) ->
+ (proportion.toFixed(4) * 100) + '%'
+
+$.renderHistogram = (histogram_data) ->
+ Morris.Area({
+ element: 'cache-age-vs-disk-histogram',
+ pointSize: 0,
+ lineWidth: 0,
+ data: histogram_data,
+ xkey: 'age',
+ ykeys: ['persisted', 'cache'],
+ labels: ['Persisted Storage Disk Utilization', 'Cached Storage Disk Utilization'],
+ ymax: 1,
+ ymin: 0,
+ xLabelFormat: cache_age_axis_label,
+ yLabelFormat: float_as_percentage,
+ dateFormat: cache_age_hover
+ })
}
var update_count = function(e) {
+ var html;
+ var this_object_uuid = $('#selection-form-content').
+ closest('form').
+ find('input[name=uuid]').val();
var lst = get_selection_list();
$("#persistent-selection-count").text(lst.length);
if (lst.length > 0) {
- $('#selection-form-content').html(
- '<li><a href="#" id="clear_selections_button">Clear selections</a></li>'
- + '<li><input type="submit" name="combine_selected_files_into_collection" '
- + ' id="combine_selected_files_into_collection" '
- + ' value="Combine selected collections and files into a new collection" /></li>'
- + '<li class="notification"><table style="width: 100%"></table></li>');
+ html = '<li><a href="#" class="btn btn-xs btn-info" id="clear_selections_button"><i class="fa fa-fw fa-ban"></i> Clear selections</a></li>';
+ if (this_object_uuid.match('-j7d0g-'))
+ html += '<li><button class="btn btn-xs btn-info" type="submit" name="copy_selections_into_folder" id="copy_selections_into_folder"><i class="fa fa-fw fa-folder-open"></i> Copy selections into this folder</button></li>';
+ html += '<li><button class="btn btn-xs btn-info" type="submit" name="combine_selected_files_into_collection" '
+ + ' id="combine_selected_files_into_collection">'
+ + '<i class="fa fa-fw fa-archive"></i> Combine selected collections and files into a new collection</button></li>'
+ + '<li class="notification"><table style="width: 100%"></table></li>';
+ $('#selection-form-content').html(html);
for (var i = 0; i < lst.length; i++) {
$('#selection-form-content > li > table').append("<tr>"
a = s[i];
var h = window.innerHeight - a.getBoundingClientRect().top - 20;
height = String(h) + "px";
- a.style.height = height;
+ a.style['max-height'] = height;
}
}
table.table-justforlayout {
margin-bottom: 0;
}
+.smaller-text {
+ font-size: .8em;
+}
.deemphasize {
font-size: .8em;
color: #888;
}
+.arvados-uuid {
+ font-size: .8em;
+ font-family: monospace;
+}
table .data-size, .table .data-size {
text-align: right;
}
text-decoration: none;
text-shadow: 0 1px 0 #ffffff;
}
-/*.navbar .nav .dropdown .dropdown-menu li a {
- padding: 2px 20px;
-}*/
-
-ul.arvados-nav {
- list-style: none;
- padding-left: 0em;
- margin-left: 0em;
-}
-
-ul.arvados-nav li ul {
- list-style: none;
- padding-left: 0;
-}
-
-ul.arvados-nav li ul li {
- list-style: none;
- padding-left: 1em;
-}
.dax {
max-width: 10%;
li.notification {
padding: 10px;
}
-.arvados-nav-container {
- top: 70px;
- height: calc(100% - 70px);
- overflow: auto;
- z-index: 2;
-}
-
-.arvados-nav-active {
- background: rgb(66, 139, 202);
-}
-
-.arvados-nav-active a, .arvados-nav-active a:hover {
- color: white;
-}
// See HeaderRowFixer in application.js
table.table-fixed-header-row {
overflow-y: auto;
}
+.row-fill-height, .row-fill-height>div[class*='col-'] {
+ display: flex;
+}
+.row-fill-height>div[class*='col-']>div {
+ width: 100%;
+}
+
+/* Show editable popover above side-nav */
+.editable-popup.popover {
+ z-index:1055;
+}
+
+.navbar-nav.side-nav {
+ box-shadow: inset -1px 0 #e7e7e7;
+}
+.navbar-nav.side-nav > li:first-child {
+ margin-top: 5px; /* keep "hover" bg below top nav bottom border */
+}
+.navbar-nav.side-nav > li > a {
+ padding-top: 10px;
+ padding-bottom: 10px;
+}
+.navbar-nav.side-nav > li.dropdown > ul.dropdown-menu > li > a {
+ padding-top: 5px;
+ padding-bottom: 5px;
+}
+.navbar-nav.side-nav a.active,
+.navbar-nav.side-nav a:hover,
+.navbar-nav.side-nav a:focus {
+ border-right: 1px solid #ffffff;
+ background: #ffffff;
+}
--- /dev/null
+.card {
+ padding-top: 20px;
+ margin: 10px 0 20px 0;
+ background-color: #ffffff;
+ border: 1px solid #d8d8d8;
+ border-top-width: 0;
+ border-bottom-width: 2px;
+ -webkit-border-radius: 3px;
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+ -webkit-box-shadow: none;
+ -moz-box-shadow: none;
+ box-shadow: none;
+ -webkit-box-sizing: border-box;
+ -moz-box-sizing: border-box;
+ box-sizing: border-box;
+}
+.card.arvados-object {
+ position: relative;
+ display: inline-block;
+ width: 170px;
+ height: 175px;
+ padding-top: 0;
+ margin-left: 20px;
+ overflow: hidden;
+ vertical-align: top;
+}
+.card.arvados-object .card-top.green {
+ background-color: #53a93f;
+}
+.card.arvados-object .card-top.blue {
+ background-color: #427fed;
+}
+.card.arvados-object .card-top {
+ position: absolute;
+ top: 0;
+ left: 0;
+ display: inline-block;
+ width: 170px;
+ height: 25px;
+ background-color: #ffffff;
+}
+.card.arvados-object .card-info {
+ position: absolute;
+ top: 25px;
+ display: inline-block;
+ width: 100%;
+ height: 101px;
+ overflow: hidden;
+ background: #ffffff;
+ -webkit-box-sizing: border-box;
+ -moz-box-sizing: border-box;
+ box-sizing: border-box;
+}
+.card.arvados-object .card-info .title {
+ display: block;
+ margin: 8px 14px 0 14px;
+ overflow: hidden;
+ font-size: 16px;
+ font-weight: bold;
+ line-height: 18px;
+ color: #404040;
+}
+.card.arvados-object .card-info .desc {
+ display: block;
+ margin: 8px 14px 0 14px;
+ overflow: hidden;
+ font-size: 12px;
+ line-height: 16px;
+ color: #737373;
+ text-overflow: ellipsis;
+}
+.card.arvados-object .card-bottom {
+ position: absolute;
+ bottom: 0;
+ left: 0;
+ display: inline-block;
+ width: 100%;
+ padding: 10px 20px;
+ line-height: 29px;
+ text-align: center;
+ -webkit-box-sizing: border-box;
+ -moz-box-sizing: border-box;
+ box-sizing: border-box;
+}
--- /dev/null
+// Place all the styles related to the folders controller here.
+// They will automatically be included in application.css.
+// You can use Sass (SCSS) here: http://sass-lang.com/
// Place all the styles related to the KeepDisks controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
+
+/* Margin allows us some space between the table above. */
+div.graph {
+ margin-top: 20px;
+}
+div.graph h3, div.graph h4 {
+ text-align: center;
+}
--- /dev/null
+/*
+Author: Start Bootstrap - http://startbootstrap.com
+'SB Admin' HTML Template by Start Bootstrap
+
+All Start Bootstrap themes are licensed under Apache 2.0.
+For more info and more free Bootstrap 3 HTML themes, visit http://startbootstrap.com!
+*/
+
+/* ATTN: This is mobile first CSS - to update 786px and up screen width use the media query near the bottom of the document! */
+
+/* Global Styles */
+
+body {
+ margin-top: 50px;
+}
+
+#wrapper {
+ padding-left: 0;
+}
+
+#page-wrapper {
+ width: 100%;
+ padding: 5px 15px;
+}
+
+/* Nav Messages */
+
+.messages-dropdown .dropdown-menu .message-preview .avatar,
+.messages-dropdown .dropdown-menu .message-preview .name,
+.messages-dropdown .dropdown-menu .message-preview .message,
+.messages-dropdown .dropdown-menu .message-preview .time {
+ display: block;
+}
+
+.messages-dropdown .dropdown-menu .message-preview .avatar {
+ float: left;
+ margin-right: 15px;
+}
+
+.messages-dropdown .dropdown-menu .message-preview .name {
+ font-weight: bold;
+}
+
+.messages-dropdown .dropdown-menu .message-preview .message {
+ font-size: 12px;
+}
+
+.messages-dropdown .dropdown-menu .message-preview .time {
+ font-size: 12px;
+}
+
+
+/* Nav Announcements */
+
+.announcement-heading {
+ font-size: 50px;
+ margin: 0;
+}
+
+.announcement-text {
+ margin: 0;
+}
+
+/* Table Headers */
+
+table.tablesorter thead {
+ cursor: pointer;
+}
+
+table.tablesorter thead tr th:hover {
+ background-color: #f5f5f5;
+}
+
+/* Flot Chart Containers */
+
+.flot-chart {
+ display: block;
+ height: 400px;
+}
+
+.flot-chart-content {
+ width: 100%;
+ height: 100%;
+}
+
+/* Edit Below to Customize Widths > 768px */
+@media (min-width:768px) {
+
+ /* Wrappers */
+
+ #wrapper {
+ padding-left: 225px;
+ }
+
+ #page-wrapper {
+ padding: 15px 25px;
+ }
+
+ /* Side Nav */
+
+ .side-nav {
+ margin-left: -225px;
+ left: 225px;
+ width: 225px;
+ position: fixed;
+ top: 50px;
+ height: calc(100% - 50px);
+ border-radius: 0;
+ border: none;
+ background-color: #f8f8f8;
+ overflow-y: auto;
+ overflow-x: hidden; /* no left nav scroll bar */
+ }
+
+ /* Bootstrap Default Overrides - Customized Dropdowns for the Side Nav */
+
+ .side-nav>li.dropdown>ul.dropdown-menu {
+ position: relative;
+ min-width: 225px;
+ margin: 0;
+ padding: 0;
+ border: none;
+ border-radius: 0;
+ background-color: transparent;
+ box-shadow: none;
+ -webkit-box-shadow: none;
+ }
+
+ .side-nav>li.dropdown>ul.dropdown-menu>li>a {
+ color: #777777;
+ padding: 15px 15px 15px 25px;
+ }
+
+ .side-nav>li.dropdown>ul.dropdown-menu>li>a:hover,
+ .side-nav>li.dropdown>ul.dropdown-menu>li>a.active,
+ .side-nav>li.dropdown>ul.dropdown-menu>li>a:focus {
+ background-color: #ffffff;
+ }
+
+ .side-nav>li>a {
+ width: 225px;
+ }
+
+ .navbar-default .navbar-nav.side-nav>li>a:hover,
+ .navbar-default .navbar-nav.side-nav>li>a:focus {
+ background-color: #ffffff;
+ }
+
+ /* Nav Messages */
+
+ .messages-dropdown .dropdown-menu {
+ min-width: 300px;
+ }
+
+ .messages-dropdown .dropdown-menu li a {
+ white-space: normal;
+ }
+
+ .navbar-collapse {
+ padding-left: 15px !important;
+ padding-right: 15px !important;
+ }
+
+}
width: 500px;
}
-#selection-form-content > li > a, #selection-form-content > li > input {
- display: block;
- padding: 3px 20px;
- clear: both;
- font-weight: normal;
- line-height: 1.42857;
- color: rgb(51, 51, 51);
- white-space: nowrap;
- border: none;
- background: transparent;
- width: 100%;
- text-align: left;
+#selection-form-content > li > a, #selection-form-content > li > button {
+ margin: 3px 20px;
}
#selection-form-content li table tr {
border-top: 1px solid rgb(221, 221, 221);
}
-#selection-form-content a:hover, #selection-form-content a:focus, #selection-form-content input:hover, #selection-form-content input:focus, #selection-form-content tr:hover {
- text-decoration: none;
- color: rgb(38, 38, 38);
- background-color: whitesmoke;
-}
\ No newline at end of file
+#selection-form-content li table tr:last-child {
+ border-bottom: 1px solid rgb(221, 221, 221);
+}
class ActionsController < ApplicationController
- skip_before_filter :find_object_by_uuid, only: :post
+ @@exposed_actions = {}
+ def self.expose_action method, &block
+ @@exposed_actions[method] = true
+ define_method method, block
+ end
+
+ def model_class
+ ArvadosBase::resource_class_for_uuid(params[:uuid])
+ end
+
+ def post
+ params.keys.collect(&:to_sym).each do |param|
+ if @@exposed_actions[param]
+ return self.send(param)
+ end
+ end
+ redirect_to :back
+ end
+
+ expose_action :copy_selections_into_folder do
+ already_named = Link.
+ filter([['tail_uuid','=',@object.uuid],
+ ['head_uuid','in',params["selection"]]]).
+ collect(&:head_uuid)
+ (params["selection"] - already_named).each do |s|
+ Link.create(tail_uuid: @object.uuid,
+ head_uuid: s,
+ link_class: 'name',
+ name: "#{s} added #{Time.now}")
+ end
+ redirect_to @object
+ end
- def combine_selected_files_into_collection
+ expose_action :combine_selected_files_into_collection do
lst = []
files = []
params["selection"].each do |s|
redirect_to controller: 'collections', action: :show, id: newc.uuid
end
- def post
- if params["combine_selected_files_into_collection"]
- combine_selected_files_into_collection
- else
- redirect_to :back
- end
- end
end
class ApiClientAuthorizationsController < ApplicationController
- def index
- m = model_class.all
- items_available = m.items_available
- offset = m.result_offset
- limit = m.result_limit
- filtered = m.to_ary.reject do |x|
- x.api_client_id == 0 or (x.expires_at and x.expires_at < Time.now) rescue false
- end
- ArvadosApiClient::patch_paging_vars(filtered, items_available, offset, limit)
- @objects = ArvadosResourceList.new(ApiClientAuthorization)
- @objects.results= filtered
- super
- end
def index_pane_list
%w(Recent Help)
class ApplicationController < ActionController::Base
respond_to :html, :json, :js
protect_from_forgery
+
+ ERROR_ACTIONS = [:render_error, :render_not_found]
+
around_filter :thread_clear
- around_filter :thread_with_mandatory_api_token, :except => [:render_exception, :render_not_found]
+ around_filter(:thread_with_mandatory_api_token,
+ except: [:index, :show] + ERROR_ACTIONS)
around_filter :thread_with_optional_api_token
- before_filter :find_object_by_uuid, :except => [:index, :render_exception, :render_not_found]
- before_filter :check_user_agreements, :except => [:render_exception, :render_not_found]
- before_filter :check_user_notifications, :except => [:render_exception, :render_not_found]
+ before_filter :check_user_agreements, except: ERROR_ACTIONS
+ before_filter :check_user_notifications, except: ERROR_ACTIONS
+ around_filter :using_reader_tokens, only: [:index, :show]
+ before_filter :find_object_by_uuid, except: [:index] + ERROR_ACTIONS
+ before_filter :check_my_folders, :except => ERROR_ACTIONS
theme :select_theme
begin
end
def index
+ @limit ||= 200
if params[:limit]
- limit = params[:limit].to_i
- else
- limit = 200
+ @limit = params[:limit].to_i
end
+ @offset ||= 0
if params[:offset]
- offset = params[:offset].to_i
- else
- offset = 0
+ @offset = params[:offset].to_i
+ end
+
+ @filters ||= []
+ if params[:filters]
+ filters = params[:filters]
+ if filters.is_a? String
+ filters = Oj.load filters
+ end
+ @filters += filters
end
- @objects ||= model_class.limit(limit).offset(offset).all
+ @objects ||= model_class
+ @objects = @objects.filter(@filters).limit(@limit).offset(@offset).all
respond_to do |f|
f.json { render json: @objects }
f.html { render }
return render_not_found("object not found")
end
respond_to do |f|
- f.json { render json: @object }
+ f.json { render json: @object.attributes.merge(href: url_for(@object)) }
f.html {
if request.method == 'GET'
render
end
def create
- @object ||= model_class.new params[model_class.to_s.underscore.singularize]
+ @new_resource_attrs ||= params[model_class.to_s.underscore.singularize]
+ @new_resource_attrs ||= {}
+ @new_resource_attrs.reject! { |k,v| k.to_s == 'uuid' }
+ @object ||= model_class.new @new_resource_attrs
@object.save!
-
- respond_to do |f|
- f.json { render json: @object }
- f.html {
- redirect_to(params[:return_to] || @object)
- }
- f.js { render }
- end
+ show
end
def destroy
end
protected
-
+
+ def redirect_to_login
+ respond_to do |f|
+ f.html {
+ if request.method == 'GET'
+ redirect_to $arvados_api_client.arvados_login_url(return_to: request.url)
+ else
+ flash[:error] = "Either you are not logged in, or your session has timed out. I can't automatically log you in and re-attempt this request."
+ redirect_to :back
+ end
+ }
+ f.json {
+ @errors = ['You do not seem to be logged in. You did not supply an API token with this request, and your session (if any) has timed out.']
+ self.render_error status: 422
+ }
+ end
+ false # For convenience to return from callbacks
+ end
+
+ def using_reader_tokens(login_optional=false)
+ if params[:reader_tokens].is_a?(Array) and params[:reader_tokens].any?
+ Thread.current[:reader_tokens] = params[:reader_tokens]
+ end
+ begin
+ yield
+ rescue ArvadosApiClient::NotLoggedInException
+ if login_optional
+ raise
+ else
+ return redirect_to_login
+ end
+ ensure
+ Thread.current[:reader_tokens] = nil
+ end
+ end
+
+ def using_specific_api_token(api_token)
+ start_values = {}
+ [:arvados_api_token, :user].each do |key|
+ start_values[key] = Thread.current[key]
+ end
+ Thread.current[:arvados_api_token] = api_token
+ Thread.current[:user] = nil
+ begin
+ yield
+ ensure
+ start_values.each_key { |key| Thread.current[key] = start_values[key] }
+ end
+ end
+
def find_object_by_uuid
if params[:id] and params[:id].match /\D/
params[:uuid] = params.delete :id
end
- if params[:uuid].is_a? String
- @object = model_class.find(params[:uuid])
+ if not model_class
+ @object = nil
+ elsif params[:uuid].is_a? String
+ if params[:uuid].empty?
+ @object = nil
+ else
+ @object = model_class.find(params[:uuid])
+ end
else
@object = model_class.where(uuid: params[:uuid]).first
end
end
if try_redirect_to_login
unless login_optional
- respond_to do |f|
- f.html {
- if request.method == 'GET'
- redirect_to $arvados_api_client.arvados_login_url(return_to: request.url)
- else
- flash[:error] = "Either you are not logged in, or your session has timed out. I can't automatically log you in and re-attempt this request."
- redirect_to :back
- end
- }
- f.json {
- @errors = ['You do not seem to be logged in. You did not supply an API token with this request, and your session (if any) has timed out.']
- self.render_error status: 422
- }
- end
+ redirect_to_login
else
# login is optional for this route so go on to the regular controller
Thread.current[:arvados_api_token] = nil
yield
else
# We skipped thread_with_mandatory_api_token. Use the optional version.
- thread_with_api_token(true) do
+ thread_with_api_token(true) do
yield
end
end
@@notification_tests = []
@@notification_tests.push lambda { |controller, current_user|
- AuthorizedKey.limit(1).where(authorized_user_uuid: current_user.uuid).each do
+ AuthorizedKey.limit(1).where(authorized_user_uuid: current_user.uuid).each do
return nil
end
return lambda { |view|
}
}
+ def check_my_folders
+ @my_top_level_folders = lambda do
+ @top_level_folders ||= Group.
+ filter([['group_class','=','folder'],
+ ['owner_uuid','=',current_user.uuid]]).
+ sort_by { |x| x.name || '' }
+ end
+ end
+
def check_user_notifications
@notification_count = 0
@notifications = []
if current_user
- @showallalerts = false
+ @showallalerts = false
@@notification_tests.each do |t|
a = t.call(self, current_user)
if a
class CollectionsController < ApplicationController
- skip_before_filter :find_object_by_uuid, :only => [:provenance]
- skip_before_filter :check_user_agreements, :only => [:show_file]
+ skip_around_filter :thread_with_mandatory_api_token, only: [:show_file]
+ skip_before_filter :find_object_by_uuid, only: [:provenance, :show_file]
+ skip_before_filter :check_user_agreements, only: [:show_file]
def show_pane_list
%w(Files Attributes Metadata Provenance_graph Used_by JSON API)
end
def show_file
- opts = params.merge(arvados_api_token: Thread.current[:arvados_api_token])
- if r = params[:file].match(/(\.\w+)/)
- ext = r[1]
+ # We pipe from arv-get to send the file to the user. Before we start it,
+ # we ask the API server if the file actually exists. This serves two
+ # purposes: it lets us return a useful status code for common errors, and
+ # helps us figure out which token to provide to arv-get.
+ coll = nil
+ usable_token = find_usable_token do
+ coll = Collection.find(params[:uuid])
end
+ if usable_token.nil?
+ return # Response already rendered.
+ elsif params[:file].nil? or not file_in_collection?(coll, params[:file])
+ return render_not_found
+ end
+ opts = params.merge(arvados_api_token: usable_token)
+ ext = File.extname(params[:file])
self.response.headers['Content-Type'] =
Rack::Mime::MIME_TYPES[ext] || 'application/octet-stream'
self.response.headers['Content-Length'] = params[:size] if params[:size]
self.response.headers['Content-Disposition'] = params[:disposition] if params[:disposition]
- self.response_body = FileStreamer.new opts
+ self.response_body = file_enumerator opts
end
def show
end
Collection.where(uuid: @object.uuid).each do |u|
- puts request
@prov_svg = ProvenanceHelper::create_provenance_graph(u.provenance, "provenance_svg",
{:request => request,
:direction => :bottom_up,
end
protected
+
+ def find_usable_token
+ # Iterate over every token available to make it the current token and
+ # yield the given block.
+ # If the block succeeds, return the token it used.
+ # Otherwise, render an error response based on the most specific
+ # error we encounter, and return nil.
+ read_tokens = [Thread.current[:arvados_api_token]].compact
+ if params[:reader_tokens].is_a? Array
+ read_tokens += params[:reader_tokens]
+ end
+ most_specific_error = [401]
+ read_tokens.each do |api_token|
+ using_specific_api_token(api_token) do
+ begin
+ yield
+ return api_token
+ rescue ArvadosApiClient::NotLoggedInException => error
+ status = 401
+ rescue => error
+ status = (error.message =~ /\[API: (\d+)\]$/) ? $1.to_i : nil
+ raise unless [401, 403, 404].include?(status)
+ end
+ if status >= most_specific_error.first
+ most_specific_error = [status, error]
+ end
+ end
+ end
+ case most_specific_error.shift
+ when 401, 403
+ redirect_to_login
+ when 404
+ render_not_found(*most_specific_error)
+ end
+ return nil
+ end
+
+ def file_in_collection?(collection, filename)
+ def normalized_path(part_list)
+ File.join(part_list).sub(%r{^\./}, '')
+ end
+ target = normalized_path([filename])
+ collection.files.each do |file_spec|
+ return true if (normalized_path(file_spec[0, 2]) == target)
+ end
+ false
+ end
+
+ def file_enumerator(opts)
+ FileStreamer.new opts
+ end
+
class FileStreamer
def initialize(opts={})
@opts = opts
--- /dev/null
+class FoldersController < ApplicationController
+ def model_class
+ Group
+ end
+
+ def index_pane_list
+ %w(My_folders Shared_with_me)
+ end
+
+ def remove_item
+ @removed_uuids = []
+ links = []
+ item = ArvadosBase.find params[:item_uuid]
+ if (item.class == Link and
+ item.link_class == 'name' and
+ item.tail_uuid = @object.uuid)
+ # Given uuid is a name link, linking an object to this
+ # folder. First follow the link to find the item we're removing,
+ # then delete the link.
+ links << item
+ item = ArvadosBase.find item.head_uuid
+ else
+ # Given uuid is an object. Delete all names.
+ links += Link.where(tail_uuid: @object.uuid,
+ head_uuid: item.uuid,
+ link_class: 'name')
+ end
+ links.each do |link|
+ @removed_uuids << link.uuid
+ link.destroy
+ end
+ if item.owner_uuid == @object.uuid
+ # Object is owned by this folder. Remove it from the folder by
+ # changing owner to the current user.
+ item.update_attributes owner_uuid: current_user
+ @removed_uuids << item.uuid
+ end
+ end
+
+ def index
+ @my_folders = []
+ @shared_with_me = []
+ @objects = Group.where(group_class: 'folder').order('name')
+ owner_of = {}
+ moretodo = true
+ while moretodo
+ moretodo = false
+ @objects.each do |folder|
+ if !owner_of[folder.uuid]
+ moretodo = true
+ owner_of[folder.uuid] = folder.owner_uuid
+ end
+ if owner_of[folder.owner_uuid]
+ if owner_of[folder.uuid] != owner_of[folder.owner_uuid]
+ owner_of[folder.uuid] = owner_of[folder.owner_uuid]
+ moretodo = true
+ end
+ end
+ end
+ end
+ @objects.each do |folder|
+ if owner_of[folder.uuid] == current_user.uuid
+ @my_folders << folder
+ else
+ @shared_with_me << folder
+ end
+ end
+ end
+
+ def show
+ @objects = @object.contents include_linked: true
+ @share_links = Link.filter([['head_uuid', '=', @object.uuid],
+ ['link_class', '=', 'permission']])
+ @logs = Log.limit(10).filter([['object_uuid', '=', @object.uuid]])
+
+ @objects_and_names = []
+ @objects.each do |object|
+ if !(name_links = @objects.links_for(object, 'name')).empty?
+ name_links.each do |name_link|
+ @objects_and_names << [object, name_link]
+ end
+ else
+ @objects_and_names << [object,
+ Link.new(tail_uuid: @object.uuid,
+ head_uuid: object.uuid,
+ link_class: "name",
+ name: "")]
+ end
+ end
+
+ super
+ end
+
+ def create
+ @new_resource_attrs = (params['folder'] || {}).merge(group_class: 'folder')
+ @new_resource_attrs[:name] ||= 'New folder'
+ super
+ end
+end
class GroupsController < ApplicationController
def index
- @groups = Group.all
+ @groups = Group.filter [['group_class', 'not in', ['folder']]]
@group_uuids = @groups.collect &:uuid
@links_from = Link.where link_class: 'permission', tail_uuid: @group_uuids
@links_to = Link.where link_class: 'permission', head_uuid: @group_uuids
end
+
+ def show
+ return redirect_to(folder_path(@object)) if @object.group_class == 'folder'
+ super
+ end
end
def index
@svg = ""
if params[:uuid]
- @jobs = Job.where(uuid: params[:uuid])
- generate_provenance(@jobs)
+ @objects = Job.where(uuid: params[:uuid])
+ generate_provenance(@objects)
else
- @jobs = Job.all
+ @limit = 20
+ super
end
end
@object = KeepDisk.new defaults.merge(params[:keep_disk] || {})
super
end
+
+ def index
+ # Retrieve cache age histogram info from logs.
+
+ # In the logs we expect to find it in an ordered list with entries
+ # of the form (mtime, disk proportion free).
+
+ # An entry of the form (1388747781, 0.52) means that if we deleted
+ # the oldest non-presisted blocks until we had 52% of the disk
+ # free, then all blocks with an mtime greater than 1388747781
+ # would be preserved.
+
+ # The chart we want to produce, will tell us how much of the disk
+ # will be free if we use a cache age of x days. Therefore we will
+ # produce output specifying the age, cache and persisted. age is
+ # specified in milliseconds. cache is the size of the cache if we
+ # delete all blocks older than age. persistent is the size of the
+ # persisted blocks. It is constant regardless of age, but it lets
+ # us show a stacked graph.
+
+ # Finally each entry in cache_age_histogram is a dictionary,
+ # because that's what our charting package wats.
+
+ @cache_age_histogram = []
+ @histogram_pretty_date = nil
+ histogram_log = Log.
+ filter([[:event_type, '=', 'block-age-free-space-histogram']]).
+ order(:created_at => :desc).
+ limit(1)
+ histogram_log.each do |log_entry|
+ # We expect this block to only execute at most once since we
+ # specified limit(1)
+ @cache_age_histogram = log_entry['properties'][:histogram]
+ # Javascript wants dates in milliseconds.
+ histogram_date_ms = log_entry['event_at'].to_i * 1000
+ @histogram_pretty_date = log_entry['event_at'].strftime('%b %-d, %Y')
+
+ total_free_cache = @cache_age_histogram[-1][1]
+ persisted_storage = 1 - total_free_cache
+ @cache_age_histogram.map! { |x| {:age => histogram_date_ms - x[0]*1000,
+ :cache => total_free_cache - x[1],
+ :persisted => persisted_storage} }
+ end
+
+ # Do the regular control work needed.
+ super
+ end
end
end
def index
- @objects ||= model_class.limit(20).all
+ @limit = 20
super
end
attrvalue = attrvalue.to_json if attrvalue.is_a? Hash or attrvalue.is_a? Array
- link_to attrvalue.to_s, '#', {
+ ajax_options = {
+ "data-pk" => {
+ id: object.uuid,
+ key: object.class.to_s.underscore
+ }
+ }
+ if object.uuid
+ ajax_options['data-url'] = url_for(action: "update", id: object.uuid, controller: object.class.to_s.pluralize.underscore)
+ else
+ ajax_options['data-url'] = url_for(action: "create", controller: object.class.to_s.pluralize.underscore)
+ ajax_options['data-pk'][:defaults] = object.attributes
+ end
+ ajax_options['data-pk'] = ajax_options['data-pk'].to_json
+
+ content_tag 'span', attrvalue.to_s, {
"data-emptytext" => "none",
"data-placement" => "bottom",
"data-type" => input_type,
- "data-url" => url_for(action: "update", id: object.uuid, controller: object.class.to_s.pluralize.underscore),
"data-title" => "Update #{attr.gsub '_', ' '}",
"data-name" => attr,
- "data-pk" => "{id: \"#{object.uuid}\", key: \"#{object.class.to_s.underscore}\"}",
+ "data-object-uuid" => object.uuid,
:class => "editable"
- }.merge(htmloptions)
+ }.merge(htmloptions).merge(ajax_options)
end
def render_pipeline_component_attribute(object, attr, subattr, value_info, htmloptions={})
--- /dev/null
+module FoldersHelper
+end
i = -1
object.components.each do |cname, c|
- puts cname, c
i += 1
pj = {index: i, name: cname}
pj[:job] = c[:job].is_a?(Hash) ? c[:job] : {}
profile_checkpoint
@@client_mtx.synchronize do
- if not @@api_client
+ if not @@api_client
@@api_client = HTTPClient.new
if Rails.configuration.arvados_insecure_https
@@api_client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
end
end
- api_token = Thread.current[:arvados_api_token]
- api_token ||= ''
-
resources_kind = class_kind(resources_kind).pluralize if resources_kind.is_a? Class
url = "#{self.arvados_v1_base}/#{resources_kind}#{action}"
# Clean up /arvados/v1/../../discovery/v1 to /discovery/v1
url.sub! '/arvados/v1/../../', '/'
- query = {"api_token" => api_token}
+ query = {
+ 'api_token' => Thread.current[:arvados_api_token] || '',
+ 'reader_tokens' => (Thread.current[:reader_tokens] || []).to_json,
+ }
if !data.nil?
data.each do |k,v|
if v.is_a? String or v.nil?
if @@profiling_enabled
query["_profile"] = "true"
end
-
+
header = {"Accept" => "application/json"}
- profile_checkpoint { "Prepare request #{url} #{query[:uuid]} #{query[:where]}" }
- msg = @@api_client.post(url,
+ profile_checkpoint { "Prepare request #{url} #{query[:uuid]} #{query[:where]} #{query[:filters]}" }
+ msg = @@api_client.post(url,
query,
header: header)
profile_checkpoint 'API transaction'
end
json = msg.content
-
+
begin
resp = Oj.load(json, :symbol_keys => true)
rescue Oj::ParseError
resp
end
- def self.patch_paging_vars(ary, items_available, offset, limit)
+ def self.patch_paging_vars(ary, items_available, offset, limit, links=nil)
if items_available
(class << ary; self; end).class_eval { attr_accessor :items_available }
ary.items_available = items_available
if limit
(class << ary; self; end).class_eval { attr_accessor :limit }
ary.limit = limit
- end
+ end
+ if links
+ (class << ary; self; end).class_eval { attr_accessor :links }
+ ary.links = links
+ end
ary
end
def unpack_api_response(j, kind=nil)
if j.is_a? Hash and j[:items].is_a? Array and j[:kind].match(/(_list|List)$/)
ary = j[:items].collect { |x| unpack_api_response x, x[:kind] }
- self.class.patch_paging_vars(ary, j[:items_available], j[:offset], j[:limit])
+ links = ArvadosResourceList.new Link
+ links.results = (j[:links] || []).collect do |x|
+ unpack_api_response x, x[:kind]
+ end
+ self.class.patch_paging_vars(ary, j[:items_available], j[:offset], j[:limit], links)
elsif j.is_a? Hash and (kind || j[:kind])
oclass = self.kind_class(kind || j[:kind])
if oclass
super(*args)
@attribute_sortkey ||= {
'id' => nil,
- 'uuid' => '000',
- 'owner_uuid' => '001',
- 'created_at' => '002',
- 'modified_at' => '003',
- 'modified_by_user_uuid' => '004',
- 'modified_by_client_uuid' => '005',
- 'name' => '050',
- 'tail_uuid' => '100',
- 'head_uuid' => '101',
- 'info' => 'zzz-000',
- 'updated_at' => 'zzz-999'
+ 'name' => '000',
+ 'owner_uuid' => '002',
+ 'event_type' => '100',
+ 'link_class' => '100',
+ 'group_class' => '100',
+ 'tail_uuid' => '101',
+ 'head_uuid' => '102',
+ 'object_uuid' => '102',
+ 'summary' => '104',
+ 'description' => '104',
+ 'properties' => '150',
+ 'info' => '150',
+ 'created_at' => '200',
+ 'modified_at' => '201',
+ 'modified_by_user_uuid' => '202',
+ 'modified_by_client_uuid' => '203',
+ 'uuid' => '999',
}
end
raise 'argument to find() must be a uuid string. Acceptable formats: warehouse locator or string with format xxxxx-xxxxx-xxxxxxxxxxxxxxx'
end
+ if self == ArvadosBase
+ # Determine type from uuid and defer to the appropriate subclass.
+ return resource_class_for_uuid(uuid).find(uuid, opts)
+ end
+
# Only do one lookup on the API side per {class, uuid, workbench
# request} unless {cache: false} is given via opts.
cache_key = "request_#{Thread.current.object_id}_#{self.to_s}_#{uuid}"
}
end
+ def class_for_display
+ self.class.to_s
+ end
+
def self.creatable?
current_user
end
def editable?
(current_user and current_user.is_active and
(current_user.is_admin or
- current_user.uuid == self.owner_uuid))
+ current_user.uuid == self.owner_uuid or
+ new_record?))
end
def attribute_editable?(attr)
elsif "uuid owner_uuid".index(attr.to_s) or current_user.is_admin
current_user.is_admin
else
- current_user.uuid == self.owner_uuid or current_user.uuid == self.uuid
+ current_user.uuid == self.owner_uuid or
+ current_user.uuid == self.uuid or
+ new_record?
end
end
(name if self.respond_to? :name) || uuid
end
+ def content_summary
+ self.class_for_display
+ end
+
def selection_label
friendly_link_name
end
class ArvadosResourceList
include Enumerable
- def initialize(resource_class)
+ def initialize resource_class=nil
@resource_class = resource_class
end
self
end
+ def collect
+ results.collect do |m|
+ yield m
+ end
+ end
+
def first
results.first
end
results.offset if results.respond_to? :offset
end
+ def result_links
+ results.links if results.respond_to? :links
+ end
+
+ # Return links provided with API response that point to the
+ # specified object, and have the specified link_class. If link_class
+ # is false or omitted, return all links pointing to the specified
+ # object.
+ def links_for item_or_uuid, link_class=false
+ return [] if !result_links
+ unless @links_for_uuid
+ @links_for_uuid = {}
+ result_links.each do |link|
+ if link.respond_to? :head_uuid
+ @links_for_uuid[link.head_uuid] ||= []
+ @links_for_uuid[link.head_uuid] << link
+ end
+ end
+ end
+ if item_or_uuid.respond_to? :uuid
+ uuid = item_or_uuid.uuid
+ else
+ uuid = item_or_uuid
+ end
+ (@links_for_uuid[uuid] || []).select do |link|
+ link_class == false or link.link_class == link_class
+ end
+ end
+
+ # Note: this arbitrarily chooses one of (possibly) multiple names.
+ def name_for item_or_uuid
+ links_for(item_or_uuid, 'name').first.andand.name
+ end
+
end
class Collection < ArvadosBase
+ include ApplicationHelper
MD5_EMPTY = 'd41d8cd98f00b204e9800998ecf8427e'
!!locator.to_s.match("^#{MD5_EMPTY}(\\+.*)?\$")
end
+ def content_summary
+ human_readable_bytes_html(total_bytes) + " " + super
+ end
+
def total_bytes
if files
tot = 0
class Group < ArvadosBase
- def self.owned_items
- res = $arvados_api_client.api self, "/#{self.uuid}/owned_items", {}
- $arvados_api_client.unpack_api_response(res)
+ def contents params={}
+ res = $arvados_api_client.api self.class, "/#{self.uuid}/contents", {
+ _method: 'GET'
+ }.merge(params)
+ ret = ArvadosResourceList.new
+ ret.results = $arvados_api_client.unpack_api_response(res)
+ ret
+ end
+
+ def class_for_display
+ group_class == 'folder' ? 'Folder' : super
+ end
+
+ def editable?
+ respond_to?(:writable_by) and
+ writable_by and
+ writable_by.index(current_user.uuid)
end
end
def attribute_editable?(attr)
false
end
+
+ def self.creatable?
+ false
+ end
end
class Log < ArvadosBase
attr_accessor :object
+ def self.creatable?
+ # Technically yes, but not worth offering: it will be empty, and
+ # you won't be able to edit it.
+ false
+ end
end
end
def attribute_editable?(attr)
- attr.to_sym == :name || (attr.to_sym == :components and self.active == nil)
+ attr && (attr.to_sym == :name ||
+ (attr.to_sym == :components and (self.state == 'New' || self.state == 'Ready')))
end
def attributes_for_display
end
end
- def owned_items
- res = $arvados_api_client.api self.class, "/#{self.uuid}/owned_items"
- $arvados_api_client.unpack_api_response(res)
- end
-
def full_name
(self.first_name || "") + " " + (self.last_name || "")
end
<% pane_list ||= %w(recent) %>
<% panes = Hash[pane_list.map { |pane|
[pane, render(partial: 'show_' + pane.downcase,
- locals: { comparable: comparable })]
+ locals: { comparable: comparable, objects: @objects })]
}.compact] %>
<ul class="nav nav-tabs">
<% if object.editable? %>
- <%= link_to({action: 'destroy', id: object.uuid}, method: :delete, remote: true, data: {confirm: "You are about to delete #{object.class} #{object.uuid}.\n\nAre you sure?"}) do %>
+ <%= link_to({action: 'destroy', id: object.uuid}, method: :delete, remote: true, data: {confirm: "You are about to delete #{object.class_for_display.downcase} '#{object.friendly_link_name}' (#{object.uuid}).\n\nAre you sure?"}) do %>
<i class="glyphicon glyphicon-trash"></i>
<% end %>
<% end %>
-<% if p.success %>
+<% if p.state == 'Complete' %>
<span class="label label-success">finished</span>
-<% elsif p.success == false %>
+<% elsif p.state == 'Failed' %>
<span class="label label-danger">failed</span>
-<% elsif p.active %>
+<% elsif p.state == 'RunningOnServer' || p.state == 'RunningOnClient' %>
<span class="label label-info">running</span>
<% else %>
<% if (p.components.select do |k,v| v[:job] end).length == 0 %>
<tr>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "uuid", attrvalue: link.uuid } %></td>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "link_class", attrvalue: link.link_class } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "name", attrvalue: link.name } %></td>
+ <td><%= render_editable_attribute link, 'name' %></td>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "properties", attrvalue: link.properties } %></td>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "head_uuid", attrvalue: link.head_uuid } %></td>
</tr>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "uuid", attrvalue: link.uuid } %></td>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "tail_uuid", attrvalue: link.tail_uuid } %></td>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "link_class", attrvalue: link.link_class } %></td>
- <td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "name", attrvalue: link.name } %></td>
+ <td><%= render_editable_attribute link, 'name' %></td>
<td><%= render partial: 'application/arvados_attr_value', locals: { obj: link, attr: "properties", attrvalue: link.properties } %></td>
</tr>
<% end %>
--- /dev/null
+<% htmloptions = {class: ''}.merge(htmloptions || {})
+ htmloptions[:class] += " btn-#{size}" rescue nil %>
+<%= link_to_if_arvados_object object, { link_text: raw('Show <i class="fa fa-fw fa-arrow-circle-right"></i>') }, { class: 'btn btn-default ' + htmloptions[:class] } %>
-<% if @objects.empty? %>
+<% if objects.empty? %>
<br/>
<p style="text-align: center">
- No <%= controller.model_class.to_s.pluralize.underscore.gsub '_', ' ' %> to display.
+ No <%= controller.controller_name.humanize.downcase %> to display.
</p>
<% else %>
-<% attr_blacklist = ' created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at' %>
+<% attr_blacklist = ' created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at owner_uuid group_class' %>
-<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+<%= render partial: "paging", locals: {results: objects, object: @object} %>
<%= form_tag do |f| %>
<thead>
<tr>
<th></th>
- <% @objects.first.attributes_for_display.each do |attr, attrvalue| %>
+ <th></th>
+ <% objects.first.attributes_for_display.each do |attr, attrvalue| %>
<% next if attr_blacklist.index(" "+attr) %>
<th class="arv-attr-<%= attr %>">
<%= controller.model_class.attribute_info[attr.to_sym].andand[:column_heading] or attr.sub /_uuid/, '' %>
</thead>
<tbody>
- <% @objects.each do |object| %>
+ <% objects.each do |object| %>
<tr data-object-uuid="<%= object.uuid %>">
<td>
<%= render :partial => "selection_checkbox", :locals => {:object => object} %>
</td>
+ <td>
+ <%= render :partial => "show_object_button", :locals => {object: object, size: 'xs'} %>
+ </td>
<% object.attributes_for_display.each do |attr, attrvalue| %>
<% next if attr_blacklist.index(" "+attr) %>
<td class="arv-object-<%= object.class.to_s %> arv-attr-<%= attr %>">
<% if attr == 'uuid' %>
- <%= link_to_if_arvados_object object %>
- <%= link_to_if_arvados_object(object, { link_text: raw('<i class="icon-hand-right"></i>') }) %>
+ <span class="arvados-uuid"><%= attrvalue %></span>
<% else %>
<% if object.attribute_editable? attr %>
<%= render_editable_attribute object, attr %>
<% end %>
-<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+<%= render partial: "paging", locals: {results: objects, object: @object} %>
<% end %>
<% content_for :page_title do %>
-<%= controller.model_class.to_s.pluralize.underscore.capitalize.gsub('_', ' ') %>
+<%= controller.controller_name.humanize.capitalize %>
<% end %>
<% content_for :tab_line_buttons do %>
'data-target' => '#user-setup-modal-window', return_to: request.url} %>
<div id="user-setup-modal-window" class="modal fade" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true"></div>
<% else %>
- <%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
- { action: 'create', return_to: request.url },
+ <%= button_to "Add a new #{controller.controller_name.singularize.humanize.downcase}",
+ { action: 'create' },
{ class: 'btn btn-primary pull-right' } %>
<% end %>
--- /dev/null
+<%= render(partial: 'show_recent',
+ locals: { comparable: comparable, objects: @my_folders }) %>
--- /dev/null
+<%= render(partial: 'show_recent',
+ locals: { comparable: comparable, objects: @shared_with_me }) %>
--- /dev/null
+<% @removed_uuids.each do |uuid| %>
+$('[data-object-uuid=<%= uuid %>]').hide('slow', function() {
+ $(this).remove();
+});
+<% end %>
--- /dev/null
+<div class="row row-fill-height">
+ <div class="col-md-6">
+ <div class="panel panel-info">
+ <div class="panel-heading">
+ <h3 class="panel-title">
+ <%= render_editable_attribute @object, 'name', nil, {data: {emptytext: "New folder"}} %>
+ </h3>
+ </div>
+ <div class="panel-body">
+ <img src="/favicon.ico" class="pull-right" alt="" style="opacity: 0.3"/>
+ <%= render_editable_attribute @object, 'description', nil, { 'data-emptytext' => "Created: #{@object.created_at.to_s(:long)}", 'data-toggle' => 'manual', 'id' => "#{@object.uuid}-description" } %>
+ <% if @object.attribute_editable? 'description' %>
+ <div style="margin-top: 1em;">
+ <a href="#" class="btn btn-xs btn-default" data-toggle="x-editable" data-toggle-selector="#<%= @object.uuid %>-description"><i class="fa fa-fw fa-pencil"></i> Edit description</a>
+ </div>
+ <% end %>
+ </div>
+ </div>
+ </div>
+ <div class="col-md-3">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h3 class="panel-title">
+ Activity
+ </h3>
+ </div>
+ <div class="panel-body smaller-text">
+ <!--
+ <input type="text" class="form-control" placeholder="Search"/>
+ -->
+ <div style="height:0.5em;"></div>
+ <% @logs[0..2].each do |log| %>
+ <p>
+ <%= time_ago_in_words(log.event_at) %> ago: <%= log.summary %>
+ <% if log.object_uuid %>
+ <%= link_to_if_arvados_object log.object_uuid, link_text: raw('<i class="fa fa-hand-o-right"></i>') %>
+ <% end %>
+ </p>
+ <% end %>
+ <% if @logs.any? %>
+ <%= link_to raw('Show all activity <i class="fa fa-fw fa-arrow-circle-right"></i>'),
+ logs_path(filters: [['object_uuid','=',@object.uuid]].to_json),
+ class: 'btn btn-xs btn-default' %>
+ <% else %>
+ <p>
+ Created: <%= @object.created_at.to_s(:long) %>
+ </p>
+ <p>
+ Last modified: <%= @object.modified_at.to_s(:long) %> by <%= link_to_if_arvados_object @object.modified_by_user_uuid, friendly_name: true %>
+ </p>
+ <% end %>
+ </div>
+ </div>
+ </div>
+ <div class="col-md-3">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h3 class="panel-title">
+ Sharing and permissions
+ </h3>
+ </div>
+ <div class="panel-body">
+ <!--
+ <input type="text" class="form-control" placeholder="Search"/>
+ -->
+ <div style="height:0.5em;"></div>
+ <p>Owner: <%= link_to_if_arvados_object @object.owner_uuid, friendly_name: true %></p>
+ <% if @share_links.any? %>
+ <p>Shared with:
+ <% @share_links.andand.each do |link| %>
+ <br /><%= link_to_if_arvados_object link.tail_uuid, friendly_name: true %>
+ <% end %>
+ </p>
+ <% end %>
+ </div>
+ </div>
+ </div>
+</div>
+
+<% if @show_cards %>
+<!-- disable cards section until we have bookmarks -->
+<div class="row">
+ <% @objects[0..3].each do |object| %>
+ <div class="card arvados-object">
+ <div class="card-top blue">
+ <a href="#">
+ <img src="/favicon.ico" alt=""/>
+ </a>
+ </div>
+ <div class="card-info">
+ <span class="title"><%= @objects.name_for(object) || object.class_for_display %></span>
+ <div class="desc"><%= object.respond_to?(:description) ? object.description : object.uuid %></div>
+ </div>
+ <div class="card-bottom">
+ <%= render :partial => "show_object_button", :locals => {object: object, htmloptions: {class: 'btn-default btn-block'}} %>
+ </div>
+ </div>
+ <% end %>
+</div>
+<!-- end disabled cards section -->
+<% end %>
+
+<div class="row">
+ <div class="col-md-12">
+ <div class="panel panel-info">
+ <div class="panel-heading">
+ <div class="row">
+ <div class="col-md-6">
+ <h3 class="panel-title" style="vertical-align:middle;">
+ Contents
+ </h3>
+ </div>
+ <div class="col-md-6">
+ <div class="input-group input-group-sm pull-right">
+ <input type="text" class="form-control search-folder-contents" placeholder="Search folder contents"/>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="panel-body">
+ <p>
+ </p>
+ <table class="table table-condensed arv-index">
+ <tbody>
+ <colgroup>
+ <col width="3%" />
+ <col width="8%" />
+ <col width="30%" />
+ <col width="15%" />
+ <col width="15%" />
+ <col width="20%" />
+ <col width="8%" />
+ </colgroup>
+ <% @objects_and_names.each do |object, name_link| %>
+ <tr data-object-uuid="<%= (name_link && name_link.uuid) || object.uuid %>">
+ <td>
+ <%= render :partial => "selection_checkbox", :locals => {object: object} %>
+ </td>
+ <td>
+ <%= render :partial => "show_object_button", :locals => {object: object, size: 'xs'} %>
+ </td>
+ <td>
+ <%= render_editable_attribute name_link, 'name', nil, {data: {emptytext: "Unnamed #{object.class_for_display}"}} %>
+ </td>
+ <td>
+ <%= object.content_summary %>
+ </td>
+ <td title="<%= object.modified_at %>">
+ <span>
+ <%= raw distance_of_time_in_words(object.modified_at, Time.now).sub('about ','~').sub(' ',' ') + ' ago' rescue object.modified_at %>
+ </span>
+ </td>
+ <td class="arvados-uuid">
+ <%= object.uuid %>
+ </td>
+ <td>
+ <% if @object.editable? %>
+ <%= link_to({action: 'remove_item', id: @object.uuid, item_uuid: ((name_link && name_link.uuid) || object.uuid)}, method: :delete, remote: true, data: {confirm: "You are about to remove #{object.class_for_display} #{object.uuid} from this folder.\n\nAre you sure?"}, class: 'btn btn-xs btn-default') do %>
+ Remove <i class="fa fa-fw fa-ban"></i>
+ <% end %>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ <thead>
+ <tr>
+ <th>
+ </th>
+ <th>
+ </th>
+ <th>
+ name
+ </th>
+ <th>
+ type
+ </th>
+ <th>
+ modified
+ </th>
+ <th>
+ uuid
+ </th>
+ <th>
+ </th>
+ </tr>
+ </thead>
+ </table>
+ <p></p>
+ </div>
+ </div>
+ </div>
+</div>
}
<% end %>
+<%= render partial: "paging", locals: {results: objects, object: @object} %>
+
<table class="topalign table">
<thead>
<tr class="contain-align-left">
</thead>
<tbody>
- <% @jobs.sort_by { |j| j[:created_at] }.reverse.each do |j| %>
+ <% @objects.sort_by { |j| j[:created_at] }.reverse.each do |j| %>
<tr class="cell-noborder">
<td>
</div>
</td>
<td>
- <%= link_to_if_arvados_object j.uuid %>
+ <%= link_to_if_arvados_object j %>
</td>
<td>
<%= j.script %>
--- /dev/null
+<% unless @histogram_pretty_date.nil? %>
+ <% content_for :tab_panes do %>
+ <%# We use protocol-relative paths here to avoid browsers refusing to load javascript over http in a page that was loaded over https. %>
+ <%= javascript_include_tag '//cdnjs.cloudflare.com/ajax/libs/raphael/2.1.2/raphael-min.js' %>
+ <%= javascript_include_tag '//cdnjs.cloudflare.com/ajax/libs/morris.js/0.4.3/morris.min.js' %>
+ <script type="text/javascript">
+ $(document).ready(function(){
+ $.renderHistogram(<%= raw @cache_age_histogram.to_json %>);
+ });
+ </script>
+ <div class='graph'>
+ <h3>Cache Age vs. Disk Utilization</h3>
+ <h4>circa <%= @histogram_pretty_date %></h4>
+ <div id='cache-age-vs-disk-histogram'>
+ </div>
+ </div>
+ <% end %>
+<% end %>
+<%= content_for :content_top %>
+<%= content_for :tab_line_buttons %>
+<%= content_for :tab_panes %>
padding-top: 70px; /* 70px to make the container go all the way to the bottom of the navbar */
}
- body > div.container-fluid > div.col-sm-9.col-sm-offset-3 {
- overflow: auto;
- }
-
@media (max-width: 979px) { body { padding-top: 0; } }
.navbar .nav li.nav-separator > span.glyphicon.glyphicon-arrow-right {
padding-top: 1.25em;
}
- @media (min-width: 768px) {
- .left-nav {
- position: fixed;
- }
- }
@media (max-width: 767px) {
.breadcrumbs {
display: none;
}
}
</style>
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
</head>
<body>
-
- <div class="navbar navbar-default navbar-fixed-top">
- <div class="container-fluid">
+ <div id="wrapper">
+ <nav class="navbar navbar-default navbar-fixed-top" role="navigation">
<div class="navbar-header">
- <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#workbench-navbar.navbar-collapse">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<a class="navbar-brand" href="/"><%= Rails.configuration.site_name rescue Rails.application.class.parent_name %></a>
</div>
- <div class="collapse navbar-collapse" id="workbench-navbar">
- <ul class="nav navbar-nav navbar-left breadcrumbs">
- <% if current_user %>
- <% if content_for?(:breadcrumbs) %>
- <%= yield(:breadcrumbs) %>
- <% else %>
- <li class="nav-separator"><span class="glyphicon glyphicon-arrow-right"></span></li>
- <li>
- <%= link_to(
- controller.model_class.to_s.pluralize.underscore.gsub('_', ' '),
- url_for({controller: params[:controller]})) %>
- </li>
- <% if params[:action] != 'index' %>
- <li class="nav-separator">
- <span class="glyphicon glyphicon-arrow-right"></span>
+ <div class="collapse navbar-collapse">
+ <% if current_user.andand.is_active %>
+ <ul class="nav navbar-nav side-nav">
+
+ <li class="<%= 'arvados-nav-active' if params[:action] == 'home' %>">
+ <a href="/"><i class="fa fa-lg fa-dashboard fa-fw"></i> Dashboard</a>
</li>
- <li>
- <%= link_to_if_arvados_object @object %>
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown"><i class="fa fa-lg fa-hand-o-up fa-fw"></i> Help <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><%= link_to raw('<i class="fa fa-book fa-fw"></i> Tutorials and User guide'), "#{Rails.configuration.arvados_docsite}/user", target: "_blank" %></li>
+ <li><%= link_to raw('<i class="fa fa-book fa-fw"></i> API Reference'), "#{Rails.configuration.arvados_docsite}/api", target: "_blank" %></li>
+ <li><%= link_to raw('<i class="fa fa-book fa-fw"></i> SDK Reference'), "#{Rails.configuration.arvados_docsite}/sdk", target: "_blank" %></li>
+ </ul>
</li>
- <li style="padding: 14px 0 14px">
- <%= form_tag do |f| %>
- <%= render :partial => "selection_checkbox", :locals => {:object => @object} %>
- <% end %>
+
+ <li class="dropdown">
+ <a href="/folders" class="dropdown-toggle" data-toggle="dropdown"><i class="fa fa-lg fa-folder-o fa-fw"></i> Folders <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+ <li><%= link_to raw('<i class="fa fa-plus fa-fw"></i> Create new folder'), folders_path, method: :post %></li>
+ <% @my_top_level_folders.call[0..7].each do |folder| %>
+ <li><%= link_to raw('<i class="fa fa-folder-open fa-fw"></i> ') + folder.name, folder_path(folder) %></li>
+ <% end %>
+ <li><a href="/folders">
+ <i class="fa fa-ellipsis-h fa-fw"></i> Show all folders
+ </a></li>
+ </ul>
</li>
- <% end %>
- <% end %>
- <% end %>
- </ul>
-
- <ul class="nav navbar-nav navbar-right">
-
- <li>
- <a><i class="rotating loading glyphicon glyphicon-refresh"></i></a>
- </li>
-
- <% if current_user %>
- <!-- XXX placeholder for this when search is implemented
- <li>
- <form class="navbar-form" role="search">
- <div class="input-group" style="width: 220px">
- <input type="text" class="form-control" placeholder="search">
- <span class="input-group-addon"><span class="glyphicon glyphicon-search"></span></span>
- </div>
- </form>
- </li>
- -->
-
- <li class="dropdown notification-menu">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="collections-menu">
- <span class="glyphicon glyphicon-paperclip"></span>
- <span class="badge" id="persistent-selection-count"></span>
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
- <%= form_tag '/actions' do %>
- <div id="selection-form-content"></div>
- <% end %>
- </ul>
- </li>
-
- <% if current_user.is_active %>
- <li class="dropdown notification-menu">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="notifications-menu">
- <span class="glyphicon glyphicon-envelope"></span>
- <span class="badge badge-alert notification-count"><%= @notification_count %></span>
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" role="menu">
- <% if (@notifications || []).length > 0 %>
- <% @notifications.each_with_index do |n, i| %>
- <% if i > 0 %><li class="divider"></li><% end %>
- <li class="notification"><%= n.call(self) %></li>
- <% end %>
- <% else %>
- <li class="notification empty">No notifications.</li>
+ <li><a href="/collections">
+ <i class="fa fa-lg fa-briefcase fa-fw"></i> Collections (data files)
+ </a></li>
+ <li><a href="/jobs">
+ <i class="fa fa-lg fa-tasks fa-fw"></i> Jobs
+ </a></li>
+ <li><a href="/pipeline_instances">
+ <i class="fa fa-lg fa-tasks fa-fw"></i> Pipeline instances
+ </a></li>
+ <li><a href="/pipeline_templates">
+ <i class="fa fa-lg fa-gears fa-fw"></i> Pipeline templates
+ </a></li>
+ <li> </li>
+ <li><a href="/repositories">
+ <i class="fa fa-lg fa-code-fork fa-fw"></i> Repositories
+ </a></li>
+ <li><a href="/virtual_machines">
+ <i class="fa fa-lg fa-terminal fa-fw"></i> Virtual machines
+ </a></li>
+ <li><a href="/humans">
+ <i class="fa fa-lg fa-male fa-fw"></i> Humans
+ </a></li>
+ <li><a href="/specimens">
+ <i class="fa fa-lg fa-flask fa-fw"></i> Specimens
+ </a></li>
+ <li><a href="/traits">
+ <i class="fa fa-lg fa-clipboard fa-fw"></i> Traits
+ </a></li>
+ <li><a href="/links">
+ <i class="fa fa-lg fa-arrows-h fa-fw"></i> Links
+ </a></li>
+ <% if current_user.andand.is_admin %>
+ <li><a href="/users">
+ <i class="fa fa-lg fa-user fa-fw"></i> Users
+ </a></li>
<% end %>
+ <li><a href="/groups">
+ <i class="fa fa-lg fa-users fa-fw"></i> Groups
+ </a></li>
+ <li><a href="/nodes">
+ <i class="fa fa-lg fa-cogs fa-fw"></i> Compute nodes
+ </a></li>
+ <li><a href="/keep_disks">
+ <i class="fa fa-lg fa-hdd-o fa-fw"></i> Keep disks
+ </a></li>
</ul>
- </li>
<% end %>
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="user-menu">
- <span class="glyphicon glyphicon-user"></span><span class="caret"></span>
- </a>
- <ul class="dropdown-menu" role="menu">
- <li role="presentation" class="dropdown-header"><%= current_user.email %></li>
- <% if current_user.is_active %>
- <li role="presentation" class="divider"></li>
- <li role="presentation"><a href="/authorized_keys" role="menuitem">Manage ssh keys</a></li>
- <li role="presentation"><a href="/api_client_authorizations" role="menuitem">Manage API tokens</a></li>
- <li role="presentation" class="divider"></li>
+ <ul class="nav navbar-nav navbar-left breadcrumbs">
+ <% if current_user %>
+ <% if content_for?(:breadcrumbs) %>
+ <%= yield(:breadcrumbs) %>
+ <% else %>
+ <li class="nav-separator"><span class="glyphicon glyphicon-arrow-right"></span></li>
+ <li>
+ <%= link_to(
+ controller.controller_name.humanize.downcase,
+ url_for({controller: params[:controller]})) %>
+ </li>
+ <% if params[:action] != 'index' %>
+ <li class="nav-separator">
+ <span class="glyphicon glyphicon-arrow-right"></span>
+ </li>
+ <li>
+ <%= link_to_if_arvados_object @object, {friendly_name: true}, {data: {object_uuid: @object.andand.uuid, name: 'name'}} %>
+ </li>
+ <li style="padding: 14px 0 14px">
+ <%= form_tag do |f| %>
+ <%= render :partial => "selection_checkbox", :locals => {:object => @object} %>
+ <% end %>
+ </li>
+ <% end %>
<% end %>
- <li role="presentation"><a href="<%= logout_path %>" role="menuitem">Log out</a></li>
- </ul>
- </li>
- <% else -%>
- <li><a href="<%= $arvados_api_client.arvados_login_url(return_to: root_url) %>">Log in</a></li>
- <% end -%>
- </ul>
- </div><!-- /.navbar-collapse -->
- </div><!-- /.container-fluid -->
- </div>
+ <% end %>
+ </ul>
- <div class="container-fluid">
- <div class="col-sm-9 col-sm-offset-3">
- <div id="content" class="body-content">
- <%= yield %>
- </div>
- </div>
- <div class="col-sm-3 left-nav">
- <div class="arvados-nav-container">
- <% if current_user.andand.is_active %>
- <div class="well">
- <ul class="arvados-nav">
- <li class="<%= 'arvados-nav-active' if params[:action] == 'home' %>">
- <a href="/">Dashboard</a>
+ <ul class="nav navbar-nav navbar-right">
+
+ <li>
+ <a><i class="rotating loading glyphicon glyphicon-refresh"></i></a>
</li>
- <% [['Data', [['collections', 'Collections (data files)'],
- ['humans'],
- ['traits'],
- ['specimens'],
- ['links']]],
- ['Activity', [['pipeline_instances', 'Recent pipeline instances'],
- ['jobs', 'Recent jobs']]],
- ['Compute', [['pipeline_templates'],
- ['repositories', 'Code repositories'],
- ['virtual_machines']]],
- ['System', [['users'],
- ['groups'],
- ['nodes', 'Compute nodes'],
- ['keep_disks']]]].each do |j| %>
- <li><%= j[0] %>
- <ul>
- <% j[1].each do |k| %>
- <% unless k[0] == 'users' and !current_user.andand.is_admin %>
- <li class="<%= 'arvados-nav-active' if (params[:controller] == k[0] && params[:action] != 'home') %>">
- <a href="/<%= k[0] %>">
- <%= if k[1] then k[1] else k[0].capitalize.gsub('_', ' ') end %>
- </a>
- </li>
+ <% if current_user %>
+ <!-- XXX placeholder for this when search is implemented
+ <li>
+ <form class="navbar-form" role="search">
+ <div class="input-group" style="width: 220px">
+ <input type="text" class="form-control" placeholder="search">
+ <span class="input-group-addon"><span class="glyphicon glyphicon-search"></span></span>
+ </div>
+ </form>
+ </li>
+ -->
+
+ <li class="dropdown notification-menu">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="collections-menu">
+ <span class="glyphicon glyphicon-paperclip"></span>
+ <span class="badge" id="persistent-selection-count"></span>
+ <span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
+ <%= form_tag '/actions' do %>
+ <%= hidden_field_tag 'uuid', @object.andand.uuid %>
+ <div id="selection-form-content"></div>
<% end %>
+ </ul>
+ </li>
+
+ <% if current_user.is_active %>
+ <li class="dropdown notification-menu">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="notifications-menu">
+ <span class="glyphicon glyphicon-envelope"></span>
+ <span class="badge badge-alert notification-count"><%= @notification_count %></span>
+ <span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu" role="menu">
+ <% if (@notifications || []).length > 0 %>
+ <% @notifications.each_with_index do |n, i| %>
+ <% if i > 0 %><li class="divider"></li><% end %>
+ <li class="notification"><%= n.call(self) %></li>
+ <% end %>
+ <% else %>
+ <li class="notification empty">No notifications.</li>
<% end %>
- </ul>
- </li>
+ </ul>
+ </li>
<% end %>
- <li>Help
- <ul>
- <li><%= link_to 'Tutorials and User guide', "#{Rails.configuration.arvados_docsite}/user", target: "_blank" %></li>
- <li><%= link_to 'API Reference', "#{Rails.configuration.arvados_docsite}/api", target: "_blank" %></li>
- <li><%= link_to 'SDK Reference', "#{Rails.configuration.arvados_docsite}/sdk", target: "_blank" %></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="user-menu">
+ <span class="glyphicon glyphicon-user"></span><span class="caret"></span>
+ </a>
+ <ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header"><%= current_user.email %></li>
+ <% if current_user.is_active %>
+ <li role="presentation" class="divider"></li>
+ <li role="presentation"><a href="/authorized_keys" role="menuitem"><i class="fa fa-key fa-fw"></i> Manage ssh keys</a></li>
+ <li role="presentation"><a href="/api_client_authorizations" role="menuitem"><i class="fa fa-ticket fa-fw"></i> Manage API tokens</a></li>
+ <li role="presentation" class="divider"></li>
+ <% end %>
+ <li role="presentation"><a href="<%= logout_path %>" role="menuitem"><i class="fa fa-sign-out fa-fw"></i> Log out</a></li>
</ul>
</li>
+ <% else %>
+ <li><a href="<%= $arvados_api_client.arvados_login_url(return_to: root_url) %>">Log in</a></li>
+ <% end %>
</ul>
- </div>
- <% end %>
- </div>
- </div>
+ </div><!-- /.navbar-collapse -->
+ </nav>
+
+ <div id="page-wrapper">
+ <%= yield %>
+ </div>
</div>
+</div>
+
<%= yield :footer_html %>
<%= piwik_tracking_tag %>
<%= javascript_tag do %>
+++ /dev/null
-<%= render :partial => 'application/arvados_object' %>
<%= content_for :content_top do %>
<h2>
- <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => 'Unnamed pipeline', 'data-mode' => 'inline' } %>
+ <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => 'Unnamed pipeline' } %>
</h2>
<% if template %>
<h4>
<% end %>
<% end %>
-<% if @object.active != nil %>
+<% if !@object.state.in? ['New', 'Ready', 'Paused'] %>
<table class="table pipeline-components-table">
<colgroup>
<col style="width: 15%" />
</tfoot>
</table>
-<% if @object.active %>
+<% if @object.state == 'RunningOnServer' || @object.state == 'RunningOnClient' %>
<% content_for :js do %>
setInterval(function(){$('a.refresh').click()}, 15000);
<% end %>
<% content_for :tab_line_buttons do %>
<%= form_tag @object, :method => :put do |f| %>
- <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => false %>
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :state, :value => 'Paused' %>
<%= button_tag "Stop pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
<% end %>
<% end %>
<% else %>
-
- <p>Please set the desired input parameters for the components of this pipeline. Parameters highlighted in red are required.</p>
+ <% if @object.state == 'New' %>
+ <p>Please set the desired input parameters for the components of this pipeline. Parameters highlighted in red are required.</p>
+ <% end %>
<% content_for :tab_line_buttons do %>
<%= form_tag @object, :method => :put do |f| %>
- <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => true %>
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :state, :value => 'RunningOnServer' %>
<%= button_tag "Run pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
<% end %>
<% end %>
- <%= render partial: 'show_components_editable', locals: {editable: true} %>
-
+ <% if @object.state.in? ['New', 'Ready'] %>
+ <%= render partial: 'show_components_editable', locals: {editable: true} %>
+ <% else %>
+ <%= render partial: 'show_components_editable', locals: {editable: false} %>
+ <% end %>
<% end %>
<col width="25%" />
<col width="20%" />
<col width="15%" />
- <col width="20%" />
+ <col width="15%" />
+ <col width="5%" />
</colgroup>
<thead>
<tr class="contain-align-left">
Owner
</th><th>
Age
+ </th><th>
</th>
</tr>
</thead>
<%= link_to_if_arvados_object ob.owner_uuid, friendly_name: true %>
</td><td>
<%= distance_of_time_in_words(ob.created_at, Time.now) %>
+ </td><td>
+ <%= render partial: 'delete_object_button', locals: {object:ob} %>
</td>
</tr>
<tr>
<td style="border-top: 0;" colspan="2">
</td>
- <td style="border-top: 0; opacity: 0.5;" colspan="5">
+ <td style="border-top: 0; opacity: 0.5;" colspan="6">
<% ob.components.each do |cname, c| %>
<% if c[:job] %>
<%= render partial: "job_status_label", locals: {:j => c[:job], :title => cname.to_s } %>
<a href="<%= collection_path(j.log) %>/<%= file[1] %>?disposition=inline&size=<%= file[2] %>">Log</a>
<% end %>
<% end %>
- <% elsif j.respond_to? :log_buffer and j.log_buffer %>
+ <% elsif j.respond_to? :log_buffer and j.log_buffer.is_a? String %>
<% buf = j.log_buffer.strip.split("\n").last %>
<span title="<%= buf %>"><%= buf %></span>
<% end %>
post 'set_persistent', on: :member
end
get '/collections/:uuid/*file' => 'collections#show_file', :format => false
+ resources :folders do
+ match 'remove/:item_uuid', on: :member, via: :delete, action: :remove_item
+ end
post 'actions' => 'actions#post'
get 'websockets' => 'websocket#index'
require 'test_helper'
class CollectionsControllerTest < ActionController::TestCase
+ def collection_params(collection_name, file_name=nil)
+ uuid = api_fixture('collections')[collection_name.to_s]['uuid']
+ params = {uuid: uuid, id: uuid}
+ params[:file] = file_name if file_name
+ params
+ end
+
+ def expected_contents(params, token)
+ unless token.is_a? String
+ token = params[:api_token] || token[:arvados_api_token]
+ end
+ [token, params[:uuid], params[:file]].join('/')
+ end
+
+ def assert_hash_includes(actual_hash, expected_hash, msg=nil)
+ expected_hash.each do |key, value|
+ assert_equal(value, actual_hash[key], msg)
+ end
+ end
+
+ def assert_no_session
+ assert_hash_includes(session, {arvados_api_token: nil},
+ "session includes unexpected API token")
+ end
+
+ def assert_session_for_auth(client_auth)
+ api_token =
+ api_fixture('api_client_authorizations')[client_auth.to_s]['api_token']
+ assert_hash_includes(session, {arvados_api_token: api_token},
+ "session token does not belong to #{client_auth}")
+ end
+
+ # Mock the collection file reader to avoid external calls and return
+ # a predictable string.
+ CollectionsController.class_eval do
+ def file_enumerator(opts)
+ [[opts[:arvados_api_token], opts[:uuid], opts[:file]].join('/')]
+ end
+ end
+
+ test "viewing a collection" do
+ params = collection_params(:foo_file)
+ sess = session_for(:active)
+ get(:show, params, sess)
+ assert_response :success
+ assert_equal([['.', 'foo', 3]], assigns(:object).files)
+ end
+
+ test "viewing a collection with a reader token" do
+ params = collection_params(:foo_file)
+ params[:reader_tokens] =
+ [api_fixture('api_client_authorizations')['active']['api_token']]
+ get(:show, params)
+ assert_response :success
+ assert_equal([['.', 'foo', 3]], assigns(:object).files)
+ assert_no_session
+ end
+
+ test "viewing the index with a reader token" do
+ params = {reader_tokens:
+ [api_fixture('api_client_authorizations')['spectator']['api_token']]
+ }
+ get(:index, params)
+ assert_response :success
+ assert_no_session
+ listed_collections = assigns(:collections).map { |c| c.uuid }
+ assert_includes(listed_collections,
+ api_fixture('collections')['bar_file']['uuid'],
+ "spectator reader token didn't list bar file")
+ refute_includes(listed_collections,
+ api_fixture('collections')['foo_file']['uuid'],
+ "spectator reader token listed foo file")
+ end
+
+ test "getting a file from Keep" do
+ params = collection_params(:foo_file, 'foo')
+ sess = session_for(:active)
+ get(:show_file, params, sess)
+ assert_response :success
+ assert_equal(expected_contents(params, sess), @response.body,
+ "failed to get a correct file from Keep")
+ end
+
+ test "can't get a file from Keep without permission" do
+ params = collection_params(:foo_file, 'foo')
+ sess = session_for(:spectator)
+ get(:show_file, params, sess)
+ assert_includes([403, 404], @response.code.to_i)
+ end
+
+ test "trying to get a nonexistent file from Keep returns a 404" do
+ params = collection_params(:foo_file, 'gone')
+ sess = session_for(:admin)
+ get(:show_file, params, sess)
+ assert_response 404
+ end
+
+ test "getting a file from Keep with a good reader token" do
+ params = collection_params(:foo_file, 'foo')
+ read_token = api_fixture('api_client_authorizations')['active']['api_token']
+ params[:reader_tokens] = [read_token]
+ get(:show_file, params)
+ assert_response :success
+ assert_equal(expected_contents(params, read_token), @response.body,
+ "failed to get a correct file from Keep using a reader token")
+ assert_not_equal(read_token, session[:arvados_api_token],
+ "using a reader token set the session's API token")
+ end
+
+ test "trying to get from Keep with an unscoped reader token prompts login" do
+ params = collection_params(:foo_file, 'foo')
+ read_token =
+ api_fixture('api_client_authorizations')['active_noscope']['api_token']
+ params[:reader_tokens] = [read_token]
+ get(:show_file, params)
+ assert_response :redirect
+ end
+
+ test "can get a file with an unpermissioned auth but in-scope reader token" do
+ params = collection_params(:foo_file, 'foo')
+ sess = session_for(:expired)
+ read_token = api_fixture('api_client_authorizations')['active']['api_token']
+ params[:reader_tokens] = [read_token]
+ get(:show_file, params, sess)
+ assert_response :success
+ assert_equal(expected_contents(params, read_token), @response.body,
+ "failed to get a correct file from Keep using a reader token")
+ assert_not_equal(read_token, session[:arvados_api_token],
+ "using a reader token set the session's API token")
+ end
end
--- /dev/null
+require 'test_helper'
+
+class FoldersControllerTest < ActionController::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
change_persist 'persistent', 'cache'
end
+ test "Collection page renders default name links" do
+ uuid = api_fixture('collections')['foo_file']['uuid']
+ coll_name = api_fixture('links')['foo_collection_name_in_afolder']['name']
+ visit page_with_token('active', "/collections/#{uuid}")
+ assert(page.has_text?(coll_name), "Collection page did not include name")
+ # Now check that the page is otherwise normal, and the collection name
+ # isn't only showing up in an error message.
+ assert(page.has_link?('foo'), "Collection page did not include file link")
+ end
end
--- /dev/null
+require 'integration_helper'
+require 'selenium-webdriver'
+require 'headless'
+
+class FoldersTest < ActionDispatch::IntegrationTest
+
+ test 'Find a folder and edit its description' do
+ Capybara.current_driver = Capybara.javascript_driver
+ visit page_with_token 'active', '/'
+ find('nav a', text: 'Folders').click
+ find('.side-nav a,button', text: 'A Folder').
+ click
+ within('.panel', text: api_fixture('groups')['afolder']['name']) do
+ find('span', text: api_fixture('groups')['afolder']['name']).click
+ find('.glyphicon-ok').click
+ find('.btn', text: 'Edit description').click
+ find('.editable-input textarea').set('I just edited this.')
+ find('.editable-submit').click
+ wait_for_ajax
+ end
+ visit current_path
+ assert(find?('.panel', text: 'I just edited this.'),
+ "Description update did not survive page refresh")
+ end
+
+ test 'Add a new name, then edit it, without creating a duplicate' do
+ Capybara.current_driver = Capybara.javascript_driver
+ folder_uuid = api_fixture('groups')['afolder']['uuid']
+ specimen_uuid = api_fixture('specimens')['owned_by_afolder_with_no_name_link']['uuid']
+ visit page_with_token 'active', '/folders/' + folder_uuid
+ within('.panel tr', text: specimen_uuid) do
+ find(".editable[data-name='name']").click
+ find('.editable-input input').set('Now I have a name.')
+ find('.glyphicon-ok').click
+ find('.editable', text: 'Now I have a name.').click
+ find('.editable-input input').set('Now I have a new name.')
+ find('.glyphicon-ok').click
+ wait_for_ajax
+ find('.editable', text: 'Now I have a new name.')
+ end
+ visit current_path
+ within '.panel', text: 'Contents' do
+ find '.editable', text: 'Now I have a new name.'
+ page.assert_no_selector '.editable', text: 'Now I have a name.'
+ end
+ end
+
+end
visit page_with_token('active_trustedclient', '/')
assert_visit_success
click_link 'user-menu'
- urls = [all_links_in('.arvados-nav'),
+ urls = [all_links_in('nav'),
all_links_in('.navbar', /^Manage /)].flatten
seen_urls = ['/']
while not (url = urls.shift).nil?
assert (text.include? 'true false'), 'Expected is_active'
end
- click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+ find('tr', text: 'zzzzz-tpzed-xurymjxw79nv3jz').
+ find('a,button', text: 'Show').
+ click
assert page.has_text? 'Attributes'
assert page.has_text? 'Metadata'
assert page.has_text? 'Admin'
# go to the Attributes tab
click_link 'Attributes'
assert page.has_text? 'modified_by_user_uuid'
- page.within(:xpath, '//a[@data-name="is_active"]') do
+ page.within(:xpath, '//span[@data-name="is_active"]') do
assert_equal "true", text, "Expected user's is_active to be true"
end
- page.within(:xpath, '//a[@data-name="is_admin"]') do
+ page.within(:xpath, '//span[@data-name="is_admin"]') do
assert_equal "false", text, "Expected user's is_admin to be false"
end
click_button "Submit"
end
- sleep(0.1)
-
- # verify that the new user showed up in the users page
- assert page.has_text? 'foo@example.com'
-
- new_user_uuid = nil
- all("tr").each do |elem|
- if elem.text.include? 'foo@example.com'
- new_user_uuid = elem.text.split[0]
- break
- end
- end
+ visit '/users'
+ # verify that the new user showed up in the users page and find
+ # the new user's UUID
+ new_user_uuid =
+ find('tr[data-object-uuid]', text: 'foo@example.com').
+ find('td', text: '-tpzed-').
+ text
assert new_user_uuid, "Expected new user uuid not found"
# go to the new user's page
- click_link new_user_uuid
+ find('tr', text: new_user_uuid).
+ find('a,button', text: 'Show').
+ click
assert page.has_text? 'modified_by_user_uuid'
- page.within(:xpath, '//a[@data-name="is_active"]') do
+ page.within(:xpath, '//span[@data-name="is_active"]') do
assert_equal "false", text, "Expected new user's is_active to be false"
end
click_link 'Users'
- assert page.has_link? 'zzzzz-tpzed-xurymjxw79nv3jz'
-
# click on active user
- click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+ find('tr', text: 'zzzzz-tpzed-xurymjxw79nv3jz').
+ find('a,button', text: 'Show').
+ click
# Setup user
click_link 'Admin'
click_link 'Users'
- assert page.has_link? 'zzzzz-tpzed-xurymjxw79nv3jz'
-
# click on active user
- click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+ find('tr', text: 'zzzzz-tpzed-xurymjxw79nv3jz').
+ find('a,button', text: 'Show').
+ click
# Verify that is_active is set
- click_link 'Attributes'
+ find('a,button', text: 'Attributes').click
assert page.has_text? 'modified_by_user_uuid'
- page.within(:xpath, '//a[@data-name="is_active"]') do
+ page.within(:xpath, '//span[@data-name="is_active"]') do
assert_equal "true", text, "Expected user's is_active to be true"
end
# Should now be back in the Attributes tab for the user
page.driver.browser.switch_to.alert.accept
assert page.has_text? 'modified_by_user_uuid'
- page.within(:xpath, '//a[@data-name="is_active"]') do
+ page.within(:xpath, '//span[@data-name="is_active"]') do
assert_equal "false", text, "Expected user's is_active to be false after unsetup"
end
click_link 'Virtual machines'
assert page.has_text? 'testvm.shell'
click_on 'Add a new virtual machine'
- assert page.has_text? 'none'
- click_link 'none'
+ find('tr', text: 'hostname').
+ find('span', text: 'none').click
assert page.has_text? 'Update hostname'
fill_in 'editable-text', with: 'testname'
click_button 'editable-submit'
require 'uri'
require 'yaml'
+module WaitForAjax
+ Capybara.default_wait_time = 5
+ def wait_for_ajax
+ Timeout.timeout(Capybara.default_wait_time) do
+ loop until finished_all_ajax_requests?
+ end
+ end
+
+ def finished_all_ajax_requests?
+ page.evaluate_script('jQuery.active').zero?
+ end
+end
+
class ActionDispatch::IntegrationTest
# Make the Capybara DSL available in all integration tests
include Capybara::DSL
include ApiFixtureLoader
+ include WaitForAjax
@@API_AUTHS = self.api_fixture('api_client_authorizations')
q_string = URI.encode_www_form('api_token' => api_token)
"#{path}#{sep}#{q_string}"
end
+
+ # Find a page element, but return false instead of raising an
+ # exception if not found. Use this with assertions to explain that
+ # the error signifies a failed test rather than an unexpected error
+ # during a testing procedure.
+ def find? *args
+ begin
+ find *args
+ rescue Capybara::ElementNotFound
+ false
+ end
+ end
end
ENV["RAILS_ENV"] = "test"
+unless ENV["NO_COVERAGE_TEST"]
+ begin
+ require 'simplecov'
+ require 'simplecov-rcov'
+ class SimpleCov::Formatter::MergedFormatter
+ def format(result)
+ SimpleCov::Formatter::HTMLFormatter.new.format(result)
+ SimpleCov::Formatter::RcovFormatter.new.format(result)
+ end
+ end
+ SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
+ SimpleCov.start do
+ add_filter '/test/'
+ add_filter 'initializers/secret_token'
+ end
+ rescue Exception => e
+ $stderr.puts "SimpleCov unavailable (#{e}). Proceeding without."
+ end
+end
+
require File.expand_path('../../config/environment', __FILE__)
require 'rails/test_help'
end
class ApiServerBackedTestRunner < MiniTest::Unit
- # Make a hash that unsets Bundle's environment variables.
- # We'll use this environment when we launch Bundle commands in the API
- # server. Otherwise, those commands will try to use Workbench's gems, etc.
- @@APIENV = Hash[ENV.map { |key, val|
- (key =~ /^BUNDLE_/) ? [key, nil] : nil
- }.compact]
-
def _system(*cmd)
- if not system(@@APIENV, *cmd)
- raise RuntimeError, "#{cmd[0]} returned exit code #{$?.exitstatus}"
+ Bundler.with_clean_env do
+ if not system({'RAILS_ENV' => 'test'}, *cmd)
+ raise RuntimeError, "#{cmd[0]} returned exit code #{$?.exitstatus}"
+ end
end
end
def _run(args=[])
Capybara.javascript_driver = :poltergeist
server_pid = Dir.chdir($ARV_API_SERVER_DIR) do |apidir|
+ ENV["NO_COVERAGE_TEST"] = "1"
_system('bundle', 'exec', 'rake', 'db:test:load')
_system('bundle', 'exec', 'rake', 'db:fixtures:load')
_system('bundle', 'exec', 'rails', 'server', '-d')
timeout = Time.now.tv_sec + 10
- begin
+ good_pid = false
+ while (not good_pid) and (Time.now.tv_sec < timeout)
sleep 0.2
begin
server_pid = IO.read(SERVER_PID_PATH).to_i
- good_pid = (server_pid > 0) and (Process.kill(0, pid) rescue false)
+ good_pid = (server_pid > 0) and (Process.kill(0, server_pid) rescue false)
rescue Errno::ENOENT
good_pid = false
end
- end while (not good_pid) and (Time.now.tv_sec < timeout)
+ end
if not good_pid
raise RuntimeError, "could not find API server Rails pid"
end
--- /dev/null
+require 'test_helper'
+
+class ResourceListTest < ActiveSupport::TestCase
+
+ test 'links_for on a resource list that does not return links' do
+ use_token :active
+ results = Specimen.all
+ assert_equal [], results.links_for(api_fixture('users')['active']['uuid'])
+ end
+
+ test 'links_for on non-empty resource list' do
+ use_token :active
+ results = Group.find(api_fixture('groups')['afolder']['uuid']).contents(include_linked: true)
+ assert_equal [], results.links_for(api_fixture('users')['active']['uuid'])
+ assert_equal [], results.links_for(api_fixture('jobs')['running_cancelled']['uuid'])
+ assert_equal [], results.links_for(api_fixture('jobs')['running']['uuid'], 'bogus-link-class')
+ assert_equal true, results.links_for(api_fixture('jobs')['running']['uuid'], 'name').any?
+ end
+
+ test 'links_for returns all link classes (simulated results)' do
+ folder_uuid = api_fixture('groups')['afolder']['uuid']
+ specimen_uuid = api_fixture('specimens')['in_afolder']['uuid']
+ api_response = {
+ kind: 'arvados#specimenList',
+ links: [{kind: 'arvados#link',
+ uuid: 'zzzzz-o0j2j-asdfasdfasdfas0',
+ tail_uuid: folder_uuid,
+ head_uuid: specimen_uuid,
+ link_class: 'name',
+ name: 'Alice'},
+ {kind: 'arvados#link',
+ uuid: 'zzzzz-o0j2j-asdfasdfasdfas1',
+ tail_uuid: folder_uuid,
+ head_uuid: specimen_uuid,
+ link_class: 'foo',
+ name: 'Bob'},
+ {kind: 'arvados#link',
+ uuid: 'zzzzz-o0j2j-asdfasdfasdfas2',
+ tail_uuid: folder_uuid,
+ head_uuid: specimen_uuid,
+ link_class: nil,
+ name: 'Clydesdale'}],
+ items: [{kind: 'arvados#specimen',
+ uuid: specimen_uuid}]
+ }
+ arl = ArvadosResourceList.new
+ arl.results = ArvadosApiClient.new.unpack_api_response(api_response)
+ assert_equal(['name', 'foo', nil],
+ (arl.
+ links_for(specimen_uuid).
+ collect { |x| x.link_class }),
+ "Expected links_for to return all link_classes")
+ end
+
+end
require 'test_helper'
-class ProjectTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+class GroupTest < ActiveSupport::TestCase
+ test "get contents with names" do
+ use_token :active
+ oi = Group.
+ find(api_fixture('groups')['asubfolder']['uuid']).
+ contents(include_linked: true)
+ assert_operator(0, :<, oi.count,
+ "Expected to find some items belonging to :active user")
+ assert_operator(0, :<, oi.items_available,
+ "Expected contents response to have items_available > 0")
+ assert_operator(0, :<, oi.result_links.count,
+ "Expected to receive name links with contents response")
+ oi_uuids = oi.collect { |i| i['uuid'] }
+
+ expect_uuid = api_fixture('specimens')['in_asubfolder']['uuid']
+ assert_includes(oi_uuids, expect_uuid,
+ "Expected '#{expect_uuid}' in asubfolder's contents")
+
+ expect_uuid = api_fixture('specimens')['in_afolder_linked_from_asubfolder']['uuid']
+ expect_name = api_fixture('links')['specimen_is_in_two_folders']['name']
+ assert_includes(oi_uuids, expect_uuid,
+ "Expected '#{expect_uuid}' in asubfolder's contents")
+ assert_equal(expect_name, oi.name_for(expect_uuid),
+ "Expected name_for '#{expect_uuid}' to be '#{expect_name}'")
+ end
end
--- /dev/null
+require 'test_helper'
+
+class FoldersHelperTest < ActionView::TestCase
+end
require 'test_helper'
class UserTest < ActiveSupport::TestCase
- test "get owned_items" do
- use_token :active
- oi = User.find(api_fixture('users')['active']['uuid']).owned_items
- assert_operator(0, :<, oi.count,
- "Expected to find some items belonging to :active user")
- assert_operator(0, :<, oi.items_available,
- "Expected owned_items response to have items_available > 0")
- oi_uuids = oi.collect { |i| i['uuid'] }
- expect = api_fixture('specimens')['owned_by_active_user']['uuid']
- assert_includes(oi_uuids, expect,
- "Expected active user's owned_items to include #{expect}")
- end
end
- sdk/perl/index.html.textile.liquid
- Ruby:
- sdk/ruby/index.html.textile.liquid
+ - Java:
+ - sdk/java/index.html.textile.liquid
- CLI:
- sdk/cli/index.html.textile.liquid
api:
- admin/cheat_sheet.html.textile.liquid
installguide:
- Install:
- - install/index.html.md.liquid
+ - install/index.html.textile.liquid
- install/install-sso.html.textile.liquid
- install/install-api-server.html.textile.liquid
- install/install-workbench-app.html.textile.liquid
- - install/client.html.textile.liquid
- install/create-standard-objects.html.textile.liquid
- install/install-crunch-dispatch.html.textile.liquid
These resources govern the Arvados infrastructure itself: Git repositories, Keep disks, active nodes, etc.
-* "CommitAncestor":schema/CommitAncestor.html
-* "Commit":schema/Commit.html
* "KeepDisk":schema/KeepDisk.html
* "Node":schema/Node.html
* "Repository":schema/Repository.html
table(table table-bordered table-condensed).
|*Parameter name*|*Value*|*Description*|
-|limit |integer|Maximum number of resources to return|
-|offset |integer|Skip the first 'offset' objects|
-|filters |array |Conditions for selecting resources to return|
-|order |array |List of fields to use to determine sorting order for returned objects|
-|select |array |Specify which fields to return|
-|distinct|boolean|true: (default) do not return duplicate objects<br> false: permitted to return duplicates|
+|limit |integer|Maximum number of resources to return.|
+|offset |integer|Skip the first 'offset' resources that match the given filter conditions.|
+|filters |array |Conditions for selecting resources to return (see below).|
+|order |array |Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order.
+Example: @["head_uuid asc","modified_at desc"]@
+Default: @["created_at desc"]@|
+|select |array |Set of attributes to include in the response.
+Example: @["head_uuid","tail_uuid"]@
+Default: all available attributes, minus "manifest_text" in the case of collections.|
+|distinct|boolean|@true@: (default) do not return duplicate objects
+@false@: permitted to return duplicates|
+
+h3. Filters
+
+The value of the @filters@ parameter is an array of conditions. The @list@ method returns only the resources that satisfy all of the given conditions. In other words, the conjunction @AND@ is implicit.
+
+Each condition is expressed as an array with three elements: @[attribute, operator, operand]@.
+
+table(table table-bordered table-condensed).
+|_. Index|_. Element|_. Type|_. Description|_. Examples|
+|0|attribute|string|Name of the attribute to compare|@script_version@, @head_uuid@|
+|1|operator|string|Comparison operator|@>@, @>=@, @like@, @not in@|
+|2|operand|string, array, or null|Value to compare with the resource attribute|@"d00220fb%"@, @"1234"@, @["foo","bar"]@, @nil@|
+
+The following operators are available.
+
+table(table table-bordered table-condensed).
+|_. Operator|_. Operand type|_. Example|
+|@<@, @<=@, @>=@, @>@, @like@|string|@["script_version","like","d00220fb%"]@|
+|@=@, @!=@|string or null|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@
+@["tail_uuid","!=",null]@|
+|@in@, @not in@|array of strings|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@is_a@|string|@["head_uuid","is_a","arvados#pipelineInstance"]@|
h2. Create
Required arguments are displayed in %{background:#ccffcc}green%.
+h2. contents
+
+Retrieve a list of items which are associated with the given group by ownership (i.e., the group owns the item) or a "name" link (i.e., a "name" link referencing the item).
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+{background:#ccffcc}.|uuid|string|The UUID of the group in question.|path||
+|include_linked|boolean|If false, results will only include items whose @owner_uuid@ attribute is the specified group. If true, results will additionally include items for which a "name" link exists.|path|{white-space:nowrap}. @false@ (default)
+@true@|
+
+If @include_linked@ is @true@, the @"links"@ field in the response will contain the "name" links referencing the objects in the @"items"@ field.
+
h2. create
Create a new Group.
|order|string|Order in which to return matching groups.|query||
|filters|array|Conditions for filtering groups.|query||
-h2. owned_items
-
-Retrieve a list of items which are owned by the given group.
-
-Arguments:
-
-table(table table-bordered table-condensed).
-|_. Argument |_. Type |_. Description |_. Location |_. Example |
-{background:#ccffcc}.|uuid|string|The UUID of the group in question.|path||
-|include_linked|boolean|If true, results will also include items on which the given group has _can_manage_ permission, even if they are owned by different users/groups.|path|{white-space:nowrap}. @false@ (default)
-@true@|
-
h2. show
show groups
h2. create
-Create a new Log.
+Create a new log entry.
Arguments:
h2. delete
-Delete an existing Log.
+Delete an existing log entry. This method can only be used by privileged (system administrator) users.
Arguments:
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
-{background:#ccffcc}.|uuid|string|The UUID of the Log in question.|path||
+{background:#ccffcc}.|uuid|string|The UUID of the log entry in question.|path||
h2. get
-Gets a Log's metadata by UUID.
+Retrieve a log entry.
Arguments:
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
-{background:#ccffcc}.|uuid|string|The UUID of the Log in question.|path||
+{background:#ccffcc}.|uuid|string|The UUID of the log entry in question.|path||
h2. list
-List logs.
+List log entries.
Arguments:
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
-|limit|integer (default 100)|Maximum number of logs to return.|query||
-|order|string|Order in which to return matching logs.|query||
-|filters|array|Conditions for filtering logs.|query||
+|limit|integer (default 100)|Maximum number of log entries to return.|query||
+|order|string|Order in which to return matching log entries.|query||
+|filters|array|Conditions for filtering log entries.|query||
h2. update
-Update attributes of an existing Log.
+Update attributes of an existing log entry. This method can only be used by privileged (system administrator) users.
Arguments:
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
-{background:#ccffcc}.|uuid|string|The UUID of the Log in question.|path||
+{background:#ccffcc}.|uuid|string|The UUID of the log entry in question.|path||
|log|object||query||
|order|string|Order in which to return matching users.|query||
|filters|array|Conditions for filtering users.|query||
-h2. owned_items
-
-Retrieve a list of items which are owned by the given user.
-
-Arguments:
-
-table(table table-bordered table-condensed).
-|_. Argument |_. Type |_. Description |_. Location |_. Example |
-{background:#ccffcc}.|uuid|string|The UUID of the user in question.|path||
-|include_linked|boolean|If true, results will also include items on which the given user has _can_manage_ permission, even if they are owned by different users/groups.|path|{white-space:nowrap}. @false@ (default)
-@true@|
-
h2. show
show users
h3. Side effects of creating a Collection
-Referenced data can be protected from garbage collection. See the section about "resources" links on the "Links":Links.html page.
+Referenced data can be protected from garbage collection. See the section about "resources" links on the "Links":Link.html page.
Data can be shared with other users via the Arvados permission model.
table(table table-bordered table-condensed).
|_. Key|_. Type|_. Description|_. Implemented|
+|docker_image|string|The name of a Docker image that this Job needs to run. If specified, Crunch will create a Docker container from this image, and run the Job's script inside that. The Keep mount and work directories will be available as volumes inside this container. You may specify the image in any format that Docker accepts, such as "arvados/jobs" or a hash identifier. If you specify a name, Crunch will try to install the latest version using @docker.io pull@.|✓|
|min_nodes|integer||✓|
|max_nodes|integer|||
|max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓|
...
-
-
-h3. Python
-
-{% include 'notebox_begin' %}
-The Python package includes the Python API client library module and the CLI utilities @arv-get@ and @arv-put@.
-{% include 'notebox_end' %}
-
-Get the arvados source tree.
-
-notextile. <pre><code>$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span></code></pre>
-
-Build and install the python package.
-
-<notextile>
-<pre><code>$ <span class="userinput">cd arvados/sdk/python</span>
-$ <span class="userinput">sudo python setup.py install</span>
-</code></pre>
-</notextile>
-
-Alternatively, build the package (without sudo) using @python setup.py bdist_egg@ and copy the @.egg@ package from @dist/@ to the target system.
-
-h3. Ruby
-
-{% include 'notebox_begin' %}
-The arvados package includes the Ruby client library module. The arvados-cli package includes the CLI utilities @arv@, @arv-run-pipeline-instance@, and @crunch-job@.
-{% include 'notebox_end' %}
-
-notextile. <pre><code>$ <span class="userinput">sudo gem install arvados arvados-cli</span></code></pre>
-
-h3. Perl
-
-{% include 'notebox_begin' %}
-The Perl client library includes the @Arvados.pm@ module and submodules.
-{% include 'notebox_end' %}
-
-<notextile>
-<pre><code>$ <span class="userinput">cd arvados/sdk/perl</span>
-$ <span class="userinput">perl Makefile.PL</span>
-$ <span class="userinput">sudo make install</span>
-</code></pre>
-</notextile>
+The "SDK Reference":{{site.baseurl}}/sdk/index.html page has installation instructions for each of the SDKs.
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Overview
-...
-
-{% include 'alert_stub' %}
-
-# Installation Overview
-
-1. Set up a cluster, or use Amazon
-1. Create and mount Keep volumes
-1. [Install the Single Sign On (SSO) server](install-sso.html)
-1. [Install the Arvados REST API server](install-api-server.html)
-1. [Install the Arvados workbench application](install-workbench-app.html)
-1. [Install the Crunch dispatcher](install-crunch-dispatch.html)
-1. [Create standard objects](create-standard-objects.html)
-1. [Install client libraries](client.html)
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Overview
+...
+
+{% include 'alert_stub' %}
+
+h2. Installation Overview
+
+# Set up a cluster, or use Amazon
+# Create and mount Keep volumes
+# "Install the Single Sign On (SSO) server":install-sso.html
+# "Install the Arvados REST API server":install-api-server.html
+# "Install the Arvados workbench application":install-workbench-app.html
+# "Install the Crunch dispatcher":install-crunch-dispatch.html
+# "Create standard objects":create-standard-objects.html
+# Install client libraries (see "SDK Reference":{{site.baseurl}}/sdk/index.html).
h2. Download the source tree
<notextile>
-<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
+<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
+~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
</code></pre></notextile>
See also: "Downloading the source code":https://arvados.org/projects/arvados/wiki/Download on the Arvados wiki.
</code></pre>
</notextile>
-h2. Add an admin user
+h2(#admin-user). Add an admin user
Point your browser to the API server's login endpoint:
h4. Perl SDK dependencies
-* @apt-get install libjson-perl libwww-perl libio-socket-ssl-perl libipc-system-simple-perl@
+Install the Perl SDK on the controller.
-Add this to @/etc/apt/sources.list@
-
-@deb http://git.oxf.freelogy.org/apt wheezy main contrib@
-
-Then
-
-@apt-get install libwarehouse-perl@
+* See "Perl SDK":{{site.baseurl}}/sdk/perl/index.html page for details.
h4. Python SDK dependencies
-On controller and all compute nodes:
+Install the Python SDK and CLI tools on controller and all compute nodes.
-* @apt-get install python-pip@
-* @pip install --upgrade virtualenv arvados-python-client@
+* See "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html page for details.
h4. Likely crunch job dependencies
h4. Repositories
-Crunch scripts must be in Git repositories in @/var/cache/git/*/.git@ (or whatever is configured in @services/api/config/environments/production.rb@).
-
-h4. Importing commits
-
-@services/api/script/import_commits.rb production@ must run periodically. Example @/var/service/arvados_import_commits/run@ script for daemontools or runit:
-
-<pre>
-#!/bin/sh
-set -e
-while sleep 60
-do
- cd /path/to/arvados/services/api
- setuidgid www-data env RAILS_ENV=production /usr/local/rvm/bin/rvm-exec 2.0.0 bundle exec ./script/import_commits.rb 2>&1
-done
-</pre>
+Crunch scripts must be in Git repositories in @/var/lib/arvados/git/*.git@ (or whatever is configured in @services/api/config/environments/production.rb@).
-Once you have imported some commits, you should be able to create a new job:
+Once you have a repository with commits -- and you have read access to the repository -- you should be able to create a new job:
<pre>
read -rd $'\000' newjob <<EOF; arv job create --job "$newjob"
{"script_parameters":{"input":"f815ec01d5d2f11cb12874ab2ed50daa"},
"script_version":"master",
- "script":"hash"}
+ "script":"hash",
+ "repository":"arvados"}
EOF
</pre>
<pre>
#!/bin/sh
set -e
+
+rvmexec=""
+## uncomment this line if you use rvm:
+#rvmexec="/usr/local/rvm/bin/rvm-exec 2.1.1"
+
export PATH="$PATH":/path/to/arvados/services/crunch
-export PERLLIB=/path/to/arvados/sdk/perl/lib:/path/to/warehouse-apps/libwarehouse-perl/lib
export ARVADOS_API_HOST={{ site.arvados_api_host }}
export CRUNCH_DISPATCH_LOCKFILE=/var/lock/crunch-dispatch
cd /path/to/arvados/services/api
export RAILS_ENV=production
-exec /usr/local/rvm/bin/rvm-exec 2.0.0 bundle exec ./script/crunch-dispatch.rb 2>&1
+exec $rvmexec bundle exec ./script/crunch-dispatch.rb 2>&1
</pre>
...
<notextile>
-<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/sso-devise-omniauth-provider.git</span>
+<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
+~$ <span class="userinput">git clone https://github.com/curoverse/sso-devise-omniauth-provider.git</span>
~$ <span class="userinput">cd sso-devise-omniauth-provider</span>
~/sso-devise-omniauth-provider$ <span class="userinput">bundle install</span>
~/sso-devise-omniauth-provider$ <span class="userinput">rake db:create</span>
h2. Download the source tree
-Please follow the instructions on the "Download page":https://arvados.org/projects/arvados/wiki/Download in the wiki.
+<notextile>
+<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
+~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
+</code></pre></notextile>
+
+See also: "Downloading the source code":https://arvados.org/projects/arvados/wiki/Download on the Arvados wiki.
The Workbench application is in @apps/workbench@ in the source tree.
* Set @secret_token@ to the string you generated with @rake secret@.
* Point @arvados_login_base@ and @arvados_v1_base@ at your "API server":install-api-server.html
* @site_name@ can be any string to identify this Workbench.
-* Assuming that the SSL certificate you use for development isn't signed by a CA, make sure @arvados_insecure_https@ is @true@.
+* If the SSL certificate you use for development isn't signed by a CA, make sure @arvados_insecure_https@ is @true@.
Copy @config/piwik.yml.example@ to @config/piwik.yml@ and edit to suit.
-h3. Apache/Passenger (optional)
+h2. Start a standalone server
-Set up Apache and Passenger. Point them to the apps/workbench directory in the source tree.
+For testing and development, the easiest way to get started is to run the web server that comes with Rails.
+
+<notextile>
+<pre><code>~/arvados/apps/workbench$ <span class="userinput">bundle exec rails server --port=3031</span>
+</code></pre>
+</notextile>
+
+Point your browser to <notextile><code>http://<b>your.host</b>:3031/</code></notextile>.
h2. Trusted client setting
-Log in to Workbench once (this ensures that the Arvados API server has a record of the Workbench client).
+Log in to Workbench once to ensure that the Arvados API server has a record of the Workbench client. (It's OK if Workbench says your account hasn't been activated yet. We'll deal with that next.)
In the API server project root, start the rails console. Locate the ApiClient record for your Workbench installation (typically, while you're setting this up, the @last@ one in the database is the one you want), then set the @is_trusted@ flag for the appropriate client record:
-<notextile><pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=development bundle exec rails console</span>
+<notextile><pre><code>~/arvados/services/api$ <span class="userinput">bundle exec rails console</span>
irb(main):001:0> <span class="userinput">wb = ApiClient.all.last; [wb.url_prefix, wb.created_at]</span>
=> ["https://workbench.example.com/", Sat, 19 Apr 2014 03:35:12 UTC +00:00]
irb(main):002:0> <span class="userinput">include CurrentApiClient</span>
=> true
</code></pre>
</notextile>
+
+h2. Activate your own account
+
+Unless you already activated your account when installing the API server, the first time you log in to Workbench you will see a message that your account is awaiting activation.
+
+Activate your own account and give yourself administrator privileges by following the instructions in the "'Add an admin user' section of the API server install page":install-api-server.html#admin-user.
* "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
* "Perl SDK":{{site.baseurl}}/sdk/perl/index.html
* "Ruby SDK":{{site.baseurl}}/sdk/ruby/index.html
+* "Java SDK":{{site.baseurl}}/sdk/java/index.html
* "Command line SDK":{{site.baseurl}}/sdk/cli/index.html ("arv")
SDKs not yet implemented:
* Rails SDK: Workbench uses an ActiveRecord-like interface to Arvados. This hasn't yet been extracted from Workbench and packaged as a gem.
-* R and Java: We plan to support these, but they have not been implemented yet.
+* R: We plan to support this, but it has not been implemented yet.
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: Java
+title: "Java SDK"
+
+...
+
+The Java SDK provides a generic set of wrappers so you can make API calls in java.
+
+h3. Introdution
+
+* The Java SDK requires Java 6 or later
+
+* The Java SDK is implemented as a maven project. Hence, you would need a working
+maven environment to be able to build the source code. If you do not have maven setup,
+you may find the "Maven in 5 Minutes":http://maven.apache.org/guides/getting-started/maven-in-five-minutes.html link useful.
+
+* In this document $ARVADOS_HOME is used to refer to the directory where
+arvados code is cloned in your system. For ex: $ARVADOS_HOME = $HOME/arvados
+
+
+h3. Setting up the environment
+
+* The SDK requires a running Arvados API server. The following information
+ about the API server needs to be passed to the SDK using environment
+ variables or during the construction of the Arvados instance.
+
+<notextile>
+<pre>
+ARVADOS_API_TOKEN: API client token to be used to authorize with API server.
+
+ARVADOS_API_HOST: Host name of the API server.
+
+ARVADOS_API_HOST_INSECURE: Set this to true if you are using self-signed
+ certificates and would like to bypass certificate validations.
+</pre>
+</notextile>
+
+* Please see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for full details.
+
+
+h3. Building the Arvados SDK
+
+<notextile>
+<pre>
+$ <code class="userinput">cd $ARVADOS_HOME/sdk/java</code>
+
+$ <code class="userinput">mvn -Dmaven.test.skip=true clean package</code>
+ This will generate arvados sdk jar file in the target directory
+</pre>
+</notextile>
+
+
+h3. Implementing your code to use SDK
+
+* The following two sample programs serve as sample implementations using the SDK.
+<code class="userinput">$ARVADOS_HOME/sdk/java/ArvadosSDKJavaExample.java</code> is a simple program
+ that makes a few calls to API server.
+<code class="userinput">$ARVADOS_HOME/sdk/java/ArvadosSDKJavaExampleWithPrompt.java</code> can be
+ used to make calls to API server interactively.
+
+Please use these implementations to see how you would want use the SDK from your java program.
+
+Also, refer to <code class="userinput">$ARVADOS_HOME/arvados/sdk/java/src/test/java/org/arvados/sdk/java/ArvadosTest.java</code>
+for more sample API invocation examples.
+
+Below are the steps to compile and run these java program.
+
+* These programs create an instance of Arvados SDK class and use it to
+make various <code class="userinput">call</code> requests.
+
+* To compile the examples
+<notextile>
+<pre>
+$ <code class="userinput">javac -cp $ARVADOS_HOME/sdk/java/target/arvados-sdk-1.0-jar-with-dependencies.jar \
+ArvadosSDKJavaExample*.java</code>
+This results in the generation of the ArvadosSDKJavaExample*.class files
+in the same directory as the java files
+</pre>
+</notextile>
+
+* To run the samples
+<notextile>
+<pre>
+$ <code class="userinput">java -cp .:$ARVADOS_HOME/sdk/java/target/arvados-sdk-1.0-jar-with-dependencies.jar \
+ArvadosSDKJavaExample</code>
+$ <code class="userinput">java -cp .:$ARVADOS_HOME/sdk/java/target/arvados-sdk-1.0-jar-with-dependencies.jar \
+ArvadosSDKJavaExampleWithPrompt</code>
+</pre>
+</notextile>
+
+
+h3. Viewing and Managing SDK logging
+
+* SDK uses log4j logging
+
+* The default location of the log file is
+ <code class="userinput">$ARVADOS_HOME/sdk/java/log/arvados_sdk_java.log</code>
+
+* Update <code class="userinput">log4j.properties</code> file to change name and location of the log file.
+
+<notextile>
+<pre>
+$ <code class="userinput">nano $ARVADOS_HOME/sdk/java/src/main/resources/log4j.properties</code>
+and modify the <code class="userinput">log4j.appender.fileAppender.File</code> property as needed.
+
+Rebuild the SDK:
+$ <code class="userinput">mvn -Dmaven.test.skip=true clean package</code>
+</pre>
+</notextile>
+
+
+h3. Using the SDK in eclipse
+
+* To develop in eclipse, you can use the provided <code class="userinput">eclipse project</code>
+
+* Install "m2eclipse":https://www.eclipse.org/m2e/ plugin in your eclipse
+
+* Set <code class="userinput">M2_REPO</code> classpath variable in eclipse to point to your local repository.
+The local repository is usually located in your home directory at <code class="userinput">$HOME/.m2/repository</code>.
+
+<notextile>
+<pre>
+In Eclipse IDE:
+Window -> Preferences -> Java -> Build Path -> Classpath Variables
+ Click on the "New..." button and add a new
+ M2_REPO variable and set it to your local Maven repository
+</pre>
+</notextile>
+
+
+* Open the SDK project in eclipse
+<notextile>
+<pre>
+In Eclipse IDE:
+File -> Import -> Existing Projects into Workspace -> Next -> Browse
+ and select $ARVADOS_HOME/sdk/java
+</pre>
+</notextile>
<notextile>
<pre>
-$ <code class="userinput">sudo apt-get install libjson-perl libio-socket-ssl-perl libwww-perl</code>
+$ <code class="userinput">sudo apt-get install libjson-perl libio-socket-ssl-perl libwww-perl libipc-system-simple-perl</code>
$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
$ <code class="userinput">cd arvados/sdk/perl</code>
$ <code class="userinput">perl Makefile.PL</code>
To use the Python SDK elsewhere, you can either install the Python SDK via PyPI or build and install the package using the arvados source tree.
+{% include 'notebox_begin' %}
+The Python SDK requires Python 2.7
+{% include 'notebox_end' %}
+
h4. Option 1: install with PyPI
<notextile>
<pre>
-$ <code class="userinput">sudo apt-get install python-pip python-dev libattr1-dev libfuse-dev pkg-config</code>
+$ <code class="userinput">sudo apt-get install python-pip python-dev libattr1-dev libfuse-dev pkg-config python-yaml</code>
$ <code class="userinput">sudo pip install arvados-python-client</code>
</pre>
</notextile>
<notextile>
<pre>
-$ <code class="userinput">sudo apt-get install python-dev libattr1-dev libfuse-dev pkg-config</code>
-$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
-$ <code class="userinput">cd arvados/sdk/python</code>
-$ <code class="userinput">./build.sh</code>
-$ <code class="userinput">sudo python setup.py install</code>
+~$ <code class="userinput">sudo apt-get install python-dev libattr1-dev libfuse-dev pkg-config</code>
+~$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+~$ <code class="userinput">cd arvados/sdk/python</code>
+~/arvados/sdk/python$ <code class="userinput">sudo python setup.py install</code>
</pre>
</notextile>
<notextile>
<pre>
$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
-$ <code class="userinput">cd arvados/sdk/cli</code>
+$ <code class="userinput">cd arvados/sdk/ruby</code>
$ <code class="userinput">gem build arvados.gemspec</code>
$ <code class="userinput">sudo gem install arvados-*.gem</code>
</pre>
BASE_DEPS = base/Dockerfile $(BASE_GENERATED)
+JOBS_DEPS = jobs/Dockerfile
+
API_DEPS = api/Dockerfile $(API_GENERATED)
DOC_DEPS = doc/Dockerfile doc/apache2_vhost
mkdir -p build
rsync -rlp --exclude=docker/ --exclude='**/log/*' --exclude='**/tmp/*' \
--chmod=Da+rx,Fa+rX ../ build/
+ find build/ -name \*.gem -delete
+ cd build/sdk/python/ && ./build.sh
+ cd build/sdk/cli && gem build arvados-cli.gemspec
+ cd build/sdk/ruby && gem build arvados.gemspec
touch build/.buildstamp
$(BASE_GENERATED): config.yml $(BUILD)
$(DOCKER_BUILD) -t arvados/doc doc
date >doc-image
+jobs-image: base-image $(BUILD) $(JOBS_DEPS)
+ $(DOCKER_BUILD) -t arvados/jobs jobs
+ date >jobs-image
+
workbench-image: passenger-image $(BUILD) $(WORKBENCH_DEPS)
mkdir -p workbench/generated
tar -czf workbench/generated/workbench.tar.gz -C build/apps workbench
--- /dev/null
+FROM arvados/base
+MAINTAINER Brett Smith <brett@curoverse.com>
+
+# Install dependencies and set up system.
+# The FUSE packages help ensure that we can install the Python SDK (arv-mount).
+RUN /usr/bin/apt-get install -q -y python-dev python-llfuse python-pip \
+ libio-socket-ssl-perl libjson-perl liburi-perl libwww-perl \
+ fuse libattr1-dev libfuse-dev && \
+ /usr/sbin/adduser --disabled-password \
+ --gecos 'Crunch execution user' crunch && \
+ /usr/bin/install -d -o crunch -g crunch -m 0700 /tmp/crunch-job && \
+ /bin/ln -s /usr/src/arvados /usr/local/src/arvados
+
+# Install Arvados packages.
+RUN find /usr/src/arvados/sdk -name '*.gem' -print0 | \
+ xargs -0rn 1 gem install && \
+ cd /usr/src/arvados/sdk/python && \
+ python setup.py install
+
+USER crunch
s.executables << "arv-run-pipeline-instance"
s.executables << "arv-crunch-job"
s.executables << "arv-tag"
+ s.required_ruby_version = '>= 2.1.0'
s.add_runtime_dependency 'arvados', '~> 0.1.0'
s.add_runtime_dependency 'google-api-client', '~> 0.6.3'
s.add_runtime_dependency 'activesupport', '~> 3.2', '>= 3.2.13'
exit
end
-request_parameters = {}.merge(method_opts)
+request_parameters = {_profile:true}.merge(method_opts)
resource_body = request_parameters.delete(resource_schema.to_sym)
if resource_body
request_body = {
resource_schema => resource_body
}
else
- request_body = {}
+ request_body = nil
end
case api_method
end
exit 0
else
- request_body[:api_token] = ENV['ARVADOS_API_TOKEN']
- request_body[:_profile] = true
result = client.execute(:api_method => eval(api_method),
:parameters => request_parameters,
:body => request_body,
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
end
begin
:parameters => {
:uuid => uuid
},
- :body => {
- :api_token => ENV['ARVADOS_API_TOKEN']
- },
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
j = JSON.parse result.body, :symbolize_names => true
unless j.is_a? Hash and j[:uuid]
debuglog "Failed to get pipeline_instance: #{j[:errors] rescue nil}", 0
def self.create(attributes)
result = $client.execute(:api_method => $arvados.pipeline_instances.create,
:body => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
:pipeline_instance => attributes
},
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
j = JSON.parse result.body, :symbolize_names => true
unless j.is_a? Hash and j[:uuid]
abort "Failed to create pipeline_instance: #{j[:errors] rescue nil} #{j.inspect}"
:uuid => @pi[:uuid]
},
:body => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
:pipeline_instance => @attributes_to_update.to_json
},
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
j = JSON.parse result.body, :symbolize_names => true
unless j.is_a? Hash and j[:uuid]
debuglog "Failed to save pipeline_instance: #{j[:errors] rescue nil}", 0
@cache ||= {}
result = $client.execute(:api_method => $arvados.jobs.get,
:parameters => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
:uuid => uuid
},
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
@cache[uuid] = JSON.parse result.body, :symbolize_names => true
end
def self.where(conditions)
result = $client.execute(:api_method => $arvados.jobs.list,
:parameters => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
:limit => 10000,
:where => conditions.to_json
},
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
list = JSON.parse result.body, :symbolize_names => true
if list and list[:items].is_a? Array
list[:items]
def self.create(job, create_params)
@cache ||= {}
result = $client.execute(:api_method => $arvados.jobs.create,
- :parameters => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
+ :body => {
:job => job.to_json
}.merge(create_params),
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
j = JSON.parse result.body, :symbolize_names => true
if j.is_a? Hash and j[:uuid]
@cache[j[:uuid]] = j
else
result = $client.execute(:api_method => $arvados.pipeline_templates.get,
:parameters => {
- :api_token => ENV['ARVADOS_API_TOKEN'],
:uuid => template
},
- :authenticated => false)
+ :authenticated => false,
+ :headers => {
+ authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
+ })
@template = JSON.parse result.body, :symbolize_names => true
if !@template[:uuid]
abort "#{$0}: fatal: failed to retrieve pipeline template #{template} #{@template[:errors].inspect rescue nil}"
end
def setup_instance
- @instance ||= PipelineInstance.
- create(:components => @components,
+ if $options[:submit]
+ @instance ||= PipelineInstance.
+ create(:components => @components,
+ :pipeline_template_uuid => @template[:uuid],
+ :state => 'New')
+ else
+ @instance ||= PipelineInstance.
+ create(:components => @components,
:pipeline_template_uuid => @template[:uuid],
- :active => true)
+ :state => 'RunningOnClient')
+ end
self
end
def run
moretodo = true
+ interrupted = false
+
while moretodo
moretodo = false
@components.each do |cname, c|
end
end
@instance[:components] = @components
- @instance[:active] = moretodo
report_status
if @options[:no_wait]
sleep 10
rescue Interrupt
debuglog "interrupt", 0
- abort
+ interrupted = true
+ break
end
end
end
end
end
- if ended == @components.length or failed > 0
- @instance[:active] = false
- @instance[:success] = (succeeded == @components.length)
+ success = (succeeded == @components.length)
+
+ if interrupted
+ if success
+ @instance[:state] = 'Complete'
+ else
+ @instance[:state] = 'Paused'
+ end
+ else
+ if ended == @components.length or failed > 0
+ @instance[:state] = success ? 'Complete' : 'Failed'
+ end
end
+ # set components_summary
+ components_summary = {"todo" => @components.length - ended, "done" => succeeded, "failed" => failed}
+ @instance[:components_summary] = components_summary
+
@instance.save
end
def cleanup
- if @instance
- @instance[:active] = false
+ if @instance and @instance[:state] == 'RunningOnClient'
+ @instance[:state] = 'Paused'
@instance.save
end
end
my $arv = Arvados->new('apiVersion' => 'v1');
-my $metastream;
+my $local_logfile;
my $User = $arv->{'users'}->{'current'}->execute;
$job_id = $Job->{'uuid'};
my $keep_logfile = $job_id . '.log.txt';
-my $local_logfile = File::Temp->new();
+$local_logfile = File::Temp->new();
$Job->{'runtime_constraints'} ||= {};
$Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0;
must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
}
-
+# If this job requires a Docker image, install that.
+my $docker_bin = "/usr/bin/docker.io";
+my $docker_image = $Job->{runtime_constraints}->{docker_image} || "";
+if ($docker_image) {
+ my $docker_pid = fork();
+ if ($docker_pid == 0)
+ {
+ srun (["srun", "--nodelist=" . join(' ', @node)],
+ [$docker_bin, 'pull', $docker_image]);
+ exit ($?);
+ }
+ while (1)
+ {
+ last if $docker_pid == waitpid (-1, WNOHANG);
+ freeze_if_want_freeze ($docker_pid);
+ select (undef, undef, undef, 0.1);
+ }
+ # If the Docker image was specified as a hash, pull will fail.
+ # Ignore that error. We'll see what happens when we try to run later.
+ if (($? != 0) && ($docker_image !~ /^[0-9a-fA-F]{5,64}$/))
+ {
+ croak("Installing Docker image $docker_image returned exit code $?");
+ }
+}
foreach (qw (script script_version script_parameters runtime_constraints))
{
qw(-n1 -c1 -N1 -D), $ENV{'TMPDIR'},
"--job-name=$job_id.$id.$$",
);
- my @execargs = qw(sh);
my $build_script_to_send = "";
my $command =
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
$command .=
"&& perl -";
}
- $command .=
- "&& exec arv-mount $ENV{TASK_KEEPMOUNT} --exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+ $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
+ if ($docker_image)
+ {
+ $command .= "$docker_bin run -i -a stdin -a stdout -a stderr ";
+ # Dynamically configure the container to use the host system as its
+ # DNS server. Get the host's global addresses from the ip command,
+ # and turn them into docker --dns options using gawk.
+ $command .=
+ q{$(ip -o address show scope global |
+ gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
+ foreach my $env_key (qw(CRUNCH_SRC CRUNCH_TMP TASK_KEEPMOUNT))
+ {
+ $command .= "-v \Q$ENV{$env_key}:$ENV{$env_key}:rw\E ";
+ }
+ while (my ($env_key, $env_val) = each %ENV)
+ {
+ $command .= "-e \Q$env_key=$env_val\E ";
+ }
+ $command .= "\Q$docker_image\E ";
+ }
+ $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
exit (111);
delete $proc{$pid};
# Load new tasks
- my $newtask_list = $arv->{'job_tasks'}->{'list'}->execute(
- 'where' => {
- 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
- },
- 'order' => 'qsequence'
- );
- foreach my $arvados_task (@{$newtask_list->{'items'}}) {
+ my $newtask_list = [];
+ my $newtask_results;
+ do {
+ $newtask_results = $arv->{'job_tasks'}->{'list'}->execute(
+ 'where' => {
+ 'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
+ },
+ 'order' => 'qsequence',
+ 'offset' => scalar(@$newtask_list),
+ );
+ push(@$newtask_list, @{$newtask_results->{items}});
+ } while (@{$newtask_results->{items}});
+ foreach my $arvados_task (@$newtask_list) {
my $jobstep = {
'level' => $arvados_task->{'sequence'},
'failures' => 0,
$message =~ s{([^ -\176])}{"\\" . sprintf ("%03o", ord($1))}ge;
$message .= "\n";
my $datetime;
- if ($metastream || -t STDERR) {
+ if ($local_logfile || -t STDERR) {
my @gmtime = gmtime;
$datetime = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d",
$gmtime[5]+1900, $gmtime[4]+1, @gmtime[3,2,1,0]);
}
print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message);
- if ($metastream) {
- print $metastream $datetime . " " . $message;
+ if ($local_logfile) {
+ print $local_logfile $datetime . " " . $message;
}
}
freeze() if @jobstep_todo;
collate_output() if @jobstep_todo;
cleanup();
- save_meta() if $metastream;
+ save_meta() if $local_logfile;
die;
}
. quotemeta($local_logfile->filename);
my $loglocator = `$cmd`;
die "system $cmd failed: $?" if $?;
+ chomp($loglocator);
$local_logfile = undef; # the temp file is automatically deleted
Log (undef, "log manifest is $loglocator");
--- /dev/null
+require 'minitest/autorun'
+
+class TestRunPipelineInstance < Minitest::Test
+ def setup
+ end
+
+ def test_run_pipeline_instance_get_help
+ out, err = capture_subprocess_io do
+ system ('arv-run-pipeline-instance -h')
+ end
+ assert_equal '', err
+ end
+
+ def test_run_pipeline_instance_with_no_such_option
+ out, err = capture_subprocess_io do
+ system ('arv-run-pipeline-instance --junk')
+ end
+ refute_equal '', err
+ end
+
+ def test_run_pipeline_instance_for_bogus_template_uuid
+ out, err = capture_subprocess_io do
+ # fails with error SSL_connect error because HOST_INSECURE is not being used
+ # system ('arv-run-pipeline-instance --template bogus-abcde-fghijklmnopqrs input=c1bad4b39ca5a924e481008009d94e32+210')
+
+ # fails with error: fatal: cannot load such file -- arvados
+ # system ('./bin/arv-run-pipeline-instance --template bogus-abcde-fghijklmnopqrs input=c1bad4b39ca5a924e481008009d94e32+210')
+ end
+ #refute_equal '', err
+ assert_equal '', err
+ end
+
+end
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry including="**/*.java" kind="src" output="target/test-classes" path="src/test/java"/>
+ <classpathentry including="**/*.java" kind="src" path="src/main/java"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/apis/google-api-services-discovery/v1-rev42-1.18.0-rc/google-api-services-discovery-v1-rev42-1.18.0-rc.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/api-client/google-api-client/1.18.0-rc/google-api-client-1.18.0-rc.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/http-client/google-http-client/1.18.0-rc/google-http-client-1.18.0-rc.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/httpcomponents/httpclient/4.0.1/httpclient-4.0.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/http-client/google-http-client-jackson2/1.18.0-rc/google-http-client-jackson2-1.18.0-rc.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/fasterxml/jackson/core/jackson-core/2.1.3/jackson-core-2.1.3.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/guava/guava/r05/guava-r05.jar"/>
+ <classpathentry kind="var" path="M2_REPO/log4j/log4j/1.2.16/log4j-1.2.16.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/googlecode/json-simple/json-simple/1.1.1/json-simple-1.1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/junit/junit/4.8.1/junit-4.8.1.jar"/>
+ <classpathentry kind="output" path="target/classes"/>
+</classpath>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>java</name>
+ <comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
+ <projects/>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
\ No newline at end of file
--- /dev/null
+#Mon Apr 28 10:33:40 EDT 2014
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.source=1.6
+org.eclipse.jdt.core.compiler.compliance=1.6
--- /dev/null
+/**
+ * This Sample test program is useful in getting started with working with Arvados Java SDK.
+ * @author radhika
+ *
+ */
+
+import org.arvados.sdk.java.Arvados;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+public class ArvadosSDKJavaExample {
+ /** Make sure the following environment variables are set before using Arvados:
+ * ARVADOS_API_TOKEN, ARVADOS_API_HOST and ARVADOS_API_HOST_INSECURE
+ * Set ARVADOS_API_HOST_INSECURE to true if you are using self-singed
+ * certificates in development and want to bypass certificate validations.
+ *
+ * If you are not using env variables, you can pass them to Arvados constructor.
+ *
+ * Please refer to http://doc.arvados.org/api/index.html for a complete list
+ * of the available API methods.
+ */
+ public static void main(String[] args) throws Exception {
+ String apiName = "arvados";
+ String apiVersion = "v1";
+
+ Arvados arv = new Arvados(apiName, apiVersion);
+
+ // Make a users list call. Here list on users is the method being invoked.
+ // Expect a Map containing the list of users as the response.
+ System.out.println("Making an arvados users.list api call");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("users", "list", params);
+ System.out.println("Arvados users.list:\n");
+ printResponse(response);
+
+ // get uuid of the first user from the response
+ List items = (List)response.get("items");
+
+ Map firstUser = (Map)items.get(0);
+ String userUuid = (String)firstUser.get("uuid");
+
+ // Make a users get call on the uuid obtained above
+ System.out.println("\n\n\nMaking a users.get call for " + userUuid);
+ params = new HashMap<String, Object>();
+ params.put("uuid", userUuid);
+ response = arv.call("users", "get", params);
+ System.out.println("Arvados users.get:\n");
+ printResponse(response);
+
+ // Make a pipeline_templates list call
+ System.out.println("\n\n\nMaking a pipeline_templates.list call.");
+
+ params = new HashMap<String, Object>();
+ response = arv.call("pipeline_templates", "list", params);
+
+ System.out.println("Arvados pipelinetempates.list:\n");
+ printResponse(response);
+ }
+
+ private static void printResponse(Map response){
+ Set<Entry<String,Object>> entrySet = (Set<Entry<String,Object>>)response.entrySet();
+ for (Map.Entry<String, Object> entry : entrySet) {
+ if ("items".equals(entry.getKey())) {
+ List items = (List)entry.getValue();
+ for (Object item : items) {
+ System.out.println(" " + item);
+ }
+ } else {
+ System.out.println(entry.getKey() + " = " + entry.getValue());
+ }
+ }
+ }
+}
\ No newline at end of file
--- /dev/null
+/**
+ * This Sample test program is useful in getting started with using Arvados Java SDK.
+ * This program creates an Arvados instance using the configured environment variables.
+ * It then provides a prompt to input method name and input parameters.
+ * The program them invokes the API server to execute the specified method.
+ *
+ * @author radhika
+ */
+
+import org.arvados.sdk.java.Arvados;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+public class ArvadosSDKJavaExampleWithPrompt {
+ /**
+ * Make sure the following environment variables are set before using Arvados:
+ * ARVADOS_API_TOKEN, ARVADOS_API_HOST and ARVADOS_API_HOST_INSECURE Set
+ * ARVADOS_API_HOST_INSECURE to true if you are using self-singed certificates
+ * in development and want to bypass certificate validations.
+ *
+ * Please refer to http://doc.arvados.org/api/index.html for a complete list
+ * of the available API methods.
+ */
+ public static void main(String[] args) throws Exception {
+ String apiName = "arvados";
+ String apiVersion = "v1";
+
+ System.out.print("Welcome to Arvados Java SDK.");
+ System.out.println("\nYou can use this example to call API methods interactively.");
+ System.out.println("\nPlease refer to http://doc.arvados.org/api/index.html for api documentation");
+ System.out.println("\nTo make the calls, enter input data at the prompt.");
+ System.out.println("When entering parameters, you may enter a simple string or a well-formed json.");
+ System.out.println("For example to get a user you may enter: user, zzzzz-12345-67890");
+ System.out.println("Or to filter links, you may enter: filters, [[ \"name\", \"=\", \"can_manage\"]]");
+
+ System.out.println("\nEnter ^C when you want to quit");
+
+ // use configured env variables for API TOKEN, HOST and HOST_INSECURE
+ Arvados arv = new Arvados(apiName, apiVersion);
+
+ while (true) {
+ try {
+ // prompt for resource
+ System.out.println("\n\nEnter Resource name (for example users)");
+ System.out.println("\nAvailable resources are: " + arv.getAvailableResourses());
+ System.out.print("\n>>> ");
+
+ // read resource name
+ BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
+ String resourceName = in.readLine().trim();
+ if ("".equals(resourceName)) {
+ throw (new Exception("No resource name entered"));
+ }
+ // read method name
+ System.out.println("\nEnter method name (for example get)");
+ System.out.println("\nAvailable methods are: " + arv.getAvailableMethodsForResourse(resourceName));
+ System.out.print("\n>>> ");
+ String methodName = in.readLine().trim();
+ if ("".equals(methodName)) {
+ throw (new Exception("No method name entered"));
+ }
+
+ // read method parameters
+ System.out.println("\nEnter parameter name, value (for example uuid, uuid-value)");
+ System.out.println("\nAvailable parameters are: " +
+ arv.getAvailableParametersForMethod(resourceName, methodName));
+
+ System.out.print("\n>>> ");
+ Map paramsMap = new HashMap();
+ String param = "";
+ try {
+ do {
+ param = in.readLine();
+ if (param.isEmpty())
+ break;
+ int index = param.indexOf(","); // first comma
+ String paramName = param.substring(0, index);
+ String paramValue = param.substring(index+1);
+ paramsMap.put(paramName.trim(), paramValue.trim());
+
+ System.out.println("\nEnter parameter name, value (for example uuid, uuid-value)");
+ System.out.print("\n>>> ");
+ } while (!param.isEmpty());
+ } catch (Exception e) {
+ System.out.println (e.getMessage());
+ System.out.println ("\nSet up a new call");
+ continue;
+ }
+
+ // Make a "call" for the given resource name and method name
+ try {
+ System.out.println ("Making a call for " + resourceName + " " + methodName);
+ Map response = arv.call(resourceName, methodName, paramsMap);
+
+ Set<Entry<String,Object>> entrySet = (Set<Entry<String,Object>>)response.entrySet();
+ for (Map.Entry<String, Object> entry : entrySet) {
+ if ("items".equals(entry.getKey())) {
+ List items = (List)entry.getValue();
+ for (Object item : items) {
+ System.out.println(" " + item);
+ }
+ } else {
+ System.out.println(entry.getKey() + " = " + entry.getValue());
+ }
+ }
+ } catch (Exception e){
+ System.out.println (e.getMessage());
+ System.out.println ("\nSet up a new call");
+ }
+ } catch (Exception e) {
+ System.out.println (e.getMessage());
+ System.out.println ("\nSet up a new call");
+ }
+ }
+ }
+}
--- /dev/null
+Welcome to Arvados Java SDK.
+
+Please refer to http://doc.arvados.org/sdk/java/index.html to get started
+ with Arvados Java SDK.
--- /dev/null
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.arvados.sdk.java</groupId>
+ <artifactId>java</artifactId>
+ <packaging>jar</packaging>
+ <version>1.0-SNAPSHOT</version>
+ <name>java</name>
+ <url>http://maven.apache.org</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-discovery</artifactId>
+ <version>v1-rev42-1.18.0-rc</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.api-client</groupId>
+ <artifactId>google-api-client</artifactId>
+ <version>1.18.0-rc</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-jackson2</artifactId>
+ <version>1.18.0-rc</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>r05</version>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.16</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <finalName>arvados-sdk-1.0</finalName>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <archive>
+ <manifest>
+ <mainClass>org.arvados.sdk.Arvados</mainClass>
+ </manifest>
+ <manifestEntries>
+ <!--<Premain-Class>Your.agent.class</Premain-Class> <Agent-Class>Your.agent.class</Agent-Class> -->
+ <Can-Redefine-Classes>true</Can-Redefine-Classes>
+ <Can-Retransform-Classes>true</Can-Retransform-Classes>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <targetPath>${basedir}/target/classes</targetPath>
+ <includes>
+ <include>log4j.properties</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </build>
+</project>
--- /dev/null
+package org.arvados.sdk.java;
+
+import com.google.api.client.http.javanet.*;
+import com.google.api.client.http.ByteArrayContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpContent;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.UriTemplate;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.util.Maps;
+import com.google.api.services.discovery.Discovery;
+import com.google.api.services.discovery.model.JsonSchema;
+import com.google.api.services.discovery.model.RestDescription;
+import com.google.api.services.discovery.model.RestMethod;
+import com.google.api.services.discovery.model.RestMethod.Request;
+import com.google.api.services.discovery.model.RestResource;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+/**
+ * This class provides a java SDK interface to Arvados API server.
+ *
+ * Please refer to http://doc.arvados.org/api/ to learn about the
+ * various resources and methods exposed by the API server.
+ *
+ * @author radhika
+ */
+public class Arvados {
+ // HttpTransport and JsonFactory are thread-safe. So, use global instances.
+ private HttpTransport httpTransport;
+ private final JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
+
+ private String arvadosApiToken;
+ private String arvadosApiHost;
+ private boolean arvadosApiHostInsecure;
+
+ private String arvadosRootUrl;
+
+ private static final Logger logger = Logger.getLogger(Arvados.class);
+
+ // Get it once and reuse on the call requests
+ RestDescription restDescription = null;
+ String apiName = null;
+ String apiVersion = null;
+
+ public Arvados (String apiName, String apiVersion) throws Exception {
+ this (apiName, apiVersion, null, null, null);
+ }
+
+ public Arvados (String apiName, String apiVersion, String token,
+ String host, String hostInsecure) throws Exception {
+ this.apiName = apiName;
+ this.apiVersion = apiVersion;
+
+ // Read needed environmental variables if they are not passed
+ if (token != null) {
+ arvadosApiToken = token;
+ } else {
+ arvadosApiToken = System.getenv().get("ARVADOS_API_TOKEN");
+ if (arvadosApiToken == null) {
+ throw new Exception("Missing environment variable: ARVADOS_API_TOKEN");
+ }
+ }
+
+ if (host != null) {
+ arvadosApiHost = host;
+ } else {
+ arvadosApiHost = System.getenv().get("ARVADOS_API_HOST");
+ if (arvadosApiHost == null) {
+ throw new Exception("Missing environment variable: ARVADOS_API_HOST");
+ }
+ }
+ arvadosRootUrl = "https://" + arvadosApiHost;
+ arvadosRootUrl += (arvadosApiHost.endsWith("/")) ? "" : "/";
+
+ if (hostInsecure != null) {
+ arvadosApiHostInsecure = Boolean.valueOf(hostInsecure);
+ } else {
+ arvadosApiHostInsecure =
+ "true".equals(System.getenv().get("ARVADOS_API_HOST_INSECURE")) ? true : false;
+ }
+
+ // Create HTTP_TRANSPORT object
+ NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
+ if (arvadosApiHostInsecure) {
+ builder.doNotValidateCertificate();
+ }
+ httpTransport = builder.build();
+
+ // initialize rest description
+ restDescription = loadArvadosApi();
+ }
+
+ /**
+ * Make a call to API server with the provide call information.
+ * @param resourceName
+ * @param methodName
+ * @param paramsMap
+ * @return Map
+ * @throws Exception
+ */
+ public Map call(String resourceName, String methodName,
+ Map<String, Object> paramsMap) throws Exception {
+ RestMethod method = getMatchingMethod(resourceName, methodName);
+
+ HashMap<String, Object> parameters = loadParameters(paramsMap, method);
+
+ GenericUrl url = new GenericUrl(UriTemplate.expand(
+ arvadosRootUrl + restDescription.getBasePath() + method.getPath(),
+ parameters, true));
+
+ try {
+ // construct the request
+ HttpRequestFactory requestFactory;
+ requestFactory = httpTransport.createRequestFactory();
+
+ // possibly required content
+ HttpContent content = null;
+
+ if (!method.getHttpMethod().equals("GET") &&
+ !method.getHttpMethod().equals("DELETE")) {
+ String objectName = resourceName.substring(0, resourceName.length()-1);
+ Object requestBody = paramsMap.get(objectName);
+ if (requestBody == null) {
+ error("POST method requires content object " + objectName);
+ }
+
+ content = new ByteArrayContent("application/json",((String)requestBody).getBytes());
+ }
+
+ HttpRequest request =
+ requestFactory.buildRequest(method.getHttpMethod(), url, content);
+
+ // make the request
+ List<String> authHeader = new ArrayList<String>();
+ authHeader.add("OAuth2 " + arvadosApiToken);
+ request.getHeaders().put("Authorization", authHeader);
+ String response = request.execute().parseAsString();
+
+ Map responseMap = jsonFactory.createJsonParser(response).parse(HashMap.class);
+
+ logger.debug(responseMap);
+
+ return responseMap;
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ /**
+ * Get all supported resources by the API
+ * @return Set
+ */
+ public Set<String> getAvailableResourses() {
+ return (restDescription.getResources().keySet());
+ }
+
+ /**
+ * Get all supported method names for the given resource
+ * @param resourceName
+ * @return Set
+ * @throws Exception
+ */
+ public Set<String> getAvailableMethodsForResourse(String resourceName)
+ throws Exception {
+ Map<String, RestMethod> methodMap = getMatchingMethodMap (resourceName);
+ return (methodMap.keySet());
+ }
+
+ /**
+ * Get the parameters for the method in the resource sought.
+ * @param resourceName
+ * @param methodName
+ * @return Set
+ * @throws Exception
+ */
+ public Set<String> getAvailableParametersForMethod(String resourceName, String methodName)
+ throws Exception {
+ RestMethod method = getMatchingMethod(resourceName, methodName);
+ Set<String> parameters = method.getParameters().keySet();
+ Request request = method.getRequest();
+ if (request != null) {
+ Object requestProperties = request.get("properties");
+ if (requestProperties != null) {
+ if (requestProperties instanceof Map) {
+ Map properties = (Map)requestProperties;
+ Set<String> propertyKeys = properties.keySet();
+ if (propertyKeys.size()>0) {
+ try {
+ propertyKeys.addAll(parameters);
+ return propertyKeys;
+ } catch (Exception e){
+ logger.error(e);
+ }
+ }
+ }
+ }
+ }
+ return parameters;
+ }
+
+ private HashMap<String, Object> loadParameters(Map<String, Object> paramsMap,
+ RestMethod method) throws Exception {
+ HashMap<String, Object> parameters = Maps.newHashMap();
+
+ // required parameters
+ if (method.getParameterOrder() != null) {
+ for (String parameterName : method.getParameterOrder()) {
+ JsonSchema parameter = method.getParameters().get(parameterName);
+ if (Boolean.TRUE.equals(parameter.getRequired())) {
+ Object parameterValue = paramsMap.get(parameterName);
+ if (parameterValue == null) {
+ error("missing required parameter: " + parameter);
+ } else {
+ putParameter(null, parameters, parameterName, parameter, parameterValue);
+ }
+ }
+ }
+ }
+
+ for (Map.Entry<String, Object> entry : paramsMap.entrySet()) {
+ String parameterName = entry.getKey();
+ Object parameterValue = entry.getValue();
+
+ if (parameterName.equals("contentType")) {
+ if (method.getHttpMethod().equals("GET") || method.getHttpMethod().equals("DELETE")) {
+ error("HTTP content type cannot be specified for this method: " + parameterName);
+ }
+ } else {
+ JsonSchema parameter = null;
+ if (restDescription.getParameters() != null) {
+ parameter = restDescription.getParameters().get(parameterName);
+ }
+ if (parameter == null && method.getParameters() != null) {
+ parameter = method.getParameters().get(parameterName);
+ }
+ putParameter(parameterName, parameters, parameterName, parameter, parameterValue);
+ }
+ }
+
+ return parameters;
+ }
+
+ private RestMethod getMatchingMethod(String resourceName, String methodName)
+ throws Exception {
+ Map<String, RestMethod> methodMap = getMatchingMethodMap(resourceName);
+
+ if (methodName == null) {
+ error("missing method name");
+ }
+
+ RestMethod method =
+ methodMap == null ? null : methodMap.get(methodName);
+ if (method == null) {
+ error("method not found: ");
+ }
+
+ return method;
+ }
+
+ private Map<String, RestMethod> getMatchingMethodMap(String resourceName)
+ throws Exception {
+ if (resourceName == null) {
+ error("missing resource name");
+ }
+
+ Map<String, RestMethod> methodMap = null;
+ Map<String, RestResource> resources = restDescription.getResources();
+ RestResource resource = resources.get(resourceName);
+ if (resource == null) {
+ error("resource not found");
+ }
+ methodMap = resource.getMethods();
+ return methodMap;
+ }
+
+ /**
+ * Not thread-safe. So, create for each request.
+ * @param apiName
+ * @param apiVersion
+ * @return
+ * @throws Exception
+ */
+ private RestDescription loadArvadosApi()
+ throws Exception {
+ try {
+ Discovery discovery;
+
+ Discovery.Builder discoveryBuilder =
+ new Discovery.Builder(httpTransport, jsonFactory, null);
+
+ discoveryBuilder.setRootUrl(arvadosRootUrl);
+ discoveryBuilder.setApplicationName(apiName);
+
+ discovery = discoveryBuilder.build();
+
+ return discovery.apis().getRest(apiName, apiVersion).execute();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ private void putParameter(String argName, Map<String, Object> parameters,
+ String parameterName, JsonSchema parameter, Object parameterValue)
+ throws Exception {
+ Object value = parameterValue;
+ if (parameter != null) {
+ if ("boolean".equals(parameter.getType())) {
+ value = Boolean.valueOf(parameterValue.toString());
+ } else if ("number".equals(parameter.getType())) {
+ value = new BigDecimal(parameterValue.toString());
+ } else if ("integer".equals(parameter.getType())) {
+ value = new BigInteger(parameterValue.toString());
+ } else if ("float".equals(parameter.getType())) {
+ value = new BigDecimal(parameterValue.toString());
+ } else if (("array".equals(parameter.getType())) ||
+ ("Array".equals(parameter.getType()))) {
+ if (parameterValue.getClass().isArray()){
+ value = getJsonValueFromArrayType(parameterValue);
+ } else if (List.class.isAssignableFrom(parameterValue.getClass())) {
+ value = getJsonValueFromListType(parameterValue);
+ }
+ } else if (("Hash".equals(parameter.getType())) ||
+ ("hash".equals(parameter.getType()))) {
+ value = getJsonValueFromMapType(parameterValue);
+ } else {
+ if (parameterValue.getClass().isArray()){
+ value = getJsonValueFromArrayType(parameterValue);
+ } else if (List.class.isAssignableFrom(parameterValue.getClass())) {
+ value = getJsonValueFromListType(parameterValue);
+ } else if (Map.class.isAssignableFrom(parameterValue.getClass())) {
+ value = getJsonValueFromMapType(parameterValue);
+ }
+ }
+ }
+
+ parameters.put(parameterName, value);
+ }
+
+ private String getJsonValueFromArrayType (Object parameterValue) {
+ String arrayStr = Arrays.deepToString((Object[])parameterValue);
+ arrayStr = arrayStr.substring(1, arrayStr.length()-1);
+ Object[] array = arrayStr.split(",");
+ Object[] trimmedArray = new Object[array.length];
+ for (int i=0; i<array.length; i++){
+ trimmedArray[i] = array[i].toString().trim();
+ }
+ String jsonString = JSONArray.toJSONString(Arrays.asList(trimmedArray));
+ String value = "["+ jsonString +"]";
+
+ return value;
+ }
+
+ private String getJsonValueFromListType (Object parameterValue) {
+ List paramList = (List)parameterValue;
+ Object[] array = new Object[paramList.size()];
+ String arrayStr = Arrays.deepToString(paramList.toArray(array));
+ arrayStr = arrayStr.substring(1, arrayStr.length()-1);
+ array = arrayStr.split(",");
+ Object[] trimmedArray = new Object[array.length];
+ for (int i=0; i<array.length; i++){
+ trimmedArray[i] = array[i].toString().trim();
+ }
+ String jsonString = JSONArray.toJSONString(Arrays.asList(trimmedArray));
+ String value = "["+ jsonString +"]";
+
+ return value;
+ }
+
+ private String getJsonValueFromMapType (Object parameterValue) {
+ JSONObject json = new JSONObject((Map)parameterValue);
+ return json.toString();
+ }
+
+ private static void error(String detail) throws Exception {
+ String errorDetail = "ERROR: " + detail;
+
+ logger.debug(errorDetail);
+ throw new Exception(errorDetail);
+ }
+
+ public static void main(String[] args){
+ System.out.println("Welcome to Arvados Java SDK.");
+ System.out.println("Please refer to http://doc.arvados.org/sdk/java/index.html to get started with the the SDK.");
+ }
+
+}
--- /dev/null
+package org.arvados.sdk.java;
+
+import com.google.api.client.util.Lists;
+import com.google.api.client.util.Sets;
+
+import java.util.ArrayList;
+import java.util.SortedSet;
+
+public class MethodDetails implements Comparable<MethodDetails> {
+ String name;
+ ArrayList<String> requiredParameters = Lists.newArrayList();
+ SortedSet<String> optionalParameters = Sets.newTreeSet();
+ boolean hasContent;
+
+ @Override
+ public int compareTo(MethodDetails o) {
+ if (o == this) {
+ return 0;
+ }
+ return name.compareTo(o.name);
+ }
+}
\ No newline at end of file
--- /dev/null
+# To change log location, change log4j.appender.fileAppender.File
+
+log4j.rootLogger=DEBUG, fileAppender
+
+log4j.appender.fileAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.fileAppender.File=${basedir}/log/arvados_sdk_java.log
+log4j.appender.fileAppender.Append=true
+log4j.appender.file.MaxFileSize=10MB
+log4j.appender.file.MaxBackupIndex=10
+log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.fileAppender.layout.ConversionPattern=[%d] %-5p %c %L %x - %m%n
--- /dev/null
+package org.arvados.sdk.java;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unit test for Arvados.
+ */
+public class ArvadosTest {
+
+ /**
+ * Test users.list api
+ * @throws Exception
+ */
+ @Test
+ public void testCallUsersList() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("users", "list", params);
+ assertEquals("Expected kind to be users.list", "arvados#userList", response.get("kind"));
+
+ List items = (List)response.get("items");
+ assertNotNull("expected users list items", items);
+ assertTrue("expected at least one item in users list", items.size()>0);
+
+ Map firstUser = (Map)items.get(0);
+ assertNotNull ("Expcted at least one user", firstUser);
+
+ assertEquals("Expected kind to be user", "arvados#user", firstUser.get("kind"));
+ assertNotNull("Expected uuid for first user", firstUser.get("uuid"));
+ }
+
+ /**
+ * Test users.get <uuid> api
+ * @throws Exception
+ */
+ @Test
+ public void testCallUsersGet() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ // call user.system and get uuid of this user
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("users", "list", params);
+
+ assertNotNull("expected users list", response);
+ List items = (List)response.get("items");
+ assertNotNull("expected users list items", items);
+
+ Map firstUser = (Map)items.get(0);
+ String userUuid = (String)firstUser.get("uuid");
+
+ // invoke users.get with the system user uuid
+ params = new HashMap<String, Object>();
+ params.put("uuid", userUuid);
+
+ response = arv.call("users", "get", params);
+
+ assertNotNull("Expected uuid for first user", response.get("uuid"));
+ assertEquals("Expected system user uuid", userUuid, response.get("uuid"));
+ }
+
+ /**
+ * Test users.create api
+ * @throws Exception
+ */
+ @Test
+ public void testCreateUser() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+ params.put("user", "{}");
+ Map response = arv.call("users", "create", params);
+
+ assertEquals("Expected kind to be user", "arvados#user", response.get("kind"));
+
+ Object uuid = response.get("uuid");
+ assertNotNull("Expected uuid for first user", uuid);
+
+ // delete the object
+ params = new HashMap<String, Object>();
+ params.put("uuid", uuid);
+ response = arv.call("users", "delete", params);
+
+ // invoke users.get with the system user uuid
+ params = new HashMap<String, Object>();
+ params.put("uuid", uuid);
+
+ Exception caught = null;
+ try {
+ arv.call("users", "get", params);
+ } catch (Exception e) {
+ caught = e;
+ }
+
+ assertNotNull ("expected exception", caught);
+ assertTrue ("Expected 404", caught.getMessage().contains("Path not found"));
+ }
+
+ @Test
+ public void testCreateUserWithMissingRequiredParam() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Exception caught = null;
+ try {
+ arv.call("users", "create", params);
+ } catch (Exception e) {
+ caught = e;
+ }
+
+ assertNotNull ("expected exception", caught);
+ assertTrue ("Expected POST method requires content object user",
+ caught.getMessage().contains("ERROR: POST method requires content object user"));
+ }
+
+ /**
+ * Test users.create api
+ * @throws Exception
+ */
+ @Test
+ public void testCreateAndUpdateUser() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+ params.put("user", "{}");
+ Map response = arv.call("users", "create", params);
+
+ assertEquals("Expected kind to be user", "arvados#user", response.get("kind"));
+
+ Object uuid = response.get("uuid");
+ assertNotNull("Expected uuid for first user", uuid);
+
+ // update this user
+ params = new HashMap<String, Object>();
+ params.put("user", "{}");
+ params.put("uuid", uuid);
+ response = arv.call("users", "update", params);
+
+ assertEquals("Expected kind to be user", "arvados#user", response.get("kind"));
+
+ uuid = response.get("uuid");
+ assertNotNull("Expected uuid for first user", uuid);
+
+ // delete the object
+ params = new HashMap<String, Object>();
+ params.put("uuid", uuid);
+ response = arv.call("users", "delete", params);
+ }
+
+ /**
+ * Test unsupported api version api
+ * @throws Exception
+ */
+ @Test
+ public void testUnsupportedApiName() throws Exception {
+ Exception caught = null;
+ try {
+ Arvados arv = new Arvados("not_arvados", "v1");
+ } catch (Exception e) {
+ caught = e;
+ }
+
+ assertNotNull ("expected exception", caught);
+ assertTrue ("Expected 404 when unsupported api is used", caught.getMessage().contains("404 Not Found"));
+ }
+
+ /**
+ * Test unsupported api version api
+ * @throws Exception
+ */
+ @Test
+ public void testUnsupportedVersion() throws Exception {
+ Exception caught = null;
+ try {
+ Arvados arv = new Arvados("arvados", "v2");
+ } catch (Exception e) {
+ caught = e;
+ }
+
+ assertNotNull ("expected exception", caught);
+ assertTrue ("Expected 404 when unsupported version is used", caught.getMessage().contains("404 Not Found"));
+ }
+
+ /**
+ * Test unsupported api version api
+ * @throws Exception
+ */
+ @Test
+ public void testCallForNoSuchResrouce() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Exception caught = null;
+ try {
+ arv.call("abcd", "list", null);
+ } catch (Exception e) {
+ caught = e;
+ }
+
+ assertNotNull ("expected exception", caught);
+ assertTrue ("Expected ERROR: 404 not found", caught.getMessage().contains("ERROR: resource not found"));
+ }
+
+ /**
+ * Test unsupported api version api
+ * @throws Exception
+ */
+ @Test
+ public void testCallForNoSuchResrouceMethod() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Exception caught = null;
+ try {
+ arv.call("users", "abcd", null);
+ } catch (Exception e) {
+ caught = e;
+ }
+
+ assertNotNull ("expected exception", caught);
+ assertTrue ("Expected ERROR: 404 not found", caught.getMessage().contains("ERROR: method not found"));
+ }
+
+ /**
+ * Test pipeline_tempates.create api
+ * @throws Exception
+ */
+ @Test
+ public void testCreateAndGetPipelineTemplate() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ File file = new File(getClass().getResource( "/first_pipeline.json" ).toURI());
+ byte[] data = new byte[(int)file.length()];
+ try {
+ FileInputStream is = new FileInputStream(file);
+ is.read(data);
+ is.close();
+ }catch(Exception e) {
+ e.printStackTrace();
+ }
+
+ Map<String, Object> params = new HashMap<String, Object>();
+ params.put("pipeline_template", new String(data));
+ Map response = arv.call("pipeline_templates", "create", params);
+
+ assertEquals("Expected kind to be user", "arvados#pipelineTemplate", response.get("kind"));
+ String uuid = (String)response.get("uuid");
+ assertNotNull("Expected uuid for pipeline template", uuid);
+
+ // get the pipeline
+ params = new HashMap<String, Object>();
+ params.put("uuid", uuid);
+ response = arv.call("pipeline_templates", "get", params);
+
+ assertEquals("Expected kind to be user", "arvados#pipelineTemplate", response.get("kind"));
+ assertEquals("Expected uuid for pipeline template", uuid, response.get("uuid"));
+
+ // delete the object
+ params = new HashMap<String, Object>();
+ params.put("uuid", uuid);
+ response = arv.call("pipeline_templates", "delete", params);
+ }
+
+ /**
+ * Test users.list api
+ * @throws Exception
+ */
+ @Test
+ public void testArvadosWithTokenPassed() throws Exception {
+ String token = System.getenv().get("ARVADOS_API_TOKEN");
+ String host = System.getenv().get("ARVADOS_API_HOST");
+ String hostInsecure = System.getenv().get("ARVADOS_API_HOST_INSECURE");
+
+ Arvados arv = new Arvados("arvados", "v1", token, host, hostInsecure);
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("users", "list", params);
+ assertEquals("Expected kind to be users.list", "arvados#userList", response.get("kind"));
+ }
+
+ /**
+ * Test users.list api
+ * @throws Exception
+ */
+ @Test
+ public void testCallUsersListWithLimit() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("users", "list", params);
+ assertEquals("Expected users.list in response", "arvados#userList", response.get("kind"));
+
+ List items = (List)response.get("items");
+ assertNotNull("expected users list items", items);
+ assertTrue("expected at least one item in users list", items.size()>0);
+
+ int numUsersListItems = items.size();
+
+ // make the request again with limit
+ params = new HashMap<String, Object>();
+ params.put("limit", numUsersListItems-1);
+
+ response = arv.call("users", "list", params);
+
+ assertEquals("Expected kind to be users.list", "arvados#userList", response.get("kind"));
+
+ items = (List)response.get("items");
+ assertNotNull("expected users list items", items);
+ assertTrue("expected at least one item in users list", items.size()>0);
+
+ int numUsersListItems2 = items.size();
+ assertEquals ("Got more users than requested", numUsersListItems-1, numUsersListItems2);
+ }
+
+ @Test
+ public void testGetLinksWithFilters() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("links", "list", params);
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+
+ String[] filters = new String[3];
+ filters[0] = "name";
+ filters[1] = "=";
+ filters[2] = "can_manage";
+
+ params.put("filters", filters);
+
+ response = arv.call("links", "list", params);
+
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+ assertFalse("Expected no can_manage in response", response.toString().contains("\"name\":\"can_manage\""));
+ }
+
+ @Test
+ public void testGetLinksWithFiltersAsList() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("links", "list", params);
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+
+ List<String> filters = new ArrayList<String>();
+ filters.add("name");
+ filters.add("is_a");
+ filters.add("can_manage");
+
+ params.put("filters", filters);
+
+ response = arv.call("links", "list", params);
+
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+ assertFalse("Expected no can_manage in response", response.toString().contains("\"name\":\"can_manage\""));
+ }
+
+ @Test
+ public void testGetLinksWithWhereClause() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map<String, String> where = new HashMap<String, String>();
+ where.put("where", "updated_at > '2014-05-01'");
+
+ params.put("where", where);
+
+ Map response = arv.call("links", "list", params);
+
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+ }
+
+ @Test
+ public void testGetAvailableResources() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+ Set<String> resources = arv.getAvailableResourses();
+ assertNotNull("Expected resources", resources);
+ assertTrue("Excected users in resrouces", resources.contains("users"));
+ }
+
+ @Test
+ public void testGetAvailableMethodsResources() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+ Set<String> methods = arv.getAvailableMethodsForResourse("users");
+ assertNotNull("Expected resources", methods);
+ assertTrue("Excected create method for users", methods.contains("create"));
+ }
+
+ @Test
+ public void testGetAvailableParametersForUsersGetMethod() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+ Set<String> parameters = arv.getAvailableParametersForMethod("users", "get");
+ assertNotNull("Expected parameters", parameters);
+ assertTrue("Excected uuid parameter for get method for users", parameters.contains("uuid"));
+ }
+
+ @Test
+ public void testGetAvailableParametersForUsersCreateMethod() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+ Set<String> parameters = arv.getAvailableParametersForMethod("users", "create");
+ assertNotNull("Expected parameters", parameters);
+ assertTrue("Excected user parameter for create method for users", parameters.contains("user"));
+ }
+
+}
\ No newline at end of file
--- /dev/null
+{
+ "name":"first pipeline",
+ "components":{
+ "do_hash":{
+ "script":"hash.py",
+ "script_parameters":{
+ "input":{
+ "required": true,
+ "dataclass": "Collection"
+ }
+ },
+ "script_version":"master",
+ "output_is_persistent":true
+ }
+ }
+}
Protocol scheme. Default: C<ARVADOS_API_PROTOCOL_SCHEME> environment
variable, or C<https>
-=item apiToken
+=item authToken
Authorization token. Default: C<ARVADOS_API_TOKEN> environment variable
{
my $self = shift;
my %req;
- $req{$self->{'method'}} = $self->{'uri'};
+ my %content;
+ my $method = $self->{'method'};
+ if ($method eq 'GET' || $method eq 'HEAD') {
+ $content{'_method'} = $method;
+ $method = 'POST';
+ }
+ $req{$method} = $self->{'uri'};
$self->{'req'} = new HTTP::Request (%req);
$self->{'req'}->header('Authorization' => ('OAuth2 ' . $self->{'authToken'})) if $self->{'authToken'};
$self->{'req'}->header('Accept' => 'application/json');
- my %content;
my ($p, $v);
while (($p, $v) = each %{$self->{'queryParams'}}) {
$content{$p} = (ref($v) eq "") ? $v : JSON::encode_json($v);
/dist/
/*.egg-info
/tmp
-setup.py
path = None
return path
-def api(version=None):
+def api(version=None, cache=True):
global services
if 'ARVADOS_DEBUG' in config.settings():
logging.basicConfig(level=logging.DEBUG)
- if not services.get(version):
+ if not cache or not services.get(version):
apiVersion = version
if not version:
apiVersion = 'v1'
ca_certs = None # use httplib2 default
http = httplib2.Http(ca_certs=ca_certs,
- cache=http_cache('discovery'))
+ cache=(http_cache('discovery') if cache else None))
http = credentials.authorize(http)
if re.match(r'(?i)^(true|1|yes)$',
config.get('ARVADOS_API_HOST_INSECURE', 'no')):
http.disable_ssl_certificate_validation=True
services[version] = apiclient.discovery.build(
'arvados', apiVersion, http=http, discoveryServiceUrl=url)
+ http.cache = None
return services[version]
--- /dev/null
+from ws4py.client.threadedclient import WebSocketClient
+import thread
+import json
+import os
+import time
+import ssl
+import re
+import config
+
+class EventClient(WebSocketClient):
+ def __init__(self, url, filters, on_event):
+ ssl_options = None
+ if re.match(r'(?i)^(true|1|yes)$',
+ config.get('ARVADOS_API_HOST_INSECURE', 'no')):
+ ssl_options={'cert_reqs': ssl.CERT_NONE}
+ else:
+ ssl_options={'cert_reqs': ssl.CERT_REQUIRED}
+
+ super(EventClient, self).__init__(url, ssl_options)
+ self.filters = filters
+ self.on_event = on_event
+
+ def opened(self):
+ self.send(json.dumps({"method": "subscribe", "filters": self.filters}))
+
+ def received_message(self, m):
+ self.on_event(json.loads(str(m)))
+
+def subscribe(api, filters, on_event):
+ url = "{}?api_token={}".format(api._rootDesc['websocketUrl'], config.get('ARVADOS_API_TOKEN'))
+ ws = EventClient(url, filters, on_event)
+ ws.connect()
+ return ws
+++ /dev/null
-#
-# FUSE driver for Arvados Keep
-#
-
-import os
-import sys
-
-import llfuse
-import errno
-import stat
-import threading
-import arvados
-import pprint
-
-from time import time
-from llfuse import FUSEError
-
-class Directory(object):
- '''Generic directory object, backed by a dict.
- Consists of a set of entries with the key representing the filename
- and the value referencing a File or Directory object.
- '''
-
- def __init__(self, parent_inode):
- self.inode = None
- self.parent_inode = parent_inode
- self._entries = {}
-
- def __getitem__(self, item):
- return self._entries[item]
-
- def __setitem__(self, key, item):
- self._entries[key] = item
-
- def __iter__(self):
- return self._entries.iterkeys()
-
- def items(self):
- return self._entries.items()
-
- def __contains__(self, k):
- return k in self._entries
-
- def size(self):
- return 0
-
-class MagicDirectory(Directory):
- '''A special directory that logically contains the set of all extant
- keep locators. When a file is referenced by lookup(), it is tested
- to see if it is a valid keep locator to a manifest, and if so, loads the manifest
- contents as a subdirectory of this directory with the locator as the directory name.
- Since querying a list of all extant keep locators is impractical, only loaded collections
- are visible to readdir().'''
-
- def __init__(self, parent_inode, inodes):
- super(MagicDirectory, self).__init__(parent_inode)
- self.inodes = inodes
-
- def __contains__(self, k):
- if k in self._entries:
- return True
- try:
- if arvados.Keep.get(k):
- return True
- else:
- return False
- except Exception as e:
- #print 'exception keep', e
- return False
-
- def __getitem__(self, item):
- if item not in self._entries:
- collection = arvados.CollectionReader(arvados.Keep.get(item))
- self._entries[item] = self.inodes.add_entry(Directory(self.inode))
- self.inodes.load_collection(self._entries[item], collection)
- return self._entries[item]
-
-class File(object):
- '''Wraps a StreamFileReader for use by Directory.'''
-
- def __init__(self, parent_inode, reader):
- self.inode = None
- self.parent_inode = parent_inode
- self.reader = reader
-
- def size(self):
- return self.reader.size()
-
-class FileHandle(object):
- '''Connects a numeric file handle to a File or Directory object that has
- been opened by the client.'''
-
- def __init__(self, fh, entry):
- self.fh = fh
- self.entry = entry
-
-class Inodes(object):
- '''Manage the set of inodes. This is the mapping from a numeric id
- to a concrete File or Directory object'''
-
- def __init__(self):
- self._entries = {}
- self._counter = llfuse.ROOT_INODE
-
- def __getitem__(self, item):
- return self._entries[item]
-
- def __setitem__(self, key, item):
- self._entries[key] = item
-
- def __iter__(self):
- return self._entries.iterkeys()
-
- def items(self):
- return self._entries.items()
-
- def __contains__(self, k):
- return k in self._entries
-
- def load_collection(self, parent_dir, collection):
- '''parent_dir is the Directory object that will be populated by the collection.
- collection is the arvados.CollectionReader to use as the source'''
- for s in collection.all_streams():
- cwd = parent_dir
- for part in s.name().split('/'):
- if part != '' and part != '.':
- if part not in cwd:
- cwd[part] = self.add_entry(Directory(cwd.inode))
- cwd = cwd[part]
- for k, v in s.files().items():
- cwd[k] = self.add_entry(File(cwd.inode, v))
-
- def add_entry(self, entry):
- entry.inode = self._counter
- self._entries[entry.inode] = entry
- self._counter += 1
- return entry
-
-class Operations(llfuse.Operations):
- '''This is the main interface with llfuse. The methods on this object are
- called by llfuse threads to service FUSE events to query and read from
- the file system.
-
- llfuse has its own global lock which is acquired before calling a request handler,
- so request handlers do not run concurrently unless the lock is explicitly released
- with llfuse.lock_released.'''
-
- def __init__(self, uid, gid):
- super(Operations, self).__init__()
-
- self.inodes = Inodes()
- self.uid = uid
- self.gid = gid
-
- # dict of inode to filehandle
- self._filehandles = {}
- self._filehandles_counter = 1
-
- # Other threads that need to wait until the fuse driver
- # is fully initialized should wait() on this event object.
- self.initlock = threading.Event()
-
- def init(self):
- # Allow threads that are waiting for the driver to be finished
- # initializing to continue
- self.initlock.set()
-
- def access(self, inode, mode, ctx):
- return True
-
- def getattr(self, inode):
- e = self.inodes[inode]
-
- entry = llfuse.EntryAttributes()
- entry.st_ino = inode
- entry.generation = 0
- entry.entry_timeout = 300
- entry.attr_timeout = 300
-
- entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
- if isinstance(e, Directory):
- entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IFDIR
- else:
- entry.st_mode |= stat.S_IFREG
-
- entry.st_nlink = 1
- entry.st_uid = self.uid
- entry.st_gid = self.gid
- entry.st_rdev = 0
-
- entry.st_size = e.size()
-
- entry.st_blksize = 1024
- entry.st_blocks = e.size()/1024
- if e.size()/1024 != 0:
- entry.st_blocks += 1
- entry.st_atime = 0
- entry.st_mtime = 0
- entry.st_ctime = 0
-
- return entry
-
- def lookup(self, parent_inode, name):
- #print "lookup: parent_inode", parent_inode, "name", name
- inode = None
-
- if name == '.':
- inode = parent_inode
- else:
- if parent_inode in self.inodes:
- p = self.inodes[parent_inode]
- if name == '..':
- inode = p.parent_inode
- elif name in p:
- inode = p[name].inode
-
- if inode != None:
- return self.getattr(inode)
- else:
- raise llfuse.FUSEError(errno.ENOENT)
-
- def open(self, inode, flags):
- if inode in self.inodes:
- p = self.inodes[inode]
- else:
- raise llfuse.FUSEError(errno.ENOENT)
-
- if (flags & os.O_WRONLY) or (flags & os.O_RDWR):
- raise llfuse.FUSEError(errno.EROFS)
-
- if isinstance(p, Directory):
- raise llfuse.FUSEError(errno.EISDIR)
-
- fh = self._filehandles_counter
- self._filehandles_counter += 1
- self._filehandles[fh] = FileHandle(fh, p)
- return fh
-
- def read(self, fh, off, size):
- #print "read", fh, off, size
- if fh in self._filehandles:
- handle = self._filehandles[fh]
- else:
- raise llfuse.FUSEError(errno.EBADF)
-
- try:
- with llfuse.lock_released:
- return handle.entry.reader.readfrom(off, size)
- except:
- raise llfuse.FUSEError(errno.EIO)
-
- def release(self, fh):
- if fh in self._filehandles:
- del self._filehandles[fh]
-
- def opendir(self, inode):
- #print "opendir: inode", inode
-
- if inode in self.inodes:
- p = self.inodes[inode]
- else:
- raise llfuse.FUSEError(errno.ENOENT)
-
- if not isinstance(p, Directory):
- raise llfuse.FUSEError(errno.ENOTDIR)
-
- fh = self._filehandles_counter
- self._filehandles_counter += 1
- if p.parent_inode in self.inodes:
- parent = self.inodes[p.parent_inode]
- else:
- parent = None
- self._filehandles[fh] = FileHandle(fh, [('.', p), ('..', parent)] + list(p.items()))
- return fh
-
- def readdir(self, fh, off):
- #print "readdir: fh", fh, "off", off
-
- if fh in self._filehandles:
- handle = self._filehandles[fh]
- else:
- raise llfuse.FUSEError(errno.EBADF)
-
- #print "handle.entry", handle.entry
-
- e = off
- while e < len(handle.entry):
- yield (handle.entry[e][0], self.getattr(handle.entry[e][1].inode), e+1)
- e += 1
-
- def releasedir(self, fh):
- del self._filehandles[fh]
-
- def statfs(self):
- st = llfuse.StatvfsData()
- st.f_bsize = 1024 * 1024
- st.f_blocks = 0
- st.f_files = 0
-
- st.f_bfree = 0
- st.f_bavail = 0
-
- st.f_ffree = 0
- st.f_favail = 0
-
- st.f_frsize = 0
- return st
-
- # The llfuse documentation recommends only overloading functions that
- # are actually implemented, as the default implementation will raise ENOSYS.
- # However, there is a bug in the llfuse default implementation of create()
- # "create() takes exactly 5 positional arguments (6 given)" which will crash
- # arv-mount.
- # The workaround is to implement it with the proper number of parameters,
- # and then everything works out.
- def create(self, p1, p2, p3, p4, p5):
- raise llfuse.FUSEError(errno.EROFS)
+++ /dev/null
-#!/usr/bin/env python
-
-from arvados.fuse import *
-import arvados
-import subprocess
-import argparse
-
-if __name__ == '__main__':
- # Handle command line parameters
- parser = argparse.ArgumentParser(
- description='Mount Keep data under the local filesystem.',
- epilog="""
-Note: When using the --exec feature, you must either specify the
-mountpoint before --exec, or mark the end of your --exec arguments
-with "--".
-""")
- parser.add_argument('mountpoint', type=str, help="""Mount point.""")
- parser.add_argument('--collection', type=str, help="""Collection locator""")
- parser.add_argument('--debug', action='store_true', help="""Debug mode""")
- parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
- dest="exec_args", metavar=('command', 'args', '...', '--'),
- help="""Mount, run a command, then unmount and exit""")
-
- args = parser.parse_args()
-
- # Create the request handler
- operations = Operations(os.getuid(), os.getgid())
-
- if args.collection != None:
- # Set up the request handler with the collection at the root
- e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE))
- operations.inodes.load_collection(e, arvados.CollectionReader(arvados.Keep.get(args.collection)))
- else:
- # Set up the request handler with the 'magic directory' at the root
- operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
-
- # FUSE options, see mount.fuse(8)
- opts = []
-
- # Enable FUSE debugging (logs each FUSE request)
- if args.debug:
- opts += ['debug']
-
- # Initialize the fuse connection
- llfuse.init(operations, args.mountpoint, opts)
-
- if args.exec_args:
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- rc = 255
- try:
- rc = subprocess.call(args.exec_args, shell=False)
- except OSError as e:
- sys.stderr.write('arv-mount: %s -- exec %s\n' % (str(e), args.exec_args))
- rc = e.errno
- except Exception as e:
- sys.stderr.write('arv-mount: %s\n' % str(e))
- finally:
- subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
-
- exit(rc)
- else:
- llfuse.main()
+++ /dev/null
-#!/bin/sh
-#
-# Apparently the only reliable way to distribute Python packages with pypi and
-# install them via pip is as source packages (sdist).
-#
-# That means that setup.py is run on the system the package is being installed on,
-# outside of the Arvados git tree.
-#
-# In turn, this means that we can not build the minor_version on the fly when
-# setup.py is being executed. Instead, we use this script to generate a 'static'
-# version of setup.py which will can be distributed via pypi.
-
-minor_version=`git log --format=format:%ct.%h -n1 .`
-
-sed "s|%%MINOR_VERSION%%|$minor_version|" < setup.py.src > setup.py
-
-google-api-python-client==1.2
-httplib2==0.8
-python-gflags==2.0
-urllib3==1.7.1
-llfuse==0.40
+google-api-python-client>=1.2
+httplib2>=0.7
+python-gflags>=1.5
+urllib3>=1.3
+ws4py>=0.3
+PyYAML>=3.0
--- /dev/null
+import subprocess
+import time
+import os
+import signal
+import yaml
+import sys
+import argparse
+import arvados.config
+import arvados.api
+import shutil
+import tempfile
+
+ARV_API_SERVER_DIR = '../../services/api'
+KEEP_SERVER_DIR = '../../services/keep'
+SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
+WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
+
+def find_server_pid(PID_PATH, wait=10):
+ now = time.time()
+ timeout = now + wait
+ good_pid = False
+ while (not good_pid) and (now <= timeout):
+ time.sleep(0.2)
+ try:
+ with open(PID_PATH, 'r') as f:
+ server_pid = int(f.read())
+ good_pid = (os.kill(server_pid, 0) == None)
+ except IOError:
+ good_pid = False
+ except OSError:
+ good_pid = False
+ now = time.time()
+
+ if not good_pid:
+ return None
+
+ return server_pid
+
+def kill_server_pid(PID_PATH, wait=10):
+ try:
+ now = time.time()
+ timeout = now + wait
+ with open(PID_PATH, 'r') as f:
+ server_pid = int(f.read())
+ while now <= timeout:
+ os.kill(server_pid, signal.SIGTERM) == None
+ os.getpgid(server_pid) # throw OSError if no such pid
+ now = time.time()
+ time.sleep(0.1)
+ except IOError:
+ good_pid = False
+ except OSError:
+ good_pid = False
+
+def run(websockets=False, reuse_server=False):
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
+
+ if websockets:
+ pid_file = WEBSOCKETS_SERVER_PID_PATH
+ else:
+ pid_file = SERVER_PID_PATH
+
+ test_pid = find_server_pid(pid_file, 0)
+
+ if test_pid == None or not reuse_server:
+ # do not try to run both server variants at once
+ stop()
+
+ # delete cached discovery document
+ shutil.rmtree(arvados.http_cache('discovery'))
+
+ # Setup database
+ os.environ["RAILS_ENV"] = "test"
+ subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
+ subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
+ subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
+
+ if websockets:
+ os.environ["ARVADOS_WEBSOCKETS"] = "true"
+ subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
+ '-out', './self-signed.pem',
+ '-keyout', './self-signed.key',
+ '-days', '3650',
+ '-subj', '/CN=localhost'])
+ subprocess.call(['bundle', 'exec',
+ 'passenger', 'start', '-d', '-p3333',
+ '--pid-file',
+ os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
+ '--ssl',
+ '--ssl-certificate', 'self-signed.pem',
+ '--ssl-certificate-key', 'self-signed.key'])
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
+ else:
+ subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
+ '--pid',
+ os.path.join(os.getcwd(), SERVER_PID_PATH),
+ '-p3001'])
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+
+ pid = find_server_pid(SERVER_PID_PATH)
+
+ os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
+ os.environ["ARVADOS_API_TOKEN"] = ""
+ os.chdir(cwd)
+
+def stop():
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
+
+ kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
+ kill_server_pid(SERVER_PID_PATH, 0)
+
+ try:
+ os.unlink('self-signed.pem')
+ except:
+ pass
+
+ try:
+ os.unlink('self-signed.key')
+ except:
+ pass
+
+ os.chdir(cwd)
+
+def _start_keep(n):
+ keep0 = tempfile.mkdtemp()
+ kp0 = subprocess.Popen(["bin/keep", "-volumes={}".format(keep0), "-listen=:{}".format(25107+n)])
+ with open("tmp/keep{}.pid".format(n), 'w') as f:
+ f.write(str(kp0.pid))
+ with open("tmp/keep{}.volume".format(n), 'w') as f:
+ f.write(keep0)
+
+def run_keep():
+ stop_keep()
+
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ if os.environ.get('GOPATH') == None:
+ os.environ["GOPATH"] = os.getcwd()
+ else:
+ os.environ["GOPATH"] = os.getcwd() + ":" + os.environ["GOPATH"]
+
+ subprocess.call(["go", "install", "keep"])
+
+ if not os.path.exists("tmp"):
+ os.mkdir("tmp")
+
+ _start_keep(0)
+ _start_keep(1)
+
+ authorize_with("admin")
+ api = arvados.api('v1', cache=False)
+ a = api.keep_disks().list().execute()
+ for d in api.keep_disks().list().execute()['items']:
+ api.keep_disks().delete(uuid=d['uuid']).execute()
+
+ api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25107} }).execute()
+ api.keep_disks().create(body={"keep_disk": {"service_host": "localhost", "service_port": 25108} }).execute()
+
+ os.chdir(cwd)
+
+def _stop_keep(n):
+ kill_server_pid("tmp/keep{}.pid".format(n), 0)
+ if os.path.exists("tmp/keep{}.volume".format(n)):
+ with open("tmp/keep{}.volume".format(n), 'r') as r:
+ shutil.rmtree(r.read(), True)
+
+def stop_keep():
+ cwd = os.getcwd()
+ os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+
+ _stop_keep(0)
+ _stop_keep(1)
+
+ shutil.rmtree("tmp", True)
+
+ os.chdir(cwd)
+
+def fixture(fix):
+ '''load a fixture yaml file'''
+ with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",
+ fix + ".yml")) as f:
+ return yaml.load(f.read())
+
+def authorize_with(token):
+ '''token is the symbolic name of the token from the api_client_authorizations fixture'''
+ arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
+ arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
+ arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
+ parser.add_argument('--websockets', action='store_true', default=False)
+ parser.add_argument('--reuse', action='store_true', default=False)
+ parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
+ args = parser.parse_args()
+
+ if args.action == 'start':
+ run(websockets=args.websockets, reuse_server=args.reuse)
+ if args.auth != None:
+ authorize_with(args.auth)
+ print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
+ print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
+ print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
+ elif args.action == 'stop':
+ stop()
+ elif args.action == 'start_keep':
+ run_keep()
+ elif args.action == 'stop_keep':
+ stop_keep()
from setuptools import setup
-import subprocess
-
-minor_version = '%%MINOR_VERSION%%'
setup(name='arvados-python-client',
- version='0.1.' + minor_version,
+ version='0.1',
description='Arvados client library',
author='Arvados',
author_email='info@arvados.org',
scripts=[
'bin/arv-get',
'bin/arv-put',
- 'bin/arv-mount',
'bin/arv-ls',
'bin/arv-normalize',
],
'google-api-python-client',
'httplib2',
'urllib3',
- 'llfuse'
+ 'ws4py'
],
zip_safe=False)
import unittest
import arvados
import os
+import run_test_server
class KeepTestCase(unittest.TestCase):
- def setUp(self):
+ @classmethod
+ def setUpClass(cls):
try:
del os.environ['KEEP_LOCAL_STORE']
except KeyError:
pass
+ run_test_server.run()
+ run_test_server.run_keep()
-class KeepBasicRWTest(KeepTestCase):
- def runTest(self):
+ @classmethod
+ def tearDownClass(cls):
+ run_test_server.stop()
+ run_test_server.stop_keep()
+
+ def test_KeepBasicRWTest(self):
foo_locator = arvados.Keep.put('foo')
self.assertEqual(foo_locator,
'acbd18db4cc2f85cedef654fccc4a4d8+3',
'foo',
'wrong content from Keep.get(md5("foo"))')
-class KeepBinaryRWTest(KeepTestCase):
- def runTest(self):
+ def test_KeepBinaryRWTest(self):
blob_str = '\xff\xfe\xf7\x00\x01\x02'
blob_locator = arvados.Keep.put(blob_str)
self.assertEqual(blob_locator,
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
-class KeepLongBinaryRWTest(KeepTestCase):
- def runTest(self):
+ def test_KeepLongBinaryRWTest(self):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
for i in range(0,23):
blob_str = blob_str + blob_str
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
-class KeepSingleCopyRWTest(KeepTestCase):
- def runTest(self):
+ def test_KeepSingleCopyRWTest(self):
blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
blob_locator = arvados.Keep.put(blob_str, copies=1)
self.assertEqual(blob_locator,
+++ /dev/null
-import unittest
-import arvados
-import arvados.fuse as fuse
-import threading
-import time
-import os
-import llfuse
-import tempfile
-import shutil
-import subprocess
-import glob
-
-class FuseMountTest(unittest.TestCase):
- def setUp(self):
- self.keeptmp = tempfile.mkdtemp()
- os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
-
- cw = arvados.CollectionWriter()
-
- cw.start_new_file('thing1.txt')
- cw.write("data 1")
- cw.start_new_file('thing2.txt')
- cw.write("data 2")
- cw.start_new_stream('dir1')
-
- cw.start_new_file('thing3.txt')
- cw.write("data 3")
- cw.start_new_file('thing4.txt')
- cw.write("data 4")
-
- cw.start_new_stream('dir2')
- cw.start_new_file('thing5.txt')
- cw.write("data 5")
- cw.start_new_file('thing6.txt')
- cw.write("data 6")
-
- cw.start_new_stream('dir2/dir3')
- cw.start_new_file('thing7.txt')
- cw.write("data 7")
-
- cw.start_new_file('thing8.txt')
- cw.write("data 8")
-
- self.testcollection = cw.finish()
-
- def runTest(self):
- # Create the request handler
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.Directory(llfuse.ROOT_INODE))
- operations.inodes.load_collection(e, arvados.CollectionReader(arvados.Keep.get(self.testcollection)))
-
- self.mounttmp = tempfile.mkdtemp()
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- # now check some stuff
- d1 = os.listdir(self.mounttmp)
- d1.sort()
- self.assertEqual(d1, ['dir1', 'dir2', 'thing1.txt', 'thing2.txt'])
-
- d2 = os.listdir(os.path.join(self.mounttmp, 'dir1'))
- d2.sort()
- self.assertEqual(d2, ['thing3.txt', 'thing4.txt'])
-
- d3 = os.listdir(os.path.join(self.mounttmp, 'dir2'))
- d3.sort()
- self.assertEqual(d3, ['dir3', 'thing5.txt', 'thing6.txt'])
-
- d4 = os.listdir(os.path.join(self.mounttmp, 'dir2/dir3'))
- d4.sort()
- self.assertEqual(d4, ['thing7.txt', 'thing8.txt'])
-
- files = {'thing1.txt': 'data 1',
- 'thing2.txt': 'data 2',
- 'dir1/thing3.txt': 'data 3',
- 'dir1/thing4.txt': 'data 4',
- 'dir2/thing5.txt': 'data 5',
- 'dir2/thing6.txt': 'data 6',
- 'dir2/dir3/thing7.txt': 'data 7',
- 'dir2/dir3/thing8.txt': 'data 8'}
-
- for k, v in files.items():
- with open(os.path.join(self.mounttmp, k)) as f:
- self.assertEqual(f.read(), v)
-
-
- def tearDown(self):
- # llfuse.close is buggy, so use fusermount instead.
- #llfuse.close(unmount=True)
- subprocess.call(["fusermount", "-u", self.mounttmp])
-
- os.rmdir(self.mounttmp)
- shutil.rmtree(self.keeptmp)
-
-class FuseMagicTest(unittest.TestCase):
- def setUp(self):
- self.keeptmp = tempfile.mkdtemp()
- os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
-
- cw = arvados.CollectionWriter()
-
- cw.start_new_file('thing1.txt')
- cw.write("data 1")
-
- self.testcollection = cw.finish()
-
- def runTest(self):
- # Create the request handler
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
-
- self.mounttmp = tempfile.mkdtemp()
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
-
- # wait until the driver is finished initializing
- operations.initlock.wait()
-
- # now check some stuff
- d1 = os.listdir(self.mounttmp)
- d1.sort()
- self.assertEqual(d1, [])
-
- d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
- d2.sort()
- self.assertEqual(d2, ['thing1.txt'])
-
- d3 = os.listdir(self.mounttmp)
- d3.sort()
- self.assertEqual(d3, [self.testcollection])
-
- files = {}
- files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
-
- for k, v in files.items():
- with open(os.path.join(self.mounttmp, k)) as f:
- self.assertEqual(f.read(), v)
-
-
- def tearDown(self):
- # llfuse.close is buggy, so use fusermount instead.
- #llfuse.close(unmount=True)
- subprocess.call(["fusermount", "-u", self.mounttmp])
-
- os.rmdir(self.mounttmp)
- shutil.rmtree(self.keeptmp)
import unittest
import arvados
import apiclient
+import run_test_server
class PipelineTemplateTest(unittest.TestCase):
+ def setUp(self):
+ run_test_server.run()
+
def runTest(self):
- pt_uuid = arvados.api('v1').pipeline_templates().create(
+ run_test_server.authorize_with("admin")
+ pt_uuid = arvados.api('v1', cache=False).pipeline_templates().create(
body={'name':__file__}
).execute()['uuid']
self.assertEqual(len(pt_uuid), 27,
'spass_box': False,
'spass-box': [True, 'Maybe', False]
}
- update_response = arvados.api('v1').pipeline_templates().update(
+ update_response = arvados.api('v1', cache=False).pipeline_templates().update(
uuid=pt_uuid,
body={'components':components}
).execute()
self.assertEqual(update_response['name'], __file__,
'update() response has a different name (%s, not %s)'
% (update_response['name'], __file__))
- get_response = arvados.api('v1').pipeline_templates().get(
+ get_response = arvados.api('v1', cache=False).pipeline_templates().get(
uuid=pt_uuid
).execute()
self.assertEqual(get_response['components'], components,
'components got munged by server (%s -> %s)'
% (components, update_response['components']))
- delete_response = arvados.api('v1').pipeline_templates().delete(
+ delete_response = arvados.api('v1', cache=False).pipeline_templates().delete(
uuid=pt_uuid
).execute()
self.assertEqual(delete_response['uuid'], pt_uuid,
'delete() response has wrong uuid (%s, not %s)'
% (delete_response['uuid'], pt_uuid))
with self.assertRaises(apiclient.errors.HttpError):
- geterror_response = arvados.api('v1').pipeline_templates().get(
+ geterror_response = arvados.api('v1', cache=False).pipeline_templates().get(
uuid=pt_uuid
).execute()
+
+ def tearDown(self):
+ run_test_server.stop()
--- /dev/null
+import run_test_server
+import unittest
+import arvados
+import arvados.events
+import time
+
+class WebsocketTest(unittest.TestCase):
+ def setUp(self):
+ run_test_server.run(websockets=True)
+
+ def on_event(self, ev):
+ if self.state == 1:
+ self.assertEqual(200, ev['status'])
+ self.state = 2
+ elif self.state == 2:
+ self.assertEqual(self.h[u'uuid'], ev[u'object_uuid'])
+ self.state = 3
+ elif self.state == 3:
+ self.fail()
+
+ def runTest(self):
+ self.state = 1
+
+ run_test_server.authorize_with("admin")
+ api = arvados.api('v1', cache=False)
+ arvados.events.subscribe(api, [['object_uuid', 'is_a', 'arvados#human']], lambda ev: self.on_event(ev))
+ time.sleep(1)
+ self.h = api.humans().create(body={}).execute()
+ time.sleep(1)
+
+ def tearDown(self):
+ run_test_server.stop()
+Gemfile.lock
arvados*gem
+++ /dev/null
-PATH
- remote: .
- specs:
- arvados (0.1.20140228213600)
- activesupport (>= 3.2.13)
- andand
- google-api-client (~> 0.6.3)
- json (>= 1.7.7)
-
-GEM
- remote: https://rubygems.org/
- specs:
- activesupport (3.2.17)
- i18n (~> 0.6, >= 0.6.4)
- multi_json (~> 1.0)
- addressable (2.3.5)
- andand (1.3.3)
- autoparse (0.3.3)
- addressable (>= 2.3.1)
- extlib (>= 0.9.15)
- multi_json (>= 1.0.0)
- extlib (0.9.16)
- faraday (0.8.9)
- multipart-post (~> 1.2.0)
- google-api-client (0.6.4)
- addressable (>= 2.3.2)
- autoparse (>= 0.3.3)
- extlib (>= 0.9.15)
- faraday (~> 0.8.4)
- jwt (>= 0.1.5)
- launchy (>= 2.1.1)
- multi_json (>= 1.0.0)
- signet (~> 0.4.5)
- uuidtools (>= 2.1.0)
- i18n (0.6.9)
- json (1.8.1)
- jwt (0.1.11)
- multi_json (>= 1.5)
- launchy (2.4.2)
- addressable (~> 2.3)
- minitest (5.2.2)
- multi_json (1.8.4)
- multipart-post (1.2.0)
- rake (10.1.1)
- signet (0.4.5)
- addressable (>= 2.2.3)
- faraday (~> 0.8.1)
- jwt (>= 0.1.5)
- multi_json (>= 1.0.0)
- uuidtools (2.1.4)
-
-PLATFORMS
- ruby
-
-DEPENDENCIES
- arvados!
- minitest (>= 5.0.0)
- rake
s.email = 'gem-dev@curoverse.com'
s.licenses = ['Apache License, Version 2.0']
s.files = ["lib/arvados.rb"]
+ s.required_ruby_version = '>= 2.1.0'
s.add_dependency('google-api-client', '~> 0.6.3')
s.add_dependency('activesupport', '>= 3.2.13')
s.add_dependency('json', '>= 1.7.7')
end
def self.api_exec(method, parameters={})
api_method = arvados_api.send(api_models_sym).send(method.name.to_sym)
- parameters = parameters.
- merge(:api_token => arvados.config['ARVADOS_API_TOKEN'])
parameters.each do |k,v|
parameters[k] = v.to_json if v.is_a? Array or v.is_a? Hash
end
execute(:api_method => api_method,
:authenticated => false,
:parameters => parameters,
- :body => body)
+ :body => body,
+ :headers => {
+ authorization: 'OAuth2 '+arvados.config['ARVADOS_API_TOKEN']
+ })
resp = JSON.parse result.body, :symbolize_names => true
if resp[:errors]
raise Arvados::TransactionFailedError.new(resp[:errors])
/Capfile*
/config/deploy*
+# SimpleCov reports
+/coverage
+
+# Dev/test SSL certificates
+/self-signed.key
+/self-signed.pem
# gem 'rails', :git => 'git://github.com/rails/rails.git'
group :test, :development do
- gem 'sqlite3'
+ # Note: "require: false" here tells bunder not to automatically
+ # 'require' the packages during application startup. Installation is
+ # still mandatory.
+ gem 'simplecov', '~> 0.7.1', require: false
+ gem 'simplecov-rcov', require: false
end
# This might not be needed in :test and :development, but we load it
gem 'themes_for_rails'
gem 'arvados-cli', '>= 0.1.20140328152103'
+
+# pg_power lets us use partial indexes in schema.rb in Rails 3
+gem 'pg_power'
addressable (2.3.6)
andand (1.3.3)
arel (3.0.3)
- arvados (0.1.20140414145041)
+ arvados (0.1.20140513131358)
activesupport (>= 3.2.13)
andand
google-api-client (~> 0.6.3)
json (>= 1.7.7)
- arvados-cli (0.1.20140414145041)
+ arvados-cli (0.1.20140513131358)
activesupport (~> 3.2, >= 3.2.13)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1.0)
railties (>= 3.0, < 5.0)
thor (>= 0.14, < 2.0)
json (1.8.1)
- jwt (0.1.11)
+ jwt (0.1.13)
multi_json (>= 1.5)
launchy (2.4.2)
addressable (~> 2.3)
mime-types (~> 1.16)
treetop (~> 1.4.8)
mime-types (1.25.1)
- multi_json (1.9.2)
+ multi_json (1.10.0)
multipart-post (1.2.0)
net-scp (1.2.0)
net-ssh (>= 2.6.5)
jwt (~> 0.1.4)
multi_json (~> 1.0)
rack (~> 1.2)
- oj (2.7.3)
+ oj (2.9.0)
omniauth (1.1.1)
hashie (~> 1.2)
rack
rack
rake (>= 0.8.1)
pg (0.17.1)
+ pg_power (1.6.4)
+ pg
+ rails (~> 3.1)
polyglot (0.3.4)
rack (1.4.5)
rack-cache (1.2)
faraday (~> 0.8.1)
jwt (>= 0.1.5)
multi_json (>= 1.0.0)
+ simplecov (0.7.1)
+ multi_json (~> 1.0)
+ simplecov-html (~> 0.7.1)
+ simplecov-html (0.7.1)
+ simplecov-rcov (0.2.3)
+ simplecov (>= 0.4.1)
sprockets (2.2.2)
hike (~> 1.2)
multi_json (~> 1.0)
rack (~> 1.0)
tilt (~> 1.1, != 1.3.0)
- sqlite3 (1.3.9)
test_after_commit (0.2.3)
themes_for_rails (0.5.1)
rails (>= 3.0.0)
omniauth-oauth2 (= 1.1.1)
passenger
pg
+ pg_power
rails (~> 3.2.0)
redis
rvm-capistrano
sass-rails (>= 3.2.0)
- sqlite3
+ simplecov (~> 0.7.1)
+ simplecov-rcov
test_after_commit
themes_for_rails
therubyracer
require File.expand_path('../config/application', __FILE__)
+begin
+ ok = PgPower
+rescue
+ abort "Hm, pg_power is missing. Make sure you use 'bundle exec rake ...'"
+end
+
Server::Application.load_tasks
before_filter :catch_redirect_hint
before_filter(:find_object_by_uuid,
except: [:index, :create] + ERROR_ACTIONS)
- before_filter :load_limit_offset_order_params, only: [:index, :owned_items]
- before_filter :load_where_param, only: [:index, :owned_items]
- before_filter :load_filters_param, only: [:index, :owned_items]
+ before_filter :load_limit_offset_order_params, only: [:index, :contents]
+ before_filter :load_where_param, only: [:index, :contents]
+ before_filter :load_filters_param, only: [:index, :contents]
before_filter :find_objects_for_index, :only => :index
before_filter :reload_object_before_update, :only => :update
before_filter(:render_404_if_no_object,
show
end
- def self._owned_items_requires_parameters
- _index_requires_parameters.
- merge({
- include_linked: {
- type: 'boolean', required: false, default: false
- },
- })
- end
-
- def owned_items
- all_objects = []
- all_available = 0
-
- # Trick apply_where_limit_order_params into applying suitable
- # per-table values. *_all are the real ones we'll apply to the
- # aggregate set.
- limit_all = @limit
- offset_all = @offset
- @orders = []
-
- ArvadosModel.descendants.
- reject(&:abstract_class?).
- sort_by(&:to_s).
- each do |klass|
- case klass.to_s
- # We might expect klass==Link etc. here, but we would be
- # disappointed: when Rails reloads model classes, we get two
- # distinct classes called Link which do not equal each
- # other. But we can still rely on klass.to_s to be "Link".
- when 'ApiClientAuthorization'
- # Do not want.
- else
- @objects = klass.readable_by(*@read_users)
- cond_sql = "#{klass.table_name}.owner_uuid = ?"
- cond_params = [@object.uuid]
- if params[:include_linked]
- @objects = @objects.
- joins("LEFT JOIN links mng_links"\
- " ON mng_links.link_class=#{klass.sanitize 'permission'}"\
- " AND mng_links.name=#{klass.sanitize 'can_manage'}"\
- " AND mng_links.tail_uuid=#{klass.sanitize @object.uuid}"\
- " AND mng_links.head_uuid=#{klass.table_name}.uuid")
- cond_sql += " OR mng_links.uuid IS NOT NULL"
- end
- @objects = @objects.where(cond_sql, *cond_params).order(:uuid)
- @limit = limit_all - all_objects.count
- apply_where_limit_order_params
- items_available = @objects.
- except(:limit).except(:offset).
- count(:id, distinct: true)
- all_available += items_available
- @offset = [@offset - items_available, 0].max
-
- all_objects += @objects.to_a
- end
- end
- @objects = all_objects || []
- @object_list = {
- :kind => "arvados#objectList",
- :etag => "",
- :self_link => "",
- :offset => offset_all,
- :limit => limit_all,
- :items_available => all_available,
- :items => @objects.as_api_response(nil)
- }
- render json: @object_list
- end
-
def catch_redirect_hint
if !current_user
if params.has_key?('redirect_to') then
class Arvados::V1::GroupsController < ApplicationController
+
+ def self._contents_requires_parameters
+ _index_requires_parameters.
+ merge({
+ include_linked: {
+ type: 'boolean', required: false, default: false
+ },
+ })
+ end
+
+ def contents
+ all_objects = []
+ all_available = 0
+
+ # Trick apply_where_limit_order_params into applying suitable
+ # per-table values. *_all are the real ones we'll apply to the
+ # aggregate set.
+ limit_all = @limit
+ offset_all = @offset
+ @orders = []
+
+ ArvadosModel.descendants.reject(&:abstract_class?).sort_by(&:to_s).
+ each do |klass|
+ case klass.to_s
+ # We might expect klass==Link etc. here, but we would be
+ # disappointed: when Rails reloads model classes, we get two
+ # distinct classes called Link which do not equal each
+ # other. But we can still rely on klass.to_s to be "Link".
+ when 'ApiClientAuthorization', 'UserAgreement', 'Link'
+ # Do not want.
+ else
+ @objects = klass.readable_by(*@read_users)
+ cond_sql = "#{klass.table_name}.owner_uuid = ?"
+ cond_params = [@object.uuid]
+ if params[:include_linked]
+ cond_sql += " OR #{klass.table_name}.uuid IN (SELECT head_uuid FROM links WHERE link_class=#{klass.sanitize 'name'} AND links.tail_uuid=#{klass.sanitize @object.uuid})"
+ end
+ @objects = @objects.where(cond_sql, *cond_params).order("#{klass.table_name}.uuid")
+ @limit = limit_all - all_objects.count
+ apply_where_limit_order_params
+ items_available = @objects.
+ except(:limit).except(:offset).
+ count(:id, distinct: true)
+ all_available += items_available
+ @offset = [@offset - items_available, 0].max
+
+ all_objects += @objects.to_a
+ end
+ end
+ @objects = all_objects || []
+ @links = Link.where('link_class=? and tail_uuid=?'\
+ ' and head_uuid in (?)',
+ 'name',
+ @object.uuid,
+ @objects.collect(&:uuid))
+ @object_list = {
+ :kind => "arvados#objectList",
+ :etag => "",
+ :self_link => "",
+ :links => @links.as_api_response(nil),
+ :offset => offset_all,
+ :limit => limit_all,
+ :items_available => all_available,
+ :items => @objects.as_api_response(nil)
+ }
+ render json: @object_list
+ end
+
end
description: "The API to interact with Arvados.",
documentationLink: "http://doc.arvados.org/api/index.html",
protocol: "rest",
- baseUrl: root_url + "/arvados/v1/",
+ baseUrl: root_url + "arvados/v1/",
basePath: "/arvados/v1/",
rootUrl: root_url,
servicePath: "arvados/v1/",
if Rails.application.config.websocket_address
discovery[:websocketUrl] = Rails.application.config.websocket_address
elsif ENV['ARVADOS_WEBSOCKETS']
- discovery[:websocketUrl] = (root_url.sub /^http/, 'ws') + "/websocket"
+ discovery[:websocketUrl] = (root_url.sub /^http/, 'ws') + "websocket"
end
ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
limit: {
type: "integer",
description: "Maximum number of #{k.to_s.underscore.pluralize} to return.",
- default: 100,
+ default: "100",
format: "int32",
- minimum: 0,
+ minimum: "0",
location: "query",
},
offset: {
type: "integer",
description: "Number of #{k.to_s.underscore.pluralize} to skip before first returned record.",
- default: 0,
+ default: "0",
format: "int32",
- minimum: 0,
+ minimum: "0",
location: "query",
},
filters: {
else
method[:parameters][k] = {}
end
+ if !method[:parameters][k][:default].nil?
+ method[:parameters][k][:default] = 'string'
+ end
method[:parameters][k][:type] ||= 'string'
method[:parameters][k][:description] ||= ''
method[:parameters][k][:location] = (route.segment_keys.include?(k) ? 'path' : 'query')
# omniauth callback method
def create
omniauth = env['omniauth.auth']
- #logger.debug "+++ #{omniauth}"
identity_url_ok = (omniauth['info']['identity_url'].length > 0) rescue false
unless identity_url_ok
# "unauthorized":
Thread.current[:user] = user
- user.save!
+ user.save or raise Exception.new(user.errors.messages)
omniauth.delete('extra')
attr_protected :modified_by_client_uuid
attr_protected :modified_at
after_initialize :log_start_state
- before_create :ensure_permission_to_create
- before_update :ensure_permission_to_update
+ before_save :ensure_permission_to_save
+ before_save :ensure_owner_uuid_is_permitted
+ before_save :ensure_ownership_path_leads_to_user
+ before_destroy :ensure_owner_uuid_is_permitted
before_destroy :ensure_permission_to_destroy
before_create :update_modified_by_fields
self.columns.select { |col| col.name == attr.to_s }.first
end
+ # Return nil if current user is not allowed to see the list of
+ # writers. Otherwise, return a list of user_ and group_uuids with
+ # write permission. (If not returning nil, current_user is always in
+ # the list because can_manage permission is needed to see the list
+ # of writers.)
+ def writable_by
+ unless (owner_uuid == current_user.uuid or
+ current_user.is_admin or
+ current_user.groups_i_can(:manage).index(owner_uuid))
+ return nil
+ end
+ [owner_uuid, current_user.uuid] + permissions.collect do |p|
+ if ['can_write', 'can_manage'].index p.name
+ p.tail_uuid
+ end
+ end.compact.uniq
+ end
+
# Return a query with read permissions restricted to the union of of the
# permissions of the members of users_list, i.e. if something is readable by
# any user in users_list, it will be readable in the query returned by this
# A permission link exists ('write' and 'manage' implicitly include
# 'read') from a member of users_list, or a group readable by users_list,
# to this row, or to the owner of this row (see join() below).
+ permitted_uuids = "(SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (#{sanitized_uuid_list}))"
+
sql_conds += ["#{table_name}.owner_uuid in (?)",
"#{table_name}.uuid in (?)",
- "permissions.head_uuid IS NOT NULL"]
+ "#{table_name}.uuid IN #{permitted_uuids}"]
sql_params += [uuid_list, user_uuids]
if self == Link and users_list.any?
if self == Log and users_list.any?
# Link head points to the object described by this row
- or_object_uuid = ", #{table_name}.object_uuid"
+ sql_conds += ["#{table_name}.object_uuid IN #{permitted_uuids}"]
# This object described by this row is owned by this user, or owned by a group readable by this user
sql_conds += ["#{table_name}.object_owner_uuid in (?)"]
# user (the identity with authorization to read)
#
# Link class is 'permission' ('write' and 'manage' implicitly include 'read')
-
- joins("LEFT JOIN links permissions ON permissions.head_uuid in (#{table_name}.owner_uuid, #{table_name}.uuid #{or_object_uuid}) AND permissions.tail_uuid in (#{sanitized_uuid_list}) AND permissions.link_class='permission'")
- .where(sql_conds.join(' OR '), *sql_params).uniq
-
+ where(sql_conds.join(' OR '), *sql_params)
else
# At least one user is admin, so don't bother to apply any restrictions.
self
end
-
end
def logged_attributes
protected
- def ensure_permission_to_create
- raise PermissionDeniedError unless permission_to_create
+ def ensure_ownership_path_leads_to_user
+ if new_record? or owner_uuid_changed?
+ uuid_in_path = {owner_uuid => true, uuid => true}
+ x = owner_uuid
+ while (owner_class = self.class.resource_class_for_uuid(x)) != User
+ begin
+ if x == uuid
+ # Test for cycles with the new version, not the DB contents
+ x = owner_uuid
+ elsif !owner_class.respond_to? :find_by_uuid
+ raise ActiveRecord::RecordNotFound.new
+ else
+ x = owner_class.find_by_uuid(x).owner_uuid
+ end
+ rescue ActiveRecord::RecordNotFound => e
+ errors.add :owner_uuid, "is not owned by any user: #{e}"
+ return false
+ end
+ if uuid_in_path[x]
+ if x == owner_uuid
+ errors.add :owner_uuid, "would create an ownership cycle"
+ else
+ errors.add :owner_uuid, "has an ownership cycle"
+ end
+ return false
+ end
+ uuid_in_path[x] = true
+ end
+ end
+ true
end
- def permission_to_create
- current_user.andand.is_active
+ def ensure_owner_uuid_is_permitted
+ raise PermissionDeniedError if !current_user
+ self.owner_uuid ||= current_user.uuid
+ if self.owner_uuid_changed?
+ if current_user.uuid == self.owner_uuid or
+ current_user.can? write: self.owner_uuid
+ # current_user is, or has :write permission on, the new owner
+ else
+ logger.warn "User #{current_user.uuid} tried to change owner_uuid of #{self.class.to_s} #{self.uuid} to #{self.owner_uuid} but does not have permission to write to #{self.owner_uuid}"
+ raise PermissionDeniedError
+ end
+ end
+ if new_record?
+ return true
+ elsif current_user.uuid == self.owner_uuid_was or
+ current_user.uuid == self.uuid or
+ current_user.can? write: self.owner_uuid_was
+ # current user is, or has :write permission on, the previous owner
+ return true
+ else
+ logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} but does not have permission to write #{self.owner_uuid_was}"
+ raise PermissionDeniedError
+ end
+ end
+
+ def ensure_permission_to_save
+ unless (new_record? ? permission_to_create : permission_to_update)
+ raise PermissionDeniedError
+ end
end
- def ensure_permission_to_update
- raise PermissionDeniedError unless permission_to_update
+ def permission_to_create
+ current_user.andand.is_active
end
def permission_to_update
logger.warn "User #{current_user.uuid} tried to change uuid of #{self.class.to_s} #{self.uuid_was} to #{self.uuid}"
return false
end
- if self.owner_uuid_changed?
- if current_user.uuid == self.owner_uuid or
- current_user.can? write: self.owner_uuid
- # current_user is, or has :write permission on, the new owner
- else
- logger.warn "User #{current_user.uuid} tried to change owner_uuid of #{self.class.to_s} #{self.uuid} to #{self.owner_uuid} but does not have permission to write to #{self.owner_uuid}"
- return false
- end
- end
- if current_user.uuid == self.owner_uuid_was or
- current_user.uuid == self.uuid or
- current_user.can? write: self.owner_uuid_was
- # current user is, or has :write permission on, the previous owner
- return true
- else
- logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} but does not have permission to write #{self.owner_uuid_was}"
- return false
- end
+ return true
end
def ensure_permission_to_destroy
def maybe_update_modified_by_fields
update_modified_by_fields if self.changed? or self.new_record?
+ true
end
def update_modified_by_fields
self.modified_at = Time.now
self.modified_by_user_uuid = current_user ? current_user.uuid : nil
self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
+ true
end
def ensure_serialized_attribute_type
t.add :name
t.add :group_class
t.add :description
+ t.add :writable_by
end
end
after_create :maybe_invalidate_permissions_cache
after_destroy :maybe_invalidate_permissions_cache
attr_accessor :head_kind, :tail_kind
+ validate :name_link_has_valid_name
api_accessible :user, extend: :common do |t|
t.add :tail_uuid
User.invalidate_permissions_cache
end
end
+
+ def name_link_has_valid_name
+ if link_class == 'name'
+ unless name.is_a? String and !name.empty?
+ errors.add('name', 'must be a non-empty string')
+ end
+ else
+ true
+ end
+ end
end
end
def start!(ping_url_method)
- ensure_permission_to_update
+ ensure_permission_to_save
ping_url = ping_url_method.call({ id: self.uuid, ping_secret: self.info[:ping_secret] })
if (Rails.configuration.compute_node_ec2run_args and
Rails.configuration.compute_node_ami)
end
# Supported states for a pipeline instance
- New = 'New'
- Ready = 'Ready'
- RunningOnServer = 'RunningOnServer'
- RunningOnClient = 'RunningOnClient'
- Paused = 'Paused'
- Failed = 'Failed'
- Complete = 'Complete'
+ States =
+ [
+ (New = 'New'),
+ (Ready = 'Ready'),
+ (RunningOnServer = 'RunningOnServer'),
+ (RunningOnClient = 'RunningOnClient'),
+ (Paused = 'Paused'),
+ (Failed = 'Failed'),
+ (Complete = 'Complete'),
+ ]
def dependencies
dependency_search(self.components).keys
end
def self.queue
- self.where('active = true')
+ self.where("state = 'RunningOnServer'")
end
protected
end
def verify_status
- if active_changed?
- if self.active
- self.state = RunningOnServer
- else
- if self.components_look_ready?
- self.state = Ready
- else
- self.state = New
- end
- end
- elsif success_changed?
- if self.success
- self.active = false
- self.state = Complete
- else
- self.active = false
- self.state = Failed
- end
- elsif state_changed?
+ changed_attributes = self.changed
+
+ if 'state'.in? changed_attributes
case self.state
when New, Ready, Paused
- self.active = false
+ self.active = nil
self.success = nil
when RunningOnServer
self.active = true
self.success = nil
when RunningOnClient
- self.active = false
+ self.active = nil
self.success = nil
when Failed
self.active = false
else
return false
end
- elsif components_changed?
- if !self.state || self.state == New || !self.active
- if self.components_look_ready?
+ elsif 'success'.in? changed_attributes
+ logger.info "pipeline_instance changed_attributes has success for #{self.uuid}"
+ if self.success
+ self.active = false
+ self.state = Complete
+ else
+ self.active = false
+ self.state = Failed
+ end
+ elsif 'active'.in? changed_attributes
+ logger.info "pipeline_instance changed_attributes has active for #{self.uuid}"
+ if self.active
+ if self.state.in? [New, Ready, Paused]
+ self.state = RunningOnServer
+ end
+ else
+ if self.state == RunningOnServer # state was RunningOnServer
+ self.active = nil
+ self.state = Paused
+ elsif self.components_look_ready?
self.state = Ready
else
self.state = New
end
end
+ elsif new_record? and self.state.nil?
+ # No state, active, or success given
+ self.state = New
+ end
+
+ if new_record? or 'components'.in? changed_attributes
+ self.state ||= New
+ if self.state == New and self.components_look_ready?
+ self.state = Ready
+ end
+ end
+
+ if self.state.in?(States)
+ true
+ else
+ errors.add :state, "'#{state.inspect} must be one of: [#{States.join ', '}]"
+ false
end
end
def set_state_before_save
- if !self.state || self.state == New
+ if !self.state || self.state == New || self.state == Ready || self.state == Paused
if self.active
self.state = RunningOnServer
- elsif self.components_look_ready?
+ elsif self.components_look_ready? && (!self.state || self.state == New)
self.state = Ready
- else
- self.state = New
end
end
end
protected
+ def ensure_ownership_path_leads_to_user
+ true
+ end
+
def permission_to_update
# users must be able to update themselves (even if they are
# inactive) in order to create sessions
assets.version: "1.0"
arvados_theme: default
+
+ # Default: do not advertise a websocket server.
+ websocket_address: false
+
+ # You can run the websocket server separately from the regular HTTP service
+ # by setting "ARVADOS_WEBSOCKETS=ws-only" in the environment before running
+ # the websocket server. When you do this, you need to set the following
+ # configuration variable so that the primary server can give out the correct
+ # address of the dedicated websocket server:
+ #websocket_address: wss://127.0.0.1:3333/websocket
common:
#git_repositories_dir: /var/cache/git
#git_internal_dir: /var/cache/arvados/internal.git
-
- # You can run the websocket server separately from the regular HTTP service
- # by setting "ARVADOS_WEBSOCKETS=ws-only" in the environment before running
- # the websocket server. When you do this, you need to set the following
- # configuration variable so that the primary server can give out the correct
- # address of the dedicated websocket server:
- #websocket_address: wss://websocket.local/websocket
require 'eventbus'
+# See application.yml for details about configuring the websocket service.
+
Server::Application.configure do
# Enables websockets if ARVADOS_WEBSOCKETS is defined with any value. If
# ARVADOS_WEBSOCKETS=ws-only, server will only accept websocket connections
:websocket_only => (ENV['ARVADOS_WEBSOCKETS'] == "ws-only")
}
end
-
- # Define websocket_address configuration option, can be overridden in config files.
- # See application.yml.example for details.
- config.websocket_address = nil
end
get 'used_by', on: :member
end
resources :groups do
- get 'owned_items', on: :member
+ get 'contents', on: :member
end
resources :humans
resources :job_tasks
post 'activate', on: :member
post 'setup', on: :collection
post 'unsetup', on: :member
- get 'owned_items', on: :member
end
resources :virtual_machines do
get 'logins', on: :member
--- /dev/null
+class AddUniqueNameIndexToLinks < ActiveRecord::Migration
+ def change
+ # Make sure PgPower is here. Otherwise the "where" will be ignored
+ # and we'll end up with a far too restrictive unique
+ # constraint. (Rails4 should work without PgPower, but that isn't
+ # tested.)
+ if not PgPower then raise "No partial column support" end
+
+ add_index(:links, [:tail_uuid, :name], unique: true,
+ where: "link_class='name'",
+ name: 'links_tail_name_unique_if_link_class_name')
+ end
+end
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140423133559) do
+ActiveRecord::Schema.define(:version => 20140501165548) do
+
+
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
add_index "links", ["created_at"], :name => "index_links_on_created_at"
add_index "links", ["head_uuid"], :name => "index_links_on_head_uuid"
add_index "links", ["modified_at"], :name => "index_links_on_modified_at"
+ add_index "links", ["tail_uuid", "name"], :name => "links_tail_name_unique_if_link_class_name", :unique => true, :where => "((link_class)::text = 'name'::text)"
add_index "links", ["tail_uuid"], :name => "index_links_on_tail_uuid"
add_index "links", ["uuid"], :name => "index_links_on_uuid", :unique => true
add_index "virtual_machines", ["hostname"], :name => "index_virtual_machines_on_hostname"
add_index "virtual_machines", ["uuid"], :name => "index_virtual_machines_on_uuid", :unique => true
+
end
def system_user
if not $system_user
real_current_user = Thread.current[:user]
- Thread.current[:user] = User.new(is_admin: true, is_active: true)
+ Thread.current[:user] = User.new(is_admin: true,
+ is_active: true,
+ uuid: system_user_uuid)
$system_user = User.where('uuid=?', system_user_uuid).first
if !$system_user
$system_user = User.new(uuid: system_user_uuid,
when String
begin
@select = Oj.load params[:select]
- raise unless @select.is_a? Array
+ raise unless @select.is_a? Array or @select.nil?
rescue
raise ArgumentError.new("Could not parse \"select\" param as an array")
end
module RecordFilters
# Input:
- # +filters+ Arvados filters as list of lists.
+ # +filters+ array of conditions, each being [column, operator, operand]
# +ar_table_name+ name of SQL table
#
# Output:
raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
end
case operator.downcase
- when '=', '<', '<=', '>', '>=', 'like'
+ when '=', '<', '<=', '>', '>=', '!=', 'like'
if operand.is_a? String
+ if operator == '!='
+ operator = '<>'
+ end
cond_out << "#{ar_table_name}.#{attr} #{operator} ?"
if (# any operator that operates on value rather than
# representation:
param_out << operand
elsif operand.nil? and operator == '='
cond_out << "#{ar_table_name}.#{attr} is null"
+ elsif operand.nil? and operator == '!='
+ cond_out << "#{ar_table_name}.#{attr} is not null"
else
raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
"for '#{operator}' operator in filters")
end
- when 'in'
+ when 'in', 'not in'
if operand.is_a? Array
- cond_out << "#{ar_table_name}.#{attr} IN (?)"
+ cond_out << "#{ar_table_name}.#{attr} #{operator} (?)"
param_out << operand
+ if operator == 'not in' and not operand.include?(nil)
+ # explicitly allow NULL
+ cond_out[-1] = "(#{cond_out[-1]} OR #{ar_table_name}.#{attr} IS NULL)"
+ end
else
raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
"for '#{operator}' operator in filters")
j_done[:wait_thr].value
jobrecord = Job.find_by_uuid(job_done.uuid)
- jobrecord.running = false
- jobrecord.finished_at ||= Time.now
- # Don't set 'jobrecord.success = false' because if the job failed to run due to an
- # issue with crunch-job or slurm, we want the job to stay in the queue.
- jobrecord.save!
+ if jobrecord.started_at
+ # Clean up state fields in case crunch-job exited without
+ # putting the job in a suitable "finished" state.
+ jobrecord.running = false
+ jobrecord.finished_at ||= Time.now
+ if jobrecord.success.nil?
+ jobrecord.success = false
+ end
+ jobrecord.save!
+ else
+ # Don't fail the job if crunch-job didn't even get as far as
+ # starting it. If the job failed to run due to an infrastructure
+ # issue with crunch-job or slurm, we want the job to stay in the
+ # queue.
+ end
# Invalidate the per-job auth token
j_done[:job_auth].update_attributes expires_at: Time.now
+++ /dev/null
-#!/usr/bin/env ruby
-
-ENV["RAILS_ENV"] = ARGV[0] || ENV["RAILS_ENV"] || "development"
-
-require File.dirname(__FILE__) + '/../config/boot'
-require File.dirname(__FILE__) + '/../config/environment'
-require 'shellwords'
-
-Commit.import_all
trusted_workbench:
uuid: zzzzz-ozdt8-teyxzyd8qllg11h
+ owner_uuid: zzzzz-tpzed-000000000000000
name: Official Workbench
url_prefix: https://official-workbench.local/
is_trusted: true
untrusted:
uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ owner_uuid: zzzzz-tpzed-000000000000000
name: Untrusted
url_prefix: https://untrusted.local/
is_trusted: false
modified_at: 2014-04-21 15:37:48 -0400
updated_at: 2014-04-21 15:37:48 -0400
name: A Subfolder
- description: Test folder belonging to active user's first test folder
+ description: "Test folder belonging to active user's first test folder"
group_class: folder
+
+bad_group_has_ownership_cycle_a:
+ uuid: zzzzz-j7d0g-cx2al9cqkmsf1hs
+ owner_uuid: zzzzz-j7d0g-0077nzts8c178lw
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-05-03 18:50:08 -0400
+ modified_at: 2014-05-03 18:50:08 -0400
+ updated_at: 2014-05-03 18:50:08 -0400
+ name: Owned by bad group b
+
+bad_group_has_ownership_cycle_b:
+ uuid: zzzzz-j7d0g-0077nzts8c178lw
+ owner_uuid: zzzzz-j7d0g-cx2al9cqkmsf1hs
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-05-03 18:50:08 -0400
+ modified_at: 2014-05-03 18:50:08 -0400
+ updated_at: 2014-05-03 18:50:08 -0400
+ name: Owned by bad group a
tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
link_class: permission
name: can_read
- head_uuid: zzzzz-2x53u-382brsig8rp3666
+ head_uuid: zzzzz-s0uqq-382brsig8rp3666
properties: {}
foo_repository_writable_by_active:
specimen_is_in_two_folders:
uuid: zzzzz-o0j2j-ryhm1bn83ni03sn
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-j7d0g-axqo7eu9pwvna1x
created_at: 2014-04-21 15:37:48 -0400
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
modified_at: 2014-04-21 15:37:48 -0400
updated_at: 2014-04-21 15:37:48 -0400
tail_uuid: zzzzz-j7d0g-axqo7eu9pwvna1x
- head_uuid: zzzzz-2x53u-5gid26432uujf79
- link_class: permission
- name: can_manage
+ head_uuid: zzzzz-j58dm-5gid26432uujf79
+ link_class: name
+ name: "I'm in a subfolder, too"
+ properties: {}
+
+template_name_in_afolder:
+ uuid: zzzzz-o0j2j-4kpwf3d6rwkeqhl
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-29 16:47:26 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2014-04-29 16:47:26 -0400
+ updated_at: 2014-04-29 16:47:26 -0400
+ tail_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ head_uuid: zzzzz-p5p6p-aox0k0ofxrystgw
+ link_class: name
+ name: "I'm a template in a folder"
+ properties: {}
+
+job_name_in_afolder:
+ uuid: zzzzz-o0j2j-1kt6dppqcxbl1yt
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-29 16:47:26 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2014-04-29 16:47:26 -0400
+ updated_at: 2014-04-29 16:47:26 -0400
+ tail_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ head_uuid: zzzzz-8i9sb-pshmckwoma9plh7
+ link_class: name
+ name: "I'm a job in a folder"
+ properties: {}
+
+foo_collection_name_in_afolder:
+ uuid: zzzzz-o0j2j-foofoldername12
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2014-04-21 15:37:48 -0400
+ updated_at: 2014-04-21 15:37:48 -0400
+ tail_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ link_class: name
+ # This should resemble the default name assigned when a
+ # Collection is added to a Folder.
+ name: "1f4b0bc7583c2a7f9102c395f4ffc5e3+45 added sometime"
properties: {}
foo_collection_tag:
link_class: tag
name: foo_tag
properties: {}
+
+active_user_can_manage_bad_group_cx2al9cqkmsf1hs:
+ uuid: zzzzz-o0j2j-ezv55ahzc9lvjwe
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-05-03 18:50:08 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-05-03 18:50:08 -0400
+ updated_at: 2014-05-03 18:50:08 -0400
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_manage
+ head_uuid: zzzzz-j7d0g-cx2al9cqkmsf1hs
+ properties: {}
new_pipeline:
+ state: New
uuid: zzzzz-d1hrv-f4gneyn6br1xize
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
has_component_with_no_script_parameters:
+ state: Ready
uuid: zzzzz-d1hrv-1xfj6xkicf2muk2
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
components:
script_parameters: {}
has_component_with_empty_script_parameters:
+ state: Ready
uuid: zzzzz-d1hrv-jq16l10gcsnyumo
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
components:
foo:
- uuid: zzzzz-2x53u-382brsig8rp3666
+ uuid: zzzzz-s0uqq-382brsig8rp3666
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz # active user
name: foo
repository2:
- uuid: zzzzz-2x53u-382brsig8rp3667
+ uuid: zzzzz-s0uqq-382brsig8rp3667
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz # active user
name: foo2
owned_by_active_user:
- uuid: zzzzz-2x53u-3zx463qyo0k4xrn
+ uuid: zzzzz-j58dm-3zx463qyo0k4xrn
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_at: 2014-04-21 15:37:48 -0400
owned_by_private_group:
- uuid: zzzzz-2x53u-5m3qwg45g3nlpu6
+ uuid: zzzzz-j58dm-5m3qwg45g3nlpu6
owner_uuid: zzzzz-j7d0g-rew6elm53kancon
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_at: 2014-04-21 15:37:48 -0400
owned_by_spectator:
- uuid: zzzzz-2x53u-3b0xxwzlbzxq5yr
+ uuid: zzzzz-j58dm-3b0xxwzlbzxq5yr
owner_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_at: 2014-04-21 15:37:48 -0400
in_afolder:
- uuid: zzzzz-2x53u-7r18rnd5nzhg5yk
+ uuid: zzzzz-j58dm-7r18rnd5nzhg5yk
owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_at: 2014-04-21 15:37:48 -0400
in_asubfolder:
- uuid: zzzzz-2x53u-c40lddwcqqr1ffs
+ uuid: zzzzz-j58dm-c40lddwcqqr1ffs
owner_uuid: zzzzz-j7d0g-axqo7eu9pwvna1x
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_at: 2014-04-21 15:37:48 -0400
in_afolder_linked_from_asubfolder:
- uuid: zzzzz-2x53u-5gid26432uujf79
+ uuid: zzzzz-j58dm-5gid26432uujf79
owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_at: 2014-04-21 15:37:48 -0400
+
+owned_by_afolder_with_no_name_link:
+ uuid: zzzzz-j58dm-ypsjlol9dofwijz
+ owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ created_at: 2014-05-05 04:11:52 -0400
+ modified_at: 2014-05-05 04:11:52 -0400
--- /dev/null
+require 'test_helper'
+
+class Arvados::V1::FiltersTest < ActionController::TestCase
+ test '"not in" filter passes null values' do
+ @controller = Arvados::V1::GroupsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['group_class', 'not in', ['folder']] ],
+ controller: 'groups',
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_includes(found.collect(&:group_class), nil,
+ "'group_class not in ['folder']' filter should pass null")
+ end
+end
test 'get group-owned objects' do
authorize_with :active
- get :owned_items, {
+ get :contents, {
id: groups(:afolder).uuid,
format: :json,
+ include_linked: true,
}
assert_response :success
assert_operator 2, :<=, json_response['items_available']
assert_operator 2, :<=, json_response['items'].count
+ kinds = json_response['items'].collect { |i| i['kind'] }.uniq
+ expect_kinds = %w'arvados#group arvados#specimen arvados#pipelineTemplate arvados#job'
+ assert_equal expect_kinds, (expect_kinds & kinds)
end
test 'get group-owned objects with limit' do
authorize_with :active
- get :owned_items, {
+ get :contents, {
id: groups(:afolder).uuid,
limit: 1,
format: :json,
test 'get group-owned objects with limit and offset' do
authorize_with :active
- get :owned_items, {
+ get :contents, {
id: groups(:afolder).uuid,
limit: 1,
offset: 12345,
test 'get group-owned objects with additional filter matching nothing' do
authorize_with :active
- get :owned_items, {
+ get :contents, {
id: groups(:afolder).uuid,
filters: [['uuid', 'in', ['foo_not_a_uuid','bar_not_a_uuid']]],
format: :json,
test 'get group-owned objects without include_linked' do
unexpected_uuid = specimens(:in_afolder_linked_from_asubfolder).uuid
authorize_with :active
- get :owned_items, {
+ get :contents, {
id: groups(:asubfolder).uuid,
format: :json,
}
test 'get group-owned objects with include_linked' do
expected_uuid = specimens(:in_afolder_linked_from_asubfolder).uuid
authorize_with :active
- get :owned_items, {
+ get :contents, {
id: groups(:asubfolder).uuid,
include_linked: true,
format: :json,
assert_response :success
uuids = json_response['items'].collect { |i| i['uuid'] }
assert_includes uuids, expected_uuid, "Did not get #{expected_uuid}"
+
+ expected_name = links(:specimen_is_in_two_folders).name
+ found_specimen_name = false
+ assert(json_response['links'].any?,
+ "Expected a non-empty array of links in response")
+ json_response['links'].each do |link|
+ if link['head_uuid'] == expected_uuid
+ if link['name'] == expected_name
+ found_specimen_name = true
+ end
+ end
+ end
+ assert(found_specimen_name,
+ "Expected to find name '#{expected_name}' in response")
+ end
+
+ [false, true].each do |inc_ind|
+ test "get all pages of group-owned #{'and -linked ' if inc_ind}objects" do
+ authorize_with :active
+ limit = 5
+ offset = 0
+ items_available = nil
+ uuid_received = {}
+ owner_received = {}
+ while true
+ # Behaving badly here, using the same controller multiple
+ # times within a test.
+ @json_response = nil
+ get :contents, {
+ id: groups(:afolder).uuid,
+ include_linked: inc_ind,
+ limit: limit,
+ offset: offset,
+ format: :json,
+ }
+ assert_response :success
+ assert_operator(0, :<, json_response['items'].count,
+ "items_available=#{items_available} but received 0 "\
+ "items with offset=#{offset}")
+ items_available ||= json_response['items_available']
+ assert_equal(items_available, json_response['items_available'],
+ "items_available changed between page #{offset/limit} "\
+ "and page #{1+offset/limit}")
+ json_response['items'].each do |item|
+ uuid = item['uuid']
+ assert_equal(nil, uuid_received[uuid],
+ "Received '#{uuid}' again on page #{1+offset/limit}")
+ uuid_received[uuid] = true
+ owner_received[item['owner_uuid']] = true
+ offset += 1
+ if not inc_ind
+ assert_equal groups(:afolder).uuid, item['owner_uuid']
+ end
+ end
+ break if offset >= items_available
+ end
+ if inc_ind
+ assert_operator 0, :<, (json_response.keys - [users(:active).uuid]).count,
+ "Set include_linked=true but did not receive any non-owned items"
+ end
+ end
+ end
+
+ %w(offset limit).each do |arg|
+ ['foo', '', '1234five', '0x10', '-8'].each do |val|
+ test "Raise error on bogus #{arg} parameter #{val.inspect}" do
+ authorize_with :active
+ get :contents, {
+ :id => groups(:afolder).uuid,
+ :format => :json,
+ arg => val,
+ }
+ assert_response 422
+ end
+ end
+ end
+
+ test 'get writable_by list for owned group' do
+ authorize_with :active
+ get :show, {
+ id: groups(:afolder).uuid,
+ format: :json
+ }
+ assert_response :success
+ assert_not_nil(json_response['writable_by'],
+ "Should receive uuid list in 'writable_by' field")
+ assert_includes(json_response['writable_by'], users(:active).uuid,
+ "owner should be included in writable_by list")
+ end
+
+ test 'no writable_by list for group with read-only access' do
+ authorize_with :rominiadmin
+ get :show, {
+ id: groups(:testusergroup_admins).uuid,
+ format: :json
+ }
+ assert_response :success
+ assert_nil(json_response['writable_by'],
+ "Should not receive uuid list in 'writable_by' field")
+ end
+
+ test 'get writable_by list by admin user' do
+ authorize_with :admin
+ get :show, {
+ id: groups(:testusergroup_admins).uuid,
+ format: :json
+ }
+ assert_response :success
+ assert_not_nil(json_response['writable_by'],
+ "Should receive uuid list in 'writable_by' field")
+ assert_includes(json_response['writable_by'],
+ users(:admin).uuid,
+ "Current user should be included in 'writable_by' field")
end
end
'zzzzz-8i9sb-pshmckwoma9plh7']
end
+ test "search jobs by uuid with 'not in' query" do
+ exclude_uuids = [jobs(:running).uuid,
+ jobs(:running_cancelled).uuid]
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', 'not in', exclude_uuids]]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_not_empty found, "'not in' query returned nothing"
+ assert_empty(found & exclude_uuids,
+ "'not in' query returned uuids I asked not to get")
+ end
+
+ ['=', '!='].each do |operator|
+ [['uuid', 'zzzzz-8i9sb-pshmckwoma9plh7'],
+ ['output', nil]].each do |attr, operand|
+ test "search jobs with #{attr} #{operator} #{operand.inspect} query" do
+ authorize_with :active
+ get :index, {
+ filters: [[attr, operator, operand]]
+ }
+ assert_response :success
+ values = assigns(:objects).collect { |x| x.send(attr) }
+ assert_not_empty values, "query should return non-empty result"
+ if operator == '='
+ assert_empty values - [operand], "query results do not satisfy query"
+ else
+ assert_empty values & [operand], "query results do not satisfy query"
+ end
+ end
+ end
+ end
+
test "search jobs by started_at with < query" do
authorize_with :active
get :index, {
assert_response :success
end
+ test "refuse duplicate name" do
+ the_name = links(:job_name_in_afolder).name
+ the_folder = links(:job_name_in_afolder).tail_uuid
+ authorize_with :active
+ post :create, link: {
+ tail_uuid: the_folder,
+ head_uuid: specimens(:owned_by_active_user).uuid,
+ link_class: 'name',
+ name: the_name,
+ properties: {this_s: "a duplicate name"}
+ }
+ assert_response 422
+ end
end
tail_uuid: system_group_uuid,
head_uuid: user_uuid).count
end
-
- test 'get user-owned objects' do
- authorize_with :active
- get :owned_items, {
- id: users(:active).uuid,
- limit: 500,
- format: :json,
- }
- assert_response :success
- assert_operator 2, :<=, json_response['items_available']
- assert_operator 2, :<=, json_response['items'].count
- kinds = json_response['items'].collect { |i| i['kind'] }.uniq
- expect_kinds = %w'arvados#group arvados#specimen arvados#pipelineTemplate arvados#job'
- assert_equal expect_kinds, (expect_kinds & kinds)
- end
-
- [false, true].each do |inc_ind|
- test "get all pages of user-owned #{'and -linked ' if inc_ind}objects" do
- authorize_with :active
- limit = 5
- offset = 0
- items_available = nil
- uuid_received = {}
- owner_received = {}
- while true
- # Behaving badly here, using the same controller multiple
- # times within a test.
- @json_response = nil
- get :owned_items, {
- id: users(:active).uuid,
- include_linked: inc_ind,
- limit: limit,
- offset: offset,
- format: :json,
- }
- assert_response :success
- assert_operator(0, :<, json_response['items'].count,
- "items_available=#{items_available} but received 0 "\
- "items with offset=#{offset}")
- items_available ||= json_response['items_available']
- assert_equal(items_available, json_response['items_available'],
- "items_available changed between page #{offset/limit} "\
- "and page #{1+offset/limit}")
- json_response['items'].each do |item|
- uuid = item['uuid']
- assert_equal(nil, uuid_received[uuid],
- "Received '#{uuid}' again on page #{1+offset/limit}")
- uuid_received[uuid] = true
- owner_received[item['owner_uuid']] = true
- offset += 1
- if not inc_ind
- assert_equal users(:active).uuid, item['owner_uuid']
- end
- end
- break if offset >= items_available
- end
- if inc_ind
- assert_operator 0, :<, (json_response.keys - [users(:active).uuid]).count,
- "Set include_linked=true but did not receive any non-owned items"
- end
- end
- end
-
- %w(offset limit).each do |arg|
- ['foo', '', '1234five', '0x10', '-8'].each do |val|
- test "Raise error on bogus #{arg} parameter #{val.inspect}" do
- authorize_with :active
- get :owned_items, {
- :id => users(:active).uuid,
- :format => :json,
- arg => val,
- }
- assert_response 422
- end
- end
- end
end
--- /dev/null
+require 'test_helper'
+
+class UserSessionsApiTest < ActionDispatch::IntegrationTest
+ test 'create new user during omniauth callback' do
+ mock = {
+ 'provider' => 'josh_id',
+ 'uid' => 'https://edward.example.com',
+ 'info' => {
+ 'identity_url' => 'https://edward.example.com',
+ 'name' => 'Edward Example',
+ 'first_name' => 'Edward',
+ 'last_name' => 'Example',
+ 'email' => 'edward@example.com',
+ },
+ }
+ client_url = 'https://wb.example.com'
+ post('/auth/josh_id/callback',
+ {return_to: client_url},
+ {'omniauth.auth' => mock})
+ assert_response :redirect, 'Did not redirect to client with token'
+ assert_equal(0, @response.redirect_url.index(client_url),
+ 'Redirected to wrong address after succesful login: was ' +
+ @response.redirect_url + ', expected ' + client_url + '[...]')
+ assert_not_nil(@response.redirect_url.index('api_token='),
+ 'Expected api_token in query string of redirect url ' +
+ @response.redirect_url)
+ end
+end
ENV["RAILS_ENV"] = "test"
+unless ENV["NO_COVERAGE_TEST"]
+ begin
+ require 'simplecov'
+ require 'simplecov-rcov'
+ class SimpleCov::Formatter::MergedFormatter
+ def format(result)
+ SimpleCov::Formatter::HTMLFormatter.new.format(result)
+ SimpleCov::Formatter::RcovFormatter.new.format(result)
+ end
+ end
+ SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
+ SimpleCov.start do
+ add_filter '/test/'
+ add_filter 'initializers/secret_token'
+ add_filter 'initializers/omniauth'
+ end
+ rescue Exception => e
+ $stderr.puts "SimpleCov unavailable (#{e}). Proceeding without."
+ end
+end
+
require File.expand_path('../../config/environment', __FILE__)
require 'rails/test_help'
require 'test_helper'
class GroupTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+
+ test "cannot set owner_uuid to object with existing ownership cycle" do
+ set_user_from_auth :active_trustedclient
+
+ # First make sure we have lots of permission on the bad group by
+ # renaming it to "{current name} is mine all mine"
+ g = groups(:bad_group_has_ownership_cycle_b)
+ g.name += " is mine all mine"
+ assert g.save, "active user should be able to modify group #{g.uuid}"
+
+ # Use the group as the owner of a new object
+ s = Specimen.
+ create(owner_uuid: groups(:bad_group_has_ownership_cycle_b).uuid)
+ assert s.valid?, "ownership should pass validation"
+ assert_equal false, s.save, "should not save object with #{g.uuid} as owner"
+
+ # Use the group as the new owner of an existing object
+ s = specimens(:in_afolder)
+ s.owner_uuid = groups(:bad_group_has_ownership_cycle_b).uuid
+ assert s.valid?, "ownership should pass validation"
+ assert_equal false, s.save, "should not save object with #{g.uuid} as owner"
+ end
+
+ test "cannot create a new ownership cycle" do
+ set_user_from_auth :active_trustedclient
+
+ g_foo = Group.create(name: "foo")
+ g_foo.save!
+
+ g_bar = Group.create(name: "bar")
+ g_bar.save!
+
+ g_foo.owner_uuid = g_bar.uuid
+ assert g_foo.save, lambda { g_foo.errors.messages }
+ g_bar.owner_uuid = g_foo.uuid
+ assert g_bar.valid?, "ownership cycle should not prevent validation"
+ assert_equal false, g_bar.save, "should not create an ownership loop"
+ assert g_bar.errors.messages[:owner_uuid].join(" ").match(/ownership cycle/)
+ end
+
+ test "cannot create a single-object ownership cycle" do
+ set_user_from_auth :active_trustedclient
+
+ g_foo = Group.create(name: "foo")
+ assert g_foo.save
+
+ # Ensure I have permission to manage this group even when its owner changes
+ perm_link = Link.create(tail_uuid: users(:active).uuid,
+ head_uuid: g_foo.uuid,
+ link_class: 'permission',
+ name: 'can_manage')
+ assert perm_link.save
+
+ g_foo.owner_uuid = g_foo.uuid
+ assert_equal false, g_foo.save, "should not create an ownership loop"
+ assert g_foo.errors.messages[:owner_uuid].join(" ").match(/ownership cycle/)
+ end
+
end
require 'test_helper'
class LinkTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ fixtures :all
+
+ setup do
+ Thread.current[:user] = users(:active)
+ end
+
+ test 'name links with the same tail_uuid must be unique' do
+ a = Link.create!(tail_uuid: groups(:afolder).uuid,
+ head_uuid: specimens(:owned_by_active_user).uuid,
+ link_class: 'name',
+ name: 'foo')
+ assert a.valid?, a.errors.to_s
+ assert_raises ActiveRecord::RecordNotUnique do
+ b = Link.create!(tail_uuid: groups(:afolder).uuid,
+ head_uuid: specimens(:owned_by_active_user).uuid,
+ link_class: 'name',
+ name: 'foo')
+ end
+ end
+
+ test 'name links with different tail_uuid need not be unique' do
+ a = Link.create!(tail_uuid: groups(:afolder).uuid,
+ head_uuid: specimens(:owned_by_active_user).uuid,
+ link_class: 'name',
+ name: 'foo')
+ assert a.valid?, a.errors.to_s
+ b = Link.create!(tail_uuid: groups(:asubfolder).uuid,
+ head_uuid: specimens(:owned_by_active_user).uuid,
+ link_class: 'name',
+ name: 'foo')
+ assert b.valid?, b.errors.to_s
+ assert_not_equal(a.uuid, b.uuid,
+ "created two links and got the same uuid back.")
+ end
+
+ [nil, '', false].each do |name|
+ test "name links cannot have name=#{name.inspect}" do
+ a = Link.create(tail_uuid: groups(:afolder).uuid,
+ head_uuid: specimens(:owned_by_active_user).uuid,
+ link_class: 'name',
+ name: name)
+ assert a.invalid?, "invalid name was accepted as valid?"
+ end
+ end
end
test "check active and success for a pipeline in new state" do
pi = pipeline_instances :new_pipeline
- assert !pi.active, 'expected active to be false for a new pipeline'
- assert !pi.success, 'expected success to be false for a new pipeline'
- assert !pi.state, 'expected state to be nil because the fixture had no state specified'
+ assert !pi.active, 'expected active to be false for :new_pipeline'
+ assert !pi.success, 'expected success to be false for :new_pipeline'
+ assert_equal 'New', pi.state, 'expected state to be New for :new_pipeline'
# save the pipeline and expect state to be New
Thread.current[:user] = users(:admin)
assert !pi.success, 'expected success to be false for a new pipeline'
end
+ test "check active and success for a newly created pipeline" do
+ set_user_from_auth :active
+
+ pi = PipelineInstance.create(state: 'Ready')
+ pi.save
+
+ assert pi.valid?, 'expected newly created empty pipeline to be valid ' + pi.errors.messages.to_s
+ assert !pi.active, 'expected active to be false for a new pipeline'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+ assert_equal 'Ready', pi.state, 'expected state to be Ready for a new empty pipeline'
+ end
+
test "update attributes for pipeline" do
Thread.current[:user] = users(:admin)
assert !pi.success, 'expected success to be false for a new pipeline'
pi.active = true
- pi.save
+ assert_equal true, pi.save, 'expected pipeline instance to save, but ' + pi.errors.messages.to_s
pi = PipelineInstance.find_by_uuid 'zzzzz-d1hrv-f4gneyn6br1xize'
assert_equal PipelineInstance::RunningOnServer, pi.state, 'expected state to be RunningOnServer after updating active to true'
assert pi.active, 'expected active to be true after update'
Thread.current[:user] = users(:active)
# Make sure we go through the "active_changed? and active" code:
- pi.update_attributes active: true
- pi.update_attributes active: false
- assert_equal PipelineInstance::Ready, pi.state
+ assert_equal true, pi.update_attributes(active: true), pi.errors.messages
+ assert_equal true, pi.update_attributes(active: false), pi.errors.messages
+ assert_equal PipelineInstance::Paused, pi.state
end
end
end
--- /dev/null
+../../sdk/python/.gitignore
\ No newline at end of file
--- /dev/null
+#
+# FUSE driver for Arvados Keep
+#
+
+import os
+import sys
+
+import llfuse
+import errno
+import stat
+import threading
+import arvados
+import pprint
+import arvados.events
+import re
+import apiclient
+import json
+
+from time import time
+from llfuse import FUSEError
+
+class FreshBase(object):
+ '''Base class for maintaining fresh/stale state to determine when to update.'''
+ def __init__(self):
+ self._stale = True
+ self._poll = False
+ self._last_update = time()
+ self._poll_time = 60
+
+ # Mark the value as stale
+ def invalidate(self):
+ self._stale = True
+
+ # Test if the entries dict is stale
+ def stale(self):
+ if self._stale:
+ return True
+ if self._poll:
+ return (self._last_update + self._poll_time) < time()
+ return False
+
+ def fresh(self):
+ self._stale = False
+ self._last_update = time()
+
+
+class File(FreshBase):
+ '''Base for file objects.'''
+
+ def __init__(self, parent_inode):
+ super(File, self).__init__()
+ self.inode = None
+ self.parent_inode = parent_inode
+
+ def size(self):
+ return 0
+
+ def readfrom(self, off, size):
+ return ''
+
+
+class StreamReaderFile(File):
+ '''Wraps a StreamFileReader as a file.'''
+
+ def __init__(self, parent_inode, reader):
+ super(StreamReaderFile, self).__init__(parent_inode)
+ self.reader = reader
+
+ def size(self):
+ return self.reader.size()
+
+ def readfrom(self, off, size):
+ return self.reader.readfrom(off, size)
+
+ def stale(self):
+ return False
+
+
+class ObjectFile(File):
+ '''Wraps a dict as a serialized json object.'''
+
+ def __init__(self, parent_inode, contents):
+ super(ObjectFile, self).__init__(parent_inode)
+ self.contentsdict = contents
+ self.uuid = self.contentsdict['uuid']
+ self.contents = json.dumps(self.contentsdict, indent=4, sort_keys=True)
+
+ def size(self):
+ return len(self.contents)
+
+ def readfrom(self, off, size):
+ return self.contents[off:(off+size)]
+
+
+class Directory(FreshBase):
+ '''Generic directory object, backed by a dict.
+ Consists of a set of entries with the key representing the filename
+ and the value referencing a File or Directory object.
+ '''
+
+ def __init__(self, parent_inode):
+ super(Directory, self).__init__()
+
+ '''parent_inode is the integer inode number'''
+ self.inode = None
+ if not isinstance(parent_inode, int):
+ raise Exception("parent_inode should be an int")
+ self.parent_inode = parent_inode
+ self._entries = {}
+
+ # Overriden by subclasses to implement logic to update the entries dict
+ # when the directory is stale
+ def update(self):
+ pass
+
+ # Only used when computing the size of the disk footprint of the directory
+ # (stub)
+ def size(self):
+ return 0
+
+ def checkupdate(self):
+ if self.stale():
+ try:
+ self.update()
+ except apiclient.errors.HttpError as e:
+ print e
+
+ def __getitem__(self, item):
+ self.checkupdate()
+ return self._entries[item]
+
+ def items(self):
+ self.checkupdate()
+ return self._entries.items()
+
+ def __iter__(self):
+ self.checkupdate()
+ return self._entries.iterkeys()
+
+ def __contains__(self, k):
+ self.checkupdate()
+ return k in self._entries
+
+ def merge(self, items, fn, same, new_entry):
+ '''Helper method for updating the contents of the directory.
+
+ items: array with new directory contents
+
+ fn: function to take an entry in 'items' and return the desired file or
+ directory name
+
+ same: function to compare an existing entry with an entry in the items
+ list to determine whether to keep the existing entry.
+
+ new_entry: function to create a new directory entry from array entry.
+ '''
+
+ oldentries = self._entries
+ self._entries = {}
+ for i in items:
+ n = fn(i)
+ if n in oldentries and same(oldentries[n], i):
+ self._entries[n] = oldentries[n]
+ del oldentries[n]
+ else:
+ self._entries[n] = self.inodes.add_entry(new_entry(i))
+ for n in oldentries:
+ llfuse.invalidate_entry(self.inode, str(n))
+ self.inodes.del_entry(oldentries[n])
+ self.fresh()
+
+
+class CollectionDirectory(Directory):
+ '''Represents the root of a directory tree holding a collection.'''
+
+ def __init__(self, parent_inode, inodes, collection_locator):
+ super(CollectionDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.collection_locator = collection_locator
+
+ def same(self, i):
+ return i['uuid'] == self.collection_locator
+
+ def update(self):
+ collection = arvados.CollectionReader(arvados.Keep.get(self.collection_locator))
+ for s in collection.all_streams():
+ cwd = self
+ for part in s.name().split('/'):
+ if part != '' and part != '.':
+ if part not in cwd._entries:
+ cwd._entries[part] = self.inodes.add_entry(Directory(cwd.inode))
+ cwd = cwd._entries[part]
+ for k, v in s.files().items():
+ cwd._entries[k] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v))
+ self.fresh()
+
+
+class MagicDirectory(Directory):
+ '''A special directory that logically contains the set of all extant keep
+ locators. When a file is referenced by lookup(), it is tested to see if it
+ is a valid keep locator to a manifest, and if so, loads the manifest
+ contents as a subdirectory of this directory with the locator as the
+ directory name. Since querying a list of all extant keep locators is
+ impractical, only collections that have already been accessed are visible
+ to readdir().
+ '''
+
+ def __init__(self, parent_inode, inodes):
+ super(MagicDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+
+ def __contains__(self, k):
+ if k in self._entries:
+ return True
+ try:
+ if arvados.Keep.get(k):
+ return True
+ else:
+ return False
+ except Exception as e:
+ #print 'exception keep', e
+ return False
+
+ def __getitem__(self, item):
+ if item not in self._entries:
+ self._entries[item] = self.inodes.add_entry(CollectionDirectory(self.inode, self.inodes, item))
+ return self._entries[item]
+
+
+class TagsDirectory(Directory):
+ '''A special directory that contains as subdirectories all tags visible to the user.'''
+
+ def __init__(self, parent_inode, inodes, api, poll_time=60):
+ super(TagsDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ try:
+ arvados.events.subscribe(self.api, [['object_uuid', 'is_a', 'arvados#link']], lambda ev: self.invalidate())
+ except:
+ self._poll = True
+ self._poll_time = poll_time
+
+ def invalidate(self):
+ with llfuse.lock:
+ super(TagsDirectory, self).invalidate()
+ for a in self._entries:
+ self._entries[a].invalidate()
+
+ def update(self):
+ tags = self.api.links().list(filters=[['link_class', '=', 'tag']], select=['name'], distinct = True).execute()
+ self.merge(tags['items'],
+ lambda i: i['name'],
+ lambda a, i: a.tag == i,
+ lambda i: TagDirectory(self.inode, self.inodes, self.api, i['name'], poll=self._poll, poll_time=self._poll_time))
+
+class TagDirectory(Directory):
+ '''A special directory that contains as subdirectories all collections visible
+ to the user that are tagged with a particular tag.
+ '''
+
+ def __init__(self, parent_inode, inodes, api, tag, poll=False, poll_time=60):
+ super(TagDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ self.tag = tag
+ self._poll = poll
+ self._poll_time = poll_time
+
+ def update(self):
+ taggedcollections = self.api.links().list(filters=[['link_class', '=', 'tag'],
+ ['name', '=', self.tag],
+ ['head_uuid', 'is_a', 'arvados#collection']],
+ select=['head_uuid']).execute()
+ self.merge(taggedcollections['items'],
+ lambda i: i['head_uuid'],
+ lambda a, i: a.collection_locator == i['head_uuid'],
+ lambda i: CollectionDirectory(self.inode, self.inodes, i['head_uuid']))
+
+
+class GroupsDirectory(Directory):
+ '''A special directory that contains as subdirectories all groups visible to the user.'''
+
+ def __init__(self, parent_inode, inodes, api, poll_time=60):
+ super(GroupsDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ try:
+ arvados.events.subscribe(self.api, [], lambda ev: self.invalidate())
+ except:
+ self._poll = True
+ self._poll_time = poll_time
+
+ def invalidate(self):
+ with llfuse.lock:
+ super(GroupsDirectory, self).invalidate()
+ for a in self._entries:
+ self._entries[a].invalidate()
+
+ def update(self):
+ groups = self.api.groups().list().execute()
+ self.merge(groups['items'],
+ lambda i: i['uuid'],
+ lambda a, i: a.uuid == i['uuid'],
+ lambda i: GroupDirectory(self.inode, self.inodes, self.api, i, poll=self._poll, poll_time=self._poll_time))
+
+
+class GroupDirectory(Directory):
+ '''A special directory that contains the contents of a group.'''
+
+ def __init__(self, parent_inode, inodes, api, uuid, poll=False, poll_time=60):
+ super(GroupDirectory, self).__init__(parent_inode)
+ self.inodes = inodes
+ self.api = api
+ self.uuid = uuid['uuid']
+ self._poll = poll
+ self._poll_time = poll_time
+
+ def invalidate(self):
+ with llfuse.lock:
+ super(GroupDirectory, self).invalidate()
+ for a in self._entries:
+ self._entries[a].invalidate()
+
+ def createDirectory(self, i):
+ if re.match(r'[0-9a-f]{32}\+\d+', i['uuid']):
+ return CollectionDirectory(self.inode, self.inodes, i['uuid'])
+ elif re.match(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}', i['uuid']):
+ return GroupDirectory(self.parent_inode, self.inodes, self.api, i, self._poll, self._poll_time)
+ elif re.match(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}', i['uuid']):
+ return ObjectFile(self.parent_inode, i)
+ return None
+
+ def update(self):
+ contents = self.api.groups().contents(uuid=self.uuid, include_linked=True).execute()
+ links = {}
+ for a in contents['links']:
+ links[a['head_uuid']] = a['name']
+
+ def choose_name(i):
+ if i['uuid'] in links:
+ return links[i['uuid']]
+ else:
+ return i['uuid']
+
+ def same(a, i):
+ if isinstance(a, CollectionDirectory):
+ return a.collection_locator == i['uuid']
+ elif isinstance(a, GroupDirectory):
+ return a.uuid == i['uuid']
+ elif isinstance(a, ObjectFile):
+ return a.uuid == i['uuid'] and not a.stale()
+ return False
+
+ self.merge(contents['items'],
+ choose_name,
+ same,
+ self.createDirectory)
+
+
+class FileHandle(object):
+ '''Connects a numeric file handle to a File or Directory object that has
+ been opened by the client.'''
+
+ def __init__(self, fh, entry):
+ self.fh = fh
+ self.entry = entry
+
+
+class Inodes(object):
+ '''Manage the set of inodes. This is the mapping from a numeric id
+ to a concrete File or Directory object'''
+
+ def __init__(self):
+ self._entries = {}
+ self._counter = llfuse.ROOT_INODE
+
+ def __getitem__(self, item):
+ return self._entries[item]
+
+ def __setitem__(self, key, item):
+ self._entries[key] = item
+
+ def __iter__(self):
+ return self._entries.iterkeys()
+
+ def items(self):
+ return self._entries.items()
+
+ def __contains__(self, k):
+ return k in self._entries
+
+ def add_entry(self, entry):
+ entry.inode = self._counter
+ self._entries[entry.inode] = entry
+ self._counter += 1
+ return entry
+
+ def del_entry(self, entry):
+ llfuse.invalidate_inode(entry.inode)
+ del self._entries[entry.inode]
+
+class Operations(llfuse.Operations):
+ '''This is the main interface with llfuse. The methods on this object are
+ called by llfuse threads to service FUSE events to query and read from
+ the file system.
+
+ llfuse has its own global lock which is acquired before calling a request handler,
+ so request handlers do not run concurrently unless the lock is explicitly released
+ with llfuse.lock_released.'''
+
+ def __init__(self, uid, gid):
+ super(Operations, self).__init__()
+
+ self.inodes = Inodes()
+ self.uid = uid
+ self.gid = gid
+
+ # dict of inode to filehandle
+ self._filehandles = {}
+ self._filehandles_counter = 1
+
+ # Other threads that need to wait until the fuse driver
+ # is fully initialized should wait() on this event object.
+ self.initlock = threading.Event()
+
+ def init(self):
+ # Allow threads that are waiting for the driver to be finished
+ # initializing to continue
+ self.initlock.set()
+
+ def access(self, inode, mode, ctx):
+ return True
+
+ def getattr(self, inode):
+ if inode not in self.inodes:
+ raise llfuse.FUSEError(errno.ENOENT)
+
+ e = self.inodes[inode]
+
+ entry = llfuse.EntryAttributes()
+ entry.st_ino = inode
+ entry.generation = 0
+ entry.entry_timeout = 300
+ entry.attr_timeout = 300
+
+ entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
+ if isinstance(e, Directory):
+ entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IFDIR
+ else:
+ entry.st_mode |= stat.S_IFREG
+
+ entry.st_nlink = 1
+ entry.st_uid = self.uid
+ entry.st_gid = self.gid
+ entry.st_rdev = 0
+
+ entry.st_size = e.size()
+
+ entry.st_blksize = 1024
+ entry.st_blocks = e.size()/1024
+ if e.size()/1024 != 0:
+ entry.st_blocks += 1
+ entry.st_atime = 0
+ entry.st_mtime = 0
+ entry.st_ctime = 0
+
+ return entry
+
+ def lookup(self, parent_inode, name):
+ #print "lookup: parent_inode", parent_inode, "name", name
+ inode = None
+
+ if name == '.':
+ inode = parent_inode
+ else:
+ if parent_inode in self.inodes:
+ p = self.inodes[parent_inode]
+ if name == '..':
+ inode = p.parent_inode
+ elif name in p:
+ inode = p[name].inode
+
+ if inode != None:
+ return self.getattr(inode)
+ else:
+ raise llfuse.FUSEError(errno.ENOENT)
+
+ def open(self, inode, flags):
+ if inode in self.inodes:
+ p = self.inodes[inode]
+ else:
+ raise llfuse.FUSEError(errno.ENOENT)
+
+ if (flags & os.O_WRONLY) or (flags & os.O_RDWR):
+ raise llfuse.FUSEError(errno.EROFS)
+
+ if isinstance(p, Directory):
+ raise llfuse.FUSEError(errno.EISDIR)
+
+ fh = self._filehandles_counter
+ self._filehandles_counter += 1
+ self._filehandles[fh] = FileHandle(fh, p)
+ return fh
+
+ def read(self, fh, off, size):
+ #print "read", fh, off, size
+ if fh in self._filehandles:
+ handle = self._filehandles[fh]
+ else:
+ raise llfuse.FUSEError(errno.EBADF)
+
+ try:
+ with llfuse.lock_released:
+ return handle.entry.readfrom(off, size)
+ except:
+ raise llfuse.FUSEError(errno.EIO)
+
+ def release(self, fh):
+ if fh in self._filehandles:
+ del self._filehandles[fh]
+
+ def opendir(self, inode):
+ #print "opendir: inode", inode
+
+ if inode in self.inodes:
+ p = self.inodes[inode]
+ else:
+ raise llfuse.FUSEError(errno.ENOENT)
+
+ if not isinstance(p, Directory):
+ raise llfuse.FUSEError(errno.ENOTDIR)
+
+ fh = self._filehandles_counter
+ self._filehandles_counter += 1
+ if p.parent_inode in self.inodes:
+ parent = self.inodes[p.parent_inode]
+ else:
+ raise llfuse.FUSEError(errno.EIO)
+
+ self._filehandles[fh] = FileHandle(fh, [('.', p), ('..', parent)] + list(p.items()))
+ return fh
+
+ def readdir(self, fh, off):
+ #print "readdir: fh", fh, "off", off
+
+ if fh in self._filehandles:
+ handle = self._filehandles[fh]
+ else:
+ raise llfuse.FUSEError(errno.EBADF)
+
+ #print "handle.entry", handle.entry
+
+ e = off
+ while e < len(handle.entry):
+ if handle.entry[e][1].inode in self.inodes:
+ yield (handle.entry[e][0], self.getattr(handle.entry[e][1].inode), e+1)
+ e += 1
+
+ def releasedir(self, fh):
+ del self._filehandles[fh]
+
+ def statfs(self):
+ st = llfuse.StatvfsData()
+ st.f_bsize = 1024 * 1024
+ st.f_blocks = 0
+ st.f_files = 0
+
+ st.f_bfree = 0
+ st.f_bavail = 0
+
+ st.f_ffree = 0
+ st.f_favail = 0
+
+ st.f_frsize = 0
+ return st
+
+ # The llfuse documentation recommends only overloading functions that
+ # are actually implemented, as the default implementation will raise ENOSYS.
+ # However, there is a bug in the llfuse default implementation of create()
+ # "create() takes exactly 5 positional arguments (6 given)" which will crash
+ # arv-mount.
+ # The workaround is to implement it with the proper number of parameters,
+ # and then everything works out.
+ def create(self, p1, p2, p3, p4, p5):
+ raise llfuse.FUSEError(errno.EROFS)
--- /dev/null
+#!/usr/bin/env python
+
+from arvados_fuse import *
+import arvados
+import subprocess
+import argparse
+import daemon
+
+if __name__ == '__main__':
+ # Handle command line parameters
+ parser = argparse.ArgumentParser(
+ description='''Mount Keep data under the local filesystem. By default, if neither
+ --collection or --tags is specified, this mounts as a virtual directory
+ under which all Keep collections are available as subdirectories named
+ with the Keep locator; however directories will not be visible to 'ls'
+ until a program tries to access them.''',
+ epilog="""
+Note: When using the --exec feature, you must either specify the
+mountpoint before --exec, or mark the end of your --exec arguments
+with "--".
+""")
+ parser.add_argument('mountpoint', type=str, help="""Mount point.""")
+ parser.add_argument('--allow-other', action='store_true',
+ help="""Let other users read the mount""")
+ parser.add_argument('--collection', type=str, help="""Mount only the specified collection at the mount point.""")
+ parser.add_argument('--tags', action='store_true', help="""Mount as a virtual directory consisting of subdirectories representing tagged
+collections on the server.""")
+ parser.add_argument('--groups', action='store_true', help="""Mount as a virtual directory consisting of subdirectories representing groups on the server.""")
+ parser.add_argument('--debug', action='store_true', help="""Debug mode""")
+ parser.add_argument('--foreground', action='store_true', help="""Run in foreground (default is to daemonize unless --exec specified)""", default=False)
+ parser.add_argument('--exec', type=str, nargs=argparse.REMAINDER,
+ dest="exec_args", metavar=('command', 'args', '...', '--'),
+ help="""Mount, run a command, then unmount and exit""")
+
+ args = parser.parse_args()
+
+ # Create the request handler
+ operations = Operations(os.getuid(), os.getgid())
+
+ if args.groups:
+ api = arvados.api('v1')
+ e = operations.inodes.add_entry(GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+ elif args.tags:
+ api = arvados.api('v1')
+ e = operations.inodes.add_entry(TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+ elif args.collection != None:
+ # Set up the request handler with the collection at the root
+ e = operations.inodes.add_entry(CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, args.collection))
+ else:
+ # Set up the request handler with the 'magic directory' at the root
+ operations.inodes.add_entry(MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
+
+ # FUSE options, see mount.fuse(8)
+ opts = [optname for optname in ['allow_other', 'debug']
+ if getattr(args, optname)]
+
+ if args.exec_args:
+ # Initialize the fuse connection
+ llfuse.init(operations, args.mountpoint, opts)
+
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ rc = 255
+ try:
+ rc = subprocess.call(args.exec_args, shell=False)
+ except OSError as e:
+ sys.stderr.write('arv-mount: %s -- exec %s\n' % (str(e), args.exec_args))
+ rc = e.errno
+ except Exception as e:
+ sys.stderr.write('arv-mount: %s\n' % str(e))
+ finally:
+ subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
+
+ exit(rc)
+ else:
+ if args.foreground:
+ # Initialize the fuse connection
+ llfuse.init(operations, args.mountpoint, opts)
+ llfuse.main()
+ else:
+ with daemon.DaemonContext():
+ # Initialize the fuse connection
+ llfuse.init(operations, args.mountpoint, opts)
+ llfuse.main()
--- /dev/null
+arvados-python-client>=0.1
+llfuse>=0.37
+python-daemon>=1.5
--- /dev/null
+../../sdk/python/run_test_server.py
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup(name='arvados_fuse',
+ version='0.1',
+ description='Arvados FUSE driver',
+ author='Arvados',
+ author_email='info@arvados.org',
+ url="https://arvados.org",
+ download_url="https://github.com/curoverse/arvados.git",
+ license='GNU Affero General Public License, version 3.0',
+ packages=['arvados_fuse'],
+ scripts=[
+ 'bin/arv-mount'
+ ],
+ install_requires=[
+ 'arvados-python-client',
+ 'llfuse',
+ 'python-daemon'
+ ],
+ zip_safe=False)
--- /dev/null
+import unittest
+import arvados
+import arvados_fuse as fuse
+import threading
+import time
+import os
+import llfuse
+import tempfile
+import shutil
+import subprocess
+import glob
+import run_test_server
+import json
+
+class MountTestBase(unittest.TestCase):
+ def setUp(self):
+ self.keeptmp = tempfile.mkdtemp()
+ os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
+ self.mounttmp = tempfile.mkdtemp()
+
+ def tearDown(self):
+ # llfuse.close is buggy, so use fusermount instead.
+ #llfuse.close(unmount=True)
+ subprocess.call(["fusermount", "-u", self.mounttmp])
+
+ os.rmdir(self.mounttmp)
+ shutil.rmtree(self.keeptmp)
+
+
+class FuseMountTest(MountTestBase):
+ def setUp(self):
+ super(FuseMountTest, self).setUp()
+
+ cw = arvados.CollectionWriter()
+
+ cw.start_new_file('thing1.txt')
+ cw.write("data 1")
+ cw.start_new_file('thing2.txt')
+ cw.write("data 2")
+ cw.start_new_stream('dir1')
+
+ cw.start_new_file('thing3.txt')
+ cw.write("data 3")
+ cw.start_new_file('thing4.txt')
+ cw.write("data 4")
+
+ cw.start_new_stream('dir2')
+ cw.start_new_file('thing5.txt')
+ cw.write("data 5")
+ cw.start_new_file('thing6.txt')
+ cw.write("data 6")
+
+ cw.start_new_stream('dir2/dir3')
+ cw.start_new_file('thing7.txt')
+ cw.write("data 7")
+
+ cw.start_new_file('thing8.txt')
+ cw.write("data 8")
+
+ self.testcollection = cw.finish()
+
+ def runTest(self):
+ # Create the request handler
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.testcollection))
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ # now check some stuff
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(['dir1', 'dir2', 'thing1.txt', 'thing2.txt'], d1)
+
+ d2 = os.listdir(os.path.join(self.mounttmp, 'dir1'))
+ d2.sort()
+ self.assertEqual(['thing3.txt', 'thing4.txt'], d2)
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'dir2'))
+ d3.sort()
+ self.assertEqual(['dir3', 'thing5.txt', 'thing6.txt'], d3)
+
+ d4 = os.listdir(os.path.join(self.mounttmp, 'dir2/dir3'))
+ d4.sort()
+ self.assertEqual(['thing7.txt', 'thing8.txt'], d4)
+
+ files = {'thing1.txt': 'data 1',
+ 'thing2.txt': 'data 2',
+ 'dir1/thing3.txt': 'data 3',
+ 'dir1/thing4.txt': 'data 4',
+ 'dir2/thing5.txt': 'data 5',
+ 'dir2/thing6.txt': 'data 6',
+ 'dir2/dir3/thing7.txt': 'data 7',
+ 'dir2/dir3/thing8.txt': 'data 8'}
+
+ for k, v in files.items():
+ with open(os.path.join(self.mounttmp, k)) as f:
+ self.assertEqual(v, f.read())
+
+
+class FuseMagicTest(MountTestBase):
+ def setUp(self):
+ super(FuseMagicTest, self).setUp()
+
+ cw = arvados.CollectionWriter()
+
+ cw.start_new_file('thing1.txt')
+ cw.write("data 1")
+
+ self.testcollection = cw.finish()
+
+ def runTest(self):
+ # Create the request handler
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
+
+ self.mounttmp = tempfile.mkdtemp()
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ # now check some stuff
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual([], d1)
+
+ d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
+ d2.sort()
+ self.assertEqual(['thing1.txt'], d2)
+
+ d3 = os.listdir(self.mounttmp)
+ d3.sort()
+ self.assertEqual([self.testcollection], d3)
+
+ files = {}
+ files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
+
+ for k, v in files.items():
+ with open(os.path.join(self.mounttmp, k)) as f:
+ self.assertEqual(v, f.read())
+
+
+class FuseTagsTest(MountTestBase):
+ def setUp(self):
+ super(FuseTagsTest, self).setUp()
+
+ cw = arvados.CollectionWriter()
+
+ cw.start_new_file('foo')
+ cw.write("foo")
+
+ self.testcollection = cw.finish()
+
+ run_test_server.run()
+
+ def runTest(self):
+ run_test_server.authorize_with("admin")
+ api = arvados.api('v1', cache=False)
+
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(['foo_tag'], d1)
+
+ d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
+ d2.sort()
+ self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'], d2)
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
+ d3.sort()
+ self.assertEqual(['foo'], d3)
+
+ files = {}
+ files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
+
+ for k, v in files.items():
+ with open(os.path.join(self.mounttmp, k)) as f:
+ self.assertEqual(v, f.read())
+
+
+ def tearDown(self):
+ run_test_server.stop()
+
+ super(FuseTagsTest, self).tearDown()
+
+class FuseTagsUpdateTestBase(MountTestBase):
+
+ def runRealTest(self):
+ run_test_server.authorize_with("admin")
+ api = arvados.api('v1', cache=False)
+
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api, poll_time=1))
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertEqual(['foo_tag'], d1)
+
+ api.links().create(body={'link': {
+ 'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ 'link_class': 'tag',
+ 'name': 'bar_tag'
+ }}).execute()
+
+ time.sleep(1)
+
+ d2 = os.listdir(self.mounttmp)
+ d2.sort()
+ self.assertEqual(['bar_tag', 'foo_tag'], d2)
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d3.sort()
+ self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
+
+ l = api.links().create(body={'link': {
+ 'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
+ 'link_class': 'tag',
+ 'name': 'bar_tag'
+ }}).execute()
+
+ time.sleep(1)
+
+ d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d4.sort()
+ self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
+
+ api.links().delete(uuid=l['uuid']).execute()
+
+ time.sleep(1)
+
+ d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
+ d5.sort()
+ self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
+
+
+class FuseTagsUpdateTestWebsockets(FuseTagsUpdateTestBase):
+ def setUp(self):
+ super(FuseTagsUpdateTestWebsockets, self).setUp()
+ run_test_server.run(True)
+
+ def runTest(self):
+ self.runRealTest()
+
+ def tearDown(self):
+ run_test_server.stop()
+ super(FuseTagsUpdateTestWebsockets, self).tearDown()
+
+
+class FuseTagsUpdateTestPoll(FuseTagsUpdateTestBase):
+ def setUp(self):
+ super(FuseTagsUpdateTestPoll, self).setUp()
+ run_test_server.run(False)
+
+ def runTest(self):
+ self.runRealTest()
+
+ def tearDown(self):
+ run_test_server.stop()
+ super(FuseTagsUpdateTestPoll, self).tearDown()
+
+
+class FuseGroupsTest(MountTestBase):
+ def setUp(self):
+ super(FuseGroupsTest, self).setUp()
+ run_test_server.run()
+
+ def runTest(self):
+ run_test_server.authorize_with("admin")
+ api = arvados.api('v1', cache=False)
+
+ operations = fuse.Operations(os.getuid(), os.getgid())
+ e = operations.inodes.add_entry(fuse.GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
+
+ llfuse.init(operations, self.mounttmp, [])
+ t = threading.Thread(None, lambda: llfuse.main())
+ t.start()
+
+ # wait until the driver is finished initializing
+ operations.initlock.wait()
+
+ d1 = os.listdir(self.mounttmp)
+ d1.sort()
+ self.assertIn('zzzzz-j7d0g-v955i6s2oi1cbso', d1)
+
+ d2 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso'))
+ d2.sort()
+ self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45 added sometime',
+ "I'm a job in a folder",
+ "I'm a template in a folder",
+ "zzzzz-j58dm-5gid26432uujf79",
+ "zzzzz-j58dm-7r18rnd5nzhg5yk",
+ "zzzzz-j58dm-ypsjlol9dofwijz",
+ "zzzzz-j7d0g-axqo7eu9pwvna1x"
+ ], d2)
+
+ d3 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', 'zzzzz-j7d0g-axqo7eu9pwvna1x'))
+ d3.sort()
+ self.assertEqual(["I'm in a subfolder, too",
+ "zzzzz-j58dm-c40lddwcqqr1ffs"
+ ], d3)
+
+ with open(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', "I'm a template in a folder")) as f:
+ j = json.load(f)
+ self.assertEqual("Two Part Pipeline Template", j['name'])
+
+ def tearDown(self):
+ run_test_server.stop()
+ super(FuseGroupsTest, self).tearDown()
projects / arvados.git / commitdiff