Documentation for the migration process from Docker 1.9 or less to Docker 1.10+
Docker changed the format of the images, and there is an arvados tool arvados-migrate-docker19,
this commit explains how to migrate
refs #11305
--- /dev/null
+// On loading of a collection, enable the "lock" button and
+// disable all file modification controls (upload, rename, delete)
+$(document).
+ ready(function(event) {
+ $(".btn-collection-file-control").addClass("disabled");
+ $(".btn-collection-rename-file-span").attr("title", "Unlock collection to rename file");
+ $(".btn-collection-remove-file-span").attr("title", "Unlock collection to remove file");
+ $(".btn-remove-selected-files").attr("title", "Unlock collection to remove selected files");
+ $(".tab-pane-Upload").addClass("disabled");
+ $(".tab-pane-Upload").attr("title", "Unlock collection to upload files");
+ $("#Upload-tab").attr("data-toggle", "disabled");
+ }).
+ on('click', '.lock-collection-btn', function(event) {
+ classes = $(event.target).attr('class')
+
+ if (classes.indexOf("fa-lock") != -1) {
+ // About to unlock; warn and get confirmation from user
+ if (confirm("Adding, renaming, and deleting files changes the portable data hash. Are you sure you want to unlock the collection?")) {
+ $(".lock-collection-btn").removeClass("fa-lock");
+ $(".lock-collection-btn").addClass("fa-unlock");
+ $(".lock-collection-btn").attr("title", "Lock collection to prevent editing files");
+ $(".btn-collection-rename-file-span").attr("title", "");
+ $(".btn-collection-remove-file-span").attr("title", "");
+ $(".btn-collection-file-control").removeClass("disabled");
+ $(".btn-remove-selected-files").attr("title", "");
+ $(".tab-pane-Upload").removeClass("disabled");
+ $(".tab-pane-Upload").attr("data-original-title", "");
+ $("#Upload-tab").attr("data-toggle", "tab");
+ } else {
+ // User clicked "no" and so do not unlock
+ }
+ } else {
+ // Lock it back
+ $(".lock-collection-btn").removeClass("fa-unlock");
+ $(".lock-collection-btn").addClass("fa-lock");
+ $(".lock-collection-btn").attr("title", "Unlock collection to edit files");
+ $(".btn-collection-rename-file-span").attr("title", "Unlock collection to rename file");
+ $(".btn-collection-remove-file-span").attr("title", "Unlock collection to remove file");
+ $(".btn-collection-file-control").addClass("disabled");
+ $(".btn-remove-selected-files").attr("title", "Unlock collection to remove selected files");
+ $(".tab-pane-Upload").addClass("disabled");
+ $(".tab-pane-Upload").attr("data-original-title", "Unlock collection to upload files");
+ $("#Upload-tab").attr("data-toggle", "disabled");
+ }
+ });
function enable_disable_selection_actions() {
var $container = $(this);
var $checked = $('.persistent-selection:checkbox:checked', $container);
+ var collection_lock_classes = $('.lock-collection-btn').attr('class')
+
$('[data-selection-action]', $container).
closest('div.btn-group-sm').
find('ul li').
toggleClass('disabled',
($checked.filter('[value*=-4zz18-]').length < 1) ||
($checked.length != $checked.filter('[value*=-4zz18-]').length));
+ $('[data-selection-action=remove-selected-files]', $container).
+ closest('li').
+ toggleClass('disabled',
+ ($checked.length < 0) ||
+ !($checked.length > 0 && collection_lock_classes && collection_lock_classes.indexOf("fa-unlock") !=-1));
}
$(document).
.btn-group.toggle-persist .btn-info.active {
background-color: $active-bg;
}
+
+.lock-collection-btn {
+ display: inline-block;
+ padding: .5em 2em;
+ margin: 0 1em;
+}
end
expose_action :combine_selected_files_into_collection do
- link_uuids, coll_ids = params["selection"].partition do |sel_s|
- ArvadosBase::resource_class_for_uuid(sel_s) == Link
- end
-
- unless link_uuids.empty?
- Link.select([:head_uuid]).where(uuid: link_uuids).each do |link|
- if ArvadosBase::resource_class_for_uuid(link.head_uuid) == Collection
- coll_ids << link.head_uuid
- end
- end
- end
-
- uuids = []
- pdhs = []
- source_paths = Hash.new { |hash, key| hash[key] = [] }
- coll_ids.each do |coll_id|
- if m = CollectionsHelper.match(coll_id)
- key = m[1] + m[2]
- pdhs << key
- source_paths[key] << m[4]
- elsif m = CollectionsHelper.match_uuid_with_optional_filepath(coll_id)
- key = m[1]
- uuids << key
- source_paths[key] << m[4]
- end
- end
-
- unless pdhs.empty?
- Collection.where(portable_data_hash: pdhs.uniq).
- select([:uuid, :portable_data_hash]).each do |coll|
- unless source_paths[coll.portable_data_hash].empty?
- uuids << coll.uuid
- source_paths[coll.uuid] = source_paths.delete(coll.portable_data_hash)
- end
- end
- end
+ uuids, source_paths = selected_collection_files params
new_coll = Arv::Collection.new
Collection.where(uuid: uuids.uniq).
end
end
+ # helper method to get the names of collection files selected
+ helper_method :selected_collection_files
+ def selected_collection_files params
+ link_uuids, coll_ids = params["selection"].partition do |sel_s|
+ ArvadosBase::resource_class_for_uuid(sel_s) == Link
+ end
+
+ unless link_uuids.empty?
+ Link.select([:head_uuid]).where(uuid: link_uuids).each do |link|
+ if ArvadosBase::resource_class_for_uuid(link.head_uuid) == Collection
+ coll_ids << link.head_uuid
+ end
+ end
+ end
+
+ uuids = []
+ pdhs = []
+ source_paths = Hash.new { |hash, key| hash[key] = [] }
+ coll_ids.each do |coll_id|
+ if m = CollectionsHelper.match(coll_id)
+ key = m[1] + m[2]
+ pdhs << key
+ source_paths[key] << m[4]
+ elsif m = CollectionsHelper.match_uuid_with_optional_filepath(coll_id)
+ key = m[1]
+ uuids << key
+ source_paths[key] << m[4]
+ end
+ end
+
+ unless pdhs.empty?
+ Collection.where(portable_data_hash: pdhs.uniq).
+ select([:uuid, :portable_data_hash]).each do |coll|
+ unless source_paths[coll.portable_data_hash].empty?
+ uuids << coll.uuid
+ source_paths[coll.uuid] = source_paths.delete(coll.portable_data_hash)
+ end
+ end
+ end
+
+ [uuids, source_paths]
+ end
+
def wiselinks_layout
'body'
end
require "arvados/keep"
+require "arvados/collection"
require "uri"
class CollectionsController < ApplicationController
sharing_popup
end
+ def remove_selected_files
+ uuids, source_paths = selected_collection_files params
+
+ arv_coll = Arv::Collection.new(@object.manifest_text)
+ source_paths[uuids[0]].each do |p|
+ arv_coll.rm "."+p
+ end
+
+ if @object.update_attributes manifest_text: arv_coll.manifest_text
+ show
+ else
+ self.render_error status: 422
+ end
+ end
+
+ def update
+ updated_attr = params[:collection].each.select {|a| a[0].andand.start_with? 'rename-file-path:'}
+
+ if updated_attr.size > 0
+ # Is it file rename?
+ file_path = updated_attr[0][0].split('rename-file-path:')[-1]
+
+ new_file_path = updated_attr[0][1]
+ if new_file_path.start_with?('./')
+ # looks good
+ elsif new_file_path.start_with?('/')
+ new_file_path = '.' + new_file_path
+ else
+ new_file_path = './' + new_file_path
+ end
+
+ arv_coll = Arv::Collection.new(@object.manifest_text)
+
+ if arv_coll.exist?(new_file_path)
+ @errors = 'Duplicate file path. Please use a different name.'
+ self.render_error status: 422
+ else
+ arv_coll.rename "./"+file_path, new_file_path
+
+ if @object.update_attributes manifest_text: arv_coll.manifest_text
+ show
+ else
+ self.render_error status: 422
+ end
+ end
+ else
+ # Not a file rename; use default
+ super
+ end
+ end
+
protected
def find_usable_token(token_list)
histogram_log = Log.
filter([[:event_type, '=', 'block-age-free-space-histogram']]).
order(:created_at => :desc).
+ with_count('none').
limit(1)
histogram_log.each do |log_entry|
# We expect this block to only execute at most once since we
filter([[:object_uuid, '=', u.uuid],
[:event_type, '=', 'user-storage-report']]).
order(:created_at => :desc).
+ with_count('none').
limit(1)
storage_log.each do |log_entry|
# We expect this block to only execute once since we specified limit(1)
"data-placement" => "bottom",
"data-type" => input_type,
"data-title" => "Edit #{attr.to_s.gsub '_', ' '}",
- "data-name" => attr,
+ "data-name" => htmloptions['selection_name'] || attr,
"data-object-uuid" => object.uuid,
"data-toggle" => "manual",
- "data-value" => attrvalue,
+ "data-value" => htmloptions['data-value'] || attrvalue,
"id" => span_id,
:class => "editable #{is_textile?( object, attr ) ? 'editable-textile' : ''}"
}.merge(htmloptions).merge(ajax_options)
RESOURCE_CLASS_ICONS = {
"Collection" => "fa-archive",
+ "ContainerRequest" => "fa-gears",
"Group" => "fa-users",
"Human" => "fa-male", # FIXME: Use a more inclusive icon.
"Job" => "fa-gears",
"Trait" => "fa-clipboard",
"User" => "fa-user",
"VirtualMachine" => "fa-terminal",
+ "Workflow" => "fa-gears",
}
DEFAULT_ICON_CLASS = "fa-cube"
ArvadosResourceList.new(self).select(*args)
end
+ def self.with_count(*args)
+ ArvadosResourceList.new(self).with_count(*args)
+ end
+
def self.distinct(*args)
ArvadosResourceList.new(self).distinct(*args)
end
self
end
+ # with_count sets the 'count' parameter to 'exact' or 'none' -- see
+ # https://doc.arvados.org/api/methods.html#index
+ def with_count(count_param='exact')
+ @count = count_param
+ self
+ end
+
def fetch_multiple_pages(f)
@fetch_multiple_pages = f
self
api_params = {
_method: 'GET'
}
+ api_params[:count] = @count if @count
api_params[:where] = @cond if @cond
api_params[:eager] = '1' if @eager
api_params[:select] = @select if @select
end
def editable_attributes
- %w(name description manifest_text)
+ %w(name description manifest_text filename)
end
def provenance
end
def stderr_log_query(limit=nil)
- query = Log.where(object_uuid: self.uuid).order("created_at DESC")
+ query = Log.where(object_uuid: self.uuid).order("created_at DESC").with_count('none')
query = query.limit(limit) if limit
query
end
def stderr_log_query(limit=nil)
query = Log.
- where(event_type: "stderr",
- object_uuid: stderr_log_object_uuids).
- order("id DESC")
+ with_count('none').
+ where(event_type: "stderr",
+ object_uuid: stderr_log_object_uuids).
+ order("created_at DESC")
unless limit.nil?
query = query.limit(limit)
end
Log.where(object_uuid: log_object_uuids).
order("created_at DESC").
limit(limit).
+ with_count('none').
select { |log| log.properties[:text].is_a? String }.
reverse.
flat_map { |log| log.properties[:text].split("\n") }
def self.goes_in_projects?
true
end
+
+ def self.creatable?
+ false
+ end
+
+ def textile_attributes
+ [ 'description' ]
+ end
end
end
%>
- <li class="<%= 'active' if i==0 %> <%= link_disabled %>" data-toggle="tooltip" data-placement="top" title="<%=tab_tooltip%>">
+ <li class="<%= 'active' if i==0 %> <%= link_disabled %> tab-pane-<%=pane_name%>" data-toggle="tooltip" data-placement="top" title="<%=tab_tooltip%>">
<a href="#<%= pane_name %>"
id="<%= pane_name %>-tab"
data-toggle="<%= data_toggle %>"
--- /dev/null
+<% if @object.editable? %>
+ <i class="fa fa-fw fa-lock lock-collection-btn btn btn-primary" title="Unlock collection to edit files"></i>
+<% end %>
'data-selection-action' => 'combine-collections',
'data-toggle' => 'dropdown'
%></li>
+ <% if object.editable? %>
+ <li><%= link_to "Remove selected files", '#',
+ method: :post,
+ 'data-href' => url_for(controller: 'collections', action: :remove_selected_files),
+ 'data-selection-param-name' => 'selection[]',
+ 'data-selection-action' => 'remove-selected-files',
+ 'data-toggle' => 'dropdown',
+ 'class' => 'btn-remove-selected-files'
+ %></li>
+ <% end %>
</ul>
</div>
<div class="btn-group btn-group-sm">
} %>
<span> </span>
<% end %>
+
+ <% if object.editable? %>
+ <span class="btn-collection-remove-file-span">
+ <%= link_to({controller: 'collections', action: 'remove_selected_files', id: object.uuid, selection: [object.portable_data_hash+'/'+file_path]}, method: :post, remote: true, data: {confirm: "Remove #{file_path}?", toggle: 'tooltip', placement: 'top'}, class: 'btn btn-sm btn-default btn-nodecorate btn-collection-file-control', title: 'Remove this file') do %>
+ <i class="fa fa-fw fa-trash-o"></i>
+ <% end %>
+ </span>
+ <% end %>
<% if CollectionsHelper::is_image(filename) %>
- <i class="fa fa-fw fa-bar-chart-o"></i> <%= filename %></div>
+ <i class="fa fa-fw fa-bar-chart-o"></i>
+ <% if object.editable? %>
+ <span class="btn-collection-rename-file-span">
+ <%= render_editable_attribute object, 'filename', filename, {'data-value' => file_path, 'data-toggle' => 'manual', 'selection_path' => 'rename-file-path:'+file_path}, {tiptitle: 'Edit name or directory or both for this file', btnclass: 'collection-file-control'} %>
+ </span>
+ <% else %>
+ <%= filename %>
+ <% end %>
+ </div>
<div class="collection_files_inline">
<%= link_to(image_tag("#{url_for object}/#{file_path}"),
link_params.merge(disposition: 'inline'),
</div>
</div>
<% else %>
- <i class="fa fa-fw fa-file" href="<%=object.uuid%>/<%=file_path%>" ></i> <%= filename %></div>
+ <% if object.editable? %>
+ <i class="fa fa-fw fa-file"></i><span class="btn-collection-rename-file-span"><%= render_editable_attribute object, 'filename', filename, {'data-value' => file_path, 'data-toggle' => 'manual', 'selection_name' => 'rename-file-path:'+file_path}, {tiptitle: 'Edit name or directory or both for this file', btnclass: 'collection-file-control'} %>
+ </span>
+ <% else %>
+ <i class="fa fa-fw fa-file" href="<%=object.uuid%>/<%=file_path%>" ></i> <%= filename %>
+ <% end %>
+ </div>
</div>
<% end %>
</li>
<% if @object.state == 'Final' %>
<%= link_to(copy_container_request_path('id' => @object.uuid),
- class: 'btn btn-primary',
+ class: 'btn btn-sm btn-primary',
title: 'Re-run',
data: {toggle: :tooltip, placement: :top}, title: 'This will make a copy and take you there. You can then make any needed changes and run it',
method: :post,
<div class="row">
<div class="col-md-3" style="word-break:break-all;">
<h4 class="panel-title">
- <a class="component-detail-panel fa fa-caret-down" data-toggle="collapse" href="#collapse<%= i %>">
- <%= current_obj.label %>
+ <a class="component-detail-panel" data-toggle="collapse" href="#collapse<%= i %>">
+ <%= current_obj.label %> <span class="caret" href="#collapse<%= i %>"></span>
</a>
</h4>
</div>
--- /dev/null
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+
+<table class="table table-condensed arv-index">
+ <colgroup>
+ <col width="10%" />
+ <col width="10%" />
+ <col width="25%" />
+ <col width="40%" />
+ <col width="15%" />
+ </colgroup>
+
+ <thead>
+ <tr class="contain-align-left">
+ <th></th>
+ <th></th>
+ <th> name </th>
+ <th> description </th>
+ <th> owner </th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <% @objects.sort_by { |ob| ob[:created_at] }.reverse.each do |ob| %>
+ <tr>
+ <td>
+ <%= button_to(choose_projects_path(id: "run-workflow-button",
+ title: 'Choose project',
+ editable: true,
+ action_name: 'Choose',
+ action_href: work_units_path,
+ action_method: 'post',
+ action_data: {'selection_param' => 'work_unit[owner_uuid]',
+ 'work_unit[template_uuid]' => ob.uuid,
+ 'success' => 'redirect-to-created-object'
+ }.to_json),
+ { class: "btn btn-default btn-xs", title: "Run #{ob.name}", remote: true, method: :get }
+ ) do %>
+ <i class="fa fa-fw fa-play"></i> Run
+ <% end %>
+ </td>
+
+ <td>
+ <%= render :partial => "show_object_button", :locals => {object: ob, size: 'xs'} %>
+ </td>
+
+ <td>
+ <%= render_editable_attribute ob, 'name' %>
+ </td>
+
+ <td>
+ <% if ob.description %>
+ <%= render_attribute_as_textile(ob, "description", ob.description, false) %>
+ <br />
+ <% end %>
+ </td>
+
+ <td>
+ <%= link_to_if_arvados_object ob.owner_uuid, friendly_name: true %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+</table>
+
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
post 'share', :on => :member
post 'unshare', :on => :member
get 'choose', on: :collection
+ post 'remove_selected_files', on: :member
end
get('/collections/download/:uuid/:reader_token/*file' => 'collections#show_file',
format: false)
collection = api_fixture('collections')['foo_file']
get :show, {id: collection['uuid']}, session_for(:active)
assert_includes @response.body, collection['name']
- assert_match /href="#{collection['uuid']}\/foo" ><\/i> foo</, @response.body
+ assert_match /not authorized to manage collection sharing links/, @response.body
end
test "No Upload tab on non-writable collection" do
assert_equal "https://download.example/c=#{id.sub '+', '-'}/_/w%20a%20z?api_token=#{tok}", @response.redirect_url
end
end
+
+ test "remove selected files from collection" do
+ use_token :active
+
+ # create a new collection to test; using existing collections will cause other tests to fail,
+ # and resetting fixtures after each test makes it take almost 4 times to run this test file.
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n./dir1 d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n"
+
+ collection = Collection.create(manifest_text: manifest_text)
+ assert_includes(collection['manifest_text'], "0:0:file1")
+
+ # now remove all files named 'file1' from the collection
+ post :remove_selected_files, {
+ id: collection['uuid'],
+ selection: ["#{collection['uuid']}/file1",
+ "#{collection['uuid']}/dir1/file1"],
+ format: :json
+ }, session_for(:active)
+ assert_response :success
+
+ # verify no 'file1' in the updated collection
+ collection = Collection.select([:uuid, :manifest_text]).where(uuid: collection['uuid']).first
+ assert_not_includes(collection['manifest_text'], "0:0:file1")
+ assert_includes(collection['manifest_text'], "0:0:file2") # but other files still exist
+ end
+
+ test "remove all files from a subdir of a collection" do
+ use_token :active
+
+ # create a new collection to test
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n./dir1 d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n"
+
+ collection = Collection.create(manifest_text: manifest_text)
+ assert_includes(collection['manifest_text'], "0:0:file1")
+
+ # now remove all files from "dir1" subdir of the collection
+ post :remove_selected_files, {
+ id: collection['uuid'],
+ selection: ["#{collection['uuid']}/dir1/file1",
+ "#{collection['uuid']}/dir1/file2"],
+ format: :json
+ }, session_for(:active)
+ assert_response :success
+
+ # verify that "./dir1" no longer exists in this collection's manifest text
+ collection = Collection.select([:uuid, :manifest_text]).where(uuid: collection['uuid']).first
+ assert_match /. d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:file1 0:0:file2\n$/, collection['manifest_text']
+ assert_not_includes(collection['manifest_text'], 'dir1')
+ end
+
+ test "rename file in a collection" do
+ use_token :active
+
+ # create a new collection to test
+ manifest_text = ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n./dir1 d41d8cd98f00b204e9800998ecf8427e+0 0:0:dir1file1 0:0:dir1file2\n"
+
+ collection = Collection.create(manifest_text: manifest_text)
+ assert_includes(collection['manifest_text'], "0:0:file1")
+
+ # rename 'file1' as 'file1renamed' and verify
+ post :update, {
+ id: collection['uuid'],
+ collection: {
+ 'rename-file-path:file1' => 'file1renamed'
+ },
+ format: :json
+ }, session_for(:active)
+ assert_response :success
+
+ collection = Collection.select([:uuid, :manifest_text]).where(uuid: collection['uuid']).first
+ assert_match /. d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:file1renamed 0:0:file2\n.\/dir1 d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:dir1file1 0:0:dir1file2\n$/, collection['manifest_text']
+
+ # now rename 'file2' such that it is moved into 'dir1'
+ @test_counter = 0
+ post :update, {
+ id: collection['uuid'],
+ collection: {
+ 'rename-file-path:file2' => 'dir1/file2'
+ },
+ format: :json
+ }, session_for(:active)
+ assert_response :success
+
+ collection = Collection.select([:uuid, :manifest_text]).where(uuid: collection['uuid']).first
+ assert_match /. d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:file1renamed\n.\/dir1 d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:dir1file1 0:0:dir1file2 0:0:file2\n$/, collection['manifest_text']
+
+ # now rename 'dir1/dir1file1' such that it is moved into a new subdir
+ @test_counter = 0
+ post :update, {
+ id: collection['uuid'],
+ collection: {
+ 'rename-file-path:dir1/dir1file1' => 'dir2/dir3/dir1file1moved'
+ },
+ format: :json
+ }, session_for(:active)
+ assert_response :success
+
+ collection = Collection.select([:uuid, :manifest_text]).where(uuid: collection['uuid']).first
+ assert_match /. d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:file1renamed\n.\/dir1 d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:dir1file2 0:0:file2\n.\/dir2\/dir3 d41d8cd98f00b204e9800998ecf8427e\+0\+A(.*) 0:0:dir1file1moved\n$/, collection['manifest_text']
+ end
+
+ test "renaming file with a duplicate name in same stream not allowed" do
+ use_token :active
+
+ # rename 'file2' as 'file1' and expect error
+ post :update, {
+ id: 'zzzzz-4zz18-pyw8yp9g3pr7irn',
+ collection: {
+ 'rename-file-path:file2' => 'file1'
+ },
+ format: :json
+ }, session_for(:active)
+ assert_response 422
+ assert_includes json_response['errors'], 'Duplicate file path'
+ end
+
+ test "renaming file with a duplicate name as another stream not allowed" do
+ use_token :active
+
+ # rename 'file1' as 'dir1/file1' and expect error
+ post :update, {
+ id: 'zzzzz-4zz18-pyw8yp9g3pr7irn',
+ collection: {
+ 'rename-file-path:file1' => 'dir1/file1'
+ },
+ format: :json
+ }, session_for(:active)
+ assert_response 422
+ assert_includes json_response['errors'], 'Duplicate file path'
+ end
end
test "Upload two empty files with the same name" do
need_selenium "to make file uploads work"
visit page_with_token 'active', sandbox_path
+
+ unlock_collection
+
find('.nav-tabs a', text: 'Upload').click
attach_file 'file_selector', testfile_path('empty.txt')
assert_selector 'div', text: 'empty.txt'
test "Upload non-empty files" do
need_selenium "to make file uploads work"
visit page_with_token 'active', sandbox_path
+
+ unlock_collection
+
find('.nav-tabs a', text: 'Upload').click
attach_file 'file_selector', testfile_path('a')
attach_file 'file_selector', testfile_path('foo.txt')
service_port: 99999)
end
visit page_with_token 'active', sandbox_path
+
+ unlock_collection
+
find('.nav-tabs a', text: 'Upload').click
attach_file 'file_selector', testfile_path('foo.txt')
assert_selector 'button:not([disabled])', text: 'Start'
# Must be an absolute path. https://github.com/jnicklas/capybara/issues/621
File.join Dir.getwd, 'tmp', filename
end
+
+ def unlock_collection
+ first('.lock-collection-btn').click
+ accept_alert
+ end
end
# Make sure we're not still on the old collection page.
refute_match(%r{/collections/#{col['uuid']}}, page.current_url)
end
+
+ test "remove a file from collection using checkbox and dropdown option" do
+ need_selenium 'to confirm unlock'
+
+ visit page_with_token('active', '/collections/zzzzz-4zz18-a21ux3541sxa8sf')
+ assert(page.has_text?('file1'), 'file not found - file1')
+
+ unlock_collection
+
+ # remove first file
+ input_files = page.all('input[type=checkbox]')
+ input_files[0].click
+
+ click_button 'Selection...'
+ within('.selection-action-container') do
+ click_link 'Remove selected files'
+ end
+
+ assert(page.has_no_text?('file1'), 'file found - file')
+ assert(page.has_text?('file2'), 'file not found - file2')
+ end
+
+ test "remove a file in collection using trash icon" do
+ need_selenium 'to confirm unlock'
+
+ visit page_with_token('active', '/collections/zzzzz-4zz18-a21ux3541sxa8sf')
+ assert(page.has_text?('file1'), 'file not found - file1')
+
+ unlock_collection
+
+ first('.fa-trash-o').click
+ accept_alert
+
+ assert(page.has_no_text?('file1'), 'file found - file')
+ assert(page.has_text?('file2'), 'file not found - file2')
+ end
+
+ test "rename a file in collection" do
+ need_selenium 'to confirm unlock'
+
+ visit page_with_token('active', '/collections/zzzzz-4zz18-a21ux3541sxa8sf')
+
+ unlock_collection
+
+ within('.collection_files') do
+ first('.fa-pencil').click
+ find('.editable-input input').set('file1renamed')
+ find('.editable-submit').click
+ end
+
+ assert(page.has_text?('file1renamed'), 'file not found - file1renamed')
+ end
+
+ test "remove/rename file options not presented if user cannot update a collection" do
+ # visit a publicly accessible collection as 'spectator'
+ visit page_with_token('spectator', '/collections/zzzzz-4zz18-uukreo9rbgwsujr')
+
+ click_button 'Selection'
+ within('.selection-action-container') do
+ assert_selector 'li', text: 'Create new collection with selected files'
+ assert_no_selector 'li', text: 'Remove selected files'
+ end
+
+ within('.collection_files') do
+ assert(page.has_text?('GNU_General_Public_License'), 'file not found - GNU_General_Public_License')
+ assert_nil first('.fa-pencil')
+ assert_nil first('.fa-trash-o')
+ end
+ end
+
+ test "unlock collection to modify files" do
+ need_selenium 'to confirm remove'
+
+ collection = api_fixture('collections')['collection_owned_by_active']
+
+ # On load, collection is locked, and upload tab, rename and remove options are disabled
+ visit page_with_token('active', "/collections/#{collection['uuid']}")
+
+ assert_selector 'a[data-toggle="disabled"]', text: 'Upload'
+
+ within('.collection_files') do
+ file_ctrls = page.all('.btn-collection-file-control')
+ assert_equal 2, file_ctrls.size
+ assert_equal true, file_ctrls[0]['class'].include?('disabled')
+ assert_equal true, file_ctrls[1]['class'].include?('disabled')
+ find('input[type=checkbox]').click
+ end
+
+ click_button 'Selection'
+ within('.selection-action-container') do
+ assert_selector 'li.disabled', text: 'Remove selected files'
+ assert_selector 'li', text: 'Create new collection with selected files'
+ end
+
+ unlock_collection
+
+ assert_no_selector 'a[data-toggle="disabled"]', text: 'Upload'
+ assert_selector 'a', text: 'Upload'
+
+ within('.collection_files') do
+ file_ctrls = page.all('.btn-collection-file-control')
+ assert_equal 2, file_ctrls.size
+ assert_equal false, file_ctrls[0]['class'].include?('disabled')
+ assert_equal false, file_ctrls[1]['class'].include?('disabled')
+
+ # previous checkbox selection won't result in firing a new event;
+ # undo and redo checkbox to fire the selection event again
+ find('input[type=checkbox]').click
+ find('input[type=checkbox]').click
+ end
+
+ click_button 'Selection'
+ within('.selection-action-container') do
+ assert_no_selector 'li.disabled', text: 'Remove selected files'
+ assert_selector 'li', text: 'Remove selected files'
+ end
+ end
+
+ def unlock_collection
+ first('.lock-collection-btn').click
+ accept_alert
+ end
end
end
end
end
+
+ test 'Run from workflows index page' do
+ visit page_with_token('active', '/workflows')
+
+ wf_count = page.all('a[data-original-title="show workflow"]').count
+ assert_equal true, wf_count>0
+
+ # Run one of the workflows
+ wf_name = 'Workflow with input specifications'
+ within('tr', text: wf_name) do
+ find('a,button', text: 'Run').click
+ end
+
+ # Choose project for the container_request being created
+ within('.modal-dialog') do
+ find('.selectable', text: 'A Project').click
+ find('button', text: 'Choose').click
+ end
+
+ # In newly created container_request page now
+ assert_text 'A Project' # CR created in "A Project"
+ assert_text "This container request was created from the workflow #{wf_name}"
+ assert_match /Provide a value for .* then click the \"Run\" button to start the workflow/, page.text
+ end
end
end
Capybara.reset_sessions!
end
+
+ def accept_alert
+ if Capybara.current_driver == :selenium
+ (0..9).each do
+ begin
+ page.driver.browser.switch_to.alert.accept
+ break
+ rescue Selenium::WebDriver::Error::NoSuchAlertError
+ sleep 0.1
+ end
+ end
+ else
+ # poltergeist returns true for confirm, so no need to accept
+ end
+ end
end
class ResourceListTest < ActiveSupport::TestCase
+ reset_api_fixtures :after_each_test, false
+
test 'links_for on a resource list that does not return links' do
use_token :active
results = Specimen.all
assert_empty c.results
end
+ test 'count=none' do
+ use_token :active
+ c = Collection.with_count('none')
+ assert_nil c.items_available
+ refute_empty c.results
+ end
end
require 'test_helper'
class LinkTest < ActiveSupport::TestCase
+
+ reset_api_fixtures :after_each_test, false
+
def uuid_for(fixture_name, object_name)
api_fixture(fixture_name)[object_name]["uuid"]
end
require 'test_helper'
class PipelineInstanceTest < ActiveSupport::TestCase
+
+ reset_api_fixtures :after_each_test, false
+
def find_pi_with(token_name, pi_name)
use_token token_name
find_fixture(PipelineInstance, pi_name)
require 'test_helper'
class WorkUnitTest < ActiveSupport::TestCase
+
+ reset_api_fixtures :after_each_test, false
+
setup do
Rails.configuration.anonymous_user_token = api_fixture('api_client_authorizations')['anonymous']['api_token']
end
#distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
debian8,ubuntu1204,centos7|python-gflags|2.0|2|python|all
-debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|google-api-python-client|1.4.2|2|python|all
-debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|oauth2client|1.5.2|2|python|all
+debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|google-api-python-client|1.6.2|2|python|all
+debian8,ubuntu1204,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
debian8,ubuntu1204,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
debian8,ubuntu1204,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|rsa|3.4.2|2|python|all
debian8,ubuntu1204,centos7|pycrypto|2.6.1|3|python|amd64
debian8,ubuntu1204,ubuntu1404,ubuntu1604|backports.ssl_match_hostname|3.5.0.1|2|python|all
debian8,ubuntu1204,ubuntu1404,centos7|llfuse|0.41.1|3|python|amd64
-debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|pycurl|7.19.5.3|3|python|amd64
+debian8,ubuntu1204,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|pyyaml|3.12|2|python|amd64
-debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|rdflib|4.2.1|2|python|all
+debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|rdflib|4.2.2|2|python|all
debian8,ubuntu1204,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|mistune|0.7.3|2|python|all
debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|typing|3.5.3.0|2|python|all
centos7|daemon|2.1.1|2|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
-centos7|sparqlwrapper|1.8.0|2|python|all
-centos7|html5lib|0.9999999|2|python|all
centos7|keepalive|0.5|2|python|all
debian8,ubuntu1204,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
all|ruamel.yaml|0.13.7|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
all|cwltest|1.0.20160907111242|3|python|all|--depends 'python-futures >= 3.0.5'
all|rdflib-jsonld|0.4.0|2|python|all
all|futures|3.0.5|2|python|all
+all|future|0.16.0|2|python|all
+all|future|0.16.0|2|python3|all
# Install RVM
RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends curl ca-certificates && \
+ DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends curl ca-certificates g++ && \
gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
curl -L https://get.rvm.io | bash -s stable && \
/usr/local/rvm/bin/rvm install 2.3 && \
chown "$WWW_OWNER:" $RELEASE_PATH/config/environment.rb
chown "$WWW_OWNER:" $RELEASE_PATH/config.ru
chown "$WWW_OWNER:" $RELEASE_PATH/Gemfile.lock
- chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp
+ chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp || true
chown -R "$WWW_OWNER:" $SHARED_PATH/log
case "$RAILSPKG_DATABASE_LOAD_TASK" in
db:schema:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/schema.rb ;;
db:structure:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/structure.sql ;;
esac
chmod 644 $SHARED_PATH/log/*
- chmod -R 2775 $RELEASE_PATH/tmp
+ chmod -R 2775 $RELEASE_PATH/tmp || true
echo "... done."
if [ -n "$RAILSPKG_DATABASE_LOAD_TASK" ]; then
ubuntu1404)
FORMAT=deb
;;
+ ubuntu1604)
+ FORMAT=deb
+ ;;
centos7)
FORMAT=rpm
;;
TARGET=debian8
COMMAND=
-RAILS_PACKAGE_ITERATION=7
-
PARSEDOPTS=$(getopt --name "$0" --longoptions \
help,build-bundle-packages,debug,target:,only-build: \
-- "" "$@")
# older packages.
LICENSE_PACKAGE_TS=20151208015500
+RAILS_PACKAGE_ITERATION=7
+
debug_echo () {
echo "$@" >"$STDOUT_IF_DEBUG"
}
echo -n 'fuse.h: '
find /usr/include -wholename '*fuse/fuse.h' \
|| fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+ echo -n 'gnutls.h: '
+ find /usr/include -wholename '*gnutls/gnutls.h' \
+ || fatal "No gnutls/gnutls.h. Try: apt-get install libgnutls28-dev"
echo -n 'pyconfig.h: '
find /usr/include -name pyconfig.h | egrep --max-count=1 . \
|| fatal "No pyconfig.h. Try: apt-get install python-dev"
export GOPATH
mkdir -p "$GOPATH/src/git.curoverse.com"
-ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+ln -sfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
|| fatal "symlink failed"
setup_virtualenv "$VENVDIR" --python python2.7
- install/install-postgresql.html.textile.liquid
- install/install-sso.html.textile.liquid
- install/install-api-server.html.textile.liquid
+ - install/install-ws.html.textile.liquid
- install/install-arv-git-httpd.html.textile.liquid
- install/install-workbench-app.html.textile.liquid
- install/install-shell-server.html.textile.liquid
</ul>
<div class="pull-right" style="padding-top: 6px">
- <form method="get" action="http://www.google.com/search">
+ <form method="get" action="https://www.google.com/search">
<div class="input-group" style="width: 220px">
<input type="text" class="form-control" name="q" placeholder="search">
<div class="input-group-addon">
|cwd|string|Initial working directory, given as an absolute path (in the container) or a path relative to the WORKDIR given in the image's Dockerfile.|Required.|
|command|array of strings|Command to execute in the container.|Required. e.g., @["echo","hello"]@|
|output_path|string|Path to a directory or file inside the container that should be preserved as container's output when it finishes. This path must be, or be inside, one of the mount targets. For best performance, point output_path to a writable collection mount. Also, see "Pre-populate output using Mount points":#pre-populate-output for details regarding optional output pre-population using mount points.|Required.|
+|output_name|string|Desired name for the output collection. If null, a name will be assigned automatically.||
+|output_ttl|integer|Desired lifetime for the output collection, in seconds. If zero, the output collection will not be deleted automatically.||
|priority|integer|Higher value means spend more resources on this container_request, i.e., go ahead of other queued containers, bring up more nodes etc.|Priority 0 means a container should not be run on behalf of this request. Clients are expected to submit container requests with zero priority in order to preview the container that will be used to satisfy it. Priority can be null if and only if state!="Committed".|
|expires_at|datetime|After this time, priority is considered to be zero.|Not yet implemented.|
|use_existing|boolean|If possible, use an existing (non-failed) container to satisfy the request instead of creating a new one.|Default is true|
h2. Set up SLURM
-Install SLURM following "the same process you used to install the Crunch dispatcher":install-crunch-dispatch.html#slurm.
+Install SLURM on the compute node using the same process you used on the API server in the "previous step":install-slurm.html.
-h2. Copy configuration files from the dispatcher (API server)
-
-The @slurm.conf@ and @/etc/munge/munge.key@ files need to be identical across the dispatcher and all compute nodes. Copy the files you created in the "Install the Crunch dispatcher":install-crunch-dispatch.html step to this compute node.
+The @slurm.conf@ and @/etc/munge/munge.key@ files must be identical on all SLURM nodes. Copy the files you created on the API server in the "previous step":install-slurm.html to each compute node.
{% include 'notebox_end' %}
+h3. CrunchRunCommand: Using host networking for containers
+
+Older Linux kernels (prior to 3.18) have bugs in network namespace handling which can lead to compute node lockups. This by is indicated by blocked kernel tasks in "Workqueue: netns cleanup_net". If you are experiencing this problem, as a workaround you can disable use of network namespaces by Docker across the cluster. Be aware this reduces container isolation, which may be a security risk.
+
+<notextile>
+<pre><code class="userinput">Client:
+ APIHost: <b>zzzzz.arvadosapi.com</b>
+ AuthToken: <b>zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</b>
+CrunchRunCommand:
+- <b>crunch-run</b>
+- <b>"-container-enable-networking=always"</b>
+- <b>"-container-network-mode=host"</b>
+</code></pre>
+</notextile>
+
h3. MinRetryPeriod: Rate-limit repeated attempts to start containers
If SLURM is unable to run a container, the dispatcher will submit it again after the next PollPeriod. If PollPeriod is very short, this can be excessive. If MinRetryPeriod is set, the dispatcher will avoid submitting the same container to SLURM more than once in the given time span.
</code></pre>
</notextile>
-h2(#set_up). Set up Web servers
+h2(#set_up). Set up Nginx and Passenger
-For best performance, we recommend you use Nginx as your Web server front-end, with a Passenger backend for the main API server and a Puma backend for API server Websockets. To do that:
+The Nginx server will serve API requests using Passenger. It will also be used to proxy SSL requests to other services which are covered later in this guide.
-<notextile>
-<ol>
-<li><a href="https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html">Install Nginx and Phusion Passenger</a>.</li>
-
-<li><p>Install runit to supervise the Puma daemon. {% include 'install_runit' %}<notextile></p></li>
-
-<li><p>Install the script below as the run script for the Puma service, modifying it as directed by the comments.</p>
-
-<pre><code>#!/bin/bash
-
-set -e
-exec 2>&1
-
-# Uncomment the line below if you're using RVM.
-#source /etc/profile.d/rvm.sh
-
-envdir="`pwd`/env"
-mkdir -p "$envdir"
-echo ws-only > "$envdir/ARVADOS_WEBSOCKETS"
-
-cd /var/www/arvados-api/current
-echo "Starting puma in `pwd`"
-
-# Change arguments below to match your deployment, "webserver-user" and
-# "webserver-group" should be changed to the user and group of the web server
-# process. This is typically "www-data:www-data" on Debian systems by default,
-# other systems may use different defaults such the name of the web server
-# software (for example, "nginx:nginx").
-exec chpst -m 1073741824 -u webserver-user:webserver-group -e "$envdir" \
- bundle exec puma -t 0:512 -e production -b tcp://127.0.0.1:8100
-</code></pre>
-</li>
+First, "Install Nginx and Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html.
-<li><p>Edit the http section of your Nginx configuration to run the Passenger server, and act as a front-end for both it and Puma. You might add a block like the following, adding SSL and logging parameters to taste:</p>
+Edit the http section of your Nginx configuration to run the Passenger server, and serve SSL requests. Add a block like the following, adding SSL and logging parameters to taste:
+<notextile>
<pre><code>server {
listen 127.0.0.1:8000;
server_name localhost-api;
server 127.0.0.1:8000 fail_timeout=10s;
}
-upstream websockets {
- # The address below must match the one specified in puma's -b option.
- server 127.0.0.1:8100 fail_timeout=10s;
-}
-
proxy_http_version 1.1;
# When Keep clients request a list of Keep services from the API server, the
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
-
-server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name ws.<span class="userinput">uuid_prefix.your.domain</span>;
-
- ssl on;
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- index index.html index.htm index.php;
-
- location / {
- proxy_pass http://websockets;
- proxy_redirect off;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
-}
</code></pre>
-</li>
+</notextile>
-<li><p>Restart Nginx:</p>
+Restart Nginx to apply the new configuration.
+<notextile>
<pre><code>~$ <span class="userinput">sudo nginx -s reload</span>
</code></pre>
-
-</li>
-
-</ol>
</notextile>
h2. Prepare the API server deployment
{% include 'notebox_begin' %}
You can safely ignore the following messages if they appear while this command runs:
-<pre>Don't run Bundler as root. Bundler can ask for sudo if it is needed, and installing your bundle as root will
-break this application for all non-root users on this machine.</pre>
-<pre>fatal: Not a git repository (or any of the parent directories): .git</pre>
+
+<notextile><pre>Don't run Bundler as root. Bundler can ask for sudo if it is needed, and installing your bundle as root will
+break this application for all non-root users on this machine.</pre></notextile>
+
+<notextile><pre>fatal: Not a git repository (or any of the parent directories): .git</pre></notextile>
{% include 'notebox_end' %}
# EC2 configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
[Daemon]
# The dispatcher can customize the start and stop procedure for
# cloud nodes. For example, the SLURM dispatcher drains nodes
# Google Compute Engine configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
[Daemon]
# Node Manager will ensure that there are at least this many nodes running at
# all times. If node manager needs to start new idle nodes for the purpose of
# Azure configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
[Daemon]
# The dispatcher can customize the start and stop procedure for
# cloud nodes. For example, the SLURM dispatcher drains nodes
title: Install the websocket server
...
-{% include 'notebox_begin_warning' %}
-
-This websocket server is an alternative to the puma server that comes with the API server. It is available as an *experimental pre-release* and is not recommended for production sites.
-
-{% include 'notebox_end' %}
-
-The arvados-ws server provides event notifications to websocket clients. It can be installed anywhere with access to Postgres database and the Arvados API server, typically behind a web proxy that provides SSL support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for additional information.
+The arvados-ws server provides event notifications to websocket clients. It can be installed anywhere with access to Postgres database and the Arvados API server, typically behind a web proxy that provides SSL support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/ws for additional information.
By convention, we use the following hostname for the websocket service.
}
</pre></notextile>
-If Nginx is already configured to proxy @ws@ requests to puma, move that configuration out of the way or change its @server_name@ so it doesn't conflict.
+{% include 'notebox_begin' %}
+If you are upgrading a cluster where Nginx is configured to proxy @ws@ requests to puma, change the @server_name@ value in the old configuration block so it doesn't conflict. When the new configuration is working, delete the old Nginx configuration sections (i.e., the "upstream websockets" block, and the "server" block that references @http://websockets@), and disable/remove the runit or systemd files for the puma server.
+{% include 'notebox_end' %}
h3. Update API server configuration
<pre>
$namespaces:
arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
</pre>
Arvados extensions must go into the @hints@ section, for example:
arv:PartitionRequirement:
partition: dev_partition
arv:APIRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: shallow_listing
</pre>
h2. arv:RunInSingleContainer
h2. arv:APIRequirement
Indicates that process wants to access to the Arvados API. Will be granted limited network access and have @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ set in the environment.
+
+h2. cwltool:LoadListingRequirement
+
+In CWL v1.0 documents, the default behavior for Directory objects is to recursively expand the @listing@ for access by parameter references an expressions. For directory trees containing many files, this can be expensive in both time and memory usage. Use @cwltool:LoadListingRequirement@ to change the behavior for expansion of directory listings in the workflow runner.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|loadListing|string|One of @no_listing@, @shallow_listing@, or @deep_listing@|
+
+*no_listing*: Do not expand directory listing at all. The @listing@ field on the Directory object will be undefined.
+
+*shallow_listing*: Only expand the first level of directory listing. The @listing@ field on the toplevel Directory object will contain the directory contents, however @listing@ will not be defined on subdirectories.
+
+*deep_listing*: Recursively expand all levels of directory listing. The @listing@ field will be provided on the toplevel object and all subdirectories.
#!/bin/sh
-exec docker build -t arvados/migrate-docker19 .
+exec docker build -t arvados/migrate-docker19:1.0 .
[ -d $CGROUP ] || mkdir $CGROUP
if mountpoint -q $CGROUP ; then
- break
+ true
else
mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP
fi
# Systemd and OpenRC (and possibly others) both create such a
# cgroup. To avoid the aforementioned bug, we symlink "foo" to
# "name=foo". This shouldn't have any adverse effect.
- echo $SUBSYS | grep -q ^name= && {
- NAME=$(echo $SUBSYS | sed s/^name=//)
- ln -s $SUBSYS $CGROUP/$NAME
- }
+ #echo $SUBSYS | grep -q ^name= && {
+ # NAME=$(echo $SUBSYS | sed s/^name=//)
+ # ln -s $SUBSYS $CGROUP/$NAME
+ #}
# Likewise, on at least one system, it has been reported that
# systemd would mount the CPU and CPU accounting controllers
read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
-if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
- docker daemon $DOCKER_DAEMON_ARGS
-fi
+exec docker daemon --storage-driver=$1 $DOCKER_DAEMON_ARGS
#!/bin/bash
-set -e
+# This script is called by arv-migrate-docker19 to perform the actual migration
+# of a single image. This works by running Docker-in-Docker (dnd.sh) to
+# download the image using Docker 1.9 and then upgrading to Docker 1.13 and
+# uploading the converted image.
-function cleanup {
- kill $(cat /var/run/docker.pid)
- sleep 1
- rm -rf /var/lib/docker/*
-}
+# When using bash in pid 1 and using "trap on EXIT"
+# it will sometimes go into an 100% CPU infinite loop.
+#
+# Using workaround from here:
+#
+# https://github.com/docker/docker/issues/4854
+if [ "$$" = 1 ]; then
+ $0 "$@"
+ exit $?
+fi
-trap cleanup EXIT
-
-/root/dnd.sh &
-sleep 2
+# -x show script
+# -e exit on error
+# -o pipefail use exit code from 1st failure in pipeline, not last
+set -x -e -o pipefail
image_tar_keepref=$1
image_id=$2
image_repo=$3
image_tag=$4
project_uuid=$5
+graph_driver=$6
+
+if [[ "$image_repo" = "<none>" ]] ; then
+ image_repo=none
+ image_tag=latest
+fi
+
+# Print free space in /var/lib/docker
+function freespace() {
+ df -B1 /var/lib/docker | tail -n1 | sed 's/ */ /g' | cut -d' ' -f4
+}
+
+# Run docker-in-docker script and then wait for it to come up
+function start_docker {
+ /root/dnd.sh $graph_driver &
+ for i in $(seq 1 10) ; do
+ if docker version >/dev/null 2>/dev/null ; then
+ return
+ fi
+ sleep 1
+ done
+ false
+}
+
+# Kill docker from pid then wait for it to be down
+function kill_docker {
+ if test -f /var/run/docker.pid ; then
+ kill $(cat /var/run/docker.pid)
+ fi
+ for i in $(seq 1 10) ; do
+ if ! docker version >/dev/null 2>/dev/null ; then
+ return
+ fi
+ sleep 1
+ done
+ false
+}
+
+# Ensure that we clean up docker graph and/or lingering cache files on exit
+function cleanup {
+ kill_docker
+ rm -rf /var/lib/docker/*
+ rm -rf /root/.cache/arvados/docker/*
+ echo "Available space after cleanup is $(freespace)"
+}
+
+trap cleanup EXIT
+
+start_docker
+
+echo "Initial available space is $(freespace)"
arv-get $image_tar_keepref | docker load
+
docker tag $image_id $image_repo:$image_tag
docker images -a
-kill $(cat /var/run/docker.pid)
-sleep 1
+kill_docker
+
+echo "Available space after image load is $(freespace)"
cd /root/pkgs
-dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb
+dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb
-/root/dnd.sh &
-sleep 2
+echo "Available space after image upgrade is $(freespace)"
+
+start_docker
docker images -a
+if [[ "$image_repo" = "none" ]] ; then
+ image_repo=$(docker images -a --no-trunc | sed 's/ */ /g' | grep ^none | cut -d' ' -f3)
+ image_tag=""
+fi
+
UUID=$(arv-keepdocker --force-image-format --project-uuid=$project_uuid $image_repo $image_tag)
+echo "Available space after arv-keepdocker is $(freespace)"
+
echo "Migrated uuid is $UUID"
# Our google-api-client dependency used to be < 0.9, but that could be
# satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
s.add_runtime_dependency 'google-api-client', '~> 0.6', '>= 0.6.3', '<0.8.9'
- s.add_runtime_dependency 'activesupport', '~> 3.2', '>= 3.2.13'
- s.add_runtime_dependency 'json', '~> 1.7', '>= 1.7.7'
+ s.add_runtime_dependency 'activesupport', '>= 3.2.13', '< 5'
+ s.add_runtime_dependency 'json', '>= 1.7.7', '<3'
s.add_runtime_dependency 'trollop', '~> 2.0'
s.add_runtime_dependency 'andand', '~> 1.3', '>= 1.3.3'
s.add_runtime_dependency 'oj', '~> 2.0', '>= 2.0.3'
# Find FUSE mounts under $CRUNCH_TMP and unmount them. Then clean
# up work directories crunch_tmp/work, crunch_tmp/opt,
# crunch_tmp/src*.
- #
- # TODO: When #5036 is done and widely deployed, we can limit mount's
- # -t option to simply fuse.keep.
my ($exited, $stdout, $stderr) = srun_sync(
["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
- ['bash', '-ec', '-o', 'pipefail', 'mount -t fuse,fuse.keep | awk "(index(\$3, \"$CRUNCH_TMP\") == 1){print \$3}" | xargs -r -n 1 fusermount -u -z; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src* $CRUNCH_TMP/*.cid'],
+ ['bash', '-ec', q{
+arv-mount --unmount-timeout 10 --unmount-all ${CRUNCH_TMP}
+rm -rf ${JOB_WORK} ${CRUNCH_INSTALL} ${CRUNCH_TMP}/task ${CRUNCH_TMP}/src* ${CRUNCH_TMP}/*.cid
+ }],
{label => "clean work dirs"});
if ($exited != 0) {
exit(EX_RETRY_UNLOCKED);
--- /dev/null
+#!/bin/bash
require 'minitest/autorun'
require 'digest/md5'
+require 'active_support'
require 'active_support/core_ext'
require 'tempfile'
assert_arv_get false, 'e796ab2294f3e48ec709ffa8d6daf58c'
end
assert_equal '', out
- assert_match /Error:/, err
+ assert_match /ERROR:/, err
end
def test_nonexistent_manifest
assert_arv_get false, 'acbd18db4cc2f85cedef654fccc4a4d8/', 'tmp/'
end
assert_equal '', out
- assert_match /Error:/, err
+ assert_match /ERROR:/, err
end
def test_manifest_root_to_dir
def test_output_collection_owner_uuid
j = jobspec :grep_local
out, err = capture_subprocess_io do
- tryjobrecord j, binstubs: ['output_coll_owner']
+ tryjobrecord j, binstubs: ['arv-mount', 'output_coll_owner']
end
assert_match /owner_uuid: #{j['owner_uuid']}/, err
end
out, err = capture_subprocess_io do
j = jobspec :grep_local
j[:script_version] = bogus_version
- tryjobrecord j
+ tryjobrecord j, binstubs: ['arv-mount']
end
assert_match /'#{bogus_version}' not found, giving up/, err
assert_jobfail $?
import hashlib
import copy
import json
+import re
from functools import partial
import pkg_resources # part of setuptools
from. runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, upload_dependencies
from .arvtool import ArvadosCommandTool
from .arvworkflow import ArvadosWorkflow, upload_workflow
-from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver
+from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache
from .perf import Perf
from .pathmapper import NoFollowPathMapper
from ._version import __version__
from cwltool.pack import pack
-from cwltool.process import shortname, UnsupportedRequirement, getListing
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
from cwltool.draft2tool import compute_checksums
from arvados.api import OrderedJsonModel
else:
self.keep_client = arvados.keep.KeepClient(api_client=self.api, num_retries=self.num_retries)
+ self.collection_cache = CollectionCache(self.api, self.keep_client, self.num_retries)
+
self.work_api = None
expected_api = ["jobs", "containers"]
for api in expected_api:
kwargs["work_api"] = self.work_api
kwargs["fetcher_constructor"] = partial(CollectionFetcher,
api_client=self.api,
- keep_client=self.keep_client)
+ fs_access=CollectionFsAccess("", collection_cache=self.collection_cache),
+ num_retries=self.num_retries)
if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
return ArvadosCommandTool(self, toolpath_object, **kwargs)
elif "class" in toolpath_object and toolpath_object["class"] == "Workflow":
if isinstance(obj, dict):
if obj.get("writable"):
raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
- if obj.get("class") == "CommandLineTool":
- if self.work_api == "containers":
- if obj.get("stdin"):
- raise SourceLine(obj, "stdin", UnsupportedRequirement).makeError("Stdin redirection currently not suppported with --api=containers")
- if obj.get("stderr"):
- raise SourceLine(obj, "stderr", UnsupportedRequirement).makeError("Stderr redirection currently not suppported with --api=containers")
if obj.get("class") == "DockerRequirement":
if obj.get("dockerOutputDirectory"):
# TODO: can be supported by containers API, but not jobs API.
keep_client=self.keep_client,
num_retries=self.num_retries)
- srccollections = {}
for k,v in generatemapper.items():
if k.startswith("_:"):
if v.type == "Directory":
raise Exception("Output source is not in keep or a literal")
sp = k.split("/")
srccollection = sp[0][5:]
- if srccollection not in srccollections:
- try:
- srccollections[srccollection] = arvados.collection.CollectionReader(
- srccollection,
- api_client=self.api,
- keep_client=self.keep_client,
- num_retries=self.num_retries)
- except arvados.errors.ArgumentError as e:
- logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
- raise
- reader = srccollections[srccollection]
try:
+ reader = self.collection_cache.get(srccollection)
srcpath = "/".join(sp[1:]) if len(sp) > 1 else "."
final.copy(srcpath, v.target, source_collection=reader, overwrite=False)
+ except arvados.errors.ArgumentError as e:
+ logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
+ raise
except IOError as e:
logger.warn("While preparing output collection: %s", e)
self.project_uuid = kwargs.get("project_uuid")
self.pipeline = None
make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess,
- api_client=self.api,
- keep_client=self.keep_client)
+ collection_cache=self.collection_cache)
self.fs_access = make_fs_access(kwargs["basedir"])
if not kwargs.get("name"):
self.set_crunch_output()
if kwargs.get("compute_checksum"):
- adjustDirObjs(self.final_output, partial(getListing, self.fs_access))
+ adjustDirObjs(self.final_output, partial(get_listing, self.fs_access))
adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
return (self.final_output, self.final_status)
return parser
def add_arv_hints():
- cache = {}
+ cwltool.draft2tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
+ cwltool.draft2tool.ACCEPTLIST_RE = cwltool.draft2tool.ACCEPTLIST_EN_RELAXED_RE
res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
- cache["http://arvados.org/cwl"] = res.read()
+ use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
res.close()
- document_loader, cwlnames, _, _ = cwltool.process.get_schema("v1.0")
- _, extnames, _, _ = schema_salad.schema.load_schema("http://arvados.org/cwl", cache=cache)
- for n in extnames.names:
- if not cwlnames.has_name("http://arvados.org/cwl#"+n, ""):
- cwlnames.add_name("http://arvados.org/cwl#"+n, "", extnames.get_name(n, ""))
- document_loader.idx["http://arvados.org/cwl#"+n] = {}
+ cwltool.process.supportedProcessRequirements.extend([
+ "http://arvados.org/cwl#RunInSingleContainer",
+ "http://arvados.org/cwl#OutputDirType",
+ "http://arvados.org/cwl#RuntimeConstraints",
+ "http://arvados.org/cwl#PartitionRequirement",
+ "http://arvados.org/cwl#APIRequirement",
+ "http://commonwl.org/cwltool#LoadListingRequirement"
+ ])
def main(args, stdout, stderr, api_client=None, keep_client=None):
parser = arg_parser()
arvargs.relax_path_checks = True
arvargs.validate = None
+ make_fs_access = partial(CollectionFsAccess,
+ collection_cache=runner.collection_cache)
+
return cwltool.main.main(args=arvargs,
stdout=stdout,
stderr=stderr,
makeTool=runner.arv_make_tool,
versionfunc=versionstring,
job_order_object=job_order_object,
- make_fs_access=partial(CollectionFsAccess,
- api_client=api_client,
- keep_client=keep_client),
+ make_fs_access=make_fs_access,
fetcher_constructor=partial(CollectionFetcher,
api_client=api_client,
- keep_client=keep_client,
+ fs_access=make_fs_access(""),
num_retries=runner.num_retries),
resolver=partial(collectionResolver, api_client, num_retries=runner.num_retries),
- logger_handler=arvados.log_handler)
+ logger_handler=arvados.log_handler,
+ custom_schema_callback=add_arv_hints)
$base: "http://arvados.org/cwl#"
+$namespaces:
+ cwl: "https://w3id.org/cwl/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: cwltool:LoadListingRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ fields:
+ class:
+ type: string
+ doc: "Always 'LoadListingRequirement'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ loadListing:
+ type:
+ - "null"
+ - type: enum
+ name: LoadListingEnum
+ symbols: [no_listing, shallow_listing, deep_listing]
+
- name: RunInSingleContainer
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Indicates that a subworkflow should run in a single container
and not be scheduled as separate steps.
- name: RuntimeConstraints
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Set Arvados-specific runtime hints.
fields:
- name: PartitionRequirement
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Select preferred compute partitions on which to run jobs.
fields:
- name: APIRequirement
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Indicates that process wants to access to the Arvados API. Will be granted
limited network access and have ARVADOS_API_HOST and ARVADOS_API_TOKEN set
import logging
import json
import os
+import urllib
import ruamel.yaml as yaml
dirs = set()
for f in self.pathmapper.files():
- pdh, p, tp = self.pathmapper.mapper(f)
+ pdh, p, tp, stg = self.pathmapper.mapper(f)
if tp == "Directory" and '/' not in pdh:
mounts[p] = {
"kind": "collection",
dirs.add(pdh)
for f in self.pathmapper.files():
- res, p, tp = self.pathmapper.mapper(f)
+ res, p, tp, stg = self.pathmapper.mapper(f)
if res.startswith("keep:"):
res = res[5:]
elif res.startswith("/keep/"):
"portable_data_hash": pdh
}
if len(sp) == 2:
- mounts[p]["path"] = sp[1]
+ mounts[p]["path"] = urllib.unquote(sp[1])
with Perf(metrics, "generatefiles %s" % self.name):
if self.generatefiles["listing"]:
container_request["environment"].update(self.environment)
if self.stdin:
- raise UnsupportedRequirement("Stdin redirection currently not suppported")
+ sp = self.stdin[6:].split("/", 1)
+ mounts["stdin"] = {"kind": "collection",
+ "portable_data_hash": sp[0],
+ "path": sp[1]}
if self.stderr:
- raise UnsupportedRequirement("Stderr redirection currently not suppported")
+ mounts["stderr"] = {"kind": "file",
+ "path": "%s/%s" % (self.outdir, self.stderr)}
if self.stdout:
mounts["stdout"] = {"kind": "file",
import functools
from arvados.api import OrderedJsonModel
-from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, getListing, normalizeFilesDirs
+from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, normalizeFilesDirs
from cwltool.load_tool import load_tool
from cwltool.errors import WorkflowException
-from .fsaccess import CollectionFetcher
+from .fsaccess import CollectionFetcher, CollectionFsAccess
logger = logging.getLogger('arvados.cwl-runner')
adjustFileObjs(job_order_object, keeppathObj)
adjustDirObjs(job_order_object, keeppathObj)
normalizeFilesDirs(job_order_object)
- adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
output_name = None
output_tags = None
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
output_name=output_name, output_tags=output_tags)
+ make_fs_access = functools.partial(CollectionFsAccess,
+ collection_cache=runner.collection_cache)
+
t = load_tool(toolpath, runner.arv_make_tool,
fetcher_constructor=functools.partial(CollectionFetcher,
- api_client=api,
- keep_client=arvados.keep.KeepClient(api_client=api, num_retries=4)))
+ api_client=runner.api,
+ fs_access=make_fs_access(""),
+ num_retries=runner.num_retries))
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
args.basedir = os.getcwd()
args.name = None
args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
+ args.make_fs_access = make_fs_access
+
runner.arv_executor(t, job_order_object, **vars(args))
except Exception as e:
if isinstance(e, WorkflowException):
import urlparse
import re
import logging
+import threading
import ruamel.yaml as yaml
logger = logging.getLogger('arvados.cwl-runner')
+class CollectionCache(object):
+ def __init__(self, api_client, keep_client, num_retries):
+ self.api_client = api_client
+ self.keep_client = keep_client
+ self.collections = {}
+ self.lock = threading.Lock()
+
+ def get(self, pdh):
+ with self.lock:
+ if pdh not in self.collections:
+ logger.debug("Creating collection reader for %s", pdh)
+ self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
+ keep_client=self.keep_client)
+ return self.collections[pdh]
+
+
class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
"""Implement the cwltool FsAccess interface for Arvados Collections."""
- def __init__(self, basedir, api_client=None, keep_client=None):
+ def __init__(self, basedir, collection_cache=None):
super(CollectionFsAccess, self).__init__(basedir)
- self.api_client = api_client
- self.keep_client = keep_client
- self.collections = {}
+ self.collection_cache = collection_cache
def get_collection(self, path):
sp = path.split("/", 1)
p = sp[0]
if p.startswith("keep:") and arvados.util.keep_locator_pattern.match(p[5:]):
pdh = p[5:]
- if pdh not in self.collections:
- self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
- keep_client=self.keep_client)
- return (self.collections[pdh], sp[1] if len(sp) == 2 else None)
+ return (self.collection_cache.get(pdh), sp[1] if len(sp) == 2 else None)
else:
return (None, path)
def exists(self, fn):
collection, rest = self.get_collection(fn)
if collection:
- return collection.exists(rest)
+ if rest:
+ return collection.exists(rest)
+ else:
+ return True
else:
return super(CollectionFsAccess, self).exists(fn)
return os.path.realpath(path)
class CollectionFetcher(DefaultFetcher):
- def __init__(self, cache, session, api_client=None, keep_client=None, num_retries=4):
+ def __init__(self, cache, session, api_client=None, fs_access=None, num_retries=4):
super(CollectionFetcher, self).__init__(cache, session)
self.api_client = api_client
- self.fsaccess = CollectionFsAccess("", api_client=api_client, keep_client=keep_client)
+ self.fsaccess = fs_access
self.num_retries = num_retries
def fetch_text(self, url):
def check_exists(self, url):
try:
+ if url.startswith("http://arvados.org/cwl"):
+ return True
if url.startswith("keep:"):
return self.fsaccess.exists(url)
if url.startswith("arvwf:"):
import logging
import uuid
import os
+import urllib
import arvados.commands.run
import arvados.collection
"""Convert container-local paths to and from Keep collection ids."""
pdh_path = re.compile(r'^keep:[0-9a-f]{32}\+\d+/.+$')
- pdh_dirpath = re.compile(r'^keep:[0-9a-f]{32}\+\d+(/.+)?$')
+ pdh_dirpath = re.compile(r'^keep:[0-9a-f]{32}\+\d+(/.*)?$')
def __init__(self, arvrunner, referenced_files, input_basedir,
collection_pattern, file_pattern, name=None, **kwargs):
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
- if srcobj["class"] == "File":
- if "#" in src:
- src = src[:src.index("#")]
- if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "File")
- if src not in self._pathmap:
+ if "#" in src:
+ src = src[:src.index("#")]
+
+ if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
+ self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
+
+ if src not in self._pathmap:
+ if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
- st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s")
+ st = arvados.commands.run.statfile("", ab,
+ fnPattern="keep:%s/%s",
+ dirPattern="keep:%s/%s")
with SourceLine(srcobj, "location", WorkflowException):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
- elif src.startswith("_:"):
- if "contents" in srcobj:
- pass
- else:
- raise WorkflowException("File literal '%s' is missing contents" % src)
- elif src.startswith("arvwf:"):
- self._pathmap[src] = MapperEnt(src, src, "File")
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
- if "secondaryFiles" in srcobj:
- for l in srcobj["secondaryFiles"]:
- self.visit(l, uploadfiles)
- elif srcobj["class"] == "Directory":
- if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
+ elif src.startswith("_:"):
+ if srcobj["class"] == "File" and "contents" not in srcobj:
+ raise WorkflowException("File literal '%s' is missing `contents`" % src)
+ if srcobj["class"] == "Directory" and "listing" not in srcobj:
+ raise WorkflowException("Directory literal '%s' is missing `listing`" % src)
+ else:
+ self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
+
+ with SourceLine(srcobj, "secondaryFiles", WorkflowException):
+ for l in srcobj.get("secondaryFiles", []):
+ self.visit(l, uploadfiles)
+ with SourceLine(srcobj, "listing", WorkflowException):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
for l in obj.get("secondaryFiles", []):
self.addentry(l, c, path, subdirs)
elif obj["class"] == "Directory":
- for l in obj["listing"]:
+ for l in obj.get("listing", []):
self.addentry(l, c, path + "/" + obj["basename"], subdirs)
subdirs.append((obj["location"], path + "/" + obj["basename"]))
elif obj["location"].startswith("_:") and "contents" in obj:
loc = k["location"]
if loc in already_uploaded:
v = already_uploaded[loc]
- self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % v.resolved[5:], "File")
+ self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % urllib.unquote(v.resolved[5:]), "File", True)
for srcobj in referenced_files:
self.visit(srcobj, uploadfiles)
project=self.arvrunner.project_uuid)
for src, ab, st in uploadfiles:
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
+ self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
+ "Directory" if os.path.isdir(ab) else "File", True)
self.arvrunner.add_uploaded(src, self._pathmap[src])
for srcobj in referenced_files:
+ subdirs = []
if srcobj["class"] == "Directory":
if srcobj["location"] not in self._pathmap:
c = arvados.collection.Collection(api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
- subdirs = []
- for l in srcobj["listing"]:
+ for l in srcobj.get("listing", []):
self.addentry(l, c, ".", subdirs)
check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
c.save_new(owner_uuid=self.arvrunner.project_uuid)
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "Directory")
- for loc, sub in subdirs:
- ab = self.file_pattern % (c.portable_data_hash(), sub[2:])
- self._pathmap[loc] = MapperEnt(ab, ab, "Directory")
+ self._pathmap[srcobj["location"]] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
elif srcobj["class"] == "File" and (srcobj.get("secondaryFiles") or
(srcobj["location"].startswith("_:") and "contents" in srcobj)):
c = arvados.collection.Collection(api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries )
- subdirs = []
self.addentry(srcobj, c, ".", subdirs)
check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
c.save_new(owner_uuid=self.arvrunner.project_uuid)
ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
- self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "File")
+ self._pathmap[srcobj["location"]] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), srcobj["basename"]),
+ ab, "File", True)
if srcobj.get("secondaryFiles"):
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt(ab, ab, "Directory")
+ self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
+
+ if subdirs:
for loc, sub in subdirs:
+ # subdirs will all start with "./", strip it off
ab = self.file_pattern % (c.portable_data_hash(), sub[2:])
- self._pathmap[loc] = MapperEnt(ab, ab, "Directory")
+ self._pathmap[loc] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), sub[2:]),
+ ab, "Directory", True)
self.keepdir = None
class StagingPathMapper(PathMapper):
_follow_dirs = True
- def visit(self, obj, stagedir, basedir, copy=False):
+ def visit(self, obj, stagedir, basedir, copy=False, staged=False):
# type: (Dict[unicode, Any], unicode, unicode, bool) -> None
loc = obj["location"]
tgt = os.path.join(stagedir, obj["basename"])
if obj["class"] == "Directory":
- self._pathmap[loc] = MapperEnt(loc, tgt, "Directory")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
if loc.startswith("_:") or self._follow_dirs:
self.visitlisting(obj.get("listing", []), tgt, basedir)
elif obj["class"] == "File":
if loc in self._pathmap:
return
if "contents" in obj and loc.startswith("_:"):
- self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile")
+ self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile", staged)
else:
if copy:
- self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile", staged)
else:
- self._pathmap[loc] = MapperEnt(loc, tgt, "File")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "File", staged)
self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir)
# with any secondary files.
self.visitlisting(referenced_files, self.stagedir, basedir)
- for path, (ab, tgt, type) in self._pathmap.items():
+ for path, (ab, tgt, type, staged) in self._pathmap.items():
if type in ("File", "Directory") and ab.startswith("keep:"):
- self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type)
+ self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type, staged)
class NoFollowPathMapper(StagingPathMapper):
from functools import partial
import logging
import json
-import re
import subprocess
from StringIO import StringIO
logger = logging.getLogger('arvados.cwl-runner')
-cwltool.draft2tool.ACCEPTLIST_RE = re.compile(r".*")
-
def trim_listing(obj):
"""Remove 'listing' field from Directory objects that are keep references.
if submit_runner_ram:
self.submit_runner_ram = submit_runner_ram
else:
- self.submit_runner_ram = 1024
+ self.submit_runner_ram = 3000
if self.submit_runner_ram <= 0:
raise Exception("Value of --submit-runner-ram must be greater than zero")
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20170213175853',
- 'schema-salad==2.2.20170208112505',
+ 'cwltool==1.0.20170413194156',
+ 'schema-salad==2.5.20170328195758',
+ 'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
- 'arvados-python-client>=0.1.20170112173420',
+ 'arvados-python-client>=0.1.20170327195441',
'setuptools'
],
data_files=[
export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
+arv-keepdocker --pull arvados/jobs latest
+
cat >/tmp/cwltest/arv-cwl-jobs <<EOF2
#!/bin/sh
exec arvados-cwl-runner --api=jobs --compute-checksum \\\$@
if ! arv-get d7514270f356df848477718d58308cc4+94 > /dev/null ; then
arv-put --portable-data-hash testdir
fi
-exec cwltest --test arvados-tests.yml --tool $PWD/runner.sh
+exec cwltest --test arvados-tests.yml --tool $PWD/runner.sh $@
}
tool: keep-dir-test-input.cwl
doc: Test directory in keep
+
+- job: dir-job2.yml
+ output:
+ "outlist": {
+ "size": 20,
+ "location": "output.txt",
+ "class": "File",
+ "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592"
+ }
+ tool: keep-dir-test-input.cwl
+ doc: Test directory in keep
+
+- job: null
+ output:
+ "outlist": {
+ "size": 20,
+ "location": "output.txt",
+ "class": "File",
+ "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592"
+ }
+ tool: keep-dir-test-input2.cwl
+ doc: Test default directory in keep
+
+- job: null
+ output:
+ "outlist": {
+ "size": 20,
+ "location": "output.txt",
+ "class": "File",
+ "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592"
+ }
+ tool: keep-dir-test-input3.cwl
+ doc: Test default directory in keep
+
+- job: octo.yml
+ output: {}
+ tool: cat.cwl
+ doc: Test hashes in filenames
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: wf/listing_shallow.cwl
+ doc: test shallow directory listing
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: wf/listing_none.cwl
+ doc: test no directory listing
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: wf/listing_deep.cwl
+ doc: test deep directory listing
--- /dev/null
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+ - id: inp
+ type: File
+ inputBinding: {}
+outputs: []
+baseCommand: cat
--- /dev/null
+indir:
+ class: Directory
+ location: keep:d7514270f356df848477718d58308cc4+94/
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ - class: ShellCommandRequirement
+inputs:
+ indir:
+ type: Directory
+ inputBinding:
+ prefix: cd
+ position: -1
+ default:
+ class: Directory
+ location: keep:d7514270f356df848477718d58308cc4+94
+outputs:
+ outlist:
+ type: File
+ outputBinding:
+ glob: output.txt
+arguments: [
+ {shellQuote: false, valueFrom: "&&"},
+ "find", ".",
+ {shellQuote: false, valueFrom: "|"},
+ "sort"]
+stdout: output.txt
\ No newline at end of file
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ - class: ShellCommandRequirement
+inputs:
+ indir:
+ type: Directory
+ inputBinding:
+ prefix: cd
+ position: -1
+ default:
+ class: Directory
+ location: keep:d7514270f356df848477718d58308cc4+94/
+outputs:
+ outlist:
+ type: File
+ outputBinding:
+ glob: output.txt
+arguments: [
+ {shellQuote: false, valueFrom: "&&"},
+ "find", ".",
+ {shellQuote: false, valueFrom: "|"},
+ "sort"]
+stdout: output.txt
\ No newline at end of file
--- /dev/null
+d:
+ class: Directory
+ location: tmp1
\ No newline at end of file
--- /dev/null
+inp:
+ class: File
+ location: "octothorpe/item %231.txt"
\ No newline at end of file
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}]
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
basedir="", make_fs_access=make_fs_access, loader=Loader({}))
arvtool.formatgraph = None
}],
"baseCommand": "ls"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
avsc_names=avsc_names, make_fs_access=make_fs_access,
loader=Loader({}))
}],
"baseCommand": "ls"
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
avsc_names=avsc_names, make_fs_access=make_fs_access,
loader=Loader({}))
for key in call_body:
self.assertEqual(call_body_expected.get(key), call_body.get(key))
+
+ # Test redirecting stdin/stdout/stderr
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_redirects(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls",
+ "stdout": "stdout.txt",
+ "stderr": "stderr.txt",
+ "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+ })
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
+ basedir="", make_fs_access=make_fs_access, loader=Loader({}))
+ arvtool.formatgraph = None
+ for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
+ make_fs_access=make_fs_access, tmpdir="/tmp"):
+ j.run()
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_redirect',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ 'use_existing': True,
+ 'priority': 1,
+ 'mounts': {
+ '/var/spool/cwl': {'kind': 'tmp'},
+ "stderr": {
+ "kind": "file",
+ "path": "/var/spool/cwl/stderr.txt"
+ },
+ "stdin": {
+ "kind": "collection",
+ "path": "file.txt",
+ "portable_data_hash": "99999999999999999999999999999996+99"
+ },
+ "stdout": {
+ "kind": "file",
+ "path": "/var/spool/cwl/stdout.txt"
+ },
+ },
+ 'state': 'Committed',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'container_image': 'arvados/jobs',
+ 'command': ['ls', '/var/spool/cwl'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ }))
+
@mock.patch("arvados.collection.Collection")
def test_done(self, col):
api = mock.MagicMock()
--- /dev/null
+import functools
+import mock
+import sys
+import unittest
+import json
+import logging
+import os
+
+import arvados
+import arvados.keep
+import arvados.collection
+import arvados_cwl
+
+from cwltool.pathmapper import MapperEnt
+from .mock_discovery import get_rootDesc
+
+from arvados_cwl.fsaccess import CollectionCache
+
+class TestFsAccess(unittest.TestCase):
+ @mock.patch("arvados.collection.CollectionReader")
+ def test_collection_cache(self, cr):
+ cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
+ c1 = cache.get("99999999999999999999999999999991+99")
+ c2 = cache.get("99999999999999999999999999999991+99")
+ self.assertIs(c1, c2)
+ self.assertEqual(1, cr.call_count)
+ c3 = cache.get("99999999999999999999999999999992+99")
+ self.assertEqual(2, cr.call_count)
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}]
})
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
basedir="", make_fs_access=make_fs_access, loader=Loader({}))
arvtool.formatgraph = None
}],
"baseCommand": "ls"
}
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
make_fs_access=make_fs_access, loader=Loader({}))
arvtool.formatgraph = None
mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
basedir="", make_fs_access=make_fs_access, loader=document_loader,
makeTool=runner.arv_make_tool, metadata=metadata)
def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
pdh = "99999999999999999999999999999991+99"
for c in files:
+ c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
c.fn = fnPattern % (pdh, os.path.basename(c.fn))
class TestPathmap(unittest.TestCase):
"location": "keep:99999999999999999999999999999991+99/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
- def test_upload(self, upl):
+ @mock.patch("arvados.commands.run.statfile")
+ def test_upload(self, statfile, upl):
"""Test pathmapper uploading files."""
arvrunner = arvados_cwl.ArvCwlRunner(self.api)
+ def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
+ st = arvados.commands.run.UploadFile("", "tests/hw.py")
+ return st
+
upl.side_effect = upload_mock
+ statfile.side_effect = statfile_mock
p = ArvPathMapper(arvrunner, [{
"class": "File",
- "location": "tests/hw.py"
+ "location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
"""Test pathmapper handling previously uploaded files."""
arvrunner = arvados_cwl.ArvCwlRunner(self.api)
- arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File'))
+ arvrunner.add_uploaded('file:tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File', staged=True))
upl.side_effect = upload_mock
p = ArvPathMapper(arvrunner, [{
"class": "File",
- "location": "tests/hw.py"
+ "location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
p = ArvPathMapper(arvrunner, [{
"class": "File",
- "location": "tests/hw.py"
+ "location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_tool.cwl dependencies',
}), ensure_unique_name=True),
- mock.call().execute(),
+ mock.call().execute(num_retries=4),
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_wf.cwl input',
}), ensure_unique_name=True),
- mock.call().execute()])
+ mock.call().execute(num_retries=4)])
arvdock.assert_has_calls([
mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_tool.cwl dependencies',
}), ensure_unique_name=True),
- mock.call().execute(),
+ mock.call().execute(num_retries=4),
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_wf.cwl input',
}), ensure_unique_name=True),
- mock.call().execute()])
+ mock.call().execute(num_retries=4)])
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
arvrunner.project_uuid = ""
api.return_value = mock.MagicMock()
arvrunner.api = api.return_value
- arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
- {"items": [], "items_available": 0, "offset": 0},
- {"items": [], "items_available": 0, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
- "link_class": "docker_image_hash",
- "name": "123456",
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [], "items_available": 0, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
+ arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
+ "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
"link_class": "docker_image_repo+tag",
"name": "arvados/jobs:"+arvados_cwl.__version__,
"owner_uuid": "",
"link_class": "docker_image_hash",
"name": "123456",
"owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} ,
+ "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
)
find_one_image_hash.return_value = "123456"
- arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
- {"items": [{"uuid": "",
+ arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
"owner_uuid": "",
"manifest_text": "",
"properties": ""
- }], "items_available": 1, "offset": 0},
- {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
+ }], "items_available": 1, "offset": 0},)
arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
+ arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: deep_listing
+ InlineJavascriptRequirement: {}
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing[0].class === 'Directory' && inputs.d.listing[0].listing[0].class === 'Directory') {return 'true';} else {return 'false';}}"]
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ cwltool: http://commonwl.org/cwltool#
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+ InlineJavascriptRequirement: {}
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
\ No newline at end of file
--- /dev/null
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ cwltool: http://commonwl.org/cwltool#
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: shallow_listing
+ InlineJavascriptRequirement: {}
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing[0].class === 'Directory' && inputs.d.listing[0].listing === undefined) {return 'true';} else {return 'false';}}"]
out: [out]
run:
class: CommandLineTool
+ id: subtool
inputs:
sleeptime:
type: int
"run": {
"baseCommand": "sleep",
"class": "CommandLineTool",
+ "id": "#main/sleep1/subtool",
"inputs": [
{
- "id": "#main/sleep1/sleeptime",
+ "id": "#main/sleep1/subtool/sleeptime",
"inputBinding": {
"position": 1
},
],
"outputs": [
{
- "id": "#main/sleep1/out",
+ "id": "#main/sleep1/subtool/out",
"outputBinding": {
"outputEval": "out"
},
import (
"encoding/json"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
"io"
"io/ioutil"
"log"
"os/signal"
"strings"
"syscall"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
)
type TaskDef struct {
}
type Job struct {
- Script_parameters Tasks `json:"script_parameters"`
+ ScriptParameters Tasks `json:"script_parameters"`
}
type Task struct {
- Job_uuid string `json:"job_uuid"`
- Created_by_job_task_uuid string `json:"created_by_job_task_uuid"`
- Parameters TaskDef `json:"parameters"`
- Sequence int `json:"sequence"`
- Output string `json:"output"`
- Success bool `json:"success"`
- Progress float32 `json:"sequence"`
+ JobUUID string `json:"job_uuid"`
+ CreatedByJobTaskUUID string `json:"created_by_job_task_uuid"`
+ Parameters TaskDef `json:"parameters"`
+ Sequence int `json:"sequence"`
+ Output string `json:"output"`
+ Success bool `json:"success"`
+ Progress float32 `json:"sequence"`
}
type IArvadosClient interface {
Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error)
}
-func setupDirectories(crunchtmpdir, taskUuid string, keepTmp bool) (tmpdir, outdir string, err error) {
+func setupDirectories(crunchtmpdir, taskUUID string, keepTmp bool) (tmpdir, outdir string, err error) {
tmpdir = crunchtmpdir + "/tmpdir"
err = os.Mkdir(tmpdir, 0700)
if err != nil {
func runner(api IArvadosClient,
kc IKeepClient,
- jobUuid, taskUuid, crunchtmpdir, keepmount string,
+ jobUUID, taskUUID, crunchtmpdir, keepmount string,
jobStruct Job, taskStruct Task) error {
var err error
// If this is task 0 and there are multiple tasks, dispatch subtasks
// and exit.
if taskStruct.Sequence == 0 {
- if len(jobStruct.Script_parameters.Tasks) == 1 {
- taskp = jobStruct.Script_parameters.Tasks[0]
+ if len(jobStruct.ScriptParameters.Tasks) == 1 {
+ taskp = jobStruct.ScriptParameters.Tasks[0]
} else {
- for _, task := range jobStruct.Script_parameters.Tasks {
+ for _, task := range jobStruct.ScriptParameters.Tasks {
err := api.Create("job_tasks",
map[string]interface{}{
- "job_task": Task{Job_uuid: jobUuid,
- Created_by_job_task_uuid: taskUuid,
- Sequence: 1,
- Parameters: task}},
+ "job_task": Task{
+ JobUUID: jobUUID,
+ CreatedByJobTaskUUID: taskUUID,
+ Sequence: 1,
+ Parameters: task}},
nil)
if err != nil {
return TempFail{err}
}
}
- err = api.Update("job_tasks", taskUuid,
+ err = api.Update("job_tasks", taskUUID,
map[string]interface{}{
- "job_task": Task{
- Output: "",
- Success: true,
- Progress: 1.0}},
+ "job_task": map[string]interface{}{
+ "output": "",
+ "success": true,
+ "progress": 1.0}},
nil)
return nil
}
}
var tmpdir, outdir string
- tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUuid, taskp.KeepTmpOutput)
+ tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUUID, taskp.KeepTmpOutput)
if err != nil {
return TempFail{err}
}
}
// Set status
- err = api.Update("job_tasks", taskUuid,
+ err = api.Update("job_tasks", taskUUID,
map[string]interface{}{
"job_task": Task{
Output: manifest,
log.Fatal(err)
}
- jobUuid := os.Getenv("JOB_UUID")
- taskUuid := os.Getenv("TASK_UUID")
+ jobUUID := os.Getenv("JOB_UUID")
+ taskUUID := os.Getenv("TASK_UUID")
tmpdir := os.Getenv("TASK_WORK")
keepmount := os.Getenv("TASK_KEEPMOUNT")
var jobStruct Job
var taskStruct Task
- err = api.Get("jobs", jobUuid, nil, &jobStruct)
+ err = api.Get("jobs", jobUUID, nil, &jobStruct)
if err != nil {
log.Fatal(err)
}
- err = api.Get("job_tasks", taskUuid, nil, &taskStruct)
+ err = api.Get("job_tasks", taskUUID, nil, &taskStruct)
if err != nil {
log.Fatal(err)
}
}
syscall.Umask(0022)
- err = runner(api, kc, jobUuid, taskUuid, tmpdir, keepmount, jobStruct, taskStruct)
+ err = runner(api, kc, jobUUID, taskUUID, tmpdir, keepmount, jobStruct, taskStruct)
if err == nil {
os.Exit(0)
package main
import (
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- . "gopkg.in/check.v1"
"io"
"io/ioutil"
"log"
"syscall"
"testing"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ . "gopkg.in/check.v1"
)
// Gocheck boilerplate
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"}}}}},
Task{Sequence: 0})
c.Check(err, IsNil)
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{
+ Job{ScriptParameters: Tasks{[]TaskDef{
{Command: []string{"echo", "bar"}},
{Command: []string{"echo", "foo"}}}}},
Task{Parameters: TaskDef{
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"cat"},
Stdout: "output.txt",
Stdin: tmpfile.Name()}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $BAR"},
Stdout: "output.txt",
Env: map[string]string{"BAR": "foo"}}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
"foo\n",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $BAR"},
Stdout: "output.txt",
Env: map[string]string{"BAR": "$(task.keep)"}}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $PATH"},
Stdout: "output.txt",
Env: map[string]string{"PATH": "foo"}}}}},
func (s *TestSuite) TestScheduleSubtask(c *C) {
api := SubtaskTestClient{c, []Task{
- {Job_uuid: "zzzz-8i9sb-111111111111111",
- Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
- Sequence: 1,
+ {JobUUID: "zzzz-8i9sb-111111111111111",
+ CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
+ Sequence: 1,
Parameters: TaskDef{
Command: []string{"echo", "bar"}}},
- {Job_uuid: "zzzz-8i9sb-111111111111111",
- Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
- Sequence: 1,
+ {JobUUID: "zzzz-8i9sb-111111111111111",
+ CreatedByJobTaskUUID: "zzzz-ot0gb-111111111111111",
+ Sequence: 1,
Parameters: TaskDef{
Command: []string{"echo", "foo"}}}},
0}
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{
+ Job{ScriptParameters: Tasks{[]TaskDef{
{Command: []string{"echo", "bar"}},
{Command: []string{"echo", "foo"}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"},
SuccessCodes: []int{0, 1}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 0"},
PermanentFailCodes: []int{0, 1}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"},
TemporaryFailCodes: []int{1}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"ls", "output.txt"},
Vwd: map[string]string{
"output.txt": tmpfile.Name()}}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"cat"},
Stdout: "output.txt",
Stdin: "$(task.keep)/file1.txt"}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"cat", "$(task.keep)/file1.txt"},
Stdout: "output.txt"}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"sleep", "4"}}}}},
Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"},
Stdout: "s ub:dir/:e vi\nl"}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{{
+ Job{ScriptParameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"},
KeepTmpOutput: true}}}},
Task{Sequence: 0})
func (*ThrottleTestSuite) TestThrottle(c *check.C) {
uuid := "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
+ t0 := throttle{}
+ c.Check(t0.Check(uuid), check.Equals, true)
+ c.Check(t0.Check(uuid), check.Equals, true)
- t := throttle{}
- c.Check(t.Check(uuid), check.Equals, true)
- c.Check(t.Check(uuid), check.Equals, true)
-
- t = throttle{hold: time.Nanosecond}
- c.Check(t.Check(uuid), check.Equals, true)
+ tNs := throttle{hold: time.Nanosecond}
+ c.Check(tNs.Check(uuid), check.Equals, true)
time.Sleep(time.Microsecond)
- c.Check(t.Check(uuid), check.Equals, true)
-
- t = throttle{hold: time.Minute}
- c.Check(t.Check(uuid), check.Equals, true)
- c.Check(t.Check(uuid), check.Equals, false)
- c.Check(t.Check(uuid), check.Equals, false)
- t.seen[uuid].last = time.Now().Add(-time.Hour)
- c.Check(t.Check(uuid), check.Equals, true)
- c.Check(t.Check(uuid), check.Equals, false)
+ c.Check(tNs.Check(uuid), check.Equals, true)
+
+ tMin := throttle{hold: time.Minute}
+ c.Check(tMin.Check(uuid), check.Equals, true)
+ c.Check(tMin.Check(uuid), check.Equals, false)
+ c.Check(tMin.Check(uuid), check.Equals, false)
+ tMin.seen[uuid].last = time.Now().Add(-time.Hour)
+ c.Check(tMin.Check(uuid), check.Equals, true)
+ c.Check(tMin.Check(uuid), check.Equals, false)
}
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
+ "log"
"math/rand"
"net"
"net/http"
+ "os"
+ "regexp"
"strings"
"time"
)
// log.Printf to DebugPrintf.
var DebugPrintf = func(string, ...interface{}) {}
+func init() {
+ var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+ if matchTrue.MatchString(os.Getenv("ARVADOS_DEBUG")) {
+ DebugPrintf = log.Printf
+ }
+}
+
type keepService struct {
Uuid string `json:"uuid"`
Hostname string `json:"service_host"`
-import gflags
import httplib
import httplib2
import logging
block_size = dl.range_size
block_end = block_start + block_size
_logger.log(RANGES_SPAM,
- "%s range_start %s block_start %s range_end %s block_end %s",
+ "L&R %s range_start %s block_start %s range_end %s block_end %s",
dl.locator, range_start, block_start, range_end, block_end)
if range_end <= block_start:
# range ends before this block starts, so don't look at any more locators
last = data_locators[-1]
if (last.range_start+last.range_size) == new_range_start:
- if last.locator == new_locator:
+ if last.locator == new_locator and (last.segment_offset+last.range_size) == new_segment_offset:
# extend last segment
last.range_size += new_range_size
else:
old_segment_start = dl.range_start
old_segment_end = old_segment_start + dl.range_size
_logger.log(RANGES_SPAM,
- "%s range_start %s segment_start %s range_end %s segment_end %s",
+ "RR %s range_start %s segment_start %s range_end %s segment_end %s",
dl, new_range_start, old_segment_start, new_range_end,
old_segment_end)
if new_range_end <= old_segment_start:
# range ends before this segment starts, so don't look at any more locators
break
- if old_segment_start <= new_range_start and new_range_end <= old_segment_end:
+ if old_segment_start <= new_range_start and new_range_end <= old_segment_end:
# new range starts and ends in old segment
# split segment into up to 3 pieces
if (new_range_start-old_segment_start) > 0:
import config
import errors
import util
+import cache
_logger = logging.getLogger('arvados.api')
return body
-def _intercept_http_request(self, uri, **kwargs):
+def _intercept_http_request(self, uri, method="GET", **kwargs):
if (self.max_request_size and
kwargs.get('body') and
self.max_request_size < len(kwargs['body'])):
kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
- retryable = kwargs.get('method', 'GET') in [
+ retryable = method in [
'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
retry_count = self._retry_count if retryable else 0
for _ in range(retry_count):
self._last_request_time = time.time()
try:
- return self.orig_http_request(uri, **kwargs)
+ return self.orig_http_request(uri, method, **kwargs)
except httplib.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
delay = delay * self._retry_delay_backoff
self._last_request_time = time.time()
- return self.orig_http_request(uri, **kwargs)
+ return self.orig_http_request(uri, method, **kwargs)
def _patch_http_request(http, api_token):
http.arvados_api_token = api_token
try:
util.mkdir_dash_p(path)
except OSError:
- path = None
- return path
+ return None
+ return cache.SafeHTTPCache(path, max_age=60*60*24*2)
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
from .errors import KeepWriteError, AssertionError, ArgumentError
from .keep import KeepLocator
from ._normalize_stream import normalize_stream
-from ._ranges import locators_and_ranges, replace_range, Range
+from ._ranges import locators_and_ranges, replace_range, Range, LocatorAndRange
from .retry import retry_method
MOD = "mod"
pos += self._filepos
elif whence == os.SEEK_END:
pos += self.size()
- self._filepos = min(max(pos, 0L), self.size())
+ if pos < 0L:
+ raise IOError(errno.EINVAL, "Tried to seek to negative file offset.")
+ self._filepos = pos
+ return self._filepos
def tell(self):
return self._filepos
+ def readable(self):
+ return True
+
+ def writable(self):
+ return False
+
+ def seekable(self):
+ return True
+
@_FileLikeObjectBase._before_close
@retry_method
def readall(self, size=2**20, num_retries=None):
return ''.join(data).splitlines(True)
def size(self):
- raise NotImplementedError()
+ raise IOError(errno.ENOSYS, "Not implemented")
def read(self, size, num_retries=None):
- raise NotImplementedError()
+ raise IOError(errno.ENOSYS, "Not implemented")
def readfrom(self, start, size, num_retries=None):
- raise NotImplementedError()
+ raise IOError(errno.ENOSYS, "Not implemented")
class StreamFileReader(ArvadosFileReaderBase):
PENDING = 1
COMMITTED = 2
ERROR = 3
+ DELETED = 4
def __init__(self, blockid, starting_capacity, owner):
"""
@synchronized
def clear(self):
+ self._state = _BufferBlock.DELETED
self.owner = None
self.buffer_block = None
self.buffer_view = None
+ @synchronized
+ def repack_writes(self):
+ """Optimize buffer block by repacking segments in file sequence.
+
+ When the client makes random writes, they appear in the buffer block in
+ the sequence they were written rather than the sequence they appear in
+ the file. This makes for inefficient, fragmented manifests. Attempt
+ to optimize by repacking writes in file sequence.
+
+ """
+ if self._state != _BufferBlock.WRITABLE:
+ raise AssertionError("Cannot repack non-writable block")
+
+ segs = self.owner.segments()
+
+ # Collect the segments that reference the buffer block.
+ bufferblock_segs = [s for s in segs if s.locator == self.blockid]
+
+ # Collect total data referenced by segments (could be smaller than
+ # bufferblock size if a portion of the file was written and
+ # then overwritten).
+ write_total = sum([s.range_size for s in bufferblock_segs])
+
+ if write_total < self.size() or len(bufferblock_segs) > 1:
+ # If there's more than one segment referencing this block, it is
+ # due to out-of-order writes and will produce a fragmented
+ # manifest, so try to optimize by re-packing into a new buffer.
+ contents = self.buffer_view[0:self.write_pointer].tobytes()
+ new_bb = _BufferBlock(None, write_total, None)
+ for t in bufferblock_segs:
+ new_bb.append(contents[t.segment_offset:t.segment_offset+t.range_size])
+ t.segment_offset = new_bb.size() - t.range_size
+
+ self.buffer_block = new_bb.buffer_block
+ self.buffer_view = new_bb.buffer_view
+ self.write_pointer = new_bb.write_pointer
+ self._locator = None
+ new_bb.clear()
+ self.owner.set_segments(segs)
+
+ def __repr__(self):
+ return "<BufferBlock %s>" % (self.blockid)
+
class NoopLock(object):
def __enter__(self):
self.copies = copies
self._pending_write_size = 0
self.threads_lock = threading.Lock()
+ self.padding_block = None
@synchronized
def alloc_bufferblock(self, blockid=None, starting_capacity=2**14, owner=None):
def _alloc_bufferblock(self, blockid=None, starting_capacity=2**14, owner=None):
if blockid is None:
- blockid = "%s" % uuid.uuid4()
+ blockid = str(uuid.uuid4())
bufferblock = _BufferBlock(blockid, starting_capacity=starting_capacity, owner=owner)
self._bufferblocks[bufferblock.blockid] = bufferblock
return bufferblock
ArvadosFile that owns the new block
"""
- new_blockid = "bufferblock%i" % len(self._bufferblocks)
+ new_blockid = str(uuid.uuid4())
bufferblock = block.clone(new_blockid, owner)
self._bufferblocks[bufferblock.blockid] = bufferblock
return bufferblock
@synchronized
def repack_small_blocks(self, force=False, sync=False, closed_file_size=0):
"""Packs small blocks together before uploading"""
+
self._pending_write_size += closed_file_size
# Check if there are enough small blocks for filling up one in full
- if force or (self._pending_write_size >= config.KEEP_BLOCK_SIZE):
+ if not (force or (self._pending_write_size >= config.KEEP_BLOCK_SIZE)):
+ return
- # Search blocks ready for getting packed together before being committed to Keep.
- # A WRITABLE block always has an owner.
- # A WRITABLE block with its owner.closed() implies that it's
- # size is <= KEEP_BLOCK_SIZE/2.
- try:
- small_blocks = [b for b in self._bufferblocks.values() if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
- except AttributeError:
- # Writable blocks without owner shouldn't exist.
- raise UnownedBlockError()
+ # Search blocks ready for getting packed together before being committed to Keep.
+ # A WRITABLE block always has an owner.
+ # A WRITABLE block with its owner.closed() implies that it's
+ # size is <= KEEP_BLOCK_SIZE/2.
+ try:
+ small_blocks = [b for b in self._bufferblocks.values()
+ if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
+ except AttributeError:
+ # Writable blocks without owner shouldn't exist.
+ raise UnownedBlockError()
+
+ if len(small_blocks) <= 1:
+ # Not enough small blocks for repacking
+ return
- if len(small_blocks) <= 1:
- # Not enough small blocks for repacking
- return
+ for bb in small_blocks:
+ bb.repack_writes()
- # Update the pending write size count with its true value, just in case
- # some small file was opened, written and closed several times.
- self._pending_write_size = sum([b.size() for b in small_blocks])
- if self._pending_write_size < config.KEEP_BLOCK_SIZE and not force:
- return
+ # Update the pending write size count with its true value, just in case
+ # some small file was opened, written and closed several times.
+ self._pending_write_size = sum([b.size() for b in small_blocks])
- new_bb = self._alloc_bufferblock()
- while len(small_blocks) > 0 and (new_bb.write_pointer + small_blocks[0].size()) <= config.KEEP_BLOCK_SIZE:
- bb = small_blocks.pop(0)
- arvfile = bb.owner
- self._pending_write_size -= bb.size()
- new_bb.append(bb.buffer_view[0:bb.write_pointer].tobytes())
- arvfile.set_segments([Range(new_bb.blockid,
- 0,
- bb.size(),
- new_bb.write_pointer - bb.size())])
- self._delete_bufferblock(bb.blockid)
- self.commit_bufferblock(new_bb, sync=sync)
+ if self._pending_write_size < config.KEEP_BLOCK_SIZE and not force:
+ return
+
+ new_bb = self._alloc_bufferblock()
+ files = []
+ while len(small_blocks) > 0 and (new_bb.write_pointer + small_blocks[0].size()) <= config.KEEP_BLOCK_SIZE:
+ bb = small_blocks.pop(0)
+ self._pending_write_size -= bb.size()
+ new_bb.append(bb.buffer_view[0:bb.write_pointer].tobytes())
+ files.append((bb, new_bb.write_pointer - bb.size()))
+
+ self.commit_bufferblock(new_bb, sync=sync)
+
+ for bb, new_bb_segment_offset in files:
+ newsegs = bb.owner.segments()
+ for s in newsegs:
+ if s.locator == bb.blockid:
+ s.locator = new_bb.locator()
+ s.segment_offset = new_bb_segment_offset+s.segment_offset
+ bb.owner.set_segments(newsegs)
+ self._delete_bufferblock(bb.blockid)
def commit_bufferblock(self, block, sync):
"""Initiate a background upload of a bufferblock.
def get_bufferblock(self, locator):
return self._bufferblocks.get(locator)
+ @synchronized
+ def get_padding_block(self):
+ """Get a bufferblock 64 MB in size consisting of all zeros, used as padding
+ when using truncate() to extend the size of a file.
+
+ For reference (and possible future optimization), the md5sum of the
+ padding block is: 7f614da9329cd3aebf59b91aadc30bf0+67108864
+
+ """
+
+ if self.padding_block is None:
+ self.padding_block = self._alloc_bufferblock(starting_capacity=config.KEEP_BLOCK_SIZE)
+ self.padding_block.write_pointer = config.KEEP_BLOCK_SIZE
+ self.commit_bufferblock(self.padding_block, False)
+ return self.padding_block
+
@synchronized
def delete_bufferblock(self, locator):
self._delete_bufferblock(locator)
@must_be_writable
@synchronized
def truncate(self, size):
- """Shrink the size of the file.
+ """Shrink or expand the size of the file.
If `size` is less than the size of the file, the file contents after
`size` will be discarded. If `size` is greater than the current size
- of the file, an IOError will be raised.
+ of the file, it will be filled with zero bytes.
"""
if size < self.size():
self._segments = new_segs
self.set_committed(False)
elif size > self.size():
- raise IOError(errno.EINVAL, "truncate() does not support extending the file size")
+ padding = self.parent._my_block_manager().get_padding_block()
+ diff = size - self.size()
+ while diff > config.KEEP_BLOCK_SIZE:
+ self._segments.append(Range(padding.blockid, self.size(), config.KEEP_BLOCK_SIZE, 0))
+ diff -= config.KEEP_BLOCK_SIZE
+ if diff > 0:
+ self._segments.append(Range(padding.blockid, self.size(), diff, 0))
+ self.set_committed(False)
+ else:
+ # size == self.size()
+ pass
def readfrom(self, offset, size, num_retries, exact=False):
"""Read up to `size` bytes from the file starting at `offset`.
return ''.join(data)
- def _repack_writes(self, num_retries):
- """Test if the buffer block has more data than actual segments.
-
- This happens when a buffered write over-writes a file range written in
- a previous buffered write. Re-pack the buffer block for efficiency
- and to avoid leaking information.
-
- """
- segs = self._segments
-
- # Sum up the segments to get the total bytes of the file referencing
- # into the buffer block.
- bufferblock_segs = [s for s in segs if s.locator == self._current_bblock.blockid]
- write_total = sum([s.range_size for s in bufferblock_segs])
-
- if write_total < self._current_bblock.size():
- # There is more data in the buffer block than is actually accounted for by segments, so
- # re-pack into a new buffer by copying over to a new buffer block.
- contents = self.parent._my_block_manager().get_block_contents(self._current_bblock.blockid, num_retries)
- new_bb = self.parent._my_block_manager().alloc_bufferblock(self._current_bblock.blockid, starting_capacity=write_total, owner=self)
- for t in bufferblock_segs:
- new_bb.append(contents[t.segment_offset:t.segment_offset+t.range_size])
- t.segment_offset = new_bb.size() - t.range_size
-
- self._current_bblock = new_bb
-
@must_be_writable
@synchronized
def writeto(self, offset, data, num_retries):
return
if offset > self.size():
- raise ArgumentError("Offset is past the end of the file")
+ self.truncate(offset)
if len(data) > config.KEEP_BLOCK_SIZE:
# Chunk it up into smaller writes
self._current_bblock = self.parent._my_block_manager().alloc_bufferblock(owner=self)
if (self._current_bblock.size() + len(data)) > config.KEEP_BLOCK_SIZE:
- self._repack_writes(num_retries)
+ self._current_bblock.repack_writes()
if (self._current_bblock.size() + len(data)) > config.KEEP_BLOCK_SIZE:
self.parent._my_block_manager().commit_bufferblock(self._current_bblock, sync=False)
self._current_bblock = self.parent._my_block_manager().alloc_bufferblock(owner=self)
if self._current_bblock and self._current_bblock.state() != _BufferBlock.COMMITTED:
if self._current_bblock.state() == _BufferBlock.WRITABLE:
- self._repack_writes(num_retries)
- self.parent._my_block_manager().commit_bufferblock(self._current_bblock, sync=sync)
+ self._current_bblock.repack_writes()
+ if self._current_bblock.state() != _BufferBlock.DELETED:
+ self.parent._my_block_manager().commit_bufferblock(self._current_bblock, sync=sync)
if sync:
to_delete = set()
normalize=False, only_committed=False):
buf = ""
filestream = []
- for segment in self.segments:
+ for segment in self._segments:
loc = segment.locator
if self.parent._my_block_manager().is_bufferblock(loc):
if only_committed:
continue
- loc = self._bufferblocks[loc].calculate_locator()
+ loc = self.parent._my_block_manager().get_bufferblock(loc).locator()
if portable_locators:
loc = KeepLocator(loc).stripped()
- filestream.append(LocatorAndRange(loc, locator_block_size(loc),
+ filestream.append(LocatorAndRange(loc, KeepLocator(loc).size,
segment.segment_offset, segment.range_size))
- buf += ' '.join(normalize_stream(stream_name, {stream_name: filestream}))
+ buf += ' '.join(normalize_stream(stream_name, {self.name: filestream}))
buf += "\n"
return buf
self.mode = mode
self.arvadosfile.add_writer(self)
+ def writable(self):
+ return True
+
@_FileLikeObjectBase._before_close
@retry_method
def write(self, data, num_retries=None):
if size is None:
size = self._filepos
self.arvadosfile.truncate(size)
- if self._filepos > self.size():
- self._filepos = self.size()
@_FileLikeObjectBase._before_close
def flush(self):
--- /dev/null
+import errno
+import md5
+import os
+import tempfile
+import time
+
+class SafeHTTPCache(object):
+ """Thread-safe replacement for httplib2.FileCache"""
+
+ def __init__(self, path=None, max_age=None):
+ self._dir = path
+ if max_age is not None:
+ try:
+ self._clean(threshold=time.time() - max_age)
+ except:
+ pass
+
+ def _clean(self, threshold=0):
+ for ent in os.listdir(self._dir):
+ fnm = os.path.join(self._dir, ent)
+ if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
+ continue
+ stat = os.lstat(fnm)
+ if stat.st_mtime < threshold:
+ try:
+ os.unlink(fnm)
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
+
+ def __str__(self):
+ return self._dir
+
+ def _filename(self, url):
+ return os.path.join(self._dir, md5.new(url).hexdigest()+'.tmp')
+
+ def get(self, url):
+ filename = self._filename(url)
+ try:
+ with open(filename, 'rb') as f:
+ return f.read()
+ except IOError, OSError:
+ return None
+
+ def set(self, url, content):
+ try:
+ fd, tempname = tempfile.mkstemp(dir=self._dir)
+ except:
+ return None
+ try:
+ try:
+ f = os.fdopen(fd, 'w')
+ except:
+ os.close(fd)
+ raise
+ try:
+ f.write(content)
+ finally:
+ f.close()
+ os.rename(tempname, self._filename(url))
+ tempname = None
+ finally:
+ if tempname:
+ os.unlink(tempname)
+
+ def delete(self, url):
+ try:
+ os.unlink(self._filename(url))
+ except OSError as err:
+ if err.errno != errno.ENOENT:
+ raise
--- /dev/null
+#!/usr/bin/env python
+
+import argparse
+import hashlib
+import os
+import re
+import string
+import sys
+import logging
+
+import arvados
+import arvados.commands._util as arv_cmd
+import arvados.util as util
+
+from arvados._version import __version__
+
+api_client = None
+logger = logging.getLogger('arvados.arv-get')
+
+parser = argparse.ArgumentParser(
+ description='Copy data from Keep to a local file or pipe.',
+ parents=[arv_cmd.retry_opt])
+parser.add_argument('--version', action='version',
+ version="%s %s" % (sys.argv[0], __version__),
+ help='Print version and exit.')
+parser.add_argument('locator', type=str,
+ help="""
+Collection locator, optionally with a file path or prefix.
+""")
+parser.add_argument('destination', type=str, nargs='?', default='-',
+ help="""
+Local file or directory where the data is to be written. Default: stdout.
+""")
+group = parser.add_mutually_exclusive_group()
+group.add_argument('--progress', action='store_true',
+ help="""
+Display human-readable progress on stderr (bytes and, if possible,
+percentage of total data size). This is the default behavior when it
+is not expected to interfere with the output: specifically, stderr is
+a tty _and_ either stdout is not a tty, or output is being written to
+named files rather than stdout.
+""")
+group.add_argument('--no-progress', action='store_true',
+ help="""
+Do not display human-readable progress on stderr.
+""")
+group.add_argument('--batch-progress', action='store_true',
+ help="""
+Display machine-readable progress on stderr (bytes and, if known,
+total data size).
+""")
+group = parser.add_mutually_exclusive_group()
+group.add_argument('--hash',
+ help="""
+Display the hash of each file as it is read from Keep, using the given
+hash algorithm. Supported algorithms include md5, sha1, sha224,
+sha256, sha384, and sha512.
+""")
+group.add_argument('--md5sum', action='store_const',
+ dest='hash', const='md5',
+ help="""
+Display the MD5 hash of each file as it is read from Keep.
+""")
+parser.add_argument('-n', action='store_true',
+ help="""
+Do not write any data -- just read from Keep, and report md5sums if
+requested.
+""")
+parser.add_argument('-r', action='store_true',
+ help="""
+Retrieve all files in the specified collection/prefix. This is the
+default behavior if the "locator" argument ends with a forward slash.
+""")
+group = parser.add_mutually_exclusive_group()
+group.add_argument('-f', action='store_true',
+ help="""
+Overwrite existing files while writing. The default behavior is to
+refuse to write *anything* if any of the output files already
+exist. As a special case, -f is not needed to write to stdout.
+""")
+group.add_argument('--skip-existing', action='store_true',
+ help="""
+Skip files that already exist. The default behavior is to refuse to
+write *anything* if any files exist that would have to be
+overwritten. This option causes even devices, sockets, and fifos to be
+skipped.
+""")
+group.add_argument('--strip-manifest', action='store_true', default=False,
+ help="""
+When getting a collection manifest, strip its access tokens before writing
+it.
+""")
+
+def parse_arguments(arguments, stdout, stderr):
+ args = parser.parse_args(arguments)
+
+ if args.locator[-1] == os.sep:
+ args.r = True
+ if (args.r and
+ not args.n and
+ not (args.destination and
+ os.path.isdir(args.destination))):
+ parser.error('Destination is not a directory.')
+ if not args.r and (os.path.isdir(args.destination) or
+ args.destination[-1] == os.path.sep):
+ args.destination = os.path.join(args.destination,
+ os.path.basename(args.locator))
+ logger.debug("Appended source file name to destination directory: %s",
+ args.destination)
+
+ if args.destination == '/dev/stdout':
+ args.destination = "-"
+
+ if args.destination == '-':
+ # Normally you have to use -f to write to a file (or device) that
+ # already exists, but "-" and "/dev/stdout" are common enough to
+ # merit a special exception.
+ args.f = True
+ else:
+ args.destination = args.destination.rstrip(os.sep)
+
+ # Turn on --progress by default if stderr is a tty and output is
+ # either going to a named file, or going (via stdout) to something
+ # that isn't a tty.
+ if (not (args.batch_progress or args.no_progress)
+ and stderr.isatty()
+ and (args.destination != '-'
+ or not stdout.isatty())):
+ args.progress = True
+ return args
+
+def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
+ global api_client
+
+ args = parse_arguments(arguments, stdout, stderr)
+ if api_client is None:
+ api_client = arvados.api('v1')
+
+ r = re.search(r'^(.*?)(/.*)?$', args.locator)
+ col_loc = r.group(1)
+ get_prefix = r.group(2)
+ if args.r and not get_prefix:
+ get_prefix = os.sep
+ try:
+ reader = arvados.CollectionReader(col_loc, num_retries=args.retries)
+ except Exception as error:
+ logger.error("failed to read collection: {}".format(error))
+ return 1
+
+ # User asked to download the collection's manifest
+ if not get_prefix:
+ if not args.n:
+ open_flags = os.O_CREAT | os.O_WRONLY
+ if not args.f:
+ open_flags |= os.O_EXCL
+ try:
+ if args.destination == "-":
+ stdout.write(reader.manifest_text(strip=args.strip_manifest))
+ else:
+ out_fd = os.open(args.destination, open_flags)
+ with os.fdopen(out_fd, 'wb') as out_file:
+ out_file.write(reader.manifest_text(strip=args.strip_manifest))
+ except (IOError, OSError) as error:
+ logger.error("can't write to '{}': {}".format(args.destination, error))
+ return 1
+ except (arvados.errors.ApiError, arvados.errors.KeepReadError) as error:
+ logger.error("failed to download '{}': {}".format(col_loc, error))
+ return 1
+ return 0
+
+ # Scan the collection. Make an array of (stream, file, local
+ # destination filename) tuples, and add up total size to extract.
+ todo = []
+ todo_bytes = 0
+ try:
+ if get_prefix == os.sep:
+ item = reader
+ else:
+ item = reader.find('.' + get_prefix)
+
+ if isinstance(item, arvados.collection.Subcollection) or isinstance(item, arvados.collection.CollectionReader):
+ # If the user asked for a file and we got a subcollection, error out.
+ if get_prefix[-1] != os.sep:
+ logger.error("requested file '{}' is in fact a subcollection. Append a trailing '/' to download it.".format('.' + get_prefix))
+ return 1
+ # If the user asked stdout as a destination, error out.
+ elif args.destination == '-':
+ logger.error("cannot use 'stdout' as destination when downloading multiple files.")
+ return 1
+ # User asked for a subcollection, and that's what was found. Add up total size
+ # to download.
+ for s, f in files_in_collection(item):
+ dest_path = os.path.join(
+ args.destination,
+ os.path.join(s.stream_name(), f.name)[len(get_prefix)+1:])
+ if (not (args.n or args.f or args.skip_existing) and
+ os.path.exists(dest_path)):
+ logger.error('Local file %s already exists.' % (dest_path,))
+ return 1
+ todo += [(s, f, dest_path)]
+ todo_bytes += f.size()
+ elif isinstance(item, arvados.arvfile.ArvadosFile):
+ todo += [(item.parent, item, args.destination)]
+ todo_bytes += item.size()
+ else:
+ logger.error("'{}' not found.".format('.' + get_prefix))
+ return 1
+ except (IOError, arvados.errors.NotFoundError) as e:
+ logger.error(e)
+ return 1
+
+ out_bytes = 0
+ for s, f, outfilename in todo:
+ outfile = None
+ digestor = None
+ if not args.n:
+ if outfilename == "-":
+ outfile = stdout
+ else:
+ if args.skip_existing and os.path.exists(outfilename):
+ logger.debug('Local file %s exists. Skipping.', outfilename)
+ continue
+ elif not args.f and (os.path.isfile(outfilename) or
+ os.path.isdir(outfilename)):
+ # Good thing we looked again: apparently this file wasn't
+ # here yet when we checked earlier.
+ logger.error('Local file %s already exists.' % (outfilename,))
+ return 1
+ if args.r:
+ arvados.util.mkdir_dash_p(os.path.dirname(outfilename))
+ try:
+ outfile = open(outfilename, 'wb')
+ except Exception as error:
+ logger.error('Open(%s) failed: %s' % (outfilename, error))
+ return 1
+ if args.hash:
+ digestor = hashlib.new(args.hash)
+ try:
+ with s.open(f.name, 'r') as file_reader:
+ for data in file_reader.readall():
+ if outfile:
+ outfile.write(data)
+ if digestor:
+ digestor.update(data)
+ out_bytes += len(data)
+ if args.progress:
+ stderr.write('\r%d MiB / %d MiB %.1f%%' %
+ (out_bytes >> 20,
+ todo_bytes >> 20,
+ (100
+ if todo_bytes==0
+ else 100.0*out_bytes/todo_bytes)))
+ elif args.batch_progress:
+ stderr.write('%s %d read %d total\n' %
+ (sys.argv[0], os.getpid(),
+ out_bytes, todo_bytes))
+ if digestor:
+ stderr.write("%s %s/%s\n"
+ % (digestor.hexdigest(), s.stream_name(), f.name))
+ except KeyboardInterrupt:
+ if outfile and (outfile.fileno() > 2) and not outfile.closed:
+ os.unlink(outfile.name)
+ break
+ finally:
+ if outfile != None and outfile != stdout:
+ outfile.close()
+
+ if args.progress:
+ stderr.write('\n')
+ return 0
+
+def files_in_collection(c):
+ # Sort first by file type, then alphabetically by file path.
+ for i in sorted(c.keys(),
+ key=lambda k: (
+ isinstance(c[k], arvados.collection.Subcollection),
+ k.upper())):
+ if isinstance(c[i], arvados.arvfile.ArvadosFile):
+ yield (c, c[i])
+ elif isinstance(c[i], arvados.collection.Subcollection):
+ for s, f in files_in_collection(c[i]):
+ yield (s, f)
from __future__ import print_function
import argparse
+import collections
+import logging
+import re
import sys
import arvados
from arvados._version import __version__
+FileInfo = collections.namedtuple('FileInfo', ['stream_name', 'name', 'size'])
+
def parse_args(args):
parser = argparse.ArgumentParser(
description='List contents of a manifest',
parents=[arv_cmd.retry_opt])
parser.add_argument('locator', type=str,
- help="""Collection UUID or locator""")
+ help="""Collection UUID or locator, optionally with a subdir path.""")
parser.add_argument('-s', action='store_true',
help="""List file sizes, in KiB.""")
parser.add_argument('--version', action='version',
return parser.parse_args(args)
def size_formatter(coll_file):
- return "{:>10}".format((coll_file.size() + 1023) / 1024)
+ return "{:>10}".format((coll_file.size + 1023) / 1024)
def name_formatter(coll_file):
- return "{}/{}".format(coll_file.stream_name(), coll_file.name)
+ return "{}/{}".format(coll_file.stream_name, coll_file.name)
-def main(args, stdout, stderr, api_client=None):
+def main(args, stdout, stderr, api_client=None, logger=None):
args = parse_args(args)
if api_client is None:
api_client = arvados.api('v1')
+ if logger is None:
+ logger = logging.getLogger('arvados.arv-ls')
+
try:
- cr = arvados.CollectionReader(args.locator, api_client=api_client,
+ r = re.search(r'^(.*?)(/.*)?$', args.locator)
+ collection = r.group(1)
+ get_prefix = r.group(2)
+
+ cr = arvados.CollectionReader(collection, api_client=api_client,
num_retries=args.retries)
- cr.normalize()
- except (arvados.errors.ArgumentError,
+ if get_prefix:
+ if get_prefix[-1] == '/':
+ get_prefix = get_prefix[:-1]
+ stream_name = '.' + get_prefix
+ reader = cr.find(stream_name)
+ if not (isinstance(reader, arvados.CollectionReader) or
+ isinstance(reader, arvados.collection.Subcollection)):
+ logger.error("'{}' is not a subdirectory".format(get_prefix))
+ return 1
+ else:
+ stream_name = '.'
+ reader = cr
+ except (arvados.errors.ApiError,
+ arvados.errors.ArgumentError,
arvados.errors.NotFoundError) as error:
- print("arv-ls: error fetching collection: {}".format(error),
- file=stderr)
+ logger.error("error fetching collection: {}".format(error))
return 1
formatters = []
formatters.append(size_formatter)
formatters.append(name_formatter)
- for f in cr.all_files():
+ for f in files_in_collection(reader, stream_name):
print(*(info_func(f) for info_func in formatters), file=stdout)
return 0
+
+def files_in_collection(c, stream_name='.'):
+ # Sort first by file type, then alphabetically by file path.
+ for i in sorted(c.keys(),
+ key=lambda k: (
+ isinstance(c[k], arvados.collection.Subcollection),
+ k.upper())):
+ if isinstance(c[i], arvados.arvfile.ArvadosFile):
+ yield FileInfo(stream_name=stream_name,
+ name=i,
+ size=c[i].size())
+ elif isinstance(c[i], arvados.collection.Subcollection):
+ for f in files_in_collection(c[i], "{}/{}".format(stream_name, i)):
+ yield f
migrate19_parser.add_argument(
'--version', action='version', version="%s %s" % (sys.argv[0], __version__),
help='Print version and exit.')
+ migrate19_parser.add_argument(
+ '--verbose', action="store_true", help="Print stdout/stderr even on success")
+ migrate19_parser.add_argument(
+ '--force', action="store_true", help="Try to migrate even if there isn't enough space")
+
+ migrate19_parser.add_argument(
+ '--storage-driver', type=str, default="overlay",
+ help="Docker storage driver, e.g. aufs, overlay, vfs")
exgroup = migrate19_parser.add_mutually_exclusive_group()
exgroup.add_argument(
'--print-unmigrated', action='store_true',
default=False, help="Print list of images needing migration.")
+ migrate19_parser.add_argument('--tempdir', help="Set temporary directory")
+
migrate19_parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
default=None, help="List of images to be migrated")
args = migrate19_parser.parse_args(arguments)
+ if args.tempdir:
+ tempfile.tempdir = args.tempdir
+
+ if args.verbose:
+ logger.setLevel(logging.DEBUG)
+
only_migrate = None
if args.infile:
only_migrate = set()
items = arvados.util.list_all(api_client.collections().list,
filters=[["uuid", "in", [img["collection"] for img in old_images]]],
- select=["uuid", "portable_data_hash"])
- uuid_to_pdh = {i["uuid"]: i["portable_data_hash"] for i in items}
+ select=["uuid", "portable_data_hash", "manifest_text", "owner_uuid"])
+ uuid_to_collection = {i["uuid"]: i for i in items}
need_migrate = {}
+ totalbytes = 0
+ biggest = 0
+ biggest_pdh = None
for img in old_images:
- pdh = uuid_to_pdh[img["collection"]]
- if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
+ i = uuid_to_collection[img["collection"]]
+ pdh = i["portable_data_hash"]
+ if pdh not in already_migrated and pdh not in need_migrate and (only_migrate is None or pdh in only_migrate):
need_migrate[pdh] = img
+ with CollectionReader(i["manifest_text"]) as c:
+ if c.values()[0].size() > biggest:
+ biggest = c.values()[0].size()
+ biggest_pdh = pdh
+ totalbytes += c.values()[0].size()
+
+
+ if args.storage_driver == "vfs":
+ will_need = (biggest*20)
+ else:
+ will_need = (biggest*2.5)
if args.print_unmigrated:
only_migrate = set()
for pdh in need_migrate:
- print pdh
+ print(pdh)
return
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
+ logger.info("Using tempdir %s", tempfile.gettempdir())
+ logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
+ logger.info("Total data to migrate about %i MiB", totalbytes/(2**20))
+
+ df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
+ ln = df_out.splitlines()[1]
+ filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
+ if int(available) <= will_need:
+ logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
+ if not args.force:
+ exit(1)
+ else:
+ logger.warn("--force provided, will migrate anyway")
if args.dry_run:
return
failures = []
count = 1
for old_image in need_migrate.values():
- if uuid_to_pdh[old_image["collection"]] in already_migrated:
+ if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated:
continue
- logger.info("[%i/%i] Migrating %s:%s (%s)", count, len(need_migrate), old_image["repo"], old_image["tag"], old_image["collection"])
+ oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"])
+ tarfile = oldcol.keys()[0]
+
+ logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
+ old_image["tag"], old_image["collection"], oldcol.values()[0].size()/(2**20))
count += 1
start = time.time()
- oldcol = CollectionReader(old_image["collection"])
- tarfile = oldcol.keys()[0]
-
varlibdocker = tempfile.mkdtemp()
+ dockercache = tempfile.mkdtemp()
try:
with tempfile.NamedTemporaryFile() as envfile:
- envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
- envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
- if "ARVADOS_API_HOST_INSECURE" in os.environ:
- envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+ envfile.write("ARVADOS_API_HOST=%s\n" % (arvados.config.get("ARVADOS_API_HOST")))
+ envfile.write("ARVADOS_API_TOKEN=%s\n" % (arvados.config.get("ARVADOS_API_TOKEN")))
+ if arvados.config.get("ARVADOS_API_HOST_INSECURE"):
+ envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (arvados.config.get("ARVADOS_API_HOST_INSECURE")))
envfile.flush()
dockercmd = ["docker", "run",
"--rm",
"--env-file", envfile.name,
"--volume", "%s:/var/lib/docker" % varlibdocker,
- "arvados/migrate-docker19",
+ "--volume", "%s:/root/.cache/arvados/docker" % dockercache,
+ "arvados/migrate-docker19:1.0",
"/root/migrate.sh",
"%s/%s" % (old_image["collection"], tarfile),
tarfile[0:40],
old_image["repo"],
old_image["tag"],
- oldcol.api_response()["owner_uuid"]]
+ uuid_to_collection[old_image["collection"]]["owner_uuid"],
+ args.storage_driver]
proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
+ initial_space = re.search(r"Initial available space is (\d+)", out)
+ imgload_space = re.search(r"Available space after image load is (\d+)", out)
+ imgupgrade_space = re.search(r"Available space after image upgrade is (\d+)", out)
+ keepdocker_space = re.search(r"Available space after arv-keepdocker is (\d+)", out)
+ cleanup_space = re.search(r"Available space after cleanup is (\d+)", out)
+
+ if initial_space:
+ isp = int(initial_space.group(1))
+ logger.info("Available space initially: %i MiB", (isp)/(2**20))
+ if imgload_space:
+ sp = int(imgload_space.group(1))
+ logger.debug("Used after load: %i MiB", (isp-sp)/(2**20))
+ if imgupgrade_space:
+ sp = int(imgupgrade_space.group(1))
+ logger.debug("Used after upgrade: %i MiB", (isp-sp)/(2**20))
+ if keepdocker_space:
+ sp = int(keepdocker_space.group(1))
+ logger.info("Used after upload: %i MiB", (isp-sp)/(2**20))
+
+ if cleanup_space:
+ sp = int(cleanup_space.group(1))
+ logger.debug("Available after cleanup: %i MiB", (sp)/(2**20))
+
if proc.returncode != 0:
logger.error("Failed with return code %i", proc.returncode)
logger.error("--- Stdout ---\n%s", out)
logger.error("--- Stderr ---\n%s", err)
raise MigrationFailed()
+ if args.verbose:
+ logger.info("--- Stdout ---\n%s", out)
+ logger.info("--- Stderr ---\n%s", err)
+
migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
if migrated:
newcol = CollectionReader(migrated.group(1))
failures.append(old_image["collection"])
finally:
shutil.rmtree(varlibdocker)
+ shutil.rmtree(dockercache)
logger.info("Successfully migrated %i images", len(success))
if failures:
with self._state_lock:
self._state['manifest'] = manifest
if self.use_cache:
- self._save_state()
+ try:
+ self._save_state()
+ except Exception as e:
+ self.logger.error("Unexpected error trying to save cache file: {}".format(e))
else:
self.bytes_written = self.bytes_skipped
# Call the reporter, if any
cache_filepath = os.path.join(
arv_cmd.make_home_conf_dir(self.CACHE_DIR, 0o700, 'raise'),
cache_filename)
- if self.resume:
+ if self.resume and os.path.exists(cache_filepath):
+ self.logger.info("Resuming upload from cache file {}".format(cache_filepath))
self._cache_file = open(cache_filepath, 'a+')
else:
# --no-resume means start with a empty cache file.
+ self.logger.info("Creating new cache file at {}".format(cache_filepath))
self._cache_file = open(cache_filepath, 'w+')
self._cache_filename = self._cache_file.name
self._lock_file(self._cache_file)
# Cache file empty, set up new cache
self._state = copy.deepcopy(self.EMPTY_STATE)
else:
+ self.logger.info("No cache usage requested for this run.")
# No cache file, set empty state
self._state = copy.deepcopy(self.EMPTY_STATE)
# Load the previous manifest so we can check if files were modified remotely.
"""
Atomically save current state into cache.
"""
+ with self._state_lock:
+ # We're not using copy.deepcopy() here because it's a lot slower
+ # than json.dumps(), and we're already needing JSON format to be
+ # saved on disk.
+ state = json.dumps(self._state)
try:
- with self._state_lock:
- # We're not using copy.deepcopy() here because it's a lot slower
- # than json.dumps(), and we're already needing JSON format to be
- # saved on disk.
- state = json.dumps(self._state)
- new_cache_fd, new_cache_name = tempfile.mkstemp(
- dir=os.path.dirname(self._cache_filename))
- self._lock_file(new_cache_fd)
- new_cache = os.fdopen(new_cache_fd, 'r+')
+ new_cache = tempfile.NamedTemporaryFile(
+ dir=os.path.dirname(self._cache_filename), delete=False)
+ self._lock_file(new_cache)
new_cache.write(state)
new_cache.flush()
os.fsync(new_cache)
- os.rename(new_cache_name, self._cache_filename)
+ os.rename(new_cache.name, self._cache_filename)
except (IOError, OSError, ResumeCacheConflict) as error:
self.logger.error("There was a problem while saving the cache file: {}".format(error))
try:
import subprocess
import logging
import sys
+import errno
import arvados.commands._util as arv_cmd
+import arvados.collection
from arvados._version import __version__
# original parameter string).
def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
absfn = os.path.abspath(fn)
- if os.path.exists(absfn):
+ try:
st = os.stat(absfn)
- if stat.S_ISREG(st.st_mode):
- sp = os.path.split(absfn)
- (pdh, branch) = is_in_collection(sp[0], sp[1])
- if pdh:
+ sp = os.path.split(absfn)
+ (pdh, branch) = is_in_collection(sp[0], sp[1])
+ if pdh:
+ if stat.S_ISREG(st.st_mode):
return ArvFile(prefix, fnPattern % (pdh, branch))
- else:
- # trim leading '/' for path prefix test later
- return UploadFile(prefix, absfn[1:])
- if stat.S_ISDIR(st.st_mode):
- sp = os.path.split(absfn)
- (pdh, branch) = is_in_collection(sp[0], sp[1])
- if pdh:
+ elif stat.S_ISDIR(st.st_mode):
return ArvFile(prefix, dirPattern % (pdh, branch))
+ else:
+ raise Exception("%s is not a regular file or directory" % absfn)
+ else:
+ # trim leading '/' for path prefix test later
+ return UploadFile(prefix, absfn[1:])
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ pass
+ else:
+ raise
return prefix+fn
-def uploadfiles(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
+def write_file(collection, pathprefix, fn):
+ with open(os.path.join(pathprefix, fn)) as src:
+ dst = collection.open(fn, "w")
+ r = src.read(1024*128)
+ while r:
+ dst.write(r)
+ r = src.read(1024*128)
+ dst.close(flush=False)
+
+def uploadfiles(files, api, dry_run=False, num_retries=0,
+ project=None,
+ fnPattern="$(file %s/%s)",
+ name=None):
# Find the smallest path prefix that includes all the files that need to be uploaded.
# This starts at the root and iteratively removes common parent directory prefixes
# until all file paths no longer have a common parent.
for c in files:
c.fn = c.fn[len(pathstep):]
- orgdir = os.getcwd()
- os.chdir(pathprefix)
-
logger.info("Upload local files: \"%s\"", '" "'.join([c.fn for c in files]))
if dry_run:
pdh = "$(input)"
else:
files = sorted(files, key=lambda x: x.fn)
- collection = arvados.CollectionWriter(api, num_retries=num_retries)
- stream = None
+ collection = arvados.collection.Collection(api_client=api, num_retries=num_retries)
+ prev = ""
for f in files:
- sp = os.path.split(f.fn)
- if sp[0] != stream:
- stream = sp[0]
- collection.start_new_stream(stream)
- collection.write_file(f.fn, sp[1])
+ localpath = os.path.join(pathprefix, f.fn)
+ if prev and localpath.startswith(prev+"/"):
+ # If this path is inside an already uploaded subdirectory,
+ # don't redundantly re-upload it.
+ # e.g. we uploaded /tmp/foo and the next file is /tmp/foo/bar
+ # skip it because it starts with "/tmp/foo/"
+ continue
+ prev = localpath
+ if os.path.isfile(localpath):
+ write_file(collection, pathprefix, f.fn)
+ elif os.path.isdir(localpath):
+ for root, dirs, iterfiles in os.walk(localpath):
+ root = root[len(pathprefix):]
+ for src in iterfiles:
+ write_file(collection, pathprefix, os.path.join(root, src))
filters=[["portable_data_hash", "=", collection.portable_data_hash()],
["name", "like", name+"%"]]
if project:
filters.append(["owner_uuid", "=", project])
- exists = api.collections().list(filters=filters).execute(num_retries=num_retries)
+ exists = api.collections().list(filters=filters, limit=1).execute(num_retries=num_retries)
if exists["items"]:
item = exists["items"][0]
- logger.info("Using collection %s", item["uuid"])
+ pdh = item["portable_data_hash"]
+ logger.info("Using collection %s (%s)", pdh, item["uuid"])
else:
- body = {"owner_uuid": project, "manifest_text": collection.manifest_text()}
- if name is not None:
- body["name"] = name
- item = api.collections().create(body=body, ensure_unique_name=True).execute()
- logger.info("Uploaded to %s", item["uuid"])
-
- pdh = item["portable_data_hash"]
+ collection.save_new(name=name, owner_uuid=project, ensure_unique_name=True)
+ pdh = collection.portable_data_hash()
+ logger.info("Uploaded to %s (%s)", pdh, collection.manifest_locator())
for c in files:
c.keepref = "%s/%s" % (pdh, c.fn)
c.fn = fnPattern % (pdh, c.fn)
- os.chdir(orgdir)
-
def main(arguments=None):
args = arvrun_parser.parse_args(arguments)
import cStringIO
+import collections
import datetime
import hashlib
import logging
except:
ua.close()
- @staticmethod
- def _socket_open(family, socktype, protocol, address=None):
+ def _socket_open(self, *args, **kwargs):
+ if len(args) + len(kwargs) == 2:
+ return self._socket_open_pycurl_7_21_5(*args, **kwargs)
+ else:
+ return self._socket_open_pycurl_7_19_3(*args, **kwargs)
+
+ def _socket_open_pycurl_7_19_3(self, family, socktype, protocol, address=None):
+ return self._socket_open_pycurl_7_21_5(
+ purpose=None,
+ address=collections.namedtuple(
+ 'Address', ['family', 'socktype', 'protocol', 'addr'],
+ )(family, socktype, protocol, address))
+
+ def _socket_open_pycurl_7_21_5(self, purpose, address):
"""Because pycurl doesn't have CURLOPT_TCP_KEEPALIVE"""
- s = socket.socket(family, socktype, protocol)
+ s = socket.socket(address.family, address.socktype, address.protocol)
s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
# Will throw invalid protocol error on mac. This test prevents that.
if hasattr(socket, 'TCP_KEEPIDLE'):
#!/usr/bin/env python
-import argparse
-import hashlib
-import os
-import re
-import string
import sys
-import logging
-import arvados
-import arvados.commands._util as arv_cmd
+from arvados.commands.get import main
-from arvados._version import __version__
-
-logger = logging.getLogger('arvados.arv-get')
-
-def abort(msg, code=1):
- print >>sys.stderr, "arv-get:", msg
- exit(code)
-
-parser = argparse.ArgumentParser(
- description='Copy data from Keep to a local file or pipe.',
- parents=[arv_cmd.retry_opt])
-parser.add_argument('--version', action='version',
- version="%s %s" % (sys.argv[0], __version__),
- help='Print version and exit.')
-parser.add_argument('locator', type=str,
- help="""
-Collection locator, optionally with a file path or prefix.
-""")
-parser.add_argument('destination', type=str, nargs='?', default='-',
- help="""
-Local file or directory where the data is to be written. Default: stdout.
-""")
-group = parser.add_mutually_exclusive_group()
-group.add_argument('--progress', action='store_true',
- help="""
-Display human-readable progress on stderr (bytes and, if possible,
-percentage of total data size). This is the default behavior when it
-is not expected to interfere with the output: specifically, stderr is
-a tty _and_ either stdout is not a tty, or output is being written to
-named files rather than stdout.
-""")
-group.add_argument('--no-progress', action='store_true',
- help="""
-Do not display human-readable progress on stderr.
-""")
-group.add_argument('--batch-progress', action='store_true',
- help="""
-Display machine-readable progress on stderr (bytes and, if known,
-total data size).
-""")
-group = parser.add_mutually_exclusive_group()
-group.add_argument('--hash',
- help="""
-Display the hash of each file as it is read from Keep, using the given
-hash algorithm. Supported algorithms include md5, sha1, sha224,
-sha256, sha384, and sha512.
-""")
-group.add_argument('--md5sum', action='store_const',
- dest='hash', const='md5',
- help="""
-Display the MD5 hash of each file as it is read from Keep.
-""")
-parser.add_argument('-n', action='store_true',
- help="""
-Do not write any data -- just read from Keep, and report md5sums if
-requested.
-""")
-parser.add_argument('-r', action='store_true',
- help="""
-Retrieve all files in the specified collection/prefix. This is the
-default behavior if the "locator" argument ends with a forward slash.
-""")
-group = parser.add_mutually_exclusive_group()
-group.add_argument('-f', action='store_true',
- help="""
-Overwrite existing files while writing. The default behavior is to
-refuse to write *anything* if any of the output files already
-exist. As a special case, -f is not needed to write to stdout.
-""")
-group.add_argument('--skip-existing', action='store_true',
- help="""
-Skip files that already exist. The default behavior is to refuse to
-write *anything* if any files exist that would have to be
-overwritten. This option causes even devices, sockets, and fifos to be
-skipped.
-""")
-
-args = parser.parse_args()
-
-if args.locator[-1] == os.sep:
- args.r = True
-if (args.r and
- not args.n and
- not (args.destination and
- os.path.isdir(args.destination))):
- parser.error('Destination is not a directory.')
-if not args.r and (os.path.isdir(args.destination) or
- args.destination[-1] == os.path.sep):
- args.destination = os.path.join(args.destination,
- os.path.basename(args.locator))
- logger.debug("Appended source file name to destination directory: %s",
- args.destination)
-
-if args.destination == '/dev/stdout':
- args.destination = "-"
-
-if args.destination == '-':
- # Normally you have to use -f to write to a file (or device) that
- # already exists, but "-" and "/dev/stdout" are common enough to
- # merit a special exception.
- args.f = True
-else:
- args.destination = args.destination.rstrip(os.sep)
-
-# Turn on --progress by default if stderr is a tty and output is
-# either going to a named file, or going (via stdout) to something
-# that isn't a tty.
-if (not (args.batch_progress or args.no_progress)
- and sys.stderr.isatty()
- and (args.destination != '-'
- or not sys.stdout.isatty())):
- args.progress = True
-
-
-r = re.search(r'^(.*?)(/.*)?$', args.locator)
-collection = r.group(1)
-get_prefix = r.group(2)
-if args.r and not get_prefix:
- get_prefix = os.sep
-api_client = arvados.api('v1')
-reader = arvados.CollectionReader(collection, num_retries=args.retries)
-
-if not get_prefix:
- if not args.n:
- open_flags = os.O_CREAT | os.O_WRONLY
- if not args.f:
- open_flags |= os.O_EXCL
- try:
- if args.destination == "-":
- sys.stdout.write(reader.manifest_text())
- else:
- out_fd = os.open(args.destination, open_flags)
- with os.fdopen(out_fd, 'wb') as out_file:
- out_file.write(reader.manifest_text())
- except (IOError, OSError) as error:
- abort("can't write to '{}': {}".format(args.destination, error))
- except (arvados.errors.ApiError, arvados.errors.KeepReadError) as error:
- abort("failed to download '{}': {}".format(collection, error))
- sys.exit(0)
-
-reader.normalize()
-
-# Scan the collection. Make an array of (stream, file, local
-# destination filename) tuples, and add up total size to extract.
-todo = []
-todo_bytes = 0
-try:
- for s in reader.all_streams():
- for f in s.all_files():
- if get_prefix and get_prefix[-1] == os.sep:
- if 0 != string.find(os.path.join(s.name(), f.name()),
- '.' + get_prefix):
- continue
- if args.destination == "-":
- dest_path = "-"
- else:
- dest_path = os.path.join(
- args.destination,
- os.path.join(s.name(), f.name())[len(get_prefix)+1:])
- if (not (args.n or args.f or args.skip_existing) and
- os.path.exists(dest_path)):
- abort('Local file %s already exists.' % (dest_path,))
- else:
- if os.path.join(s.name(), f.name()) != '.' + get_prefix:
- continue
- dest_path = args.destination
- todo += [(s, f, dest_path)]
- todo_bytes += f.size()
-except arvados.errors.NotFoundError as e:
- abort(e)
-
-# Read data, and (if not -n) write to local file(s) or pipe.
-
-out_bytes = 0
-for s,f,outfilename in todo:
- outfile = None
- digestor = None
- if not args.n:
- if outfilename == "-":
- outfile = sys.stdout
- else:
- if args.skip_existing and os.path.exists(outfilename):
- logger.debug('Local file %s exists. Skipping.', outfilename)
- continue
- elif not args.f and (os.path.isfile(outfilename) or
- os.path.isdir(outfilename)):
- # Good thing we looked again: apparently this file wasn't
- # here yet when we checked earlier.
- abort('Local file %s already exists.' % (outfilename,))
- if args.r:
- arvados.util.mkdir_dash_p(os.path.dirname(outfilename))
- try:
- outfile = open(outfilename, 'wb')
- except Exception as error:
- abort('Open(%s) failed: %s' % (outfilename, error))
- if args.hash:
- digestor = hashlib.new(args.hash)
- try:
- for data in f.readall():
- if outfile:
- outfile.write(data)
- if digestor:
- digestor.update(data)
- out_bytes += len(data)
- if args.progress:
- sys.stderr.write('\r%d MiB / %d MiB %.1f%%' %
- (out_bytes >> 20,
- todo_bytes >> 20,
- (100
- if todo_bytes==0
- else 100.0*out_bytes/todo_bytes)))
- elif args.batch_progress:
- sys.stderr.write('%s %d read %d total\n' %
- (sys.argv[0], os.getpid(),
- out_bytes, todo_bytes))
- if digestor:
- sys.stderr.write("%s %s/%s\n"
- % (digestor.hexdigest(), s.name(), f.name()))
- except KeyboardInterrupt:
- if outfile and (outfile.fileno() > 2) and not outfile.closed:
- os.unlink(outfile.name)
- break
-
-if args.progress:
- sys.stderr.write('\n')
+sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))
('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
],
install_requires=[
- 'google-api-python-client==1.4.2',
- 'oauth2client >=1.4.6, <2',
+ 'google-api-python-client==1.6.2, <1.7',
'ciso8601',
- 'httplib2',
- 'pycurl >=7.19.5.1, <7.21.5',
- 'python-gflags<3.0',
+ 'httplib2 >= 0.9.2',
+ 'pycurl >=7.19.5.1',
'setuptools',
- 'ws4py',
- 'ruamel.yaml==0.13.7'
+ 'ws4py<0.4',
+ 'ruamel.yaml>=0.13.7'
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],
ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
if 'GOPATH' in os.environ:
+ # Add all GOPATH bin dirs to PATH -- but insert them after the
+ # ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
+ # command, not the golang.org/x/tools/cmd/bundle command.
gopaths = os.environ['GOPATH'].split(':')
- gobins = [os.path.join(path, 'bin') for path in gopaths]
- os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
+ addbins = [os.path.join(path, 'bin') for path in gopaths]
+ newbins = []
+ for path in os.environ['PATH'].split(':'):
+ newbins.append(path)
+ if os.path.exists(os.path.join(path, 'bundle')):
+ newbins += addbins
+ addbins = []
+ newbins += addbins
+ os.environ['PATH'] = ':'.join(newbins)
TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
if not os.path.exists(TEST_TMPDIR):
# This will clear cached docs that belong to other processes (like
# concurrent test suites) even if they're still running. They should
# be able to tolerate that.
- for fn in glob.glob(os.path.join(arvados.http_cache('discovery'),
- '*,arvados,v1,rest,*')):
+ for fn in glob.glob(os.path.join(
+ str(arvados.http_cache('discovery')),
+ '*,arvados,v1,rest,*')):
os.unlink(fn)
pid_file = _pidfile('api')
--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import io
+import os
+import re
+import shutil
+import tempfile
+
+import arvados
+import arvados.collection as collection
+import arvados.commands.get as arv_get
+import run_test_server
+
+from arvados_testutil import redirected_streams
+
+class ArvadosGetTestCase(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+ KEEP_SERVER = {}
+
+ def setUp(self):
+ super(ArvadosGetTestCase, self).setUp()
+ self.tempdir = tempfile.mkdtemp()
+ self.col_loc, self.col_pdh, self.col_manifest = self.write_test_collection()
+
+ def tearDown(self):
+ super(ArvadosGetTestCase, self).tearDown()
+ shutil.rmtree(self.tempdir)
+
+ def write_test_collection(self,
+ strip_manifest=False,
+ contents = {
+ 'foo.txt' : 'foo',
+ 'bar.txt' : 'bar',
+ 'subdir/baz.txt' : 'baz',
+ }):
+ c = collection.Collection()
+ for path, data in contents.items():
+ with c.open(path, 'w') as f:
+ f.write(data)
+ c.save_new()
+ return (c.manifest_locator(),
+ c.portable_data_hash(),
+ c.manifest_text(strip=strip_manifest))
+
+ def run_get(self, args):
+ self.stdout = io.BytesIO()
+ self.stderr = io.BytesIO()
+ return arv_get.main(args, self.stdout, self.stderr)
+
+ def test_version_argument(self):
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_get(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+
+ def test_get_single_file(self):
+ # Get the file using the collection's locator
+ r = self.run_get(["{}/subdir/baz.txt".format(self.col_loc), '-'])
+ self.assertEqual(0, r)
+ self.assertEqual('baz', self.stdout.getvalue())
+ # Then, try by PDH
+ r = self.run_get(["{}/subdir/baz.txt".format(self.col_pdh), '-'])
+ self.assertEqual(0, r)
+ self.assertEqual('baz', self.stdout.getvalue())
+
+ def test_get_multiple_files(self):
+ # Download the entire collection to the temp directory
+ r = self.run_get(["{}/".format(self.col_loc), self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, "foo.txt"), "r") as f:
+ self.assertEqual("foo", f.read())
+ with open(os.path.join(self.tempdir, "bar.txt"), "r") as f:
+ self.assertEqual("bar", f.read())
+ with open(os.path.join(self.tempdir, "subdir", "baz.txt"), "r") as f:
+ self.assertEqual("baz", f.read())
+
+ def test_get_collection_unstripped_manifest(self):
+ dummy_token = "+Axxxxxxx"
+ # Get the collection manifest by UUID
+ r = self.run_get([self.col_loc, self.tempdir])
+ self.assertEqual(0, r)
+ m_from_collection = re.sub(r"\+A[0-9a-f@]+", dummy_token, self.col_manifest)
+ with open(os.path.join(self.tempdir, self.col_loc), "r") as f:
+ # Replace manifest tokens before comparison to avoid races
+ m_from_file = re.sub(r"\+A[0-9a-f@]+", dummy_token, f.read())
+ self.assertEqual(m_from_collection, m_from_file)
+ # Get the collection manifest by PDH
+ r = self.run_get([self.col_pdh, self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, self.col_pdh), "r") as f:
+ # Replace manifest tokens before comparison to avoid races
+ m_from_file = re.sub(r"\+A[0-9a-f@]+", dummy_token, f.read())
+ self.assertEqual(m_from_collection, m_from_file)
+
+ def test_get_collection_stripped_manifest(self):
+ col_loc, col_pdh, col_manifest = self.write_test_collection(strip_manifest=True)
+ # Get the collection manifest by UUID
+ r = self.run_get(['--strip-manifest', col_loc, self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, col_loc), "r") as f:
+ self.assertEqual(col_manifest, f.read())
+ # Get the collection manifest by PDH
+ r = self.run_get(['--strip-manifest', col_pdh, self.tempdir])
+ self.assertEqual(0, r)
+ with open(os.path.join(self.tempdir, col_pdh), "r") as f:
+ self.assertEqual(col_manifest, f.read())
+
+ def test_invalid_collection(self):
+ # Asking for an invalid collection should generate an error.
+ r = self.run_get(['this-uuid-seems-to-be-fake', self.tempdir])
+ self.assertNotEqual(0, r)
+
+ def test_invalid_file_request(self):
+ # Asking for an inexistant file within a collection should generate an error.
+ r = self.run_get(["{}/im-not-here.txt".format(self.col_loc), self.tempdir])
+ self.assertNotEqual(0, r)
+
+ def test_invalid_destination(self):
+ # Asking to place the collection's files on a non existant directory
+ # should generate an error.
+ r = self.run_get([self.col_loc, "/fake/subdir/"])
+ self.assertNotEqual(0, r)
+
+ def test_preexistent_destination(self):
+ # Asking to place a file with the same path as a local one should
+ # generate an error and avoid overwrites.
+ with open(os.path.join(self.tempdir, "foo.txt"), "w") as f:
+ f.write("another foo")
+ r = self.run_get(["{}/foo.txt".format(self.col_loc), self.tempdir])
+ self.assertNotEqual(0, r)
+ with open(os.path.join(self.tempdir, "foo.txt"), "r") as f:
+ self.assertEqual("another foo", f.read())
+
for supported, img_id, expect_ok in [
(['v1'], old_id, True),
(['v1'], new_id, False),
- (None, old_id, True),
- ([], old_id, True),
- ([], new_id, True),
+ (None, old_id, False),
+ ([], old_id, False),
+ ([], new_id, False),
(['v1', 'v2'], new_id, True),
(['v1'], new_id, False),
(['v2'], new_id, True)]:
api_client.collections().get().execute.return_value = coll_info
return coll_info, api_client
- def run_ls(self, args, api_client):
+ def run_ls(self, args, api_client, logger=None):
self.stdout = io.BytesIO()
self.stderr = io.BytesIO()
- return arv_ls.main(args, self.stdout, self.stderr, api_client)
+ return arv_ls.main(args, self.stdout, self.stderr, api_client, logger)
def test_plain_listing(self):
collection, api_client = self.mock_api_for_manifest(
def test_locator_failure(self):
api_client = mock.MagicMock(name='mock_api_client')
+ error_mock = mock.MagicMock()
+ logger = mock.MagicMock()
+ logger.error = error_mock
api_client.collections().get().execute.side_effect = (
arv_error.NotFoundError)
- self.assertNotEqual(0, self.run_ls([self.FAKE_UUID], api_client))
- self.assertNotEqual('', self.stderr.getvalue())
+ self.assertNotEqual(0, self.run_ls([self.FAKE_UUID], api_client, logger))
+ self.assertEqual(1, error_mock.call_count)
def test_version_argument(self):
err = io.BytesIO()
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
+
+ def test_truncate2(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+ "replication_desired":None},
+ {"uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+ "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"})
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 10)
+ self.assertEqual("0123456789", writer.read(12))
+
+ # extend file size
+ writer.truncate(12)
+
+ self.assertEqual(writer.size(), 12)
+ writer.seek(0, os.SEEK_SET)
+ self.assertEqual(b"0123456789\x00\x00", writer.read(12))
+
+ self.assertIsNone(c.manifest_locator())
+ self.assertTrue(c.modified())
+ c.save_new("test_truncate2")
+ self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+ self.assertFalse(c.modified())
+
+ def test_truncate3(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
+ "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
+ api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "replication_desired":None},
+ {"uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 20)
+ self.assertEqual("0123456789ab", writer.read(12))
+ self.assertEqual(12, writer.tell())
+
+ writer.truncate(8)
+
+ # Make sure reading off the end doesn't break
+ self.assertEqual(12, writer.tell())
+ self.assertEqual("", writer.read(12))
+
+ self.assertEqual(writer.size(), 8)
+ self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
+ self.assertEqual("234567", writer.read(12))
+
+ self.assertIsNone(c.manifest_locator())
+ self.assertTrue(c.modified())
+ c.save_new("test_truncate")
+ self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+ self.assertFalse(c.modified())
+
+
+
def test_write_to_end(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
- writer.seek(5, os.SEEK_SET)
+ self.assertEqual(5, writer.seek(5, os.SEEK_SET))
self.assertEqual("56789", writer.read(8))
writer.seek(10, os.SEEK_SET)
self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
+ def test_sparse_write(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({})
+ api = ArvadosFileWriterTestCase.MockApi({}, {})
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 0)
+
+ text = "0123456789"
+ writer.seek(2)
+ writer.write(text)
+ self.assertEqual(writer.size(), 12)
+ writer.seek(0, os.SEEK_SET)
+ self.assertEqual(writer.read(), b"\x00\x00"+text)
+
+ self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:2:count.txt 67108864:10:count.txt\n")
+
+
+ def test_sparse_write2(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({})
+ api = ArvadosFileWriterTestCase.MockApi({}, {})
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 0)
+
+ text = "0123456789"
+ writer.seek((arvados.config.KEEP_BLOCK_SIZE*2) + 2)
+ writer.write(text)
+ self.assertEqual(writer.size(), (arvados.config.KEEP_BLOCK_SIZE*2) + 12)
+ writer.seek(0, os.SEEK_SET)
+
+ self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:67108864:count.txt 0:67108864:count.txt 0:2:count.txt 67108864:10:count.txt\n")
+
+
+ def test_sparse_write3(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({})
+ api = ArvadosFileWriterTestCase.MockApi({}, {})
+ for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]:
+ with Collection() as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 0)
+
+ for i in r:
+ w = ("%s" % i) * 10
+ writer.seek(i*10)
+ writer.write(w)
+ writer.seek(0)
+ self.assertEqual(writer.read(), "00000000001111111111222222222233333333334444444444")
+
+ def test_sparse_write4(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({})
+ api = ArvadosFileWriterTestCase.MockApi({}, {})
+ for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
+ with Collection() as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 0)
+
+ for i in r:
+ w = ("%s" % i) * 10
+ writer.seek(i*10)
+ writer.write(w)
+ writer.seek(0)
+ self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
+
+
def test_rewrite_on_empty_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
def test_write_large_rewrite(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
- "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
+ "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
"replication_desired":None},
{"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
+ "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
"portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
--- /dev/null
+from __future__ import print_function
+
+import md5
+import mock
+import os
+import random
+import shutil
+import sys
+import tempfile
+import threading
+import unittest
+
+import arvados.cache
+import arvados
+import run_test_server
+
+
+def _random(n):
+ return bytearray(random.getrandbits(8) for _ in xrange(n))
+
+
+class CacheTestThread(threading.Thread):
+ def __init__(self, dir):
+ super(CacheTestThread, self).__init__()
+ self._dir = dir
+
+ def run(self):
+ c = arvados.cache.SafeHTTPCache(self._dir)
+ url = 'http://example.com/foo'
+ self.ok = True
+ for x in range(16):
+ try:
+ data_in = _random(128)
+ data_in = md5.new(data_in).hexdigest() + "\n" + str(data_in)
+ c.set(url, data_in)
+ data_out = c.get(url)
+ digest, content = data_out.split("\n", 1)
+ if digest != md5.new(content).hexdigest():
+ self.ok = False
+ except Exception as err:
+ self.ok = False
+ print("cache failed: {}".format(err), file=sys.stderr)
+
+
+class CacheTest(unittest.TestCase):
+ def setUp(self):
+ self._dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self._dir)
+
+ def test_cache_create_error(self):
+ _, filename = tempfile.mkstemp()
+ home_was = os.environ['HOME']
+ os.environ['HOME'] = filename
+ try:
+ c = arvados.http_cache('test')
+ self.assertEqual(None, c)
+ finally:
+ os.environ['HOME'] = home_was
+ os.unlink(filename)
+
+ def test_cache_crud(self):
+ c = arvados.cache.SafeHTTPCache(self._dir, max_age=0)
+ url = 'https://example.com/foo?bar=baz'
+ data1 = _random(256)
+ data2 = _random(128)
+ self.assertEqual(None, c.get(url))
+ c.delete(url)
+ c.set(url, data1)
+ self.assertEqual(data1, c.get(url))
+ c.delete(url)
+ self.assertEqual(None, c.get(url))
+ c.set(url, data1)
+ c.set(url, data2)
+ self.assertEqual(data2, c.get(url))
+
+ def test_cache_threads(self):
+ threads = []
+ for _ in range(64):
+ t = CacheTestThread(dir=self._dir)
+ t.start()
+ threads.append(t)
+ for t in threads:
+ t.join()
+ self.assertTrue(t.ok)
+
+
+class CacheIntegrationTest(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+
+ def test_cache_used_by_default_client(self):
+ with mock.patch('arvados.cache.SafeHTTPCache.get') as getter:
+ arvados.api('v1')._rootDesc.get('foobar')
+ getter.assert_called()
def test_only_small_blocks_are_packed_together(self):
c = Collection()
- # Write a couple of small files,
+ # Write a couple of small files,
f = c.open("count.txt", "w")
f.write("0123456789")
f.close(flush=False)
c.manifest_text("."),
'. 2d303c138c118af809f39319e5d507e9+34603008 a8430a058b8fbf408e1931b794dbd6fb+13 0:34603008:bigfile.txt 34603008:10:count.txt 34603018:3:foo.txt\n')
+ def test_flush_after_small_block_packing(self):
+ c = Collection()
+ # Write a couple of small files,
+ f = c.open("count.txt", "w")
+ f.write("0123456789")
+ f.close(flush=False)
+ foo = c.open("foo.txt", "w")
+ foo.write("foo")
+ foo.close(flush=False)
+
+ self.assertEqual(
+ c.manifest_text(),
+ '. a8430a058b8fbf408e1931b794dbd6fb+13 0:10:count.txt 10:3:foo.txt\n')
+
+ f = c.open("count.txt", "r+")
+ f.close(flush=True)
+
+ self.assertEqual(
+ c.manifest_text(),
+ '. a8430a058b8fbf408e1931b794dbd6fb+13 0:10:count.txt 10:3:foo.txt\n')
+
+ def test_write_after_small_block_packing2(self):
+ c = Collection()
+ # Write a couple of small files,
+ f = c.open("count.txt", "w")
+ f.write("0123456789")
+ f.close(flush=False)
+ foo = c.open("foo.txt", "w")
+ foo.write("foo")
+ foo.close(flush=False)
+
+ self.assertEqual(
+ c.manifest_text(),
+ '. a8430a058b8fbf408e1931b794dbd6fb+13 0:10:count.txt 10:3:foo.txt\n')
+
+ f = c.open("count.txt", "r+")
+ f.write("abc")
+ f.close(flush=False)
+
+ self.assertEqual(
+ c.manifest_text(),
+ '. 900150983cd24fb0d6963f7d28e17f72+3 a8430a058b8fbf408e1931b794dbd6fb+13 0:3:count.txt 6:7:count.txt 13:3:foo.txt\n')
+
+
+ def test_small_block_packing_with_overwrite(self):
+ c = Collection()
+ c.open("b1", "w").close()
+ c["b1"].writeto(0, "b1", 0)
+
+ c.open("b2", "w").close()
+ c["b2"].writeto(0, "b2", 0)
+
+ c["b1"].writeto(0, "1b", 0)
+
+ self.assertEquals(c.manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1 2:2:b2\n")
+ self.assertEquals(c["b1"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1\n")
+ self.assertEquals(c["b2"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 2:2:b2\n")
+
class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
def test_seek_min_zero(self):
sfile = self.make_count_reader()
- sfile.seek(-2, os.SEEK_SET)
+ self.assertEqual(0, sfile.tell())
+ with self.assertRaises(IOError):
+ sfile.seek(-2, os.SEEK_SET)
self.assertEqual(0, sfile.tell())
def test_seek_max_size(self):
sfile = self.make_count_reader()
sfile.seek(2, os.SEEK_END)
- self.assertEqual(9, sfile.tell())
+ # POSIX permits seeking past end of file.
+ self.assertEqual(11, sfile.tell())
def test_size(self):
self.assertEqual(9, self.make_count_reader().size())
s.add_dependency('google-api-client', '>= 0.7', '< 0.8.9')
# work around undeclared dependency on i18n in some activesupport 3.x.x:
s.add_dependency('i18n', '~> 0')
- s.add_dependency('json', '~> 1.7', '>= 1.7.7')
+ s.add_dependency('json', '>= 1.7.7', '<3')
s.add_runtime_dependency('jwt', '<2', '>= 0.1.5')
s.homepage =
'https://arvados.org'
end
end
end
+
+# Work around Rails3+PostgreSQL9.5 incompatibility (pg_dump used to
+# accept -i as a no-op, but now it's not accepted at all).
+module Kernel
+ alias_method :orig_backtick, :`
+ def `(*args) #`#` sorry, parsers
+ args[0].sub!(/\Apg_dump -i /, 'pg_dump ') rescue nil
+ orig_backtick(*args)
+ end
+end
attr_writer :resource_attrs
- MAX_UNIQUE_NAME_ATTEMPTS = 10
-
begin
rescue_from(Exception,
ArvadosModel::PermissionDeniedError,
def create
@object = model_class.new resource_attrs
- if @object.respond_to? :name and params[:ensure_unique_name]
- # Record the original name. See below.
- name_stem = @object.name
- retries = MAX_UNIQUE_NAME_ATTEMPTS
+ if @object.respond_to?(:name) && params[:ensure_unique_name]
+ @object.save_with_unique_name!
else
- retries = 0
- end
-
- begin
@object.save!
- rescue ActiveRecord::RecordNotUnique => rn
- raise unless retries > 0
- retries -= 1
-
- # Dig into the error to determine if it is specifically calling out a
- # (owner_uuid, name) uniqueness violation. In this specific case, and
- # the client requested a unique name with ensure_unique_name==true,
- # update the name field and try to save again. Loop as necessary to
- # discover a unique name. It is necessary to handle name choosing at
- # this level (as opposed to the client) to ensure that record creation
- # never fails due to a race condition.
- raise unless rn.original_exception.is_a? PG::UniqueViolation
-
- # Unfortunately ActiveRecord doesn't abstract out any of the
- # necessary information to figure out if this the error is actually
- # the specific case where we want to apply the ensure_unique_name
- # behavior, so the following code is specialized to Postgres.
- err = rn.original_exception
- detail = err.result.error_field(PG::Result::PG_DIAG_MESSAGE_DETAIL)
- raise unless /^Key \(owner_uuid, name\)=\([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}, .*?\) already exists\./.match detail
-
- @object.uuid = nil
-
- new_name = "#{name_stem} (#{db_current_time.utc.iso8601(3)})"
- if new_name == @object.name
- # If the database is fast enough to do two attempts in the
- # same millisecond, we need to wait to ensure we try a
- # different timestamp on each attempt.
- sleep 0.002
- new_name = "#{name_stem} (#{db_current_time.utc.iso8601(3)})"
- end
- @object.name = new_name
- retry
end
+
show
end
@objects = model_class.where('last_ping_at >= ?', db_current_time - 1.hours)
end
super
- job_uuids = @objects.map { |n| n[:job_uuid] }.compact
- assoc_jobs = readable_job_uuids(job_uuids)
- @objects.each do |node|
- node.job_readable = assoc_jobs.include?(node[:job_uuid])
+ if @select.nil? or @select.include? 'job_uuid'
+ job_uuids = @objects.map { |n| n[:job_uuid] }.compact
+ assoc_jobs = readable_job_uuids(job_uuids)
+ @objects.each do |node|
+ node.job_readable = assoc_jobs.include?(node[:job_uuid])
+ end
end
end
permission_link_classes: ['permission', 'resources'])
end
+ def save_with_unique_name!
+ uuid_was = uuid
+ name_was = name
+ max_retries = 2
+ transaction do
+ conn = ActiveRecord::Base.connection
+ conn.exec_query 'SAVEPOINT save_with_unique_name'
+ begin
+ save!
+ rescue ActiveRecord::RecordNotUnique => rn
+ raise if max_retries == 0
+ max_retries -= 1
+
+ conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
+
+ # Dig into the error to determine if it is specifically calling out a
+ # (owner_uuid, name) uniqueness violation. In this specific case, and
+ # the client requested a unique name with ensure_unique_name==true,
+ # update the name field and try to save again. Loop as necessary to
+ # discover a unique name. It is necessary to handle name choosing at
+ # this level (as opposed to the client) to ensure that record creation
+ # never fails due to a race condition.
+ err = rn.original_exception
+ raise unless err.is_a?(PG::UniqueViolation)
+
+ # Unfortunately ActiveRecord doesn't abstract out any of the
+ # necessary information to figure out if this the error is actually
+ # the specific case where we want to apply the ensure_unique_name
+ # behavior, so the following code is specialized to Postgres.
+ detail = err.result.error_field(PG::Result::PG_DIAG_MESSAGE_DETAIL)
+ raise unless /^Key \(owner_uuid, name\)=\([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}, .*?\) already exists\./.match detail
+
+ new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
+ if new_name == name
+ # If the database is fast enough to do two attempts in the
+ # same millisecond, we need to wait to ensure we try a
+ # different timestamp on each attempt.
+ sleep 0.002
+ new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
+ end
+
+ self[:name] = new_name
+ self[:uuid] = nil if uuid_was.nil? && !uuid.nil?
+ conn.exec_query 'SAVEPOINT save_with_unique_name'
+ retry
+ ensure
+ conn.exec_query 'RELEASE SAVEPOINT save_with_unique_name'
+ end
+ end
+ end
+
def logged_attributes
attributes.except(*Rails.configuration.unlogged_attributes)
end
raise PermissionDeniedError
end
- # Verify "write" permission on old owner
- # default fail unless one of:
- # owner_uuid did not change
- # previous owner_uuid is nil
- # current user is the old owner
- # current user is this object
- # current user can_write old owner
- unless !owner_uuid_changed? or
- owner_uuid_was.nil? or
- current_user.uuid == self.owner_uuid_was or
- current_user.uuid == self.uuid or
- current_user.can? write: self.owner_uuid_was
- logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{uuid} but does not have permission to write old owner_uuid #{owner_uuid_was}"
- errors.add :owner_uuid, "cannot be changed without write permission on old owner"
- raise PermissionDeniedError
- end
-
- # Verify "write" permission on new owner
- # default fail unless one of:
- # current_user is this object
- # current user can_write new owner, or this object if owner unchanged
- if new_record? or owner_uuid_changed? or is_a?(ApiClientAuthorization)
- write_target = owner_uuid
+ if new_record? || owner_uuid_changed?
+ # Permission on owner_uuid_was is needed to move an existing
+ # object away from its previous owner (which implies permission
+ # to modify this object itself, so we don't need to check that
+ # separately). Permission on the new owner_uuid is also needed.
+ [['old', owner_uuid_was],
+ ['new', owner_uuid]
+ ].each do |which, check_uuid|
+ if check_uuid.nil?
+ # old_owner_uuid is nil? New record, no need to check.
+ elsif !current_user.can?(write: check_uuid)
+ logger.warn "User #{current_user.uuid} tried to set ownership of #{self.class.to_s} #{self.uuid} but does not have permission to write #{which} owner_uuid #{check_uuid}"
+ errors.add :owner_uuid, "cannot be set or changed without write permission on #{which} owner"
+ raise PermissionDeniedError
+ end
+ end
else
- write_target = uuid
- end
- unless current_user == self or current_user.can? write: write_target
- logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{uuid} but does not have permission to write new owner_uuid #{owner_uuid}"
- errors.add :owner_uuid, "cannot be changed without write permission on new owner"
- raise PermissionDeniedError
+ # If the object already existed and we're not changing
+ # owner_uuid, we only need write permission on the object
+ # itself.
+ if !current_user.can?(write: self.uuid)
+ logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} without write permission"
+ errors.add :uuid, "is not writable"
+ raise PermissionDeniedError
+ end
end
true
false
elsif portable_data_hash[0..31] != computed_pdh[0..31]
errors.add(:portable_data_hash,
- "does not match computed hash #{computed_pdh}")
+ "'#{portable_data_hash}' does not match computed hash '#{computed_pdh}'")
false
else
# Ignore the client-provided size part: always store
true
end
- # If trash_at is updated without touching delete_at, automatically
- # update delete_at to a sensible value.
def default_trash_interval
if trash_at_changed? && !delete_at_changed?
+ # If trash_at is updated without touching delete_at,
+ # automatically update delete_at to a sensible value.
if trash_at.nil?
self.delete_at = nil
else
self.delete_at = trash_at + Rails.configuration.default_trash_lifetime.seconds
end
+ elsif !trash_at || !delete_at || trash_at > delete_at
+ # Not trash, or bogus arguments? Just validate in
+ # validate_trash_and_delete_timing.
+ elsif delete_at_changed? && delete_at >= trash_at
+ # Fix delete_at if needed, so it's not earlier than the expiry
+ # time on any permission tokens that might have been given out.
+
+ # In any case there are no signatures expiring after now+TTL.
+ # Also, if the existing trash_at time has already passed, we
+ # know we haven't given out any signatures since then.
+ earliest_delete = [
+ @validation_timestamp,
+ trash_at_was,
+ ].compact.min + Rails.configuration.blob_signature_ttl.seconds
+
+ # The previous value of delete_at is also an upper bound on the
+ # longest-lived permission token. For example, if TTL=14,
+ # trash_at_was=now-7, delete_at_was=now+7, then it is safe to
+ # set trash_at=now+6, delete_at=now+8.
+ earliest_delete = [earliest_delete, delete_at_was].compact.min
+
+ # If delete_at is too soon, use the earliest possible time.
+ if delete_at < earliest_delete
+ self.delete_at = earliest_delete
+ end
end
end
def validate_trash_and_delete_timing
if trash_at.nil? != delete_at.nil?
errors.add :delete_at, "must be set if trash_at is set, and must be nil otherwise"
- end
-
- earliest_delete = ([@validation_timestamp, trash_at_was].compact.min +
- Rails.configuration.blob_signature_ttl.seconds)
- if delete_at && delete_at < earliest_delete
- errors.add :delete_at, "#{delete_at} is too soon: earliest allowed is #{earliest_delete}"
- end
-
- if delete_at && delete_at < trash_at
+ elsif delete_at && delete_at < trash_at
errors.add :delete_at, "must not be earlier than trash_at"
end
-
true
end
end
end
end
+ # Create a new container (or find an existing one) to satisfy the
+ # given container request.
+ def self.resolve(req)
+ c_attrs = {
+ command: req.command,
+ cwd: req.cwd,
+ environment: req.environment,
+ output_path: req.output_path,
+ container_image: resolve_container_image(req.container_image),
+ mounts: resolve_mounts(req.mounts),
+ runtime_constraints: resolve_runtime_constraints(req.runtime_constraints),
+ scheduling_parameters: req.scheduling_parameters,
+ }
+ act_as_system_user do
+ if req.use_existing && (reusable = find_reusable(c_attrs))
+ reusable
+ else
+ Container.create!(c_attrs)
+ end
+ end
+ end
+
+ # Return a runtime_constraints hash that complies with requested but
+ # is suitable for saving in a container record, i.e., has specific
+ # values instead of ranges.
+ #
+ # Doing this as a step separate from other resolutions, like "git
+ # revision range to commit hash", makes sense only when there is no
+ # opportunity to reuse an existing container (e.g., container reuse
+ # is not implemented yet, or we have already found that no existing
+ # containers are suitable).
+ def self.resolve_runtime_constraints(runtime_constraints)
+ rc = {}
+ defaults = {
+ 'keep_cache_ram' =>
+ Rails.configuration.container_default_keep_cache_ram,
+ }
+ defaults.merge(runtime_constraints).each do |k, v|
+ if v.is_a? Array
+ rc[k] = v[0]
+ else
+ rc[k] = v
+ end
+ end
+ rc
+ end
+
+ # Return a mounts hash suitable for a Container, i.e., with every
+ # readonly collection UUID resolved to a PDH.
+ def self.resolve_mounts(mounts)
+ c_mounts = {}
+ mounts.each do |k, mount|
+ mount = mount.dup
+ c_mounts[k] = mount
+ if mount['kind'] != 'collection'
+ next
+ end
+ if (uuid = mount.delete 'uuid')
+ c = Collection.
+ readable_by(current_user).
+ where(uuid: uuid).
+ select(:portable_data_hash).
+ first
+ if !c
+ raise ArvadosModel::UnresolvableContainerError.new "cannot mount collection #{uuid.inspect}: not found"
+ end
+ if mount['portable_data_hash'].nil?
+ # PDH not supplied by client
+ mount['portable_data_hash'] = c.portable_data_hash
+ elsif mount['portable_data_hash'] != c.portable_data_hash
+ # UUID and PDH supplied by client, but they don't agree
+ raise ArgumentError.new "cannot mount collection #{uuid.inspect}: current portable_data_hash #{c.portable_data_hash.inspect} does not match #{c['portable_data_hash'].inspect} in request"
+ end
+ end
+ end
+ return c_mounts
+ end
+
+ # Return a container_image PDH suitable for a Container.
+ def self.resolve_container_image(container_image)
+ coll = Collection.for_latest_docker_image(container_image)
+ if !coll
+ raise ArvadosModel::UnresolvableContainerError.new "docker image #{container_image.inspect} not found"
+ end
+ coll.portable_data_hash
+ end
+
def self.find_reusable(attrs)
candidates = Container.
where_serialized(:command, attrs[:command]).
where('cwd = ?', attrs[:cwd]).
where_serialized(:environment, attrs[:environment]).
where('output_path = ?', attrs[:output_path]).
- where('container_image = ?', attrs[:container_image]).
- where_serialized(:mounts, attrs[:mounts]).
- where_serialized(:runtime_constraints, attrs[:runtime_constraints])
+ where('container_image = ?', resolve_container_image(attrs[:container_image])).
+ where_serialized(:mounts, resolve_mounts(attrs[:mounts])).
+ where_serialized(:runtime_constraints, resolve_runtime_constraints(attrs[:runtime_constraints]))
# Check for Completed candidates whose output and log are both readable.
select_readable_pdh = Collection.
before_validation :validate_scheduling_parameters
before_validation :set_container
validates :command, :container_image, :output_path, :cwd, :presence => true
+ validates :output_ttl, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validate :validate_state_change
- validate :validate_change
+ validate :check_update_whitelist
after_save :update_priority
after_save :finalize_if_needed
before_create :set_requesting_container_uuid
t.add :output_name
t.add :output_path
t.add :output_uuid
+ t.add :output_ttl
t.add :priority
t.add :properties
t.add :requesting_container_uuid
Committed => [Final]
}
+ AttrsPermittedAlways = [:owner_uuid, :state, :name, :description]
+ AttrsPermittedBeforeCommit = [:command, :container_count_max,
+ :container_image, :cwd, :environment, :filters, :mounts,
+ :output_path, :priority, :properties, :requesting_container_uuid,
+ :runtime_constraints, :state, :container_uuid, :use_existing,
+ :scheduling_parameters, :output_name, :output_ttl]
+
def state_transitions
State_transitions
end
['output', 'log'].each do |out_type|
pdh = c.send(out_type)
next if pdh.nil?
- if self.output_name and out_type == 'output'
- coll_name = self.output_name
- else
- coll_name = "Container #{out_type} for request #{uuid}"
+ coll_name = "Container #{out_type} for request #{uuid}"
+ trash_at = nil
+ if out_type == 'output'
+ if self.output_name
+ coll_name = self.output_name
+ end
+ if self.output_ttl > 0
+ trash_at = db_current_time + self.output_ttl
+ end
end
manifest = Collection.unscoped do
Collection.where(portable_data_hash: pdh).first.manifest_text
end
- begin
- coll = Collection.create!(owner_uuid: owner_uuid,
- manifest_text: manifest,
- portable_data_hash: pdh,
- name: coll_name,
- properties: {
- 'type' => out_type,
- 'container_request' => uuid,
- })
- rescue ActiveRecord::RecordNotUnique => rn
- # In case this is executed as part of a transaction: When a Postgres exception happens,
- # the following statements on the same transaction become invalid, so a rollback is
- # needed. One example are Unit Tests, every test is enclosed inside a transaction so
- # that the database can be reverted before every new test starts.
- # See: http://api.rubyonrails.org/classes/ActiveRecord/Transactions/ClassMethods.html#module-ActiveRecord::Transactions::ClassMethods-label-Exception+handling+and+rolling+back
- ActiveRecord::Base.connection.execute 'ROLLBACK'
- raise unless out_type == 'output' and self.output_name
- # Postgres specific unique name check. See ApplicationController#create for
- # a detailed explanation.
- raise unless rn.original_exception.is_a? PG::UniqueViolation
- err = rn.original_exception
- detail = err.result.error_field(PG::Result::PG_DIAG_MESSAGE_DETAIL)
- raise unless /^Key \(owner_uuid, name\)=\([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}, .*?\) already exists\./.match detail
- # Output collection name collision detected: append a timestamp.
- coll_name = "#{self.output_name} #{Time.now.getgm.strftime('%FT%TZ')}"
- retry
- end
+
+ coll = Collection.new(owner_uuid: owner_uuid,
+ manifest_text: manifest,
+ portable_data_hash: pdh,
+ name: coll_name,
+ trash_at: trash_at,
+ delete_at: trash_at,
+ properties: {
+ 'type' => out_type,
+ 'container_request' => uuid,
+ })
+ coll.save_with_unique_name!
if out_type == 'output'
out_coll = coll.uuid
else
self.cwd ||= "."
self.container_count_max ||= Rails.configuration.container_count_max
self.scheduling_parameters ||= {}
- end
-
- # Create a new container (or find an existing one) to satisfy this
- # request.
- def resolve
- c_mounts = mounts_for_container
- c_runtime_constraints = runtime_constraints_for_container
- c_container_image = container_image_for_container
- c = act_as_system_user do
- c_attrs = {command: self.command,
- cwd: self.cwd,
- environment: self.environment,
- output_path: self.output_path,
- container_image: c_container_image,
- mounts: c_mounts,
- runtime_constraints: c_runtime_constraints}
-
- reusable = self.use_existing ? Container.find_reusable(c_attrs) : nil
- if not reusable.nil?
- reusable
- else
- c_attrs[:scheduling_parameters] = self.scheduling_parameters
- Container.create!(c_attrs)
- end
- end
- self.container_uuid = c.uuid
- end
-
- # Return a runtime_constraints hash that complies with
- # self.runtime_constraints but is suitable for saving in a container
- # record, i.e., has specific values instead of ranges.
- #
- # Doing this as a step separate from other resolutions, like "git
- # revision range to commit hash", makes sense only when there is no
- # opportunity to reuse an existing container (e.g., container reuse
- # is not implemented yet, or we have already found that no existing
- # containers are suitable).
- def runtime_constraints_for_container
- rc = {}
- runtime_constraints.each do |k, v|
- if v.is_a? Array
- rc[k] = v[0]
- else
- rc[k] = v
- end
- end
- rc
- end
-
- # Return a mounts hash suitable for a Container, i.e., with every
- # readonly collection UUID resolved to a PDH.
- def mounts_for_container
- c_mounts = {}
- mounts.each do |k, mount|
- mount = mount.dup
- c_mounts[k] = mount
- if mount['kind'] != 'collection'
- next
- end
- if (uuid = mount.delete 'uuid')
- c = Collection.
- readable_by(current_user).
- where(uuid: uuid).
- select(:portable_data_hash).
- first
- if !c
- raise ArvadosModel::UnresolvableContainerError.new "cannot mount collection #{uuid.inspect}: not found"
- end
- if mount['portable_data_hash'].nil?
- # PDH not supplied by client
- mount['portable_data_hash'] = c.portable_data_hash
- elsif mount['portable_data_hash'] != c.portable_data_hash
- # UUID and PDH supplied by client, but they don't agree
- raise ArgumentError.new "cannot mount collection #{uuid.inspect}: current portable_data_hash #{c.portable_data_hash.inspect} does not match #{c['portable_data_hash'].inspect} in request"
- end
- end
- end
- return c_mounts
- end
-
- # Return a container_image PDH suitable for a Container.
- def container_image_for_container
- coll = Collection.for_latest_docker_image(container_image)
- if !coll
- raise ArvadosModel::UnresolvableContainerError.new "docker image #{container_image.inspect} not found"
- end
- coll.portable_data_hash
+ self.output_ttl ||= 0
end
def set_container
return false
end
if state_changed? and state == Committed and container_uuid.nil?
- resolve
+ self.container_uuid = Container.resolve(self).uuid
end
if self.container_uuid != self.container_uuid_was
if self.container_count_changed?
def validate_runtime_constraints
case self.state
when Committed
- ['vcpus', 'ram'].each do |k|
- if not (runtime_constraints.include? k and
- runtime_constraints[k].is_a? Integer and
- runtime_constraints[k] > 0)
- errors.add :runtime_constraints, "#{k} must be a positive integer"
+ [['vcpus', true],
+ ['ram', true],
+ ['keep_cache_ram', false]].each do |k, required|
+ if !required && !runtime_constraints.include?(k)
+ next
+ end
+ v = runtime_constraints[k]
+ unless (v.is_a?(Integer) && v > 0)
+ errors.add(:runtime_constraints,
+ "[#{k}]=#{v.inspect} must be a positive integer")
end
- end
-
- if runtime_constraints.include? 'keep_cache_ram' and
- (!runtime_constraints['keep_cache_ram'].is_a?(Integer) or
- runtime_constraints['keep_cache_ram'] <= 0)
- errors.add :runtime_constraints, "keep_cache_ram must be a positive integer"
- elsif !runtime_constraints.include? 'keep_cache_ram'
- runtime_constraints['keep_cache_ram'] = Rails.configuration.container_default_keep_cache_ram
end
end
end
end
end
- def validate_change
- permitted = [:owner_uuid]
+ def check_update_whitelist
+ permitted = AttrsPermittedAlways.dup
- case self.state
- when Uncommitted
- # Permit updating most fields
- permitted.push :command, :container_count_max,
- :container_image, :cwd, :description, :environment,
- :filters, :mounts, :name, :output_path, :priority,
- :properties, :requesting_container_uuid, :runtime_constraints,
- :state, :container_uuid, :use_existing, :scheduling_parameters,
- :output_name
+ if self.new_record? || self.state_was == Uncommitted
+ # Allow create-and-commit in a single operation.
+ permitted.push *AttrsPermittedBeforeCommit
+ end
+ case self.state
when Committed
- if container_uuid.nil?
- errors.add :container_uuid, "has not been resolved to a container."
- end
+ permitted.push :priority, :container_count_max, :container_uuid
- if priority.nil?
- errors.add :priority, "cannot be nil"
+ if self.container_uuid.nil?
+ self.errors.add :container_uuid, "has not been resolved to a container."
end
- # Can update priority, container count, name and description
- permitted.push :priority, :container_count, :container_count_max, :container_uuid,
- :name, :description
+ if self.priority.nil?
+ self.errors.add :priority, "cannot be nil"
+ end
- if self.state_changed?
- # Allow create-and-commit in a single operation.
- permitted.push :command, :container_image, :cwd, :description, :environment,
- :filters, :mounts, :name, :output_path, :properties,
- :requesting_container_uuid, :runtime_constraints,
- :state, :container_uuid, :use_existing, :scheduling_parameters,
- :output_name
+ # Allow container count to increment by 1
+ if (self.container_uuid &&
+ self.container_uuid != self.container_uuid_was &&
+ self.container_count == 1 + (self.container_count_was || 0))
+ permitted.push :container_count
end
when Final
- if not current_user.andand.is_admin and not (self.name_changed? || self.description_changed?)
- errors.add :state, "of container request can only be set to Final by system."
+ if self.state_changed? and not current_user.andand.is_admin
+ self.errors.add :state, "of container request can only be set to Final by system."
end
- if self.state_changed? || self.name_changed? || self.description_changed? || self.output_uuid_changed? || self.log_uuid_changed?
- permitted.push :state, :name, :description, :output_uuid, :log_uuid
- else
- errors.add :state, "does not allow updates"
+ if self.state_was == Committed
+ permitted.push :output_uuid, :log_uuid
end
- else
- errors.add :state, "invalid value"
end
- check_update_whitelist permitted
+ super(permitted)
end
def update_priority
+require 'audit_logs'
+
class Log < ArvadosModel
include HasUuid
include KindAndEtag
serialize :properties, Hash
before_validation :set_default_event_at
after_save :send_notify
+ after_commit { AuditLogs.tidy_in_background }
api_accessible :user, extend: :common do |t|
t.add :id
def send_notify
connection.execute "NOTIFY logs, '#{self.id}'"
end
-
end
+require 'tempfile'
+
class Node < ArvadosModel
include HasUuid
include KindAndEtag
}
if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template
+ tmpfile = nil
begin
begin
template = IO.read(Rails.configuration.dns_server_conf_template)
- rescue => e
+ rescue IOError, SystemCallError => e
logger.error "Reading #{Rails.configuration.dns_server_conf_template}: #{e.message}"
raise
end
hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
- File.open hostfile+'.tmp', 'w' do |f|
+ Tempfile.open(["#{hostname}-", ".conf.tmp"],
+ Rails.configuration.dns_server_conf_dir) do |f|
+ tmpfile = f.path
f.puts template % template_vars
end
- File.rename hostfile+'.tmp', hostfile
- rescue => e
+ File.rename tmpfile, hostfile
+ rescue IOError, SystemCallError => e
logger.error "Writing #{hostfile}: #{e.message}"
ok = false
+ ensure
+ if tmpfile and File.file? tmpfile
+ # Cleanup remaining temporary file.
+ File.unlink tmpfile
+ end
end
end
# Typically, this is used to trigger a dns server restart
f.puts Rails.configuration.dns_server_reload_command
end
- rescue => e
+ rescue IOError, SystemCallError => e
logger.error "Unable to write #{restartfile}: #{e.message}"
ok = false
end
# crunchstat logs from the logs table.
clean_container_log_rows_after: <%= 30.days %>
+ # Time to keep audit logs, in seconds. (An audit log is a row added
+ # to the "logs" table in the PostgreSQL database each time an
+ # Arvados object is created, modified, or deleted.)
+ #
+ # Currently, websocket event notifications rely on audit logs, so
+ # this should not be set lower than 600 (5 minutes).
+ max_audit_log_age: 1209600
+
+ # Maximum number of log rows to delete in a single SQL transaction.
+ #
+ # If max_audit_log_delete_batch is 0, log entries will never be
+ # deleted by Arvados. Cleanup can be done by an external process
+ # without affecting any Arvados system processes, as long as very
+ # recent (<5 minutes old) logs are not deleted.
+ #
+ # 100000 is a reasonable batch size for most sites.
+ max_audit_log_delete_batch: 0
+
# The maximum number of compute nodes that can be in use simultaneously
# If this limit is reduced, any existing nodes with slot number >= new limit
# will not be counted against the new limit. In other words, the new limit
--- /dev/null
+class AddPortableDataHashIndexToCollections < ActiveRecord::Migration
+ def change
+ add_index :collections, :portable_data_hash
+ end
+end
--- /dev/null
+class AddOutputTtlToContainerRequests < ActiveRecord::Migration
+ def change
+ add_column :container_requests, :output_ttl, :integer, default: 0, null: false
+ end
+end
--- /dev/null
+class AddCreatedByJobTaskIndexToJobTasks < ActiveRecord::Migration
+ def change
+ add_index :job_tasks, :created_by_job_task_uuid
+ end
+end
--- /dev/null
+class AddObjectOwnerIndexToLogs < ActiveRecord::Migration
+ def change
+ add_index :logs, :object_owner_uuid
+ end
+end
--- /dev/null
+class AddRequestingContainerIndexToContainerRequests < ActiveRecord::Migration
+ def change
+ add_index :container_requests, :requesting_container_uuid
+ end
+end
scheduling_parameters text,
output_uuid character varying(255),
log_uuid character varying(255),
- output_name character varying(255) DEFAULT NULL::character varying
+ output_name character varying(255) DEFAULT NULL::character varying,
+ output_ttl integer DEFAULT 0 NOT NULL
);
CREATE UNIQUE INDEX index_collections_on_owner_uuid_and_name ON collections USING btree (owner_uuid, name) WHERE (is_trashed = false);
+--
+-- Name: index_collections_on_portable_data_hash; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX index_collections_on_portable_data_hash ON collections USING btree (portable_data_hash);
+
+
--
-- Name: index_collections_on_trash_at; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE INDEX index_container_requests_on_owner_uuid ON container_requests USING btree (owner_uuid);
+--
+-- Name: index_container_requests_on_requesting_container_uuid; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX index_container_requests_on_requesting_container_uuid ON container_requests USING btree (requesting_container_uuid);
+
+
--
-- Name: index_container_requests_on_uuid; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE INDEX index_job_tasks_on_created_at ON job_tasks USING btree (created_at);
+--
+-- Name: index_job_tasks_on_created_by_job_task_uuid; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX index_job_tasks_on_created_by_job_task_uuid ON job_tasks USING btree (created_by_job_task_uuid);
+
+
--
-- Name: index_job_tasks_on_job_uuid; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
CREATE INDEX index_logs_on_modified_at ON logs USING btree (modified_at);
+--
+-- Name: index_logs_on_object_owner_uuid; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX index_logs_on_object_owner_uuid ON logs USING btree (object_owner_uuid);
+
+
--
-- Name: index_logs_on_object_uuid; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
INSERT INTO schema_migrations (version) VALUES ('20170216170823');
-INSERT INTO schema_migrations (version) VALUES ('20170301225558');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20170301225558');
+
+INSERT INTO schema_migrations (version) VALUES ('20170328215436');
+
+INSERT INTO schema_migrations (version) VALUES ('20170330012505');
+
+INSERT INTO schema_migrations (version) VALUES ('20170419173031');
+
+INSERT INTO schema_migrations (version) VALUES ('20170419173712');
+
+INSERT INTO schema_migrations (version) VALUES ('20170419175801');
\ No newline at end of file
--- /dev/null
+require 'current_api_client'
+require 'db_current_time'
+
+module AuditLogs
+ extend CurrentApiClient
+ extend DbCurrentTime
+
+ def self.delete_old(max_age:, max_batch:)
+ act_as_system_user do
+ if !File.owned?(Rails.root.join('tmp'))
+ Rails.logger.warn("AuditLogs: not owner of #{Rails.root}/tmp, skipping")
+ return
+ end
+ lockfile = Rails.root.join('tmp', 'audit_logs.lock')
+ File.open(lockfile, File::RDWR|File::CREAT, 0600) do |f|
+ return unless f.flock(File::LOCK_NB|File::LOCK_EX)
+
+ sql = "select clock_timestamp() - interval '#{'%.9f' % max_age} seconds'"
+ threshold = ActiveRecord::Base.connection.select_value(sql).to_time.utc
+ Rails.logger.info "AuditLogs: deleting logs older than #{threshold}"
+
+ did_total = 0
+ loop do
+ sql = Log.unscoped.
+ select(:id).
+ order(:created_at).
+ where('event_type in (?)', ['create', 'update', 'destroy', 'delete']).
+ where('created_at < ?', threshold).
+ limit(max_batch).
+ to_sql
+ did = Log.unscoped.where("id in (#{sql})").delete_all
+ did_total += did
+
+ Rails.logger.info "AuditLogs: deleted batch of #{did}"
+ break if did == 0
+ end
+ Rails.logger.info "AuditLogs: deleted total #{did_total}"
+ end
+ end
+ end
+
+ def self.tidy_in_background
+ max_age = Rails.configuration.max_audit_log_age
+ max_batch = Rails.configuration.max_audit_log_delete_batch
+ return if max_age <= 0 || max_batch <= 0
+
+ exp = (max_age/14).seconds
+ need = false
+ Rails.cache.fetch('AuditLogs', expires_in: exp) do
+ need = true
+ end
+ return if !need
+
+ Thread.new do
+ Thread.current.abort_on_exception = false
+ begin
+ delete_old(max_age: max_age, max_batch: max_batch)
+ rescue => e
+ Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
+ ensure
+ ActiveRecord::Base.connection.close
+ end
+ end
+ end
+end
jobrecord = Job.find_by_uuid(job_done.uuid)
if exit_status == EXIT_RETRY_UNLOCKED or (exit_tempfail and @job_retry_counts.include? jobrecord.uuid)
+ $stderr.puts("dispatch: job #{jobrecord.uuid} was interrupted by node failure")
# Only this crunch-dispatch process can retry the job:
# it's already locked, and there's no way to put it back in the
# Queued state. Put it in our internal todo list unless the job
# has failed this way excessively.
@job_retry_counts[jobrecord.uuid] += 1
exit_tempfail = @job_retry_counts[jobrecord.uuid] <= RETRY_UNLOCKED_LIMIT
+ do_what_next = "give up now"
if exit_tempfail
@todo_job_retries[jobrecord.uuid] = jobrecord
- else
- $stderr.puts("dispatch: job #{jobrecord.uuid} exceeded node failure retry limit -- giving up")
+ do_what_next = "re-attempt"
end
+ $stderr.puts("dispatch: job #{jobrecord.uuid} has been interrupted " +
+ "#{@job_retry_counts[jobrecord.uuid]}x, will #{do_what_next}")
end
if !exit_tempfail
# An array of job_uuids in squeue
def squeue_jobs
if Rails.configuration.crunch_job_wrapper == :slurm_immediate
- File.popen(['squeue', '-a', '-h', '-o', '%j']).readlines.map do |line|
- line.strip
+ p = IO.popen(['squeue', '-a', '-h', '-o', '%j'])
+ begin
+ p.readlines.map {|line| line.strip}
+ ensure
+ p.close
end
else
[]
def scancel slurm_name
cmd = sudo_preface + ['scancel', '-n', slurm_name]
- puts File.popen(cmd).read
+ IO.popen(cmd) do |scancel_pipe|
+ puts scancel_pipe.read
+ end
if not $?.success?
Rails.logger.error "scancel #{slurm_name.shellescape}: $?"
end
# The following four collections are used to test combining collections with repeated filenames
collection_with_repeated_filenames_and_contents_in_two_dirs_1:
uuid: zzzzz-4zz18-duplicatenames1
- portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ portable_data_hash: f3a67fad3a19c31c658982fb8158fa58+144
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
collection_with_repeated_filenames_and_contents_in_two_dirs_2:
uuid: zzzzz-4zz18-duplicatenames2
- portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ portable_data_hash: f3a67fad3a19c31c658982fb8158fa58+144
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
name: collection_not_readable_by_active
+collection_to_remove_and_rename_files:
+ uuid: zzzzz-4zz18-a21ux3541sxa8sf
+ portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:file1 0:0:file2\n"
+ name: collection to remove and rename files
+
# Test Helper trims the rest of the file
object_uuid: zzzzz-2x53u-382brsig8rp3667 # repository foo
object_owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz # active user
event_at: <%= 2.minute.ago.to_s(:db) %>
+ event_type: update
admin_changes_specimen: # admin changes specimen owned_by_spectator
id: 3
object_uuid: zzzzz-2x53u-3b0xxwzlbzxq5yr # specimen owned_by_spectator
object_owner_uuid: zzzzz-tpzed-l1s2piq4t4mps8r # spectator user
event_at: <%= 3.minute.ago.to_s(:db) %>
+ event_type: update
system_adds_foo_file: # foo collection added, readable by active through link
id: 4
object_uuid: zzzzz-4zz18-znfnqtbbv4spc3w # foo file
object_owner_uuid: zzzzz-tpzed-000000000000000 # system user
event_at: <%= 4.minute.ago.to_s(:db) %>
+ event_type: create
system_adds_baz: # baz collection added, readable by active and spectator through group 'all users' group membership
id: 5
object_uuid: zzzzz-4zz18-y9vne9npefyxh8g # baz file
object_owner_uuid: zzzzz-tpzed-000000000000000 # system user
event_at: <%= 5.minute.ago.to_s(:db) %>
+ event_type: create
log_owned_by_active:
id: 6
authorize_with :inactive
get :index
assert_response :success
- node_items = JSON.parse(@response.body)['items']
- assert_equal 0, node_items.size
+ assert_equal 0, json_response['items'].size
+ assert_equal 0, json_response['items_available']
end
# active user sees non-secret attributes of up and recently-up nodes
authorize_with :active
get :index
assert_response :success
- node_items = JSON.parse(@response.body)['items']
- assert_not_equal 0, node_items.size
+ assert_operator 0, :<, json_response['items_available']
+ node_items = json_response['items']
+ assert_operator 0, :<, node_items.size
found_busy_node = false
node_items.each do |node|
assert_nil node['info'].andand['ping_secret']
authorize_with user
get :index, {select: ['domain']}
assert_response :success
+ assert_operator 0, :<, json_response['items_available']
end
end
include UsersTestHelper
setup do
- @all_links_at_start = Link.all
+ @initial_link_count = Link.count
@vm_uuid = virtual_machines(:testvm).uuid
end
assert_nil created['identity_url'], 'expected no identity_url'
# arvados#user, repo link and link add user to 'All users' group
- verify_num_links @all_links_at_start, 4
+ verify_links_added 4
verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
created['uuid'], created['email'], 'arvados#user', false, 'User'
assert_equal response_object['email'], 'foo@example.com', 'expected given email'
# four extra links; system_group, login, group and repo perms
- verify_num_links @all_links_at_start, 4
+ verify_links_added 4
end
test "setup user with fake vm and expect error" do
assert_equal response_object['email'], 'foo@example.com', 'expected given email'
# five extra links; system_group, login, group, vm, repo
- verify_num_links @all_links_at_start, 5
+ verify_links_added 5
end
test "setup user with valid email, no vm and no repo as input" do
assert_equal response_object['email'], 'foo@example.com', 'expected given email'
# three extra links; system_group, login, and group
- verify_num_links @all_links_at_start, 3
+ verify_links_added 3
verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
response_object['uuid'], response_object['email'], 'arvados#user', false, 'User'
'expecting first name'
# five extra links; system_group, login, group, repo and vm
- verify_num_links @all_links_at_start, 5
+ verify_links_added 5
end
test "setup user with an existing user email and check different object is created" do
'expected different uuid after create operation'
assert_equal inactive_user['email'], response_object['email'], 'expected given email'
# system_group, openid, group, and repo. No vm link.
- verify_num_links @all_links_at_start, 4
+ verify_links_added 4
end
test "setup user with openid prefix" do
# verify links
# four new links: system_group, arvados#user, repo, and 'All users' group.
- verify_num_links @all_links_at_start, 4
+ verify_links_added 4
verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
created['uuid'], created['email'], 'arvados#user', false, 'User'
# five new links: system_group, arvados#user, repo, vm and 'All
# users' group link
- verify_num_links @all_links_at_start, 5
+ verify_links_added 5
verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
created['uuid'], created['email'], 'arvados#user', false, 'User'
"admin's filtered index did not return inactive user")
end
- def verify_num_links (original_links, expected_additional_links)
- assert_equal expected_additional_links, Link.all.size-original_links.size,
- "Expected #{expected_additional_links.inspect} more links"
+ def verify_links_added more
+ assert_equal @initial_link_count+more, Link.count,
+ "Started with #{@initial_link_count} links, expected #{more} more"
end
def find_obj_in_resp (response_items, object_type, head_kind=nil)
end
end
+ now = Time.now
[['trash-to-delete interval negative',
:collection_owned_by_active,
- {trash_at: Time.now+2.weeks, delete_at: Time.now},
+ {trash_at: now+2.weeks, delete_at: now},
{state: :invalid}],
- ['trash-to-delete interval too short',
+ ['now-to-delete interval short',
:collection_owned_by_active,
- {trash_at: Time.now+3.days, delete_at: Time.now+7.days},
- {state: :invalid}],
+ {trash_at: now+3.days, delete_at: now+7.days},
+ {state: :trash_future}],
+ ['now-to-delete interval short, trash=delete',
+ :collection_owned_by_active,
+ {trash_at: now+3.days, delete_at: now+3.days},
+ {state: :trash_future}],
['trash-to-delete interval ok',
:collection_owned_by_active,
- {trash_at: Time.now, delete_at: Time.now+15.days},
+ {trash_at: now, delete_at: now+15.days},
{state: :trash_now}],
['trash-to-delete interval short, but far enough in future',
:collection_owned_by_active,
- {trash_at: Time.now+13.days, delete_at: Time.now+15.days},
+ {trash_at: now+13.days, delete_at: now+15.days},
{state: :trash_future}],
['trash by setting is_trashed bool',
:collection_owned_by_active,
{state: :trash_now}],
['trash in future by setting just trash_at',
:collection_owned_by_active,
- {trash_at: Time.now+1.week},
+ {trash_at: now+1.week},
{state: :trash_future}],
['trash in future by setting trash_at and delete_at',
:collection_owned_by_active,
- {trash_at: Time.now+1.week, delete_at: Time.now+4.weeks},
+ {trash_at: now+1.week, delete_at: now+4.weeks},
{state: :trash_future}],
['untrash by clearing is_trashed bool',
:expired_collection,
end
updates_ok = c.update_attributes(updates)
expect_valid = expect[:state] != :invalid
- assert_equal updates_ok, expect_valid, c.errors.full_messages.to_s
+ assert_equal expect_valid, updates_ok, c.errors.full_messages.to_s
case expect[:state]
when :invalid
refute c.valid?
class ContainerRequestTest < ActiveSupport::TestCase
include DockerMigrationHelper
+ include DbCurrentTime
def create_minimal_req! attrs={}
defaults = {
cr.reload
+ assert_equal({"vcpus" => 2, "ram" => 30}, cr.runtime_constraints)
+
assert_not_nil cr.container_uuid
c = Container.find_by_uuid cr.container_uuid
assert_not_nil c
lambda { |resolved| resolved["ram"] == 1234234234 }],
].each do |rc, okfunc|
test "resolve runtime constraint range #{rc} to values" do
- cr = ContainerRequest.new(runtime_constraints: rc)
- resolved = cr.send :runtime_constraints_for_container
+ resolved = Container.resolve_runtime_constraints(rc)
assert(okfunc.call(resolved),
"container runtime_constraints was #{resolved.inspect}")
end
].each do |mounts, okfunc|
test "resolve mounts #{mounts.inspect} to values" do
set_user_from_auth :active
- cr = ContainerRequest.new(mounts: mounts)
- resolved = cr.send :mounts_for_container
+ resolved = Container.resolve_mounts(mounts)
assert(okfunc.call(resolved),
- "mounts_for_container returned #{resolved.inspect}")
+ "Container.resolve_mounts returned #{resolved.inspect}")
end
end
"path" => "/foo",
},
}
- cr = ContainerRequest.new(mounts: m)
assert_raises(ArvadosModel::UnresolvableContainerError) do
- cr.send :mounts_for_container
+ Container.resolve_mounts(m)
end
end
"path" => "/foo",
},
}
- cr = ContainerRequest.new(mounts: m)
assert_raises(ArgumentError) do
- cr.send :mounts_for_container
+ Container.resolve_mounts(m)
end
end
'arvados/apitestfixture',
'd8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678',
].each do |tag|
- test "container_image_for_container(#{tag.inspect})" do
+ test "Container.resolve_container_image(#{tag.inspect})" do
set_user_from_auth :active
- cr = ContainerRequest.new(container_image: tag)
- resolved = cr.send :container_image_for_container
+ resolved = Container.resolve_container_image(tag)
assert_equal resolved, collections(:docker_image).portable_data_hash
end
end
- test "container_image_for_container(pdh)" do
+ test "Container.resolve_container_image(pdh)" do
set_user_from_auth :active
[[:docker_image, 'v1'], [:docker_image_1_12, 'v2']].each do |coll, ver|
Rails.configuration.docker_image_formats = [ver]
pdh = collections(coll).portable_data_hash
- cr = ContainerRequest.new(container_image: pdh)
- resolved = cr.send :container_image_for_container
+ resolved = Container.resolve_container_image(pdh)
assert_equal resolved, pdh
end
end
].each do |img|
test "container_image_for_container(#{img.inspect}) => 422" do
set_user_from_auth :active
- cr = ContainerRequest.new(container_image: img)
assert_raises(ArvadosModel::UnresolvableContainerError) do
- cr.send :container_image_for_container
+ Container.resolve_container_image(img)
end
end
end
set_user_from_auth :active
cr = create_minimal_req!(command: ["true", "1"],
container_image: collections(:docker_image).portable_data_hash)
- assert_equal(cr.send(:container_image_for_container),
+ assert_equal(Container.resolve_container_image(cr.container_image),
collections(:docker_image_1_12).portable_data_hash)
cr = create_minimal_req!(command: ["true", "2"],
container_image: links(:docker_image_collection_tag).name)
- assert_equal(cr.send(:container_image_for_container),
+ assert_equal(Container.resolve_container_image(cr.container_image),
collections(:docker_image_1_12).portable_data_hash)
end
set_user_from_auth :active
cr = create_minimal_req!(command: ["true", "1"],
container_image: collections(:docker_image).portable_data_hash)
- assert_equal(cr.send(:container_image_for_container),
+ assert_equal(Container.resolve_container_image(cr.container_image),
collections(:docker_image).portable_data_hash)
cr = create_minimal_req!(command: ["true", "2"],
container_image: links(:docker_image_collection_tag).name)
- assert_equal(cr.send(:container_image_for_container),
+ assert_equal(Container.resolve_container_image(cr.container_image),
collections(:docker_image).portable_data_hash)
end
cr = create_minimal_req!(command: ["true", "1"],
container_image: collections(:docker_image_1_12).portable_data_hash)
assert_raises(ArvadosModel::UnresolvableContainerError) do
- cr.send(:container_image_for_container)
+ Container.resolve_container_image(cr.container_image)
end
end
cr = create_minimal_req!(command: ["true", "1"],
container_image: collections(:docker_image).portable_data_hash)
assert_raises(ArvadosModel::UnresolvableContainerError) do
- cr.send(:container_image_for_container)
+ Container.resolve_container_image(cr.container_image)
end
cr = create_minimal_req!(command: ["true", "2"],
container_image: links(:docker_image_collection_tag).name)
assert_raises(ArvadosModel::UnresolvableContainerError) do
- cr.send(:container_image_for_container)
+ Container.resolve_container_image(cr.container_image)
end
end
command: ["echo", "hello"],
output_path: "test",
runtime_constraints: {"vcpus" => 4,
- "ram" => 12000000000,
- "keep_cache_ram" => 268435456},
+ "ram" => 12000000000},
mounts: {"test" => {"kind" => "json"}}}
set_user_from_auth :active
cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed,
test "Output collection name setting using output_name with name collision resolution" do
set_user_from_auth :active
- output_name = collections(:foo_file).name
+ output_name = 'unimaginative name'
+ Collection.create!(name: output_name)
cr = create_minimal_req!(priority: 1,
state: ContainerRequest::Committed,
output_name: output_name)
- act_as_system_user do
- c = Container.find_by_uuid(cr.container_uuid)
- c.update_attributes!(state: Container::Locked)
- c.update_attributes!(state: Container::Running)
- c.update_attributes!(state: Container::Complete,
- exit_code: 0,
- output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
- log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
- end
- cr.save
+ run_container(cr)
+ cr.reload
assert_equal ContainerRequest::Final, cr.state
output_coll = Collection.find_by_uuid(cr.output_uuid)
# Make sure the resulting output collection name include the original name
# plus the date
assert_not_equal output_name, output_coll.name,
- "It shouldn't exist more than one collection with the same owner and name '${output_name}'"
+ "more than one collection with the same owner and name"
assert output_coll.name.include?(output_name),
"New name should include original name"
- assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, output_coll.name,
+ assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
"New name should include ISO8601 date"
end
- test "Finalize committed request when reusing a finished container" do
- set_user_from_auth :active
- cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
- cr.reload
- assert_equal ContainerRequest::Committed, cr.state
+ [[0, :check_output_ttl_0],
+ [1, :check_output_ttl_1s],
+ [365*86400, :check_output_ttl_1y],
+ ].each do |ttl, checker|
+ test "output_ttl=#{ttl}" do
+ act_as_user users(:active) do
+ cr = create_minimal_req!(priority: 1,
+ state: ContainerRequest::Committed,
+ output_name: 'foo',
+ output_ttl: ttl)
+ run_container(cr)
+ cr.reload
+ output = Collection.find_by_uuid(cr.output_uuid)
+ send(checker, db_current_time, output.trash_at, output.delete_at)
+ end
+ end
+ end
+
+ def check_output_ttl_0(now, trash, delete)
+ assert_nil(trash)
+ assert_nil(delete)
+ end
+
+ def check_output_ttl_1s(now, trash, delete)
+ assert_not_nil(trash)
+ assert_not_nil(delete)
+ assert_in_delta(trash, now + 1.second, 10)
+ assert_in_delta(delete, now + Rails.configuration.blob_signature_ttl.second, 10)
+ end
+
+ def check_output_ttl_1y(now, trash, delete)
+ year = (86400*365).second
+ assert_not_nil(trash)
+ assert_not_nil(delete)
+ assert_in_delta(trash, now + year, 10)
+ assert_in_delta(delete, now + year, 10)
+ end
+
+ def run_container(cr)
act_as_system_user do
c = Container.find_by_uuid(cr.container_uuid)
c.update_attributes!(state: Container::Locked)
exit_code: 0,
output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
+ c
end
+ end
+
+ test "Finalize committed request when reusing a finished container" do
+ set_user_from_auth :active
+ cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
+ cr.reload
+ assert_equal ContainerRequest::Committed, cr.state
+ run_container(cr)
cr.reload
assert_equal ContainerRequest::Final, cr.state
assert_equal ContainerRequest::Final, cr3.state
end
- [
- [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => 100}, ContainerRequest::Committed, 100],
- [{"vcpus" => 1, "ram" => 123}, ContainerRequest::Uncommitted],
- [{"vcpus" => 1, "ram" => 123}, ContainerRequest::Committed],
- [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => -1}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
- [{"vcpus" => 1, "ram" => 123, "keep_cache_ram" => '123'}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
- ].each do |rc, state, expected|
- test "create container request with #{rc} in state #{state} and verify keep_cache_ram #{expected}" do
- common_attrs = {cwd: "test",
- priority: 1,
- command: ["echo", "hello"],
- output_path: "test",
- runtime_constraints: rc,
- mounts: {"test" => {"kind" => "json"}}}
- set_user_from_auth :active
-
- if expected == ActiveRecord::RecordInvalid
- assert_raises(ActiveRecord::RecordInvalid) do
- create_minimal_req!(common_attrs.merge({state: state}))
- end
- else
- cr = create_minimal_req!(common_attrs.merge({state: state}))
- expected = Rails.configuration.container_default_keep_cache_ram if state == ContainerRequest::Committed and expected.nil?
- assert_equal expected, cr.runtime_constraints['keep_cache_ram']
- end
- end
- end
-
[
[{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Committed, ActiveRecord::RecordInvalid],
[{"partitions" => ["fastcpu","vfastcpu", 100]}, ContainerRequest::Uncommitted],
end
end
end
+
+ [['Committed', true, {name: "foobar", priority: 123}],
+ ['Committed', false, {container_count: 2}],
+ ['Committed', false, {container_count: 0}],
+ ['Committed', false, {container_count: nil}],
+ ['Final', false, {state: ContainerRequest::Committed, name: "foobar"}],
+ ['Final', false, {name: "foobar", priority: 123}],
+ ['Final', false, {name: "foobar", output_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
+ ['Final', false, {name: "foobar", log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
+ ['Final', false, {log_uuid: "zzzzz-4zz18-znfnqtbbv4spc3w"}],
+ ['Final', false, {priority: 123}],
+ ['Final', false, {mounts: {}}],
+ ['Final', false, {container_count: 2}],
+ ['Final', true, {name: "foobar"}],
+ ['Final', true, {name: "foobar", description: "baz"}],
+ ].each do |state, permitted, updates|
+ test "state=#{state} can#{'not' if !permitted} update #{updates.inspect}" do
+ act_as_user users(:active) do
+ cr = create_minimal_req!(priority: 1,
+ state: "Committed",
+ container_count_max: 1)
+ case state
+ when 'Committed'
+ # already done
+ when 'Final'
+ act_as_system_user do
+ Container.find_by_uuid(cr.container_uuid).
+ update_attributes!(state: Container::Cancelled)
+ end
+ cr.reload
+ else
+ raise 'broken test case'
+ end
+ assert_equal state, cr.state
+ if permitted
+ assert cr.update_attributes!(updates)
+ else
+ assert_raises(ActiveRecord::RecordInvalid) do
+ cr.update_attributes!(updates)
+ end
+ end
+ end
+ end
+ end
end
runtime_constraints: {"vcpus" => 1, "ram" => 1},
}
- REUSABLE_COMMON_ATTRS = {container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
- cwd: "test",
- command: ["echo", "hello"],
- output_path: "test",
- runtime_constraints: {"vcpus" => 4,
- "ram" => 12000000000},
- mounts: {"test" => {"kind" => "json"}},
- environment: {"var" => 'val'}}
+ REUSABLE_COMMON_ATTRS = {
+ container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
+ cwd: "test",
+ command: ["echo", "hello"],
+ output_path: "test",
+ runtime_constraints: {
+ "ram" => 12000000000,
+ "vcpus" => 4,
+ },
+ mounts: {
+ "test" => {"kind" => "json"},
+ },
+ environment: {
+ "var" => "val",
+ },
+ }
def minimal_new attrs={}
cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
test "Container serialized hash attributes sorted before save" do
env = {"C" => 3, "B" => 2, "A" => 1}
m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
- rc = {"vcpus" => 1, "ram" => 1}
+ rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
}
- set_user_from_auth :dispatch1
-
- c_output1 = Container.create common_attrs
- c_output2 = Container.create common_attrs
- assert_not_equal c_output1.uuid, c_output2.uuid
-
cr = ContainerRequest.new common_attrs
+ cr.use_existing = false
cr.state = ContainerRequest::Committed
- cr.container_uuid = c_output1.uuid
cr.save!
+ c_output1 = Container.where(uuid: cr.container_uuid).first
cr = ContainerRequest.new common_attrs
+ cr.use_existing = false
cr.state = ContainerRequest::Committed
- cr.container_uuid = c_output2.uuid
cr.save!
+ c_output2 = Container.where(uuid: cr.container_uuid).first
+
+ assert_not_equal c_output1.uuid, c_output2.uuid
+
+ set_user_from_auth :dispatch1
out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
log1 = collections(:real_log_collection).portable_data_hash
c_output2.update_attributes!({state: Container::Running})
c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
- reused = Container.find_reusable(common_attrs)
- assert_not_nil reused
- assert_equal reused.uuid, c_output1.uuid
+ reused = Container.resolve(ContainerRequest.new(common_attrs))
+ assert_equal c_output1.uuid, reused.uuid
end
test "find_reusable method should select running container by start date" do
act_as_system_user do
dispatch = CrunchDispatch.new
- squeue_resp = File.popen("echo zzzzz-8i9sb-pshmckwoma9plh7\necho thisisnotvalidjobuuid\necho zzzzz-8i9sb-4cf0abc123e809j\n")
- scancel_resp = File.popen("true")
+ squeue_resp = IO.popen("echo zzzzz-8i9sb-pshmckwoma9plh7\necho thisisnotvalidjobuuid\necho zzzzz-8i9sb-4cf0abc123e809j\n")
+ scancel_resp = IO.popen("true")
- File.expects(:popen).
+ IO.expects(:popen).
with(['squeue', '-a', '-h', '-o', '%j']).
returns(squeue_resp)
- File.expects(:popen).
+ IO.expects(:popen).
with(dispatch.sudo_preface + ['scancel', '-n', 'zzzzz-8i9sb-4cf0abc123e809j']).
returns(scancel_resp)
test 'cancel slurm jobs' do
Rails.configuration.crunch_job_wrapper = :slurm_immediate
Rails.configuration.crunch_job_user = 'foobar'
- fake_squeue = File.popen("echo #{@job[:before_reboot].uuid}")
- fake_scancel = File.popen("true")
- File.expects(:popen).
+ fake_squeue = IO.popen("echo #{@job[:before_reboot].uuid}")
+ fake_scancel = IO.popen("true")
+ IO.expects(:popen).
with(['squeue', '-a', '-h', '-o', '%j']).
returns(fake_squeue)
- File.expects(:popen).
+ IO.expects(:popen).
with(includes('sudo', '-u', 'foobar', 'scancel', '-n', @job[:before_reboot].uuid)).
returns(fake_scancel)
@dispatch.fail_jobs(before: Time.at(BOOT_TIME).to_s)
require 'test_helper'
+require 'audit_logs'
class LogTest < ActiveSupport::TestCase
include CurrentApiClient
end
end
end
+
+ def assert_no_logs_deleted
+ logs_before = Log.unscoped.all.count
+ yield
+ assert_equal logs_before, Log.unscoped.all.count
+ end
+
+ def remaining_audit_logs
+ Log.unscoped.where('event_type in (?)', %w(create update destroy delete))
+ end
+
+ # Default settings should not delete anything -- some sites rely on
+ # the original "keep everything forever" behavior.
+ test 'retain old audit logs with default settings' do
+ assert_no_logs_deleted do
+ AuditLogs.delete_old(
+ max_age: Rails.configuration.max_audit_log_age,
+ max_batch: Rails.configuration.max_audit_log_delete_batch)
+ end
+ end
+
+ # Batch size 0 should retain all logs -- even if max_age is very
+ # short, and even if the default settings (and associated test) have
+ # changed.
+ test 'retain old audit logs with max_audit_log_delete_batch=0' do
+ assert_no_logs_deleted do
+ AuditLogs.delete_old(max_age: 1, max_batch: 0)
+ end
+ end
+
+ # We recommend a more conservative age of 5 minutes for production,
+ # but 3 minutes suits our test data better (and is test-worthy in
+ # that it's expected to work correctly in production).
+ test 'delete old audit logs with production settings' do
+ initial_log_count = Log.unscoped.all.count
+ AuditLogs.delete_old(max_age: 180, max_batch: 100000)
+ assert_operator remaining_audit_logs.count, :<, initial_log_count
+ end
+
+ test 'delete all audit logs in multiple batches' do
+ AuditLogs.delete_old(max_age: 0.00001, max_batch: 2)
+ assert_equal [], remaining_audit_logs.collect(&:uuid)
+ end
+
+ test 'delete old audit logs in thread' do
+ begin
+ Rails.configuration.max_audit_log_age = 20
+ Rails.configuration.max_audit_log_delete_batch = 100000
+ Rails.cache.delete 'AuditLogs'
+ initial_log_count = Log.unscoped.all.count + 1
+ act_as_system_user do
+ Log.create!()
+ initial_log_count += 1
+ end
+ deadline = Time.now + 10
+ while remaining_audit_logs.count == initial_log_count
+ if Time.now > deadline
+ raise "timed out"
+ end
+ sleep 0.1
+ end
+ assert_operator remaining_audit_logs.count, :<, initial_log_count
+ ensure
+ # The test framework rolls back our transactions, but that
+ # doesn't undo the deletes we did from separate threads.
+ ActiveRecord::Base.connection.exec_query 'ROLLBACK'
+ Thread.new do
+ begin
+ dc = DatabaseController.new
+ dc.define_singleton_method :render do |*args| end
+ dc.reset
+ ensure
+ ActiveRecord::Base.connection.close
+ end
+ end.join
+ end
+ end
end
require 'test_helper'
+require 'tmpdir'
+require 'tempfile'
class NodeTest < ActiveSupport::TestCase
def ping_node(node_name, ping_data)
assert Node.dns_server_update 'compute65535', '127.0.0.127'
end
+ test "don't leave temp files behind if there's an error writing them" do
+ Rails.configuration.dns_server_conf_template = Rails.root.join 'config', 'unbound.template'
+ Tempfile.any_instance.stubs(:puts).raises(IOError)
+ Dir.mktmpdir do |tmpdir|
+ Rails.configuration.dns_server_conf_dir = tmpdir
+ refute Node.dns_server_update 'compute65535', '127.0.0.127'
+ assert_empty Dir.entries(tmpdir).select{|f| File.file? f}
+ end
+ end
+
test "ping new node with no hostname and default config" do
node = ping_node(:new_with_no_hostname, {})
slot_number = node.slot_number
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/git-httpd/git-httpd.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=notify
ExecStart=/usr/bin/arvados-git-httpd
Restart=always
+RestartSec=1
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=notify
ExecStart=/usr/bin/crunch-dispatch-slurm
Restart=always
+RestartSec=1
[Install]
WantedBy=multi-user.target
package main
import (
+ "bytes"
+ "context"
"encoding/json"
"errors"
"flag"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
- "github.com/curoverse/dockerclient"
+
+ dockertypes "github.com/docker/docker/api/types"
+ dockercontainer "github.com/docker/docker/api/types/container"
+ dockernetwork "github.com/docker/docker/api/types/network"
+ dockerclient "github.com/docker/docker/client"
)
// IArvadosClient is the minimal Arvados API methods used by crunch-run.
Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error
+ CallRaw(method string, resourceType string, uuid string, action string, parameters arvadosclient.Dict) (reader io.ReadCloser, err error)
Discovery(key string) (interface{}, error)
}
// ThinDockerClient is the minimal Docker client interface used by crunch-run.
type ThinDockerClient interface {
- StopContainer(id string, timeout int) error
- InspectImage(id string) (*dockerclient.ImageInfo, error)
- LoadImage(reader io.Reader) error
- CreateContainer(config *dockerclient.ContainerConfig, name string, authConfig *dockerclient.AuthConfig) (string, error)
- StartContainer(id string, config *dockerclient.HostConfig) error
- AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error)
- Wait(id string) <-chan dockerclient.WaitResult
- RemoveImage(name string, force bool) ([]*dockerclient.ImageDelete, error)
+ ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error)
+ ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig,
+ networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error)
+ ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error
+ ContainerStop(ctx context.Context, container string, timeout *time.Duration) error
+ ContainerWait(ctx context.Context, container string) (int64, error)
+ ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error)
+ ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error)
+ ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error)
+}
+
+// ThinDockerClientProxy is a proxy implementation of ThinDockerClient
+// that executes the docker requests on dockerclient.Client
+type ThinDockerClientProxy struct {
+ Docker *dockerclient.Client
+}
+
+// ContainerAttach invokes dockerclient.Client.ContainerAttach
+func (proxy ThinDockerClientProxy) ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error) {
+ return proxy.Docker.ContainerAttach(ctx, container, options)
+}
+
+// ContainerCreate invokes dockerclient.Client.ContainerCreate
+func (proxy ThinDockerClientProxy) ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig,
+ networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error) {
+ return proxy.Docker.ContainerCreate(ctx, config, hostConfig, networkingConfig, containerName)
+}
+
+// ContainerStart invokes dockerclient.Client.ContainerStart
+func (proxy ThinDockerClientProxy) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
+ return proxy.Docker.ContainerStart(ctx, container, options)
+}
+
+// ContainerStop invokes dockerclient.Client.ContainerStop
+func (proxy ThinDockerClientProxy) ContainerStop(ctx context.Context, container string, timeout *time.Duration) error {
+ return proxy.Docker.ContainerStop(ctx, container, timeout)
+}
+
+// ContainerWait invokes dockerclient.Client.ContainerWait
+func (proxy ThinDockerClientProxy) ContainerWait(ctx context.Context, container string) (int64, error) {
+ return proxy.Docker.ContainerWait(ctx, container)
+}
+
+// ImageInspectWithRaw invokes dockerclient.Client.ImageInspectWithRaw
+func (proxy ThinDockerClientProxy) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
+ return proxy.Docker.ImageInspectWithRaw(ctx, image)
+}
+
+// ImageLoad invokes dockerclient.Client.ImageLoad
+func (proxy ThinDockerClientProxy) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
+ return proxy.Docker.ImageLoad(ctx, input, quiet)
+}
+
+// ImageRemove invokes dockerclient.Client.ImageRemove
+func (proxy ThinDockerClientProxy) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) {
+ return proxy.Docker.ImageRemove(ctx, image, options)
}
// ContainerRunner is the main stateful struct used for a single execution of a
ArvClient IArvadosClient
Kc IKeepClient
arvados.Container
- dockerclient.ContainerConfig
- dockerclient.HostConfig
+ ContainerConfig dockercontainer.Config
+ dockercontainer.HostConfig
token string
ContainerID string
ExitCode *int
loggingDone chan bool
CrunchLog *ThrottledLogger
Stdout io.WriteCloser
- Stderr *ThrottledLogger
+ Stderr io.WriteCloser
LogCollection *CollectionWriter
LogsPDH *string
RunArvMount
CleanupTempDir []string
Binds []string
OutputPDH *string
- CancelLock sync.Mutex
- Cancelled bool
SigChan chan os.Signal
ArvMountExit chan error
finalState string
// parent to be X" feature even on sites where the "specify
// cgroup parent" feature breaks.
setCgroupParent string
+
+ cStateLock sync.Mutex
+ cStarted bool // StartContainer() succeeded
+ cCancelled bool // StopContainer() invoked
+
+ enableNetwork string // one of "default" or "always"
+ networkMode string // passed through to HostConfig.NetworkMode
}
// SetupSignals sets up signal handling to gracefully terminate the underlying
// stop the underlying Docker container.
func (runner *ContainerRunner) stop() {
- runner.CancelLock.Lock()
- defer runner.CancelLock.Unlock()
- if runner.Cancelled {
+ runner.cStateLock.Lock()
+ defer runner.cStateLock.Unlock()
+ if runner.cCancelled {
return
}
- runner.Cancelled = true
- if runner.ContainerID != "" {
- err := runner.Docker.StopContainer(runner.ContainerID, 10)
+ runner.cCancelled = true
+ if runner.cStarted {
+ timeout := time.Duration(10)
+ err := runner.Docker.ContainerStop(context.TODO(), runner.ContainerID, &(timeout))
if err != nil {
log.Printf("StopContainer failed: %s", err)
}
runner.CrunchLog.Printf("Using Docker image id '%s'", imageID)
- _, err = runner.Docker.InspectImage(imageID)
+ _, _, err = runner.Docker.ImageInspectWithRaw(context.TODO(), imageID)
if err != nil {
runner.CrunchLog.Print("Loading Docker image from keep")
return fmt.Errorf("While creating ManifestFileReader for container image: %v", err)
}
- err = runner.Docker.LoadImage(readCloser)
+ response, err := runner.Docker.ImageLoad(context.TODO(), readCloser, false)
if err != nil {
return fmt.Errorf("While loading container image into Docker: %v", err)
}
+ response.Body.Close()
} else {
runner.CrunchLog.Print("Docker image is available")
}
for _, bind := range binds {
mnt := runner.Container.Mounts[bind]
- if bind == "stdout" {
+ if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
- return fmt.Errorf("Unsupported mount kind '%s' for stdout. Only 'file' is supported.", mnt.Kind)
+ return fmt.Errorf("Unsupported mount kind '%s' for %s. Only 'file' is supported.", mnt.Kind, bind)
}
// Does path start with OutputPath?
prefix += "/"
}
if !strings.HasPrefix(mnt.Path, prefix) {
- return fmt.Errorf("Stdout path does not start with OutputPath: %s, %s", mnt.Path, prefix)
+ return fmt.Errorf("%s path does not start with OutputPath: %s, %s", strings.Title(bind), mnt.Path, prefix)
+ }
+ }
+
+ if bind == "stdin" {
+ // Is it a "collection" mount kind?
+ if mnt.Kind != "collection" && mnt.Kind != "json" {
+ return fmt.Errorf("Unsupported mount kind '%s' for stdin. Only 'collection' or 'json' are supported.", mnt.Kind)
}
}
}
switch {
- case mnt.Kind == "collection":
+ case mnt.Kind == "collection" && bind != "stdin":
var src string
if mnt.UUID != "" && mnt.PortableDataHash != "" {
return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
runner.statReporter.Start()
}
-// AttachLogs connects the docker container stdout and stderr logs to the
-// Arvados logger which logs to Keep and the API server logs table.
+type infoCommand struct {
+ label string
+ cmd []string
+}
+
+// Gather node information and store it on the log for debugging
+// purposes.
+func (runner *ContainerRunner) LogNodeInfo() (err error) {
+ w := runner.NewLogWriter("node-info")
+ logger := log.New(w, "node-info", 0)
+
+ commands := []infoCommand{
+ infoCommand{
+ label: "Host Information",
+ cmd: []string{"uname", "-a"},
+ },
+ infoCommand{
+ label: "CPU Information",
+ cmd: []string{"cat", "/proc/cpuinfo"},
+ },
+ infoCommand{
+ label: "Memory Information",
+ cmd: []string{"cat", "/proc/meminfo"},
+ },
+ infoCommand{
+ label: "Disk Space",
+ cmd: []string{"df", "-m", "/", os.TempDir()},
+ },
+ infoCommand{
+ label: "Disk INodes",
+ cmd: []string{"df", "-i", "/", os.TempDir()},
+ },
+ }
+
+ // Run commands with informational output to be logged.
+ var out []byte
+ for _, command := range commands {
+ out, err = exec.Command(command.cmd[0], command.cmd[1:]...).CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("While running command %q: %v",
+ command.cmd, err)
+ }
+ logger.Println(command.label)
+ for _, line := range strings.Split(string(out), "\n") {
+ logger.Println(" ", line)
+ }
+ }
+
+ err = w.Close()
+ if err != nil {
+ return fmt.Errorf("While closing node-info logs: %v", err)
+ }
+ return nil
+}
+
+// Get and save the raw JSON container record from the API server
+func (runner *ContainerRunner) LogContainerRecord() (err error) {
+ w := &ArvLogWriter{
+ runner.ArvClient,
+ runner.Container.UUID,
+ "container",
+ runner.LogCollection.Open("container.json"),
+ }
+ // Get Container record JSON from the API Server
+ reader, err := runner.ArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil)
+ if err != nil {
+ return fmt.Errorf("While retrieving container record from the API server: %v", err)
+ }
+ defer reader.Close()
+ // Read the API server response as []byte
+ json_bytes, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return fmt.Errorf("While reading container record API server response: %v", err)
+ }
+ // Decode the JSON []byte
+ var cr map[string]interface{}
+ if err = json.Unmarshal(json_bytes, &cr); err != nil {
+ return fmt.Errorf("While decoding the container record JSON response: %v", err)
+ }
+ // Re-encode it using indentation to improve readability
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ if err = enc.Encode(cr); err != nil {
+ return fmt.Errorf("While logging the JSON container record: %v", err)
+ }
+ err = w.Close()
+ if err != nil {
+ return fmt.Errorf("While closing container.json log: %v", err)
+ }
+ return nil
+}
+
+// AttachStreams connects the docker container stdin, stdout and stderr logs
+// to the Arvados logger which logs to Keep and the API server logs table.
func (runner *ContainerRunner) AttachStreams() (err error) {
runner.CrunchLog.Print("Attaching container streams")
- var containerReader io.Reader
- containerReader, err = runner.Docker.AttachContainer(runner.ContainerID,
- &dockerclient.AttachOptions{Stream: true, Stdout: true, Stderr: true})
+ // If stdin mount is provided, attach it to the docker container
+ var stdinRdr keepclient.Reader
+ var stdinJson []byte
+ if stdinMnt, ok := runner.Container.Mounts["stdin"]; ok {
+ if stdinMnt.Kind == "collection" {
+ var stdinColl arvados.Collection
+ collId := stdinMnt.UUID
+ if collId == "" {
+ collId = stdinMnt.PortableDataHash
+ }
+ err = runner.ArvClient.Get("collections", collId, nil, &stdinColl)
+ if err != nil {
+ return fmt.Errorf("While getting stding collection: %v", err)
+ }
+
+ stdinRdr, err = runner.Kc.ManifestFileReader(manifest.Manifest{Text: stdinColl.ManifestText}, stdinMnt.Path)
+ if os.IsNotExist(err) {
+ return fmt.Errorf("stdin collection path not found: %v", stdinMnt.Path)
+ } else if err != nil {
+ return fmt.Errorf("While getting stdin collection path %v: %v", stdinMnt.Path, err)
+ }
+ } else if stdinMnt.Kind == "json" {
+ stdinJson, err = json.Marshal(stdinMnt.Content)
+ if err != nil {
+ return fmt.Errorf("While encoding stdin json data: %v", err)
+ }
+ }
+ }
+
+ stdinUsed := stdinRdr != nil || len(stdinJson) != 0
+ response, err := runner.Docker.ContainerAttach(context.TODO(), runner.ContainerID,
+ dockertypes.ContainerAttachOptions{Stream: true, Stdin: stdinUsed, Stdout: true, Stderr: true})
if err != nil {
return fmt.Errorf("While attaching container stdout/stderr streams: %v", err)
}
runner.loggingDone = make(chan bool)
if stdoutMnt, ok := runner.Container.Mounts["stdout"]; ok {
- stdoutPath := stdoutMnt.Path[len(runner.Container.OutputPath):]
- index := strings.LastIndex(stdoutPath, "/")
- if index > 0 {
- subdirs := stdoutPath[:index]
- if subdirs != "" {
- st, err := os.Stat(runner.HostOutputDir)
- if err != nil {
- return fmt.Errorf("While Stat on temp dir: %v", err)
- }
- stdoutPath := path.Join(runner.HostOutputDir, subdirs)
- err = os.MkdirAll(stdoutPath, st.Mode()|os.ModeSetgid|0777)
- if err != nil {
- return fmt.Errorf("While MkdirAll %q: %v", stdoutPath, err)
- }
- }
- }
- stdoutFile, err := os.Create(path.Join(runner.HostOutputDir, stdoutPath))
+ stdoutFile, err := runner.getStdoutFile(stdoutMnt.Path)
if err != nil {
- return fmt.Errorf("While creating stdout file: %v", err)
+ return err
}
runner.Stdout = stdoutFile
} else {
runner.Stdout = NewThrottledLogger(runner.NewLogWriter("stdout"))
}
- runner.Stderr = NewThrottledLogger(runner.NewLogWriter("stderr"))
- go runner.ProcessDockerAttach(containerReader)
+ if stderrMnt, ok := runner.Container.Mounts["stderr"]; ok {
+ stderrFile, err := runner.getStdoutFile(stderrMnt.Path)
+ if err != nil {
+ return err
+ }
+ runner.Stderr = stderrFile
+ } else {
+ runner.Stderr = NewThrottledLogger(runner.NewLogWriter("stderr"))
+ }
+
+ if stdinRdr != nil {
+ go func() {
+ _, err := io.Copy(response.Conn, stdinRdr)
+ if err != nil {
+ runner.CrunchLog.Print("While writing stdin collection to docker container %q", err)
+ runner.stop()
+ }
+ stdinRdr.Close()
+ response.CloseWrite()
+ }()
+ } else if len(stdinJson) != 0 {
+ go func() {
+ _, err := io.Copy(response.Conn, bytes.NewReader(stdinJson))
+ if err != nil {
+ runner.CrunchLog.Print("While writing stdin json to docker container %q", err)
+ runner.stop()
+ }
+ response.CloseWrite()
+ }()
+ }
+
+ go runner.ProcessDockerAttach(response.Reader)
return nil
}
+func (runner *ContainerRunner) getStdoutFile(mntPath string) (*os.File, error) {
+ stdoutPath := mntPath[len(runner.Container.OutputPath):]
+ index := strings.LastIndex(stdoutPath, "/")
+ if index > 0 {
+ subdirs := stdoutPath[:index]
+ if subdirs != "" {
+ st, err := os.Stat(runner.HostOutputDir)
+ if err != nil {
+ return nil, fmt.Errorf("While Stat on temp dir: %v", err)
+ }
+ stdoutPath := path.Join(runner.HostOutputDir, subdirs)
+ err = os.MkdirAll(stdoutPath, st.Mode()|os.ModeSetgid|0777)
+ if err != nil {
+ return nil, fmt.Errorf("While MkdirAll %q: %v", stdoutPath, err)
+ }
+ }
+ }
+ stdoutFile, err := os.Create(path.Join(runner.HostOutputDir, stdoutPath))
+ if err != nil {
+ return nil, fmt.Errorf("While creating file %q: %v", stdoutPath, err)
+ }
+
+ return stdoutFile, nil
+}
+
// CreateContainer creates the docker container.
func (runner *ContainerRunner) CreateContainer() error {
runner.CrunchLog.Print("Creating Docker container")
for k, v := range runner.Container.Environment {
runner.ContainerConfig.Env = append(runner.ContainerConfig.Env, k+"="+v)
}
+
+ runner.HostConfig = dockercontainer.HostConfig{
+ Binds: runner.Binds,
+ Cgroup: dockercontainer.CgroupSpec(runner.setCgroupParent),
+ LogConfig: dockercontainer.LogConfig{
+ Type: "none",
+ },
+ }
+
if wantAPI := runner.Container.RuntimeConstraints.API; wantAPI != nil && *wantAPI {
tok, err := runner.ContainerToken()
if err != nil {
"ARVADOS_API_HOST="+os.Getenv("ARVADOS_API_HOST"),
"ARVADOS_API_HOST_INSECURE="+os.Getenv("ARVADOS_API_HOST_INSECURE"),
)
- runner.ContainerConfig.NetworkDisabled = false
+ runner.HostConfig.NetworkMode = dockercontainer.NetworkMode(runner.networkMode)
} else {
- runner.ContainerConfig.NetworkDisabled = true
+ if runner.enableNetwork == "always" {
+ runner.HostConfig.NetworkMode = dockercontainer.NetworkMode(runner.networkMode)
+ } else {
+ runner.HostConfig.NetworkMode = dockercontainer.NetworkMode("none")
+ }
}
- var err error
- runner.ContainerID, err = runner.Docker.CreateContainer(&runner.ContainerConfig, "", nil)
+ _, stdinUsed := runner.Container.Mounts["stdin"]
+ runner.ContainerConfig.OpenStdin = stdinUsed
+ runner.ContainerConfig.StdinOnce = stdinUsed
+ runner.ContainerConfig.AttachStdin = stdinUsed
+ runner.ContainerConfig.AttachStdout = true
+ runner.ContainerConfig.AttachStderr = true
+
+ createdBody, err := runner.Docker.ContainerCreate(context.TODO(), &runner.ContainerConfig, &runner.HostConfig, nil, runner.Container.UUID)
if err != nil {
return fmt.Errorf("While creating container: %v", err)
}
- runner.HostConfig = dockerclient.HostConfig{
- Binds: runner.Binds,
- CgroupParent: runner.setCgroupParent,
- LogConfig: dockerclient.LogConfig{
- Type: "none",
- },
- }
+ runner.ContainerID = createdBody.ID
return runner.AttachStreams()
}
// StartContainer starts the docker container created by CreateContainer.
func (runner *ContainerRunner) StartContainer() error {
runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID)
- err := runner.Docker.StartContainer(runner.ContainerID, &runner.HostConfig)
+ runner.cStateLock.Lock()
+ defer runner.cStateLock.Unlock()
+ if runner.cCancelled {
+ return ErrCancelled
+ }
+ err := runner.Docker.ContainerStart(context.TODO(), runner.ContainerID,
+ dockertypes.ContainerStartOptions{})
if err != nil {
return fmt.Errorf("could not start container: %v", err)
}
+ runner.cStarted = true
return nil
}
func (runner *ContainerRunner) WaitFinish() error {
runner.CrunchLog.Print("Waiting for container to finish")
- waitDocker := runner.Docker.Wait(runner.ContainerID)
+ waitDocker, err := runner.Docker.ContainerWait(context.TODO(), runner.ContainerID)
+ if err != nil {
+ return fmt.Errorf("container wait: %v", err)
+ }
+
+ runner.CrunchLog.Printf("Container exited with code: %v", waitDocker)
+ code := int(waitDocker)
+ runner.ExitCode = &code
+
waitMount := runner.ArvMountExit
- for waitDocker != nil {
- select {
- case err := <-waitMount:
- runner.CrunchLog.Printf("arv-mount exited before container finished: %v", err)
- waitMount = nil
- runner.stop()
- case wr := <-waitDocker:
- if wr.Error != nil {
- return fmt.Errorf("While waiting for container to finish: %v", wr.Error)
- }
- runner.ExitCode = &wr.ExitCode
- waitDocker = nil
- }
+ select {
+ case err := <-waitMount:
+ runner.CrunchLog.Printf("arv-mount exited before container finished: %v", err)
+ waitMount = nil
+ runner.stop()
+ default:
}
// wait for stdout/stderr to complete
// UpdateContainerRunning updates the container state to "Running"
func (runner *ContainerRunner) UpdateContainerRunning() error {
- runner.CancelLock.Lock()
- defer runner.CancelLock.Unlock()
- if runner.Cancelled {
+ runner.cStateLock.Lock()
+ defer runner.cStateLock.Unlock()
+ if runner.cCancelled {
return ErrCancelled
}
return runner.ArvClient.Update("containers", runner.Container.UUID,
// IsCancelled returns the value of Cancelled, with goroutine safety.
func (runner *ContainerRunner) IsCancelled() bool {
- runner.CancelLock.Lock()
- defer runner.CancelLock.Unlock()
- return runner.Cancelled
+ runner.cStateLock.Lock()
+ defer runner.cStateLock.Unlock()
+ return runner.cCancelled
}
// NewArvLogWriter creates an ArvLogWriter
return
}
+ // Gather and record node information
+ err = runner.LogNodeInfo()
+ if err != nil {
+ return
+ }
+ // Save container.json record on log collection
+ err = runner.LogContainerRecord()
+ if err != nil {
+ return
+ }
+
runner.StartCrunchstat()
if runner.IsCancelled() {
cgroupParent := flag.String("cgroup-parent", "docker", "name of container's parent cgroup (ignored if -cgroup-parent-subsystem is used)")
cgroupParentSubsystem := flag.String("cgroup-parent-subsystem", "", "use current cgroup for given subsystem as parent cgroup for container")
caCertsPath := flag.String("ca-certs", "", "Path to TLS root certificates")
+ enableNetwork := flag.String("container-enable-networking", "default",
+ `Specify if networking should be enabled for container. One of 'default', 'always':
+ default: only enable networking if container requests it.
+ always: containers always have networking enabled
+ `)
+ networkMode := flag.String("container-network-mode", "default",
+ `Set networking mode for container. Corresponds to Docker network mode (--net).
+ `)
flag.Parse()
containerId := flag.Arg(0)
}
kc.Retries = 4
- var docker *dockerclient.DockerClient
- docker, err = dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
+ var docker *dockerclient.Client
+ // API version 1.21 corresponds to Docker 1.9, which is currently the
+ // minimum version we want to support.
+ docker, err = dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil)
if err != nil {
log.Fatalf("%s: %v", containerId, err)
}
- cr := NewContainerRunner(api, kc, docker, containerId)
+ dockerClientProxy := ThinDockerClientProxy{Docker: docker}
+
+ cr := NewContainerRunner(api, kc, dockerClientProxy, containerId)
cr.statInterval = *statInterval
cr.cgroupRoot = *cgroupRoot
cr.expectCgroupParent = *cgroupParent
+ cr.enableNetwork = *enableNetwork
+ cr.networkMode = *networkMode
if *cgroupParentSubsystem != "" {
p := findCgroup(*cgroupParentSubsystem)
cr.setCgroupParent = p
package main
import (
+ "bufio"
"bytes"
+ "context"
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
+ "net"
"os"
"os/exec"
"path/filepath"
+ "runtime/pprof"
"sort"
"strings"
"sync"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/sdk/go/manifest"
- "github.com/curoverse/dockerclient"
+
+ dockertypes "github.com/docker/docker/api/types"
+ dockercontainer "github.com/docker/docker/api/types/container"
+ dockernetwork "github.com/docker/docker/api/types/network"
. "gopkg.in/check.v1"
)
logReader io.ReadCloser
logWriter io.WriteCloser
fn func(t *TestDockerClient)
- finish chan dockerclient.WaitResult
+ finish int
stop chan bool
cwd string
env []string
api *ArvTestClient
}
-func NewTestDockerClient() *TestDockerClient {
+func NewTestDockerClient(exitCode int) *TestDockerClient {
t := &TestDockerClient{}
t.logReader, t.logWriter = io.Pipe()
- t.finish = make(chan dockerclient.WaitResult)
+ t.finish = exitCode
t.stop = make(chan bool)
t.cwd = "/"
return t
}
-func (t *TestDockerClient) StopContainer(id string, timeout int) error {
- t.stop <- true
- return nil
+type MockConn struct {
+ net.Conn
}
-func (t *TestDockerClient) InspectImage(id string) (*dockerclient.ImageInfo, error) {
- if t.imageLoaded == id {
- return &dockerclient.ImageInfo{}, nil
- } else {
- return nil, errors.New("")
- }
+func (m *MockConn) Write(b []byte) (int, error) {
+ return len(b), nil
}
-func (t *TestDockerClient) LoadImage(reader io.Reader) error {
- _, err := io.Copy(ioutil.Discard, reader)
- if err != nil {
- return err
- } else {
- t.imageLoaded = hwImageId
- return nil
- }
+func NewMockConn() *MockConn {
+ c := &MockConn{}
+ return c
}
-func (t *TestDockerClient) CreateContainer(config *dockerclient.ContainerConfig, name string, authConfig *dockerclient.AuthConfig) (string, error) {
+func (t *TestDockerClient) ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error) {
+ return dockertypes.HijackedResponse{Conn: NewMockConn(), Reader: bufio.NewReader(t.logReader)}, nil
+}
+
+func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig, networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error) {
if config.WorkingDir != "" {
t.cwd = config.WorkingDir
}
t.env = config.Env
- return "abcde", nil
+ return dockercontainer.ContainerCreateCreatedBody{ID: "abcde"}, nil
}
-func (t *TestDockerClient) StartContainer(id string, config *dockerclient.HostConfig) error {
- if id == "abcde" {
+func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
+ if container == "abcde" {
go t.fn(t)
return nil
} else {
}
}
-func (t *TestDockerClient) AttachContainer(id string, options *dockerclient.AttachOptions) (io.ReadCloser, error) {
- return t.logReader, nil
+func (t *TestDockerClient) ContainerStop(ctx context.Context, container string, timeout *time.Duration) error {
+ t.stop <- true
+ return nil
+}
+
+func (t *TestDockerClient) ContainerWait(ctx context.Context, container string) (int64, error) {
+ return int64(t.finish), nil
}
-func (t *TestDockerClient) Wait(id string) <-chan dockerclient.WaitResult {
- return t.finish
+func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
+ if t.imageLoaded == image {
+ return dockertypes.ImageInspect{}, nil, nil
+ } else {
+ return dockertypes.ImageInspect{}, nil, errors.New("")
+ }
+}
+
+func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
+ _, err := io.Copy(ioutil.Discard, input)
+ if err != nil {
+ return dockertypes.ImageLoadResponse{}, err
+ } else {
+ t.imageLoaded = hwImageId
+ return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil
+ }
}
-func (*TestDockerClient) RemoveImage(name string, force bool) ([]*dockerclient.ImageDelete, error) {
+func (*TestDockerClient) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) {
return nil, nil
}
}
}
+func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
+ parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
+ j := []byte(`{
+ "command": ["sleep", "1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`)
+ return ioutil.NopCloser(bytes.NewReader(j)), nil
+}
+
func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
if resourceType == "collections" {
if uuid == hwPDH {
rdr := ioutil.NopCloser(&bytes.Buffer{})
client.Called = true
return FileWrapper{rdr, 1321984}, nil
+ } else if filename == "/file1_in_main.txt" {
+ rdr := ioutil.NopCloser(strings.NewReader("foo"))
+ client.Called = true
+ return FileWrapper{rdr, 3}, nil
}
return nil, nil
}
func (s *TestSuite) TestLoadImage(c *C) {
kc := &KeepTestClient{}
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(0)
cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- _, err := cr.Docker.RemoveImage(hwImageId, true)
+ _, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
- _, err = cr.Docker.InspectImage(hwImageId)
+ _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
c.Check(err, NotNil)
cr.Container.ContainerImage = hwPDH
c.Check(err, IsNil)
defer func() {
- cr.Docker.RemoveImage(hwImageId, true)
+ cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
}()
c.Check(kc.Called, Equals, true)
c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
- _, err = cr.Docker.InspectImage(hwImageId)
+ _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
c.Check(err, IsNil)
// (2) Test using image that's already loaded
return errors.New("ArvError")
}
+func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
+ parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
+ return nil, errors.New("ArvError")
+}
+
func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
return errors.New("ArvError")
}
func (s *TestSuite) TestLoadImageKeepError(c *C) {
// (2) Keep error
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(0)
cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.Container.ContainerImage = hwPDH
func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
// (4) Collection doesn't contain image
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(0)
cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
cr.Container.ContainerImage = hwPDH
}
func (s *TestSuite) TestRunContainer(c *C) {
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(0)
docker.fn = func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, "Hello world\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{}
}
cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
api := &ArvTestClient{}
kc := &KeepTestClient{}
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
- cr.Cancelled = true
+ cr.cCancelled = true
cr.finalState = "Cancelled"
err := cr.UpdateContainerFinal()
// Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
// dress rehearsal of the Run() function, starting from a JSON container record.
-func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
+func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
rec := arvados.Container{}
err := json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(exitCode)
docker.fn = fn
- docker.RemoveImage(hwImageId, true)
+ docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
docker.api = api
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, "hello world\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
- }`, nil, func(t *TestDockerClient) {
+ }`, nil, 0, func(t *TestDockerClient) {
time.Sleep(time.Second)
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
}
+func (s *TestSuite) TestNodeInfoLog(c *C) {
+ api, _, _ := FullRunHelper(c, `{
+ "command": ["sleep", "1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`, nil, 0,
+ func(t *TestDockerClient) {
+ time.Sleep(time.Second)
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+
+ c.Assert(api.Logs["node-info"], NotNil)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
+}
+
+func (s *TestSuite) TestContainerRecordLog(c *C) {
+ api, _, _ := FullRunHelper(c, `{
+ "command": ["sleep", "1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`, nil, 0,
+ func(t *TestDockerClient) {
+ time.Sleep(time.Second)
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+
+ c.Assert(api.Logs["container"], NotNil)
+ c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
+}
+
func (s *TestSuite) TestFullRunStderr(c *C) {
api, _, _ := FullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 1, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, "hello\n"))
t.logWriter.Write(dockerLog(2, "world\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 1}
})
final := api.CalledWith("container.state", "Complete")
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
func (s *TestSuite) TestStopOnSignal(c *C) {
s.testStopContainer(c, func(cr *ContainerRunner) {
go func() {
- for cr.ContainerID == "" {
+ for !cr.cStarted {
time.Sleep(time.Millisecond)
}
cr.SigChan <- syscall.SIGINT
err := json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(0)
docker.fn = func(t *TestDockerClient) {
<-t.stop
t.logWriter.Write(dockerLog(1, "foo\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
}
- docker.RemoveImage(hwImageId, true)
+ docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api := &ArvTestClient{Container: rec}
cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
}()
select {
case <-time.After(20 * time.Second):
+ pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
c.Fatal("timed out")
case err = <-done:
c.Check(err, IsNil)
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
cr.CleanupDirs()
checkEmpty()
}
+
+ // Only mount point of kind 'collection' is allowed for stdin
+ {
+ i = 0
+ cr.ArvMountPoint = ""
+ cr.Container.Mounts = make(map[string]arvados.Mount)
+ cr.Container.Mounts = map[string]arvados.Mount{
+ "stdin": {Kind: "tmp"},
+ }
+
+ err := cr.SetupMounts()
+ c.Check(err, NotNil)
+ c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+ cr.CleanupDirs()
+ checkEmpty()
+ }
}
func (s *TestSuite) TestStdout(c *C) {
"runtime_constraints": {}
}`
- api, _, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
+ api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
err = json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
- docker := NewTestDockerClient()
+ docker := NewTestDockerClient(0)
docker.fn = fn
- docker.RemoveImage(hwImageId, true)
+ docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {"API": true}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
"output_path": "/tmp",
"priority": 1,
"runtime_constraints": {"API": true}
-}`, nil, func(t *TestDockerClient) {
+}`, nil, 0, func(t *TestDockerClient) {
t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
"a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
}
- api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
"b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
}
- api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
}
}
}
+
+func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
+ helperRecord := `{
+ "command": ["/bin/sh", "-c", "echo $FROBIZ"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": "/bin",
+ "environment": {"FROBIZ": "bilbo"},
+ "mounts": {
+ "/tmp": {"kind": "tmp"},
+ "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
+ "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`
+
+ extraMounts := []string{
+ "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
+ }
+
+ api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
+ t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ for _, v := range api.Content {
+ if v["collection"] != nil {
+ collection := v["collection"].(arvadosclient.Dict)
+ if strings.Index(collection["name"].(string), "output") == 0 {
+ manifest := collection["manifest_text"].(string)
+ c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+`)
+ }
+ }
+ }
+}
+
+func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
+ helperRecord := `{
+ "command": ["/bin/sh", "-c", "echo $FROBIZ"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": "/bin",
+ "environment": {"FROBIZ": "bilbo"},
+ "mounts": {
+ "/tmp": {"kind": "tmp"},
+ "stdin": {"kind": "json", "content": "foo"},
+ "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`
+
+ api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
+ t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
+ t.logWriter.Close()
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ for _, v := range api.Content {
+ if v["collection"] != nil {
+ collection := v["collection"].(arvadosclient.Dict)
+ if strings.Index(collection["name"].(string), "output") == 0 {
+ manifest := collection["manifest_text"].(string)
+ c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+`)
+ }
+ }
+ }
+}
+
+func (s *TestSuite) TestStderrMount(c *C) {
+ api, _, _ := FullRunHelper(c, `{
+ "command": ["/bin/sh", "-c", "echo hello;exit 1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"},
+ "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
+ "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+}`, nil, 1, func(t *TestDockerClient) {
+ t.logWriter.Write(dockerLog(1, "hello\n"))
+ t.logWriter.Write(dockerLog(2, "oops\n"))
+ t.logWriter.Close()
+ })
+
+ final := api.CalledWith("container.state", "Complete")
+ c.Assert(final, NotNil)
+ c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
+ c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
+
+ c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
+}
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/docker-cleaner/docker-cleaner.json
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=simple
import arvados.commands._util as arv_cmd
from arvados_fuse import crunchstat
from arvados_fuse import *
+from arvados_fuse.unmount import unmount
from arvados_fuse._version import __version__
class ArgumentParser(argparse.ArgumentParser):
self.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
+ unmount = self.add_mutually_exclusive_group()
+ unmount.add_argument('--unmount', action='store_true', default=False,
+ help="Forcefully unmount the specified mountpoint (if it's a fuse mount) and exit. If --subtype is given, unmount only if the mount has the specified subtype. WARNING: This command can affect any kind of fuse mount, not just arv-mount.")
+ unmount.add_argument('--unmount-all', action='store_true', default=False,
+ help="Forcefully unmount every fuse mount at or below the specified path and exit. If --subtype is given, unmount only mounts that have the specified subtype. Exit non-zero if any other types of mounts are found at or below the given path. WARNING: This command can affect any kind of fuse mount, not just arv-mount.")
+ unmount.add_argument('--replace', action='store_true', default=False,
+ help="If a fuse mount is already present at mountpoint, forcefully unmount it before mounting")
self.add_argument('--unmount-timeout',
type=float, default=2.0,
help="Time to wait for graceful shutdown after --exec program exits and filesystem is unmounted")
class Mount(object):
def __init__(self, args, logger=logging.getLogger('arvados.arv-mount')):
+ self.daemon = False
self.logger = logger
self.args = args
self.listen_for_events = False
exit(1)
def __enter__(self):
+ if self.args.replace:
+ unmount(path=self.args.mountpoint,
+ timeout=self.args.unmount_timeout)
llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+ if self.daemon:
+ daemon.DaemonContext(
+ working_directory=os.path.dirname(self.args.mountpoint),
+ files_preserve=range(
+ 3, resource.getrlimit(resource.RLIMIT_NOFILE)[1])
+ ).open()
if self.listen_for_events and not self.args.disable_event_listening:
self.operations.listen_for_events()
self.llfuse_thread = threading.Thread(None, lambda: self._llfuse_main())
self.args.unmount_timeout)
def run(self):
- if self.args.exec_args:
+ if self.args.unmount or self.args.unmount_all:
+ unmount(path=self.args.mountpoint,
+ subtype=self.args.subtype,
+ timeout=self.args.unmount_timeout,
+ recursive=self.args.unmount_all)
+ elif self.args.exec_args:
self._run_exec()
else:
self._run_standalone()
def _run_standalone(self):
try:
- llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
-
- if not self.args.foreground:
- self.daemon_ctx = daemon.DaemonContext(
- working_directory=os.path.dirname(self.args.mountpoint),
- files_preserve=range(
- 3, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))
- self.daemon_ctx.open()
-
- # Subscribe to change events from API server
- if self.listen_for_events and not self.args.disable_event_listening:
- self.operations.listen_for_events()
-
- self._llfuse_main()
+ self.daemon = not self.args.foreground
+ with self:
+ self.llfuse_thread.join(timeout=None)
except Exception as e:
self.logger.exception('arv-mount: exception during mount: %s', e)
exit(getattr(e, 'errno', 1))
--- /dev/null
+import collections
+import errno
+import os
+import subprocess
+import time
+
+
+MountInfo = collections.namedtuple(
+ 'MountInfo', ['is_fuse', 'major', 'minor', 'mnttype', 'path'])
+
+
+def mountinfo():
+ mi = []
+ with open('/proc/self/mountinfo') as f:
+ for m in f.readlines():
+ mntid, pmntid, dev, root, path, extra = m.split(" ", 5)
+ mnttype = extra.split(" - ")[1].split(" ", 1)[0]
+ major, minor = dev.split(":")
+ mi.append(MountInfo(
+ is_fuse=(mnttype == "fuse" or mnttype.startswith("fuse.")),
+ major=major,
+ minor=minor,
+ mnttype=mnttype,
+ path=path,
+ ))
+ return mi
+
+
+def unmount(path, subtype=None, timeout=10, recursive=False):
+ """Unmount the fuse mount at path.
+
+ Unmounting is done by writing 1 to the "abort" control file in
+ sysfs to kill the fuse driver process, then executing "fusermount
+ -u -z" to detach the mount point, and repeating these steps until
+ the mount is no longer listed in /proc/self/mountinfo.
+
+ This procedure should enable a non-root user to reliably unmount
+ their own fuse filesystem without risk of deadlock.
+
+ Returns True if unmounting was successful, False if it wasn't a
+ fuse mount at all. Raises an exception if it cannot be unmounted.
+ """
+
+ path = os.path.realpath(path)
+
+ if subtype is None:
+ mnttype = None
+ elif subtype == '':
+ mnttype = 'fuse'
+ else:
+ mnttype = 'fuse.' + subtype
+
+ if recursive:
+ paths = []
+ for m in mountinfo():
+ if m.path == path or m.path.startswith(path+"/"):
+ paths.append(m.path)
+ if not (m.is_fuse and (mnttype is None or
+ mnttype == m.mnttype)):
+ raise Exception(
+ "cannot unmount {}: mount type is {}".format(
+ path, m.mnttype))
+ for path in sorted(paths, key=len, reverse=True):
+ unmount(path, timeout=timeout, recursive=False)
+ return len(paths) > 0
+
+ was_mounted = False
+ attempted = False
+ if timeout is None:
+ deadline = None
+ else:
+ deadline = time.time() + timeout
+
+ while True:
+ mounted = False
+ for m in mountinfo():
+ if m.is_fuse and (mnttype is None or mnttype == m.mnttype):
+ try:
+ if os.path.realpath(m.path) == path:
+ was_mounted = True
+ mounted = True
+ break
+ except OSError:
+ continue
+ if not mounted:
+ return was_mounted
+
+ if attempted:
+ delay = 1
+ if deadline:
+ delay = min(delay, deadline - time.time())
+ if delay <= 0:
+ raise Exception("timed out")
+ time.sleep(delay)
+
+ try:
+ with open('/sys/fs/fuse/connections/{}/abort'.format(m.minor),
+ 'w') as f:
+ f.write("1")
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ attempted = True
+ try:
+ subprocess.check_call(["fusermount", "-u", "-z", path])
+ except subprocess.CalledProcessError:
+ pass
--- /dev/null
+import subprocess
+
+from integration_test import IntegrationTest
+
+
+class CrunchstatTest(IntegrationTest):
+ def test_crunchstat(self):
+ output = subprocess.check_output(
+ ['./bin/arv-mount',
+ '--crunchstat-interval', '1',
+ self.mnt,
+ '--exec', 'echo', 'ok'])
+ self.assertEqual("ok\n", output)
-import arvados
-import arvados.safeapi
-import arvados_fuse as fuse
-import glob
import json
import llfuse
+import logging
+import mock
import os
-import shutil
import subprocess
-import sys
-import tempfile
-import threading
import time
import unittest
-import logging
-import multiprocessing
+
+import arvados
+import arvados_fuse as fuse
import run_test_server
-import mock
-import re
from mount_test_base import MountTestBase
with open(path, 'w') as f:
f.write(content)
self.assertRegexpMatches(current_manifest(tmpdir), expect)
+
+ @IntegrationTest.mount(argv=mnt_args)
+ def test_tmp_rewrite(self):
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ @staticmethod
+ def _test_tmp_rewrite(self, tmpdir):
+ with open(os.path.join(tmpdir, "b1"), 'w') as f:
+ f.write("b1")
+ with open(os.path.join(tmpdir, "b2"), 'w') as f:
+ f.write("b2")
+ with open(os.path.join(tmpdir, "b1"), 'w') as f:
+ f.write("1b")
+ self.assertRegexpMatches(current_manifest(tmpdir), "^\. ed4f3f67c70b02b29c50ce1ea26666bd\+4(\+\S+)? 0:2:b1 2:2:b2\n$")
--- /dev/null
+import os
+import subprocess
+import time
+
+from integration_test import IntegrationTest
+
+class UnmountTest(IntegrationTest):
+ def setUp(self):
+ super(UnmountTest, self).setUp()
+ self.tmp = self.mnt
+ self.to_delete = []
+
+ def tearDown(self):
+ for d in self.to_delete:
+ os.rmdir(d)
+ super(UnmountTest, self).tearDown()
+
+ def test_replace(self):
+ subprocess.check_call(
+ ['./bin/arv-mount', '--subtype', 'test', '--replace',
+ self.mnt])
+ subprocess.check_call(
+ ['./bin/arv-mount', '--subtype', 'test', '--replace',
+ '--unmount-timeout', '10',
+ self.mnt])
+ subprocess.check_call(
+ ['./bin/arv-mount', '--subtype', 'test', '--replace',
+ '--unmount-timeout', '10',
+ self.mnt,
+ '--exec', 'true'])
+ for m in subprocess.check_output(['mount']).splitlines():
+ self.assertNotIn(' '+self.mnt+' ', m)
+
+ def _mounted(self, mounts):
+ all_mounts = subprocess.check_output(['mount'])
+ return [m for m in mounts
+ if ' '+m+' ' in all_mounts]
+
+ def _wait_for_mounts(self, mounts):
+ deadline = time.time() + 10
+ while self._mounted(mounts) != mounts:
+ time.sleep(0.1)
+ self.assertLess(time.time(), deadline)
+
+ def test_unmount_subtype(self):
+ mounts = []
+ for d in ['foo', 'bar']:
+ mnt = self.tmp+'/'+d
+ os.mkdir(mnt)
+ self.to_delete.insert(0, mnt)
+ mounts.append(mnt)
+ subprocess.check_call(
+ ['./bin/arv-mount', '--subtype', d, mnt])
+
+ self._wait_for_mounts(mounts)
+ self.assertEqual(mounts, self._mounted(mounts))
+ subprocess.call(['./bin/arv-mount', '--subtype', 'baz', '--unmount-all', self.tmp])
+ self.assertEqual(mounts, self._mounted(mounts))
+ subprocess.call(['./bin/arv-mount', '--subtype', 'bar', '--unmount', mounts[0]])
+ self.assertEqual(mounts, self._mounted(mounts))
+ subprocess.call(['./bin/arv-mount', '--subtype', '', '--unmount', self.tmp])
+ self.assertEqual(mounts, self._mounted(mounts))
+ subprocess.check_call(['./bin/arv-mount', '--subtype', 'foo', '--unmount', mounts[0]])
+ self.assertEqual(mounts[1:], self._mounted(mounts))
+ subprocess.check_call(['./bin/arv-mount', '--subtype', '', '--unmount-all', mounts[0]])
+ self.assertEqual(mounts[1:], self._mounted(mounts))
+ subprocess.check_call(['./bin/arv-mount', '--subtype', 'bar', '--unmount-all', self.tmp])
+ self.assertEqual([], self._mounted(mounts))
+
+ def test_unmount_children(self):
+ for d in ['foo', 'foo/bar', 'bar']:
+ mnt = self.tmp+'/'+d
+ os.mkdir(mnt)
+ self.to_delete.insert(0, mnt)
+ mounts = []
+ for d in ['bar', 'foo/bar']:
+ mnt = self.tmp+'/'+d
+ mounts.append(mnt)
+ subprocess.check_call(
+ ['./bin/arv-mount', '--subtype', 'test', mnt])
+
+ self._wait_for_mounts(mounts)
+ self.assertEqual(mounts, self._mounted(mounts))
+ subprocess.check_call(['./bin/arv-mount', '--unmount', self.tmp])
+ self.assertEqual(mounts, self._mounted(mounts))
+ subprocess.check_call(['./bin/arv-mount', '--unmount-all', self.tmp])
+ self.assertEqual([], self._mounted(mounts))
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=simple
httpserver.Log(remoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
}()
+ if r.Method == "OPTIONS" {
+ method := r.Header.Get("Access-Control-Request-Method")
+ if method != "GET" && method != "POST" {
+ statusCode = http.StatusMethodNotAllowed
+ return
+ }
+ w.Header().Set("Access-Control-Allow-Headers", "Range")
+ w.Header().Set("Access-Control-Allow-Methods", "GET, POST")
+ w.Header().Set("Access-Control-Allow-Origin", "*")
+ w.Header().Set("Access-Control-Max-Age", "86400")
+ statusCode = http.StatusOK
+ return
+ }
+
if r.Method != "GET" && r.Method != "POST" {
statusCode, statusText = http.StatusMethodNotAllowed, r.Method
return
} else if len(pathParts) >= 3 && pathParts[0] == "collections" {
if len(pathParts) >= 5 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
- targetID = pathParts[2]
+ targetID = parseCollectionIDFromURL(pathParts[2])
tokens = []string{pathParts[3]}
targetPath = pathParts[4:]
pathToken = true
} else {
// /collections/ID/PATH...
- targetID = pathParts[1]
+ targetID = parseCollectionIDFromURL(pathParts[1])
tokens = h.Config.AnonymousTokens
targetPath = pathParts[2:]
}
- } else {
+ }
+
+ if targetID == "" {
statusCode = http.StatusNotFound
return
}
type UnitSuite struct{}
+func (s *UnitSuite) TestCORSPreflight(c *check.C) {
+ h := handler{Config: &Config{}}
+ u, _ := url.Parse("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
+ req := &http.Request{
+ Method: "OPTIONS",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Origin": {"https://workbench.example"},
+ "Access-Control-Request-Method": {"POST"},
+ },
+ }
+
+ // Check preflight for an allowed request
+ resp := httptest.NewRecorder()
+ h.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
+ c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "GET, POST")
+ c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Range")
+
+ // Check preflight for a disallowed request
+ resp = httptest.NewRecorder()
+ req.Header.Set("Access-Control-Request-Method", "DELETE")
+ h.ServeHTTP(resp, req)
+ c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
+}
+
+func (s *UnitSuite) TestInvalidUUID(c *check.C) {
+ bogusID := strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "-"
+ token := arvadostest.ActiveToken
+ for _, trial := range []string{
+ "http://keep-web/c=" + bogusID + "/foo",
+ "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
+ "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
+ "http://keep-web/collections/" + bogusID + "/foo",
+ "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
+ "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
+ } {
+ c.Log(trial)
+ u, err := url.Parse(trial)
+ c.Assert(err, check.IsNil)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ }
+ resp := httptest.NewRecorder()
+ h := handler{Config: &Config{
+ AnonymousTokens: []string{arvadostest.AnonymousToken},
+ }}
+ h.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusNotFound)
+ }
+}
+
func mustParseURL(s string) *url.URL {
r, err := url.Parse(s)
if err != nil {
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/keep-web/keep-web.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=notify
ExecStart=/usr/bin/keep-web
Restart=always
+RestartSec=1
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/keepproxy/keepproxy.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=notify
ExecStart=/usr/bin/keepproxy
Restart=always
+RestartSec=1
[Install]
WantedBy=multi-user.target
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/keepstore/keepstore.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=notify
ExecStart=/usr/bin/keepstore
Restart=always
+RestartSec=1
[Install]
WantedBy=multi-user.target
self.cloud_size.name)
self.cloud_node = self._cloud.create_node(self.cloud_size,
self.arvados_node)
- if not self.cloud_node.size:
- self.cloud_node.size = self.cloud_size
+
+ # The information included in the node size object we get from libcloud
+ # is inconsistent between cloud providers. Replace libcloud NodeSize
+ # object with compatible CloudSizeWrapper object which merges the size
+ # info reported from the cloud with size information from the
+ # configuration file.
+ self.cloud_node.size = self.cloud_size
+
self._logger.info("Cloud node %s created.", self.cloud_node.id)
self._later.update_arvados_node_properties()
if self.arvados_node is None:
return 'unpaired'
- # This node is indicated as non-functioning by the cloud
- if self._cloud.broken(self.cloud_node):
- return 'down'
-
state = self.arvados_node['crunch_worker_state']
# If state information is not available because it is missing or the
return (isinstance(exception, cls.CLOUD_ERRORS) or
type(exception) is Exception)
+ def destroy_node(self, cloud_node):
+ try:
+ return self.real.destroy_node(cloud_node)
+ except self.CLOUD_ERRORS as destroy_error:
+ # Sometimes the destroy node request succeeds but times out and
+ # raises an exception instead of returning success. If this
+ # happens, we get a noisy stack trace. Check if the node is still
+ # on the node list. If it is gone, we can declare victory.
+ try:
+ self.search_for_now(cloud_node.id, 'list_nodes')
+ except ValueError:
+ # If we catch ValueError, that means search_for_now didn't find
+ # it, which means destroy_node actually succeeded.
+ return True
+ # The node is still on the list. Re-raise.
+ raise
+
# Now that we've defined all our own methods, delegate generic, public
# attributes of libcloud drivers that we haven't defined ourselves.
def _delegate_to_real(attr_name):
raise
def sync_node(self, cloud_node, arvados_node):
+ # Update the cloud node record to ensure we have the correct metadata
+ # fingerprint.
+ cloud_node = self.real.ex_get_node(cloud_node.name, cloud_node.extra['zone'])
+
# We can't store the FQDN on the name attribute or anything like it,
# because (a) names are static throughout the node's life (so FQDN
# isn't available because we don't know it at node creation time) and
self._find_metadata(metadata_items, 'hostname')['value'] = hostname
except KeyError:
metadata_items.append({'key': 'hostname', 'value': hostname})
- response = self.real.connection.async_request(
- '/zones/{}/instances/{}/setMetadata'.format(
- cloud_node.extra['zone'].name, cloud_node.name),
- method='POST', data=metadata_req)
- if not response.success():
- raise Exception("setMetadata error: {}".format(response.error))
+
+ self.real.ex_set_node_metadata(cloud_node, metadata_items)
@classmethod
def node_fqdn(cls, node):
'node_stale_after': str(60 * 60 * 2),
'watchdog': '600',
'node_mem_scaling': '0.95'},
+ 'Manage': {'address': '127.0.0.1',
+ 'port': '-1'},
'Logging': {'file': '/dev/stderr',
'level': 'WARNING'},
}.iteritems():
import pykka
from . import computenode as cnode
+from . import status
from .computenode import dispatch
from .config import actor_class
def try_pairing(self):
for record in self.cloud_nodes.unpaired():
for arv_rec in self.arvados_nodes.unpaired():
- if record.actor.offer_arvados_pair(arv_rec.arvados_node).get():
+ if record.actor is not None and record.actor.offer_arvados_pair(arv_rec.arvados_node).get():
self._pair_nodes(record, arv_rec.arvados_node)
break
states.append("shutdown")
return states + pykka.get_all(proxy_states)
+ def _update_tracker(self):
+ updates = {
+ k: 0
+ for k in status.tracker.keys()
+ if k.startswith('nodes_')
+ }
+ for s in self._node_states(size=None):
+ updates.setdefault('nodes_'+s, 0)
+ updates['nodes_'+s] += 1
+ updates['nodes_wish'] = len(self.last_wishlist)
+ status.tracker.update(updates)
+
def _state_counts(self, size):
states = self._node_states(size)
counts = {
elif (nodes_wanted < 0) and self.booting:
self._later.stop_booting_node(size)
except Exception as e:
- self._logger.exception("while calculating nodes wanted for size %s", size)
+ self._logger.exception("while calculating nodes wanted for size %s", getattr(size, "id", "(id not available)"))
+ try:
+ self._update_tracker()
+ except:
+ self._logger.exception("while updating tracker")
def _check_poll_freshness(orig_func):
"""Decorator to inhibit a method when poll information is stale.
@_check_poll_freshness
def node_can_shutdown(self, node_actor):
- if self._nodes_excess(node_actor.cloud_node.get().size) > 0:
- self._begin_node_shutdown(node_actor, cancellable=True)
- elif self.cloud_nodes.nodes.get(node_actor.cloud_node.get().id).arvados_node is None:
- # Node is unpaired, which means it probably exceeded its booting
- # grace period without a ping, so shut it down so we can boot a new
- # node in its place.
- self._begin_node_shutdown(node_actor, cancellable=False)
- elif node_actor.in_state('down').get():
- # Node is down and unlikely to come back.
- self._begin_node_shutdown(node_actor, cancellable=False)
+ try:
+ if self._nodes_excess(node_actor.cloud_node.get().size) > 0:
+ self._begin_node_shutdown(node_actor, cancellable=True)
+ elif self.cloud_nodes.nodes.get(node_actor.cloud_node.get().id).arvados_node is None:
+ # Node is unpaired, which means it probably exceeded its booting
+ # grace period without a ping, so shut it down so we can boot a new
+ # node in its place.
+ self._begin_node_shutdown(node_actor, cancellable=False)
+ elif node_actor.in_state('down').get():
+ # Node is down and unlikely to come back.
+ self._begin_node_shutdown(node_actor, cancellable=False)
+ except pykka.ActorDeadError as e:
+ # The monitor actor sends shutdown suggestions every time the
+ # node's state is updated, and these go into the daemon actor's
+ # message queue. It's possible that the node has already been shut
+ # down (which shuts down the node monitor actor). In that case,
+ # this message is stale and we'll get ActorDeadError when we try to
+ # access node_actor. Log the error.
+ self._logger.debug("ActorDeadError in node_can_shutdown: %s", e)
def node_finished_shutdown(self, shutdown_actor):
try:
import libcloud
from . import config as nmconfig
+from . import status
from .baseactor import WatchdogActor
from .daemon import NodeManagerDaemonActor
from .jobqueue import JobQueueMonitorActor, ServerCalculator
for sigcode in [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]:
signal.signal(sigcode, shutdown_signal)
+ status.Server(config).start()
+
try:
root_logger = setup_logging(config.get('Logging', 'file'), **config.log_levels())
root_logger.info("%s %s, libcloud %s", sys.argv[0], __version__, libcloud.__version__)
--- /dev/null
+from __future__ import absolute_import, print_function
+from future import standard_library
+
+import http.server
+import json
+import logging
+import socketserver
+import threading
+
+_logger = logging.getLogger('status.Handler')
+
+
+class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
+ def __init__(self, config):
+ port = config.getint('Manage', 'port')
+ self.enabled = port >= 0
+ if not self.enabled:
+ _logger.warning("Management server disabled. "+
+ "Use [Manage] config section to enable.")
+ return
+ self._config = config
+ self._tracker = tracker
+ super(Server, self).__init__(
+ (config.get('Manage', 'address'), port), Handler)
+ self._thread = threading.Thread(target=self.serve_forever)
+ self._thread.daemon = True
+
+ def start(self):
+ if self.enabled:
+ self._thread.start()
+
+
+class Handler(http.server.BaseHTTPRequestHandler, object):
+ def do_GET(self):
+ if self.path == '/status.json':
+ self.send_response(200)
+ self.send_header('Content-type', 'application/json')
+ self.end_headers()
+ self.wfile.write(tracker.get_json())
+ else:
+ self.send_response(404)
+
+ def log_message(self, fmt, *args, **kwargs):
+ _logger.info(fmt, *args, **kwargs)
+
+
+class Tracker(object):
+ def __init__(self):
+ self._mtx = threading.Lock()
+ self._latest = {}
+
+ def get_json(self):
+ with self._mtx:
+ return json.dumps(self._latest)
+
+ def keys(self):
+ with self._mtx:
+ return self._latest.keys()
+
+ def update(self, updates):
+ with self._mtx:
+ self._latest.update(updates)
+
+
+tracker = Tracker()
# Azure configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
[Daemon]
# The dispatcher can customize the start and stop procedure for
# cloud nodes. For example, the SLURM dispatcher drains nodes
# EC2 configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
[Daemon]
# The dispatcher can customize the start and stop procedure for
# cloud nodes. For example, the SLURM dispatcher drains nodes
# Google Compute Engine configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.
+[Manage]
+# The management server responds to http://addr:port/status.json with
+# a snapshot of internal state.
+
+# Management server listening address (default 127.0.0.1)
+#address = 0.0.0.0
+
+# Management server port number (default -1, server is disabled)
+#port = 8989
+
[Daemon]
# Node Manager will ensure that there are at least this many nodes running at
# all times. If node manager needs to start new idle nodes for the purpose of
# is through the API server Rails console: load the Node object, set its
# IP address to 10.10.0.N (where N is the cloud node's ID), and save.
+[Manage]
+address = 0.0.0.0
+port = 8989
+
[Daemon]
min_nodes = 0
max_nodes = 8
('share/doc/arvados-node-manager', ['agpl-3.0.txt', 'README.rst']),
],
install_requires=[
- 'apache-libcloud>=0.16',
- 'arvados-python-client>=0.1.20150206225333',
- 'pykka',
- 'python-daemon',
- 'setuptools'
- ],
- dependency_links = [
+ 'apache-libcloud>=0.16',
+ 'arvados-python-client>=0.1.20150206225333',
+ 'future',
+ 'pykka',
+ 'python-daemon',
+ 'setuptools'
+ ],
+ dependency_links=[
"https://github.com/curoverse/libcloud/archive/apache-libcloud-0.18.1.dev4.zip"
],
test_suite='tests',
- tests_require=['pbr<1.7.0', 'mock>=1.0', "apache-libcloud==0.18.1.dev4"],
+ tests_require=[
+ 'requests',
+ 'pbr<1.7.0',
+ 'mock>=1.0',
+ 'apache-libcloud==0.18.1.dev4',
+ ],
zip_safe=False,
cmdclass={'egg_info': tagger},
)
cloud_node = testutil.cloud_node_mock(
2, metadata=start_metadata.copy(),
zone=testutil.cloud_object_mock('testzone'))
+ self.driver_mock().ex_get_node.return_value = cloud_node
driver = self.new_driver()
driver.sync_node(cloud_node, arv_node)
- args, kwargs = self.driver_mock().connection.async_request.call_args
- self.assertEqual('/zones/testzone/instances/2/setMetadata', args[0])
- for key in ['kind', 'fingerprint']:
- self.assertEqual(start_metadata[key], kwargs['data'][key])
+ args, kwargs = self.driver_mock().ex_set_node_metadata.call_args
+ self.assertEqual(cloud_node, args[0])
plain_metadata['hostname'] = 'compute1.zzzzz.arvadosapi.com'
self.assertEqual(
plain_metadata,
- {item['key']: item['value'] for item in kwargs['data']['items']})
+ {item['key']: item['value'] for item in args[1]})
def test_sync_node_updates_hostname_tag(self):
self.check_sync_node_updates_hostname_tag(
arv_node = testutil.arvados_node_mock(8)
cloud_node = testutil.cloud_node_mock(
9, metadata={}, zone=testutil.cloud_object_mock('failzone'))
- mock_response = self.driver_mock().connection.async_request()
- mock_response.success.return_value = False
- mock_response.error = 'sync error test'
+ mock_response = self.driver_mock().ex_set_node_metadata.side_effect = (Exception('sync error test'),)
driver = self.new_driver()
with self.assertRaises(Exception) as err_check:
driver.sync_node(cloud_node, arv_node)
import pykka
import arvnodeman.daemon as nmdaemon
+import arvnodeman.status as status
from arvnodeman.jobqueue import ServerCalculator
from arvnodeman.computenode.dispatch import ComputeNodeMonitorActor
from . import testutil
+from . import test_status
import logging
class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
monitor = self.monitor_list()[0].proxy()
self.daemon.update_server_wishlist([])
self.daemon.node_can_shutdown(monitor).get(self.TIMEOUT)
+ self.daemon.update_server_wishlist([]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
self.assertTrue(self.node_shutdown.start.called,
"daemon did not shut down booted node on offer")
+ with test_status.TestServer() as srv:
+ self.assertEqual(0, srv.get_status().get('nodes_unpaired', None))
+ self.assertEqual(1, srv.get_status().get('nodes_shutdown', None))
+ self.assertEqual(0, srv.get_status().get('nodes_wish', None))
+
def test_booted_node_lifecycle(self):
cloud_node = testutil.cloud_node_mock(6)
setup = self.start_node_boot(cloud_node, id_num=6)
self.daemon.node_finished_shutdown(self.last_shutdown).get(self.TIMEOUT)
self.assertEqual(0, self.alive_monitor_count())
- def test_broken_node_not_counted(self):
- size = testutil.MockSize(8)
- cloud_node = testutil.cloud_node_mock(8, size=size)
- wishlist = [size]
- self.make_daemon([cloud_node], [testutil.arvados_node_mock(8)],
- wishlist, avail_sizes=[(size, {"cores":1})])
- self.assertEqual(1, self.alive_monitor_count())
- self.assertFalse(self.node_setup.start.called)
- monitor = self.monitor_list()[0].proxy()
- shutdown_proxy = self.node_shutdown.start().proxy
- shutdown_proxy().cloud_node.get.return_value = cloud_node
- shutdown_proxy().success.get.return_value = False
- self.cloud_factory().broken.return_value = True
- self.daemon.update_server_wishlist([]).get(self.TIMEOUT)
- self.daemon.node_can_shutdown(monitor).get(self.TIMEOUT)
- self.daemon.node_finished_shutdown(shutdown_proxy()).get(self.TIMEOUT)
- self.daemon.update_cloud_nodes([cloud_node]).get(self.TIMEOUT)
- self.daemon.update_server_wishlist(wishlist).get(self.TIMEOUT)
- self.stop_proxy(self.daemon)
- self.assertEqual(1, self.node_setup.start.call_count)
-
def test_nodes_shutting_down_replaced_below_max_nodes(self):
size = testutil.MockSize(6)
cloud_node = testutil.cloud_node_mock(6, size=size)
def ping(self):
# Called by WatchdogActorTest, this delay is longer than the test timeout
# of 1 second, which should cause the watchdog ping to fail.
- time.sleep(2)
+ time.sleep(4)
return True
class ActorUnhandledExceptionTest(testutil.ActorTestMixin, unittest.TestCase):
--- /dev/null
+#!/usr/bin/env python
+
+from __future__ import absolute_import, print_function
+from future import standard_library
+
+import requests
+import unittest
+
+import arvnodeman.status as status
+import arvnodeman.config as config
+
+
+class TestServer(object):
+ def __enter__(self):
+ cfg = config.NodeManagerConfig()
+ cfg.set('Manage', 'port', '0')
+ cfg.set('Manage', 'address', '127.0.0.1')
+ self.srv = status.Server(cfg)
+ self.srv.start()
+ addr, port = self.srv.server_address
+ self.srv_base = 'http://127.0.0.1:'+str(port)
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.srv.shutdown()
+
+ def get_status_response(self):
+ return requests.get(self.srv_base+'/status.json')
+
+ def get_status(self):
+ return self.get_status_response().json()
+
+
+class StatusServerUpdates(unittest.TestCase):
+ def test_updates(self):
+ with TestServer() as srv:
+ for n in [1, 2, 3]:
+ status.tracker.update({'nodes_'+str(n): n})
+ r = srv.get_status_response()
+ self.assertEqual(200, r.status_code)
+ self.assertEqual('application/json', r.headers['content-type'])
+ resp = r.json()
+ self.assertEqual(n, resp['nodes_'+str(n)])
+ self.assertEqual(1, resp['nodes_1'])
+
+
+class StatusServerDisabled(unittest.TestCase):
+ def test_config_disabled(self):
+ cfg = config.NodeManagerConfig()
+ cfg.set('Manage', 'port', '-1')
+ cfg.set('Manage', 'address', '127.0.0.1')
+ self.srv = status.Server(cfg)
+ self.srv.start()
+ self.assertFalse(self.srv.enabled)
+ self.assertFalse(getattr(self.srv, '_thread', False))
Documentation=https://doc.arvados.org/
After=network.target
AssertPathExists=/etc/arvados/ws/ws.yml
+# systemd<230
+StartLimitInterval=0
+# systemd>=230
+StartLimitIntervalSec=0
[Service]
Type=notify
ExecStart=/usr/bin/arvados-ws
Restart=always
+RestartSec=1
[Install]
WantedBy=multi-user.target
projects / arvados.git / commitdiff