+// filterable.js shows/hides content when the user operates
+// search/select widgets. For "infinite scroll" content, it passes the
+// filters to the server and retrieves new content. For other content,
+// it filters the existing DOM elements using jQuery show/hide.
+//
+// Usage:
+//
+// 1. Add the "filterable" class to each filterable content item.
+// Typically, each item is a 'tr' or a 'div class="row"'.
+//
+// <div id="results">
+// <div class="filterable row">First row</div>
+// <div class="filterable row">Second row</div>
+// </div>
+//
+// 2. Add the "filterable-control" class to each search/select widget.
+// Also add a data-filterable-target attribute with a jQuery selector
+// for an ancestor of the filterable items, i.e., the container in
+// which this widget should apply filtering.
+//
+// <input class="filterable-control" data-filterable-target="#results"
+// type="text" />
+//
+// Supported widgets:
+//
+// <input type="text" ... />
+//
+// The input value is used as a regular expression. Rows with content
+// matching the regular expression are shown.
+//
+// <select ... data-filterable-attribute="data-example-attr">
+// <option value="foo">Foo</option>
+// <option value="">Show all</option>
+// </select>
+//
+// When the user selects the "Foo" option, rows with
+// data-example-attr="foo" are shown, and all others are hidden. When
+// the user selects the "Show all" option, all rows are shown.
+//
+// Notes:
+//
+// When multiple filterable-control widgets operate on the same
+// data-filterable-target, items must pass _all_ filters in order to
+// be shown.
+//
+// If one data-filterable-target is the parent of another
+// data-filterable-target, results are undefined. Don't do this.
+//
+// Combining "select" filterable-controls with infinite-scroll is not
+// yet supported.
+
$(document).
on('paste keyup input', 'input[type=text].filterable-control', function() {
var $target = $($(this).attr('data-filterable-target'));
usable_token = find_usable_token(tokens) do
coll = Collection.find(params[:uuid])
end
+
+ file_name = params[:file].andand.sub(/^(\.\/|\/|)/, './')
if usable_token.nil?
return # Response already rendered.
- elsif params[:file].nil? or not coll.manifest.has_file?(params[:file])
+ elsif file_name.nil? or not coll.manifest.has_file?(file_name)
return render_not_found
end
if component[:script_parameters]
component[:script_parameters].each do |param, value_info|
if value_info.is_a? Hash
- if resource_class_for_uuid(value_info[:value]) == Link
+ value_info_class = resource_class_for_uuid(value_info[:value])
+ if value_info_class == Link
# Use the link target, not the link itself, as script
# parameter; but keep the link info around as well.
link = Link.find value_info[:value]
value_info[:link_uuid] = nil
value_info[:link_name] = nil
end
+ if value_info_class == Collection
+ # to ensure reproducibility, the script_parameter for a
+ # collection should be the portable_data_hash
+ # keep the collection name and uuid for human-readability
+ obj = Collection.find value_info[:value]
+ value_info[:value] = obj.portable_data_hash
+ value_info[:selection_uuid] = obj.uuid
+ value_info[:selection_name] = obj.name
+ end
end
end
end
# page, and use the last item on this page as a filter for
# retrieving the next page. Ideally the API would do this for
# us, but it doesn't (yet).
- nextpage_operator = /\bdesc$/i =~ @order[0] ? '<' : '>'
+
+ # To avoid losing items that have the same created_at as the
+ # last item on this page, we retrieve an overlapping page with a
+ # "created_at <= last_created_at" filter, then remove duplicates
+ # with a "uuid not in [...]" filter (see below).
+ nextpage_operator = /\bdesc$/i =~ @order[0] ? '<=' : '>='
+
@objects = []
@name_link_for = {}
kind_filters.each do |attr,op,val|
end
end
@objects = @objects.to_a.sort_by(&:created_at)
- @objects.reverse! if nextpage_operator == '<'
+ @objects.reverse! if nextpage_operator == '<='
@objects = @objects[0..@limit-1]
@next_page_filters = @filters.reject do |attr,op,val|
- attr == 'created_at' and op == nextpage_operator
+ (attr == 'created_at' and op == nextpage_operator) or
+ (attr == 'uuid' and op == 'not in')
end
+
if @objects.any?
+ last_created_at = @objects.last.created_at
+
+ last_uuids = []
+ @objects.each do |obj|
+ last_uuids << obj.uuid if obj.created_at.eql?(last_created_at)
+ end
+
@next_page_filters += [['created_at',
nextpage_operator,
- @objects.last.created_at]]
+ last_created_at]]
+ @next_page_filters += [['uuid', 'not in', last_uuids]]
@next_page_href = url_for(partial: :contents_rows,
+ limit: @limit,
filters: @next_page_filters.to_json)
else
@next_page_href = nil
include_linked: true,
filters: @filters,
offset: @offset)
- @next_page_href = next_page_href(partial: :contents_rows)
+ @next_page_href = next_page_href(partial: :contents_rows, filters: @filters.to_json)
end
preload_links_for_objects(@objects.to_a)
display_value = link.name
elsif value_info[:link_name]
display_value = value_info[:link_name]
+ elsif value_info[:selection_name]
+ display_value = value_info[:selection_name]
end
end
if (attr == :components) and (subattr.size > 2)
end
def attribute_editable?(name, ever=nil)
- (ever or %w(New Ready).include?(state)) and super
+ if name.to_s == "components"
+ (ever or %w(New Ready).include?(state)) and super
+ else
+ super
+ end
end
def attributes_for_display
</ul>
</div>
</div>
+ <div class="pull-right col-lg-3">
+ <%= form_tag collection_path(@object.uuid), {method: 'get'} do %>
+ <div class="input-group">
+ <input class="form-control" id="file_regex" name="file_regex" placeholder="regular expression" value="<%= params[:file_regex] %>" type="text"/>
+ <span class="input-group-btn">
+ <button id="file_regex_submit" type="submit" class="btn btn-primary" autofocus>Filter</button>
+ </span>
+ </div>
+ <% end %>
+ </div>
</div>
<p/>
<% end %>
+<% file_regex = nil %>
+<% if params[:file_regex] %>
+ <% begin %>
+ <% file_regex = Regexp.new(params[:file_regex]) %>
+ <% rescue RegexpError %>
+ <% # If the pattern is not a valid regex, quote it %>
+ <% # (i.e. use it as a simple substring search) %>
+ <div class="alert alert-info">
+ <p>The search term <code><%= params[:file_regex] %></code> could not be parsed as a regular expression.</p>
+ <p>Searching for files named <code><%= params[:file_regex] %></code> instead.</p>
+ </div>
+ <% file_regex = Regexp.new(Regexp.quote(params[:file_regex])) %>
+ <% end %>
+<% end %>
+
<% file_tree = @object.andand.files_tree %>
<% if file_tree.nil? or file_tree.empty? %>
<p>This collection is empty.</p>
<% else %>
<ul id="collection_files" class="collection_files">
<% dirstack = [file_tree.first.first] %>
- <% file_tree.take(10000).each_with_index do |(dirname, filename, size), index| %>
+ <% file_tree.reject { |(dirname, filename, size)|
+ # Eliminate any files that don't match file_regex
+ # (or accept all files if no file_regex was given)
+ size and file_regex and !file_regex.match(filename)
+ }
+ .take(10000)
+ .each_with_index do |(dirname, filename, size), index| %>
<% file_path = CollectionsHelper::file_path([dirname, filename]) %>
<% while dirstack.any? and (dirstack.last != dirname) %>
<% dirstack.pop %></ul></li>
This job is next in the queue to run.
<% elsif current_job[:queue_position] == 1 %>
There is 1 job in the queue ahead of this one.
- <% else %>
+ <% elsif current_job[:queue_position] %>
There are <%= current_job[:queue_position] %> jobs in the queue ahead of this one.
<% end %>
<% rescue %>
<%= render_pane 'tab_contents', to_string: true, locals: {
+ limit: 50,
filters: [['uuid', 'is_a', ["arvados#job", "arvados#pipelineInstance"]]]
}.merge(local_assigns) %>
<col width="60%" style="width: 60%;" />
<col width="40%" style="width: 40%;" />
</colgroup>
- <tbody data-infinite-scroller="#<%= tab_pane %>-scroll" data-infinite-content-href="<%= url_for partial: :contents_rows %>" data-infinite-content-params-projecttab="<%= {filters: filters}.to_json %>">
+ <tbody data-infinite-scroller="#<%= tab_pane %>-scroll" data-infinite-content-href="<%= url_for partial: :contents_rows %>" data-infinite-content-params-projecttab="<%= local_assigns.to_json %>">
</tbody>
<thead>
<tr>
arvados_login_base: https://arvados.local:3030/login
arvados_v1_base: https://arvados.local:3030/arvados/v1
arvados_insecure_https: true
+
+production:
+ # At minimum, you need a nice long randomly generated secret_token here.
+ secret_token: ~
+
+ # You probably also want to point to your API server.
+ arvados_login_base: https://arvados.local:3030/login
+ arvados_v1_base: https://arvados.local:3030/arvados/v1
+ arvados_insecure_https: false
config.serve_static_assets = false
# Compress JavaScripts and CSS
- config.assets.js_compressor = :yui
+ config.assets.js_compressor = :uglifier
# Don't fallback to assets pipeline if a precompiled asset is missed
config.assets.compile = false
# runs.
@response.body.length
end
+
+ test "show file in a subdirectory of a collection" do
+ params = collection_params(:collection_with_files_in_subdir, 'subdir2/subdir3/subdir4/file1_in_subdir4.txt')
+ expect_content = stub_file_content
+ get(:show_file, params, session_for(:user1_with_load))
+ assert_response :success
+ assert_equal(expect_content, @response.body, "failed to get a correct file from Keep")
+ end
end
assert page.has_no_text?("Activity")
assert page.has_no_text?("Sharing and permissions")
end
+
+ test "Filtering collection files by regexp" do
+ col = api_fixture('collections', 'multilevel_collection_1')
+ visit page_with_token('active', "/collections/#{col['uuid']}")
+
+ # Test when only some files match the regex
+ page.find_field('file_regex').set('file[12]')
+ find('button#file_regex_submit').click
+ assert page.has_text?("file1")
+ assert page.has_text?("file2")
+ assert page.has_no_text?("file3")
+
+ # Test all files matching the regex
+ page.find_field('file_regex').set('file[123]')
+ find('button#file_regex_submit').click
+ assert page.has_text?("file1")
+ assert page.has_text?("file2")
+ assert page.has_text?("file3")
+
+ # Test no files matching the regex
+ page.find_field('file_regex').set('file9')
+ find('button#file_regex_submit').click
+ assert page.has_no_text?("file1")
+ assert page.has_no_text?("file2")
+ assert page.has_no_text?("file3")
+ # make sure that we actually are looking at the collections
+ # page and not e.g. a fiddlesticks
+ assert page.has_text?("multilevel_collection_1")
+ assert page.has_text?(col['portable_data_hash'])
+
+ # Syntactically invalid regex
+ # Page loads, but does not match any files
+ page.find_field('file_regex').set('file[2')
+ find('button#file_regex_submit').click
+ assert page.has_text?('could not be parsed as a regular expression')
+ assert page.has_no_text?("file1")
+ assert page.has_no_text?("file2")
+ assert page.has_no_text?("file3")
+ end
end
end
wait_for_ajax
+ # Ensure that the collection's portable_data_hash, uuid and name
+ # are saved in the desired places. (#4015)
+
+ # foo_collection_in_aproject is the collection tagged with foo_tag.
+ col = api_fixture('collections', 'foo_collection_in_aproject')
+ click_link 'Advanced'
+ click_link 'API response'
+ api_response = JSON.parse(find('div#advanced_api_response pre').text)
+ input_params = api_response['components']['part-one']['script_parameters']['input']
+ assert_equal input_params['value'], col['portable_data_hash']
+ assert_equal input_params['selection_name'], col['name']
+ assert_equal input_params['selection_uuid'], col['uuid']
+
# "Run" button is now enabled
page.assert_no_selector 'a.disabled,button.disabled', text: 'Run'
create_and_run_pipeline_in_aproject true
end
- # Create a pipeline instance from within a project and run
+ # Create a pipeline instance from outside of a project
test 'Run a pipeline from dashboard' do
visit page_with_token('active_trustedclient')
create_and_run_pipeline_in_aproject false
end
wait_for_ajax
+ # Ensure that the collection's portable_data_hash, uuid and name
+ # are saved in the desired places. (#4015)
+
+ # foo_collection_in_aproject is the collection tagged with foo_tag.
+ col = api_fixture('collections', 'foo_collection_in_aproject')
+ click_link 'Advanced'
+ click_link 'API response'
+ api_response = JSON.parse(find('div#advanced_api_response pre').text)
+ input_params = api_response['components']['part-one']['script_parameters']['input']
+ assert_equal input_params['value'], col['portable_data_hash']
+ assert_equal input_params['selection_name'], col['name']
+ assert_equal input_params['selection_uuid'], col['uuid']
+
# "Run" button present and enabled
page.assert_no_selector 'a.disabled,button.disabled', text: 'Run'
first('a,button', text: 'Run').click
[
['project with 10 pipelines', 10, 0],
-# ['project with 200 jobs and 10 pipelines', 2, 200],
+ ['project with 2 pipelines and 60 jobs', 2, 60],
['project with 25 pipelines', 25, 0],
].each do |project_name, num_pipelines, num_jobs|
test "scroll pipeline instances tab for #{project_name} with #{num_pipelines} pipelines and #{num_jobs} jobs" do
require 'test_helper'
class PipelineInstanceTest < ActiveSupport::TestCase
+ def attribute_editable_for?(token_name, pi_name, attr_name, ever=nil)
+ use_token token_name
+ find_fixture(PipelineInstance, pi_name).attribute_editable?(attr_name, ever)
+ end
+
test "admin can edit name" do
- use_token :admin
- assert(find_fixture(PipelineInstance, "new_pipeline_in_subproject")
- .attribute_editable?("name"),
+ assert(attribute_editable_for?(:admin, "new_pipeline_in_subproject",
+ "name"),
"admin not allowed to edit pipeline instance name")
end
test "project owner can edit name" do
- use_token :active
- assert(find_fixture(PipelineInstance, "new_pipeline_in_subproject")
- .attribute_editable?("name"),
+ assert(attribute_editable_for?(:active, "new_pipeline_in_subproject",
+ "name"),
"project owner not allowed to edit pipeline instance name")
end
test "project admin can edit name" do
- use_token :subproject_admin
- assert(find_fixture(PipelineInstance, "new_pipeline_in_subproject")
- .attribute_editable?("name"),
+ assert(attribute_editable_for?(:subproject_admin,
+ "new_pipeline_in_subproject", "name"),
"project admin not allowed to edit pipeline instance name")
end
test "project viewer cannot edit name" do
- use_token :project_viewer
- refute(find_fixture(PipelineInstance, "new_pipeline_in_subproject")
- .attribute_editable?("name"),
+ refute(attribute_editable_for?(:project_viewer,
+ "new_pipeline_in_subproject", "name"),
"project viewer allowed to edit pipeline instance name")
end
+
+ test "name editable on completed pipeline" do
+ assert(attribute_editable_for?(:active, "has_component_with_completed_jobs",
+ "name"),
+ "name not editable on complete pipeline")
+ end
+
+ test "components editable on new pipeline" do
+ assert(attribute_editable_for?(:active, "new_pipeline", "components"),
+ "components not editable on new pipeline")
+ end
+
+ test "components not editable on completed pipeline" do
+ refute(attribute_editable_for?(:active, "has_component_with_completed_jobs",
+ "components"),
+ "components not editable on new pipeline")
+ end
end
- Docker:
- install/install-docker.html.textile.liquid
- Manual installation:
- - install/install-keep.html.textile.liquid
- - install/install-sso.html.textile.liquid
+ - install/install-manual-overview.html.textile.liquid
+ - install/install-manual-prerequisites.html.textile.liquid
- install/install-api-server.html.textile.liquid
- install/install-workbench-app.html.textile.liquid
+ - install/install-shell-server.html.textile.liquid
- install/create-standard-objects.html.textile.liquid
+ - install/install-keepstore.html.textile.liquid
+ - install/install-keepproxy.html.textile.liquid
- install/install-crunch-dispatch.html.textile.liquid
+ - install/install-compute-node.html.textile.liquid
+ - Software prerequisites:
+ - install/install-manual-prerequisites-ruby.html.textile.liquid
+ - install/install-sso.html.textile.liquid
...
+Next, we're going to use the Arvados CLI tools on the <strong>shell server</strong> to create some standard objects.
h3. "All users" group
The convention is to add every active user to this group. We give it a distinctive UUID that looks like an IP broadcast address.
-<pre>
-prefix=`arv --format=uuid user current | cut -d- -f1`
-
-echo "Site prefix is '$prefix'"
-# (Make sure it matches your configured 5-character site prefix.)
-
-read -rd $'\000' newgroup <<EOF; arv group create --group "$newgroup"
-{
+<notextile>
+<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
+~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
+~$ <span class="userinput">read -rd $'\000' newgroup <<EOF; arv group create --group "$newgroup"</span>
+<span class="userinput">{
"uuid":"$prefix-j7d0g-fffffffffffffff",
"name":"All users"
-}
+}</span>
EOF
-</pre>
+</code></pre></notextile>
h3. "arvados" repository
This will be readable by the "All users" group, and therefore by every active user. This makes it possible for users to run the bundled Crunch scripts by specifying @"script_version":"master","repository":"arvados"@ rather than pulling the Arvados source tree into their own repositories.
-<pre>
-prefix=`arv --format=uuid user current | cut -d- -f1`
-
-echo "Site prefix is '$prefix'"
-# (Make sure it matches your configured 5-character site prefix.)
-
-all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
-repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados"}'`
-echo "Arvados repository uuid is '$repo_uuid'"
-
-read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
-{
+<notextile>
+<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
+~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
+~$ <span class="userinput">all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"</span>
+~$ <span class="userinput">repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados"}'`</span>
+~$ <span class="userinput">echo "Arvados repository uuid is '$repo_uuid'"</span>
+~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
+<span class="userinput">{
"tail_uuid":"$all_users_group_uuid",
"head_uuid":"$repo_uuid",
"link_class":"permission",
"name":"can_read"
}
-EOF
-</pre>
+EOF</span>
+</code></pre></notextile>
-h3. Keep disks
-
-Currently, you need to tell Arvados about Keep services manually. You'll need at least two "disk" services.
-
-Example:
-
-<pre>
-prefix=`arv --format=uuid user current | cut -d- -f1`
-echo "Site prefix is '$prefix'"
-# (Make sure it matches your configured 5-character site prefix.)
-
-read -rd $'\000' keepservice <<EOF; arv keep_service create --keep-service "$keepservice"
-{
- "service_host":"keep0.$prefix.arvadosapi.com",
- "service_port":25107,
- "service_ssl_flag":false,
- "service_type":"disk"
-}
-EOF
-</pre>
Arvados can be installed in multiple ways. Arvados does not depend on any particular cloud operating stack. Arvados runs on one or more GNU/Linux system(s). Arvados is being developed on Debian and Ubuntu GNU/Linux.
-The simplest way to try out Arvados is to use the Docker-based installation, which installs Arvados in a series of Docker containers.
+The simplest way to try out Arvados is to use the "Docker-based installation":install-docker.html, which installs Arvados in a series of Docker containers.
-For larger scale installations, a manual installation is more appropriate.
-
-h2. Docker
-
-"Installing with Docker":install-docker.html
-
-h2. Manual installation
-
-{% include 'alert_stub' %}
-
-# Set up a cluster, or use Amazon
-# "Install Keep":install-keep.html
-# "Install the Single Sign On (SSO) server":install-sso.html
-# "Install the Arvados REST API server":install-api-server.html
-# "Install the Arvados workbench application":install-workbench-app.html
-# "Install the Crunch dispatcher":install-crunch-dispatch.html
-# "Create standard objects":create-standard-objects.html
-# Install client libraries (see "SDK Reference":{{site.baseurl}}/sdk/index.html).
+For production use or evaluation at scale, a "Manual Installation":install-manual-overview.html is more appropriate.
title: Install the API server
...
-h2. Prerequisites:
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
-# A GNU/Linux (virtual) machine
-# A domain name for your api server
-
-h2(#dependencies). Install dependencies
+h2. Install prerequisites
<notextile>
<pre><code>~$ <span class="userinput">sudo apt-get install \
bison build-essential gettext libcurl3 libcurl3-gnutls \
libcurl4-openssl-dev libpcre3-dev libpq-dev libreadline-dev \
- libsqlite3-dev libssl-dev libxslt1.1 postgresql sqlite3 sudo \
- wget zlib1g-dev
+ libssl-dev libxslt1.1 postgresql sudo wget zlib1g-dev
</span></code></pre></notextile>
-h2(#ruby). Install Ruby and bundler
-
-We recommend Ruby >= 2.1.
-
-<notextile>
-<pre><code><span class="userinput">mkdir -p ~/src
-cd ~/src
-wget http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.2.tar.gz
-tar xzf ruby-2.1.2.tar.gz
-cd ruby-2.1.2
-./configure
-make
-sudo make install
-
-sudo gem install bundler</span>
-</code></pre></notextile>
+Also make sure you have "Ruby and bundler":install-manual-prerequisites-ruby.html installed.
h2. Download the source tree
See also: "Downloading the source code":https://arvados.org/projects/arvados/wiki/Download on the Arvados wiki.
+The API server is in @services/api@ in the source tree.
+
h2. Install gem dependencies
<notextile>
~/arvados/services/api$ <span class="userinput">bundle install</span>
</code></pre></notextile>
+h2. Choose your environment
+
+The API server can be run in @development@ or in @production@ mode. Unless this installation is going to be used for development on the Arvados API server itself, you should run it in @production@ mode.
+
+Copy the example environment file for your environment. For example, if you choose @production@:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">cp -i config/environments/production.rb.example config/environments/production.rb</span>
+</code></pre></notextile>
+
h2. Configure the API server
-Edit the main configuration:
+First, copy the example configuration file:
<notextile>
<pre><code>~/arvados/services/api$ <span class="userinput">cp -i config/application.yml.example config/application.yml</span>
</code></pre></notextile>
-Choose a unique 5-character alphanumeric string to use as your @uuid_prefix@. An example is given that generates a 5-character string based on a hash of your hostname. The @uuid_prefix@ is a unique identifier for your API server. It also serves as the first part of the hostname for your API server.
+The API server reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the API server and is provided for installation convenience, only.
-For a development site, use your own domain instead of arvadosapi.com.
+Consult @config/application.default.yml@ for a full list of configuration options. Always put your local configuration in @config/application.yml@, never edit @config/application.default.yml@.
-Make sure a clone of the arvados repository exists in @git_repositories_dir@:
+h3(#uuid_prefix). uuid_prefix
+
+It is recommended to explicitly define your @uuid_prefix@ in @config/application.yml@, by setting the 'uuid_prefix' field in the section for your environment.
+
+h3(#git_repositories_dir). git_repositories_dir
+
+This field defaults to @/var/lib/arvados/git@. You can override the value by defining it in @config/application.yml@.
+
+Make sure a clone of the arvados repository exists in @git_repositories_dir@.
<notextile>
-<pre><code>~/arvados/services/api$ <span class="userinput">sudo mkdir -p /var/cache/git</span>
-~/arvados/services/api$ <span class="userinput">sudo git clone --bare ../../.git /var/cache/git/arvados.git</span>
+<pre><code>~/arvados/services/api$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git</span>
+~/arvados/services/api$ <span class="userinput">sudo git clone --bare ../../.git /var/lib/arvados/git/arvados.git</span>
</code></pre></notextile>
+h3. secret_token
+
Generate a new secret token for signing cookies:
<notextile>
zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
</code></pre></notextile>
-If you want access control on your Keep server(s), you should set @blob_signing_key@ to the same value as the permission key you provided to your "Keep server(s)":install-keep.html.
+Then put that value in the @secret_token@ field.
-Put it in @config/application.yml@ in the production or common section:
+h3. blob_signing_key
-<notextile>
-<pre><code><span class="userinput"> secret_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</span>
-</code></pre>
-</notextile>
+If you want access control on your "Keepstore":install-keepstore.html server(s), you should set @blob_signing_key@ to the same value as the permission key you provide to your Keepstore daemon(s).
+
+h3. workbench_address
+
+Fill in the url of your workbench application in in @workbench_address@, for example
+
+ https://workbench.@prefix_uuid@.your.domain
+
+h3. other options
Consult @application.default.yml@ for a full list of configuration options. Always put your local configuration in @application.yml@ instead of editing @application.default.yml@.
+h2. Set up the database
+
Generate a new database password. Nobody ever needs to memorize it or type it, so we'll make a strong one:
<notextile>
~/arvados/services/api$ <span class="userinput">edit config/database.yml</span>
</code></pre></notextile>
-Create and initialize the database.
+Create and initialize the database. If you are planning a production system, choose the @production@ rails environment, otherwise use @development@.
<notextile>
-<pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=development bundle exec rake db:setup</span>
+<pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=production bundle exec rake db:setup</span>
</code></pre></notextile>
-Set up omniauth:
+Alternatively, if the database user you intend to use for the API server is not allowed to create new databases, you can create the database first and then populate it with rake. Be sure to adjust the database name if you are using the @development@ environment. This sequence of commands is functionally equivalent to the rake db:setup command above.
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">su postgres createdb arvados_production -E UTF8 -O arvados</span>
+~/arvados/services/api$ <span class="userinput">RAILS_ENV=production bundle exec rake db:structure:load</span>
+~/arvados/services/api$ <span class="userinput">RAILS_ENV=production bundle exec rake db:seed</span>
+</code></pre></notextile>
+
+<div class="alert alert-block alert-info">
+ <button type="button" class="close" data-dismiss="alert">×</button>
+ <h4>Note!</h4>
+You can safely ignore the following error message you may see when loading the database structure:
+<notextile>
+<pre><code>ERROR: must be owner of extension plpgsql</code></pre></notextile>
+</div>
+
+h2. Set up omniauth
+
+First copy the omniauth configuration file:
<notextile>
<pre><code>~/arvados/services/api$ <span class="userinput">cp -i config/initializers/omniauth.rb.example config/initializers/omniauth.rb
</code></pre></notextile>
-Edit @config/initializers/omniauth.rb@, and tell your api server to use the Curoverse SSO server for authentication:
+Edit @config/initializers/omniauth.rb@, and tell your api server to use the Curoverse SSO server for authentication. Use the @APP_SECRET@ specified in the snippet below.
<notextile>
<pre><code>APP_ID = 'local_docker_installation'
<p>You can also run your own SSO server. However, the SSO server codebase currently uses OpenID 2.0 to talk to Google's authentication service. Google <a href="https://developers.google.com/accounts/docs/OpenID2">has deprecated that protocol</a>. This means that new clients will not be allowed to talk to Google's authentication services anymore over OpenID 2.0, and they will phase out the use of OpenID 2.0 completely in the coming monts. We are working on upgrading the SSO server codebase to a newer protocol. That work should be complete by the end of November 2014. In the mean time, anyone is free to use the existing Curoverse SSO server for any local Arvados installation.</p>
</div>
-You can now run the development server:
+h2. Start the API server
+
+h3. Development environment
+
+If you plan to run in development mode, you can now run the development server this way:
<notextile>
<pre><code>~/arvados/services/api$ <span class="userinput">bundle exec rails server --port=3030
</code></pre></notextile>
-h3. Apache/Passenger (optional)
+h3. Production environment
-You can use "Passenger":https://www.phusionpassenger.com/ for deployment. Point it to the services/api directory in the source tree.
+We recommend "Passenger":https://www.phusionpassenger.com/ to run the API server in production.
-To enable streaming so users can monitor crunch jobs in real time, add to your Passenger configuration in Apache:
+Point it to the services/api directory in the source tree.
-<notextile>
-<pre><code><span class="userinput">PassengerBufferResponse off</span>
-</code></pre>
-</notextile>
-
-h2(#admin-user). Add an admin user
-
-Point your browser to the API server's login endpoint:
+To enable streaming so users can monitor crunch jobs in real time, make sure to add the following to your Passenger configuration:
<notextile>
-<pre><code><span class="userinput">https://localhost:3030/login</span>
+<pre><code><span class="userinput">PassengerBufferResponse off</span>
</code></pre>
</notextile>
-
-Log in with your google account.
-
-Use the rails console to give yourself admin privileges:
-
-<notextile>
-<pre><code>~/arvados/services/api$ <span class="userinput">bundle exec rails console</span>
-irb(main):001:0> <span class="userinput">Thread.current[:user] = User.all.select(&:identity_url).last</span>
-irb(main):002:0> <span class="userinput">Thread.current[:user].is_admin = true</span>
-irb(main):003:0> <span class="userinput">Thread.current[:user].update_attributes is_admin: true, is_active: true</span>
-irb(main):004:0> <span class="userinput">User.where(is_admin: true).collect &:email</span>
-=> ["root", "<b>your_address@example.com</b>"]
-</code></pre></notextile>
* @pip install --upgrade pyvcf@
-h4. Redis
-
-On controller:
-
-* @apt-get install redis-server@
-
h4. Crunch user account
On compute nodes and controller:
h4. Repositories
-Crunch scripts must be in Git repositories in @/var/lib/arvados/git/*.git@ (or whatever is configured in @services/api/config/environments/production.rb@).
+Crunch scripts must be in Git repositories in the directory configured as @git_repositories_dir@/*.git (see the "API server installation":install-api-server.html#git_repositories_dir).
Once you have a repository with commits -- and you have read access to the repository -- you should be able to create a new job:
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install Keep
-...
-
-This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
-
-First add the Arvados apt repository, and then install the Keep package.
-
-<notextile>
-<pre><code>~$ <span class="userinput">echo "# apt.arvados.org" > /etc/apt/sources.list.d/apt.arvados.org.list</span>
-~$ <span class="userinput">echo "deb http://apt.arvados.org/ wheezy main" >> /etc/apt/sources.list.d/apt.arvados.org.list</span>
-~$ <span class="userinput">/usr/bin/apt-key adv --keyserver pgp.mit.edu --recv 1078ECD7</span>
-~$ <span class="userinput">/usr/bin/apt-get update</span>
-~$ <span class="userinput">/usr/bin/apt-get install keepstore</span>
-</code></pre>
-</notextile>
-
-Verify that Keep is functional:
-
-<notextile>
-<pre><code>~$ <span class="userinput">keepstore -h</span>
-2014/07/24 15:38:27 Keep started: pid 13606
-Usage of keepstore:
- -data-manager-token-file="": File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
- -enforce-permissions=false: Enforce permission signatures on requests.
- -listen=":25107": Interface on which to listen for requests, in the format ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port to listen on all network interfaces.
- -never-delete=false: If set, nothing will be deleted. HTTP 405 will be returned for valid DELETE requests.
- -permission-key-file="": File containing the secret key for generating and verifying permission signatures.
- -permission-ttl=1209600: Expiration time (in seconds) for newly generated permission signatures.
- -pid="": Path to write pid file
- -serialize=false: If set, all read and write operations on local Keep volumes will be serialized.
- -volumes="": Comma-separated list of directories to use for Keep volumes, e.g. -volumes=/var/keep1,/var/keep2. If empty or not supplied, Keep will scan mounted filesystems for volumes with a /keep top-level directory.
-</code></pre>
-</notextile>
-
-If you want access control on your Keep server(s), you should provide a permission key. The @-permission-key-file@ argument should contain the path to a file that contains a single line with a long random alphanumeric string. It should be the same as the @blob_signing_key@ that can be set in the "API server":install-api-server.html config/application.yml file.
-
-Prepare one or more volumes for Keep to use. Simply create a /keep directory on all the partitions you would like Keep to use, and then start Keep. For example, using 2 tmpfs volumes:
-
-<notextile>
-<pre><code>~$ <span class="userinput">keepstore</span>
-2014/07/24 11:41:37 Keep started: pid 20736
-2014/07/24 11:41:37 adding Keep volume: /tmp/tmp.vwSCtUCyeH/keep
-2014/07/24 11:41:37 adding Keep volume: /tmp/tmp.Lsn4w8N3Xv/keep
-2014/07/24 11:41:37 Running without a PermissionSecret. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions.
-2014/07/24 11:41:37 To fix this, run Keep with --permission-key-file=<path> to define the location of a file containing the permission key.
-
-</code></pre>
-</notextile>
-
-It's recommended to run Keep under "runit":https://packages.debian.org/search?keywords=runit or something similar.
-
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Keepproxy server
+...
+
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
+
+The Keepproxy server is a gateway into your Keep storage. Unlike the Keepstore servers, which are only accessible on the local LAN, Keepproxy is designed to provide secure access into Keep from anywhere on the internet.
+
+By convention, we use the following hostname for the Keepproxy:
+
+<div class="offset1">
+table(table table-bordered table-condensed).
+|_Hostname_|
+|keep.@uuid_prefix@.your.domain|
+</div>
+
+This hostname should resolve from anywhere on the internet.
+
+h2. Install Keepproxy
+
+First add the Arvados apt repository, and then install the Keepproxy package.
+
+<notextile>
+<pre><code>~$ <span class="userinput">echo "# apt.arvados.org" > /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">echo "deb http://apt.arvados.org/ wheezy main" >> /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">/usr/bin/apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7</span>
+~$ <span class="userinput">/usr/bin/apt-get update</span>
+~$ <span class="userinput">/usr/bin/apt-get install keepproxy</span>
+</code></pre>
+</notextile>
+
+Verify that Keepproxy is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">keepproxy -h</span>
+Usage of default:
+ -default-replicas=2: Default number of replicas to write if not specified by the client.
+ -listen=":25107": Interface on which to listen for requests, in the format ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port to listen on all network interfaces.
+ -no-get=false: If set, disable GET operations
+ -no-put=false: If set, disable PUT operations
+ -pid="": Path to write pid file
+</code></pre>
+</notextile>
+
+It's recommended to run Keepproxy under "runit":https://packages.debian.org/search?keywords=runit or something similar.
+
+h3. Create an API token for the Keepproxy server
+
+The Keepproxy server needs a token to talk to the API server.
+
+On the <strong>API server</strong>, use the following command to create the token:
+
+<notextile>
+<pre><code>~/arvados/services/api/script$ <span class="userinput">RAILS_ENV=production ./get_anonymous_user_token.rb</span>
+hoShoomoo2bai3Ju1xahg6aeng1siquuaZ1yae2gi2Uhaeng2r
+</code></pre></notextile>
+
+The value for the @api_token@ field should be added to Keepproxy's environment as ARVADOS_API_TOKEN. Make sure to also set ARVADOS_API_HOST to @uuid_prefix@.your.domain.
+
+h3. Set up a reverse proxy with SSL support
+
+Because the Keepproxy is intended for access from anywhere on the internet, it is recommended to use SSL for transport encryption.
+
+This is best achieved by putting a reverse proxy with SSL support in front of Keepproxy. Keepproxy itself runs on port 25107 by default; your reverse proxy can run on port 443 and pass requests to Keepproxy on port 25107.
+
+h3. Tell the API server about the Keepproxy server
+
+The API server needs to be informed about the presence of your Keepproxy server. Please execute the following commands on your <strong>shell server</strong>.
+
+<notextile>
+<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
+~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
+~$ <span class="userinput">read -rd $'\000' keepservice <<EOF; arv keep_service create --keep-service "$keepservice"</span>
+<span class="userinput">{
+ "service_host":"keep.$prefix.your.domain",
+ "service_port":443,
+ "service_ssl_flag":true,
+ "service_type":"proxy"
+}
+EOF</span>
+</code></pre></notextile>
+
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Keepstore servers
+...
+
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
+
+We are going to install two Keepstore servers. By convention, we use the following hostname pattern:
+
+<div class="offset1">
+table(table table-bordered table-condensed).
+|_Hostname_|
+|keep0.@uuid_prefix@.your.domain|
+|keep1.@uuid_prefix@.your.domain|
+</div>
+
+Because the Keepstore servers are not directly accessible from the internet, these hostnames only need to resolve on the local network.
+
+h2. Install Keepstore
+
+First add the Arvados apt repository, and then install the Keepstore package.
+
+<notextile>
+<pre><code>~$ <span class="userinput">echo "# apt.arvados.org" > /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">echo "deb http://apt.arvados.org/ wheezy main" >> /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">/usr/bin/apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7</span>
+~$ <span class="userinput">/usr/bin/apt-get update</span>
+~$ <span class="userinput">/usr/bin/apt-get install keepstore</span>
+</code></pre>
+</notextile>
+
+Verify that Keepstore is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">keepstore -h</span>
+2014/10/29 14:23:38 Keep started: pid 6848
+Usage of keepstore:
+ -data-manager-token-file="": File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
+ -enforce-permissions=false: Enforce permission signatures on requests.
+ -listen=":25107": Interface on which to listen for requests, in the format ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port to listen on all network interfaces.
+ -never-delete=false: If set, nothing will be deleted. HTTP 405 will be returned for valid DELETE requests.
+ -permission-key-file="": File containing the secret key for generating and verifying permission signatures.
+ -permission-ttl=1209600: Expiration time (in seconds) for newly generated permission signatures.
+ -pid="": Path to write pid file
+ -serialize=false: If set, all read and write operations on local Keep volumes will be serialized.
+ -volumes="": Comma-separated list of directories to use for Keep volumes, e.g. -volumes=/var/keep1,/var/keep2. If empty or not supplied, Keep will scan mounted filesystems for volumes with a /keep top-level directory.
+</code></pre>
+</notextile>
+
+If you want access control on your Keepstore server(s), you should provide a permission key. The @-permission-key-file@ argument should contain the path to a file that contains a single line with a long random alphanumeric string. It should be the same as the @blob_signing_key@ that can be set in the "API server":install-api-server.html config/application.yml file.
+
+Prepare one or more volumes for Keepstore to use. Simply create a /keep directory on all the partitions you would like Keepstore to use, and then start Keepstore. For example, using 2 tmpfs volumes:
+
+<notextile>
+<pre><code>~$ <span class="userinput">keepstore</span>
+2014/10/29 11:41:37 Keep started: pid 20736
+2014/10/29 11:41:37 adding Keep volume: /tmp/tmp.vwSCtUCyeH/keep
+2014/10/29 11:41:37 adding Keep volume: /tmp/tmp.Lsn4w8N3Xv/keep
+2014/10/29 11:41:37 Running without a PermissionSecret. Block locators returned by this server will not be signed, and will be rejected by a server that enforces permissions.
+2014/10/29 11:41:37 To fix this, run Keep with --permission-key-file=<path> to define the location of a file containing the permission key.
+
+</code></pre>
+</notextile>
+
+It's recommended to run Keepstore under "runit":https://packages.debian.org/search?keywords=runit or something similar.
+
+Repeat this section for each Keepstore server you are setting up.
+
+h3. Tell the API server about the Keepstore servers
+
+The API server needs to be informed about the presence of your Keepstore servers. For each of the Keepstore servers you have created, please execute the following commands on your <strong>shell server</strong>.
+
+Make sure to update the @service_host@ value to match each of your Keepstore servers.
+
+<notextile>
+<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
+~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
+~$ <span class="userinput">read -rd $'\000' keepservice <<EOF; arv keep_service create --keep-service "$keepservice"</span>
+<span class="userinput">{
+ "service_host":"keep0.$prefix.your.domain",
+ "service_port":25107,
+ "service_ssl_flag":false,
+ "service_type":"disk"
+}
+EOF</span>
+</code></pre></notextile>
+
+
+
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Overview
+...
+
+{% include 'alert_stub' %}
+
+The manual installation guide will walk you through setting up a basic Arvados cluster on a number of (virtual) GNU/Linux systems. This installation method is intended for evaluation or production use at scale.
+
+<div class="alert alert-block alert-info">
+ <button type="button" class="close" data-dismiss="alert">×</button>
+ <h4>Note</h4>
+ <p>If you are looking to evaluate Arvados on one machine, we recommend the "Docker installation method":install-docker.html instead.</p>
+</div>
+
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Ruby and bundler
+...
+
+We recommend Ruby >= 2.1.
+
+h2(#rvm). Option 1: Install with rvm
+
+<notextile>
+<pre><code>~$ <span class="userinput">\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.1</span>
+~$ <span class="userinput">gem install bundler
+</span></code></pre></notextile>
+
+h2(#fromsource). Option 2: Install from source
+
+<notextile>
+<pre><code><span class="userinput">mkdir -p ~/src
+cd ~/src
+wget http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.3.tar.gz
+tar xzf ruby-2.1.3.tar.gz
+cd ruby-2.1.3
+./configure
+make
+sudo make install
+
+sudo gem install bundler</span>
+</code></pre></notextile>
+
+
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Prerequisites
+...
+
+h2. Hardware (or virtual machines)
+
+This guide assumes you have seven systems available in the same network subnet:
+
+<div class="offset1">
+table(table table-bordered table-condensed).
+|_Function_|_Number of nodes_|
+|Arvados REST API, Websockets, Workbench and Crunch dispatcher|1|
+|Arvados SSO server|1|
+|Arvados Keepproxy server|1|
+|Arvados Keepstore servers|2|
+|Arvados shell server|1|
+|Arvados compute node|1|
+</div>
+
+The number of Keepstore, shell and compute nodes listed above is a minimum. In a real production installation, you will likely run many more of each of those types of nodes. In such a scenario, you would probably also want to dedicate a node to the Workbench server and Crunch dispatcher, respectively. For performance reasons, you may want to run the database server on a separate node as well.
+
+h2. A unique identifier
+
+Each Arvados installation should have a globally unique identifier, which is a unique 5-character alphanumeric string. Here is a snippet of ruby that generates such a string based on the hostname of your computer:
+
+<pre>
+Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4]
+</pre>
+
+You may also use a different method to pick the unique identifier. The unique identifier will be part of the hostname of the services in your Arvados cluster. The rest of this documentation will refer to it as your @uuid_prefix@.
+
+
+h2. SSL certificates
+
+There are four public-facing services that will require an SSL certificate. If you do not have official SSL certificates, you can use self-signed certificates. By convention, we use the following hostname pattern:
+
+<div class="offset1">
+table(table table-bordered table-condensed).
+|_Function_|_Hostname_|
+|Arvados REST API|@uuid_prefix@.your.domain|
+|Arvados Websockets endpoint|ws.@uuid_prefix@.your.domain|
+|Arvados Keepproxy server|keep.@uuid_prefix@.your.domain|
+|Arvados Workbench|workbench.@uuid_prefix@.your.domain|
+</div>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install a shell server
+...
+
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
+
+There is nothing inherently special about an Arvados shell server. It is just a GNU/Linux machine with the Arvados SDKs installed. For optimal performance, the Arvados shell server should be on the same LAN as the Arvados cluster, but that is not required.
+
+h2. Install API tokens
+
+Please follow the "API token guide":{{site.baseurl}}/user/reference/api-tokens.html to get API tokens for your user and install them on your shell server. We will use those tokens to test the SDKs as we install them.
+
+h2. Install the SDKs
+
+Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html and the "Command line SDK":{{site.baseurl}}/sdk/cli/index.html
h2(#dependencies). Install dependencies
-You need to have ruby 2.1 or higher and the bundler gem installed.
-
-One way to install those dependencies is:
-
-<notextile>
-<pre><code>~$ <span class="userinput">\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.1</span>
-~$ <span class="userinput">gem install bundler
-</span></code></pre></notextile>
+Make sure you have "Ruby and bundler":install-manual-prerequisites-ruby.html installed.
h2(#install). Install SSO server
---
layout: default
navsection: installguide
-title: Install the Arvados Workbench application
+title: Install Workbench
...
-h2. Prerequisites
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
-# A GNU/linux (virtual) machine (can be shared with the API server)
-# A hostname for your Workbench application
+h2. Install prerequisites
-h2. Install dependencies
-
-If you haven't already installed the API server on the same host:
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install \
+ bison build-essential gettext libcurl3 libcurl3-gnutls \
+ libcurl4-openssl-dev libpcre3-dev libpq-dev libreadline-dev \
+ libssl-dev libxslt1.1 sudo wget zlib1g-dev graphviz
+</span></code></pre></notextile>
-* Install Ruby 2.1 and Bundler: see the "dependencies" and "Ruby" sections on the "API server installation page":install-api-server.html#dependencies for details.
-* Omit postgresql. Workbench doesn't need its own database.
+Also make sure you have "Ruby and bundler":install-manual-prerequisites-ruby.html installed.
-Install graphviz.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install graphviz</span>
-</code></pre>
-</notextile>
+Workbench doesn't need its own database, so it does not need to have PostgreSQL installed.
h2. Download the source tree
Using themes_for_rails (0.5.1) from https://github.com/holtkampw/themes_for_rails (at 1fd2d78)
</code></pre></notextile>
+h2. Choose your environment
+
+The Workbench application can be run in @development@ or in @production@ mode. Unless this installation is going to be used for development on the Workbench applicatoin itself, you should run it in @production@ mode.
+
+Copy the example environment file for your environment. For example, if you choose @production@:
+
+<notextile>
+<pre><code>~/arvados/apps/workbench$ <span class="userinput">cp -i config/environments/production.rb.example config/environments/production.rb</span>
+</code></pre></notextile>
+
h2. Configure the Workbench application
+First, copy the example configuration file:
+
+<notextile>
+<pre><code>~/arvados/apps/workbench$ <span class="userinput">cp -i config/application.yml.example config/application.yml</span>
+</code></pre></notextile>
+
+The Workbench application reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the Workbench application and is provided for installation convenience, only.
+
+Consult @config/application.default.yml@ for a full list of configuration options. Always put your local configuration in @config/application.yml@, never edit @config/application.default.yml@.
+
+h3. secret_token
+
This application needs a secret token. Generate a new secret:
<notextile>
</code></pre>
</notextile>
-Copy @config/application.yml.example@ to @config/application.yml@ and edit it appropriately for your environment.
+Then put that value in the @secret_token@ field.
+
+h3. arvados_login_base and arvados_v1_base
-* Set @secret_token@ to the string you generated with @rake secret@.
-* Point @arvados_login_base@ and @arvados_v1_base@ at your "API server":install-api-server.html, like this:
+Point @arvados_login_base@ and @arvados_v1_base@ at your "API server":install-api-server.html. For example like this:
<notextile>
-<pre><code>arvados_login_base: https://your.host:3030/login
-arvados_v1_base: https://your.host:3030/arvados/v1
+<pre><code>arvados_login_base: https://prefix_uuid.your.domain/login
+arvados_v1_base: https://prefix_uuid.your.domain/arvados/v1
</code></pre>
</notextile>
-* @site_name@ can be any string to identify this Workbench.
-* If the SSL certificate you use for development isn't signed by a CA, make sure @arvados_insecure_https@ is @true@.
+h3. site_name
+
+@site_name@ can be set to any arbitrary string. It is used to identify this Workbench to people visiting it.
+
+h3. arvados_insecure_https
+
+If the SSL certificate you use for your API server isn't an official certificate signed by a CA, make sure @arvados_insecure_https@ is @true@.
+
+h3. other options
+
+Consult @application.default.yml@ for a full list of configuration options. Always put your local configuration in @application.yml@ instead of editing @application.default.yml@.
Copy @config/piwik.yml.example@ to @config/piwik.yml@ and edit to suit.
-h2. Start a standalone server
+h2. Start the Workbench application
+
+h3. Development environment
-For testing and development, the easiest way to get started is to run the web server that comes with Rails.
+If you plan to run in development mode, you can now run the development server this way:
<notextile>
<pre><code>~/arvados/apps/workbench$ <span class="userinput">bundle exec rails server --port=3031</span>
-</code></pre>
-</notextile>
+</code></pre></notextile>
+
+h3. Production environment
-Point your browser to <notextile><code>http://<b>your.host</b>:3031/</code></notextile>.
+We recommend "Passenger":https://www.phusionpassenger.com/ to run the API server in production.
+
+Point it to the apps/workbench directory in the source tree.
h2. Trusted client setting
Log in to Workbench once to ensure that the Arvados API server has a record of the Workbench client. (It's OK if Workbench says your account hasn't been activated yet. We'll deal with that next.)
-In the API server project root, start the rails console. Locate the ApiClient record for your Workbench installation (typically, while you're setting this up, the @last@ one in the database is the one you want), then set the @is_trusted@ flag for the appropriate client record:
+In the <strong>API server</strong> project root, start the rails console. Locate the ApiClient record for your Workbench installation (typically, while you're setting this up, the @last@ one in the database is the one you want), then set the @is_trusted@ flag for the appropriate client record:
-<notextile><pre><code>~/arvados/services/api$ <span class="userinput">bundle exec rails console</span>
+<notextile><pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=production bundle exec rails console</span>
irb(main):001:0> <span class="userinput">wb = ApiClient.all.last; [wb.url_prefix, wb.created_at]</span>
=> ["https://workbench.example.com/", Sat, 19 Apr 2014 03:35:12 UTC +00:00]
irb(main):002:0> <span class="userinput">include CurrentApiClient</span>
</code></pre>
</notextile>
-h2. Activate your own account
+h2(#admin-user). Add an admin user
+
+Next, we're going to use the rails console on the <strong>API server</strong> to activate our own account and give yourself admin privileges:
-Unless you already activated your account when installing the API server, the first time you log in to Workbench you will see a message that your account is awaiting activation.
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=production bundle exec rails console</span>
+irb(main):001:0> <span class="userinput">Thread.current[:user] = User.all.select(&:identity_url).last</span>
+irb(main):002:0> <span class="userinput">Thread.current[:user].is_admin = true</span>
+irb(main):003:0> <span class="userinput">Thread.current[:user].update_attributes is_admin: true, is_active: true</span>
+irb(main):004:0> <span class="userinput">User.where(is_admin: true).collect &:email</span>
+=> ["root", "<b>your_address@example.com</b>"]
+</code></pre></notextile>
-Activate your own account and give yourself administrator privileges by following the instructions in the "'Add an admin user' section of the API server install page":install-api-server.html#admin-user.
+At this point, you should have a working Workbench login with administrator privileges. Revisit your Workbench URL in a browser and reload the page to access it.
MAINTAINER Tim Pierce <twp@curoverse.com>
# Install postgres and apache.
-RUN apt-get update && \
- apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \
- supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
- libjson-perl cron
+RUN apt-get update -qq
+RUN apt-get install -qqy \
+ procps postgresql postgresql-server-dev-9.1 apache2 slurm-llnl munge \
+ supervisor sudo libwww-perl libio-socket-ssl-perl libcrypt-ssleay-perl \
+ libjson-perl cron
ADD munge.key /etc/munge/
RUN chown munge:munge /etc/munge/munge.key && chmod 600 /etc/munge/munge.key
workbench_address: @@API_WORKBENCH_ADDRESS@@
+ auto_setup_new_users: true
+
test:
uuid_prefix: zzzzz
secret_token: <%= rand(2**512).to_s(36) %>
while [ ${#keep_volumes[*]} -lt 2 ]
do
new_keep=$(mktemp -d)
- echo >&2 "mounting 512M tmpfs keep volume in $new_keep"
- sudo mount -t tmpfs -o size=512M tmpfs $new_keep
+ echo >&2 "mounting 2G tmpfs keep volume in $new_keep"
+ sudo mount -t tmpfs -o size=2G tmpfs $new_keep
mkdir $new_keep/keep
keep_volumes+=($new_keep)
done
$start_keep == false ]]
then
start_doc=9898
- start_sso=9901
+ #the sso server is currently not used by default so don't start it unless explicitly requested
+ #start_sso=9901
start_api=9900
start_compute=2
start_workbench=9899
if [[ $start_api != false ]]
then
+ if [[ $start_sso != false ]]; then
start_container "$start_api:443" "api_server" '' "sso_server:sso" "arvados/api"
+ else
+ start_container "$start_api:443" "api_server" '' '' "arvados/api"
+ fi
fi
if [[ $start_nameserver != false ]]
start_container "$start_workbench:80" "workbench_server" '' "api_server:api" "arvados/workbench"
fi
- if [ -d $HOME/.config/arvados ] || mkdir -p $HOME/.config/arvados
+ if [[ $start_api != false ]]
then
- cat >$HOME/.config/arvados/settings.conf <<EOF
+ if [ -d $HOME/.config/arvados ] || mkdir -p $HOME/.config/arvados
+ then
+ cat >$HOME/.config/arvados/settings.conf <<EOF
ARVADOS_API_HOST=$(ip_address "api_server")
ARVADOS_API_HOST_INSECURE=yes
ARVADOS_API_TOKEN=$(cat api/generated/superuser_token)
EOF
+ fi
fi
}
# * git, curl, rvm
# * Arvados source code in /usr/src/arvados, for preseeding gem installation
-RUN apt-get update && \
- apt-get -q -y install -q -y openssh-server apt-utils git curl \
+ADD apt.arvados.org.list /etc/apt/sources.list.d/
+RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
+RUN apt-get update -qq
+
+RUN apt-get install -qqy openssh-server apt-utils git curl \
libcurl3 libcurl3-gnutls libcurl4-openssl-dev locales \
- postgresql-server-dev-9.1 && \
+ postgresql-server-dev-9.1 python-arvados-python-client
+
+RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
/bin/mkdir -p /root/.ssh && \
/bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
/usr/sbin/locale-gen && \
/usr/local/rvm/bin/rvm alias create default ruby && \
/bin/mkdir -p /usr/src/arvados
-ADD apt.arvados.org.list /etc/apt/sources.list.d/
-RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
-RUN apt-get update && apt-get -qqy install python-arvados-python-client
-
ADD generated/arvados.tar.gz /usr/src/arvados/
# Update gem. This (hopefully) fixes
FROM arvados/slurm
MAINTAINER Ward Vandewege <ward@curoverse.com>
-RUN apt-get update && apt-get -qqy install supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse crunchstat python-arvados-fuse cron
+RUN apt-get update -qq
+RUN apt-get install -qqy supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse crunchstat python-arvados-fuse cron dnsmasq
ADD fuse.conf /etc/fuse.conf
+RUN chmod 644 /etc/fuse.conf
RUN /usr/local/rvm/bin/rvm-exec default gem install arvados-cli arvados
-# Install Docker from the Docker Inc. repository
-RUN apt-get update -qq && apt-get install -qqy iptables ca-certificates lxc apt-transport-https
-RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
-RUN apt-get update -qq && apt-get install -qqy lxc-docker
+# Install Docker from the Arvados package repository (cf. arvados/base)
+RUN apt-get install -qqy iptables ca-certificates lxc apt-transport-https docker.io
RUN addgroup --gid 4005 crunch && mkdir /home/crunch && useradd --uid 4005 --gid 4005 crunch && usermod crunch -G fuse,docker && chown crunch:crunch /home/crunch
user=root
command=/usr/local/bin/wrapdocker.sh
+[program:dnsmasq]
+user=root
+command=/etc/init.d/dnsmasq start
+startsecs=0
+
# Install packages
RUN /bin/mkdir -p /usr/src/arvados && \
- apt-get update && \
- apt-get install -q -y curl procps apache2-mpm-worker
+ apt-get update -qq && \
+ apt-get install -qqy curl procps apache2-mpm-worker
ADD generated/doc.tar.gz /usr/src/arvados/
USER root
-RUN apt-get update && \
- apt-get install -y -q openjdk-7-jre-headless && \
+RUN apt-get update -qq
+RUN apt-get install -qqy openjdk-7-jre-headless && \
cd /tmp && \
curl --location http://downloads.sourceforge.net/project/bio-bwa/bwa-0.7.9a.tar.bz2 -o bwa-0.7.9a.tar.bz2 && \
tar xjf bwa-0.7.9a.tar.bz2 && \
MAINTAINER Ward Vandewege <ward@curoverse.com>
# Install packages and build the passenger apache module
-RUN apt-get update && \
- apt-get install -q -y apt-utils git curl procps apache2-mpm-worker \
- libcurl4-openssl-dev apache2-threaded-dev \
- libapr1-dev libaprutil1-dev && \
- cd /usr/src/arvados/services/api && \
+
+RUN apt-get update -qq
+RUN apt-get install -qqy \
+ apt-utils git curl procps apache2-mpm-worker \
+ libcurl4-openssl-dev apache2-threaded-dev \
+ libapr1-dev libaprutil1-dev
+
+RUN cd /usr/src/arvados/services/api && \
/usr/local/rvm/bin/rvm-exec default bundle exec passenger-install-apache2-module --auto
RUN cd /usr/src/arvados/services/api && \
FROM arvados/base
MAINTAINER Ward Vandewege <ward@curoverse.com>
-RUN apt-get update && apt-get -qqy install supervisor python-pip python-pyvcf python-gflags python-google-api-python-client python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse crunchstat python-arvados-fuse cron vim
+RUN apt-get update -qq
+RUN apt-get install -qqy \
+ python-pip python-pyvcf python-gflags python-google-api-python-client \
+ python-virtualenv libattr1-dev libfuse-dev python-dev python-llfuse fuse \
+ crunchstat python-arvados-fuse cron vim supervisor
ADD fuse.conf /etc/fuse.conf
+RUN chmod 644 /etc/fuse.conf
ADD generated/superuser_token /tmp/superuser_token
FROM arvados/base
MAINTAINER Ward Vandewege <ward@curoverse.com>
-RUN apt-get update && apt-get -q -y install slurm-llnl munge
+RUN apt-get update -qq
+RUN apt-get install -qqy slurm-llnl munge
ADD munge.key /etc/munge/
RUN chown munge:munge /etc/munge/munge.key && chmod 600 /etc/munge/munge.key
MAINTAINER Ward Vandewege <ward@curoverse.com>
# We need graphviz for the provenance graphs
-RUN apt-get update && apt-get -qqy install graphviz
+RUN apt-get update -qq
+RUN apt-get install -qqy graphviz
# Update Arvados source
RUN /bin/mkdir -p /usr/src/arvados/apps
s.executables << "arv-tag"
s.required_ruby_version = '>= 2.1.0'
s.add_runtime_dependency 'arvados', '~> 0.1', '>= 0.1.0'
- s.add_runtime_dependency 'google-api-client', '~> 0.6', '>= 0.6.3'
+ s.add_runtime_dependency 'google-api-client', '~> 0.6.3', '>= 0.6.3'
s.add_runtime_dependency 'activesupport', '~> 3.2', '>= 3.2.13'
s.add_runtime_dependency 'json', '~> 1.7', '>= 1.7.7'
s.add_runtime_dependency 'trollop', '~> 2.0'
arv_edit client, arvados, global_opts, remaining_opts
when 'keep'
@sub = remaining_opts.shift
- if ['get', 'put', 'ls', 'normalize'].index @sub then
+ if ['get', 'put', 'ls', 'normalize', 'copy'].index @sub then
# Native Arvados
exec `which arv-#{@sub}`.strip, *remaining_opts
elsif ['less', 'check'].index @sub then
--- /dev/null
+../../python/bin/arv-copy
\ No newline at end of file
use POSIX qw(strftime);
use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
use Arvados;
+use Data::Dumper;
use Digest::MD5 qw(md5_hex);
use Getopt::Long;
use IPC::Open2;
if ($cleanpid == 0)
{
srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
- ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
+ ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep $CRUNCH_TMP/task/*.keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_INSTALL $CRUNCH_TMP/task $CRUNCH_TMP/src*']);
exit (1);
}
while (1)
my @execargs = ("sh", "-c",
"mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
- # Note: this section is almost certainly unnecessary if we're
- # running tasks in docker containers.
my $installpid = fork();
if ($installpid == 0)
{
}
$ENV{"TASK_SLOT_NODE"} = $slot[$childslot]->{node}->{name};
$ENV{"TASK_SLOT_NUMBER"} = $slot[$childslot]->{cpu};
- $ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/$id.$$";
+ $ENV{"TASK_WORK"} = $ENV{"CRUNCH_TMP"}."/task/$childslotname";
$ENV{"HOME"} = $ENV{"TASK_WORK"};
$ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}.".keep";
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
$command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
if ($docker_hash)
{
- $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=10000 ";
- $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --user=crunch --cidfile=$ENV{TASK_WORK}/docker.cid ";
+ my $cidfile = "$ENV{CRUNCH_TMP}/$ENV{TASK_UUID}.cid";
+ $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$cidfile -poll=10000 ";
+ $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --attach=stdin -i --user=crunch --cidfile=$cidfile --sig-proxy ";
+
# Dynamically configure the container to use the host system as its
# DNS server. Get the host's global addresses from the ip command,
# and turn them into docker --dns options using gawk.
$command .=
q{$(ip -o address show scope global |
gawk 'match($4, /^([0-9\.:]+)\//, x){print "--dns", x[1]}') };
- $command .= "--volume=\Q$ENV{CRUNCH_SRC}:/tmp/crunch-src:ro\E ";
+
+ # The source tree and $destdir directory (which we have
+ # installed on the worker host) are available in the container,
+ # under the same path.
+ $command .= "--volume=\Q$ENV{CRUNCH_SRC}:$ENV{CRUNCH_SRC}:ro\E ";
+ $command .= "--volume=\Q$ENV{CRUNCH_INSTALL}:$ENV{CRUNCH_INSTALL}:ro\E ";
+
+ # Currently, we make arv-mount's mount point appear at /keep
+ # inside the container (instead of using the same path as the
+ # host like we do with CRUNCH_SRC and CRUNCH_INSTALL). However,
+ # crunch scripts and utilities must not rely on this. They must
+ # use $TASK_KEEPMOUNT.
$command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
- $command .= "--env=\QHOME=/home/crunch\E ";
+ $ENV{TASK_KEEPMOUNT} = "/keep";
+
+ # TASK_WORK is a plain docker data volume: it starts out empty,
+ # is writable, and persists until no containers use it any
+ # more. We don't use --volumes-from to share it with other
+ # containers: it is only accessible to this task, and it goes
+ # away when this task stops.
+ $command .= "--volume=\Q$ENV{TASK_WORK}\E ";
+
+ # JOB_WORK is also a plain docker data volume for now. TODO:
+ # Share a single JOB_WORK volume across all task containers on a
+ # given worker node, and delete it when the job ends (and, in
+ # case that doesn't work, when the next job starts).
+ $command .= "--volume=\Q$ENV{JOB_WORK}\E ";
+
while (my ($env_key, $env_val) = each %ENV)
{
- if ($env_key =~ /^(ARVADOS|JOB|TASK)_/) {
- if ($env_key eq "TASK_WORK") {
- $command .= "--env=\QTASK_WORK=/tmp/crunch-job\E ";
- }
- elsif ($env_key eq "TASK_KEEPMOUNT") {
- $command .= "--env=\QTASK_KEEPMOUNT=/keep\E ";
- }
- else {
- $command .= "--env=\Q$env_key=$env_val\E ";
- }
+ if ($env_key =~ /^(ARVADOS|CRUNCH|JOB|TASK)_/) {
+ $command .= "--env=\Q$env_key=$env_val\E ";
}
}
- $command .= "--env=\QCRUNCH_NODE_SLOTS=$ENV{CRUNCH_NODE_SLOTS}\E ";
- $command .= "--env=\QCRUNCH_SRC=/tmp/crunch-src\E ";
+ $command .= "--env=\QHOME=$ENV{HOME}\E ";
$command .= "\Q$docker_hash\E ";
$command .= "stdbuf --output=0 --error=0 ";
- $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
+ $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
} else {
# Non-docker run
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
# exec() failed, we assume nothing happened.
- Log(undef, "srun() failed on build script");
- die;
+ die "srun() failed on build script\n";
}
close("writer");
if (!defined $childpid)
my $opts = shift || {};
my $stdin = shift;
my $args = $have_slurm ? [@$srunargs, @$execargs] : $execargs;
- print STDERR (join (" ",
- map { / / ? "'$_'" : $_ }
- (@$args)),
- "\n")
- if $ENV{CRUNCH_DEBUG};
+
+ $Data::Dumper::Terse = 1;
+ $Data::Dumper::Indent = 0;
+ my $show_cmd = Dumper($args);
+ $show_cmd =~ s/(TOKEN\\*=)[^\s\']+/${1}[...]/g;
+ $show_cmd =~ s/\n/ /g;
+ warn "starting: $show_cmd\n";
if (defined $stdin) {
my $child = open STDIN, "-|";
# checkout-and-build
use Fcntl ':flock';
-use File::Path qw( make_path );
+use File::Path qw( make_path remove_tree );
my $destdir = $ENV{"CRUNCH_SRC"};
my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
my $task_work = $ENV{"TASK_WORK"};
for my $dir ($destdir, $task_work) {
- if ($dir) {
- make_path $dir;
- -e $dir or die "Failed to create temporary directory ($dir): $!";
- }
+ if ($dir) {
+ make_path $dir;
+ -e $dir or die "Failed to create temporary directory ($dir): $!";
+ }
}
+if ($task_work) {
+ remove_tree($task_work, {keep_root => 1});
+}
+
+
open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
flock L, LOCK_EX;
if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
}
unlink "$destdir.commit";
+open STDERR_ORIG, ">&STDERR";
open STDOUT, ">", "$destdir.log";
open STDERR, ">&STDOUT";
if ($ENV{"DEBUG"}) {
print STDERR "@_\n";
}
- system (@_) == 0
- or die "@_ failed: $! exit 0x".sprintf("%x",$?);
+ if (system (@_) != 0) {
+ my $err = $!;
+ my $exitstatus = sprintf("exit %d signal %d", $? >> 8, $? & 0x7f);
+ open STDERR, ">&STDERR_ORIG";
+ system ("cat $destdir.log >&2");
+ die "@_ failed ($err): $exitstatus";
+ }
}
__DATA__
--- /dev/null
+#! /usr/bin/env python
+
+# arv-copy [--recursive] [--no-recursive] object-uuid src dst
+#
+# Copies an object from Arvados instance src to instance dst.
+#
+# By default, arv-copy recursively copies any dependent objects
+# necessary to make the object functional in the new instance
+# (e.g. for a pipeline instance, arv-copy copies the pipeline
+# template, input collection, docker images, git repositories). If
+# --no-recursive is given, arv-copy copies only the single record
+# identified by object-uuid.
+#
+# The user must have files $HOME/.config/arvados/{src}.conf and
+# $HOME/.config/arvados/{dst}.conf with valid login credentials for
+# instances src and dst. If either of these files is not found,
+# arv-copy will issue an error.
+
+import argparse
+import getpass
+import os
+import re
+import shutil
+import sys
+import logging
+import tempfile
+
+import arvados
+import arvados.config
+import arvados.keep
+import arvados.util
+import arvados.commands._util as arv_cmd
+import arvados.commands.keepdocker
+
+logger = logging.getLogger('arvados.arv-copy')
+
+# local_repo_dir records which git repositories from the Arvados source
+# instance have been checked out locally during this run, and to which
+# directories.
+# e.g. if repository 'twp' from src_arv has been cloned into
+# /tmp/gitfHkV9lu44A then local_repo_dir['twp'] = '/tmp/gitfHkV9lu44A'
+#
+local_repo_dir = {}
+
+# List of collections that have been copied in this session, and their
+# destination collection UUIDs.
+collections_copied = {}
+
+def main():
+ copy_opts = argparse.ArgumentParser(add_help=False)
+
+ copy_opts.add_argument(
+ '-v', '--verbose', dest='verbose', action='store_true',
+ help='Verbose output.')
+ copy_opts.add_argument(
+ '--progress', dest='progress', action='store_true',
+ help='Report progress on copying collections. (default)')
+ copy_opts.add_argument(
+ '--no-progress', dest='progress', action='store_false',
+ help='Do not report progress on copying collections.')
+ copy_opts.add_argument(
+ '-f', '--force', dest='force', action='store_true',
+ help='Perform copy even if the object appears to exist at the remote destination.')
+ copy_opts.add_argument(
+ '--src', dest='source_arvados', required=True,
+ help='The name of the source Arvados instance (required). May be either a pathname to a config file, or the basename of a file in $HOME/.config/arvados/instance_name.conf.')
+ copy_opts.add_argument(
+ '--dst', dest='destination_arvados', required=True,
+ help='The name of the destination Arvados instance (required). May be either a pathname to a config file, or the basename of a file in $HOME/.config/arvados/instance_name.conf.')
+ copy_opts.add_argument(
+ '--recursive', dest='recursive', action='store_true',
+ help='Recursively copy any dependencies for this object. (default)')
+ copy_opts.add_argument(
+ '--no-recursive', dest='recursive', action='store_false',
+ help='Do not copy any dependencies. NOTE: if this option is given, the copied object will need to be updated manually in order to be functional.')
+ copy_opts.add_argument(
+ '--dst-git-repo', dest='dst_git_repo',
+ help='The name of the destination git repository. Required when copying a pipeline recursively.')
+ copy_opts.add_argument(
+ '--project-uuid', dest='project_uuid',
+ help='The UUID of the project at the destination to which the pipeline should be copied.')
+ copy_opts.add_argument(
+ 'object_uuid',
+ help='The UUID of the object to be copied.')
+ copy_opts.set_defaults(progress=True)
+ copy_opts.set_defaults(recursive=True)
+
+ parser = argparse.ArgumentParser(
+ description='Copy a pipeline instance, template or collection from one Arvados instance to another.',
+ parents=[copy_opts, arv_cmd.retry_opt])
+ args = parser.parse_args()
+
+ if args.verbose:
+ logger.setLevel(logging.DEBUG)
+ else:
+ logger.setLevel(logging.INFO)
+
+ # Create API clients for the source and destination instances
+ src_arv = api_for_instance(args.source_arvados)
+ dst_arv = api_for_instance(args.destination_arvados)
+
+ # Identify the kind of object we have been given, and begin copying.
+ t = uuid_type(src_arv, args.object_uuid)
+ if t == 'Collection':
+ result = copy_collection(args.object_uuid,
+ src_arv, dst_arv,
+ args)
+ elif t == 'PipelineInstance':
+ result = copy_pipeline_instance(args.object_uuid,
+ src_arv, dst_arv,
+ args)
+ elif t == 'PipelineTemplate':
+ result = copy_pipeline_template(args.object_uuid,
+ src_arv, dst_arv, args)
+ else:
+ abort("cannot copy object {} of type {}".format(args.object_uuid, t))
+
+ # Clean up any outstanding temp git repositories.
+ for d in local_repo_dir.values():
+ shutil.rmtree(d, ignore_errors=True)
+
+ # If no exception was thrown and the response does not have an
+ # error_token field, presume success
+ if 'error_token' in result or 'uuid' not in result:
+ logger.error("API server returned an error result: {}".format(result))
+ exit(1)
+
+ logger.info("")
+ logger.info("Success: created copy with uuid {}".format(result['uuid']))
+ exit(0)
+
+# api_for_instance(instance_name)
+#
+# Creates an API client for the Arvados instance identified by
+# instance_name.
+#
+# If instance_name contains a slash, it is presumed to be a path
+# (either local or absolute) to a file with Arvados configuration
+# settings.
+#
+# Otherwise, it is presumed to be the name of a file in
+# $HOME/.config/arvados/instance_name.conf
+#
+def api_for_instance(instance_name):
+ if '/' in instance_name:
+ config_file = instance_name
+ else:
+ config_file = os.path.join(os.environ['HOME'], '.config', 'arvados', "{}.conf".format(instance_name))
+
+ try:
+ cfg = arvados.config.load(config_file)
+ except (IOError, OSError) as e:
+ abort(("Could not open config file {}: {}\n" +
+ "You must make sure that your configuration tokens\n" +
+ "for Arvados instance {} are in {} and that this\n" +
+ "file is readable.").format(
+ config_file, e, instance_name, config_file))
+
+ if 'ARVADOS_API_HOST' in cfg and 'ARVADOS_API_TOKEN' in cfg:
+ api_is_insecure = (
+ cfg.get('ARVADOS_API_HOST_INSECURE', '').lower() in set(
+ ['1', 't', 'true', 'y', 'yes']))
+ client = arvados.api('v1',
+ host=cfg['ARVADOS_API_HOST'],
+ token=cfg['ARVADOS_API_TOKEN'],
+ insecure=api_is_insecure,
+ cache=False)
+ else:
+ abort('need ARVADOS_API_HOST and ARVADOS_API_TOKEN for {}'.format(instance_name))
+ return client
+
+# copy_pipeline_instance(pi_uuid, src, dst, args)
+#
+# Copies a pipeline instance identified by pi_uuid from src to dst.
+#
+# If the args.recursive option is set:
+# 1. Copies all input collections
+# * For each component in the pipeline, include all collections
+# listed as job dependencies for that component)
+# 2. Copy docker images
+# 3. Copy git repositories
+# 4. Copy the pipeline template
+#
+# The only changes made to the copied pipeline instance are:
+# 1. The original pipeline instance UUID is preserved in
+# the 'properties' hash as 'copied_from_pipeline_instance_uuid'.
+# 2. The pipeline_template_uuid is changed to the new template uuid.
+# 3. The owner_uuid of the instance is changed to the user who
+# copied it.
+#
+def copy_pipeline_instance(pi_uuid, src, dst, args):
+ # Fetch the pipeline instance record.
+ pi = src.pipeline_instances().get(uuid=pi_uuid).execute()
+
+ if args.recursive:
+ if not args.dst_git_repo:
+ abort('--dst-git-repo is required when copying a pipeline recursively.')
+ # Copy the pipeline template and save the copied template.
+ if pi.get('pipeline_template_uuid', None):
+ pt = copy_pipeline_template(pi['pipeline_template_uuid'],
+ src, dst, args)
+
+ # Copy input collections, docker images and git repos.
+ pi = copy_collections(pi, src, dst, args)
+ copy_git_repos(pi, src, dst, args.dst_git_repo)
+ copy_docker_images(pi, src, dst, args)
+
+ # Update the fields of the pipeline instance with the copied
+ # pipeline template.
+ if pi.get('pipeline_template_uuid', None):
+ pi['pipeline_template_uuid'] = pt['uuid']
+
+ else:
+ # not recursive
+ logger.info("Copying only pipeline instance %s.", pi_uuid)
+ logger.info("You are responsible for making sure all pipeline dependencies have been updated.")
+
+ # Update the pipeline instance properties, and create the new
+ # instance at dst.
+ pi['properties']['copied_from_pipeline_instance_uuid'] = pi_uuid
+ pi['description'] = "Pipeline copied from {}\n\n{}".format(
+ pi_uuid,
+ pi['description'] if pi.get('description', None) else '')
+ if args.project_uuid:
+ pi['owner_uuid'] = args.project_uuid
+ else:
+ del pi['owner_uuid']
+ del pi['uuid']
+
+ new_pi = dst.pipeline_instances().create(body=pi, ensure_unique_name=True).execute()
+ return new_pi
+
+# copy_pipeline_template(pt_uuid, src, dst, args)
+#
+# Copies a pipeline template identified by pt_uuid from src to dst.
+#
+# If args.recursive is True, also copy any collections, docker
+# images and git repositories that this template references.
+#
+# The owner_uuid of the new template is changed to that of the user
+# who copied the template.
+#
+# Returns the copied pipeline template object.
+#
+def copy_pipeline_template(pt_uuid, src, dst, args):
+ # fetch the pipeline template from the source instance
+ pt = src.pipeline_templates().get(uuid=pt_uuid).execute()
+
+ if args.recursive:
+ if not args.dst_git_repo:
+ abort('--dst-git-repo is required when copying a pipeline recursively.')
+ # Copy input collections, docker images and git repos.
+ pt = copy_collections(pt, src, dst, args)
+ copy_git_repos(pt, src, dst, args.dst_git_repo)
+ copy_docker_images(pt, src, dst, args)
+
+ pt['description'] = "Pipeline template copied from {}\n\n{}".format(
+ pt_uuid,
+ pt['description'] if pt.get('description', None) else '')
+ pt['name'] = "{} copied from {}".format(pt.get('name', ''), pt_uuid)
+ del pt['uuid']
+ del pt['owner_uuid']
+
+ return dst.pipeline_templates().create(body=pt, ensure_unique_name=True).execute()
+
+# copy_collections(obj, src, dst, args)
+#
+# Recursively copies all collections referenced by 'obj' from src
+# to dst. obj may be a dict or a list, in which case we run
+# copy_collections on every value it contains. If it is a string,
+# search it for any substring that matches a collection hash or uuid
+# (this will find hidden references to collections like
+# "input0": "$(file 3229739b505d2b878b62aed09895a55a+142/HWI-ST1027_129_D0THKACXX.1_1.fastq)")
+#
+# Returns a copy of obj with any old collection uuids replaced by
+# the new ones.
+#
+def copy_collections(obj, src, dst, args):
+
+ def copy_collection_fn(collection_match):
+ """Helper function for regex substitution: copies a single collection,
+ identified by the collection_match MatchObject, to the
+ destination. Returns the destination collection uuid (or the
+ portable data hash if that's what src_id is).
+
+ """
+ src_id = collection_match.group(0)
+ if src_id not in collections_copied:
+ dst_col = copy_collection(src_id, src, dst, args)
+ if src_id in [dst_col['uuid'], dst_col['portable_data_hash']]:
+ collections_copied[src_id] = src_id
+ else:
+ collections_copied[src_id] = dst_col['uuid']
+ return collections_copied[src_id]
+
+ if isinstance(obj, basestring):
+ # Copy any collections identified in this string to dst, replacing
+ # them with the dst uuids as necessary.
+ obj = arvados.util.portable_data_hash_pattern.sub(copy_collection_fn, obj)
+ obj = arvados.util.collection_uuid_pattern.sub(copy_collection_fn, obj)
+ return obj
+ elif type(obj) == dict:
+ return {v: copy_collections(obj[v], src, dst, args) for v in obj}
+ elif type(obj) == list:
+ return [copy_collections(v, src, dst, args) for v in obj]
+ return obj
+
+# copy_git_repos(p, src, dst, dst_repo)
+#
+# Copies all git repositories referenced by pipeline instance or
+# template 'p' from src to dst.
+#
+# For each component c in the pipeline:
+# * Copy git repositories named in c['repository'] and c['job']['repository'] if present
+# * Rename script versions:
+# * c['script_version']
+# * c['job']['script_version']
+# * c['job']['supplied_script_version']
+# to the commit hashes they resolve to, since any symbolic
+# names (tags, branches) are not preserved in the destination repo.
+#
+# The pipeline object is updated in place with the new repository
+# names. The return value is undefined.
+#
+def copy_git_repos(p, src, dst, dst_repo):
+ copied = set()
+ for c in p['components']:
+ component = p['components'][c]
+ if 'repository' in component:
+ repo = component['repository']
+ script_version = component.get('script_version', None)
+ if repo not in copied:
+ copy_git_repo(repo, src, dst, dst_repo, script_version)
+ copied.add(repo)
+ component['repository'] = dst_repo
+ if script_version:
+ repo_dir = local_repo_dir[repo]
+ component['script_version'] = git_rev_parse(script_version, repo_dir)
+ if 'job' in component:
+ j = component['job']
+ if 'repository' in j:
+ repo = j['repository']
+ script_version = j.get('script_version', None)
+ if repo not in copied:
+ copy_git_repo(repo, src, dst, dst_repo, script_version)
+ copied.add(repo)
+ j['repository'] = dst_repo
+ repo_dir = local_repo_dir[repo]
+ if script_version:
+ j['script_version'] = git_rev_parse(script_version, repo_dir)
+ if 'supplied_script_version' in j:
+ j['supplied_script_version'] = git_rev_parse(j['supplied_script_version'], repo_dir)
+
+def total_collection_size(manifest_text):
+ """Return the total number of bytes in this collection (excluding
+ duplicate blocks)."""
+
+ total_bytes = 0
+ locators_seen = {}
+ for line in manifest_text.splitlines():
+ words = line.split()
+ for word in words[1:]:
+ try:
+ loc = arvados.KeepLocator(word)
+ except ValueError:
+ continue # this word isn't a locator, skip it
+ if loc.md5sum not in locators_seen:
+ locators_seen[loc.md5sum] = True
+ total_bytes += loc.size
+
+ return total_bytes
+
+# copy_collection(obj_uuid, src, dst, args)
+#
+# Copies the collection identified by obj_uuid from src to dst.
+# Returns the collection object created at dst.
+#
+# If args.progress is True, produce a human-friendly progress
+# report.
+#
+# If a collection with the desired portable_data_hash already
+# exists at dst, and args.force is False, copy_collection returns
+# the existing collection without copying any blocks. Otherwise
+# (if no collection exists or if args.force is True)
+# copy_collection copies all of the collection data blocks from src
+# to dst.
+#
+# For this application, it is critical to preserve the
+# collection's manifest hash, which is not guaranteed with the
+# arvados.CollectionReader and arvados.CollectionWriter classes.
+# Copying each block in the collection manually, followed by
+# the manifest block, ensures that the collection's manifest
+# hash will not change.
+#
+def copy_collection(obj_uuid, src, dst, args):
+ c = src.collections().get(uuid=obj_uuid).execute()
+
+ # If a collection with this hash already exists at the
+ # destination, and 'force' is not true, just return that
+ # collection.
+ if not args.force:
+ if 'portable_data_hash' in c:
+ colhash = c['portable_data_hash']
+ else:
+ colhash = c['uuid']
+ dstcol = dst.collections().list(
+ filters=[['portable_data_hash', '=', colhash]]
+ ).execute()
+ if dstcol['items_available'] > 0:
+ logger.debug("Skipping collection %s (already at dst)", obj_uuid)
+ return dstcol['items'][0]
+
+ # Fetch the collection's manifest.
+ manifest = c['manifest_text']
+ logger.debug("Copying collection %s with manifest: <%s>", obj_uuid, manifest)
+
+ # Copy each block from src_keep to dst_keep.
+ # Use the newly signed locators returned from dst_keep to build
+ # a new manifest as we go.
+ src_keep = arvados.keep.KeepClient(api_client=src, num_retries=args.retries)
+ dst_keep = arvados.keep.KeepClient(api_client=dst, num_retries=args.retries)
+ dst_manifest = ""
+ dst_locators = {}
+ bytes_written = 0
+ bytes_expected = total_collection_size(manifest)
+ if args.progress:
+ progress_writer = ProgressWriter(human_progress)
+ else:
+ progress_writer = None
+
+ for line in manifest.splitlines(True):
+ words = line.split()
+ dst_manifest_line = words[0]
+ for word in words[1:]:
+ try:
+ loc = arvados.KeepLocator(word)
+ blockhash = loc.md5sum
+ # copy this block if we haven't seen it before
+ # (otherwise, just reuse the existing dst_locator)
+ if blockhash not in dst_locators:
+ logger.debug("Copying block %s (%s bytes)", blockhash, loc.size)
+ if progress_writer:
+ progress_writer.report(obj_uuid, bytes_written, bytes_expected)
+ data = src_keep.get(word)
+ dst_locator = dst_keep.put(data)
+ dst_locators[blockhash] = dst_locator
+ bytes_written += loc.size
+ dst_manifest_line += ' ' + dst_locators[blockhash]
+ except ValueError:
+ # If 'word' can't be parsed as a locator,
+ # presume it's a filename.
+ dst_manifest_line += ' ' + word
+ dst_manifest += dst_manifest_line
+ if line.endswith("\n"):
+ dst_manifest += "\n"
+
+ if progress_writer:
+ progress_writer.report(obj_uuid, bytes_written, bytes_expected)
+ progress_writer.finish()
+
+ # Copy the manifest and save the collection.
+ logger.debug('saving %s with manifest: <%s>', obj_uuid, dst_manifest)
+ dst_keep.put(dst_manifest)
+
+ if 'uuid' in c:
+ del c['uuid']
+ if 'owner_uuid' in c:
+ del c['owner_uuid']
+ c['manifest_text'] = dst_manifest
+ return dst.collections().create(body=c, ensure_unique_name=True).execute()
+
+# copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version)
+#
+# Copies commits from git repository 'src_git_repo' on Arvados
+# instance 'src' to 'dst_git_repo' on 'dst'. Both src_git_repo
+# and dst_git_repo are repository names, not UUIDs (i.e. "arvados"
+# or "jsmith")
+#
+# All commits will be copied to a destination branch named for the
+# source repository URL.
+#
+# Because users cannot create their own repositories, the
+# destination repository must already exist.
+#
+# The user running this command must be authenticated
+# to both repositories.
+#
+def copy_git_repo(src_git_repo, src, dst, dst_git_repo, script_version):
+ # Identify the fetch and push URLs for the git repositories.
+ r = src.repositories().list(
+ filters=[['name', '=', src_git_repo]]).execute()
+ if r['items_available'] != 1:
+ raise Exception('cannot identify source repo {}; {} repos found'
+ .format(src_git_repo, r['items_available']))
+ src_git_url = r['items'][0]['fetch_url']
+ logger.debug('src_git_url: {}'.format(src_git_url))
+
+ r = dst.repositories().list(
+ filters=[['name', '=', dst_git_repo]]).execute()
+ if r['items_available'] != 1:
+ raise Exception('cannot identify destination repo {}; {} repos found'
+ .format(dst_git_repo, r['items_available']))
+ dst_git_push_url = r['items'][0]['push_url']
+ logger.debug('dst_git_push_url: {}'.format(dst_git_push_url))
+
+ # script_version is the "script_version" parameter from the source
+ # component or job. It is used here to tie the destination branch
+ # to the commit that was used on the source. If no script_version
+ # was supplied in the component or job, it is a mistake in the pipeline,
+ # but for the purposes of copying the repository, default to "master".
+ #
+ if not script_version:
+ script_version = "master"
+
+ dst_branch = re.sub(r'\W+', '_', "{}_{}".format(src_git_url, script_version))
+
+ # Copy git commits from src repo to dst repo (but only if
+ # we have not already copied this repo in this session).
+ #
+ if src_git_repo in local_repo_dir:
+ logger.debug('already copied src repo %s, skipping', src_git_repo)
+ else:
+ tmprepo = tempfile.mkdtemp()
+ local_repo_dir[src_git_repo] = tmprepo
+ arvados.util.run_command(
+ ["git", "clone", "--bare", src_git_url, tmprepo],
+ cwd=os.path.dirname(tmprepo))
+ arvados.util.run_command(
+ ["git", "branch", dst_branch, script_version],
+ cwd=tmprepo)
+ arvados.util.run_command(["git", "remote", "add", "dst", dst_git_push_url], cwd=tmprepo)
+ arvados.util.run_command(["git", "push", "dst", dst_branch], cwd=tmprepo)
+
+
+def copy_docker_images(pipeline, src, dst, args):
+ """Copy any docker images named in the pipeline components'
+ runtime_constraints field from src to dst."""
+
+ logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
+ for c_name, c_info in pipeline['components'].iteritems():
+ if ('runtime_constraints' in c_info and
+ 'docker_image' in c_info['runtime_constraints']):
+ copy_docker_image(
+ c_info['runtime_constraints']['docker_image'],
+ c_info['runtime_constraints'].get('docker_image_tag', 'latest'),
+ src, dst, args)
+
+
+def copy_docker_image(docker_image, docker_image_tag, src, dst, args):
+ """Copy the docker image identified by docker_image and
+ docker_image_tag from src to dst. Create appropriate
+ docker_image_repo+tag and docker_image_hash links at dst.
+
+ """
+
+ logger.debug('copying docker image {}:{}'.format(docker_image, docker_image_tag))
+
+ # Find the link identifying this docker image.
+ docker_image_list = arvados.commands.keepdocker.list_images_in_arv(
+ src, args.retries, docker_image, docker_image_tag)
+ image_uuid, image_info = docker_image_list[0]
+ logger.debug('copying collection {} {}'.format(image_uuid, image_info))
+
+ # Copy the collection it refers to.
+ dst_image_col = copy_collection(image_uuid, src, dst, args)
+
+ # Create docker_image_repo+tag and docker_image_hash links
+ # at the destination.
+ lk = dst.links().create(
+ body={
+ 'head_uuid': dst_image_col['uuid'],
+ 'link_class': 'docker_image_repo+tag',
+ 'name': "{}:{}".format(docker_image, docker_image_tag),
+ }
+ ).execute(num_retries=args.retries)
+ logger.debug('created dst link {}'.format(lk))
+
+ lk = dst.links().create(
+ body={
+ 'head_uuid': dst_image_col['uuid'],
+ 'link_class': 'docker_image_hash',
+ 'name': dst_image_col['portable_data_hash'],
+ }
+ ).execute(num_retries=args.retries)
+ logger.debug('created dst link {}'.format(lk))
+
+
+# git_rev_parse(rev, repo)
+#
+# Returns the 40-character commit hash corresponding to 'rev' in
+# git repository 'repo' (which must be the path of a local git
+# repository)
+#
+def git_rev_parse(rev, repo):
+ gitout, giterr = arvados.util.run_command(
+ ['git', 'rev-parse', rev], cwd=repo)
+ return gitout.strip()
+
+# uuid_type(api, object_uuid)
+#
+# Returns the name of the class that object_uuid belongs to, based on
+# the second field of the uuid. This function consults the api's
+# schema to identify the object class.
+#
+# It returns a string such as 'Collection', 'PipelineInstance', etc.
+#
+# Special case: if handed a Keep locator hash, return 'Collection'.
+#
+def uuid_type(api, object_uuid):
+ if re.match(r'^[a-f0-9]{32}\+[0-9]+(\+[A-Za-z0-9+-]+)?$', object_uuid):
+ return 'Collection'
+ p = object_uuid.split('-')
+ if len(p) == 3:
+ type_prefix = p[1]
+ for k in api._schema.schemas:
+ obj_class = api._schema.schemas[k].get('uuidPrefix', None)
+ if type_prefix == obj_class:
+ return k
+ return None
+
+def abort(msg, code=1):
+ logger.info("arv-copy: %s", msg)
+ exit(code)
+
+
+# Code for reporting on the progress of a collection upload.
+# Stolen from arvados.commands.put.ArvPutCollectionWriter
+# TODO(twp): figure out how to refactor into a shared library
+# (may involve refactoring some arvados.commands.arv_copy.copy_collection
+# code)
+
+def machine_progress(obj_uuid, bytes_written, bytes_expected):
+ return "{} {}: {} {} written {} total\n".format(
+ sys.argv[0],
+ os.getpid(),
+ obj_uuid,
+ bytes_written,
+ -1 if (bytes_expected is None) else bytes_expected)
+
+def human_progress(obj_uuid, bytes_written, bytes_expected):
+ if bytes_expected:
+ return "\r{}: {}M / {}M {:.1%} ".format(
+ obj_uuid,
+ bytes_written >> 20, bytes_expected >> 20,
+ float(bytes_written) / bytes_expected)
+ else:
+ return "\r{}: {} ".format(obj_uuid, bytes_written)
+
+class ProgressWriter(object):
+ _progress_func = None
+ outfile = sys.stderr
+
+ def __init__(self, progress_func):
+ self._progress_func = progress_func
+
+ def report(self, obj_uuid, bytes_written, bytes_expected):
+ if self._progress_func is not None:
+ self.outfile.write(
+ self._progress_func(obj_uuid, bytes_written, bytes_expected))
+
+ def finish(self):
+ self.outfile.write("\n")
+
+if __name__ == '__main__':
+ main()
t = s[0] + s[1][-1:]
return datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%SZ")
-def list_images_in_arv(api_client, num_retries):
+def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None):
+ """List all Docker images known to the api_client with image_name and
+ image_tag. If no image_name is given, defaults to listing all
+ Docker images.
+
+ Returns a list of tuples representing matching Docker images,
+ sorted in preference order (i.e. the first collection in the list
+ is the one that the API server would use). Each tuple is a
+ (collection_uuid, collection_info) pair, where collection_info is
+ a dict with fields "dockerhash", "repo", "tag", and "timestamp".
+
+ """
+ docker_image_filters = [['link_class', 'in', ['docker_image_hash', 'docker_image_repo+tag']]]
+ if image_name:
+ image_link_name = "{}:{}".format(image_name, image_tag or 'latest')
+ docker_image_filters.append(['name', '=', image_link_name])
+
existing_links = api_client.links().list(
- filters=[['link_class', 'in', ['docker_image_hash', 'docker_image_repo+tag']]]
+ filters=docker_image_filters
).execute(num_retries=num_retries)['items']
images = {}
for link in existing_links:
else:
images[collection_uuid]["timestamp"] = ptimestamp(link["created_at"])
- st = sorted(images.items(), lambda a, b: cmp(b[1]["timestamp"], a[1]["timestamp"]))
+ return sorted(images.items(), lambda a, b: cmp(b[1]["timestamp"], a[1]["timestamp"]))
- fmt = "{:30} {:10} {:12} {:29} {:20}"
- print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
- for i, j in st:
- print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
def main(arguments=None):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
- list_images_in_arv(api, args.retries)
+ fmt = "{:30} {:10} {:12} {:29} {:20}"
+ print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
+ for i, j in list_images_in_arv(api, args.retries):
+ print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
def initialize(config_file=default_config_file):
global _settings
_settings = {}
- if os.path.exists(config_file):
- with open(config_file, "r") as f:
- for config_line in f:
- if re.match('^\s*#', config_line):
- continue
- var, val = config_line.rstrip().split('=', 2)
- _settings[var] = val
+
+ # load the specified config file if available
+ try:
+ _settings = load(config_file)
+ except IOError:
+ pass
+
+ # override any settings with environment vars
for var in os.environ:
if var.startswith('ARVADOS_'):
_settings[var] = os.environ[var]
+def load(config_file):
+ cfg = {}
+ with open(config_file, "r") as f:
+ for config_line in f:
+ if re.match('^\s*$', config_line):
+ continue
+ if re.match('^\s*#', config_line):
+ continue
+ var, val = config_line.rstrip().split('=', 2)
+ cfg[var] = val
+ return cfg
+
def flag_is_true(key):
return get(key, '').lower() in set(['1', 't', 'true', 'y', 'yes'])
if proxy is None:
proxy = config.get('ARVADOS_KEEP_PROXY')
if api_token is None:
- api_token = config.get('ARVADOS_API_TOKEN')
+ if api_client is None:
+ api_token = config.get('ARVADOS_API_TOKEN')
+ else:
+ api_token = api_client.api_token
elif api_client is not None:
raise ValueError(
"can't build KeepClient with both API client and token")
--- /dev/null
+#!/usr/bin/env python
+
+from arvados.commands.arv_copy import main
+main()
license='Apache 2.0',
packages=find_packages(),
scripts=[
+ 'bin/arv-copy',
'bin/arv-get',
'bin/arv-keepdocker',
'bin/arv-ls',
blob_str,
'wrong content from Keep.get(md5(<binarydata>))')
+ def test_KeepEmptyCollectionTest(self):
+ blob_locator = self.keep_client.put('', copies=1)
+ self.assertRegexpMatches(
+ blob_locator,
+ '^d41d8cd98f00b204e9800998ecf8427e\+0',
+ ('wrong locator from Keep.put(""): ' + blob_locator))
+
class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
s.licenses = ['Apache License, Version 2.0']
s.files = ["lib/arvados.rb", "lib/arvados/keep.rb"]
s.required_ruby_version = '>= 2.1.0'
- s.add_dependency('google-api-client', '~> 0.6', '>= 0.6.3')
+ s.add_dependency('google-api-client', '~> 0.6.3', '>= 0.6.3')
s.add_dependency('activesupport', '~> 3.2', '>= 3.2.13')
s.add_dependency('json', '~> 1.7', '>= 1.7.7')
s.add_dependency('andand', '~> 1.3', '>= 1.3.3')
return to_enum(__method__) unless block_given?
@text.each_line do |line|
tokens = line.split
+ next if tokens.empty?
stream_name = unescape(tokens.shift)
blocks = []
while loc = Locator.parse(tokens.first)
"./dir1/subdir #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
"./dir2 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n"].join("")
+ def check_stream(stream, exp_name, exp_blocks, exp_files)
+ assert_equal(exp_name, stream.first)
+ assert_equal(exp_blocks, stream[1].map(&:to_s))
+ assert_equal(exp_files, stream.last)
+ end
+
def test_simple_each_line_array
manifest = Keep::Manifest.new(SIMPLEST_MANIFEST)
stream_name, block_s, file = SIMPLEST_MANIFEST.strip.split
stream_a = manifest.each_line.to_a
assert_equal(1, stream_a.size, "wrong number of streams")
- assert_equal(stream_name, stream_a[0][0])
- assert_equal([block_s], stream_a[0][1].map(&:to_s))
- assert_equal([file], stream_a[0][2])
+ check_stream(stream_a.first, stream_name, [block_s], [file])
end
def test_simple_each_line_block
assert_empty(Keep::Manifest.new("").each_line.to_a)
end
+ def test_empty_line_within_manifest
+ block_s = random_block
+ manifest = Keep::Manifest.
+ new([". #{block_s} 0:1:file1 1:2:file2\n",
+ "\n",
+ ". #{block_s} 3:3:file3 6:4:file4\n"].join(""))
+ streams = manifest.each_line.to_a
+ assert_equal(2, streams.size)
+ check_stream(streams[0], ".", [block_s], ["0:1:file1", "1:2:file2"])
+ check_stream(streams[1], ".", [block_s], ["3:3:file3", "6:4:file4"])
+ end
+
def test_backslash_escape_parsing
m_text = "./dir\\040name #{random_block} 0:0:file\\\\name\\011\\here.txt\n"
manifest = Keep::Manifest.new(m_text)
end
end
- @objects = @objects.order("#{klass.table_name}.uuid")
+ @objects = @objects.order("#{klass.table_name}.created_at desc")
@limit = limit_all - all_objects.count
apply_where_limit_order_params klass
klass_items_available = @objects.
def auto_setup_new_user
return true if !Rails.configuration.auto_setup_new_users
return true if !self.email
+ return true if self.uuid == system_user_uuid
+ return true if self.uuid == anonymous_user_uuid
if Rails.configuration.auto_setup_new_users_with_vm_uuid ||
Rails.configuration.auto_setup_new_users_with_repository
# Mandatory site secrets. See application.default.yml for more info.
secret_token: ~
blob_signing_key: ~
+ uuid_prefix: bogus
workbench_address: https://localhost:3031
production:
#!/usr/bin/env ruby
+require 'shellwords'
include Process
$options = {}
nodelist
end
+ def fail_job job, message
+ $stderr.puts "dispatch: #{job.uuid}: #{message}"
+ begin
+ Log.new(object_uuid: job.uuid,
+ event_type: 'dispatch',
+ owner_uuid: job.owner_uuid,
+ summary: message,
+ properties: {"text" => message}).save!
+ rescue
+ $stderr.puts "dispatch: log.create failed"
+ end
+ job.state = "Failed"
+ if not job.save
+ $stderr.puts "dispatch: job.save failed"
+ end
+ end
+
def start_jobs
@todo.each do |job|
next if @running[job.uuid]
"GEM_PATH=#{ENV['GEM_PATH']}")
end
- job_auth = ApiClientAuthorization.
- new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
- api_client_id: 0)
- if not job_auth.save
- $stderr.puts "dispatch: job_auth.save failed"
- next
+ @authorizations ||= {}
+ if @authorizations[job.uuid] and
+ @authorizations[job.uuid].user.uuid != job.modified_by_user_uuid
+ # We already made a token for this job, but we need a new one
+ # because modified_by_user_uuid has changed (the job will run
+ # as a different user).
+ @authorizations[job.uuid].update_attributes expires_at: Time.now
+ @authorizations[job.uuid] = nil
+ end
+ if not @authorizations[job.uuid]
+ auth = ApiClientAuthorization.
+ new(user: User.where('uuid=?', job.modified_by_user_uuid).first,
+ api_client_id: 0)
+ if not auth.save
+ $stderr.puts "dispatch: auth.save failed"
+ next
+ end
+ @authorizations[job.uuid] = auth
end
crunch_job_bin = (ENV['CRUNCH_JOB_BIN'] || `which arv-crunch-job`.strip)
raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
end
- require 'shellwords'
-
arvados_internal = Rails.configuration.git_internal_dir
if not File.exists? arvados_internal
$stderr.puts `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init --bare`
end
- repo_root = Rails.configuration.git_repositories_dir
- src_repo = File.join(repo_root, job.repository + '.git')
- if not File.exists? src_repo
- src_repo = File.join(repo_root, job.repository, '.git')
+ git = "git --git-dir=#{arvados_internal.shellescape}"
+
+ # @fetched_commits[V]==true if we know commit V exists in the
+ # arvados_internal git repository.
+ @fetched_commits ||= {}
+ if !@fetched_commits[job.script_version]
+
+ repo_root = Rails.configuration.git_repositories_dir
+ src_repo = File.join(repo_root, job.repository + '.git')
if not File.exists? src_repo
- $stderr.puts "dispatch: No #{job.repository}.git or #{job.repository}/.git at #{repo_root}"
- sleep 1
- next
+ src_repo = File.join(repo_root, job.repository, '.git')
+ if not File.exists? src_repo
+ fail_job job, "No #{job.repository}.git or #{job.repository}/.git at #{repo_root}"
+ next
+ end
end
- end
-
- git = "git --git-dir=#{arvados_internal.shellescape}"
- # check if the commit needs to be fetched or not
- commit_rev = `#{git} rev-list -n1 #{job.script_version.shellescape} 2>/dev/null`.chomp
- unless $? == 0 and commit_rev == job.script_version
- # commit does not exist in internal repository, so import the source repository using git fetch-pack
- cmd = "#{git} fetch-pack --no-progress --all #{src_repo.shellescape}"
- $stderr.puts cmd
- $stderr.puts `#{cmd}`
- unless $? == 0
- $stderr.puts "dispatch: git fetch-pack failed"
- sleep 1
- next
+ # check if the commit needs to be fetched or not
+ commit_rev = `#{git} rev-list -n1 #{job.script_version.shellescape} 2>/dev/null`.chomp
+ unless $? == 0 and commit_rev == job.script_version
+ # commit does not exist in internal repository, so import the source repository using git fetch-pack
+ cmd = "#{git} fetch-pack --no-progress --all #{src_repo.shellescape}"
+ $stderr.puts cmd
+ $stderr.puts `#{cmd}`
+ unless $? == 0
+ fail_job job, "git fetch-pack failed"
+ next
+ end
end
+ @fetched_commits[job.script_version] = true
end
- # check if the commit needs to be tagged with this job uuid
- tag_rev = `#{git} rev-list -n1 #{job.uuid.shellescape} 2>/dev/null`.chomp
- if $? != 0
- # no job tag found, so create one
- cmd = "#{git} tag #{job.uuid.shellescape} #{job.script_version.shellescape}"
- $stderr.puts cmd
- $stderr.puts `#{cmd}`
- unless $? == 0
- $stderr.puts "dispatch: git tag failed"
- sleep 1
- next
- end
- else
- # job tag found, check that it has the expected revision
- unless tag_rev == job.script_version
- # Uh oh, the tag doesn't point to the revision we were expecting.
- # Someone has been monkeying with the job record and/or git.
- $stderr.puts "dispatch: Already a tag #{job.script_version} pointing to commit #{tag_rev} but expected commit #{job.script_version}"
- job.state = "Failed"
- if not job.save
- $stderr.puts "dispatch: job.save failed"
+ # @job_tags[J]==V if we know commit V has been tagged J in the
+ # arvados_internal repository. (J is a job UUID, V is a commit
+ # sha1.)
+ @job_tags ||= {}
+ if not @job_tags[job.uuid]
+ # check if the commit needs to be tagged with this job uuid
+ tag_rev = `#{git} rev-list -n1 #{job.uuid.shellescape} 2>/dev/null`.chomp
+ if $? != 0
+ # no job tag found, so create one
+ cmd = "#{git} tag #{job.uuid.shellescape} #{job.script_version.shellescape}"
+ $stderr.puts cmd
+ $stderr.puts `#{cmd}`
+ unless $? == 0
+ fail_job job, "git tag failed"
+ next
+ end
+ else
+ # job tag found, check that it has the expected revision
+ unless tag_rev == job.script_version
+ # Uh oh, the tag doesn't point to the revision we were expecting.
+ # Someone has been monkeying with the job record and/or git.
+ fail_job job, "Existing tag #{job.uuid} points to commit #{tag_rev} but expected commit #{job.script_version}"
next
end
- next
end
+ @job_tags[job.uuid] = job.script_version
+ elsif @job_tags[job.uuid] != job.script_version
+ fail_job job, "Existing tag #{job.uuid} points to commit #{@job_tags[job.uuid]} but this job uses commit #{job.script_version}"
end
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
- cmd_args << job_auth.api_token
+ cmd_args << @authorizations[job.uuid].api_token
cmd_args << '--job'
cmd_args << job.uuid
cmd_args << '--git-dir'
buf: {stderr: '', stdout: ''},
started: false,
sent_int: 0,
- job_auth: job_auth,
+ job_auth: @authorizations[job.uuid],
stderr_buf_to_flush: '',
stderr_flushed_at: Time.new(0),
bytes_logged: 0,
end
else
refresh_todo unless did_recently(:refresh_todo, 1.0)
- update_node_status
+ update_node_status unless did_recently(:update_node_status, 1.0)
unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
start_jobs
end
user: user1_with_load
api_token: 1234k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi
expires_at: 2038-01-01 00:00:00
+
+fuse:
+ api_client: untrusted
+ user: fuse
+ api_token: 4nagbkv8eap0uok7pxm72nossq5asihls3yn5p4xmvqx5t5e7p
+ expires_at: 2038-01-01 00:00:00
+
multilevel_collection_1:
uuid: zzzzz-4zz18-pyw8yp9g3pr7irn
portable_data_hash: 1fd08fc162a5c6413070a8bd0bffc818+150
- owner_uuid: qr1hi-tpzed-000000000000000
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
uuid: zzzzz-4zz18-45xf9hw1sxkhl6q
# All of this collection's files are deep in subdirectories.
portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
- owner_uuid: qr1hi-tpzed-000000000000000
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
manifest_text: ". 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz\n"
name: "baz file"
+collection_1_owned_by_fuse:
+ uuid: zzzzz-4zz18-ovx05bfzormx3bg
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-0fusedrivertest
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: "collection #1 owned by FUSE"
+
+collection_2_owned_by_fuse:
+ uuid: zzzzz-4zz18-8ubpy4w74twtwzr
+ portable_data_hash: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ owner_uuid: zzzzz-tpzed-0fusedrivertest
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo\n"
+ name: "collection #2 owned by FUSE"
+
+collection_in_fuse_project:
+ uuid: zzzzz-4zz18-vx4mtkjqfrb534f
+ portable_data_hash: ea10d51bcf88862dbcc36eb292017dfd+45
+ owner_uuid: zzzzz-j7d0g-0000ownedbyfuse
+ created_at: 2014-02-03T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-02-03T17:22:54Z
+ updated_at: 2014-02-03T17:22:54Z
+ manifest_text: ". 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz\n"
+ name: "collection in FUSE project"
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
description: project with 10 pipelines
group_class: project
-project_with_2_pipelines_and_200_jobs:
+project_with_2_pipelines_and_60_jobs:
uuid: zzzzz-j7d0g-nnjobspipelines
owner_uuid: zzzzz-tpzed-user1withloadab
created_at: 2014-04-21 15:37:48 -0400
modified_by_user_uuid: zzzzz-tpzed-user1withloadab
modified_at: 2014-04-21 15:37:48 -0400
updated_at: 2014-04-21 15:37:48 -0400
- name: project with 2 pipelines and 200 jobs
+ name: project with 2 pipelines and 60 jobs
description: This will result in two pages in the display
group_class: project
name: project with 25 pipelines
description: project with 25 pipelines
group_class: project
+
+fuse_owned_project:
+ uuid: zzzzz-j7d0g-0000ownedbyfuse
+ owner_uuid: zzzzz-tpzed-0fusedrivertest
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-0fusedrivertest
+ modified_at: 2014-04-21 15:37:48 -0400
+ updated_at: 2014-04-21 15:37:48 -0400
+ name: FUSE Test Project
+ description: Test project belonging to FUSE test user
+ group_class: project
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
-# jobs in project_with_2_pipelines_and_200_jobs
-<% for i in 1..200 do %>
-job_<%=i%>_of_200:
- uuid: zzzzz-8i9sb-0vsrcqi7whch<%= i.to_s.rjust(3, '0') %>
- created_at: <%= i.minute.ago.to_s(:db) %>
+# jobs in project_with_2_pipelines_and_60_jobs
+<% for i in 1..60 do %>
+job_<%=i%>_of_60:
+ uuid: zzzzz-8i9sb-oneof100jobs<%= i.to_s.rjust(3, '0') %>
+ created_at: <%= ((i+5)/5).minute.ago.to_s(:db) %>
owner_uuid: zzzzz-j7d0g-nnjobspipelines
script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
state: Complete
name: can_read
head_uuid: zzzzz-j7d0g-fffffffffffffff
properties: {}
+
+empty_collection_name_in_fuse_user_home_project:
+ uuid: zzzzz-o0j2j-hw3mcg3c8pwo6ar
+ owner_uuid: zzzzz-tpzed-0fusedrivertest
+ created_at: 2014-08-06 22:11:51.242392533 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-0fusedrivertest
+ modified_at: 2014-08-06 22:11:51.242150425 Z
+ tail_uuid: zzzzz-tpzed-0fusedrivertest
+ link_class: name
+ name: Empty collection
+ head_uuid: d41d8cd98f00b204e9800998ecf8427e+0
+ properties: {}
+ updated_at: 2014-08-06 22:11:51.242010312 Z
+
dataclass: Collection
title: foo instance input
+pipeline_instance_owned_by_fuse:
+ state: Complete
+ uuid: zzzzz-d1hrv-ri9dvgkgqs9y09j
+ owner_uuid: zzzzz-tpzed-0fusedrivertest
+ pipeline_template_uuid: zzzzz-p5p6p-vq4wuvy84xvaq2r
+ created_at: 2014-09-15 12:00:00
+ name: "pipeline instance owned by FUSE"
+ components:
+ foo:
+ script: foo
+ script_version: master
+ script_parameters:
+ input:
+ required: true
+ dataclass: Collection
+ title: foo instance input
+
+pipeline_instance_in_fuse_project:
+ state: Complete
+ uuid: zzzzz-d1hrv-scarxiyajtshq3l
+ owner_uuid: zzzzz-j7d0g-0000ownedbyfuse
+ pipeline_template_uuid: zzzzz-p5p6p-vq4wuvy84xvaq2r
+ created_at: 2014-09-15 12:00:00
+ name: "pipeline instance in FUSE project"
+ components:
+ foo:
+ script: foo
+ script_version: master
+ script_parameters:
+ input:
+ required: true
+ dataclass: Collection
+ title: foo instance input
+
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
title: foo instance input
<% end %>
-# pipelines in project_with_2_pipelines_and_200_jobs
+# pipelines in project_with_2_pipelines_and_100_jobs
<% for i in 0..1 do %>
-pipeline_<%=i%>_of_2_pipelines_and_200_jobs:
+pipeline_<%=i%>_of_2_pipelines_and_100_jobs:
name: pipeline_<%= i %>
state: New
uuid: zzzzz-d1hrv-abcgneyn6brx<%= i.to_s.rjust(3, '0') %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-09-14 12:00:00
modified_at: 2014-09-16 12:00:00
+ name: Pipeline Template Newer Than Instance
components:
foo:
script: foo
required: true
dataclass: Collection
title: bar template input
+
+pipeline_template_in_fuse_project:
+ uuid: zzzzz-p5p6p-templinfuseproj
+ owner_uuid: zzzzz-j7d0g-0000ownedbyfuse
+ created_at: 2014-04-14 12:35:04 -0400
+ updated_at: 2014-04-14 12:35:04 -0400
+ modified_at: 2014-04-14 12:35:04 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-0fusedrivertest
+ name: pipeline template in FUSE project
+ components:
+ foo_component:
+ script: foo
+ script_version: master
+ script_parameters:
+ input:
+ required: true
+ dataclass: Collection
+ title: "default input"
+ description: "input collection"
+
profile:
organization: example.com
role: IT
+
+fuse:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zzzzz-tpzed-0fusedrivertest
+ email: fuse@arvados.local
+ first_name: FUSE
+ last_name: User
+ identity_url: https://fuse.openid.local
+ is_active: true
+ is_admin: false
+ prefs:
+ profile:
+ organization: example.com
+ role: IT
# wait until the driver is finished initializing
operations.initlock.wait()
- d1 = os.listdir(self.mounttmp)
- d1.sort()
- self.assertIn('Active User', d1)
-
- d2 = os.listdir(os.path.join(self.mounttmp, 'Active User'))
- d2.sort()
- self.assertEqual(['A Project',
- "Empty collection",
- "Empty collection.link",
- "Pipeline Template with Input Parameter with Search.pipelineTemplate",
- "Pipeline Template with Jobspec Components.pipelineTemplate",
- "collection_expires_in_future",
- "collection_with_same_name_in_aproject_and_home_project",
- "owned_by_active",
- "pipeline_to_merge_params.pipelineInstance",
- "pipeline_with_job.pipelineInstance",
- "pipeline_with_tagged_collection_input.pipelineInstance",
- "real_log_collection"
- ], d2)
-
- d3 = os.listdir(os.path.join(self.mounttmp, 'Active User', 'A Project'))
- d3.sort()
- self.assertEqual(["A Subproject",
- "Two Part Pipeline Template.pipelineTemplate",
- "collection_to_move_around",
- "collection_with_same_name_in_aproject_and_home_project",
- "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
- ], d3)
-
- with open(os.path.join(self.mounttmp, 'Active User', "A Project", "Two Part Pipeline Template.pipelineTemplate")) as f:
+ # shared_dirs is a list of the directories exposed
+ # by fuse.SharedDirectory (i.e. any object visible
+ # to the current user)
+ shared_dirs = os.listdir(self.mounttmp)
+ shared_dirs.sort()
+ self.assertIn('FUSE User', shared_dirs)
+
+ # fuse_user_objs is a list of the objects owned by the FUSE
+ # test user (which present as files in the 'FUSE User'
+ # directory)
+ fuse_user_objs = os.listdir(os.path.join(self.mounttmp, 'FUSE User'))
+ fuse_user_objs.sort()
+ self.assertEqual(['Empty collection.link', # permission link on collection
+ 'FUSE Test Project', # project owned by user
+ 'collection #1 owned by FUSE', # collection owned by user
+ 'collection #2 owned by FUSE', # collection owned by user
+ 'pipeline instance owned by FUSE.pipelineInstance', # pipeline instance owned by user
+ ], fuse_user_objs)
+
+ # test_proj_files is a list of the files in the FUSE Test Project.
+ test_proj_files = os.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
+ test_proj_files.sort()
+ self.assertEqual(['collection in FUSE project',
+ 'pipeline instance in FUSE project.pipelineInstance',
+ 'pipeline template in FUSE project.pipelineTemplate'
+ ], test_proj_files)
+
+ # Double check that we can open and read objects in this folder as a file,
+ # and that its contents are what we expect.
+ with open(os.path.join(
+ self.mounttmp,
+ 'FUSE User',
+ 'FUSE Test Project',
+ 'pipeline template in FUSE project.pipelineTemplate')) as f:
j = json.load(f)
- self.assertEqual("Two Part Pipeline Template", j['name'])
+ self.assertEqual("pipeline template in FUSE project", j['name'])
class FuseHomeTest(MountTestBase):
if job['uuid'] not in self.logged_jobs:
self.logged_jobs.add(job['uuid'])
self.logger.debug("job %s not satisfiable", job['uuid'])
- elif (want_count < self.max_nodes):
+ elif (want_count <= self.max_nodes):
servers.extend([cloud_size.real] * max(1, want_count))
self.logged_jobs.intersection_update(seen_jobs)
return servers
self.node_actor = cnode.ComputeNodeMonitorActor.start(
self.cloud_mock, start_time, self.shutdowns, self.timer,
self.updates, arv_node).proxy()
- self.subscription = self.node_actor.subscribe(self.subscriber)
+ self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
def test_init_shutdown_scheduling(self):
self.make_actor()
- self.subscription.get(self.TIMEOUT)
self.assertTrue(self.timer.schedule.called)
self.assertEqual(300, self.timer.schedule.call_args[0][0])
{'min_scratch_mb_per_node': 200})
self.assertEqual(6, len(servlist))
+ def test_job_requesting_max_nodes_accepted(self):
+ servcalc = self.make_calculator([1], max_nodes=4)
+ servlist = self.calculate(servcalc, {'min_nodes': 4})
+ self.assertEqual(4, len(servlist))
+
class JobQueueMonitorActorTestCase(testutil.RemotePollLoopActorTestMixin,
unittest.TestCase):
projects / arvados.git / commitdiff