end
log.info { "vm uuid: " + vm[:uuid] }
-# Look up the "all users" group (we expect uuid *-*-fffffffffffffff).
+# Look up the "All users" group (we expect uuid *-*-fffffffffffffff).
group = arv.group.list(where: {name: 'All users'})[:items].select do |g|
g[:uuid].match /-f+$/
end.first
if not group
- abort "Could not look up the 'all users' group with uuid '*-*-fffffffffffffff'. Stop."
+ abort "Could not look up the 'All users' group with uuid '*-*-fffffffffffffff'. Stop."
end
-log.info { "\"all users\" group uuid: " + group[:uuid] }
+log.info { "\"All users\" group uuid: " + group[:uuid] }
# Look for signs the user has already been activated / set up.
</div>
<% else %>
- <%= image_tag "dax.png", style: "max-width=40%" %>
+
+ <div class="row-fluid">
+ <div class="col-sm-4">
+ <%= image_tag "dax.png", style: "max-width:100%" %>
+ </div>
+ <div class="col-sm-8">
+ <h2>Welcome to Arvados, <%= current_user.first_name %>!</h2>
+ <div class="well">
+ <p>
+ Your account must be activated by an Arvados administrator. If this
+ is your first time accessing Arvados and would like to request
+ access, or you believe you are seeing the page in error, please
+ <%= link_to "contact us", Rails.configuration.activation_contact_link %>.
+ You should receive an email at the address you used to log in when
+ your account is activated. In the mean time, you can
+ <%= link_to "learn more about Arvados", "https://arvados.org/projects/arvados/wiki/Introduction_to_Arvados" %>,
+ and <%= link_to "read the Arvados user guide", "http://doc.arvados.org/user" %>.
+ </p>
+ <p style="padding-bottom: 1em">
+ <%= link_to raw('Contact us ✉'),
+ Rails.configuration.activation_contact_link, class: "pull-right btn btn-primary" %></p>
+ </div>
+ </div>
+ </div>
<% end %>
<% content_for :js do %>
require "rubygems"
require "colorize"
-Rake.application.rake_require "zenweb/tasks"
+require "zenweb/tasks"
load "zenweb-textile.rb"
load "zenweb-liquid.rb"
-exclude: ["Rakefile", "tmp", "vendor"]
+# baseurl is the location of the generated site from the browser's
+# perspective (e.g., http://doc.arvados.org or
+# file:///tmp/arvados/doc/.site). To make docs show up inside
+# workbench, use /doc here and add a symlink at
+# apps/workbench/public/doc pointing to ../../../doc/.site
+
baseurl: /doc
+
+exclude: ["Rakefile", "tmp", "vendor"]
arvados_api_host: qr1hi.arvadosapi.com
navbar:
- user/tutorials/tutorial-job1.html.textile.liquid
- user/tutorials/tutorial-firstscript.html.textile.liquid
- user/tutorials/tutorial-job-debug.html.textile.liquid
- - user/tutorials/tutorial-new-pipeline.html.textile.liquid
- user/tutorials/tutorial-parallel.html.textile.liquid
+ - user/tutorials/tutorial-new-pipeline.html.textile.liquid
- user/tutorials/tutorial-trait-search.html.textile.liquid
- user/tutorials/tutorial-gatk-variantfiltration.html.textile.liquid
- user/tutorials/running-external-program.html.textile.liquid
</div>
<div class="collapse navbar-collapse" id="bs-navbar-collapse">
<ul class="nav navbar-nav">
- <li {% if page.navsection == 'userguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/user/">User Guide</a></li>
- <li {% if page.navsection == 'sdk' %} class="active" {% endif %}><a href="{{ site.baseurl }}/sdk/">SDK Reference</a></li>
- <li {% if page.navsection == 'api' %} class="active" {% endif %}><a href="{{ site.baseurl }}/api/">API Reference</a></li>
- <li {% if page.navsection == 'adminguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/admin/">Admin Guide</a></li>
- <li {% if page.navsection == 'installguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/install/">Install Guide</a></li>
+ <li {% if page.navsection == 'userguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/user/index.html">User Guide</a></li>
+ <li {% if page.navsection == 'sdk' %} class="active" {% endif %}><a href="{{ site.baseurl }}/sdk/index.html">SDK Reference</a></li>
+ <li {% if page.navsection == 'api' %} class="active" {% endif %}><a href="{{ site.baseurl }}/api/index.html">API Reference</a></li>
+ <li {% if page.navsection == 'adminguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/admin/index.html">Admin Guide</a></li>
+ <li {% if page.navsection == 'installguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/install/index.html">Install Guide</a></li>
<li><a href="https://arvados.org/" style="padding-left: 2em">arvados.org »</a></li>
</ul>
</div>
import hashlib
import arvados
-# Jobs consist of one of more tasks. A task is a single invocation of
+# Jobs consist of one or more tasks. A task is a single invocation of
# a crunch script.
# Get the current task
## Configure the API server
-First configure the database:
+First install the gems:
cd arvados/services/api
+ bundle install
+
+Next, configure the database:
+
cp config/database.yml.sample config/database.yml
Edit database.yml to your liking and make sure the database and db user exist.
Server::Application.config.secret_token = 'your-new-secret-here'
-Finally, edit your
-
- environments/production.rb
+Finally, edit the main configuration:
-file.
+ cp config/environments/production.rb.example config/environments/production.rb
First, you want to make sure that
* @apt-get install libjson-perl libwww-perl libio-socket-ssl-perl libipc-system-simple-perl@
+Add this to @/etc/apt/sources.list@
+
+@deb http://git.oxf.freelogy.org/apt wheezy main contrib@
+
+Then
+
+@apt-get install libwarehouse-perl@
+
h4. Python SDK dependencies
On controller and all compute nodes:
h4. Test installation
-If the SDK is installed and your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.basedoc}}/user/reference/api-tokens.html for details), @import arvados@ should produce no errors:
+If the SDK is installed and your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), @import arvados@ should produce no errors:
<notextile>
<pre>$ <code class="userinput">python</code>
h1. Checking your environment
-First you should "log into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login if you have not already done so.
+First you should "log into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login if you have not already done so.
If @arv user current@ is able to access the API server, it will print out information about your account. Check that you are able to access the Arvados API server using the following command:
bc. ARVADOS_API_HOST and ARVADOS_API_TOKEN need to be defined as environment variables
-Then follow the instructions for "getting an API token,":{{site.basedoc}}/user/reference/api-tokens.html and try @arv user current@ again.
+Then follow the instructions for "getting an API token,":{{site.baseurl}}/user/reference/api-tokens.html and try @arv user current@ again.
-Once you are able to access the API server, you are ready proceed to the first tutorial: "Storing and retrieving data using Arvados Keep.":{{site.basedoc}}/user/tutorials/tutorial-keep.html
+Once you are able to access the API server, you are ready proceed to the first tutorial: "Storing and retrieving data using Arvados Keep.":{{site.baseurl}}/user/tutorials/tutorial-keep.html
(Replace @{{ site.arvados_api_host }}@ with the hostname of your local Arvados instance if necessary.)
-First, open a shell on the system on which you intend to use the Arvados client (this may be your local workstation, or an Arvados VM, refer to "Accessing Arvados over ssh":{{site.basedoc}}/user/getting_started/ssh-access.html ) .
+First, open a shell on the system on which you intend to use the Arvados client (this may be your local workstation, or an Arvados VM, refer to "Accessing Arvados over ssh":{{site.baseurl}}/user/getting_started/ssh-access.html ) .
Click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item _Manage API token_ to go to the "api client authorizations" page.
h1. Reference: Command Line Interface
-*First, you should be "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*First, you should be "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
h3. Usage
h4. Global options
-- @--format=json@ := Output response as JSON
+- @--format=json@ := Output response as JSON. This is the default format.
+
- @--format=yaml@ := Output response as YAML
+
- @--format=uuid@ := Output only the UUIDs of object(s) in the API response, one per line.
This tutorial demonstrates how to use Crunch to run an external program by writting a wrapper using the Python SDK.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
In this tutorial, you will use the external program @md5sum@ to compute hashes instead of the built-in Python library used in earlier tutorials.
In this tutorial, we will write the "hash" script demonstrated in the first tutorial.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
This tutorial uses *@you@* to denote your username. Replace *@you@* with your user name in all the following examples.
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x hash.py</span></code></pre>
+{% include 'notebox_begin' %}
+The below steps describe how to execute the script after committing changes to git. To test the script locally, please see the "debugging a crunch script":tutorial-job-debug.html page.
+
+{% include 'notebox_end' %}
+
Next, add the file to @git@ staging. This tells @git@ that the file should be included on the next commit.
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git add hash.py</span></code></pre>
h1. Using GATK with Arvados
-This tutorials demonstrates how to use the Genome Analysis Toolkit (GATK) with Arvados. In this example we will install GATK and then create a VariantFiltration job to assign pass/fail scores to variants in a VCF file.
+This tutorial demonstrates how to use the Genome Analysis Toolkit (GATK) with Arvados. In this example we will install GATK and then create a VariantFiltration job to assign pass/fail scores to variants in a VCF file.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
h2. Installing GATK
To test changes to a script by running a job, the change must be pushed into @git@, the job queued asynchronously, and the actual execution may be run on any compute server. As a result, debugging a script can be difficult and time consuming. This tutorial demonstrates using @arv-crunch-job@ to run your job in your local VM. This avoids the job queue and allows you to execute the script from your uncomitted git tree.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
This tutorial uses _you_ to denote your username. Replace _you_ with your user name in all the following examples.
This tutorial introduces the concepts and use of the Crunch job system using the @arv@ command line tool and Arvados Workbench.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
In "retrieving data using Keep,":tutorial-keep.html we downloaded a file from Keep and did some computation with it (specifically, computing the md5 hash of the complete file). While a straightforward way to accomplish a computational task, there are several obvious drawbacks to this approach:
* Large files require significant time to download.
</code></pre>
</notextile>
-The job is new queued and will start running as soon as it reaches the front of the queue. Fields to pay attention to include:
+The job is now queued and will start running as soon as it reaches the front of the queue. Fields to pay attention to include:
* @"uuid"@ is the unique identifier for this specific job
* @"script_version"@ is the actual revision of the script used. This is useful if the version was described using the "repository:branch" format.
This tutorial introduces you to the Arvados file storage system.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
The Arvados distributed file system is called *Keep*. Keep is a content-addressable file system. This means that files are managed using special unique identifiers derived from the _contents_ of the file, rather than human-assigned file names (specifically, the md5 hash). This has a number of advantages:
* Files can be stored and replicated across a cluster of servers without requiring a central name server.
A pipeline in Arvados is a collection of crunch scripts, in which the output from one script may be used as the input to another script.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
h2. Create a new script
"script_parameters":{
"input": "887cd41e9c613463eab2f0d885c6dd96+83"
},
- "script_version":"tetron:master"
+ "script_version":"you:master"
},
"filter":{
"script":"0-filter.py",
"output_of":"do_hash"
}
},
- "script_version":"tetron:master"
+ "script_version":"you:master"
}
}
}
What if we want to run the pipeline on a different input block? One option is to define a new pipeline template, but would potentially result in clutter with many pipeline templates defined for one-off jobs. Instead, you can override values in the input of the component like this:
<notextile>
-<pre><code>$ <span class="userinput">arv pipeline run --template qr1hi-d1hrv-vxzkp38nlde9yyr do_hash::input=33a9f3842b01ea3fdf27cc582f5ea2af</span>
+<pre><code>$ <span class="userinput">arv pipeline run --template qr1hi-d1hrv-vxzkp38nlde9yyr do_hash::input=33a9f3842b01ea3fdf27cc582f5ea2af+242</span>
2013-12-17 20:31:24 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
do_hash qr1hi-8i9sb-rffhuay4jryl2n2 queued 2013-12-17T20:31:24Z
filter - -
h2. The one job per file pattern
-This example demonstrates how to schedule a new task per file. Because this is a common pattern, the Crunch Python API contains a convenience function to "queue a task for each input file":{{site.basedoc}}/sdk/python/crunch-utility-libraries.html#one_task_per_input which reduces the amount of boilerplate code required to handle parallel jobs.
+This example demonstrates how to schedule a new task per file. Because this is a common pattern, the Crunch Python API contains a convenience function to "queue a task for each input file":{{site.baseurl}}/sdk/python/crunch-utility-libraries.html#one_task_per_input which reduces the amount of boilerplate code required to handle parallel jobs.
+
+Next, "Constructing a Crunch pipeline":tutorial-new-pipeline.html
+
This tutorial introduces the Arvados Metadata Database. The Metadata Database stores information about files in Keep. This example will use the Python SDK to find public WGS (Whole Genome Sequencing) data for people who have reported a certain medical condition.
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.basedoc}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.basedoc}}/user/getting_started/check-environment.html*
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
In the tutorial examples, three angle brackets (>>>) will be used to denote code to enter at the interactive Python prompt.
if ($skip_install)
{
$ENV{"CRUNCH_SRC"} = $Job->{script_version};
- system("virtualenv", "$ENV{CRUNCH_TMP}/opt") == 0
- or croak ("virtualenv $ENV{CRUNCH_TMP}/opt failed: exit ".($?>>8));
- for my $src_path ("$ENV{CRUNCH_SRC}/arvados/sdk/python",
- "$ENV{CRUNCH_SRC}/sdk/python") {
+ for my $src_path ("$ENV{CRUNCH_SRC}/arvados/sdk/python") {
if (-d $src_path) {
- system ("cd $src_path && \$CRUNCH_TMP/opt/bin/python setup.py install")
+ system("virtualenv", "$ENV{CRUNCH_TMP}/opt") == 0
+ or croak ("virtualenv $ENV{CRUNCH_TMP}/opt failed: exit ".($?>>8));
+ system ("cd $src_path && ./build.sh && \$CRUNCH_TMP/opt/bin/python setup.py install")
== 0
or croak ("setup.py in $src_path failed: exit ".($?>>8));
}
open STDERR, ">&STDOUT";
mkdir $destdir;
-open TARX, "|-", "tar", "-C", $destdir, "-xf", "-";
-print TARX <DATA>;
-if(!close(TARX)) {
- die "'tar -C $destdir -xf -' exited $?: $!";
+my @git_archive_data = <DATA>;
+if (@git_archive_data) {
+ open TARX, "|-", "tar", "-C", $destdir, "-xf", "-";
+ print TARX @git_archive_data;
+ if(!close(TARX)) {
+ die "'tar -C $destdir -xf -' exited $?: $!";
+ }
}
my $pwd;
my $install_dir = $ENV{"CRUNCH_INSTALL"} || "$pwd/opt";
mkdir $install_dir;
-shell_or_die ("virtualenv", $install_dir);
-for my $src_path ("$destdir/arvados/sdk/python", "$destdir/sdk/python") {
+for my $src_path ("$destdir/arvados/sdk/python") {
if (-d $src_path) {
- shell_or_die ("cd $src_path && $install_dir/bin/python setup.py install");
+ shell_or_die ("virtualenv", $install_dir);
+ shell_or_die ("cd $src_path && ./build.sh && $install_dir/bin/python setup.py install");
}
}
import stat
import threading
import arvados
-import argparse
import pprint
from time import time
from arvados.fuse import *
import arvados
import subprocess
+import argparse
if __name__ == '__main__':
- api = arvados.api()
-
# Handle command line parameters
parser = argparse.ArgumentParser(
description='Mount Keep data under the local filesystem.',
PATH
remote: .
specs:
- arvados (0.1.20140127093947)
+ arvados (0.1.20140228213600)
activesupport (>= 3.2.13)
andand
google-api-client (~> 0.6.3)
json (>= 1.7.7)
- minitest (>= 5.0.0)
- rake
GEM
remote: https://rubygems.org/
specs:
- activesupport (3.2.16)
+ activesupport (3.2.17)
i18n (~> 0.6, >= 0.6.4)
multi_json (~> 1.0)
addressable (2.3.5)
DEPENDENCIES
arvados!
+ minitest (>= 5.0.0)
+ rake
end
def cancel
+ reload_object_before_update
@object.update_attributes cancelled_at: Time.now
show
end
class CollectionsApiTest < ActionDispatch::IntegrationTest
fixtures :all
- def jresponse
- @jresponse ||= ActiveSupport::JSON.decode @response.body
- end
-
test "should get index" do
get "/arvados/v1/collections", {:format => :json}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
assert_response :success
--- /dev/null
+require 'test_helper'
+
+class JobsApiTest < ActionDispatch::IntegrationTest
+ fixtures :all
+
+ test "cancel job" do
+ post "/arvados/v1/jobs/#{jobs(:running).uuid}/cancel", {:format => :json}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ assert_response :success
+ assert_equal "arvados#job", jresponse['kind']
+ assert_not_nil jresponse['cancelled_at']
+ end
+
+ test "cancel someone else's visible job" do
+ post "/arvados/v1/jobs/#{jobs(:barbaz).uuid}/cancel", {:format => :json}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:spectator).api_token}"}
+ assert_response 403
+ end
+
+ test "cancel someone else's invisible job" do
+ post "/arvados/v1/jobs/#{jobs(:running).uuid}/cancel", {:format => :json}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:spectator).api_token}"}
+ assert_response 404
+ end
+
+end
# Add more helper methods to be used by all tests here...
end
+class ActionDispatch::IntegrationTest
+ def jresponse
+ @jresponse ||= ActiveSupport::JSON.decode @response.body
+ end
+end
+
# Ensure permissions are computed from the test fixtures.
User.invalidate_permissions_cache
projects / arvados.git / commitdiff