cp -f -rlt ubuntu2004/generated common-generated/*
GOTARBALL=go1.13.4.linux-amd64.tar.gz
-NODETARBALL=node-v6.11.2-linux-x64.tar.xz
+NODETARBALL=node-v10.23.1-linux-x64.tar.xz
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
wget -cqO common-generated/$(GOTARBALL) https://dl.google.com/go/$(GOTARBALL)
common-generated/$(NODETARBALL): common-generated
- wget -cqO common-generated/$(NODETARBALL) https://nodejs.org/dist/v6.11.2/$(NODETARBALL)
+ wget -cqO common-generated/$(NODETARBALL) https://nodejs.org/dist/v10.23.1/$(NODETARBALL)
common-generated/$(RVMKEY1): common-generated
wget -cqO common-generated/$(RVMKEY1) https://rvm.io/mpapis.asc
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
+ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
# Need to "touch" RPM database to workaround bug in interaction between
# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
+ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
+ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
+ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
+ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
COMMAND_ARR+=('--license' "$LICENSE_STRING")
+ if [[ "$FORMAT" != "rpm" ]]; then
+ COMMAND_ARR+=('--conflicts' "python-$PKG")
+ fi
+
if [[ "$DEBUG" != "0" ]]; then
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
|| fatal "No gitolite. Try: apt-get install gitolite3"
echo -n 'npm: '
npm --version \
- || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v6.11.2/node-v6.11.2-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v6.11.2-linux-x64/bin/{node,npm} /usr/local/bin/"
+ || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v10.23.1-linux-x64/bin/{node,npm} /usr/local/bin/"
echo -n 'cadaver: '
cadaver --version | grep -w cadaver \
|| fatal "No cadaver. Try: apt-get install cadaver"
task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/R/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
vars = ['baseurl', 'arvados_cluster_uuid', 'arvados_api_host', 'arvados_workbench_host']
+ if ! ENV.key?('baseurl') || ENV['baseurl'] == ""
+ if !ENV.key?('WORKSPACE') || ENV['WORKSPACE'] == ""
+ puts "The `baseurl` variable was not specified and the `WORKSPACE` environment variable is not set. Defaulting `baseurl` to file://#{pwd}/.site"
+ ENV['baseurl'] = "file://#{pwd}/.site/"
+ else
+ puts "The `baseurl` variable was not specified, defaulting to a value derived from the `WORKSPACE` environment variable"
+ ENV['baseurl'] = "file://#{ENV['WORKSPACE']}/doc/.site/"
+ end
+ end
vars.each do |v|
if ENV[v]
website.config.h[v] = ENV[v]
- install/install-shell-server.html.textile.liquid
- install/install-webshell.html.textile.liquid
- install/install-arv-git-httpd.html.textile.liquid
- - Containers API:
+ - Containers API (cloud):
- install/install-jobs-image.html.textile.liquid
- install/crunch2-cloud/install-compute-node.html.textile.liquid
- install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid
- - install/crunch2-slurm/install-compute-node.html.textile.liquid
+ - Containers API (slurm):
- install/crunch2-slurm/install-dispatch.html.textile.liquid
+ - install/crunch2-slurm/configure-slurm.html.textile.liquid
+ - install/crunch2-slurm/install-compute-node.html.textile.liquid
- install/crunch2-slurm/install-test.html.textile.liquid
- External dependencies:
- install/install-postgresql.html.textile.liquid
To enable cgroups accounting, you must boot Linux with the command line parameters @cgroup_enable=memory swapaccount=1@.
+Currently Arvados is not compatible with the new cgroups accounting, also known as cgroups v2. Currently, all supported GNU/Linux distributions don't use cgroups v2 as default
+If you are using a distribution in the compute nodes that ships with cgroups v2 enabled, make sure to disable it by booting Linux with the command line parameters @systemd.unified_cgroup_hierarchy=0@.
+
After making changes, reboot the system to make these changes effective.
h3. Red Hat and CentOS
<notextile>
-<pre><code>~$ <span class="userinput">sudo grubby --update-kernel=ALL --args='cgroup_enable=memory swapaccount=1'</span>
+<pre><code>~$ <span class="userinput">sudo grubby --update-kernel=ALL --args='cgroup_enable=memory swapaccount=1 systemd.unified_cgroup_hierarchy=0'</span>
</code></pre>
</notextile>
h3. Debian and Ubuntu
-Open the file @/etc/default/grub@ in an editor. Find where the string @GRUB_CMDLINE_LINUX@ is set. Add @cgroup_enable=memory swapaccount=1@ to that string. Save the file and exit the editor. Then run:
+Open the file @/etc/default/grub@ in an editor. Find where the string @GRUB_CMDLINE_LINUX@ is set. Add @cgroup_enable=memory swapaccount=1 systemd.unified_cgroup_hierarchy=0@ to that string. Save the file and exit the editor. Then run:
<notextile>
<pre><code>~$ <span class="userinput">sudo update-grub</span>
+++ /dev/null
-user-management.html.textile.liquid
\ No newline at end of file
Similar settings should be added to @clsr2@ & @clsr3@ hosts, so that all clusters in the federation can talk to each other.
-The @ActivateUsers@ setting indicates whether users from a given cluster are automatically activated or they require manual activation. User activation is covered in more detail in the "user activation section":{{site.baseurl}}/admin/activation.html. In the current example, users from @clsr2@ would be automatically, activated, but users from @clsr3@ would require an admin to activate the account.
+The @ActivateUsers@ setting indicates whether users from a given cluster are automatically activated or they require manual activation. User activation is covered in more detail in the "user activation section":{{site.baseurl}}/admin/user-management.html. In the current example, users from @clsr2@ would be automatically, activated, but users from @clsr3@ would require an admin to activate the account.
+
+Note: The @Proxy:@ variable is intended for future use, and should always be set to @true@.
h2(#LoginCluster). User management
+++ /dev/null
-logging.html.textile.liquid
\ No newline at end of file
In these examples, @zzzzz-tpzed-3kz0nwtjehhl0u4@ is the sample user account. Replace with the uuid of the user you wish to manipulate.
-See "user management":{{site.baseurl}}/admin/activation.html for an overview of how to use these commands.
+See "user management":{{site.baseurl}}/admin/user-management.html for an overview of how to use these commands.
h3. Setup a user
h2. Adding Permissions
-h3. VM login
+h3(#vm-login). VM login
-Give @$user_uuid@ permission to log in to @$vm_uuid@ as @$target_username@
+Give @$user_uuid@ permission to log in to @$vm_uuid@ as @$target_username@ and make sure that @$target_username@ is a member of the @docker@ group
<pre>
user_uuid=xxxxxxxchangeme
"head_uuid":"$vm_uuid",
"link_class":"permission",
"name":"can_login",
-"properties":{"username":"$target_username"}
+"properties":{"username":"$target_username", "groups": [ "docker" ]}
}
EOF
</pre>
notextile. <div class="spaced-out">
# A new user record is not set up, and not active. An inactive user cannot create or update any object, but can read Arvados objects that the user account has permission to read (such as publicly available items readable by the "anonymous" user).
-# Using Workbench or the "command line":{{site.baseurl}}/install/cheat_sheet.html , the admin invokes @setup@ on the user. The setup method adds the user to the "All users" group.
+# Using Workbench or the "command line":{{site.baseurl}}/admin/user-management-cli.html , the admin invokes @setup@ on the user. The setup method adds the user to the "All users" group.
- If "Users.AutoSetupNewUsers":config.html is true, this happens automatically during user creation, so in that case new users start at step (3).
- If "Users.AutoSetupNewUsersWithRepository":config.html is true, a new git repo is created for the user.
- If "Users.AutoSetupNewUsersWithVmUUID":config.html is set, the user is given login permission to the specified shell node
notextile. </div>
-User management can be performed through the web using Workbench or the command line. See "user management at the CLI":{{site.baseurl}}/install/cheat_sheet.html for specific examples.
+User management can be performed through the web using Workbench or the command line. See "user management at the CLI":{{site.baseurl}}/admin/user-management-cli.html for specific examples.
h2(#user_agreements). User agreements and self-activation
h3. setup
-Set up a user. Adds the user to the "All users" group. Enables the user to invoke @activate@. See "user management":{{site.baseurl}}/admin/activation.html for details.
+Set up a user. Adds the user to the "All users" group. Enables the user to invoke @activate@. See "user management":{{site.baseurl}}/admin/user-management.html for details.
Arguments:
h3. activate
-Check that a user has is set up and has signed all the user agreements. If so, activate the user. Users can invoke this for themselves. See "user agreements":{{site.baseurl}}/admin/activation.html#user_agreements for details.
+Check that a user has is set up and has signed all the user agreements. If so, activate the user. Users can invoke this for themselves. See "user agreements":{{site.baseurl}}/admin/user-management.html#user_agreements for details.
Arguments:
h3. unsetup
-Remove the user from the "All users" group and deactivate the user. See "user management":{{site.baseurl}}/admin/activation.html for details.
+Remove the user from the "All users" group and deactivate the user. See "user management":{{site.baseurl}}/admin/user-management.html for details.
Arguments:
* @owner_uuid@ of the system user.
* @link_class@ "permission"
-* @name@ one of *can_read*, *can_write* or *can_manage*
+* @name@ one of *can_read*, *can_write*, *can_manage* or *can_login*
* @head_uuid@ of some Arvados object
* @tail_uuid@ of a User or Group. For Group, the @group_class@ must be a "role".
If a User has *can_manage* permission on some object, the user has the ability to read, create, update and delete permission links with @head_uuid@ of the managed object. In other words, the user has the ability to modify the permission grants on the object.
+The *can_login* @name@ is only meaningful on a permission link with with @tail_uuid@ a user UUID and @head_uuid@ a Virtual Machine UUID. A permission link of this type gives the user UUID permission to log into the Virtual Machine UUID. The username for the VM is specified in the @properties@ field. Group membership can be specified that way as well, optionally. See the "VM login section on the CLI cheat sheet":/install/cheat_sheet.html#vm-login for an example.
+
h3. Transitive permissions
Permissions can be obtained indirectly through nested ownership (*can_manage*) or by following multiple permission links.
h2. User activation
-"Creation and activation of new users is described here.":{{site.baseurl}}/admin/activation.html
+"Creation and activation of new users is described here.":{{site.baseurl}}/admin/user-management.html
h2. Creating tokens via the API
+++ /dev/null
-../admin/user-management-cli.html.textile.liquid
\ No newline at end of file
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install client libraries
-
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The "SDK Reference":{{site.baseurl}}/sdk/index.html page has installation instructions for each of the SDKs.
-
---
layout: default
navsection: installguide
-title: Set up Slurm
+title: Configure Slurm
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'notebox_begin_warning' %}
+crunch-dispatch-slurm is only relevant for on premises clusters that will spool jobs to Slurm. Skip this section if you are installing a cloud cluster.
+{% include 'notebox_end' %}
+
Containers can be dispatched to a Slurm cluster. The dispatcher sends work to the cluster using Slurm's @sbatch@ command, so it works in a variety of SLURM configurations.
In order to run containers, you must run the dispatcher as a user that has permission to set up FUSE mounts and run Docker containers on each compute node. This install guide refers to this user as the @crunch@ user. We recommend you create this user on each compute node with the same UID and GID, and add it to the @fuse@ and @docker@ system groups to grant it the necessary permissions. However, you can run the dispatcher under any account with sufficient permissions across the cluster.
+We will assume that you have Slurm and munge running.
-On the API server, install Slurm and munge, and generate a munge key.
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo /usr/bin/apt-get install slurm-llnl munge</span>
-~$ <span class="userinput">sudo /usr/sbin/create-munge-key</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install slurm munge slurm-munge</span>
-</code></pre>
-</notextile>
+h3. Sample Slurm configuration file
-Now we need to give Slurm a configuration file. On Debian-based systems, this is installed at @/etc/slurm-llnl/slurm.conf@. On Red Hat-based systems, this is installed at @/etc/slurm/slurm.conf@. Here's an example @slurm.conf@:
+Here's an example @slurm.conf@ for use with Arvados:
<notextile>
<pre><code>
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Containers API Slurm prerequisites
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
-If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token. Use the following command on the <strong>API server</strong> to create an anonymous user token. {% include 'install_rails_command' %}
+If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token.
+
+# First, generate a long random string and put it in the @config.yml@ file, in the @AnonymousUserToken@ field.
+# Then, use the following command on the <strong>API server</strong> to register the anonymous user token in the database. {% include 'install_rails_command' %}
<notextile>
<pre><code> Users:
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Overview
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{% comment %}
-Obsolete page, no longer in nav.
-{% endcomment %}
-
-<notextile><script>
-window.location = "install-manual-prerequisites.html";
-</script></notextile>
-
-Please proceed to "Prerequisites":install-manual-prerequisites.html.
|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional, but required by Workflow Composer.|
|\3=. *Crunch (running containers)*|
|"arvados-dispatch-cloud":crunch2-cloud/install-dispatch-cloud.html |Allocate and free cloud VM instances on demand based on workload.|Optional, not needed for a static Slurm cluster such as on-premises HPC.|
-|"crunch-dispatch-slurm":crunch2-slurm/install-prerequisites.html |Run analysis workflows using Docker containers distributed across a Slurm cluster.|Optional, not needed for a Cloud installation, or if you wish to use Arvados for data management only.|
+|"crunch-dispatch-slurm":crunch2-slurm/install-dispatch.html |Run analysis workflows using Docker containers distributed across a Slurm cluster.|Optional, not needed for a Cloud installation, or if you wish to use Arvados for data management only.|
h2(#identity). Identity provider
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install pre-built Docker images
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-This method is intended for evaluation and development on a local workstation. It is not suitable for production use in a cluster deployment.
-
-{% include 'notebox_begin' %}
-* The automatic network configuration allows you to log in to Workbench from a browser _running on the same host as Docker_. Connecting from other hosts requires additional configuration (not covered here).
-* Your data will be stored inside the Docker containers. You may stop and restart the containers without loss, but if you delete the container, your data will be gone.
-* Updating the Arvados software inside the Docker containers is not supported. You may download updated Docker images, but migrating data to updated containers is not yet supported.
-{% include 'notebox_end' %}
-
-h2. Prerequisites
-
-# A GNU/Linux x64 (virtual) machine
-# A working Docker installation (see "Installing Docker":https://docs.docker.com/installation/)
-# curl
-
-h2. Verify prerequisites
-
-Make sure that @curl@ and @docker@ are installed on your system, and that you are in the docker group (see "Installing Docker":https://docs.docker.com/installation/).
-
-<notextile><pre><code>~$ <span class="userinput">which curl</span>
-/usr/bin/curl
-~$ <span class="userinput">docker.io --version</span>
-Docker version 1.2.0-dev, build dc243c8
-~$ <span class="userinput">groups</span>
-yourusername sudo fuse docker
-</code></pre></notextile>
-
-h2. Download and install Arvados.
-
-<notextile>
-<pre><code>~$ <span class="userinput">\curl -sSL get.arvados.org | bash</span>
-</code></pre></notextile>
-
-This command will download the latest build of the Arvados docker images. It also gets the @arvdock@ command and saves it in the current working directory. It then uses @arvdock@ to spin up Arvados. Note that the Arvados Docker images are large and may take a while to download.
-
-If you prefer, you can also download and inspect the installation script before running it. @get.arvados.org@ redirects to "https://raw.githubusercontent.com/curoverse/arvados-dev/master/install/easy-docker-install.sh":https://raw.githubusercontent.com/curoverse/arvados-dev/master/install/easy-docker-install.sh, which is the installation script.
-
-The @arvdock@ command usage is listed here:
-
-<pre>
-usage: ./arvdock (start|stop|restart|reset|test) [options]
-
-start run new or restart stopped arvados containers
-stop stop arvados containers
-restart stop and then start arvados containers
-reset stop and delete containers WARNING: this will delete the data inside Arvados!
-test run tests
-
-./arvdock start/stop/restart options:
- -d[port], --doc[=port] Documentation server (default port 9898)
- -w[port], --workbench[=port] Workbench server (default port 9899)
- -s[port], --sso[=port] SSO server (default port 9901)
- -a[port], --api[=port] API server (default port 9900)
- -c, --compute Compute nodes (starts 2)
- -v, --vm Shell server
- -n, --nameserver Nameserver
- -k, --keep Keep servers
- -p, --keepproxy Keepproxy server
- -h, --help Display this help and exit
-
- If no options are given, the action is applied to all servers.
-
-./arvdock test [testname] [testname] ...
- By default, all tests are run.
-</pre>
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: Create a Workflow with Composer
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The Arvados Workflow Composer is a graphical interface for building Common Workflow Language (CWL) workflows to run on Arvados.
-
-This tutorial will demonstrate:
-
-# Creating a new git repository through Arvados to store the workflow
-# Creating CommandLineTools for "sort" and "uniq"
-# Creating a Workflow which uses "sort" and "uniq" to remove duplicate lines from a text file
-# Submitting the Workflow to run on Arvados
-
-h3. 1. Access from workbench
-
-!(screenshot)c1.png!
-
-h3. 2. Composer starting page
-
-!(screenshot)c2.png!
-
-h3. 3. Manage git repositories (opens Workbench in new tab)
-
-!(screenshot)c2b.png!
-
-h3. 4. Add a new repository
-
-!(screenshot)c4.png!
-
-!(screenshot)c3.png!
-
-h3. 5. Return to Composer. Use refresh button to discover new repository (may take a few moments to show up).
-
-!(screenshot)c2c.png!
-
-h3. 6. Create a new Command Line Tool
-
-!(screenshot)c5.png!
-
-!(screenshot)c20.png!
-
-h3. 7. Set Docker image, base command, and input port for "sort" tool
-
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:10@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
-
-!(screenshot)c6.png!
-
-h3. 8. Redirect stdout to a file
-
-!(screenshot)c7.png!
-
-h3. 9. Capture output file
-
-!(screenshot)c8.png!
-
-h3. 10. Save Command Line Tool
-
-!(screenshot)c22.png!
-
-h3. 11. Repeat steps 6-10 for "uniq" tool
-
-Create a new tool with a "base command" of "uniq".
-
-h3. 12. Switch back to "Home" tab and create workflow
-
-!(screenshot)c24.png!
-
-!(screenshot)c9.png!
-
-!(screenshot)c10.png!
-
-h3. 13. Drag and drop tools into Workflow
-
-!(screenshot)c11.png!
-
-h3. 14. Drag from input port of "sort" to empty space to create workflow input
-
-!(screenshot)c21.png!
-
-h3. 15. Drag from output port of "sort" to input port of "uniq"
-
-!(screenshot)c13.png!
-
-h3. 16. Drag from output port of "uniq" to empty space to create workflow output
-
-!(screenshot)c14.png!
-
-h3. 17. Save Workflow
-
-!(screenshot)c23.png!
-
-h3. 18. Click on "Test" tab then click "Run"
-
-!(screenshot)c15.png!
-
-h3. 19. Choose input file
-
-You may need to "upload an input file":{{site.baseurl}}/user/tutorials/tutorial-keep.html
-
-!(screenshot)c16.png!
-
-h3. 20. Run the workflow
-
-!(screenshot)c17.png!
-
-h3. 21. Monitor progress (may take several minutes)
-
-!(screenshot)c18.png!
-
-h3. 22. Get workflow output
-
-!(screenshot)c19.png!
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Getting started at the command line
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Many operations in Arvados can be performed using either the web Workbench or through command line tools. Some operations can only be done using the command line.
+
+To use the command line tools, you can either log into an Arvados-managed VM instance where those tools are pre-installed, or install the Arvados tools on your own system.
+
+To log into an Arvados-managed VM, see instructions for "Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html or "Unix":{{site.baseurl}}/user/getting_started/ssh-access-unix.html or "Windows":{{site.baseurl}}/user/getting_started/ssh-access-windows.html .
+
+To install the Arvados tools on your own system, you should install the "Command line SDK":{{site.baseurl}}/sdk/cli/install.html (requires Ruby) and "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html (requires Python). You may also want to install "arvados-cwl-runner":{{site.baseurl}}/sdk/python/arvados-cwl-runner.html to submit workflows and "arvados-fuse":{{site.baseurl}}/sdk/python/arvados-fuse.html to mount keep as a filesystem.
+
+Once you are logged in or have command line tools installed, see "getting an API token":{{site.baseurl}}/user/reference/api-tokens.html and "check your environment":{{site.baseurl}}/user/getting_started/check-environment.html .
+++ /dev/null
----
-layout: default
-navsection: userguide
-title: Introduction to Crunch
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The Arvados "Crunch" framework is designed to support processing very large data batches (gigabytes to terabytes) efficiently, and provides the following benefits:
-* Increase concurrency by running tasks asynchronously, using many CPUs and network interfaces at once (especially beneficial for CPU-bound and I/O-bound tasks respectively).
-* Track inputs, outputs, and settings so you can verify that the inputs, settings, and sequence of programs you used to arrive at an output is really what you think it was.
-* Ensure that your programs and workflows are repeatable with different versions of your code, OS updates, etc.
-* Interrupt and resume long-running jobs consisting of many short tasks.
-* Maintain timing statistics automatically, so they're there when you want them.
-
-h2. Prerequisites
-
-To get the most value out of this section, you should be comfortable with the following:
-
-# Using a secure shell client such as SSH or PuTTY to log on to a remote server
-# Using the Unix command line shell, Bash
-# Viewing and editing files using a unix text editor such as vi, Emacs, or nano
-# Revision control using Git
-
-We also recommend you read the "Arvados Platform Overview":https://dev.arvados.org/projects/arvados/wiki#Platform-Overview for an introduction and background information about Arvados.
# The e-mail address of the user you would like to become marked as an admin
# user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses OAuth2 against Google's servers, so in that case this
- # should be an address associated with a Google account.
AutoAdminUserWithEmail: ""
# If AutoAdminFirstUser is set to true, the first user to log in when no
NewUserNotificationRecipients: {}
NewInactiveUserNotificationRecipients: {}
- # Set AnonymousUserToken to enable anonymous user access. You can get
- # the token by running "bundle exec ./script/get_anonymous_user_token.rb"
- # in the directory where your API server is running.
+ # Set AnonymousUserToken to enable anonymous user access. Populate this
+ # field with a long random string. Then run "bundle exec
+ # ./script/get_anonymous_user_token.rb" in the directory where your API
+ # server is running to record the token in the database.
AnonymousUserToken: ""
# If a new user has an alternate email address (local@domain)
# The e-mail address of the user you would like to become marked as an admin
# user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses OAuth2 against Google's servers, so in that case this
- # should be an address associated with a Google account.
AutoAdminUserWithEmail: ""
# If AutoAdminFirstUser is set to true, the first user to log in when no
NewUserNotificationRecipients: {}
NewInactiveUserNotificationRecipients: {}
- # Set AnonymousUserToken to enable anonymous user access. You can get
- # the token by running "bundle exec ./script/get_anonymous_user_token.rb"
- # in the directory where your API server is running.
+ # Set AnonymousUserToken to enable anonymous user access. Populate this
+ # field with a long random string. Then run "bundle exec
+ # ./script/get_anonymous_user_token.rb" in the directory where your API
+ # server is running to record the token in the database.
AnonymousUserToken: ""
# If a new user has an alternate email address (local@domain)
"google.golang.org/api/people/v1"
)
-const (
+var (
tokenCacheSize = 1000
tokenCacheNegativeTTL = time.Minute * 5
tokenCacheTTL = time.Minute * 10
+ tokenCacheRaceWindow = time.Minute
)
type oidcLoginController struct {
return origFunc(ctx, opts)
}
// Check each token in the incoming request. If any
- // are OAuth2 access tokens, swap them out for Arvados
- // tokens.
+ // are valid OAuth2 access tokens, insert/update them
+ // in the database so RailsAPI's auth code accepts
+ // them.
for _, tok := range creds.Tokens {
err = ta.registerToken(ctx, tok)
if err != nil {
// Expiry time for our token is one minute longer than our
// cache TTL, so we don't pass it through to RailsAPI just as
// it's expiring.
- exp := time.Now().UTC().Add(tokenCacheTTL + time.Minute)
+ exp := time.Now().UTC().Add(tokenCacheTTL + tokenCacheRaceWindow)
var aca arvados.APIClientAuthorization
if updating {
if err != nil {
return err
}
+ aca.ExpiresAt = exp.Format(time.RFC3339Nano)
ta.cache.Add(tok, aca)
return nil
}
import (
"bytes"
"context"
+ "crypto/hmac"
+ "crypto/sha256"
"encoding/json"
"fmt"
+ "io"
"net/http"
"net/http/httptest"
"net/url"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/jmoiron/sqlx"
check "gopkg.in/check.v1"
)
c.Check(resp.RedirectLocation, check.Equals, "")
}
+func (s *OIDCLoginSuite) TestOIDCAuthorizer(c *check.C) {
+ s.cluster.Login.Google.Enable = false
+ s.cluster.Login.OpenIDConnect.Enable = true
+ json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
+ s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
+ s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
+ s.fakeProvider.ValidClientID = "oidc#client#id"
+ s.fakeProvider.ValidClientSecret = "oidc#client#secret"
+ db := arvadostest.DB(c, s.cluster)
+
+ tokenCacheTTL = time.Millisecond
+ tokenCacheRaceWindow = time.Millisecond
+
+ oidcAuthorizer := OIDCAccessTokenAuthorizer(s.cluster, func(context.Context) (*sqlx.DB, error) { return db, nil })
+ accessToken := s.fakeProvider.ValidAccessToken()
+
+ mac := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
+ io.WriteString(mac, accessToken)
+ hmac := fmt.Sprintf("%x", mac.Sum(nil))
+
+ cleanup := func() {
+ _, err := db.Exec(`delete from api_client_authorizations where api_token=$1`, hmac)
+ c.Check(err, check.IsNil)
+ }
+ cleanup()
+ defer cleanup()
+
+ ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{accessToken}})
+ var exp1 time.Time
+ oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ creds, ok := auth.FromContext(ctx)
+ c.Assert(ok, check.Equals, true)
+ c.Assert(creds.Tokens, check.HasLen, 1)
+ c.Check(creds.Tokens[0], check.Equals, accessToken)
+
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp1)
+ c.Check(err, check.IsNil)
+ c.Check(exp1.Sub(time.Now()) > -time.Second, check.Equals, true)
+ c.Check(exp1.Sub(time.Now()) < time.Second, check.Equals, true)
+ return nil, nil
+ })(ctx, nil)
+
+ // If the token is used again after the in-memory cache
+ // expires, oidcAuthorizer must re-checks the token and update
+ // the expires_at value in the database.
+ time.Sleep(3 * time.Millisecond)
+ oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
+ var exp time.Time
+ err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, hmac).Scan(&exp)
+ c.Check(err, check.IsNil)
+ c.Check(exp.Sub(exp1) > 0, check.Equals, true)
+ c.Check(exp.Sub(exp1) < time.Second, check.Equals, true)
+ return nil, nil
+ })(ctx, nil)
+}
+
func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
s.cluster.Login.Google.Enable = false
s.cluster.Login.OpenIDConnect.Enable = true
// Already finalized.
return
}
- mt, err := runner.LogCollection.MarshalManifest(".")
- if err != nil {
- err = fmt.Errorf("error creating log manifest: %v", err)
- return
- }
updates := arvadosclient.Dict{
- "name": "logs for " + runner.Container.UUID,
- "manifest_text": mt,
+ "name": "logs for " + runner.Container.UUID,
}
+ mt, err1 := runner.LogCollection.MarshalManifest(".")
+ if err1 == nil {
+ // Only send updated manifest text if there was no
+ // error.
+ updates["manifest_text"] = mt
+ }
+
+ // Even if flushing the manifest had an error, we still want
+ // to update the log record, if possible, to push the trash_at
+ // and delete_at times into the future. Details on bug
+ // #17293.
if final {
updates["is_trashed"] = true
} else {
updates["delete_at"] = exp
}
reqBody := arvadosclient.Dict{"collection": updates}
+ var err2 error
if runner.logUUID == "" {
reqBody["ensure_unique_name"] = true
- err = runner.DispatcherArvClient.Create("collections", reqBody, &response)
+ err2 = runner.DispatcherArvClient.Create("collections", reqBody, &response)
} else {
- err = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
+ err2 = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
}
- if err != nil {
- return
+ if err2 == nil {
+ runner.logUUID = response.UUID
+ }
+
+ if err1 != nil || err2 != nil {
+ err = fmt.Errorf("error recording logs: %q, %q", err1, err2)
}
- runner.logUUID = response.UUID
return
}
}
}
- nodejsversion := "v8.15.1"
+ nodejsversion := "v10.23.1"
if havenodejsversion, err := exec.Command("/usr/local/bin/node", "--version").CombinedOutput(); err == nil && string(havenodejsversion) == nodejsversion+"\n" {
logger.Print("nodejs " + nodejsversion + " already installed")
} else {
logger.info("%s submitted container_request %s", self.arvrunner.label(self), response["uuid"])
+ workbench1 = self.arvrunner.api.config()["Services"]["Workbench1"]["ExternalURL"]
+ workbench2 = self.arvrunner.api.config()["Services"]["Workbench2"]["ExternalURL"]
+ url = ""
+ if workbench2:
+ url = "{}processes/{}".format(workbench2, response["uuid"])
+ elif workbench1:
+ url = "{}container_requests/{}".format(workbench1, response["uuid"])
+ if url:
+ logger.info("Monitor workflow progress at %s", url)
+
+
def done(self, record):
try:
container = self.arvrunner.api.containers().get(
def arv_executor(self, updated_tool, job_order, runtimeContext, logger=None):
self.debug = runtimeContext.debug
- logger.info("Using cluster %s (%s)", self.api.config()["ClusterID"], self.api.config()["Services"]["Controller"]["ExternalURL"])
+ workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"]
+ workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"]
+ controller = self.api.config()["Services"]["Controller"]["ExternalURL"]
+ logger.info("Using cluster %s (%s)", self.api.config()["ClusterID"], workbench2 or workbench1 or controller)
updated_tool.visit(self.check_features)
if runtimeContext.submit and isinstance(tool, Runner):
logger.info("Final output collection %s", tool.final_output)
+ if workbench2 or workbench1:
+ logger.info("Output at %scollections/%s", workbench2 or workbench1, tool.final_output)
else:
if self.output_name is None:
self.output_name = "Output of %s" % (shortname(tool.tool["id"]))
req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
req.Header.Set("X-Amz-Date", date)
req.Host = "host.example.com"
+ c.Assert(err, check.IsNil)
obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
if !c.Check(err, check.IsNil) {
log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
if notFoundBlocks > 0 {
- return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)
+ return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix", notFoundBlocks, totalBlocks)
}
return nil
allLocators = append(allLocators, TestHash2)
err := performKeepBlockCheck(kc, blobSignatureTTL, "", allLocators, true)
c.Check(err, NotNil)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
err = performKeepBlockCheck(kc, blobSignatureTTL, "", locators, true)
c.Check(err, NotNil)
// Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
}
setupKeepBlockCheck(c, true, "")
setupTestData(c)
err := performKeepBlockCheck(kc, blobSignatureTTL, "badblobsigningkey", []string{TestHash, TestHash2}, false)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403")
// verbose logging not requested
c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, false)
func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) {
setupKeepBlockCheck(c, false, testKeepServicesJSON)
err := performKeepBlockCheck(kc, blobSignatureTTL, "", []string{TestHash, TestHash2}, true)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "")
}
args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
err := doMain(args)
c.Check(err, NotNil)
- c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true)
}
projects / arvados.git / commitdiff
