tgt = Dir.pwd
Dir.mkdir("sdk/R")
Dir.mkdir("sdk/R/arvados")
+ puts("tgt", tgt)
+ cp('css/R.css', 'sdk/R/arvados')
docfiles = []
Dir.chdir("../sdk/R/") do
STDERR.puts `Rscript createDoc.R README.Rmd #{tgt}/sdk/R/README.md 2>&1`
Dir.chdir("../sdk/java-v2") do
STDERR.puts `gradle javadoc 2>&1`
raise if $? != 0
+ puts `sed -i "s/@import.*dejavu.css.*//g" build/docs/javadoc/stylesheet.css`
+ raise if $? != 0
end
cp_r("../sdk/java-v2/build/docs/javadoc", "sdk/java-v2")
raise if $? != 0
- R:
- sdk/R/index.html.md
- sdk/R/arvados/index.html.textile.liquid
- - Perl:
- - sdk/perl/index.html.textile.liquid
- - sdk/perl/example.html.textile.liquid
- Ruby:
- sdk/ruby/index.html.textile.liquid
- sdk/ruby/example.html.textile.liquid
- Java v1:
- sdk/java/index.html.textile.liquid
- sdk/java/example.html.textile.liquid
+ - Perl:
+ - sdk/perl/index.html.textile.liquid
+ - sdk/perl/example.html.textile.liquid
api:
- Concepts:
- api/index.html.textile.liquid
admin:
- Topics:
- admin/index.html.textile.liquid
- - Configuration:
- - admin/config.html.textile.liquid
- - admin/federation.html.textile.liquid
- - Upgrading and migrations:
- - admin/upgrading.html.textile.liquid
- - admin/config-migration.html.textile.liquid
- Users and Groups:
- admin/user-management.html.textile.liquid
- admin/reassign-ownership.html.textile.liquid
- admin/user-management-cli.html.textile.liquid
- admin/group-management.html.textile.liquid
+ - admin/federation.html.textile.liquid
- admin/merge-remote-account.html.textile.liquid
- admin/migrating-providers.html.textile.liquid
- user/topics/arvados-sync-groups.html.textile.liquid
- install/arvados-on-kubernetes.html.textile.liquid
- Manual installation:
- install/install-manual-prerequisites.html.textile.liquid
- - install/install-components.html.textile.liquid
+ - install/packages.html.textile.liquid
+ - admin/upgrading.html.textile.liquid
+ - Configuration:
+ - install/config.html.textile.liquid
+ - admin/config-migration.html.textile.liquid
+ - admin/config.html.textile.liquid
- Core:
- - install/install-postgresql.html.textile.liquid
- install/install-api-server.html.textile.liquid
- - install/install-controller.html.textile.liquid
- Keep:
- install/install-keepstore.html.textile.liquid
- install/configure-fs-storage.html.textile.liquid
- install/install-keep-web.html.textile.liquid
- install/install-keep-balance.html.textile.liquid
- User interface:
+ - install/setup-login.html.textile.liquid
- install/install-sso.html.textile.liquid
- install/install-workbench-app.html.textile.liquid
- install/install-workbench2-app.html.textile.liquid
- install/install-composer.html.textile.liquid
- Additional services:
- install/install-ws.html.textile.liquid
- - install/install-shell-server.html.textile.liquid
- install/install-arv-git-httpd.html.textile.liquid
- - Containers API support on SLURM:
- - install/crunch2-slurm/install-prerequisites.html.textile.liquid
- - install/crunch2-slurm/install-slurm.html.textile.liquid
+ - install/install-shell-server.html.textile.liquid
+ - Containers API:
- install/crunch2-slurm/install-compute-node.html.textile.liquid
+ - install/install-jobs-image.html.textile.liquid
+ - install/install-dispatch-cloud.html.textile.liquid
- install/crunch2-slurm/install-dispatch.html.textile.liquid
- install/crunch2-slurm/install-test.html.textile.liquid
- - install/install-nodemanager.html.textile.liquid
- - install/install-compute-ping.html.textile.liquid
- - Containers API support on cloud (beta):
- - install/install-dispatch-cloud.html.textile.liquid
+ - External dependencies:
+ - install/install-postgresql.html.textile.liquid
+ - install/ruby.html.textile.liquid
+ - install/nginx.html.textile.liquid
+ - install/google-auth.html.textile.liquid
+ - install/install-docker.html.textile.liquid
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Note that each volume has a UUID, like @zzzzz-nyw5e-0123456789abcde@. You assign these manually: replace @zzzzz@ with your cluster ID, and replace @0123456789abcde@ with an arbitrary string of 15 alphanumerics. Once assigned, UUIDs should not be changed.
+Note that each volume has a UUID, like @zzzzz-nyw5e-0123456789abcde@. You assign these manually: replace @zzzzz@ with your Cluster ID, and replace @0123456789abcde@ with an arbitrary unique string of 15 alphanumerics. Once assigned, UUIDs should not be changed.
+
+Essential configuration values are highlighted in <span class="userinput">red</span>. Remaining parameters are provided for documentation, with their default values.
\ No newline at end of file
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Install Docker
+h2(#cgroups). Configure Linux cgroups accounting
-Compute nodes must have Docker installed to run containers. This requires a relatively recent version of Linux (at least upstream version 3.10, or a distribution version with the appropriate patches backported). Follow the "Docker Engine installation documentation":https://docs.docker.com/ for your distribution.
-
-For Debian-based systems, the Arvados package repository includes a backported @docker.io@ package with a known-good version you can install.
-
-h2(#configure_docker_daemon). Configure the Docker daemon
-
-Crunch runs Docker containers with relatively little configuration. You may need to start the Docker daemon with specific options to make sure these jobs run smoothly in your environment. This section highlights options that are useful to most installations. Refer to the "Docker daemon reference":https://docs.docker.com/reference/commandline/daemon/ for complete information about all available options.
-
-The best way to configure these options varies by distribution.
-
-* If you're using our backported @docker.io@ package, you can list these options in the @DOCKER_OPTS@ setting in @/etc/default/docker.io@.
-* If you're using another Debian-based package, you can list these options in the @DOCKER_OPTS@ setting in @/etc/default/docker@.
-* On Red Hat-based distributions, you can list these options in the @other_args@ setting in @/etc/sysconfig/docker@.
-
-h3. Default ulimits
-
-Docker containers inherit ulimits from the Docker daemon. However, the ulimits for a single Unix daemon may not accommodate a long-running Crunch job. You may want to increase default limits for compute containers by passing @--default-ulimit@ options to the Docker daemon. For example, to allow containers to open 10,000 files, set @--default-ulimit nofile=10000:10000@.
+Linux can report what compute resources are used by processes in a specific cgroup or Docker container. Crunch can use these reports to share that information with users running compute work. This can help pipeline authors debug and optimize their workflows.
-h3. DNS
+To enable cgroups accounting, you must boot Linux with the command line parameters @cgroup_enable=memory swapaccount=1@.
-Your containers must be able to resolve the hostname of your API server and any hostnames returned in Keep service records. If these names are not in public DNS records, you may need to specify a DNS resolver for the containers by setting the @--dns@ address to an IP address of an appropriate nameserver. You may specify this option more than once to use multiple nameservers.
+After making changes, reboot the system to make these changes effective.
-h2. Configure Linux cgroups accounting
+h3. Red Hat and CentOS
-Linux can report what compute resources are used by processes in a specific cgroup or Docker container. Crunch can use these reports to share that information with users running compute work. This can help pipeline authors debug and optimize their workflows.
+<notextile>
+<pre><code>~$ <span class="userinput">sudo grubby --update-kernel=ALL --args='cgroup_enable=memory swapaccount=1'</span>
+</code></pre>
+</notextile>
-To enable cgroups accounting, you must boot Linux with the command line parameters @cgroup_enable=memory swapaccount=1@.
+h3. Debian and Ubuntu
-On Debian-based systems, open the file @/etc/default/grub@ in an editor. Find where the string @GRUB_CMDLINE_LINUX@ is set. Add @cgroup_enable=memory swapaccount=1@ to that string. Save the file and exit the editor. Then run:
+Open the file @/etc/default/grub@ in an editor. Find where the string @GRUB_CMDLINE_LINUX@ is set. Add @cgroup_enable=memory swapaccount=1@ to that string. Save the file and exit the editor. Then run:
<notextile>
<pre><code>~$ <span class="userinput">sudo update-grub</span>
</code></pre>
</notextile>
-On Red Hat-based systems, run:
+h2(#install_docker). Install Docker
+
+Compute nodes must have Docker installed to run containers. This requires a relatively recent version of Linux (at least upstream version 3.10, or a distribution version with the appropriate patches backported). Follow the "Docker Engine installation documentation":https://docs.docker.com/install/ for your distribution.
+
+Make sure Docker is enabled to start on boot:
<notextile>
-<pre><code>~$ <span class="userinput">sudo grubby --update-kernel=ALL --args='cgroup_enable=memory swapaccount=1'</span>
+<pre><code># <span class="userinput">systemctl enable --now docker</span>
</code></pre>
</notextile>
-Finally, reboot the system to make these changes effective.
-
-h2. Create a project for Docker images
+h2(#configure_docker_daemon). Configure the Docker daemon
-Here we create a default project for the standard Arvados Docker images, and give all users read access to it. The project is owned by the system user.
+Depending on your anticipated workload or cluster configuration, you may need to tweak Docker options.
-<notextile>
-<pre><code>~$ <span class="userinput">uuid_prefix=`arv --format=uuid user current | cut -d- -f1`</span>
-~$ <span class="userinput">project_uuid=`arv --format=uuid group create --group "{\"owner_uuid\":\"$uuid_prefix-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}"`</span>
-~$ <span class="userinput">echo "Arvados project uuid is '$project_uuid'"</span>
-~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
-<span class="userinput">{
- "tail_uuid":"$all_users_group_uuid",
- "head_uuid":"$project_uuid",
- "link_class":"permission",
- "name":"can_read"
-}
-EOF</span>
-</code></pre></notextile>
-
-h2. Download and tag the latest arvados/jobs docker image
-
-In order to start workflows from workbench, there needs to be Docker image tagged @arvados/jobs:latest@. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep, and tags it as 'latest'. In this example @$project_uuid@ should be the UUID of the "Arvados Standard Docker Images" project.
+For information about how to set configuration options for the Docker daemon, see https://docs.docker.com/config/daemon/systemd/
-<notextile>
-<pre><code>~$ <span class="userinput">arv-keepdocker --pull arvados/jobs latest --project-uuid $project_uuid</span>
-</code></pre></notextile>
+h3. Changing ulimits
-If the image needs to be downloaded from Docker Hub, the command can take a few minutes to complete, depending on available network bandwidth.
+Docker containers inherit ulimits from the Docker daemon. However, the ulimits for a single Unix daemon may not accommodate a long-running Crunch job. You may want to increase default limits for compute containers by passing @--default-ulimit@ options to the Docker daemon. For example, to allow containers to open 10,000 files, set @--default-ulimit nofile=10000:10000@.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Configure FUSE
+h2(#fuse). Update fuse.conf
FUSE must be configured with the @user_allow_other@ option enabled for Crunch to set up Keep mounts that are readable by containers. Install this file as @/etc/fuse.conf@:
<notextile>
<pre>
-# Set the maximum number of FUSE mounts allowed to non-root users.
-# The default is 1000.
-#
-#mount_max = 1000
-
# Allow non-root users to specify the 'allow_other' or 'allow_root'
# mount options.
-#
user_allow_other
</pre>
</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+
+
<notextile>
-<pre><code>~$ <span class="userinput">sudo /usr/bin/apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7</span>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install gnupg</span>
+# <span class="userinput">/usr/bin/apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7</span>
</code></pre>
</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Configure the Docker cleaner
+h2(#docker-cleaner). Update docker-cleaner.json
-The arvados-docker-cleaner program removes least recently used Docker images as needed to keep disk usage below a configured limit.
-
-{% include 'notebox_begin' %}
-This also removes all containers as soon as they exit, as if they were run with @docker run --rm@. If you need to debug or inspect containers after they stop, temporarily stop arvados-docker-cleaner or configure it with @"RemoveStoppedContainers":"never"@.
-{% include 'notebox_end' %}
+The @arvados-docker-cleaner@ program removes least recently used Docker images as needed to keep disk usage below a configured limit.
Create a file @/etc/arvados/docker-cleaner/docker-cleaner.json@ in an editor, with the following contents.
*Choosing a quota:* Most deployments will want a quota that's at least 10G. From there, a larger quota can help reduce compute overhead by preventing reloading the same Docker image repeatedly, but will leave less space for other files on the same storage (usually Docker volumes). Make sure the quota is less than the total space available for Docker images.
-Restart the service after updating the configuration file.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-docker-cleaner</span>
-</code></pre>
-</notextile>
-
-*If you are using a different daemon supervisor,* or if you want to test the daemon in a terminal window, run @arvados-docker-cleaner@. Run @arvados-docker-cleaner --help@ for more configuration options.
+{% include 'notebox_begin' %}
+This also removes all containers as soon as they exit, as if they were run with @docker run --rm@. If you need to debug or inspect containers after they stop, temporarily stop arvados-docker-cleaner or configure it with @"RemoveStoppedContainers":"never"@.
+{% include 'notebox_end' %}
+++ /dev/null
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-On a Debian-based system, install the following packages:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install git curl</span>
-</code></pre>
-</notextile>
-
-On a Red Hat-based system, install the following packages:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install git curl</span>
-</code></pre>
-</notextile>
--- /dev/null
+{% comment %}
+packages_to_install should be a list
+fallback on arvados_component if not defined
+{% endcomment %}
+
+{% if package_to_install == nil %}
+ {% assign packages_to_install = arvados_component | split: " " %}
+{% endif %}
+
+h2(#install-packages). Install {{packages_to_install | join: " and " }}
+
+h3. Red Hat and Centos
+
+<notextile>
+<pre><code># <span class="userinput">yum install {{packages_to_install | join: " "}}</span>
+</code></pre>
+</notextile>
+
+h3. Debian and Ubuntu
+
+<notextile>
+<pre><code># <span class="userinput">apt-get install {{packages_to_install | join " "}}</span>
+</code></pre>
+</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-<ol>
+<ol class=>
<li>Start a shell for the postgres user:
-<notextile><pre>~$ <span class="userinput">sudo -u postgres bash</span></pre></notextile>
+<notextile><pre># <span class="userinput">su postgres</span></pre></notextile>
</li>
<li>Generate a new database password:
-<notextile><pre>$ <span class="userinput">ruby -e 'puts rand(2**128).to_s(36)'</span>
+<notextile><pre>postgres$ <span class="userinput"><span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c25; echo</span>
yourgeneratedpassword
</pre></notextile> Record this. You'll need it when you set up the Rails server later.
</li>
<li>Create a database user with the password you generated:
- <notextile><pre><code>$ <span class="userinput">createuser --encrypted -R -S --pwprompt {{service_role}}</span>
+ <notextile><pre><code>postgres$ <span class="userinput">createuser --encrypted --no-createrole --no-superuser --pwprompt {{service_role}}</span>
Enter password for new role: <span class="userinput">yourgeneratedpassword</span>
Enter it again: <span class="userinput">yourgeneratedpassword</span></code></pre></notextile>
</li>
<li>Create a database owned by the new user:
- <notextile><pre><code>$ <span class="userinput">createdb {{service_database}} -T template0 -E UTF8 -O {{service_role}}</span></code></pre></notextile>
+ <notextile><pre><code>postgres$ <span class="userinput">createdb {{service_database}} -T template0 -E UTF8 -O {{service_role}}</span></code></pre></notextile>
</li>
{% if use_contrib %}
<li>Enable the pg_trgm extension
- <notextile><pre>$ <span class="userinput">psql {{service_database}} -c "CREATE EXTENSION IF NOT EXISTS pg_trgm"</span></pre></notextile>
+ <notextile><pre>postgres$ <span class="userinput">psql {{service_database}} -c "CREATE EXTENSION IF NOT EXISTS pg_trgm"</span></pre></notextile>
</li>
{% endif %}
<li>Exit the postgres user shell:
- <notextile><pre>$ <span class="userinput">exit</span></pre></notextile>
+ <notextile><pre>postgres$ <span class="userinput">exit</span></pre></notextile>
</li>
</ol>
{% assign railscmd = "bundle exec rails console" %}
{% endunless %}
-Using RVM:
-
-<notextile>
-<pre><code>{{railshost}}~$ <span class="userinput">cd {{railsdir}}</span>
-{{railshost}}{{railsdir}}$ <span class="userinput">sudo -u <b>webserver-user</b> RAILS_ENV=production `which rvm-exec` default {{railscmd}}</span>
-{% if railsout %}{{railsout}}
-{% endif %}</code></pre>
-</notextile>
-
-Not using RVM:
-
<notextile>
<pre><code>{{railshost}}~$ <span class="userinput">cd {{railsdir}}</span>
{{railshost}}{{railsdir}}$ <span class="userinput">sudo -u <b>webserver-user</b> RAILS_ENV=production {{railscmd}}</span>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The Curoverse signing key fingerprint is
+The Arvados signing key fingerprint is
<notextile>
-<pre><code>
-pub 2048R/1078ECD7 2010-11-15 Curoverse, Inc Automatic Signing Key <sysadmin@curoverse.com>
- Key fingerprint = B2DA 2991 656E B4A5 0314 CA2B 5716 5911 1078 ECD7
-sub 2048R/5A8C5A93 2010-11-15
+<pre><code>pub rsa2048 2010-11-15 [SC]
+ B2DA 2991 656E B4A5 0314 CA2B 5716 5911 1078 ECD7
+uid [ unknown] Arvados Automatic Signing Key <sysadmin@arvados.org>
+uid [ unknown] Curoverse, Inc Automatic Signing Key <sysadmin@curoverse.com>
+sub rsa2048 2010-11-15 [E]
</code></pre>
</notextile>
Ruby 2.5 is recommended; Ruby 2.3 is also known to work.
-h4(#rvm). *Option 1: Install with RVM*
+* "Option 1: Install from packages":#packages
+* "Option 2: Install with RVM":#rvm
+* "Option 3: Install from source":#fromsource
+
+h2(#packages). Option 1: Install from packages
+
+h3. Centos 7
+
+The Ruby version shipped with Centos 7 is too old. Use "RVM.":#rvm
+
+h3. Debian and Ubuntu
+
+<notextile>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install ruby-dev bundler</span></code></pre>
+</notextile>
+
+h2(#rvm). Option 2: Install with RVM
+
+h3. Install gpg and curl
+
+h4. Centos 7
+
+<pre>
+yum install gpg curl which
+</pre>
+
+h4. Debian and Ubuntu
+
+<pre>
+apt-get --no-install-recommends install gpg curl
+</pre>
+
+h3. Install RVM
+
+<notextile>
+<pre><code># <span class="userinput">gpg --keyserver pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.5
+</span></code></pre></notextile>
+
+To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
-<pre><code><span class="userinput">sudo gpg --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | sudo bash -s stable --ruby=2.5
+<pre><code><span class="userinput">. /usr/local/rvm/scripts/rvm
</span></code></pre></notextile>
-Either log out and log back in to activate RVM, or explicitly load it in all open shells like this:
+Alternately you can use @rvm-exec@ (the first parameter is the ruby version to use, or "default"), for example:
<notextile>
-<pre><code><span class="userinput">source /usr/local/rvm/scripts/rvm
+<pre><code><span class="userinput">rvm-exec default rails console
</span></code></pre></notextile>
-Once RVM is activated in your shell, install Bundler:
+Finally, install Bundler:
<notextile>
<pre><code>~$ <span class="userinput">gem install bundler</span>
</code></pre></notextile>
-h4(#fromsource). *Option 2: Install from source*
+h2(#fromsource). Option 3: Install from source
Install prerequisites for Debian 8:
}
</code></pre>|
|Temporary directory|@tmp@|@"capacity"@: capacity (in bytes) of the storage device.
-@"device_type"@ (optional, default "network"): one of @{"ram", "ssd", "disk", "network"}@ indicating the acceptable level of performance.
+@"device_type"@ (optional, default "network"): one of @{"ram", "ssd", "disk", "network"}@ indicating the acceptable level of performance. (*note: not yet implemented as of v1.5*)
At container startup, the target path will be empty. When the container finishes, the content will be discarded. This will be backed by a storage mechanism no slower than the specified type.|<pre><code>{
"kind":"tmp",
"capacity":100000000000
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% assign highlighturl = "" %}
+{% for section in site.navbar[page.navsection] %}
+ {% for entry in section %}
+ {% comment %}
+ Want to highlight the current page on the left nav.
+ But some pages have been renamed with a symlink from the old page to the new one.
+ Then the URL won't match.
+ So if the URL doesn't match, as a fallback look for a page with a matching title.
+ {% endcomment %}
+
+ {% for item in entry[1] %}
+ {% if site.pages[item].url == page.url %}
+ {% assign highlighturl = site.pages[item].url %}
+ {% endif %}
+ {% endfor %}
+
+ {% if highlighturl == "" %}
+ {% for item in entry[1] %}
+ {% if site.pages[item].title == page.title %}
+ {% assign highlighturl = site.pages[item].url %}
+ {% endif %}
+ {% endfor %}
+ {% endif %}
+ {% endfor %}
+{% endfor %}
+
<div class="col-sm-3">
<div class="well">
<ol class="nav nav-list">
<li><span class="nav-header">{{ entry[0] }}</span>
<ol class="nav nav-list">
{% for item in entry[1] %}
- {% assign p = site.pages[item] %}
- <li {% if p.url == page.url or p.title == page.title %} class="active activesubnav" {% elsif p.title == page.subnavsection %} class="activesubnav" {% endif %}>
- <a href="{{ site.baseurl }}{{ p.url }}">{{ p.title }}</a></li>
+ {% assign p = site.pages[item] %}
+ <li {% if p.url == highlighturl %} class="active activesubnav" {% elsif p.title == page.subnavsection %} class="activesubnav" {% endif %}>
+
<a href="{{ site.baseurl }}{{ p.url }}">{{ p.title }}</a></li>
{% endfor %}
</ol>
{% endfor %}
<li><a href="https://arvados.org" style="padding-left: 2em">arvados.org »</a></li>
</ul>
- <div class="pull-right" style="padding-top: 6px">
+ <div class="pull-right" style="padding-top: 6px; padding-right: 25px">
<form method="get" action="https://www.google.com/search">
<div class="input-group" style="width: 220px">
<input type="text" class="form-control" name="q" placeholder="search">
</form>
</div>
</div>
+
+ <div class="alert alert-block alert-info" style="display: none;" id="annotate-notify">
+ <div style="margin-top: -26px; font-size: 12pt">Hey! You can use the annotation sidebar from <a href="https://hypothes.is">hypothes.is</a> to make public comments and private notes
+ <span style="font-size: 32pt">→</span></div>
+ <button type="button" class="close" onclick="dismissAnnotateNotify()">Got it</button>
+ </div>
+
+ <script>
+ function dismissAnnotateNotify() {
+ window.sessionStorage.setItem("dismiss-annotate-notify", "true");
+ $('#annotate-notify').attr('style', "display: none;");
+ }
+ if (window.sessionStorage.getItem("dismiss-annotate-notify") === "true") {
+ dismissAnnotateNotify();
+ } else {
+ $('#annotate-notify').attr('style', "display: inline-block;");
+ }
+ </script>
+
</div>
</div>
--- /dev/null
+h2(#restart-api). Restart the API server and controller
+
+*Make sure the cluster config file is up to date on the API server host* then restart the API server and controller processes to ensure the configuration changes are visible to the whole cluster.
+
+<notextile>
+<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
+</code></pre>
+</notextile>
--- /dev/null
+h2(#start-service). Start the service
+
+<notextile>
+<pre><code># <span class="userinput">systemctl enable --now {{arvados_component}}</span>
+# <span class="userinput">systemctl status {{arvados_component}}</span>
+[...]
+</code></pre>
+</notextile>
+
+If @systemctl status@ indicates it is not running, use @journalctl@ to check logs for errors:
+
+<notextile>
+<pre><code># <span class="userinput">journalctl -n12 --unit {{arvados_component}}</span>
+</code></pre>
+</notextile>
<link href="{{ site.baseurl }}/css/carousel-override.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/button-override.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/images.css" rel="stylesheet">
+ <script src="{{ site.baseurl }}/js/jquery.min.js"></script>
+ <script src="https://hypothes.is/embed.js" async></script>
<style>
html {
height:100%;
padding-top: 61px;
margin-top: -61px;
}
+
+ #annotate-notify { position: fixed; right: 40px; top: 3px; }
</style>
<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
{{ content }}
{% else %}
- <div class="container-fluid">
+ <div class="container-fluid" style="padding-right: 30px">
+
<div class="row">
{% include 'navbar_left' %}
<div class="col-sm-9">
</div>
{% endif %}
- <script src="{{ site.baseurl }}/js/jquery.min.js"></script>
- <script src="{{ site.baseurl }}/js/bootstrap.min.js"></script>
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
</p>
{% endif %}
-
</body>
</html>
---
layout: default
-navsection: admin
-title: Migrating Configuration
+navsection: installguide
+title: Migrating Configuration from v1.4 to v1.5
...
{% comment %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados is migrating to a centralized configuration file for all components. The centralized Arvados configuration is @/etc/arvados/config.yml@. Components that support the new centralized configuration are listed below. Components not listed here do not yet support centralized configuration. During the migration period, legacy configuration files will continue to be loaded and take precedence over the centralized configuration file.
+{% include 'notebox_begin_warning' %}
+_New installations of Arvados 1.5+ can skip this section_
+{% include 'notebox_end' %}
+
+Arvados 1.5 migrates to a centralized configuration file for all components. The centralized Arvados configuration is @/etc/arvados/config.yml@. Components that support the new centralized configuration are listed below. During the migration period, legacy configuration files are still loaded and take precedence over the centralized configuration file.
h2. API server
This command will also report if no migrations are required.
-h2. crunch-dispatch-slurm
-
-Currently only reads @InstanceTypes@ from centralized configuration. Still requires component-specific configuration file.
-
-h2(#keepstore). keepstore
-
-The legacy keepstore config (loaded from @/etc/arvados/keepstore/keepstore.yml@ or a different location specified via -legacy-keepstore-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/keepstore/keepstore.yml@ and stop using the -legacy-keepstore-config argument.
+h2. keepstore, keep-web, crunch-dispatch-slurm, arvados-ws, keepproxy, arv-git-httpd, keep-balance
-To migrate a keepstore node's configuration, first install @arvados-server@. Run @arvados-server config-check@, review and apply the recommended changes to @/etc/arvados/config.yml@, and run @arvados-server config-check@ again to check for additional warnings and recommendations. When you are satisfied, delete the legacy config file, restart keepstore, and check its startup logs. Copy the updated centralized config file to your next keepstore server, and repeat the process there.
+The legacy config for each component (loaded from @/etc/arvados/component/component.yml@ or a different location specified via the -legacy-component-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/component/component.yml@ and/or stop using the corresponding -legacy-component-config argument.
-After migrating and removing all legacy keepstore config files, make sure the @/etc/arvados/config.yml@ file is identical across all system nodes -- API server, keepstore, etc. -- and restart all services to make sure they are using the latest configuration.
+To migrate a component configuration, do this on each node that runs an Arvados service:
-h2(#keepproxy). keepproxy
+# Ensure that the latest @config.yml@ is installed on the current node
+# Install @arvados-server@ using @apt-get@ or @yum@.
+# Run @arvados-server config-check@, review and apply the recommended changes to @/etc/arvados/config.yml@
+# After applying changes, re-run @arvados-server config-check@ again to check for additional warnings and recommendations.
+# When you are satisfied, delete the legacy config file, restart the service, and check its startup logs.
+# Copy the updated @config.yml@ file to your next node, and repeat the process there.
-The legacy keepproxy config (loaded from @/etc/arvados/keepproxy/keepproxy.yml@ or a different location specified via -legacy-keepproxy-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/keepproxy/keepproxy.yml@ and stop using the -legacy-keepproxy-config argument.
+After migrating and removing all legacy config files, make sure the @/etc/arvados/config.yml@ file is identical across all system nodes -- API server, keepstore, etc. -- and restart all services to make sure they are using the latest configuration.
-h2(#arv-git-httpd). arv-git-httpd
+h2. Cloud installations only: node manager
-The legacy arv-git-httpd config (loaded from @/etc/arvados/git-httpd/git-httpd.yml@ or a different location specified via -legacy-git-httpd-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/git-httpd/git-httpd.yml@ and stop using the -legacy-git-httpd-config argument.
+Node manager is deprecated and replaced by @arvados-dispatch-cloud@. No automated config migration is available. Follow the instructions to "install the cloud dispatcher":../install/install-dispatch-cloud.html
-h2(#keepbalance). keep-balance
+*Only one dispatch process should be running at a time.* If you are migrating a system that currently runs Node manager and @crunch-dispatch-slurm@, it is safest to remove the @crunch-dispatch-slurm@ service entirely before installing @arvados-dispatch-cloud@.
-The legacy keep-balance config (loaded from @/etc/arvados/keep-balance/keep-balance.yml@ or a different location specified via -legacy-keepbalance-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/keep-balance/keep-balance.yml@ and stop using the -legacy-keepbalance-config argument.
-
-h2. arvados-controller
-
-Already uses centralized config exclusively. No migration needed.
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl --now disable crunch-dispatch-slurm</span>
+~$ <span class="userinput">sudo apt-get remove crunch-dispatch-slurm</span>
+</code></pre>
+</notextile>
-h2. arvados-dispatch-cloud
+h2. arvados-controller, arvados-dispatch-cloud
Already uses centralized config exclusively. No migration needed.
---
layout: default
-navsection: admin
+navsection: installguide
title: Configuration reference
...
The service @arvados-health@ performs health checks on all configured services and returns a single value of @OK@ or @ERROR@ for the entire cluster. It exposes the endpoint @/_health/all@ .
-The healthcheck aggregator uses the @NodeProfile@ section of the cluster-wide @arvados.yml@ configuration file. Here is an example.
-
-<pre>
-Cluster:
- # The cluster uuid prefix
- zzzzz:
- ManagementToken: xyzzy
- NodeProfile:
- # For each node, the profile name corresponds to a
- # locally-resolvable hostname, and describes which Arvados
- # services are available on that machine.
- api:
- arvados-controller:
- Listen: :8000
- arvados-api-server:
- Listen: :8001
- manage:
- arvados-node-manager:
- Listen: :8002
- workbench:
- arvados-workbench:
- Listen: :8003
- arvados-ws:
- Listen: :8004
- keep:
- keep-web:
- Listen: :8005
- keepproxy:
- Listen: :8006
- keep-balance:
- Listen: :9005
- keep0:
- keepstore:
- Listen: :25107
- keep1:
- keepstore:
- Listen: :25107
-</pre>
+The healthcheck aggregator uses the @Services@ section of the cluster-wide @config.yml@ configuration file.
<notextile>
<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
+ <span class="userinput">ClusterID</span>:
# Token to be included in all healthcheck requests. Disabled by default.
# Server expects request header of the format "Authorization: Bearer xxx"
ManagementToken: xxx
<pre>
Clusters:
- uuid_prefix:
+ ClusterID:
Containers:
UsePreemptibleInstances: true
InstanceTypes:
When @UsePreemptibleInstances@ is enabled, child containers (workflow steps) will automatically be made preemptible. Note that because preempting the workflow runner would cancel the entire workflow, the workflow runner runs in a reserved (non-preemptible) instance.
-If you are using "crunch-dispatch-cloud":{{site.baseurl}}/install/install-dispatch-cloud.html no additional configuration is required.
+If you are using "arvados-dispatch-cloud":{{site.baseurl}}/install/install-dispatch-cloud.html no additional configuration is required.
If you are using the legacy Nodemanager, "see below":#nodemanager .
---
layout: default
-navsection: admin
+navsection: installguide
title: "Upgrading Arvados and Release notes"
...
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
-h3(#master). development master (as of 2019-08-12)
+h3(#master). development master (as of 2019-12-17)
+
+h4. Migrating to centralized config.yml
+
+See "Migrating Configuration":config-migration.html for notes on migrating legacy per-component configuration files to the new centralized @/etc/arvados/config.yml@. To ensure a smooth transition, the per-component config files continue to be read, and take precedence over the centralized configuration.
+
+h4. Cloud installations only: node manager replaced by arvados-dispatch-cloud
+
+Node manager is deprecated and replaced by @arvados-dispatch-cloud@. No automated config migration is available. Follow the instructions to "install the cloud dispatcher":../install/install-dispatch-cloud.html
+
+*Only one dispatch process should be running at a time.* If you are migrating a system that currently runs Node manager and @crunch-dispatch-slurm@, it is safest to remove the @crunch-dispatch-slurm@ service entirely before installing @arvados-dispatch-cloud@.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl --now disable crunch-dispatch-slurm</span>
+~$ <span class="userinput">sudo apt-get remove crunch-dispatch-slurm</span>
+</code></pre>
+</notextile>
h4. Delete "keep_services" records
keepstore now supports the legacy @keepstore.yml@ config format (used by Arvados 1.4) and the new cluster config file format. Please check the "keepstore config migration notes":{{site.baseurl}}/admin/config-migration.html#keepstore and "keepstore install guide":{{site.baseurl}}/install/install-keepstore.html for more details.
-h4. Jobs API is read-only
+h4. Keepproxy configuration migration
-(task "#15133":https://dev.arvados.org/issues/15133 ) The legacy 'jobs' API is now read-only. It has long been superceded by containers / container_requests (aka crunch v2). Arvados installations since the end of 2017 (v1.1.0) have probably only used containers, and are unaffected by this change.
+(feature "#14715":https://dev.arvados.org/issues/14715 ) Keepproxy can now be configured using the centralized config at @/etc/arvados/config.yml@. Configuration via individual command line arguments is no longer available and the @DisableGet@, @DisablePut@, and @PIDFile@ configuration options are no longer supported. If you are still using the legacy config and @DisableGet@ or @DisablePut@ are set to true or @PIDFile@ has a value, keepproxy will produce an error and fail to start. Please see "keepproxy's config migration guide":{{site.baseurl}}/admin/config-migration.html#keepproxy for more details.
-So that older Arvados sites don't lose access to legacy records, the API has been converted to read-only. Creating and updating jobs (and related types job_task, pipeline_template and pipeline_instance) is disabled and much of the business logic related has been removed, along with various other code specific to the jobs API. Specifically, the following programs associated with the jobs API have been removed: @crunch-dispatch.rb@, @crunch-job@, @crunchrunner@, @arv-run-pipeline-instance@, @arv-run@.
+h4. Jobs API is read-only
-h4. Keepproxy configuration migration
+(task "#15133":https://dev.arvados.org/issues/15133 ) The legacy 'jobs' API is now read-only. It has been superceded since Arvados 1.1 by containers / container_requests (aka crunch v2). Arvados installations since the end of 2017 (v1.1.0) have probably only used containers, and are unaffected by this change.
-(feature "#14715":https://dev.arvados.org/issues/14715 ) Keepproxy can now be configured using the centralized config at @/etc/arvados/config.yml@. Configuration via individual command line arguments is no longer available and the @DisableGet@, @DisablePut@, and @PIDFile@ configuration options are no longer supported. If you are still using the legacy config and @DisableGet@ or @DisablePut@ are set to true or @PIDFile@ has a value, keepproxy will produce an error and fail to start. Please see "keepproxy's config migration guide":{{site.baseurl}}/admin/config-migration.html#keepproxy for more details.
+So that older Arvados sites don't lose access to legacy records, the API has been converted to read-only. Creating and updating jobs (and related types job_task, pipeline_template and pipeline_instance) is disabled and much of the business logic related has been removed, along with various other code specific to the jobs API. Specifically, the following programs associated with the jobs API have been removed: @crunch-dispatch.rb@, @crunch-job@, @crunchrunner@, @arv-run-pipeline-instance@, @arv-run@.
h4. No longer stripping ':' from strings in serialized database columns
Subsequently, the <code class="userinput">psql -d 'arvados_production' -c '\dx'</code> command will display the installed extensions for the arvados_production database. This list should now contain @pg_trgm@.
-h4. Migrating to centralized config.yml
-
-See "Migrating Configuration":config-migration.html for notes on migrating legacy per-component configuration files to the new centralized @/etc/arvados/config.yml@. To ensure a smooth transition, the per-component config files continue to be read, and take precedence over the centralized configuration.
h3(#v1_4_1). v1.4.1 (2019-09-20)
This release includes several database migrations, which will be executed automatically as part of the API server upgrade. On large Arvados installations, these migrations will take a while. We've seen the upgrade take 30 minutes or more on installations with a lot of collections.
-The @arvados-controller@ component now requires the /etc/arvados/config.yml file to be present. See <a href="{{ site.baseurl }}/install/install-controller.html#configuration">the @arvados-controller@ installation instructions</a>.
+The @arvados-controller@ component now requires the /etc/arvados/config.yml file to be present.
Support for the deprecated "jobs" API is broken in this release. Users who rely on it should not upgrade. This will be fixed in an upcoming 1.3.1 patch release, however users are "encouraged to migrate":upgrade-crunch2.html as support for the "jobs" API will be dropped in an upcoming release. Users who are already using the "containers" API are not affected.
Commit "db5107dca":https://dev.arvados.org/projects/arvados/repository/revisions/db5107dca adds a new system service, arvados-controller. More detail is available in story "#13496":https://dev.arvados.org/issues/13497.
-To add the Arvados Controller to your system please refer to the "installation instructions":../install/install-controller.html after upgrading your system to 1.2.0.
+To add the Arvados Controller to your system please refer to the "installation instructions":../install/install-api-server.html after upgrading your system to 1.2.0.
Verify your setup by confirming that API calls appear in the controller's logs (_e.g._, @journalctl -fu arvados-controller@) while loading a workbench page.
Commit "1e2ace5":https://dev.arvados.org/projects/arvados/repository/revisions/1e2ace5 changes recommended config for keep-web (see "#5824":https://dev.arvados.org/issues/5824)
-* proxy/dns/ssl config should be updated to route "https://download.uuid_prefix.arvadosapi.com/" requests to keep-web (alongside the existing "collections" routing)
-* keep-web command line adds @-attachment-only-host download.uuid_prefix.arvadosapi.com@
+* proxy/dns/ssl config should be updated to route "https://download.ClusterID.example.com/" requests to keep-web (alongside the existing "collections" routing)
+* keep-web command line adds @-attachment-only-host download.ClusterID.example.com@
* Workbench config adds @keep_web_download_url@
* More info on the (still beta/non-TOC-linked) "keep-web doc page":http://doc.arvados.org/install/install-keep-web.html
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-p=. *Legacy. The job APIs are read-only and disabled by default in new installations. Use "container requests":container_requests.html.textile.liquid .*
+p=. *Legacy. The job APIs are read-only and disabled by default in new installations. Use "container requests":methods/container_requests.html .*
h2. Crunch scripts
* For automated tests purposes, use "z****"
* For experimental/local-only/private clusters that won't ever be visible on the public Internet, use "x****"
-* For long-lived clusters, we recommend reserving a cluster id. Contact "mailto:support@curoverse.com":support@curoverse.com
+* For long-lived clusters, we recommend reserving a cluster id. Contact "info@curii.com":mailto:info@curii.com
Cluster identifiers are mapped API server hosts one of two ways:
-* Through DNS resolution, under the @arvadosapi.com@ domain. For example, the API server for the cluster @qr1hi@ can be found at @qr1hi.arvadosapi.com@. To register a cluster id for free under @arvadosapi.com@, contact "mailto:support@curoverse.com":support@curoverse.com
+* Through DNS resolution, under the @arvadosapi.com@ domain. For example, the API server for the cluster @qr1hi@ can be found at @qr1hi.arvadosapi.com@. To register a cluster id for free under @arvadosapi.com@, contact "info@curii.com":mailto:info@curii.com
* Through explicit configuration:
The @RemoteClusters@ section of @/etc/arvados/config.yml@ (for arvados-controller)
--- /dev/null
+body {
+ background: white;
+ color: black;
+}
+
+a:link {
+ background: white;
+ color: blue;
+}
+
+a:visited {
+ background: white;
+ color: rgb(50%, 0%, 50%);
+}
+
+h1 {
+ background: white;
+ color: rgb(55%, 55%, 55%);
+ font-family: monospace;
+ font-size: x-large;
+ text-align: center;
+}
+
+h2 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-size: large;
+ text-align: center;
+}
+
+h3 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-size: large;
+}
+
+h4 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-style: italic;
+ font-size: large;
+}
+
+h5 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+}
+
+h6 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-style: italic;
+}
+
+img.toplogo {
+ width: 4em;
+ vertical-align: middle;
+}
+
+img.arrow {
+ width: 30px;
+ height: 30px;
+ border: 0;
+}
+
+span.acronym {
+ font-size: small;
+}
+
+span.env {
+ font-family: monospace;
+}
+
+span.file {
+ font-family: monospace;
+}
+
+span.option{
+ font-family: monospace;
+}
+
+span.pkg {
+ font-weight: bold;
+}
+
+span.samp{
+ font-family: monospace;
+}
+
+div.vignettes a:hover {
+ background: rgb(85%, 85%, 85%);
+}
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.2" width="279.4mm" height="63.5mm" viewBox="0 0 27940 6350" preserveAspectRatio="xMidYMid" fill-rule="evenodd" stroke-width="28.222" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" xmlns:ooo="http://xml.openoffice.org/svg/export" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:presentation="http://sun.com/xmlns/staroffice/presentation" xmlns:smil="http://www.w3.org/2001/SMIL20/" xmlns:anim="urn:oasis:names:tc:opendocument:xmlns:animation:1.0" xml:space="preserve">
+ <defs class="ClipPathGroup">
+ <clipPath id="presentation_clip_path" clipPathUnits="userSpaceOnUse">
+ <rect x="0" y="0" width="27940" height="6350"/>
+ </clipPath>
+ <clipPath id="presentation_clip_path_shrink" clipPathUnits="userSpaceOnUse">
+ <rect x="27" y="6" width="27885" height="6338"/>
+ </clipPath>
+ </defs>
+ <defs>
+ <font id="EmbeddedFont_1" horiz-adv-x="2048">
+ <font-face font-family="Liberation Sans embedded" units-per-em="2048" font-weight="normal" font-style="normal" ascent="1852" descent="423"/>
+ <missing-glyph horiz-adv-x="2048" d="M 0,0 L 2047,0 2047,2047 0,2047 0,0 Z"/>
+ <glyph unicode="x" horiz-adv-x="1006" d="M 801,0 L 510,444 217,0 23,0 408,556 41,1082 240,1082 510,661 778,1082 979,1082 612,558 1002,0 801,0 Z"/>
+ <glyph unicode="v" horiz-adv-x="1033" d="M 613,0 L 400,0 7,1082 199,1082 437,378 C 442,363 447,346 454,325 460,304 466,282 473,259 480,236 486,215 492,194 497,173 502,155 506,141 510,155 515,173 522,194 528,215 534,236 541,258 548,280 555,302 562,323 569,344 575,361 580,376 L 826,1082 1017,1082 613,0 Z"/>
+ <glyph unicode="t" horiz-adv-x="531" d="M 554,8 C 527,1 499,-5 471,-10 442,-14 409,-16 372,-16 228,-16 156,66 156,229 L 156,951 31,951 31,1082 163,1082 216,1324 336,1324 336,1082 536,1082 536,951 336,951 336,268 C 336,216 345,180 362,159 379,138 408,127 450,127 467,127 484,128 501,131 517,134 535,137 554,141 L 554,8 Z"/>
+ <glyph unicode="s" horiz-adv-x="901" d="M 950,299 C 950,248 940,203 921,164 901,124 872,91 835,64 798,37 752,16 698,2 643,-13 581,-20 511,-20 448,-20 392,-15 342,-6 291,4 247,20 209,41 171,62 139,91 114,126 88,161 69,203 57,254 L 216,285 C 231,227 263,185 311,158 359,131 426,117 511,117 550,117 585,120 618,125 650,130 678,140 701,153 724,166 743,183 756,205 769,226 775,253 775,285 775,318 767,345 752,366 737,387 715,404 688,418 661,432 628,444 589,455 550,465 507,476 460,489 417,500 374,513 331,527 288,541 250,560 216,583 181,606 153,634 132,668 111,702 100,745 100,796 100,895 135,970 206,1022 276,1073 378,1099 513,1099 632,1099 727,1078 798,1036 868,994 912,927 931,834 L 769,814 C 763,842 752,866 736,885 720,904 701,919 678,931 655,942 630,951 602,956 573,961 544,963 513,963 432,963 372,951 333,926 294,901 275,864 275,814 275,785 282,761 297,742 311,723 331,707 357,694 382,681 413,669 449,660 485,650 525,640 568,629 597,622 626,614 656,606 686,597 715,587 744,576 772,564 799,550 824,535 849,519 870,500 889,478 908,456 923,430 934,401 945,372 950,338 950,299 Z"/>
+ <glyph unicode="r" horiz-adv-x="530" d="M 142,0 L 142,830 C 142,853 142,876 142,900 141,923 141,946 140,968 139,990 139,1011 138,1030 137,1049 137,1067 136,1082 L 306,1082 C 307,1067 308,1049 309,1030 310,1010 311,990 312,969 313,948 313,929 314,910 314,891 314,874 314,861 L 318,861 C 331,902 344,938 359,969 373,999 390,1024 409,1044 428,1063 451,1078 478,1088 505,1097 537,1102 575,1102 590,1102 604,1101 617,1099 630,1096 641,1094 648,1092 L 648,927 C 636,930 622,933 606,935 590,936 572,937 552,937 511,937 476,928 447,909 418,890 394,865 376,832 357,799 344,759 335,714 326,668 322,618 322,564 L 322,0 142,0 Z"/>
+ <glyph unicode="o" horiz-adv-x="980" d="M 1053,542 C 1053,353 1011,212 928,119 845,26 724,-20 565,-20 490,-20 422,-9 363,14 304,37 254,71 213,118 172,165 140,223 119,294 97,364 86,447 86,542 86,915 248,1102 571,1102 655,1102 728,1090 789,1067 850,1044 900,1009 939,962 978,915 1006,857 1025,787 1044,717 1053,635 1053,542 Z M 864,542 C 864,626 858,695 845,750 832,805 813,848 788,881 763,914 732,937 696,950 660,963 619,969 574,969 528,969 487,962 450,949 413,935 381,912 355,879 329,846 309,802 296,747 282,692 275,624 275,542 275,458 282,389 297,334 312,279 332,235 358,202 383,169 414,146 449,133 484,120 522,113 563,113 609,113 651,120 688,133 725,146 757,168 783,201 809,234 829,278 843,333 857,388 864,458 864,542 Z"/>
+ <glyph unicode="n" horiz-adv-x="874" d="M 825,0 L 825,686 C 825,739 821,783 814,818 806,853 793,882 776,904 759,925 736,941 708,950 679,959 644,963 602,963 559,963 521,956 487,941 452,926 423,904 399,876 374,847 355,812 342,771 329,729 322,681 322,627 L 322,0 142,0 142,851 C 142,874 142,898 142,923 141,948 141,971 140,994 139,1016 139,1035 138,1051 137,1067 137,1077 136,1082 L 306,1082 C 307,1079 307,1070 308,1055 309,1040 310,1024 311,1005 312,986 312,966 313,947 314,927 314,910 314,897 L 317,897 C 334,928 353,957 374,982 395,1007 419,1029 446,1047 473,1064 505,1078 540,1088 575,1097 616,1102 663,1102 723,1102 775,1095 818,1080 861,1065 897,1043 925,1012 953,981 974,942 987,894 1000,845 1006,788 1006,721 L 1006,0 825,0 Z"/>
+ <glyph unicode="l" horiz-adv-x="187" d="M 138,0 L 138,1484 318,1484 318,0 138,0 Z"/>
+ <glyph unicode="i" horiz-adv-x="187" d="M 137,1312 L 137,1484 317,1484 317,1312 137,1312 Z M 137,0 L 137,1082 317,1082 317,0 137,0 Z"/>
+ <glyph unicode="g" horiz-adv-x="927" d="M 548,-425 C 486,-425 431,-419 383,-406 335,-393 294,-375 260,-352 226,-328 198,-300 177,-267 156,-234 140,-198 131,-158 L 312,-132 C 324,-182 351,-220 392,-248 433,-274 486,-288 553,-288 594,-288 631,-282 664,-271 697,-260 726,-241 749,-217 772,-191 790,-159 803,-119 816,-79 822,-30 822,27 L 822,201 820,201 C 807,174 790,148 771,123 751,98 727,75 699,56 670,37 637,21 600,10 563,-2 520,-8 472,-8 403,-8 345,4 296,27 247,50 207,84 176,130 145,176 122,233 108,302 93,370 86,449 86,539 86,626 93,704 108,773 122,842 145,901 178,950 210,998 252,1035 304,1061 355,1086 418,1099 492,1099 569,1099 635,1082 692,1047 748,1012 791,962 822,897 L 824,897 C 824,914 825,933 826,953 827,974 828,994 829,1012 830,1031 831,1046 832,1060 833,1073 835,1080 836,1080 L 1007,1080 C 1006,1074 1006,1064 1005,1050 1004,1035 1004,1018 1003,998 1002,978 1002,956 1002,932 1001,907 1001,882 1001,856 L 1001,30 C 1001,-121 964,-234 890,-311 815,-387 701,-425 548,-425 Z M 822,541 C 822,616 814,681 798,735 781,788 760,832 733,866 706,900 676,925 642,941 607,957 572,965 536,965 490,965 451,957 418,941 385,925 357,900 336,866 314,831 298,787 288,734 277,680 272,616 272,541 272,463 277,398 288,345 298,292 314,249 335,216 356,183 383,160 416,146 449,132 488,125 533,125 569,125 604,133 639,148 673,163 704,188 731,221 758,254 780,297 797,350 814,403 822,466 822,541 Z"/>
+ <glyph unicode="e" horiz-adv-x="980" d="M 276,503 C 276,446 282,394 294,347 305,299 323,258 348,224 372,189 403,163 441,144 479,125 525,115 578,115 656,115 719,131 766,162 813,193 844,233 861,281 L 1019,236 C 1008,206 992,176 972,146 951,115 924,88 890,64 856,39 814,19 763,4 712,-12 650,-20 578,-20 418,-20 296,28 213,123 129,218 87,360 87,548 87,649 100,735 125,806 150,876 185,933 229,977 273,1021 324,1053 383,1073 442,1092 504,1102 571,1102 662,1102 738,1087 799,1058 860,1029 909,988 946,937 983,885 1009,824 1025,754 1040,684 1048,608 1048,527 L 1048,503 276,503 Z M 862,641 C 852,755 823,838 775,891 727,943 658,969 568,969 538,969 507,964 474,955 441,945 410,928 382,903 354,878 330,845 311,803 292,760 281,706 278,641 L 862,641 Z"/>
+ <glyph unicode="d" horiz-adv-x="927" d="M 821,174 C 788,105 744,55 689,25 634,-5 565,-20 484,-20 347,-20 247,26 183,118 118,210 86,349 86,536 86,913 219,1102 484,1102 566,1102 634,1087 689,1057 744,1027 788,979 821,914 L 823,914 C 823,921 823,931 823,946 822,960 822,975 822,991 821,1006 821,1021 821,1035 821,1049 821,1059 821,1065 L 821,1484 1001,1484 1001,223 C 1001,197 1001,172 1002,148 1002,124 1002,102 1003,82 1004,62 1004,45 1005,31 1006,16 1006,6 1007,0 L 835,0 C 834,7 833,16 832,29 831,41 830,55 829,71 828,87 827,104 826,122 825,139 825,157 825,174 L 821,174 Z M 275,542 C 275,467 280,403 289,350 298,297 313,253 334,219 355,184 381,159 413,143 445,127 484,119 530,119 577,119 619,127 656,142 692,157 722,182 747,217 771,251 789,296 802,351 815,406 821,474 821,554 821,631 815,696 802,749 789,802 771,844 746,877 721,910 691,933 656,948 620,962 579,969 532,969 488,969 450,961 418,946 386,931 359,906 338,872 317,838 301,794 291,740 280,685 275,619 275,542 Z"/>
+ <glyph unicode="c" horiz-adv-x="901" d="M 275,546 C 275,484 280,427 289,375 298,323 313,278 334,241 355,203 384,174 419,153 454,132 497,122 548,122 612,122 666,139 709,174 752,209 778,262 788,334 L 970,322 C 964,277 951,234 931,193 911,152 884,115 850,84 815,53 773,28 724,9 675,-10 618,-20 553,-20 468,-20 396,-6 337,23 278,52 230,91 193,142 156,192 129,251 112,320 95,388 87,462 87,542 87,615 93,679 105,735 117,790 134,839 156,881 177,922 203,957 232,986 261,1014 293,1037 328,1054 362,1071 398,1083 436,1091 474,1098 512,1102 551,1102 612,1102 666,1094 713,1077 760,1060 801,1038 836,1009 870,980 898,945 919,906 940,867 955,824 964,779 L 779,765 C 770,825 746,873 708,908 670,943 616,961 546,961 495,961 452,953 418,936 383,919 355,893 334,859 313,824 298,781 289,729 280,677 275,616 275,546 Z"/>
+ <glyph unicode="a" horiz-adv-x="1060" d="M 414,-20 C 305,-20 224,9 169,66 114,123 87,202 87,302 87,373 101,432 128,478 155,523 190,559 234,585 277,611 327,629 383,639 439,649 496,655 554,656 L 797,660 797,719 C 797,764 792,802 783,833 774,864 759,890 740,909 721,928 697,943 668,952 639,961 604,965 565,965 530,965 499,963 471,958 443,953 419,944 398,931 377,918 361,900 348,878 335,855 327,827 323,793 L 135,810 C 142,853 154,892 173,928 192,963 218,994 253,1020 287,1046 330,1066 382,1081 433,1095 496,1102 569,1102 705,1102 807,1071 876,1009 945,946 979,856 979,738 L 979,272 C 979,219 986,179 1000,152 1014,125 1041,111 1080,111 1090,111 1100,112 1110,113 1120,114 1130,116 1139,118 L 1139,6 C 1116,1 1094,-3 1072,-6 1049,-9 1025,-10 1000,-10 966,-10 937,-5 913,4 888,13 868,26 853,45 838,63 826,86 818,113 810,140 805,171 803,207 L 797,207 C 778,172 757,141 734,113 711,85 684,61 653,42 622,22 588,7 549,-4 510,-15 465,-20 414,-20 Z M 455,115 C 512,115 563,126 606,147 649,168 684,194 713,227 741,260 762,295 776,334 790,373 797,410 797,445 L 797,534 600,530 C 556,529 514,526 475,521 435,515 400,504 370,487 340,470 316,447 299,417 281,387 272,348 272,299 272,240 288,195 320,163 351,131 396,115 455,115 Z"/>
+ <glyph unicode="S" horiz-adv-x="1192" d="M 1272,389 C 1272,330 1261,275 1238,225 1215,175 1179,132 1131,96 1083,59 1023,31 950,11 877,-10 790,-20 690,-20 515,-20 378,11 280,72 182,133 120,222 93,338 L 278,375 C 287,338 302,305 321,275 340,245 367,219 400,198 433,176 473,159 522,147 571,135 629,129 697,129 754,129 806,134 853,144 900,153 941,168 975,188 1009,208 1036,234 1055,266 1074,297 1083,335 1083,379 1083,425 1073,462 1052,491 1031,520 1001,543 963,562 925,581 880,596 827,609 774,622 716,635 652,650 613,659 573,668 534,679 494,689 456,701 420,716 383,730 349,747 317,766 285,785 257,809 234,836 211,863 192,894 179,930 166,965 159,1006 159,1053 159,1120 173,1177 200,1225 227,1272 264,1311 312,1342 360,1373 417,1395 482,1409 547,1423 618,1430 694,1430 781,1430 856,1423 918,1410 980,1396 1032,1375 1075,1348 1118,1321 1152,1287 1178,1247 1203,1206 1224,1159 1239,1106 L 1051,1073 C 1042,1107 1028,1137 1011,1164 993,1191 970,1213 941,1231 912,1249 878,1263 837,1272 796,1281 747,1286 692,1286 627,1286 572,1280 528,1269 483,1257 448,1241 421,1221 394,1201 374,1178 363,1151 351,1124 345,1094 345,1063 345,1021 356,987 377,960 398,933 426,910 462,892 498,874 540,859 587,847 634,835 685,823 738,811 781,801 825,791 868,781 911,770 952,758 991,744 1030,729 1067,712 1102,693 1136,674 1166,650 1191,622 1216,594 1236,561 1251,523 1265,485 1272,440 1272,389 Z"/>
+ <glyph unicode="Q" horiz-adv-x="1430" d="M 1495,711 C 1495,612 1482,521 1457,439 1431,356 1394,284 1346,222 1297,160 1238,110 1168,71 1097,32 1017,6 928,-6 942,-49 958,-85 976,-115 993,-145 1013,-169 1036,-189 1059,-207 1084,-221 1112,-231 1139,-239 1170,-244 1204,-244 1223,-244 1243,-243 1264,-240 1285,-237 1304,-234 1319,-231 L 1319,-365 C 1294,-371 1266,-376 1236,-381 1205,-385 1174,-387 1141,-387 1084,-387 1034,-378 991,-362 948,-344 911,-320 879,-289 846,-257 818,-218 795,-172 772,-126 751,-74 733,-16 628,-11 535,11 456,50 376,88 310,139 257,204 204,268 164,343 137,430 110,516 97,610 97,711 97,821 112,920 143,1009 174,1098 219,1173 278,1236 337,1298 411,1346 498,1380 585,1413 684,1430 797,1430 909,1430 1009,1413 1096,1379 1183,1345 1256,1297 1315,1234 1374,1171 1418,1096 1449,1007 1480,918 1495,820 1495,711 Z M 1300,711 C 1300,796 1289,873 1268,942 1246,1011 1214,1071 1172,1120 1129,1169 1077,1207 1014,1234 951,1261 879,1274 797,1274 713,1274 639,1261 576,1234 513,1207 460,1169 418,1120 375,1071 344,1011 323,942 302,873 291,796 291,711 291,626 302,549 324,479 345,408 377,348 420,297 462,246 515,206 578,178 641,149 713,135 795,135 883,135 959,149 1023,178 1086,207 1139,247 1180,298 1221,349 1251,409 1271,480 1290,551 1300,628 1300,711 Z"/>
+ <glyph unicode="P" horiz-adv-x="1112" d="M 1258,985 C 1258,924 1248,867 1228,814 1207,761 1177,715 1137,676 1096,637 1046,606 985,583 924,560 854,549 773,549 L 359,549 359,0 168,0 168,1409 761,1409 C 844,1409 917,1399 979,1379 1041,1358 1093,1330 1134,1293 1175,1256 1206,1211 1227,1159 1248,1106 1258,1048 1258,985 Z M 1066,983 C 1066,1072 1039,1140 984,1187 929,1233 847,1256 738,1256 L 359,1256 359,700 746,700 C 856,700 937,724 989,773 1040,822 1066,892 1066,983 Z"/>
+ <glyph unicode="N" horiz-adv-x="1165" d="M 1082,0 L 328,1200 C 329,1167 331,1135 333,1103 334,1076 336,1047 337,1017 338,986 338,959 338,936 L 338,0 168,0 168,1409 390,1409 1152,201 C 1150,234 1148,266 1146,299 1145,327 1143,358 1142,391 1141,424 1140,455 1140,485 L 1140,1409 1312,1409 1312,0 1082,0 Z"/>
+ <glyph unicode="L" horiz-adv-x="927" d="M 168,0 L 168,1409 359,1409 359,156 1071,156 1071,0 168,0 Z"/>
+ <glyph unicode="I" horiz-adv-x="213" d="M 189,0 L 189,1409 380,1409 380,0 189,0 Z"/>
+ <glyph unicode="C" horiz-adv-x="1324" d="M 792,1274 C 712,1274 641,1261 580,1234 518,1207 466,1169 425,1120 383,1071 351,1011 330,942 309,873 298,796 298,711 298,626 310,549 333,479 356,408 389,348 432,297 475,246 527,207 590,179 652,151 722,137 800,137 855,137 905,144 950,159 995,173 1035,193 1072,219 1108,245 1140,276 1169,312 1198,347 1223,387 1245,430 L 1401,352 C 1376,299 1344,250 1307,205 1270,160 1226,120 1176,87 1125,54 1068,28 1005,9 941,-10 870,-20 791,-20 677,-20 577,-2 492,35 406,71 334,122 277,187 219,252 176,329 147,418 118,507 104,605 104,711 104,821 119,920 150,1009 180,1098 224,1173 283,1236 341,1298 413,1346 498,1380 583,1413 681,1430 790,1430 940,1430 1065,1401 1166,1342 1267,1283 1341,1196 1388,1081 L 1207,1021 C 1194,1054 1176,1086 1153,1117 1130,1147 1102,1174 1068,1197 1034,1220 994,1239 949,1253 903,1267 851,1274 792,1274 Z"/>
+ <glyph unicode="A" horiz-adv-x="1377" d="M 1167,0 L 1006,412 364,412 202,0 4,0 579,1409 796,1409 1362,0 1167,0 Z M 768,1026 C 757,1053 747,1080 738,1107 728,1134 719,1159 712,1182 705,1204 699,1223 694,1238 689,1253 686,1262 685,1265 684,1262 681,1252 676,1237 671,1222 665,1203 658,1180 650,1157 641,1132 632,1105 622,1078 612,1051 602,1024 L 422,561 949,561 768,1026 Z"/>
+ <glyph unicode=" " horiz-adv-x="556"/>
+ </font>
+ </defs>
+ <defs class="TextShapeIndex">
+ <g ooo:slide="id1" ooo:id-list="id3 id4 id5 id6 id7 id8 id9 id10 id11 id12"/>
+ </defs>
+ <defs class="EmbeddedBulletChars">
+ <g id="bullet-char-template-57356" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 580,1141 L 1163,571 580,0 -4,571 580,1141 Z"/>
+ </g>
+ <g id="bullet-char-template-57354" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 8,1128 L 1137,1128 1137,0 8,0 8,1128 Z"/>
+ </g>
+ <g id="bullet-char-template-10146" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 174,0 L 602,739 174,1481 1456,739 174,0 Z M 1358,739 L 309,1346 659,739 1358,739 Z"/>
+ </g>
+ <g id="bullet-char-template-10132" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 2015,739 L 1276,0 717,0 1260,543 174,543 174,936 1260,936 717,1481 1274,1481 2015,739 Z"/>
+ </g>
+ <g id="bullet-char-template-10007" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 0,-2 C -7,14 -16,27 -25,37 L 356,567 C 262,823 215,952 215,954 215,979 228,992 255,992 264,992 276,990 289,987 310,991 331,999 354,1012 L 381,999 492,748 772,1049 836,1024 860,1049 C 881,1039 901,1025 922,1006 886,937 835,863 770,784 769,783 710,716 594,584 L 774,223 C 774,196 753,168 711,139 L 727,119 C 717,90 699,76 672,76 641,76 570,178 457,381 L 164,-76 C 142,-110 111,-127 72,-127 30,-127 9,-110 8,-76 1,-67 -2,-52 -2,-32 -2,-23 -1,-13 0,-2 Z"/>
+ </g>
+ <g id="bullet-char-template-10004" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 285,-33 C 182,-33 111,30 74,156 52,228 41,333 41,471 41,549 55,616 82,672 116,743 169,778 240,778 293,778 328,747 346,684 L 369,508 C 377,444 397,411 428,410 L 1163,1116 C 1174,1127 1196,1133 1229,1133 1271,1133 1292,1118 1292,1087 L 1292,965 C 1292,929 1282,901 1262,881 L 442,47 C 390,-6 338,-33 285,-33 Z"/>
+ </g>
+ <g id="bullet-char-template-9679" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 813,0 C 632,0 489,54 383,161 276,268 223,411 223,592 223,773 276,916 383,1023 489,1130 632,1184 813,1184 992,1184 1136,1130 1245,1023 1353,916 1407,772 1407,592 1407,412 1353,268 1245,161 1136,54 992,0 813,0 Z"/>
+ </g>
+ <g id="bullet-char-template-8226" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 346,457 C 273,457 209,483 155,535 101,586 74,649 74,723 74,796 101,859 155,911 209,963 273,989 346,989 419,989 480,963 531,910 582,859 608,796 608,723 608,648 583,586 532,535 482,483 420,457 346,457 Z"/>
+ </g>
+ <g id="bullet-char-template-8211" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M -4,459 L 1135,459 1135,606 -4,606 -4,459 Z"/>
+ </g>
+ <g id="bullet-char-template-61548" transform="scale(0.00048828125,-0.00048828125)">
+ <path d="M 173,740 C 173,903 231,1043 346,1159 462,1274 601,1332 765,1332 928,1332 1067,1274 1183,1159 1299,1043 1357,903 1357,740 1357,577 1299,437 1183,322 1067,206 928,148 765,148 601,148 462,206 346,322 231,437 173,577 173,740 Z"/>
+ </g>
+ </defs>
+ <defs class="TextEmbeddedBitmaps"/>
+ <g>
+ <g id="id2" class="Master_Slide">
+ <g id="bg-id2" class="Background"/>
+ <g id="bo-id2" class="BackgroundObjects"/>
+ </g>
+ </g>
+ <g class="SlideGroup">
+ <g>
+ <g id="container-id1">
+ <g id="id1" class="Slide" clip-path="url(#presentation_clip_path)">
+ <g class="Page">
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id3">
+ <rect class="BoundingBox" stroke="none" fill="none" x="1760" y="1380" width="3433" height="2584"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 2067,2236 C 2003,1917 2300,1616 2597,1616 2691,1614 2788,1645 2867,1691 2944,1547 3085,1458 3244,1458 3349,1463 3461,1506 3541,1584 3598,1456 3718,1381 3849,1381 3958,1381 4058,1435 4122,1519 4195,1433 4304,1381 4419,1381 4605,1381 4762,1516 4795,1704 4975,1757 5105,1928 5105,2124 5105,2183 5095,2241 5068,2296 5144,2391 5191,2510 5191,2630 5191,2904 4986,3135 4722,3174 4722,3436 4519,3641 4265,3641 4177,3641 4095,3616 4022,3568 3955,3799 3744,3962 3507,3962 3331,3962 3164,3865 3064,3712 2971,3770 3020,3805 2751,3805 2531,3805 2327,3684 2221,3488 1967,3484 1837,3328 1837,3132 1837,3041 1870,2959 1930,2891 1821,2834 1761,2720 1761,2590 1761,2407 1894,2256 2067,2236 L 2067,2236 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 2067,2236 C 2003,1917 2300,1616 2597,1616 2691,1614 2788,1645 2867,1691 2944,1547 3085,1458 3244,1458 3349,1463 3461,1506 3541,1584 3598,1456 3718,1381 3849,1381 3958,1381 4058,1435 4122,1519 4195,1433 4304,1381 4419,1381 4605,1381 4762,1516 4795,1704 4975,1757 5105,1928 5105,2124 5105,2183 5095,2241 5068,2296 5144,2391 5191,2510 5191,2630 5191,2904 4986,3135 4722,3174 4722,3436 4519,3641 4265,3641 4177,3641 4095,3616 4022,3568 3955,3799 3744,3962 3507,3962 3331,3962 3164,3865 3064,3712 2971,3770 3020,3805 2751,3805 2531,3805 2327,3684 2221,3488 1967,3484 1837,3328 1837,3132 1837,3041 1870,2959 1930,2891 1821,2834 1761,2720 1761,2590 1761,2407 1894,2256 2067,2236 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 2067,2236 C 2070,2266 2084,2299 2092,2327"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 2867,1691 C 2904,1714 2948,1745 2978,1776"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 3541,1584 C 3528,1609 3520,1639 3512,1667"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4122,1519 C 4098,1548 4085,1586 4069,1621"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4795,1704 C 4798,1726 4814,1774 4808,1784"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 5068,2296 C 5041,2357 5005,2411 4954,2455"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4724,3174 C 4736,3077 4663,2838 4460,2749"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 4022,3568 C 4034,3529 4039,3493 4042,3455"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 3066,3712 C 3040,3681 3025,3645 3009,3608"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 2221,3488 C 2251,3484 2281,3476 2310,3466"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 1930,2891 C 1983,2922 2043,2949 2130,2939"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="2549" y="2835"><tspan fill="rgb(0,0,0)" stroke="none">Client</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id4">
+ <rect class="BoundingBox" stroke="none" fill="none" x="6939" y="1674" width="3178" height="2035"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 8528,3707 L 6940,3707 6940,1675 10115,1675 10115,3707 8528,3707 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 8528,3707 L 6940,3707 6940,1675 10115,1675 10115,3707 8528,3707 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="7719" y="2912"><tspan fill="rgb(0,0,0)" stroke="none">Nginx</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id5">
+ <rect class="BoundingBox" stroke="none" fill="none" x="11766" y="1674" width="3305" height="2035"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 13418,3707 L 11767,3707 11767,1675 15069,1675 15069,3707 13418,3707 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 13418,3707 L 11767,3707 11767,1675 15069,1675 15069,3707 13418,3707 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="12256" y="2556"><tspan fill="rgb(0,0,0)" stroke="none">Arvados</tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="12116" y="3267"><tspan fill="rgb(0,0,0)" stroke="none">controller</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id6">
+ <rect class="BoundingBox" stroke="none" fill="none" x="16719" y="1674" width="3305" height="2035"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 18371,3707 L 16720,3707 16720,1675 20022,1675 20022,3707 18371,3707 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 18371,3707 L 16720,3707 16720,1675 20022,1675 20022,3707 18371,3707 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="17120" y="2556"><tspan fill="rgb(0,0,0)" stroke="none">Arvados </tspan></tspan></tspan><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="16889" y="3267"><tspan fill="rgb(0,0,0)" stroke="none">API server</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.CustomShape">
+ <g id="id7">
+ <rect class="BoundingBox" stroke="none" fill="none" x="21545" y="1674" width="4067" height="2035"/>
+ <path fill="rgb(114,159,207)" stroke="none" d="M 23578,3707 L 21546,3707 21546,1675 25610,1675 25610,3707 23578,3707 Z"/>
+ <path fill="none" stroke="rgb(52,101,164)" d="M 23578,3707 L 21546,3707 21546,1675 25610,1675 25610,3707 23578,3707 Z"/>
+ <text class="TextShape"><tspan class="TextParagraph" font-family="Liberation Sans, sans-serif" font-size="635px" font-weight="400"><tspan class="TextPosition" x="21851" y="2912"><tspan fill="rgb(0,0,0)" stroke="none">PostgreSQL</tspan></tspan></tspan></text>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id8">
+ <rect class="BoundingBox" stroke="none" fill="none" x="5190" y="2535" width="1752" height="302"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 5191,2671 L 6511,2686"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 6941,2691 L 6493,2536 6489,2836 6941,2691 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id9">
+ <rect class="BoundingBox" stroke="none" fill="none" x="10107" y="2527" width="1661" height="329"/>
+ <path fill="none" stroke="rgb(0,0,0)" stroke-width="18" stroke-linejoin="round" d="M 10116,2691 L 11424,2691"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 11428,2853 L 11753,2708 11763,2701 11767,2692 11763,2681 11755,2674 11428,2529 11424,2528 11421,2528 11414,2529 11408,2533 11404,2538 11402,2545 11402,2837 11404,2844 11408,2850 11414,2853 11421,2855 11424,2855 11428,2853 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id10">
+ <rect class="BoundingBox" stroke="none" fill="none" x="15068" y="2541" width="1653" height="301"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 15069,2691 L 16290,2691"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 16720,2691 L 16270,2541 16270,2841 16720,2691 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id11">
+ <rect class="BoundingBox" stroke="none" fill="none" x="20021" y="2541" width="1526" height="301"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 20022,2691 L 21116,2691"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 21546,2691 L 21096,2541 21096,2841 21546,2691 Z"/>
+ </g>
+ </g>
+ <g class="com.sun.star.drawing.ConnectorShape">
+ <g id="id12">
+ <rect class="BoundingBox" stroke="none" fill="none" x="13417" y="3706" width="10312" height="1273"/>
+ <path fill="none" stroke="rgb(0,0,0)" d="M 13418,3707 L 13418,4977 23578,4977 23578,4137"/>
+ <path fill="rgb(0,0,0)" stroke="none" d="M 23578,3707 L 23428,4157 23728,4157 23578,3707 Z"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+ </g>
+</svg>
\ No newline at end of file
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Configuration files
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+h2. Arados /etc/arvados/config.yml
+
+The configuration file is normally found at @/etc/arvados/config.yml@ and will be referred to as just @config.yml@ in this guide. This configuration file must be kept in sync across every service node in the cluster, but not shell and compute nodes (which do not require config.yml).
+
+h3. Syntax
+
+The configuration file is in "YAML":https://yaml.org/ format. This is a block syntax where indentation is significant (similar to Python). By convention we use two space indent. The first line of the file is always "Clusters:", underneath it at the first indent level is the Cluster ID. All the actual cluster configuration follows under the Cluster ID. This means all configuration parameters are indented by at least two levels (four spaces). Comments start with @#@ .
+
+We recommend a YAML-syntax plugin for your favorite text editor, such as @yaml-mode@ (Emacs) or @yaml-vim@.
+
+Example file:
+
+<pre>
+Clusters: # Clusters block, everything else is listed under this
+ abcde: # Cluster ID, everything under it is configuration for this cluster
+ ExampleConfigKey: "fghijk" # An example configuration key
+ ExampleConfigGroup: # A group of keys
+ ExampleDurationConfig: 12s # Example duration
+ ExampleSizeConfig: 99KiB # Example with a size suffix
+</pre>
+
+Each configuration group may only appear once. When a configuration key is within a config group, it will be written with the group name leading, for example @ExampleConfigGroup.ExampleSizeConfig@.
+
+Duration suffixes are s=seconds, m=minutes or h=hours.
+
+Size suffixes are K=10 ^3^, Ki=2 ^10^ , M=10 ^6^, Mi=2 ^20^, G=10 ^9^, Gi=2 ^30^, T=10 ^12^, Ti=2 ^40^, P=10 ^15^, Pi=2 ^50^, E=10 ^18^, Ei=2 ^60^. You can optionally follow with a "B" (eg "MB" or "MiB") for readability (it does not affect the units.)
+
+h3(#empty). Create empty configuration file
+
+Change @webserver-user@ to the user that runs your web server process. This is @www-data@ on Debian-based systems, and @nginx@ on Red Hat-based systems.
+
+<notextile>
+<pre><code># <span class="userinput">export ClusterID=xxxxx</span>
+# <span class="userinput">umask 027</span>
+# <span class="userinput">mkdir -p /etc/arvados</span>
+# <span class="userinput">cat > /etc/arvados/config.yml <<EOF
+Clusters:
+ ${ClusterID}:
+EOF</span>
+# <span class="userinput">chgrp webserver-user /etc/arvados /etc/arvados/config.yml</span>
+</span></code></pre>
+</notextile>
+
+h2. Nginx configuration
+
+This guide will also cover setting up "Nginx":https://www.nginx.com/ as a reverse proxy for Arvados services. Nginx performs two main functions: TLS termination and virtual host routing. The virtual host configuration for each component will go in its own file in @/etc/nginx/conf.d/@.
+
+h2. Synchronizing config file
+
+The Arvados configuration file must be kept in sync across every service node in the cluster. We strongly recommend using a devops configuration management tool such as "Puppet":https://puppet.com/open-source/ to synchronize the config file. Alternately, something like the following script to securely copy the configuration file to each node may be helpful. Replace the @ssh@ targets with your nodes.
+
+<notextile>
+<pre><code>#!/bin/sh
+sudo cat /etc/arvados/config.yml | ssh <span class="userinput">10.0.0.2</span> sudo sh -c "'cat > /etc/arvados/config.yml'"
+sudo cat /etc/arvados/config.yml | ssh <span class="userinput">10.0.0.3</span> sudo sh -c "'cat > /etc/arvados/config.yml'"
+</code></pre>
+</notextile>
<notextile>
<pre><code>~$ <span class="userinput">azure config mode arm</span>
-~$ <span class="userinput">azure login</span>
-~$ <span class="userinput">azure group create exampleGroupName eastus</span>
-~$ <span class="userinput">azure storage account create --type LRS --location eastus --resource-group exampleGroupName exampleStorageAccountName</span>
-~$ <span class="userinput">azure storage account keys list --resource-group exampleGroupName exampleStorageAccountName</span>
-info: Executing command storage account keys list
-+ Getting storage account keys
-data: Primary: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz==
-data: Secondary: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy==
-info: storage account keys list command OK
+~$ <span class="userinput">az login</span>
+~$ <span class="userinput">az group create exampleGroupName eastus2</span>
+~$ <span class="userinput">az storage account create --sku Standard_LRS --kind BlobStorage --encryption-services blob --access-tier Hot --https-only true --location eastus2 --resource-group exampleGroupName --name exampleStorageAccountName</span>
+~$ <span class="userinput">az storage account keys list --resource-group exampleGroupName --account-name exampleStorageAccountName
+[
+ {
+ "keyName": "key1",
+ "permissions": "Full",
+ "value": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=="
+ },
+ {
+ "keyName": "key2",
+ "permissions": "Full",
+ "value": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy=="
+ }
+]</span>
~$ <span class="userinput">AZURE_STORAGE_ACCOUNT="exampleStorageAccountName" \
AZURE_STORAGE_ACCESS_KEY="zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz==" \
-azure storage container create exampleContainerName</span>
+azure storage container create --name exampleContainerName</span>
</code></pre>
</notextile>
{% include 'assign_volume_uuid' %}
-<notextile><pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Volumes:
- <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000000</span>:
+<notextile><pre><code> Volumes:
+ <span class="userinput">ClusterID</span>-nyw5e-<span class="userinput">000000000000000</span>:
AccessViaHosts:
# This section determines which keepstore servers access the
# volume. In this example, keep0 has read/write access, and
# If the AccessViaHosts section is empty or omitted, all
# keepstore servers will have read/write access to the
# volume.
- "http://<span class="userinput">keep0.uuid_prefix.example.com</span>:25107/": {}
- "http://<span class="userinput">keep1.uuid_prefix.example.com</span>:25107/": {ReadOnly: true}
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
- Driver: Azure
+ Driver: <span class="userinput">Azure</span>
DriverParameters:
# Storage account name and secret key, used for
# authentication.
- StorageAccountName: exampleStorageAccountName
- StorageAccountKey: zzzzzzzzzzzzzzzzzzzzzzzzzz
+ StorageAccountName: <span class="userinput">exampleStorageAccountName</span>
+ StorageAccountKey: <span class="userinput">zzzzzzzzzzzzzzzzzzzzzzzzzz</span>
+
+ # Storage container name.
+ ContainerName: <span class="userinput">exampleContainerName</span>
# The cloud environment to use,
# e.g. "core.chinacloudapi.cn". Defaults to
# "core.windows.net" if blank or omitted.
StorageBaseURL: ""
- # Storage container name.
- ContainerName: exampleContainerName
-
# Time to wait for an upstream response before failing the
# request.
RequestTimeout: 10m
{% include 'assign_volume_uuid' %}
-Note that each volume has an AccessViaHosts section indicating that (for example) keep0's /mnt/local-disk directory is volume 0, while keep1's /mnt/local-disk directory is volume 1.
+Note that each volume entry has an @AccessViaHosts@ section indicating which Keepstore instance(s) will serve that volume. In this example, keep0 and keep1 each have their own data disk. The @/mnt/local-disk@ directory on keep0 is volume @ClusterID-nyw5e-000000000000000@, and the @/mnt/local-disk@ directory on keep1 is volume @ClusterID-nyw5e-000000000000001@ .
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Volumes:
- <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000000</span>:
+<pre><code> Volumes:
+ <span class="userinput">ClusterID</span>-nyw5e-<span class="userinput">000000000000000</span>:
AccessViaHosts:
- "http://<span class="userinput">keep0.uuid_prefix.example.com</span>:25107": {}
- Driver: Directory
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+ Driver: <span class="userinput">Directory</span>
DriverParameters:
# The directory that will be used as the backing store.
- Root: /mnt/local-disk
-
- # When true, read and write operations (for whole 64MiB
- # blocks) on an individual volume will queued and issued
- # serially. When false, read and write operations will be
- # issued concurrently.
- #
- # May improve throughput if you experience contention when
- # there are multiple requests to the same volume.
- #
- # When using SSDs, RAID, or a shared network filesystem, you
- # probably don't want this.
- Serialize: false
+ Root: <span class="userinput">/mnt/local-disk</span>
# How much replication is performed by the underlying
# filesystem. (for example, a network filesystem may provide
# reads.
ReadOnly: false
- # Storage classes to associate with this volume. See "Storage
- # classes" in the "Admin" section of doc.arvados.org.
+ # <a href="{{site.baseurl}}/admin/storage-classes.html">Storage classes</a> to associate with this volume.
StorageClasses: null
- <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000001</span>:
+ <span class="userinput">ClusterID</span>-nyw5e-<span class="userinput">000000000000001</span>:
AccessViaHosts:
- "http://keep1.<span class="userinput">uuid_prefix</span>.example.com:25107": {}
- Driver: Directory
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {}
+ Driver: <span class="userinput">Directory</span>
DriverParameters:
- Root: /mnt/local-disk
+ Root: <span class="userinput">/mnt/local-disk</span>
</code></pre></notextile>
-In the case of a network-attached filesystem, the AccessViaHosts section can have multiple entries. If the filesystem is accessible by all keepstore servers, the AccessViaHosts section can be empty, or omitted entirely.
+In the case of a network-attached filesystem, the @AccessViaHosts@ section can have multiple entries. If the filesystem is accessible by all keepstore servers, the AccessViaHosts section can be empty, or omitted entirely. In this example, the underlying storage system performs replication, so specifying @Replication: 2@ means a block is considered to be stored twice for the purposes of data integrity, while only stored on a single volume from the perspective of Keep.
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Volumes:
- <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000002</span>:
+<pre><code> Volumes:
+ <span class="userinput">ClusterID</span>-nyw5e-<span class="userinput">000000000000002</span>:
AccessViaHosts:
# This section determines which keepstore servers access the
# volume. In this example, keep0 has read/write access, and
# If the AccessViaHosts section is empty or omitted, all
# keepstore servers will have read/write access to the
# volume.
- "http://<span class="userinput">keep0.uuid_prefix.example.com</span>:25107/": {}
- "http://<span class="userinput">keep1.uuid_prefix.example.com</span>:25107/": {ReadOnly: true}
- Driver: Directory
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
+ Driver: <span class="userinput">Directory</span>
DriverParameters:
- Root: /mnt/network-attached-filesystem
+ Root: <span class="userinput">/mnt/network-attached-filesystem</span>
Replication: 2
</code></pre></notextile>
{% include 'assign_volume_uuid' %}
-<notextile><pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Volumes:
- <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000000</span>:
+<notextile><pre><code> Volumes:
+ <span class="userinput">ClusterID</span>-nyw5e-<span class="userinput">000000000000000</span>:
AccessViaHosts:
# This section determines which keepstore servers access the
# volume. In this example, keep0 has read/write access, and
# If the AccessViaHosts section is empty or omitted, all
# keepstore servers will have read/write access to the
# volume.
- "http://<span class="userinput">keep0.uuid_prefix.example.com</span>:25107/": {}
- "http://<span class="userinput">keep1.uuid_prefix.example.com</span>:25107/": {ReadOnly: true}
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
- Driver: S3
+ Driver: <span class="userinput">S3</span>
DriverParameters:
+ # Bucket name.
+ Bucket: <span class="userinput">example-bucket-name</span>
+
# IAM role name to use when retrieving credentials from
# instance metadata. It can be omitted, in which case the
# role name itself will be retrieved from instance metadata
# -- but setting it explicitly may protect you from using
# the wrong credentials in the event of an
# installation/configuration error.
- IAMRole: ""
+ IAMRole: <span class="userinput">""</span>
# If you are not using an IAM role for authentication,
# specify access credentials here instead.
- AccessKey: ""
- SecretKey: ""
+ AccessKey: <span class="userinput">""</span>
+ SecretKey: <span class="userinput">""</span>
+
+ # Storage provider region. For Google Cloud Storage, use ""
+ # or omit.
+ Region: <span class="userinput">us-east-1a</span>
# Storage provider endpoint. For Amazon S3, use "" or
# omit. For Google Cloud Storage, use
# "https://storage.googleapis.com".
Endpoint: ""
- # Storage provider region. For Google Cloud Storage, use ""
- # or omit.
- Region: us-east-1a
-
# Change to true if the region requires a LocationConstraint
# declaration.
LocationConstraint: false
- # Bucket name.
- Bucket: example-bucket-name
-
# Requested page size for "list bucket contents" requests.
IndexPageSize: 1000
# Maximum eventual consistency latency
RaceWindow: 24h
- # Enable deletion (garbage collection) even when the
- # configured BlobTrashLifetime is zero. WARNING: eventual
- # consistency may result in race conditions that can cause
- # data loss. Do not enable this unless you understand and
- # accept the risk.
- UnsafeDelete: false
-
# How much replication is provided by the underlying bucket.
# This is used to inform replication decisions at the Keep
# layer.
</code></pre>
</notextile>
-You can now copy the pipeline template from *qr1hi* to *your cluster*. Replace *dst_cluster* with the *uuid_prefix* of your cluster.
+You can now copy the pipeline template from *qr1hi* to *your cluster*. Replace *dst_cluster* with the *ClusterID* of your cluster.
<notextile>
<pre><code>~$ <span class="userinput"> arv-copy --no-recursive --src qr1hi --dst dst_cluster qr1hi-p5p6p-9pkaxt6qjnkxhhu</span>
If you already have an account in the Arvados Playground, you can follow the instructions in the "*Using arv-copy*":http://doc.arvados.org/user/topics/arv-copy.html user guide to get your *Current token* for source and destination clusters, and use them to create the source *qr1hi.conf* and dst_cluster.conf configuration files.
-You can now copy the pipeline template from *qr1hi* to *your cluster* with or without recursion. Replace *dst_cluster* with the *uuid_prefix* of your cluster.
+You can now copy the pipeline template from *qr1hi* to *your cluster* with or without recursion. Replace *dst_cluster* with the *ClusterID* of your cluster.
*Non-recursive copy:*
<notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Install dependencies
+# "Introduction":#introduction
+# "Set up Docker":#docker
+# "Update fuse.conf":#fuse
+# "Update docker-cleaner.json":#docker-cleaner
+# "Configure Linux cgroups accounting":#cgroups
+# "Install Docker":#install_docker
+# "Configure the Docker daemon":#configure_docker_daemon
+# "Install'python-arvados-fuse and crunch-run and arvados-docker-cleaner":#install-packages
-First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
+h2(#introduction). Introduction
-{% include 'note_python_sc' %}
+This page describes how to configure a compute node so that it can be used to run containers dispatched by Arvados.
-On Red Hat-based systems:
+* If you are using the cloud dispatcher, apply these step and then save a compute node virtual machine image. The virtual machine image id will go in @config.yml@.
+* If you are using SLURM on a static custer, these steps must be duplicated on every compute node, preferrably using a devops tool such as Puppet.
-<notextile>
-<pre><code>~$ <span class="userinput">echo 'exclude=python2-llfuse' | sudo tee -a /etc/yum.conf</span>
-~$ <span class="userinput">sudo yum install python-arvados-fuse crunch-run arvados-docker-cleaner</span>
-</code></pre>
-</notextile>
+h2(#docker). Set up Docker
-On Debian-based systems:
+See "Set up Docker":../install-docker.html
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install python-arvados-fuse crunch-run arvados-docker-cleaner</span>
-</code></pre>
-</notextile>
-
-{% include 'install_compute_docker' %}
+{% assign arvados_component = 'python-arvados-fuse crunch-run arvados-docker-cleaner' %}
{% include 'install_compute_fuse' %}
{% include 'install_docker_cleaner' %}
-h2. Set up SLURM
+{% include 'install_packages' %}
-Install SLURM on the compute node using the same process you used on the API server in the "previous step":install-slurm.html.
+{% assign arvados_component = 'arvados-docker-cleaner' %}
-The @slurm.conf@ and @/etc/munge/munge.key@ files must be identical on all SLURM nodes. Copy the files you created on the API server in the "previous step":install-slurm.html to each compute node.
+{% include 'start_service' %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The SLURM dispatcher can run on any node that can submit requests to both the Arvados API server and the SLURM controller. It is not resource-intensive, so you can run it on the API server node.
-
-h2. Install the dispatcher
+{% include 'notebox_begin_warning' %}
+crunch-dispatch-slurm is only relevant for on premise clusters that will spool jobs to Slurm. Skip this section if you are installing a cloud cluster.
+{% include 'notebox_end' %}
-First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
+# "Introduction":#introduction
+# "Update config.yml":#update-config
+# "Install crunch-dispatch-slurm":#install-packages
+# "Start the service":#start-service
+# "Restart the API server and controller":#restart-api
-On Red Hat-based systems:
+h2(#introduction). Introduction
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install crunch-dispatch-slurm</span>
-~$ <span class="userinput">sudo systemctl enable crunch-dispatch-slurm</span>
-</code></pre>
-</notextile>
+This assumes you already have a SLURM cluster, and have "set up all of your compute nodes":install-compute-node.html . For information on installing SLURM, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html
-On Debian-based systems:
+The Arvados SLURM dispatcher can run on any node that can submit requests to both the Arvados API server and the SLURM controller (via @sbatch@). It is not resource-intensive, so you can run it on the API server node.
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install crunch-dispatch-slurm</span>
-</code></pre>
-</notextile>
+h2(#update-config). Update config.yml (optional)
-h2. Configure the dispatcher (optional)
+Crunch-dispatch-slurm reads the common configuration file at @config.yml@.
-Crunch-dispatch-slurm reads the common configuration file at @/etc/arvados/config.yml@. The essential configuration parameters will already be set by previous install steps, so no additional configuration is required. The following sections describe optional configuration parameters.
+The following configuration parameters are optional.
h3(#PollPeriod). Containers.PollInterval
crunch-dispatch-slurm polls the API server periodically for new containers to run. The @PollInterval@ option controls how often this poll happens. Set this to a string of numbers suffixed with one of the time units @ns@, @us@, @ms@, @s@, @m@, or @h@. For example:
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
<code class="userinput">PollInterval: <b>3m30s</b>
</code></pre>
</notextile>
Supports suffixes @KB@, @KiB@, @MB@, @MiB@, @GB@, @GiB@, @TB@, @TiB@, @PB@, @PiB@, @EB@, @EiB@ (where @KB@ is 10[^3^], @KiB@ is 2[^10^], @MB@ is 10[^6^], @MiB@ is 2[^20^] and so forth).
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
<code class="userinput">ReserveExtraRAM: <b>256MiB</b></code>
</pre>
</notextile>
If SLURM is unable to run a container, the dispatcher will submit it again after the next PollPeriod. If PollPeriod is very short, this can be excessive. If MinRetryPeriod is set, the dispatcher will avoid submitting the same container to SLURM more than once in the given time span.
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
<code class="userinput">MinRetryPeriod: <b>30s</b></code>
</pre>
</notextile>
Some Arvados installations run a local keepstore on each compute node to handle all Keep traffic. To override Keep service discovery and access the local keep server instead of the global servers, set ARVADOS_KEEP_SERVICES in SbatchEnvironmentVariables:
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
SLURM:
<span class="userinput">SbatchEnvironmentVariables:
ARVADOS_KEEP_SERVICES: "http://127.0.0.1:25107"</span>
The smallest usable value is @1@. The default value of @10@ is used if this option is zero or negative. Example:
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
SLURM:
<code class="userinput">PrioritySpread: <b>1000</b></code></pre>
</notextile>
When crunch-dispatch-slurm invokes @sbatch@, you can add arguments to the command by specifying @SbatchArguments@. You can use this to send the jobs to specific cluster partitions or add resource requests. Set @SbatchArguments@ to an array of strings. For example:
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
SLURM:
<code class="userinput">SbatchArgumentsList:
- <b>"--partition=PartitionName"</b></code>
If your SLURM cluster uses the @task/cgroup@ TaskPlugin, you can configure Crunch's Docker containers to be dispatched inside SLURM's cgroups. This provides consistent enforcement of resource constraints. To do this, use a crunch-dispatch-slurm configuration like the following:
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
<code class="userinput">CrunchRunArgumentsList:
- <b>"-cgroup-parent-subsystem=memory"</b></code>
</pre>
Older Linux kernels (prior to 3.18) have bugs in network namespace handling which can lead to compute node lockups. This by is indicated by blocked kernel tasks in "Workqueue: netns cleanup_net". If you are experiencing this problem, as a workaround you can disable use of network namespaces by Docker across the cluster. Be aware this reduces container isolation, which may be a security risk.
<notextile>
-<pre>
-Clusters:
- zzzzz:
- Containers:
+<pre> Containers:
<code class="userinput">CrunchRunArgumentsList:
- <b>"-container-enable-networking=always"</b>
- <b>"-container-network-mode=host"</b></code>
</pre>
</notextile>
-h2. Restart the dispatcher
+{% assign arvados_component = 'crunch-dispatch-slurm' %}
-{% include 'notebox_begin' %}
-
-The crunch-dispatch-slurm package includes configuration files for systemd. If you're using a different init system, you'll need to configure a service to start and stop a @crunch-dispatch-slurm@ process as desired. The process should run from a directory where the @crunch@ user has write permission on all compute nodes, such as its home directory or @/tmp@. You do not need to specify any additional switches or environment variables.
-
-{% include 'notebox_end' %}
+{% include 'install_packages' %}
-Restart the dispatcher to run with your new configuration:
+{% include 'start_service' %}
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart crunch-dispatch-slurm</span>
-</code></pre>
-</notextile>
+{% include 'restart_api' %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-
-Containers can be dispatched to a SLURM cluster. The dispatcher sends work to the cluster using SLURM's @sbatch@ command, so it works in a variety of SLURM configurations.
-
-In order to run containers, you must run the dispatcher as a user that has permission to set up FUSE mounts and run Docker containers on each compute node. This install guide refers to this user as the @crunch@ user. We recommend you create this user on each compute node with the same UID and GID, and add it to the @fuse@ and @docker@ system groups to grant it the necessary permissions. However, you can run the dispatcher under any account with sufficient permissions across the cluster.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+Containers can be dispatched to a SLURM cluster. The dispatcher sends work to the cluster using SLURM's @sbatch@ command, so it works in a variety of SLURM configurations.
+
+In order to run containers, you must run the dispatcher as a user that has permission to set up FUSE mounts and run Docker containers on each compute node. This install guide refers to this user as the @crunch@ user. We recommend you create this user on each compute node with the same UID and GID, and add it to the @fuse@ and @docker@ system groups to grant it the necessary permissions. However, you can run the dispatcher under any account with sufficient permissions across the cluster.
+
+
On the API server, install SLURM and munge, and generate a munge key.
On Debian-based systems:
Now we need to give SLURM a configuration file. On Debian-based systems, this is installed at @/etc/slurm-llnl/slurm.conf@. On Red Hat-based systems, this is installed at @/etc/slurm/slurm.conf@. Here's an example @slurm.conf@:
<notextile>
-<pre>
-ControlMachine=uuid_prefix.your.domain
+<pre><code>
+ControlMachine=<span class="userinput">ClusterID.example.com</class>
SlurmctldPort=6817
SlurmdPort=6818
AuthType=auth/munge
NodeName=compute[0-255]
PartitionName=compute Nodes=compute[0-255] Default=YES Shared=YES
-</pre>
+</code></pre>
</notextile>
h3. SLURM configuration essentials
Each hostname in @slurm.conf@ must also resolve correctly on all SLURM worker nodes as well as the controller itself. Furthermore, the hostnames used in the configuration file must match the hostnames reported by @hostname@ or @hostname -s@ on the nodes themselves. This applies to the ControlMachine as well as the worker nodes.
For example:
-* In @slurm.conf@ on control and worker nodes: @ControlMachine=uuid_prefix.your.domain@
+* In @slurm.conf@ on control and worker nodes: @ControlMachine=ClusterID.example.com@
* In @slurm.conf@ on control and worker nodes: @NodeName=compute[0-255]@
-* In @/etc/resolv.conf@ on control and worker nodes: @search uuid_prefix.your.domain@
-* On the control node: @hostname@ reports @uuid_prefix.your.domain@
-* On worker node 123: @hostname@ reports @compute123.uuid_prefix.your.domain@
+* In @/etc/resolv.conf@ on control and worker nodes: @search ClusterID.example.com@
+* On the control node: @hostname@ reports @ClusterID.example.com@
+* On worker node 123: @hostname@ reports @compute123.ClusterID.example.com@
h3. Automatic hostname assignment
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+{% include 'notebox_begin_warning' %}
+crunch-dispatch-slurm is only relevant for on premise clusters that will spool jobs to Slurm. Skip this section if you are installing a cloud cluster.
+{% include 'notebox_end' %}
+
h2. Test compute node setup
You should now be able to submit SLURM jobs that run in Docker containers. On the node where you're running the dispatcher, you can test this by running:
</code></pre>
</notextile>
-*On your shell server*, submit a simple container request:
+Submit a simple container request:
<notextile>
<pre><code>shell:~$ <span class="userinput">arv container_request create --container-request '{
</code></pre>
</notextile>
-This command should return a record with a @container_uuid@ field. Once crunch-dispatch-slurm polls the API server for new containers to run, you should see it dispatch that same container. It will log messages like:
+This command should return a record with a @container_uuid@ field. Once @crunch-dispatch-slurm@ polls the API server for new containers to run, you should see it dispatch that same container. It will log messages like:
<notextile>
<pre><code>2016/08/05 13:52:54 Monitoring container zzzzz-dz642-hdp2vpu9nq14tx0 started
</code></pre>
</notextile>
-If you do not see crunch-dispatch-slurm try to dispatch the container, double-check that it is running and that the API hostname and token in @/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml@ are correct.
-
Before the container finishes, SLURM's @squeue@ command will show the new job in the list of queued and running jobs. For example, you might see:
<notextile>
</code></pre>
</notextile>
-If the container does not dispatch successfully, refer to the crunch-dispatch-slurm logs for information about why it failed.
+If the container does not dispatch successfully, refer to the @crunch-dispatch-slurm@ logs for information about why it failed.
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Setting up Google auth
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+In order to use Google for authentication, you must use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
+
+# Go to the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> and select or create a project; this will take you to the project page.
+# Click on *+ Enable APIs and Services*
+## Search for *People API* and click on *Enable API*.
+# Navigate back to the main "APIs & Services" page
+# On the sidebar, click on *OAuth consent screen*
+## On consent screen settings, enter your identifying details
+## Under *Authorized domains* add @example.com@
+## Click on *Save*.
+# On the sidebar, click on *Credentials*; then click on *Create credentials*→*OAuth Client ID*
+# Under *Application type* select *Web application*.
+# You must set the authorization origins. Edit @auth.example.com@ to the appropriate hostname that you will use to access the SSO service:
+## JavaScript origin should be @https://ClusterID.example.com/@ (using Arvados-controller based login) or @https://auth.example.com/@ (for the SSO server)
+## Redirect URI should be @https://ClusterID.example.com/login@ (using Arvados-controller based login) or @https://auth.example.com/users/auth/google_oauth2/callback@ (for the SSO server)
+# Copy the values of *Client ID* and *Client secret* from the Google Developers Console and add them to the appropriate configuration.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados components run on GNU/Linux systems, and supports multiple cloud operating stacks. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS. "Arvados is Free Software":{{site.baseurl}}/copying/copying.html and self-install installations are not limited in any way. Commercial support and development are also available from "Curii Corporation.":mailto:info@curii.com
+Arvados components run on GNU/Linux systems, and supports AWS, GCP and Azure cloud platforms as well as on-premises installs. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS. "Arvados is Free Software":{{site.baseurl}}/user/copying/copying.html and self-install installations are not limited in any way. Commercial support and development are also available from "Curii Corporation.":mailto:info@curii.com
Arvados components can be installed and configured in a number of different ways.
<div class="offset1">
table(table table-bordered table-condensed).
|||\5=. Appropriate for|
-||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados System Testing|
+||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
---
layout: default
navsection: installguide
-title: Install the API server
+title: Install API server and Controller
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Install prerequisites
+# "Introduction":#introduction
+# "Install dependencies":#dependencies
+# "Set up database":#database-setup
+# "Update config.yml":#update-config
+# "Update nginx configuration":#update-nginx
+# "Install arvados-api-server and arvados-controller":#install-packages
+# "Confirm working installation":#confirm-working
-The Arvados package repository includes an API server package that can help automate much of the deployment.
+h2(#introduction). Introduction
-h3(#install_ruby_and_bundler). Install Ruby and Bundler
+The Arvados core API server consists of four services: PostgreSQL, Arvados Rails API, Arvados Controller, and Nginx.
-{% include 'install_ruby_and_bundler' %}
+Here is a simplified diagram showing the relationship between the core services. Client requests arrive at the public-facing Nginx reverse proxy. The request is forwarded to Arvados controller. The controller is able handle some requests itself, the rest are forwarded to the Arvados Rails API. The Rails API server implements the majority of business logic, communicating with the PostgreSQL database to fetch data and make transactional updates. All services are stateless, except the PostgreSQL database. This guide assumes all of these services will be installed on the same node, but it is possible to install these services across multiple nodes.
-h2(#install_apiserver). Install API server and dependencies
+!(full-width){{site.baseurl}}/images/proxy-chain.svg!
-On a Debian-based system, install the following packages:
+h2(#dependencies). Install dependencies
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install bison build-essential libcurl4-openssl-dev git arvados-api-server</span>
-</code></pre>
-</notextile>
-
-On a Red Hat-based system, install the following packages:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install bison make automake gcc gcc-c++ libcurl-devel git arvados-api-server</span>
-</code></pre>
-</notextile>
+# "Install PostgreSQL":install-postgresql.html
+# "Install Ruby and Bundler":ruby.html
+# "Install nginx":nginx.html
+# "Install Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html
-{% include 'install_git' %}
+h2(#database-setup). Set up database
-h2(#configure_application). Configure the API server
+{% assign service_role = "arvados" %}
+{% assign service_database = "arvados_production" %}
+{% assign use_contrib = true %}
+{% include 'install_postgres_database' %}
-Edit @/etc/arvados/config.yml@ to set the keys below. Only the most important configuration options are listed here. The example configuration fragments given below should be merged into a single configuration structure. Correct indentation is important. The full set of configuration options are listed in "config.yml":{{site.baseurl}}/admin/config.html
+h2(#update-config). Update config.yml
-h3(#uuid_prefix). ClusterID
+Starting from an "empty config.yml file,":config.html#empty add the following configuration keys.
-The @ClusterID@ is used for all database identifiers to identify the record as originating from this site. It is the first key under @Clusters@ in @config.yml@. It must be exactly 5 lowercase ASCII letters and digits. All configuration items go under the cluster id key (replace @zzzzz@ with your cluster id in the examples below).
+h3. Tokens
<notextile>
-<pre><code>Clusters:
- <span class="userinput">zzzzz</span>:
- ...</code></pre>
-</notextile>
-
-h3(#configure). PostgreSQL.Connection
-
-Replace the @xxxxxxxx@ database password placeholder with the "password you generated during database setup":install-postgresql.html#api.
-
-<notextile>
-<pre><code>Clusters:
- zzzzz:
- PostgreSQL:
- Connection:
- host: <span class="userinput">localhost</span>
- user: <span class="userinput">arvados</span>
- password: <span class="userinput">xxxxxxxx</span>
- dbname: <span class="userinput">arvados_production</span>
- </code></pre>
-</notextile>
-
-h3. API.RailsSessionSecretToken
-
-The @API.RailsSessionSecretToken@ is used for for signing cookies. IMPORTANT: This is a site secret. It should be at least 50 characters. Generate a random value and set it in @config.yml@:
-
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy
-</code></pre></notextile>
-
-Example @config.yml@:
-
-<notextile>
-<pre><code>Clusters:
- zzzzz:
+<pre><code> SystemRootToken: <span class="userinput">"$system_root_token"</span>
+ ManagementToken: <span class="userinput">"$management_token"</span>
API:
- RailsSessionSecretToken: <span class="userinput">yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy</span></code></pre>
-</notextile>
-
-h3(#blob_signing_key). Collections.BlobSigningKey
-
-The @Collections.BlobSigningKey@ is used to enforce access control to Keep blocks. This same key must be provided to the Keepstore daemons when "installing Keepstore servers.":install-keepstore.html IMPORTANT: This is a site secret. It should be at least 50 characters. Generate a random value and set it in @config.yml@:
-
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-</code></pre></notextile>
-
-Example @config.yml@:
-
-<notextile>
-<pre><code>Clusters:
- zzzzz:
+ RailsSessionSecretToken: <span class="userinput">"$rails_secret_token"</span>
Collections:
- BlobSigningKey: <span class="userinput">xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx</span></code></pre>
+ BlobSigningKey: <span class="userinput">"blob_signing_key"</span>
+</code></pre>
</notextile>
-h3(#omniauth). Login.ProviderAppID, Login.ProviderAppSecret, Services.SSO.ExternalURL
+@SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
-The following settings enable the API server to communicate with the "Single Sign On (SSO) server":install-sso.html to authenticate user log in.
+@ManagementToken@ is used to authenticate access to system metrics.
-Set @Services.SSO.ExternalURL@ to the base URL where your SSO server is installed. This should be a URL consisting of the scheme and host (and optionally, port), without a trailing slash.
+@API.RailsSessionSecretToken@ is required by the API server.
-Set @Login.ProviderAppID@ and @Login.ProviderAppSecret@ to the corresponding values for @app_id@ and @app_secret@ used in the "Create arvados-server client for Single Sign On (SSO)":install-sso.html#client step.
+@Collections.BlobSigningKey@ is used to control access to Keep blocks.
-Example @config.yml@:
+You can generate a random token for each of these items at the command line like this:
<notextile>
-<pre><code>Clusters:
- zzzzz:
- Services:
- SSO:
- ExternalURL: <span class="userinput">https://sso.example.com</span>
- Login:
- ProviderAppID: <span class="userinput">arvados-server</span>
- ProviderAppSecret: <span class="userinput">wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww</span></code></pre>
-</notextile>
-
-h3. Services.Workbench1.ExternalURL
-
-Set @Services.Workbench1.ExternalURL@ to the URL of your workbench application after following "Install Workbench.":install-workbench-app.html
-
-Example @config.yml@:
-
-<notextile>
-<pre><code>Clusters:
- zzzzz:
- Services:
- Workbench1:
- ExternalURL: <span class="userinput">https://workbench.zzzzz.example.com</span></code></pre>
+<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50; echo</span>
+</code></pre>
</notextile>
-h3. Services.Websocket.ExternalURL
-
-Set @Services.Websocket.ExternalURL@ to the @wss://@ URL of the API server websocket endpoint after following "Install the websocket server":install-ws.html .
-
-Example @config.yml@:
+h3. PostgreSQL.Connection
<notextile>
-<pre><code>Clusters:
- zzzzz:
- Services:
- Websocket:
- ExternalURL: <span class="userinput">wss://ws.zzzzz.example.com</span></code></pre>
+<pre><code> PostgreSQL:
+ Connection:
+ host: <span class="userinput">localhost</span>
+ user: <span class="userinput">arvados</span>
+ password: <span class="userinput">$postgres_password</span>
+ dbname: <span class="userinput">arvados_production</span>
+</code></pre>
</notextile>
-h3(#git_repositories_dir). Git.Repositories
-
-The @Git.Repositories@ setting specifies the directory where user git repositories will be stored.
+Replace the @$postgres_password@ placeholder with the password you generated during "database setup":#database-setup .
-The git server setup process is covered on "its own page":install-arv-git-httpd.html. For now, create an empty directory in the default location:
+h3. Services
<notextile>
-<pre><code>~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git/repositories</span>
-</code></pre></notextile>
-
-If you intend to store your git repositories in a different location, specify that location in @config.yml@. Example:
-
-<notextile>
-<pre><code>Clusters:
- zzzzz:
- Git:
- Repositories: <span class="userinput">/var/lib/arvados/git/repositories</span></code></pre>
+<pre><code> Services:
+ Controller:
+ ExternalURL: <span class="userinput">"https://ClusterID.example.com"</span>
+ InternalURLs:
+ <span class="userinput">"http://localhost:8003": {}</span>
+ RailsAPI:
+ # Does not have an ExternalURL
+ InternalURLs:
+ <span class="userinput">"http://localhost:8004": {}</span>
+</code></pre>
</notextile>
-h3(#enable_legacy_jobs_api). Containers.JobsAPI.Enable
+Replace @ClusterID.example.com@ with the hostname that you previously selected for the API server.
-Enable the legacy "Jobs API":install-crunch-dispatch.html . Note: new installations should use the "Containers API":crunch2-slurm/install-prerequisites.html
+The @Services@ section of the configuration helps Arvados components contact one another (service discovery). Each service has one or more @InternalURLs@ and an @ExternalURL@. The @InternalURLs@ describe where the service runs, and how the Nginx reverse proxy will connect to it. The @ExternalURL@ is how external clients contact the service.
-Disabling the jobs API means methods involving @jobs@, @job_tasks@, @pipeline_templates@ and @pipeline_instances@ are disabled. This functionality is superceded by the containers API which consists of @container_requests@, @containers@ and @workflows@. Arvados clients (such as @arvados-cwl-runner@) detect which APIs are available and adjust behavior accordingly. Note the configuration value must be a quoted string.
+h2(#update-nginx). Update nginx configuration
-* 'auto' -- (default) enable the Jobs API only if it has been used before (i.e., there are job records in the database), otherwise disable jobs API .
-* 'true' -- enable the Jobs API even if there are no existing job records.
-* 'false' -- disable the Jobs API even in the presence of existing job records.
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-api-and-controller.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
<notextile>
-<pre><code>Clusters:
- zzzzz:
- Containers:
- JobsAPI:
- Enable: <span class="userinput">'auto'</span></code></pre>
-</notextile>
-
-h4(#git_internal_dir). Containers.JobsAPI.GitInternalDir
+<pre><code>proxy_http_version 1.1;
-Only required if the legacy "Jobs API" is enabled, otherwise you should skip this.
+# When Keep clients request a list of Keep services from the API
+# server, use the origin IP address to determine if the request came
+# from the internal subnet or it is an external client. This sets the
+# $external_client variable which in turn is used to set the
+# X-External-Client header.
+#
+# The API server uses this header to choose whether to respond to a
+# "available keep services" request with either a list of internal keep
+# servers (0) or with the keepproxy (1).
+#
+# <span class="userinput">Following the example here, update the 10.20.30.0/24 netmask</span>
+# <span class="userinput">to match your private subnet.</span>
+# <span class="userinput">Update 1.2.3.4 and add lines as necessary with the public IP</span>
+# <span class="userinput">address of all servers that can also access the private network to</span>
+# <span class="userinput">ensure they are not considered 'external'.</span>
-The @Containers.JobsAPI.GitInternalDir@ setting specifies the location of Arvados' internal git repository. By default this is @/var/lib/arvados/internal.git@. This repository stores git commits that have been used to run Crunch jobs. It should _not_ be a subdirectory of the directory in @Git.Repositories@.
+geo $external_client {
+ default 1;
+ 127.0.0.0/24 0;
+ <span class="userinput">10.20.30.0/24</span> 0;
+ <span class="userinput">1.2.3.4/32</span> 0;
+}
-Example @config.yml@:
+# This is the port where nginx expects to contact arvados-controller.
+upstream controller {
+ server localhost:8003 fail_timeout=10s;
+}
-<notextile>
-<pre><code>Clusters:
- zzzzz:
- Containers:
- JobsAPI:
- GitInternalDir: <span class="userinput">/var/lib/arvados/internal.git</span></code></pre>
-</notextile>
+server {
+ # This configures the public https port that clients will actually connect to,
+ # the request is reverse proxied to the upstream 'controller'
-h2(#set_up). Set up Nginx and Passenger
+ listen *:443 ssl;
+ server_name <span class="userinput">xxxxx.example.com</span>;
-The Nginx server will serve API requests using Passenger. It will also be used to proxy SSL requests to other services which are covered later in this guide.
+ ssl on;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-First, "Install Nginx and Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html.
+ # Refer to the comment about this setting in the passenger (arvados
+ # api server) section of your Nginx configuration.
+ client_max_body_size 128m;
-Edit the http section of your Nginx configuration to run the Passenger server. Add a block like the following, adding SSL and logging parameters to taste:
+ location / {
+ proxy_pass http://controller;
+ proxy_redirect off;
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-External-Client $external_client;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+}
-<notextile>
-<pre><code>
server {
- listen 127.0.0.1:8000;
+ # This configures the Arvados API server. It is written using Ruby
+ # on Rails and uses the Passenger application server.
+
+ listen <span class="userinput">localhost:8004</span>;
server_name localhost-api;
root /var/www/arvados-api/current/public;
index index.html index.htm index.php;
passenger_enabled on;
- # If you're using RVM, uncomment the line below.
+
+ # <span class="userinput">If you are using RVM, uncomment the line below.</span>
+ # <span class="userinput">If you're using system ruby, leave it commented out.</span>
#passenger_ruby /usr/local/rvm/wrappers/default/ruby;
# This value effectively limits the size of API objects users can
# create, especially collections. If you change this, you should
# also ensure the following settings match it:
- # * `client_max_body_size` in the server section below
- # * `client_max_body_size` in the Workbench Nginx configuration (twice)
+ # * `client_max_body_size` in the previous server section
# * `API.MaxRequestSize` in config.yml
client_max_body_size 128m;
}
+</code></pre>
+</notextile>
-upstream api {
- server 127.0.0.1:8000 fail_timeout=10s;
-}
+{% assign arvados_component = 'arvados-api-server arvados-controller' %}
-proxy_http_version 1.1;
+{% include 'install_packages' %}
-# When Keep clients request a list of Keep services from the API server, the
-# server will automatically return the list of available proxies if
-# the request headers include X-External-Client: 1. Following the example
-# here, at the end of this section, add a line for each netmask that has
-# direct access to Keep storage daemons to set this header value to 0.
-geo $external_client {
- default 1;
- <span class="userinput">10.20.30.0/24</span> 0;
-}
-</code></pre>
-</notextile>
+{% assign arvados_component = 'arvados-controller' %}
-Restart Nginx to apply the new configuration.
+{% include 'start_service' %}
-<notextile>
-<pre><code>~$ <span class="userinput">sudo nginx -s reload</span>
-</code></pre>
-</notextile>
+h2(#confirm-working). Confirm working installation
-h2. Prepare the API server deployment
+Confirm working controller:
-{% assign railspkg = "arvados-api-server" %}
-{% include 'install_rails_reconfigure' %}
+<notextile><pre><code>$ curl https://<span class="userinput">ClusterID.example.com</span>/arvados/v1/config
+</code></pre></notextile>
-{% include 'notebox_begin' %}
-You can safely ignore the following messages if they appear while this command runs:
+Confirm working Rails API server:
+
+<notextile><pre><code>$ curl https://<span class="userinput">ClusterID.example.com</span>/discovery/v1/apis/arvados/v1/rest
+</code></pre></notextile>
+
+Confirm that you can use the system root token to act as the system root user:
+
+<notextile><pre><code>
+$ curl -H "Authorization: Bearer $system_root_token" https://<span class="userinput">ClusterID.example.com</span>/arvados/v1/users/current
+</code></pre></notextile>
-<notextile><pre>Don't run Bundler as root. Bundler can ask for sudo if it is needed, and installing your bundle as root will
-break this application for all non-root users on this machine.</pre></notextile>
+h3. Troubleshooting
-<notextile><pre>fatal: Not a git repository (or any of the parent directories): .git</pre></notextile>
-{% include 'notebox_end' %}
+If you are getting TLS errors, make sure the @ssl_certificate@ directive in your nginx configuration has the "full certificate chain":http://nginx.org/en/docs/http/configuring_https_servers.html#chains
-h2. Troubleshooting
+Logs can be found in @/var/www/arvados-api/current/log/production.log@ and using @journalctl -u arvados-controller@.
-Once you have the API Server up and running you may need to check it back if dealing with client related issues. Please read our "admin troubleshooting notes":{{site.baseurl}}/admin/troubleshooting.html on how requests can be tracked down between services.
\ No newline at end of file
+See also the admin page on "Logging":{{site.baseurl}}/admin/logging.html .
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados allows users to create their own private and public git repositories, and clone/push them using SSH and HTTPS.
+# "Introduction":#introduction
+# "Install dependencies":#dependencies
+# "Create "git" user and storage directory":#create
+# "Install gitolite":#gitolite
+# "Configure gitolite":#config-gitolite
+# "Configure git synchronization":#sync
+# "Update config.yml":#update-config
+# "Update nginx configuration":#update-nginx
+# "Install arvados-git-httpd package":#install-packages
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
+
+h2(#introduction). Introduction
+
+Arvados support for git repository management enables using Arvados permissions to control access to git repositories. Users can create their own private and public git repositories and share them with others.
The git hosting setup involves three components.
* The "arvados-git-sync.rb" script polls the API server for the current list of repositories, creates bare repositories, and updates the local permission cache used by gitolite.
-* Gitolite provides SSH access.
-* arvados-git-http provides HTTPS access.
+* Gitolite provides SSH access. Users authenticate by SSH keys.
+* arvados-git-http provides HTTPS access. Users authenticate by Arvados tokens.
-It is not strictly necessary to deploy _both_ SSH and HTTPS access, but we recommend deploying both:
-* SSH is a more appropriate way to authenticate from a user's workstation because it does not require managing tokens on the client side;
-* HTTPS is a more appropriate way to authenticate from a shell VM because it does not depend on SSH agent forwarding (SSH clients' agent forwarding features tend to behave as if the remote machine is fully trusted).
-* HTTPS is also used by Arvados Composer to access git repositories from the browser.
+Git services must be installed on the same host as the Arvados Rails API server.
-The HTTPS instructions given below will not work if you skip the SSH setup steps.
+h2(#dependencies). Install dependencies
-h2. Set up DNS
-
-By convention, we use the following hostname for the git service:
+h3. Centos 7
<notextile>
-<pre><code>git.<span class="userinput">uuid_prefix</span>.your.domain
+<pre><code># <span class="userinput">yum install git perl-Data-Dumper openssh-server</span>
</code></pre>
</notextile>
-{% include 'notebox_begin' %}
-Here, we show how to install the git hosting services *on the same host as your API server.* Using a different host is not yet fully supported. On this page we will refer to it as your git server.
-{% include 'notebox_end' %}
-
-DNS and network configuration should be set up so port 443 reaches your HTTPS proxy, and port 22 reaches the OpenSSH service on your git server.
-
-h2. Generate an API token
-
-{% assign railshost = "gitserver" %}
-{% assign railscmd = "bundle exec ./script/create_superuser_token.rb" %}
-{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
-Use the following command to generate an API token. {% include 'install_rails_command' %}
-
-Copy that token; you'll need it in a minute.
-
-h2. Install git and other dependencies
-
-On Debian-based systems:
+h3. Debian and Ubuntu
<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo apt-get install git openssh-server</span>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install git openssh-server</span>
</code></pre>
</notextile>
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo yum install git perl-Data-Dumper openssh-server</span>
-</code></pre>
-</notextile>
-
-{% include 'install_git' %}
-
-h2. Create a "git" user and a storage directory
+h2(#create). Create "git" user and storage directory
Gitolite and some additional scripts will be installed in @/var/lib/arvados/git@, which means hosted repository data will be stored in @/var/lib/arvados/git/repositories@. If you choose to install gitolite in a different location, make sure to update the @git_repositories_dir@ entry in your API server's @application.yml@ file accordingly: for example, if you install gitolite at @/data/gitolite@ then your @git_repositories_dir@ will be @/data/gitolite/repositories@.
</code></pre>
</notextile>
-h2. Install gitolite
+h2(#gitolite). Install gitolite
-Check "https://github.com/sitaramc/gitolite/tags":https://github.com/sitaramc/gitolite/tags for the latest stable version. This guide was tested with @v3.6.4@. _Versions below 3.0 are missing some features needed by Arvados, and should not be used._
+Check "https://github.com/sitaramc/gitolite/tags":https://github.com/sitaramc/gitolite/tags for the latest stable version. This guide was tested with @v3.6.11@. _Versions below 3.0 are missing some features needed by Arvados, and should not be used._
Download and install the version you selected.
<notextile>
-<pre><code>git@gitserver:~$ <span class="userinput">echo 'PATH=$HOME/bin:$PATH' >.profile</span>
-git@gitserver:~$ <span class="userinput">source .profile</span>
-git@gitserver:~$ <span class="userinput">git clone --branch <b>v3.6.4</b> https://github.com/sitaramc/gitolite</span>
+<pre><code>$ <span class="userinput">sudo -u git -i bash</span>
+git@gitserver:~$ <span class="userinput">echo 'PATH=$HOME/bin:$PATH' >.profile</span>
+git@gitserver:~$ <span class="userinput">. .profile</span>
+git@gitserver:~$ <span class="userinput">git clone --branch <b>v3.6.11</b> https://github.com/sitaramc/gitolite</span>
...
Note: checking out '5d24ae666bfd2fa9093d67c840eb8d686992083f'.
...
</code></pre>
</notextile>
-h3. Configure gitolite
+h2(#config-gitolite). Configure gitolite
Configure gitolite to look up a repository name like @username/reponame.git@ and find the appropriate bare repository storage directory.
</span></code></pre>
</notextile>
-h2. Configure git synchronization
+h2(#sync). Configure git synchronization
Create a configuration file @/var/www/arvados-api/current/config/arvados-clients.yml@ using the following template, filling in the appropriate values for your system.
-* For @arvados_api_token@, use the token you generated above.
+* For @arvados_api_token@, use @SystemRootToken@
* For @gitolite_arvados_git_user_key@, provide the public key you generated above, i.e., the contents of @~git/.ssh/id_rsa.pub@.
<notextile>
<pre><code>production:
gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
gitolite_tmp: /var/lib/arvados/git
- arvados_api_host: <span class="userinput">uuid_prefix.example.com</span>
+ arvados_api_host: <span class="userinput">ClusterID.example.com</span>
arvados_api_token: "<span class="userinput">zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</span>"
arvados_api_host_insecure: <span class="userinput">false</span>
gitolite_arvados_git_user_key: "<span class="userinput">ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7aBIDAAgMQN16Pg6eHmvc+D+6TljwCGr4YGUBphSdVb25UyBCeAEgzqRiqy0IjQR2BLtSirXr+1SJAcQfBgI/jwR7FG+YIzJ4ND9JFEfcpq20FvWnMMQ6XD3y3xrZ1/h/RdBNwy4QCqjiXuxDpDB7VNP9/oeAzoATPZGhqjPfNS+RRVEQpC6BzZdsR+S838E53URguBOf9yrPwdHvosZn7VC0akeWQerHqaBIpSfDMtaM4+9s1Gdsz0iP85rtj/6U/K/XOuv2CZsuVZZ52nu3soHnEX2nx2IaXMS3L8Z+lfOXB2T6EaJgXF7Z9ME5K1tx9TSNTRcYCiKztXLNLSbp git@gitserver</span>"
</code></pre>
</notextile>
-h3. Enable the synchronization script
+<pre>
+$ sudo chown git:git /var/www/arvados-api/current/config/arvados-clients.yml
+$ sudo chmod og-rwx /var/www/arvados-api/current/config/arvados-clients.yml
+</pre>
-The API server package includes a script that retrieves the current set of repository names and permissions from the API, writes them to @arvadosaliases.pl@ in a format usable by gitolite, and triggers gitolite hooks which create new empty repositories if needed. This script should run every 2 to 5 minutes.
+h3. Test configuration
-If you are using RVM, create @/etc/cron.d/arvados-git-sync@ with the following content:
+notextile. <pre><code>$ <span class="userinput">sudo -u git -i bash -c 'cd /var/www/arvados-api/current && bundle exec script/arvados-git-sync.rb production'</span></code></pre>
-<notextile>
-<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && /usr/local/rvm/bin/rvm-exec default bundle exec script/arvados-git-sync.rb production</span>
-</code></pre>
-</notextile>
+h3. Enable the synchronization script
-Otherwise, create @/etc/cron.d/arvados-git-sync@ with the following content:
+The API server package includes a script that retrieves the current set of repository names and permissions from the API, writes them to @arvadosaliases.pl@ in a format usable by gitolite, and triggers gitolite hooks which create new empty repositories if needed. This script should run every 2 to 5 minutes.
+
+Create @/etc/cron.d/arvados-git-sync@ with the following content:
<notextile>
<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && bundle exec script/arvados-git-sync.rb production</span>
</code></pre>
</notextile>
-h3. Configure the API server to advertise the correct SSH URLs
+h2(#update-config). Update config.yml
-Edit the cluster config at @/etc/arvados/config.yml@ and set @Services.GitSSH.ExternalURL@. Replace @uuid_prefix@ with your cluster id.
+Edit the cluster config at @config.yml@ .
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
+<pre><code> Services:
GitSSH:
- ExternalURL: <span class="userinput">git@git.uuid_prefix.your.domain:</span>
-</code></pre>
-</notextile>
-
-Make sure to include the trailing colon.
-
-h2. Install the arvados-git-httpd package
-
-This is needed only for HTTPS access.
-
-The arvados-git-httpd package provides HTTP access, using Arvados authentication tokens instead of passwords. It is intended to be installed on the system where your git repositories are stored, and accessed through a web proxy that provides SSL support.
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install git arvados-git-httpd</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install git arvados-git-httpd</span>
-~$ <span class="userinput">sudo systemctl enable arvados-git-httpd</span>
-</code></pre>
-</notextile>
-
-Verify that @arvados-git-httpd@ and @git-http-backend@ can be run:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arvados-git-httpd -h</span>
-[...]
-Usage: arvados-git-httpd [-config path/to/arvados/git-httpd.yml]
-[...]
-~$ <span class="userinput">git http-backend</span>
-Status: 500 Internal Server Error
-Expires: Fri, 01 Jan 1980 00:00:00 GMT
-Pragma: no-cache
-Cache-Control: no-cache, max-age=0, must-revalidate
-
-fatal: No REQUEST_METHOD from server
-</code></pre>
-</notextile>
-
-h3. Enable arvados-git-httpd
-
-{% include 'notebox_begin' %}
-
-The arvados-git-httpd package includes configuration files for systemd. If you're using a different init system, you'll need to configure a service to start and stop an @arvados-git-httpd@ process as desired.
-
-{% include 'notebox_end' %}
-
-Edit the cluster config at @/etc/arvados/config.yml@ and set the following values. Replace @uuid_prefix@ with your cluster id.
-
-<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
+ ExternalURL: "<span class="userinput">ssh://git@git.ClusterID.example.com</span>"
GitHTTP:
- ExternalURL: <span class="userinput">https://git.uuid_prefix.your.domain/</span>
+ ExternalURL: <span class="userinput">https://git.ClusterID.example.com/</span>
InternalURLs:
- <span class="userinput">"http://localhost:9001": {}</span>
+ "http://localhost:9001": {}
Git:
GitCommand: <span class="userinput">/var/lib/arvados/git/gitolite/src/gitolite-shell</span>
GitoliteHome: <span class="userinput">/var/lib/arvados/git</span>
</code></pre>
</notextile>
-Make sure to include the trailing slash for @Services.GitHTTP.ExternalURL@.
-
-Restart the systemd service to ensure the new configuration is used.
-
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-git-httpd</span>
-</code></pre>
-</notextile>
-
-h3. Set up a reverse proxy to provide SSL service
-
-The arvados-git-httpd service will be accessible from anywhere on the internet, so we recommend using SSL.
-
-This is best achieved by putting a reverse proxy with SSL support in front of arvados-git-httpd, running on port 443 and passing requests to @arvados-git-httpd@ on port 9001 (or whichever port you used in your run script).
+h2(#update-nginx). Update nginx configuration
-Add the following configuration to the @http@ section of your Nginx configuration:
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-git.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
<notextile>
-<pre><code>
-upstream arvados-git-httpd {
+<pre><code>upstream arvados-git-httpd {
server 127.0.0.1:<span class="userinput">9001</span>;
}
server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name git.<span class="userinput">uuid_prefix.your.domain</span>;
+ listen *:443 ssl;
+ server_name git.<span class="userinput">ClusterID.example.com</span>;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
# The server needs to accept potentially large refpacks from push clients.
- client_max_body_size 50m;
+ client_max_body_size 128m;
location / {
proxy_pass http://arvados-git-httpd;
</code></pre>
</notextile>
-h2. Restart Nginx
+h2(#install-packages). Install the arvados-git-httpd package
+
+The arvados-git-httpd package provides HTTP access, using Arvados authentication tokens instead of passwords. It must be installed on the system where your git repositories are stored.
+
+h3. Centos 7
+
+<notextile>
+<pre><code># <span class="userinput">yum install arvados-git-httpd</span>
+</code></pre>
+</notextile>
-Restart Nginx to make the Nginx and API server configuration changes take effect.
+h3. Debian and Ubuntu
<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo nginx -s reload</span>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install arvados-git-httpd</span>
</code></pre>
</notextile>
-h2. Clone Arvados repository
+h2(#restart-api). Restart the API server and controller
-Here we create a repository object which will be used to set up a hosted clone of the arvados repository on this cluster.
+After adding Workbench to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
<notextile>
-<pre><code>~$ <span class="userinput">uuid_prefix=`arv --format=uuid user current | cut -d- -f1`</span>
-~$ <span class="userinput">echo "Site prefix is '$uuid_prefix'"</span>
-~$ <span class="userinput">all_users_group_uuid="$uuid_prefix-j7d0g-fffffffffffffff"</span>
-~$ <span class="userinput">repo_uuid=`arv --format=uuid repository create --repository "{\"owner_uuid\":\"$uuid_prefix-tpzed-000000000000000\", \"name\":\"arvados\"}"`</span>
-~$ <span class="userinput">echo "Arvados repository uuid is '$repo_uuid'"</span>
-</code></pre></notextile>
+<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
+</code></pre>
+</notextile>
-Create a link object to make the repository object readable by the "All users" group, and therefore by every active user. This makes it possible for users to run the bundled Crunch scripts by specifying @"script_version":"master","repository":"arvados"@ rather than pulling the Arvados source tree into their own repositories.
+h2(#confirm-working). Confirm working installation
+
+Create 'testrepo' in the Arvados database.
<notextile>
-<pre><code>~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
-<span class="userinput">{
- "tail_uuid":"$all_users_group_uuid",
- "head_uuid":"$repo_uuid",
- "link_class":"permission",
- "name":"can_read"
-}
-EOF</span>
+<pre><code>~$ <span class="userinput">arv --format=uuid repository create --repository '{"name":"myusername/testrepo"}'</span>
</code></pre></notextile>
-In a couple of minutes, your arvados-git-sync cron job will create an empty repository on your git server. Seed it with the real arvados repository. If your git credential helpers were configured correctly when you "set up your shell server":install-shell-server.html, the "git push" command will use your API token instead of prompting you for a username and password.
+The arvados-git-sync cron job will notice the new repository record and create a repository on disk. Because it is on a timer (default 5 minutes) you may have to wait a minute or two for it to show up.
+
+h3. SSH
+
+Before you do this, go to Workbench and choose *SSH Keys* from the menu, and upload your public key. Arvados uses the public key to identify you when you access the git repo.
<notextile>
-<pre><code>~$ <span class="userinput">cd /tmp</span>
-/tmp$ <span class="userinput">git clone --bare https://github.com/arvados/arvados.git</span>
-/tmp <span class="userinput">git --git-dir arvados.git push https://git.<b>uuid_prefix.your.domain</b>/arvados.git '*:*'</span>
+<pre><code>~$ <span class="userinput">git clone git@git.ClusterID.example.com:username/testrepo.git</span>
</code></pre>
</notextile>
-If you did not set up a HTTPS service, you can push to <code>git@git.uuid_prefix.your.domain:arvados.git</code> using your SSH key, or by logging in to your git server and using sudo.
+h3. HTTP
+
+Set up git credential helpers as described in "install shell server":install-shell-server.html#config-git for the git command to use your API token instead of prompting you for a username and password.
<notextile>
-<pre><code>gitserver:~$ <span class="userinput">sudo -u git -i bash</span>
-git@gitserver:~$ <span class="userinput">git clone --bare https://github.com/arvados/arvados.git /tmp/arvados.git</span>
-git@gitserver:~$ <span class="userinput">cd /tmp/arvados.git</span>
-git@gitserver:/tmp/arvados.git$ <span class="userinput">gitolite push /var/lib/arvados/git/repositories/<b>your_arvados_repo_uuid</b>.git '*:*'</span>
+<pre><code>~$ <span class="userinput">git clone https://git.ClusterID.example.com/username/testrepo.git</span>
</code></pre>
</notextile>
Arvados Composer is a web-based javascript application for building Common Workflow Languge (CWL) Workflows.
-h2. Prerequisites
+# "Install dependencies":#dependencies
+# "Update config.yml":#update-config
+# "Update Nginx configuration":#update-nginx
+# "Install arvados-composer":#install-packages
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
-In addition to Arvados core services, Composer requires "Arvados hosted git repositories":install-arv-git-httpd.html which are used for storing workflow files.
+h2(#dependencies). Install dependencies
-h2. Install
+In addition to Arvados core services, Composer requires "Arvados hosted git repositories":install-arv-git-httpd.html which are used for storing workflow files.
-Composer may be installed on the same host as Workbench, or on a different host. Composer communicates directly with the Arvados API server. It does not require its own backend and should be served as a static file.
+h2(#configure). Update config.yml
-On a Debian-based system, install the following package:
+Edit @config.yml@ and set @Services.Composer.ExternalURL@ to the location from which it is served:
<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-composer</span>
-</code></pre>
+<pre><code> Services:
+ Composer:
+ ExternalURL: <span class="userinput">https://workbench.CusterID.example.com/composer</span></code></pre>
</notextile>
-On a Red Hat-based system, install the following package:
+h2(#update-nginx). Update nginx configuration
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-composer</span>
-</code></pre>
-</notextile>
+Composer may be served from the same host as Workbench. Composer communicates directly with the Arvados API server. It does not require its own backend and should be served as a static file.
-h2. Configure
+Add the following @location@ sections to @/etc/nginx/conf.d/arvados-workbench.conf@ .
-h3. Nginx
+<notextile>
+<pre><code>server {
+ [...]
-Add Composer to your Nginx configuration. This example will host Composer at @/composer@.
+ location /composer {
+ root /var/www/arvados-composer;
+ index index.html;
+ }
-<pre>
-location /composer {
- root /var/www/arvados-composer
- index index.html
+ location /composer/composer.yml {
+ return 200 '{ "API_HOST": "<span class="userinput">ClusterID.example.com</span>" }';
+ }
}
-</pre>
-
-h3. composer.yml
+</code></pre>
+</notextile>
-Create @/var/www/arvados-composer/composer.yml@ and set @API_HOST@ to your API server:
+{% assign arvados_component = 'arvados-composer' %}
-<pre>
-API_HOST: zzzzz.arvadosapi.com
-</pre>
+{% include 'install_packages' %}
-h3. Workbench link to composer
+{% include 'restart_api' %}
-Edit @config.yml@ and set @Services.Composer.ExternalURL@ to the location from which it is served:
+h2(#confirm-working). Confirm working installation
-<notextile>
-<pre><code>Clusters:
- zzzzz:
- Services:
- Composer:
- ExternalURL: <span class="userinput">https://workbench.zzzzz.arvadosapi.com/composer</span></code></pre>
-</notextile>
+Visit @https://workbench.ClusterID.example.com/composer@ in a browser. You should be able to log in using the login method you configured previously.
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install the controller
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-The arvados-controller service must be installed on your API server node.
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-controller</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-controller</span>
-</code></pre>
-</notextile>
-
-Verify the @arvados-controller@ program is functional:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arvados-controller -h</span>
-Usage:
- -config file
-[...]
-</code></pre>
-</notextile>
-
-h3. Configure Nginx to route requests to the controller
-
-Add @upstream@ and @server@ definitions inside the @http@ section of your Nginx configuration using the following template.
-
-{% include 'notebox_begin' %}
-
-If you are adding arvados-controller to an existing system as part of the upgrade procedure, do not add a new "server" part here. Instead, add only the "upstream" part as shown here, and update your existing "server" section by changing its @proxy_pass@ directive from @http://api@ to @http://controller@.
-
-{% include 'notebox_end' %}
-
-<notextile>
-<pre><code>upstream controller {
- server 127.0.0.1:9004 fail_timeout=10s;
-}
-
-server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name <span class="userinput">uuid_prefix.your.domain</span>;
-
- ssl on;
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- # Refer to the comment about this setting in the passenger (arvados
- # api server) section of your Nginx configuration.
- client_max_body_size 128m;
-
- location / {
- proxy_pass http://controller;
- proxy_redirect off;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host $http_host;
- proxy_set_header X-External-Client $external_client;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
-}
-</code></pre>
-</notextile>
-
-Restart Nginx to apply the new configuration.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo nginx -s reload</span>
-</code></pre>
-</notextile>
-
-h3(#configuration). Configure arvados-controller
-
-Create the cluster configuration file @/etc/arvados/config.yml@ using the following template.
-
-<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
- Controller:
- InternalURLs:
- "http://localhost:<span class="userinput">9004</span>": {} # must match the "upstream controller" section of your Nginx config
- RailsAPI:
- arvados-api-server:
- "http://localhost:<span class="userinput">8000</span>": {} # must match the "upstream api" section of your Nginx config
- PostgreSQL:
- ConnectionPool: 128
- Connection:
- host: localhost
- dbname: arvados_production
- user: arvados
- password: <span class="userinput">xxxxxxxx</span>
- sslmode: require
-</code></pre>
-</notextile>
-
-Create the host configuration file @/etc/arvados/environment@.
-
-<notextile>
-<pre><code>ARVADOS_NODE_PROFILE=apiserver
-</code></pre>
-</notextile>
-
-h3. Start the service (option 1: systemd)
-
-If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
-
-If your system uses systemd, the arvados-controller service should already be set up. Restart it to load the new configuration file, and check its status:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-controller</span>
-~$ <span class="userinput">sudo systemctl status arvados-controller</span>
-● arvados-controller.service - Arvados controller
- Loaded: loaded (/lib/systemd/system/arvados-controller.service; enabled; vendor preset: enabled)
- Active: active (running) since Tue 2018-07-31 13:17:44 UTC; 3s ago
- Docs: https://doc.arvados.org/
- Main PID: 25066 (arvados-control)
- CGroup: /system.slice/arvados-controller.service
- └─25066 /usr/bin/arvados-controller
-
-Jul 31 13:17:44 zzzzz systemd[1]: Starting Arvados controller...
-Jul 31 13:17:44 zzzzz arvados-controller[25191]: {"Listen":"[::]:9004","Service":"arvados-controller","level":"info","msg":"listening","time":"2018-07-31T13:17:44.521694195Z"}
-Jul 31 13:17:44 zzzzz systemd[1]: Started Arvados controller.
-</code></pre>
-</notextile>
-
-Skip ahead to "confirm the service is working":#confirm.
-
-h3(#runit). Start the service (option 2: runit)
-
-Install runit to supervise the arvados-controller daemon. {% include 'install_runit' %}
-
-Create a supervised service.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/arvados-controller</span>
-~$ <span class="userinput">cd /etc/service/arvados-controller</span>
-~$ <span class="userinput">sudo mkdir log log/main</span>
-~$ <span class="userinput">printf '#!/bin/sh\nset -a\n. /etc/arvados/environment\nexec arvados-controller 2>&1\n' | sudo tee run</span>
-~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
-~$ <span class="userinput">sudo chmod +x run log/run</span>
-~$ <span class="userinput">sudo sv exit .</span>
-~$ <span class="userinput">cd -</span>
-</code></pre>
-</notextile>
-
-Use @sv stat@ and check the log file to verify the service is running.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/arvados-controller</span>
-run: /etc/service/arvados-controller: (pid 12520) 2s; run: log: (pid 12519) 2s
-~$ <span class="userinput">tail /etc/service/arvados-controller/log/main/current</span>
-{"Listen":"[::]:9004","Service":"arvados-controller","level":"info","msg":"listening","time":"2018-07-31T13:17:44.521694195Z"}
-</code></pre>
-</notextile>
-
-h3(#confirm). Confirm the service is working
-
-Confirm the service is listening on its assigned port and responding to requests.
-
-<notextile>
-<pre><code>~$ <span class="userinput">curl -X OPTIONS http://0.0.0.0:<b>9004</b>/login</span>
-{"errors":["Forbidden"],"error_token":"1533044555+684b532c"}
-</code></pre>
-</notextile>
-
-h3(#confirm-config). Confirm the public configuration is OK
-
-Confirm the publicly accessible configuration endpoint does not reveal any sensitive information (e.g., a secret that was mistakenly entered under the wrong configuration key). Use the jq program, if you have installed it, to make the JSON document easier to read.
-
-<notextile>
-<pre><code>~$ <span class="userinput">curl http://0.0.0.0:<b>9004</b>/arvados/v1/config | jq .</span>
-{
- "API": {
- "MaxItemsPerResponse": 1000,
- "MaxRequestAmplification": 4,
- "RequestTimeout": "5m"
- },
- ...
-</code></pre>
-</notextile>
layout: default
navsection: installguide
title: Install the cloud dispatcher
-
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The cloud dispatch service is an *experimental* service for running containers on cloud VMs. It eliminates the need for SLURM, Node Manager, and SLURM dispatcher. It works with Microsoft Azure and Amazon EC2; future versions will also support Google Compute Engine.
+{% include 'notebox_begin_warning' %}
+arvados-dispatch-cloud is only relevant for cloud installations. Skip this section if you are installing a on premise cluster that will spool jobs to Slurm.
+{% include 'notebox_end' %}
+
+# "Introduction":#introduction
+# "Create compute node VM image":#create-image
+# "Update config.yml":#update-config
+# "Install arvados-dispatch-cloud":#install-packages
+# "Start the service":#start-service
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
+
+h2(#introduction). Introduction
+
+The cloud dispatch service is for running containers on cloud VMs. It works with Microsoft Azure and Amazon EC2; future versions will also support Google Compute Engine.
The cloud dispatch service can run on any node that can connect to the Arvados API service, the cloud provider's API, and the SSH service on cloud VMs. It is not resource-intensive, so you can run it on the API server node.
-*Only one dispatch process should be running at a time.* If you are migrating a system that currently runs @crunch-dispatch-slurm@, it is safest to remove the @crunch-dispatch-slurm@ service entirely before installing @arvados-dispatch-cloud@.
+h2(#create-image). Create compute node VM image
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl --now disable crunch-dispatch-slurm</span>
-~$ <span class="userinput">sudo apt-get remove crunch-dispatch-slurm</span>
-</code></pre>
-</notextile>
+Create a VM image following the steps "to set up a compute node":crunch2-slurm/install-compute-node.html
-h2. Create a dispatcher token
+On the compute VM image, add the API server's internal IP address to @/etc/hosts@, this will ensure that it contacts the API server on the private network and not through the public interface. For example:
-If you haven't already done so, create an Arvados superuser token to use as SystemRootToken in your cluster config file.
+<notextile><pre><code>10.20.30.40 <span class="userinput">ClusterID.example.com</span>
+</code></pre></notextile>
-{% include 'create_superuser_token' %}
+h2(#update-config). Update config.yml
-h2. Create a private key
+h3. Create a private key
Generate an SSH private key with no passphrase. Save it in the cluster configuration file (see @PrivateKey@ in the example below).
</code></pre>
</notextile>
-h2. Configure the dispatcher
+h3. Configure CloudVMs
-Add or update the following portions of your cluster configuration file, @/etc/arvados/config.yml@. Refer to "config.defaults.yml":{{site.baseurl}}/admin/config.html for information about additional configuration options.
+Add or update the following portions of your cluster configuration file, @config.yml@. Refer to "config.defaults.yml":{{site.baseurl}}/admin/config.html for information about additional configuration options.
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- ManagementToken: xyzzy
- SystemRootToken: <span class="userinput">zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</span>
- Services:
- Controller:
- ExternalURL: "https://<span class="userinput">uuid_prefix.arvadosapi.com</span>"
+<pre><code> Services:
DispatchCloud:
InternalURLs:
"http://localhost:9006": {}
</code></pre>
</notextile>
-Minimal configuration example for Amazon EC2:
+h4. Minimal configuration example for Amazon EC2
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Containers:
+<pre><code> Containers:
CloudVMs:
ImageID: ami-01234567890abcdef
Driver: ec2
DriverParameters:
- AccessKeyID: EALMF21BJC7MKNF9FVVR
- SecretAccessKey: yKJAPmoCQOMtYWzEUQ1tKTyrocTcbH60CRvGP3pM
+ AccessKeyID: XXXXXXXXXXXXXXXXXXXX
+ SecretAccessKey: YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
SecurityGroupIDs:
- sg-0123abcd
SubnetID: subnet-0123abcd
Region: us-east-1
EBSVolumeType: gp2
- AdminUsername: debian
+ AdminUsername: arvados
</code></pre>
</notextile>
-Minimal configuration example for Azure:
+h4. Minimal configuration example for Azure
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Containers:
+<pre><code> Containers:
CloudVMs:
ImageID: "https://zzzzzzzz.blob.core.windows.net/system/Microsoft.Compute/Images/images/zzzzz-compute-osDisk.55555555-5555-5555-5555-555555555555.vhd"
Driver: azure
DriverParameters:
SubscriptionID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
ClientID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
- ClientSecret: 2WyXt0XFbEtutnf2hp528t6Wk9S5bOHWkRaaWwavKQo=
+ ClientSecret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
TenantID: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
CloudEnvironment: AzurePublicCloud
ResourceGroup: zzzzz
</code></pre>
</notextile>
-h2. Test your configuration
+Get the @SubscriptionID@ and @TenantID@:
-First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
+<pre>
+$ az account list
+[
+ {
+ "cloudName": "AzureCloud",
+ "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXX",
+ "isDefault": true,
+ "name": "Your Subscription",
+ "state": "Enabled",
+ "tenantId": "YYYYYYYY-YYYY-YYYY-YYYYYYYY",
+ "user": {
+ "name": "you@example.com",
+ "type": "user"
+ }
+ }
+]
+</pre>
-Next, install the arvados-server package.
+You will need to create a "service principal" to use as a delegated authority for API access.
-On Red Hat-based systems:
+<notextile><pre><code>$ az ad app create --display-name "Arvados Dispatch Cloud (<span class="userinput">ClusterID</span>)" --homepage "https://arvados.org" --identifier-uris "https://<span class="userinput">ClusterID.example.com</span>" --end-date 2299-12-31 --password <span class="userinput">Your_Password</span>
+$ az ad sp create "<span class="userinput">appId</span>"
+(appId is part of the response of the previous command)
+$ az role assignment create --assignee "<span class="userinput">objectId</span>" --role Owner --scope /subscriptions/{subscriptionId}/
+(objectId is part of the response of the previous command)
+</code></pre></notextile>
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-server</span>
-</code></pre>
-</notextile>
+Now update your @config.yml@ file:
-On Debian-based systems:
+@ClientID@ is the 'appId' value.
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-server</span>
-</code></pre>
-</notextile>
+@ClientSecret@ is what was provided as <span class="userinput">Your_Password</span>.
+
+h3. Test your configuration
Run the @cloudtest@ tool to verify that your configuration works. This creates a new cloud VM, confirms that it boots correctly and accepts your configured SSH private key, and shuts it down.
Refer to the "cloudtest tool documentation":../admin/cloudtest.html for more information.
-h2. Install the dispatcher
+{% assign arvados_component = 'arvados-dispatch-cloud' %}
+
+{% include 'install_packages' %}
+
+{% include 'start_service' %}
+
+{% include 'restart_api' %}
+
+h2(#confirm-working). Confirm working installation
-On Red Hat-based systems:
+On the dispatch node, start monitoring the arvados-dispatch-cloud logs:
<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-dispatch-cloud</span>
-~$ <span class="userinput">sudo systemctl enable arvados-dispatch-cloud</span>
+<pre><code>~$ <span class="userinput">sudo journalctl -o cat -fu arvados-dispatch-cloud.service</span>
</code></pre>
</notextile>
-On Debian-based systems:
+"Make sure to install the arvados/jobs image.":install-jobs-image.html
+
+Submit a simple container request:
<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-dispatch-cloud</span>
+<pre><code>shell:~$ <span class="userinput">arv container_request create --container-request '{
+ "name": "test",
+ "state": "Committed",
+ "priority": 1,
+ "container_image": "arvados/jobs:latest",
+ "command": ["echo", "Hello, Crunch!"],
+ "output_path": "/out",
+ "mounts": {
+ "/out": {
+ "kind": "tmp",
+ "capacity": 1000
+ }
+ },
+ "runtime_constraints": {
+ "vcpus": 1,
+ "ram": 1048576
+ }
+}'</span>
</code></pre>
</notextile>
-{% include 'notebox_begin' %}
+This command should return a record with a @container_uuid@ field. Once @arvados-dispatch-cloud@ polls the API server for new containers to run, you should see it dispatch that same container.
-The arvados-dispatch-cloud package includes configuration files for systemd. If you're using a different init system, configure a service to start and stop an @arvados-dispatch-cloud@ process as desired.
+The @arvados-dispatch-cloud@ API a list of queued and running jobs. For example:
-{% include 'notebox_end' %}
+<notextile>
+<pre><code>~$ <span class="userinput">curl ...</span>
+</code></pre>
+</notextile>
-h2. Verify the dispatcher is running
+When the container finishes, the dispatcher will log it.
-Use your ManagementToken to test the dispatcher's metrics endpoint.
+After the container finishes, you can get the container record by UUID *from a shell server* to see its results:
<notextile>
-<pre><code>~$ <span class="userinput">token="xyzzy"</span>
-~$ <span class="userinput">curl -H "Authorization: Bearer $token" http://localhost:9006/metrics</span>
-# HELP arvados_dispatchcloud_containers_running Number of containers reported running by cloud VMs.
-# TYPE arvados_dispatchcloud_containers_running gauge
-arvados_dispatchcloud_containers_running 0
-[...]
+<pre><code>shell:~$ <span class="userinput">arv get <b>zzzzz-dz642-hdp2vpu9nq14tx0</b></span>
+{
+ ...
+ "exit_code":0,
+ "log":"a01df2f7e5bc1c2ad59c60a837e90dc6+166",
+ "output":"d41d8cd98f00b204e9800998ecf8427e+0",
+ "state":"Complete",
+ ...
+}
+</code></pre>
+</notextile>
+
+You can use standard Keep tools to view the container's output and logs from their corresponding fields. For example, to see the logs from the collection referenced in the @log@ field:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep ls <b>a01df2f7e5bc1c2ad59c60a837e90dc6+166</b></span>
+./crunch-run.txt
+./stderr.txt
+./stdout.txt
+~$ <span class="userinput">arv-get <b>a01df2f7e5bc1c2ad59c60a837e90dc6+166</b>/stdout.txt</span>
+2016-08-05T13:53:06.201011Z Hello, Crunch!
</code></pre>
</notextile>
+
+If the container does not dispatch successfully, refer to the @arvados-dispatch-cloud@ logs for information about why it failed.
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Set up Docker
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% include 'install_compute_docker' %}
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install arvados/jobs image
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+h2. Create a project for Docker images
+
+Here we create a default project for the standard Arvados Docker images, and give all users read access to it. The project is owned by the system user.
+
+<notextile>
+<pre><code>~$ <span class="userinput">uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)</span>
+~$ <span class="userinput">project_uuid=$(arv --format=uuid group create --group '{"owner_uuid":"'$uuid_prefix'-tpzed-000000000000000", "group_class":"project", "name":"Arvados Standard Docker Images"}')</span>
+~$ <span class="userinput">echo "Arvados project uuid is '$project_uuid'"</span>
+~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
+<span class="userinput">{
+ "tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
+ "head_uuid":"$project_uuid",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF</span>
+</code></pre></notextile>
+
+h2. Import the arvados/jobs docker image
+
+In order to start workflows from workbench, there needs to be Docker image @arvados/jobs@ tagged with the version of Arvados you are installing. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep. In this example @$project_uuid@ should be the UUID of the "Arvados Standard Docker Images" project.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv-keepdocker --pull arvados/jobs latest --project-uuid $project_uuid</span>
+</code></pre></notextile>
+
+If the image needs to be downloaded from Docker Hub, the command can take a few minutes to complete, depending on available network bandwidth.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Keep-balance deletes unreferenced and overreplicated blocks from Keep servers, makes additional copies of underreplicated blocks, and moves blocks into optimal locations as needed (e.g., after adding new servers). See "Balancing Keep servers":{{site.baseurl}}/admin/keep-balance.html for usage details.
-
-{% include 'notebox_begin' %}
+# "Introduction":#introduction
+# "Update config.yml":#update-config
+# "Install keep-balance package":#install-packages
+# "Start the service":#start-service
-If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, edit your keep-balance startup script to use the flags @-commit-pulls=false -commit-trash=false@.
-
-{% include 'notebox_end' %}
+h2(#introduction). Introduction
-h2. Install keep-balance
+Keep-balance deletes unreferenced and overreplicated blocks from Keep servers, makes additional copies of underreplicated blocks, and moves blocks into optimal locations as needed (e.g., after adding new servers). See "Balancing Keep servers":{{site.baseurl}}/admin/keep-balance.html for usage details.
Keep-balance can be installed anywhere with network access to Keep services. Typically it runs on the same host as keepproxy.
-*A cluster should have only one keep-balance process running at a time.*
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install keep-balance</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
+*A cluster should have only one instance of keep-balance running at a time.*
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install keep-balance</span>
-</code></pre>
-</notextile>
+{% include 'notebox_begin' %}
-Verify that @keep-balance@ is functional:
+If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, edit your keep-balance startup script to use the flags @-commit-pulls=false -commit-trash=false@.
-<notextile>
-<pre><code>~$ <span class="userinput">keep-balance -h</span>
-...
-Usage of ./keep-balance:
- -commit-pulls
- send pull requests (make more replicas of blocks that are underreplicated or are not in optimal rendezvous probe order)
- -commit-trash
- send trash requests (delete unreferenced old blocks, and excess replicas of overreplicated blocks)
-...
-</code></pre>
-</notextile>
+{% include 'notebox_end' %}
-h3. Update the cluster config
+h2(#update-config). Update the cluster config
-Edit the cluster config at @/etc/arvados/config.yml@ and set @Services.Keepbalance.InternalURLs@. Replace @uuid_prefix@ with your cluster id.
+Edit the cluster config at @config.yml@ and set @Services.Keepbalance.InternalURLs@. This port is only used to publish metrics.
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
+<pre><code> Services:
Keepbalance:
InternalURLs:
- "http://localhost:9005/": {}
- TLS:
- Insecure: false
+ "http://<span class="userinput">keep.ClusterID.example.com</span>:9005/": {}
</code></pre>
</notextile>
-Set @TLS.Insecure: true@ if your API server’s TLS certificate is not signed by a recognized CA.
-
-h3. Start the service (option 1: systemd)
-
-If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
-
-If your system uses systemd, the keep-balance service should already be set up. Start it and check its status:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart keep-balance</span>
-~$ <span class="userinput">sudo systemctl status keep-balance</span>
-● keep-balance.service - Arvados Keep Balance
- Loaded: loaded (/lib/systemd/system/keep-balance.service; enabled)
- Active: active (running) since Sat 2017-02-14 18:46:01 UTC; 3 days ago
- Docs: https://doc.arvados.org/
- Main PID: 541 (keep-balance)
- CGroup: /system.slice/keep-balance.service
- └─541 /usr/bin/keep-balance -commit-pulls -commit-trash
-
-Feb 14 18:46:01 zzzzz.arvadosapi.com keep-balance[541]: 2017/02/14 18:46:01 starting up: will scan every 10m0s and on SIGUSR1
-Feb 14 18:56:01 zzzzz.arvadosapi.com keep-balance[541]: 2017/02/14 18:56:01 Run: start
-Feb 14 18:56:01 zzzzz.arvadosapi.com keep-balance[541]: 2017/02/14 18:56:01 skipping zzzzz-bi6l4-rbtrws2jxul6i4t with service type "proxy"
-Feb 14 18:56:01 zzzzz.arvadosapi.com keep-balance[541]: 2017/02/14 18:56:01 clearing existing trash lists, in case the new rendezvous order differs from previous run
-</code></pre>
-</notextile>
-
-h3(#runit). Start the service (option 2: runit)
-
-Install runit to supervise the keep-balance daemon. {% include 'install_runit' %}
-
-Create a supervised service.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/keep-balance</span>
-~$ <span class="userinput">cd /etc/service/keep-balance</span>
-~$ <span class="userinput">sudo mkdir log log/main</span>
-~$ <span class="userinput">printf '#!/bin/sh\nexec keep-balance -commit-pulls -commit-trash 2>&1\n' | sudo tee run</span>
-~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
-~$ <span class="userinput">sudo chmod +x run log/run</span>
-~$ <span class="userinput">sudo sv exit .</span>
-~$ <span class="userinput">cd -</span>
-</code></pre>
-</notextile>
-
-Use @sv stat@ and check the log file to verify the service is running.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/keep-balance</span>
-run: /etc/service/keep-balance: (pid 12520) 2s; run: log: (pid 12519) 2s
-~$ <span class="userinput">tail /etc/service/keep-balance/log/main/current</span>
-2017/02/14 18:46:01 starting up: will scan every 10m0s and on SIGUSR1
-2017/02/14 18:56:01 Run: start
-2017/02/14 18:56:01 skipping zzzzz-bi6l4-rbtrws2jxul6i4t with service type "proxy"
-2017/02/14 18:56:01 clearing existing trash lists, in case the new rendezvous order differs from previous run
-</code></pre>
-</notextile>
-
-h2. Enable garbage collection
-
Ensure your cluster configuration has @Collections.BlobTrash: true@ (this is the default).
<notextile>
-<pre><code>~$ arvados-server config-dump | grep BlobTrash:
+<pre><code># arvados-server config-dump | grep BlobTrash:
BlobTrash: true
</code></pre>
</notextile>
If BlobTrash is false, unneeded blocks will be counted and logged by keep-balance, but they will not be deleted.
+
+{% assign arvados_component = 'keep-balance' %}
+
+{% include 'install_packages' %}
+
+{% include 'start_service' %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/arvados/arvados/services/keep-web for more detail.
+# "Introduction":#introduction
+# "Configure DNS":#introduction
+# "Configure anonymous user token.yml":#update-config
+# "Update nginx configuration":#update-nginx
+# "Install keep-web package":#install-packages
+# "Start the service":#start-service
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
-By convention, we use the following hostnames for the Keep-web service:
+h2(#introduction). Introduction
+
+The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+
+h2(#dns). Configure DNS
+
+It is important to properly configure the keep-web service to so it does not open up cross-site-scripting (XSS) attacks. A HTML file can be stored in collection. If an attacker causes a victim to visit that HTML file through Workbench, it will be rendered by the browser. If all collections are served at the same domain, the browser will consider collections as coming from the same origin and thus have access to the same browsing data (such as API token), enabling malicious Javascript in the HTML file to access Arvados as the victim.
+
+There are two approaches to mitigate this.
+
+# The service can tell the browser that all files should go to download instead of in-browser preview, except in situations where an attacker is unlikely to be able to gain access to anything they didn't already have access to.
+# Each each collection served by @keep-web@ is served on its own virtual host. This allows for file with executable content to be displayed in-browser securely. The virtual host embeds the collection uuid or portable data hash in the hostname. For example, a collection with uuid @xxxxx-4zz18-tci4vn4fa95w0zx@ could be served as @xxxxx-4zz18-tci4vn4fa95w0zx.collections.ClusterID.example.com@ . The portable data hash @dd755dbc8d49a67f4fe7dc843e4f10a6+54@ could be served at @dd755dbc8d49a67f4fe7dc843e4f10a6-54.collections.ClusterID.example.com@ . This requires "wildcard DNS record":https://en.wikipedia.org/wiki/Wildcard_DNS_record and "wildcard TLS certificate.":https://en.wikipedia.org/wiki/Wildcard_certificate
+
+h3. Collections download URL
+
+Downloads links will served from the the URL in @Services.WebDAVDownload.ExternalURL@ . The collection uuid or PDH is put in the URL path.
+
+If blank, serve links to WebDAV with @disposition=attachment@ query param. Unlike preview links, browsers do not render attachments, so there is no risk of XSS.
+
+If @WebDAVDownload@ is blank, and @WebDAV@ has a single origin (not wildcard, see below), then Workbench will show an error page
<notextile>
-<pre><code>download.<span class="userinput">uuid_prefix</span>.your.domain
-collections.<span class="userinput">uuid_prefix</span>.your.domain
-*.collections.<span class="userinput">uuid_prefix</span>.your.domain
+<pre><code> Services:
+ WebDAVDownload:
+ ExternalURL: https://<span class="userinput">download.ClusterID.example.com</span>
</code></pre>
</notextile>
-The above hostnames should resolve from anywhere on the internet.
+h3. Collections preview URL
+
+Collections will be served using the URL pattern in @Services.WebDAV.ExternalURL@ . If blank, use @Services.WebDAVDownload.ExternalURL@ instead, and disable inline preview. If both are empty, downloading collections from workbench will be impossible. When wildcard domains configured, credentials are still required to access non-public data.
+
+h4. In their own subdomain
+
+Collections can be served from their own subdomain:
-h2. Install Keep-web
+<notextile>
+<pre><code> Services:
+ WebDAV:
+ ExternalURL: https://<span class="userinput">*.collections.ClusterID.example.com/</span>
+</code></pre>
+</notextile>
-Typically Keep-web runs on the same host as Keepproxy.
+h4. Under the main domain
-On Debian-based systems:
+Alternately, they can go under the main domain by including @--@:
<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install keep-web</span>
+<pre><code> Services:
+ WebDAV:
+ ExternalURL: https://<span class="userinput">*--collections.ClusterID.example.com/</span>
</code></pre>
</notextile>
-On Red Hat-based systems:
+h4. From a single domain
+
+Serve preview links from a single domain, setting uuid or pdh in the path (similar to downloads). This configuration only allows previews of public data (data accessible by the anonymous user) and collection-sharing links (where the token is already embedded in the URL); it will ignore authorization headers, so a request for non-public data may return "404 Not Found" even if normally valid credentials were provided.
<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install keep-web</span>
+<pre><code> Services:
+ WebDAV:
+ ExternalURL: https://<span class="userinput">collections.ClusterID.example.com/</span>
</code></pre>
</notextile>
-Verify that @Keep-web@ is functional:
+Note the trailing slash.
+
+h2. Set InternalURLs
<notextile>
-<pre><code>~$ <span class="userinput">keep-web -h</span>
-Usage of keep-web:
- -config file
- Site configuration file (default may be overridden by setting an ARVADOS_CONFIG environment variable) (default "/etc/arvados/config.yml")
- -dump-config
- write current configuration to stdout and exit
-[...]
- -version
- print version information and exit.
+<pre><code> Services:
+ WebDAV:
+ InternalURLs:
+ http://"<span class="userinput">localhost:9002</span>": {}
</code></pre>
</notextile>
-h3. Set up a reverse proxy with TLS support
+h2(#update-config). Configure anonymous user token
-The Keep-web service will be accessible from anywhere on the internet, so we recommend using TLS for transport encryption.
+{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
+{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
+If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token. Use the following command on the <strong>API server</strong> to create an anonymous user token. {% include 'install_rails_command' %}
-This is best achieved by putting a reverse proxy with TLS support in front of Keep-web, running on port 443 and passing requests to Keep-web on port 9002 (or whatever port you chose in your run script).
+<notextile>
+<pre><code> Users:
+ AnonymousUserToken: <span class="userinput">"{{railsout}}"</span>
+</code></pre>
+</notextile>
+
+Set @Users.AnonymousUserToken: ""@ (empty string) or leave it out if you do not want to serve public data.
+
+h3. Update nginx configuration
-Note: A wildcard TLS certificate is required in order to support a full-featured secure Keep-web service. Without it, Keep-web can offer file downloads for all Keep data; however, in order to avoid cross-site scripting vulnerabilities, Keep-web refuses to serve private data as web content except when it is accessed using a "secret link" share. With a wildcard TLS certificate and DNS configured appropriately, all data can be served as web content.
+Put a reverse proxy with SSL support in front of keep-web. Keep-web itself runs on the port 25107 (or whatever is specified in @Services.Keepproxy.InternalURL@) the reverse proxy runs on port 443 and forwards requests to Keepproxy.
-For example, using Nginx:
+Use a text editor to create a new file @/etc/nginx/conf.d/keep-web.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
<notextile><pre>
upstream keep-web {
}
server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name download.<span class="userinput">uuid_prefix</span>.your.domain
- collections.<span class="userinput">uuid_prefix</span>.your.domain
- *.collections.<span class="userinput">uuid_prefix</span>.your.domain
- ~.*--collections.<span class="userinput">uuid_prefix</span>.your.domain;
+ listen *:443 ssl;
+ server_name <span class="userinput">download.ClusterID.example.com</span>
+ <span class="userinput">collections.ClusterID.example.com</span>
+ <span class="userinput">*.collections.ClusterID.example.com</span>
+ <span class="userinput">~.*--collections.ClusterID.example.com</span>;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
ssl on;
- ssl_certificate <span class="userinput"/>YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput"/>YOUR/PATH/TO/cert.key</span>;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
location / {
proxy_pass http://keep-web;
</pre></notextile>
{% include 'notebox_begin' %}
-If you restrict access to your Arvados services based on network topology -- for example, your proxy server is not reachable from the public internet -- additional proxy configuration might be needed to thwart cross-site scripting attacks that would circumvent your restrictions. Read the "'Intranet mode' section of the Keep-web documentation":https://godoc.org/github.com/arvados/arvados/services/keep-web#hdr-Intranet_mode now.
-{% include 'notebox_end' %}
-
-h3(#dns). Configure DNS
-
-Configure your DNS servers so the following names resolve to your Nginx proxy's public IP address.
-* @download.uuid_prefix.your.domain@
-* @collections.uuid_prefix.your.domain@
-* @*--collections.uuid_prefix.your.domain@, if you have a wildcard TLS certificate valid for @*.uuid_prefix.your.domain@ and your DNS server allows this without interfering with other DNS names.
-* @*.collections.uuid_prefix.your.domain@, if you have a wildcard TLS certificate valid for these names.
-
-If neither of the above wildcard options is feasible, you have two choices:
-# Serve web content at @collections.uuid_prefix.your.domain@, but only for unauthenticated requests (public data and collection sharing links). Authenticated requests will always result in file downloads, using the @download@ name. For example, the Workbench "preview" button and the "view entire log file" link will invoke file downloads instead of displaying content in the browser window.
-# In the special case where you know you are immune to XSS exploits, you can enable the "trust all content" mode in Keep-web and Workbench (setting @Collections.TrustAllContent: true@ on the config file). With this enabled, inline web content can be served from a single @collections@ host name; no wildcard DNS or certificate is needed. Do not do this without understanding the security implications described in the "Keep-web documentation":http://godoc.org/github.com/arvados/arvados/services/keep-web.
+If you restrict access to your Arvados services based on network topology -- for example, your proxy server is not reachable from the public internet -- additional proxy configuration might be needed to thwart cross-site scripting attacks that would circumvent your restrictions.
-h2. Configure Keep-web
+Normally, Keep-web accepts requests for multiple collections using the same host name, provided the client's credentials are not being used. This provides insufficient XSS protection in an installation where the "anonymously accessible" data is not truly public, but merely protected by network topology.
-{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
-{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
-If you intend to use Keep-web to serve public data to anonymous clients, configure it with an anonymous token. You can use the same one you used when you set up your Keepproxy server, or use the following command on the <strong>API server</strong> to create another. {% include 'install_rails_command' %}
-
-Set the cluster config file like the following:
-
-<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
- Controller:
- ExternalURL: "https://<span class="userinput">uuid_prefix</span>.your.domain"
- WebDAV:
- InternalURLs:
- "http://keep_web_hostname_goes_here:9002/": {}
- ExternalURL: "https://collections.<span class="userinput">uuid_prefix</span>.your.domain"
- WebDAVDownload:
- InternalURLs:
- "http://keep_web_hostname_goes_here:9002/": {}
- ExternalURL: "https://download.<span class="userinput">uuid_prefix</span>.your.domain"
- Users:
- AnonymousUserToken: "{{railsout}}"
- Collections:
- TrustAllContent: false
- TLS:
- Insecure: false
-</code></pre>
-</notextile>
-
-Set @Users.AnonymousUserToken: ""@ (empty string) if you do not want to serve public data.
-
-Set @TLS.Insecure: true@ if your API server's TLS certificate is not signed by a recognized CA.
-
-Workbench has features like "download file from collection" and "show image" which work better if the content is served by Keep-web rather than Workbench itself. We recommend using the two different hostnames ("download" and "collections" above) for file downloads and inline content respectively.
-
-The following entry on your cluster configuration file (@/etc/arvados/config.yml@) details the URL that will be used for file downloads.
-
-<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
- WebDAVDownload:
- ExternalURL: "https://download.<span class="userinput">uuid_prefix</span>.your.domain"
-</code></pre>
-</notextile>
-
-Additionally, one of the following entries on your cluster configuration file (depending on your DNS setup) tells Workbench which URL will be used to serve user content that can be displayed in the browser, like image previews and static HTML pages.
-
-<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
- WebDAV:
- ExternalURL: "https://*--collections.<span class="userinput">uuid_prefix</span>.your.domain"
- ExternalURL: "https://*.collections.<span class="userinput">uuid_prefix</span>.your.domain"
- ExternalURL: "https://collections.<span class="userinput">uuid_prefix</span>.your.domain"
-</code></pre>
-</notextile>
+In such cases -- for example, a site which is not reachable from the internet, where some data is world-readable from Arvados's perspective but is intended to be available only to users within the local network -- the downstream proxy should configured to return 401 for all paths beginning with "/c="
+{% include 'notebox_end' %}
-h2. Run Keep-web
+{% assign arvados_component = 'keep-web' %}
-h3. Start the service (option 1: systemd)
+{% include 'install_packages' %}
-If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+{% include 'start_service' %}
-If your system uses systemd, the keep-web service should already be set up. Start it and check its status:
+{% include 'restart_api' %}
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart keep-web</span>
-~$ <span class="userinput">sudo systemctl status keep-web</span>
-● keep-web.service - Arvados Keep web gateway
- Loaded: loaded (/lib/systemd/system/keep-web.service; enabled)
- Active: active (running) since Sat 2019-08-10 10:33:21 UTC; 3 days ago
- Docs: https://doc.arvados.org/
- Main PID: 4242 (keep-web)
- CGroup: /system.slice/keep-web.service
- └─4242 /usr/bin/keep-web
-[...]
-</code></pre>
-</notextile>
+h2(#confirm-working). Confirm working installation
-h3(#runit). Start the service (option 2: runit)
+<notextile><code><pre>
+$ curl -H "Authorization: Bearer $system_root_token" https://<span class="userinput">download.ClusterID.example.com</span>/c=59389a8f9ee9d399be35462a0f92541c-53/_/hello.txt
+</code></pre></notextile>
-Install runit to supervise the Keep-web daemon. {% include 'install_runit' %}
+If wildcard collections domains are configured:
-The basic command to start Keep-web in the service run script is:
+<notextile><code><pre>
+$ curl -H "Authorization: Bearer $system_root_token" https://<span class="userinput">59389a8f9ee9d399be35462a0f92541c-53.collections.ClusterID.example.com</span>/hello.txt
+</code></pre></notextile>
-<notextile>
-<pre><code>exec keep-web
-</code></pre>
-</notextile>
+If using a single collections preview domain:
+<notextile><code><pre>
+$ curl https://<span class="userinput">collections.ClusterID.example.com</span>/c=59389a8f9ee9d399be35462a0f92541c-53/t=$system_root_token/_/hello.txt
+</code></pre></notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+# "Introduction":#introduction
+# "Update config.yml":#update-config
+# "Update nginx configuration":#update-nginx
+# "Install keepproxy package":#install-packages
+# "Start the service":#start-service
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
+
+h2(#introduction). Introduction
+
The Keepproxy server is a gateway into your Keep storage. Unlike the Keepstore servers, which are only accessible on the local LAN, Keepproxy is suitable for clients located elsewhere on the internet. Specifically, in contrast to Keepstore:
-* A client writing through Keepproxy generates less network traffic: the client sends a single copy of a data block, and Keepproxy sends copies to the appropriate Keepstore servers.
+* A client writing through Keepproxy sends a single copy of a data block, and Keepproxy distributes copies to the appropriate Keepstore servers.
* A client can write through Keepproxy without precomputing content hashes. Notably, the browser-based upload feature in Workbench requires Keepproxy.
* Keepproxy checks API token validity before processing requests. (Clients that can connect directly to Keepstore can use it as scratch space even without a valid API token.)
<div class="offset1">
table(table table-bordered table-condensed).
-|_Hostname_|
-|keep.@uuid_prefix@.your.domain|
+|_. Hostname|
+|@keep.ClusterID.example.com@|
</div>
This hostname should resolve from anywhere on the internet.
-h2. Install Keepproxy
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install keepproxy</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install keepproxy</span>
-</code></pre>
-</notextile>
-
-Verify that Keepproxy is functional:
-
-<notextile>
-<pre><code>~$ <span class="userinput">keepproxy -h</span>
-Usage of keepproxy:
- -config file
- Site configuration file (default may be overridden by setting an ARVADOS_CONFIG environment variable) (default "/etc/arvados/config.yml")
- -dump-config
- write current configuration to stdout and exit
-[...]
- -version
- print version information and exit.
-</code></pre>
-</notextile>
+h2(#update-config). Update config.yml
-h3. Update the cluster config
-
-Edit the cluster config at @/etc/arvados/config.yml@ and set @Services.Keepproxy.ExternalURL@ and @Services.Keepproxy.InternalURLs@. Replace @uuid_prefix@ with your cluster id.
+Edit the cluster config at @config.yml@ and set @Services.Keepproxy.ExternalURL@ and @Services.Keepproxy.InternalURLs@.
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- Services:
+<pre><code> Services:
Keepproxy:
- ExternalURL: <span class="userinput">https://keep.uuid_prefix.your.domain</span>
+ ExternalURL: <span class="userinput">https://keep.ClusterID.example.com</span>
InternalURLs:
- <span class="userinput">"http://localhost:25107": {}</span>
+ <span class="userinput">"http://localhost:25107": {}</span>
</span></code></pre>
</notextile>
-h3. Set up a reverse proxy with SSL support
+h2(#update-nginx). Update Nginx configuration
-Because the Keepproxy is intended for access from anywhere on the internet, it is recommended to use SSL for transport encryption.
+Put a reverse proxy with SSL support in front of Keepproxy. Keepproxy itself runs on the port 25107 (or whatever is specified in @Services.Keepproxy.InternalURL@) the reverse proxy runs on port 443 and forwards requests to Keepproxy.
-This is best achieved by putting a reverse proxy with SSL support in front of Keepproxy. Keepproxy itself runs on port 25107 by default; your reverse proxy can run on port 443 and pass requests to Keepproxy on port 25107.
+Use a text editor to create a new file @/etc/nginx/conf.d/keepproxy.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
-<notextile><pre>
-upstream keepproxy {
+<notextile><pre><code>upstream keepproxy {
server 127.0.0.1:<span class="userinput">25107</span>;
}
server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name keep.<span class="userinput">uuid_prefix</span>.your.domain;
+ listen *:443 ssl;
+ server_name <span class="userinput">keep.ClusterID.example.com</span>;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
proxy_http_version 1.1;
proxy_request_buffering off;
- ssl on;
- ssl_certificate /etc/nginx/keep.<span class="userinput">uuid_prefix</span>.your.domain-ssl.crt;
- ssl_certificate_key /etc/nginx/keep.<span class="userinput">uuid_prefix</span>.your.domain-ssl.key;
+ ssl on;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
# Clients need to be able to upload blocks of data up to 64MiB in size.
client_max_body_size 64m;
proxy_pass http://keepproxy;
}
}
-</pre></notextile>
+</code></pre></notextile>
Note: if the Web uploader is failing to upload data and there are no logs from keepproxy, be sure to check the nginx proxy logs. In addition to "GET" and "PUT", The nginx proxy must pass "OPTIONS" requests to keepproxy, which should respond with appropriate Cross-origin resource sharing headers. If the CORS headers are not present, brower security policy will cause the upload request to silently fail. The CORS headers are generated by keepproxy and should not be set in nginx.
-h3. Tell the API server about the Keepproxy server
+{% assign arvados_component = 'keepproxy' %}
-The API server needs to be informed about the presence of your Keepproxy server.
+{% include 'install_packages' %}
-First, if you don't already have an admin token, create a superuser token.
+{% include 'start_service' %}
-{% include 'create_superuser_token' %}
+{% include 'restart_api' %}
-Configure your environment to run @arv@ using the output of create_superuser_token.rb:
+h2(#confirm-working). Confirm working installation
-<pre>
-export ARVADOS_API_HOST=zzzzz.example.com
-export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</pre>
+Log into a host that is on a network external to your private Arvados network. The host should be able to contact your keepproxy server (eg @keep.ClusterID.example.com@), but not your keepstore servers (eg keep[0-9].ClusterID.example.com).
-<notextile>
-<pre><code>~$ <span class="userinput">uuid_prefix=`arv --format=uuid user current | cut -d- -f1`</span>
-~$ <span class="userinput">echo "Site prefix is '$uuid_prefix'"</span>
-~$ <span class="userinput">read -rd $'\000' keepservice <<EOF; arv keep_service create --keep-service "$keepservice"</span>
-<span class="userinput">{
- "service_host":"<strong>keep.$uuid_prefix.your.domain</strong>",
- "service_port":443,
- "service_ssl_flag":true,
- "service_type":"proxy"
-}
-EOF</span>
-</code></pre></notextile>
+@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ must be set in the environment.
-h2. Run Keepproxy
+@ARVADOS_API_HOST@ should be the hostname of the API server.
-h3. Start the service (option 1: systemd)
+@ARVADOS_API_TOKEN@ should be the system root token.
-If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+Install the "Command line SDK":{{site.baseurl}}/sdk/cli/install.html
-If your system uses systemd, the keepproxy service should already be set up. Start it and check its status:
+Check that the keepproxy server is in the @keep_service@ "accessible" list:
<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart keepproxy</span>
-~$ <span class="userinput">sudo systemctl status keepproxy</span>
-● keepproxy.service - Arvados Keep Proxy
- Loaded: loaded (/lib/systemd/system/keepproxy.service; enabled)
- Active: active (running) since Tue 2019-07-23 09:33:47 EDT; 3 weeks 1 days ago
- Docs: https://doc.arvados.org/
- Main PID: 1150 (Keepproxy)
- CGroup: /system.slice/keepproxy.service
- └─1150 /usr/bin/keepproxy
+<pre><code>
+$ <span class="userinput">arv keep_service accessible</span>
[...]
</code></pre>
</notextile>
-h3(#runit). Start the service (option 2: runit)
-
-Install runit to supervise the Keep-web daemon. {% include 'install_runit' %}
-
-h3. Testing keepproxy
-
-Log into a host that is on an external network from your private Arvados network. The host should be able to contact your keepproxy server (eg keep.$uuid_prefix.arvadosapi.com), but not your keepstore servers (eg keep[0-9].$uuid_prefix.arvadosapi.com).
+If keepstore does not show up in the "accessible" list, and you are accessing it from within the private network, check that you have "properly configured the @geo@ block for the API server":install-api-server.html#update-nginx .
Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
-@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ must be set in the environment.
+You should now be able to use @arv-put@ to upload collections and @arv-get@ to fetch collections. Be sure to execute this from _outside_ the cluster's private network.
-You should now be able to use @arv-put@ to upload collections and @arv-get@ to fetch collections, for an example see "Testing keep.":install-keepstore.html#testing on the keepstore install page.
+{% include 'arv_put_example' %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+# "Introduction":#introduction
+# "Update config.yml":#update-config
+# "Install keepstore package":#install-packages
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
+# "Note on storage management":#note
+
+h2. Introduction
+
Keepstore provides access to underlying storage for reading and writing content-addressed blocks, with enforcement of Arvados permissions. Keepstore supports a variety of cloud object storage and POSIX filesystems for its backing store.
h3. Plan your storage layout
<div class="offset1">
table(table table-bordered table-condensed).
-|_Hostname_|
-|keep0.@uuid_prefix@.your.domain|
-|keep1.@uuid_prefix@.your.domain|
+|_. Hostname|
+|@keep0.ClusterID.example.com@|
+|@keep1.ClusterID.example.com@|
</div>
Keepstore servers should not be directly accessible from the Internet (they are accessed via "keepproxy":install-keepproxy.html), so the hostnames only need to resolve on the private network.
-h2. Install Keepstore
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install keepstore</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install keepstore</span>
-</code></pre>
-</notextile>
-
-Verify that Keepstore is functional:
-
-<notextile>
-<pre><code>~$ <span class="userinput">keepstore --version</span>
-</code></pre>
-</notextile>
+h2(#update-config). Update cluster config
-h3. Create a superuser token
+h3. Configure storage volumes
-If you haven't already done so, create a superuser token.
+Fill in the @Volumes@ section of @config.yml@ for each storage volume. Available storage volume types include POSIX filesystems and cloud object storage. It is possible to have different volume types in the same cluster.
-{% include 'create_superuser_token' %}
+* To use a POSIX filesystem, including both local filesystems (ext4, xfs) and network file system such as GPFS or Lustre, follow the setup instructions on "Filesystem storage":configure-fs-storage.html
+* If you are using S3-compatible object storage (including Amazon S3, Google Cloud Storage, and Ceph RADOS), follow the setup instructions on "S3 Object Storage":configure-s3-object-storage.html
+* If you are using Azure Blob Storage, follow the setup instructions on "Azure Blob Storage":configure-azure-blob-storage.html
-h3. Update cluster config file
+h3. List services
-Add or update the following sections of @/etc/arvados/config.yml@ as needed. Refer to the examples and comments in the "default config.yml file":{{site.baseurl}}/admin/config.html for more information.
+Add each keepstore server to the @Services.Keepstore@ section of @/etc/arvados/config.yml@ .
<notextile>
-<pre><code>Clusters:
- <span class="userinput">uuid_prefix</span>:
- SystemRootToken: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
- Services:
+<pre><code> Services:
Keepstore:
+ # No ExternalURL because they are only accessed by the internal subnet.
InternalURLs:
- "http://<span class="userinput">keep0.uuid_prefix.example.com</span>:25107/": {}
- API:
- MaxKeepBlobBuffers: 128
+ "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
+ "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {}
+ # and so forth
</code></pre>
</notextile>
-h3. Note on storage management
-
-On its own, a keepstore server never deletes data. Instead, the keep-balance service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash. Please see the "Balancing Keep servers":{{site.baseurl}}/admin/keep-balance.html for more details.
-
-h3. Configure storage volumes
+{% assign arvados_component = 'keepstore' %}
-Available storage volume types include POSIX filesystems and cloud object storage.
+{% include 'install_packages' %}
-* To use a POSIX filesystem, including both local filesystems (ext4, xfs) and network file system such as GPFS or Lustre, follow the setup instructions on "Filesystem storage":configure-fs-storage.html
-* If you are using S3-compatible object storage (including Amazon S3, Google Cloud Storage, and Ceph RADOS), follow the setup instructions on "S3 Object Storage":configure-s3-object-storage.html
-* If you are using Azure Blob Storage, follow the setup instructions on "Azure Blob Storage":configure-azure-blob-storage.html
+{% include 'start_service' %}
-h2. Run keepstore as a supervised service
+{% include 'restart_api' %}
-h3. Start the service (option 1: systemd)
+h2(#confirm-working). Confirm working installation
-If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+Log into a host that is on your private Arvados network. The host should be able to contact your your keepstore servers (eg keep[0-9].ClusterID.example.com).
-If your system uses systemd, the keepstore service should already be set up. Restart it to read the updated configuration, and check its status:
+@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ must be set in the environment.
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart keepstore</span>
-~$ <span class="userinput">sudo systemctl status keepstore</span>
-● keepstore.service - Arvados Keep Storage Daemon
- Loaded: loaded (/etc/systemd/system/keepstore.service; enabled; vendor preset: enabled)
- Active: active (running) since Tue 2019-09-10 14:16:29 UTC; 1s ago
- Docs: https://doc.arvados.org/
- Main PID: 25465 (keepstore)
- Tasks: 9 (limit: 4915)
- CGroup: /system.slice/keepstore.service
- └─25465 /usr/bin/keepstore
-[...]
-</code></pre>
-</notextile>
+@ARVADOS_API_HOST@ should be the hostname of the API server.
-h3(#runit). Start the service (option 2: runit)
+@ARVADOS_API_TOKEN@ should be the system root token.
-Install runit to supervise the keepstore daemon. {% include 'install_runit' %}
+Install the "Command line SDK":{{site.baseurl}}/sdk/cli/install.html
-Install this script as the run script @/etc/sv/keepstore/run@ for the keepstore service:
+Check that the keepstore server is in the @keep_service@ "accessible" list:
<notextile>
-<pre><code>#!/bin/sh
-
-exec 2>&1
-GOGC=10 exec keepstore
+<pre><code>
+$ <span class="userinput">arv keep_service accessible</span>
+[...]
</code></pre>
</notextile>
-h2. Set up additional servers
-
-Repeat the above sections to prepare volumes and bring up supervised services on each Keepstore server you are setting up.
+If keepstore does not show up in the "accessible" list, and you are accessing it from within the private network, check that you have "properly configured the @geo@ block for the API server":install-api-server.html#update-nginx .
-h2. Restart the API server and controller
+Next, install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
-After adding all of your keepstore servers to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
+You should now be able to use @arv-put@ to upload collections and @arv-get@ to fetch collections. Be sure to execute this from _inside_ the cluster's private network. You will be able to access keep from _outside_ the private network after setting up "keepproxy":install-keepproxy.html .
-<pre>
-sudo systemctl restart nginx arvados-controller
-</pre>
+{% include 'arv_put_example' %}
-h2(#testing). Testing keep
-
-Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
-
-@ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ must be set in the environment.
+h2(#note). Note on storage management
-You should now be able to use @arv-put@ to upload collections and @arv-get@ to fetch collections:
-
-<pre>
-$ echo "hello world!" > hello.txt
-
-$ arv-put --portable-data-hash hello.txt
-2018-07-12 13:35:25 arvados.arv_put[28702] INFO: Creating new cache file at /home/example/.cache/arvados/arv-put/1571ec0adb397c6a18d5c74cc95b3a2a
-0M / 0M 100.0% 2018-07-12 13:35:27 arvados.arv_put[28702] INFO:
-
-2018-07-12 13:35:27 arvados.arv_put[28702] INFO: Collection saved as 'Saved at 2018-07-12 17:35:25 UTC by example@example'
-59389a8f9ee9d399be35462a0f92541c+53
-
-$ arv-get 59389a8f9ee9d399be35462a0f92541c+53/hello.txt
-hello world!
-</pre>
+On its own, a keepstore server never deletes data. Instead, the keep-balance service determines which blocks are candidates for deletion and instructs the keepstore to move those blocks to the trash. Please see the "Balancing Keep servers":{{site.baseurl}}/admin/keep-balance.html for more details.
---
layout: default
navsection: installguide
-title: Prerequisites
+title: Planning and prerequisites
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Supported Cloud and HPC platforms
+Before attempting installation, you should begin by reviewing supported platforms, choosing backends for identity, storage, and scheduling, and decide how you will distribute Arvados services onto machines. You should also choose an Arvados Cluster ID, choose your hostnames, and aquire TLS certificates. It may be helpful to make notes as you go along.
-Arvados can run in a variety of configurations. For compute scheduling, Arvados supports HPC clusters using @slurm@, and supports elastic cloud computing on AWS, Google and Azure. For storage, Arvados can store blocks on regular file systems such as ext4 or xfs, on network file systems such as GPFS, or object storage such as Azure blob storage, Amazon S3, and other object storage that supports the S3 API including Google Cloud Storage and Ceph.
+The Arvados storage subsystem is called "keep". The compute subsystem is called "crunch".
-h2. Hardware (or virtual machines)
+# "Supported GNU/Linux distributions":#supportedlinux
+# "Choosing which components to install":#components
+# "Identity provider":#identity
+# "Storage backend (Keep)":#storage
+# "Container compute scheduler (Crunch)":#scheduler
+# "Hardware or virtual machines":#machines
+# "Arvados Cluster ID":#clusterid
+# "DNS and TLS":#dnstls
-This guide assumes you have seven systems available in the same network subnet:
-
-<div class="offset1">
-table(table table-bordered table-condensed).
-|_. Function|_. Number of nodes|
-|Arvados API, Crunch dispatcher, Git, Websockets and Workbench|1|
-|Arvados Compute node|1|
-|Arvados Keepproxy and Keep-web server|1|
-|Arvados Keepstore servers|2|
-|Arvados Shell server|1|
-|Arvados SSO server|1|
-</div>
-
-The number of Keepstore, shell and compute nodes listed above is a minimum. In a real production installation, you will likely run many more of each of those types of nodes. In such a scenario, you would probably also want to dedicate a node to the Workbench server and Crunch dispatcher, respectively. For performance reasons, you may want to run the database server on a separate node as well.
-
-h2. Supported GNU/Linux distributions
+h2(#supportedlinux). Supported GNU/Linux distributions
table(table table-bordered table-condensed).
|_. Distribution|_. State|_. Last supported version|
Arvados packages are published for current Debian releases (until the EOL date), current Ubuntu LTS releases (until the end of standard support), and the latest version of CentOS.
-h2(#repos). Arvados package repositories
-
-On any host where you install Arvados software, you'll need to set up an Arvados package repository. They're available for several popular distributions.
-
-h3. CentOS
+h2(#components). Choosing which components to install
-Packages are available for CentOS 7. To install them with yum, save this configuration block in @/etc/yum.repos.d/arvados.repo@:
+Arvados consists of many components, some of which may be omitted (at the cost of reduced functionality.) It may also be helpful to review the "Arvados Architecture":{{site.baseurl}}/architecture to understand how these components interact.
-<notextile>
-<pre><code>[arvados]
-name=Arvados
-baseurl=http://rpm.arvados.org/CentOS/$releasever/os/$basearch/
-gpgcheck=1
-gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-curoverse
-</code></pre>
-</notextile>
+table(table table-bordered table-condensed).
+|\3=. *Core*|
+|"Postgres database":install-postgresql.html |Stores data for the API server.|Required.|
+|"API server":install-api-server.html |Core Arvados logic for managing users, groups, collections, containers, and enforcing permissions.|Required.|
+|\3=. *Keep (storage)*|
+|"Keepstore":install-keepstore.html |Stores content-addressed blocks in a variety of backends (local filesystem, cloud object storage).|Required.|
+|"Keepproxy":install-keepproxy.html |Gateway service to access keep servers from external networks.|Required to be able to use arv-put, arv-get, or arv-mount outside the private Arvados network.|
+|"Keep-web":install-keep-web.html |Gateway service providing read/write HTTP and WebDAV support on top of Keep.|Required to access files from Workbench.|
+|"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
+|\3=. *User interface*|
+|"Single Sign On server":install-sso.html |Web based login to Workbench.|Depends on identity provider. Not required for Google. Required for LDAP or standalone database.|
+|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional. Depends on API server, SSO server, keep-web, websockets server.|
+|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional. Depends on git server (arv-git-httpd).|
+|\3=. *Additional services*|
+|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
+|"Shell server":install-shell-server.html |Synchronize (create/delete/configure) Unix shell accounts with Arvados users.|Optional.|
+|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional, but required by Workflow Composer.|
+|\3=. *Crunch (running containers)*|
+|"crunch-dispatch-slurm":crunch2-slurm/install-prerequisites.html |Run analysis workflows using Docker containers distributed across a SLURM cluster.|Optional if you wish to use Arvados for data management only.|
+|"Node Manager":install-nodemanager.html, "arvados-dispatch-cloud":install-dispatch-cloud.html |Allocate and free cloud VM instances on demand based on workload.|Optional, not needed for a static SLURM cluster (such as on-premise HPC).|
-{% include 'install_redhat_key' %}
+h2(#identity). Identity provider
-h3. Debian and Ubuntu
+Choose which backend you will use to authenticate users.
-Packages are available for Debian 9 ("stretch"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
+* Google login to authenticate users with a Google account. Note: if you only use this identity provider, login can be handled by @arvados-controller@ (recommended), and you do not need to install the Arvados Single Sign-On server (SSO).
+* LDAP login to authenticate users using the LDAP protocol, supported by many services such as OpenLDAP and Active Directory. Supports username/password authentication.
+* Standalone SSO server user database. Supports username/password authentication. Supports new user sign-up.
-First, register the Curoverse signing key in apt's database:
+h2(#storage). Storage backend
-{% include 'install_debian_key' %}
+Choose which backend you will use for storing and retrieving content-addressed Keep blocks.
-Configure apt to retrieve packages from the Arvados package repository. This command depends on your OS vendor and version:
+* File systems storage, such as ext4 or xfs, or network file systems such as GPFS or Lustre
+* Amazon S3, or other object storage that supports the S3 API including Google Cloud Storage and Ceph.
+* Azure blob storage
-table(table table-bordered table-condensed).
-|_. OS version|_. Command|
-|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+You should also determine the desired replication factor for your data. A replication factor of 1 means only a single copy of a given data block is kept. With a conventional file system backend and a replication factor of 1, a hard drive failure is likely to lose data. For this reason the default replication factor is 2 (two copies are kept).
-{% include 'notebox_begin' %}
+A backend may have its own replication factor (such as durability guarantees of cloud buckets) and Arvados will take this into account when writing a new data block.
-fn1. Arvados packages for Ubuntu may depend on third-party packages in Ubuntu's "universe" repository. If you're installing on Ubuntu, make sure you have the universe sources uncommented in @/etc/apt/sources.list@.
+h2(#scheduler). Container compute scheduler
-{% include 'notebox_end' %}
+Choose which backend you will use to schedule computation.
-Retrieve the package list:
+* On AWS EC2 and Azure, you probably want to use @arvados-dispatch-cloud@ to manage the full lifecycle of cloud compute nodes: starting up nodes sized to the container request, executing containers on those nodes, and shutting nodes down when no longer needed.
+* For on-premise HPC clusters using "slurm":https://slurm.schedmd.com/ use @crunch-dispatch-slurm@ to execute containers with slurm job submissions.
+* For single node demos, use @crunch-dispatch-local@ to execute containers directly.
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get update</span>
-</code></pre>
-</notextile>
+h2(#machines). Hardware (or virtual machines)
-h2. A unique identifier
+Choose how to allocate Arvados services to machines. We recommend that each machine start with a clean installation of a supported GNU/Linux distribution.
-Each Arvados installation should have a globally unique identifier, which is a unique 5-character lowercase alphanumeric string. For testing purposes, here is one way to make a random 5-character string:
+For a production installation, this is a reasonable starting point:
-<notextile>
-<pre><code>~$ <span class="userinput">tr -dc 0-9a-z </dev/urandom | head -c5; echo</span>
-</code></pre>
-</notextile>
-
-You may also use a different method to pick the unique identifier. The unique identifier will be part of the hostname of the services in your Arvados cluster. The rest of this documentation will refer to it as your @uuid_prefix@.
-
-
-h2. SSL certificates
+<div class="offset1">
+table(table table-bordered table-condensed).
+|_. Function|_. Number of nodes|_. Recommended specs|
+|Postgres database, Arvados API server, Arvados controller, Git, Websockets, Container dispatcher|1|16+ GiB RAM, 4+ cores, fast disk for database|
+|Single Sign-On (SSO) server ^1^|1|2 GiB RAM|
+|Workbench, Keepproxy, Keep-web, Keep-balance|1|8 GiB RAM, 2+ cores|
+|Keepstore servers ^2^|2+|4 GiB RAM|
+|Compute worker nodes ^2^|0+ |Depends on workload; scaled dynamically in the cloud|
+|User shell nodes ^3^|0+|Depends on workload|
+</div>
-There are six public-facing services that require an SSL certificate. If you do not have official SSL certificates, you can use self-signed certificates.
+^1^ May be omitted when using Google login support in @arvados-controller@
+^2^ Should be scaled up as needed
+^3^ Refers to shell nodes managed by Arvados, that provide ssh access for users to interact with Arvados at the command line. Optional.
{% include 'notebox_begin' %}
+For a small demo installation, it is possible to run all the Arvados services on a single node. Special considerations for single-node installs will be noted in boxes like this.
+{% include 'notebox_end' %}
-Most Arvados clients and services will accept self-signed certificates when the @ARVADOS_API_HOST_INSECURE@ environment variable is set to @true@. However, web browsers generally do not make it easy for users to accept self-signed certificates from Web sites.
+h2(#clusterid). Arvados Cluster ID
-Users who log in through Workbench will visit at least three sites: the SSO server, the API server, and Workbench itself. When a browser visits each of these sites, it will warn the user if the site uses a self-signed certificate, and the user must accept it before continuing. This procedure usually only needs to be done once in a browser.
+Each Arvados installation should have a cluster identifier, which is a unique 5-character lowercase alphanumeric string. Here is one way to make a random 5-character string:
-After that's done, Workbench includes JavaScript clients for other Arvados services. Users are usually not warned if these client connections are refused because the server uses a self-signed certificate, and it is especially difficult to accept those cerficiates:
+<notextile>
+<pre><code>~$ <span class="userinput">tr -dc 0-9a-z </dev/urandom | head -c5; echo</span>
+</code></pre>
+</notextile>
-* JavaScript connects to the Websockets server to provide incremental page updates and view logs from running jobs.
-* JavaScript connects to the API and Keepproxy servers to upload local files to collections.
-* JavaScript connects to the Keep-web server to download log files.
+You may also use a different method to pick the cluster identifier. The cluster identifier will be part of the hostname of the services in your Arvados cluster. The rest of this documentation will refer to it as your @ClusterID@. Whenever @ClusterID@ appears in a configuration example, replace it with your five-character cluster identifier.
-In sum, Workbench will be much less pleasant to use in a cluster that uses self-signed certificates. You should avoid using self-signed certificates unless you plan to deploy a cluster without Workbench; you are deploying only to evaluate Arvados as an individual system administrator; or you can push configuration to users' browsers to trust your self-signed certificates.
+h2(#dnstls). DNS entries and TLS certificates
-{% include 'notebox_end' %}
+The following services are normally public-facing and require DNS entries and corresponding TLS certificates. Get certificates from your preferred TLS certificate provider. We recommend using "Let's Encrypt":https://letsencrypt.org/. You can run several services on same node, but each distinct hostname requires its own TLS certificate.
-By convention, we use the following hostname pattern:
+This guide uses the following hostname conventions. A later part of this guide will describe how to set up Nginx virtual hosts.
<div class="offset1">
table(table table-bordered table-condensed).
|_. Function|_. Hostname|
-|Arvados API|@uuid_prefix@.your.domain|
-|Arvados Git server|git.@uuid_prefix@.your.domain|
-|Arvados Keepproxy server|keep.@uuid_prefix@.your.domain|
-|Arvados Keep-web server|download.@uuid_prefix@.your.domain
+|Arvados API|@ClusterID.example.com@|
+|Arvados Git server|git.@ClusterID.example.com@|
+|Arvados Websockets endpoint|ws.@ClusterID.example.com@|
+|Arvados SSO Server|@auth.example.com@|
+|Arvados Workbench|workbench.@ClusterID.example.com@|
+|Arvados Workbench 2|workbench2.@ClusterID.example.com@|
+|Arvados Keepproxy server|keep.@ClusterID.example.com@|
+|Arvados Keep-web server|download.@ClusterID.example.com@
_and_
-*.collections.@uuid_prefix@.your.domain or
-*<notextile>--</notextile>collections.@uuid_prefix@.your.domain or
-collections.@uuid_prefix@.your.domain (see the "keep-web install docs":install-keep-web.html)|
-|Arvados SSO Server|auth.your.domain|
-|Arvados Websockets endpoint|ws.@uuid_prefix@.your.domain|
-|Arvados Workbench|workbench.@uuid_prefix@.your.domain|
+*.collections.@ClusterID.example.com@ or
+*<notextile>--</notextile>collections.@ClusterID.example.com@ or
+collections.@ClusterID.example.com@ (see the "keep-web install docs":install-keep-web.html)|
</div>
+
+{% include 'notebox_begin' %}
+It is also possible to create your own certificate authority, issue server certificates, and install a custom root certificate in the browser. This is out of scope for this guide.
+{% include 'notebox_end' %}
---
layout: default
navsection: installguide
-title: Set up PostgreSQL databases
+title: Install PostgreSQL 9.4+
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Two Arvados Rails servers store data in a PostgreSQL database: the SSO server, and the API server. The API server requires at least version *9.4* of PostgreSQL. Beyond that, you have the flexibility to deploy PostgreSQL any way that the Rails servers will be able to connect to it. Our recommended deployment strategy is:
+Arvados requires at least version *9.4* of PostgreSQL.
-* Install PostgreSQL on the same host as the SSO server, and dedicate that install to hosting the SSO database. This provides the best security for the SSO server, because the database does not have to accept any client connections over the network. Typical load on the SSO server is light enough that deploying both it and its database on the same host does not compromise performance.
-* If you want to provide the most scalability for your Arvados cluster, install PostgreSQL for the API server on a dedicated host. This gives you the most flexibility to avoid resource contention, and tune performance separately for the API server and its database. If performance is less of a concern for your installation, you can install PostgreSQL on the API server host directly, as with the SSO server.
-
-Find the section for your distribution below, and follow it to install PostgreSQL on each host where you will deploy it. Then follow the steps in the later section(s) to set up PostgreSQL for the Arvados service(s) that need it.
-
-It is important to make sure that autovacuum is enabled for the PostgreSQL database that backs the API server. Autovacuum is enabled by default since PostgreSQL 8.3.
-
-h2. Install PostgreSQL 9.4+
-
-The API server requires at least version *9.4* of PostgreSQL.
+* "CentOS 7":#centos7
+* "Debian or Ubuntu":#debian
h3(#centos7). CentOS 7
{% assign rh_version = "7" %}
{% include 'note_python_sc' %}
-# Install PostgreSQL:
- <notextile><pre>~$ <span class="userinput">sudo yum install rh-postgresql95 rh-postgresql95-postgresql-contrib</span>
+# Install PostgreSQL
+ <notextile><pre># <span class="userinput">yum install rh-postgresql95 rh-postgresql95-postgresql-contrib</span>
~$ <span class="userinput">scl enable rh-postgresql95 bash</span></pre></notextile>
-# Initialize the database:
- <notextile><pre>~$ <span class="userinput">sudo postgresql-setup initdb</span></pre></notextile>
-# Configure the database to accept password connections:
- <notextile><pre><code>~$ <span class="userinput">sudo sed -ri -e 's/^(host +all +all +(127\.0\.0\.1\/32|::1\/128) +)ident$/\1md5/' /var/lib/pgsql/data/pg_hba.conf</span></code></pre></notextile>
-# Configure the database to launch at boot:
- <notextile><pre>~$ <span class="userinput">sudo systemctl enable rh-postgresql95-postgresql</span></pre></notextile>
-# Start the database:
- <notextile><pre>~$ <span class="userinput">sudo systemctl start rh-postgresql95-postgresql</span></pre></notextile>
-# "Set up Arvados credentials and databases":#rails_setup for the services that will use this PostgreSQL install.
+# Initialize the database
+ <notextile><pre># <span class="userinput">postgresql-setup initdb</span></pre></notextile>
+# Configure the database to accept password connections
+ <notextile><pre><code># <span class="userinput">sed -ri -e 's/^(host +all +all +(127\.0\.0\.1\/32|::1\/128) +)ident$/\1md5/' /var/lib/pgsql/data/pg_hba.conf</span></code></pre></notextile>
+# Configure the database to launch at boot and start now
+ <notextile><pre># <span class="userinput">systemctl enable --now rh-postgresql95-postgresql</span></pre></notextile>
h3(#debian). Debian or Ubuntu
Ubuntu 14.04 (Trusty) requires an updated PostgreSQL version, see "the PostgreSQL ubuntu repository":https://www.postgresql.org/download/linux/ubuntu/
-# Install PostgreSQL:
- <notextile><pre>~$ <span class="userinput">sudo apt-get install postgresql postgresql-contrib</span></pre></notextile>
-# "Set up Arvados credentials and databases":#rails_setup for the services that will use this PostgreSQL install.
-
-<a name="rails_setup"></a>
-
-h2(#sso). Set up SSO server credentials and database
-
-{% assign service_role = "arvados_sso" %}
-{% assign service_database = "arvados_sso_production" %}
-{% assign use_contrib = false %}
-{% include 'install_postgres_database' %}
-
-h2(#api). Set up API server credentials and database
-
-{% assign service_role = "arvados" %}
-{% assign service_database = "arvados_production" %}
-{% assign use_contrib = true %}
-{% include 'install_postgres_database' %}
+# Install PostgreSQL
+ <notextile><pre># <span class="userinput">apt-get --no-install-recommends install postgresql postgresql-contrib</span></pre></notextile>
+# Configure the database to launch at boot and start now
+ <notextile><pre># <span class="userinput">systemctl enable --now postgresql</span></pre></notextile>
---
layout: default
navsection: installguide
-title: Install a shell server
+title: Set up a shell node
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-There is nothing inherently special about an Arvados shell server. It is just a GNU/Linux machine with Arvados utilites and SDKs installed. For optimal performance, the Arvados shell server should be on the same LAN as the Arvados cluster, but that is not required.
+# "Introduction":#introduction
+# "Install Dependecies and SDKs":#dependencies
+# "Install git and curl":#install-packages
+# "Update Git Config":#config-git
+# "Create record for VM":#vm-record
+# "Create scoped token":#scoped-token
+# "Install arvados-login-sync":#arvados-login-sync
+# "Confirm working installation":#confirm-working
-h2. Install API tokens
+h2(#introduction). Introduction
-Please follow the "API token guide":../user/reference/api-tokens.html to get API tokens for your Arvados account and install them on your shell server. We will use those tokens to test the SDKs as we install them.
+Arvados support for shell nodes allows you to use Arvados permissions to grant Linux shell accounts to users.
-h2. Install the Ruby SDK and utilities
+A shell node runs the @arvados-login-sync@ service, and has some additional configuration to make it convenient for users to use Arvados utilites and SDKs. Users are allowed to log in and run arbitrary programs. For optimal performance, the Arvados shell server should be on the same LAN as the Arvados cluster.
-First, install the curl development libraries necessary to build the Arvados Ruby SDK. On Debian-based systems:
+Because it _contains secrets_ shell nodes should *not* have a copy of the complete @config.yml@. For example, if users have access to the @docker@ daemon, it is trival to gain *root* access to any file on the system. Users sharing a shell node should be implicitly trusted, or not given access to Docker. In more secure environments, the admin should allocate a separate VM for each user.
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install libcurl4-openssl-dev</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install libcurl-devel</span>
-</code></pre>
-</notextile>
-
-Next, install the arvados-cli Ruby gem. If you're using RVM:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo /usr/local/rvm/bin/rvm-exec default gem install arvados-cli</span>
-</code></pre>
-</notextile>
-
-If you're not using RVM:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo -i gem install arvados-cli</span>
-</code></pre>
-</notextile>
-
-h2. Install the Python SDK and utilities
+h2(#dependencies). Install Dependecies and SDKs
-{% assign rh_version = "7" %}
-{% include 'note_python_sc' %}
+# "Install Ruby and Bundler":ruby.html
+# "Install the Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
+# "Install the CLI":{{site.baseurl}}/sdk/cli/install.html
+# "Install the R SDK":{{site.baseurl}}/sdk/R/index.html (optional)
+# "Install Docker":install-docker.html (optional)
-On Red Hat-based systems:
+{% assign arvados_component = 'git curl' %}
-<notextile>
-<pre><code>~$ <span class="userinput">echo 'exclude=python2-llfuse' | sudo tee -a /etc/yum.conf</span>
-~$ <span class="userinput">sudo yum install python-arvados-python-client python-arvados-fuse crunchrunner</span>
-</code></pre>
-</notextile>
+{% include 'install_packages' %}
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install python-arvados-python-client python-arvados-fuse crunchrunner</span>
-</code></pre>
-</notextile>
-
-h2. Install Git and curl
-
-{% include 'install_git_curl' %}
-
-h2. Update Git Config
+h2(#config-git). Update Git Config
Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arv-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers.
<notextile>
<pre>
-<code>~$ <span class="userinput">sudo git config --system 'credential.https://git.<b>uuid_prefix.your.domain</b>/.username' none</span></code>
-<code>~$ <span class="userinput">sudo git config --system 'credential.https://git.<b>uuid_prefix.your.domain</b>/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
+<code># <span class="userinput">git config --system 'credential.https://git.<b>ClusterID.example.com</b>/.username' none</span></code>
+<code># <span class="userinput">git config --system 'credential.https://git.<b>ClusterID.example.com</b>/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
</pre>
</notextile>
-h2. Install arvados-login-sync
+h2(#vm-record). Create record for VM
This program makes it possible for Arvados users to log in to the shell server -- subject to permissions assigned by the Arvados administrator -- using the SSH keys they upload to Workbench. It sets up login accounts, updates group membership, and adds users' public keys to the appropriate @authorized_keys@ files.
</pre>
</notextile>
-Create a token that is allowed to read login information for this VM.
+h2(#scoped-token). Create scoped token
+
+As an admin arvados user (such as the system root user), create a token that is restricted to only reading login information for this VM.
<notextile>
<pre>
Note the UUID and the API token output by the above commands: you will need them in a minute.
-Install the arvados-login-sync program.
-
-If you're using RVM:
-
-<notextile>
-<pre>
-<code>shellserver:~$ <span class="userinput">sudo -i `which rvm-exec` default gem install arvados-login-sync</span></code>
-</pre>
-</notextile>
+h2(#arvados-login-sync). Install arvados-login-sync
-If you're not using RVM:
+Install the arvados-login-sync program from RubyGems.
<notextile>
<pre>
-<code>shellserver:~$ <span class="userinput">sudo -i gem install arvados-login-sync</span></code>
+<code>shellserver:# <span class="userinput">gem install arvados-login-sync</span></code>
</pre>
</notextile>
-Install cron.
-
-On Red Hat-based distributions:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install cronie</span>
-~$ <span class="userinput">sudo systemctl enable crond</span>
-~$ <span class="userinput">sudo systemctl start crond</span>
-</code></pre>
-</notextile>
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install cron</span>
-</code></pre>
-</notextile>
-
Configure cron to run the @arvados-login-sync@ program every 2 minutes.
-If you're using RVM:
-
<notextile>
<pre>
-<code>shellserver:~$ <span class="userinput">sudo bash -c 'umask 077; tee /etc/cron.d/arvados-login-sync' <<'EOF'
-ARVADOS_API_HOST="<strong>uuid_prefix.your.domain</strong>"
-ARVADOS_API_TOKEN="<strong>the_token_you_created_above</strong>"
-ARVADOS_VIRTUAL_MACHINE_UUID="<strong>zzzzz-2x53u-zzzzzzzzzzzzzzz</strong>"
-*/2 * * * * root /usr/local/rvm/bin/rvm-exec default arvados-login-sync
-EOF</span></code>
-</pre>
-</notextile>
-
-If you're not using RVM:
-
-<notextile>
-<pre>
-<code>shellserver:~$ <span class="userinput">sudo bash -c 'umask 077; tee /etc/cron.d/arvados-login-sync' <<'EOF'
-ARVADOS_API_HOST="<strong>uuid_prefix.your.domain</strong>"
+<code>shellserver:# <span class="userinput">umask 077; tee /etc/cron.d/arvados-login-sync <<EOF
+ARVADOS_API_HOST="<strong>ClusterID.example.com</strong>"
ARVADOS_API_TOKEN="<strong>the_token_you_created_above</strong>"
ARVADOS_VIRTUAL_MACHINE_UUID="<strong>zzzzz-2x53u-zzzzzzzzzzzzzzz</strong>"
*/2 * * * * root arvados-login-sync
</pre>
</notextile>
+h2(#confirm-working). Confirm working installation
+
A user should be able to log in to the shell server when the following conditions are satisfied:
-* The user has uploaded an SSH public key: Workbench → Account menu → "SSH keys" item → "Add new SSH key" button.
-* As an admin user, you have given the user permission to log in: Workbench → Admin menu → "Users" item → "Show" button → "Admin" tab → "Setup shell account" button.
-* Two minutes have elapsed since the above conditions were satisfied, and the cron job has had a chance to run.
+
+# The user has uploaded an SSH public key: Workbench → Account menu → "SSH keys" item → "Add new SSH key" button.
+# As an admin user, you have given the user permission to log in using the Workbench → Admin menu → "Users" item → "Show" button → "Admin" tab → "Setup account" button.
+# The cron job has run.
+
+See also "how to add a VM login permission link at the command line":../admin/user-management-cli.html
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2(#dependencies). Install prerequisites
+{% include 'notebox_begin_warning' %}
+Skip this section if you are using Google login via @arvados-controller@.
+{% include 'notebox_end' %}
-The Arvados package repository includes an SSO server package that can help automate much of the deployment.
+# "Install dependencies":#dependencies
+# "Set up database":#database-setup
+# "Update config.yml":#update-config
+# "Configure the SSO server":#create-application-yml
+# "Update Nginx configuration":#update-nginx
+# "Install arvados-sso-server":#install-packages
+# "Create arvados-server client record":#client
+# "Restart the API server and controller":#restart-api
-h3(#install_ruby_and_bundler). Install Ruby and Bundler
+h2(#dependencies). Install dependencies
-{% include 'install_ruby_and_bundler_sso' %}
+# "Install PostgreSQL":install-postgresql.html
+# "Install Ruby and Bundler":ruby.html Important! The Single Sign On server only supports Ruby 2.3, to avoid version conflicts we recommend installing it on a different server from the API server. When installing Ruby, ensure that you get the right version by installing the "ruby2.3" package, or by using RVM with @--ruby=2.3@
+# "Install nginx":nginx.html
+# "Install Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html
-h3(#install_web_server). Set up a Web server
+h2(#database-setup). Set up the database
-For best performance, we recommend you use Nginx as your Web server frontend with a Passenger backend to serve the SSO server. The Passenger team provides "Nginx + Passenger installation instructions":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html.
+{% assign service_role = "arvados_sso" %}
+{% assign service_database = "arvados_sso_production" %}
+{% assign use_contrib = false %}
+{% include 'install_postgres_database' %}
-Follow the instructions until you see the section that says you are ready to deploy your Ruby application on the production server.
+Now create @/etc/arvados/sso/database.yml@
-h2(#install). Install the SSO server
+<pre>
+production:
+ adapter: postgresql
+ encoding: utf8
+ database: arvados_sso_production
+ username: arvados_sso
+ password: $password
+ host: localhost
+ template: template0
+</pre>
-On a Debian-based system, install the following package:
+h2(#update-config). Update config.yml
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-sso-server</span>
-</code></pre>
-</notextile>
+<pre>
+ Services:
+ SSO:
+ ExternalURL: auth.ClusterID.example.com
+ Login:
+ ProviderAppID: "arvados-server"
+ ProviderAppSecret: $app_secret
+</pre>
-On a Red Hat-based system, install the following package:
+Generate @ProviderAppSecret@:
<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-sso-server</span>
-</code></pre>
-</notextile>
-
-h2(#configure). Configure the SSO server
+<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
+zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+</code></pre></notextile>
-The package has installed three configuration files in @/etc/arvados/sso@:
+h2(#create-application-yml). Configure the SSO server
-<notextile>
-<pre><code>/etc/arvados/sso/application.yml
-/etc/arvados/sso/database.yml
-/etc/arvados/sso/production.rb
-</code></pre>
-</notextile>
-
-The SSO server runs from the @/var/www/arvados-sso/current/@ directory. The files @/var/www/arvados-sso/current/config/application.yml@, @/var/www/arvados-sso/current/config/database.yml@ and @/var/www/arvados-sso/current/config/environments/production.rb@ are symlinked to the configuration files in @/etc/arvados/sso/@.
+The SSO server runs from the @/var/www/arvados-sso/current/@ directory. The files @/var/www/arvados-sso/current/config/application.yml@ and @/var/www/arvados-sso/current/config/database.yml@ will be symlinked to the configuration files in @/etc/arvados/sso/@.
The SSO server reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the SSO server and is provided for installation convenience only.
Consult @config/application.default.yml@ for a full list of configuration options. Local configuration goes in @/etc/arvados/sso/application.yml@, do not edit @config/application.default.yml@.
-h3(#uuid_prefix). uuid_prefix
+Create @/etc/arvados/sso/application.yml@ and add these keys:
-Generate a uuid prefix for the single sign on service. This prefix is used to identify user records as originating from this site. It must be exactly 5 lowercase ASCII letters and/or digits. You may use the following snippet to generate a uuid prefix:
+<pre>
+production:
+ uuid_prefix: xxxxx
+ secret_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+</pre>
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts "#{rand(2**64).to_s(36)[0,5]}"'</span>
-abcde
-</code></pre></notextile>
+h3(#uuid_prefix). uuid_prefix
-Edit @/etc/arvados/sso/application.yml@ and set @uuid_prefix@ in the "common" section.
+Most of the time, you want this to be the same as your @ClusterID@. If not, generate a new one from the command line listed previously.
h3(#secret_token). secret_token
zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
</code></pre></notextile>
-Edit @/etc/arvados/sso/application.yml@ and set @secret_token@ in the "common" section.
-
-There are other configuration options in @/etc/arvados/sso/application.yml@. See the "Authentication methods":install-sso.html#authentication_methods section below for more details.
-
-h2(#database). Set up the database
-
-Configure the SSO server to connect to your database by updating @/etc/arvados/sso/database.yml@. Replace the @xxxxxxxx@ database password placeholder with the "password you generated during database setup":install-postgresql.html#sso. Be sure to update the @production@ section.
-
-<notextile>
-<pre><code>~$ <span class="userinput">editor /etc/arvados/sso/database.yml</span>
-</code></pre></notextile>
-
-h2(#reconfigure_package). Reconfigure the package
-
-{% assign railspkg = "arvados-sso-server" %}
-{% include 'install_rails_reconfigure' %}
+h3(#authentication_methods). Authentication methods
-h2(#client). Create arvados-server client
+Authentication methods are configured in @application.yml@. Currently three authentication methods are supported: local accounts, LDAP, and Google. If neither Google nor LDAP are enabled, the SSO server defaults to local user accounts. Only one authentication mechanism should be in use at a time. Choose your authentication method and add the listed configuration items to the @production@ section.
-{% assign railshost = "" %}
-{% assign railsdir = "/var/www/arvados-sso/current" %}
-Use @rails console@ to create a @Client@ record that will be used by the Arvados API server. {% include 'install_rails_command' %}
-
-Enter the following commands at the console. The values that appear after you assign @app_id@ and @app_secret@ correspond to the values for @sso_app_id@ and @sso_app_secret@, respectively, in the "API server's SSO settings":install-api-server.html#omniauth.
-
-<notextile>
-<pre><code>:001 > <span class="userinput">c = Client.new</span>
-:002 > <span class="userinput">c.name = "joshid"</span>
-:003 > <span class="userinput">c.app_id = "arvados-server"</span>
-:004 > <span class="userinput">c.app_secret = rand(2**400).to_s(36)</span>
-=> "<strong>save this string for your API server's sso_app_secret</strong>"
-:005 > <span class="userinput">c.save!</span>
-:006 > <span class="userinput">quit</span>
-</code></pre>
-</notextile>
-
-h2(#configure_web_server). Configure your web server
-
-Edit the http section of your Nginx configuration to run the Passenger server and act as a frontend for it. You might add a block like the following, adding SSL and logging parameters to taste:
-
-<notextile>
-<pre><code>server {
- listen 127.0.0.1:8900;
- server_name localhost-sso;
-
- root /var/www/arvados-sso/current/public;
- index index.html;
-
- passenger_enabled on;
- # If you're not using RVM, comment out the line below.
- passenger_ruby /usr/local/rvm/wrappers/default/ruby;
-}
-
-upstream sso {
- server 127.0.0.1:8900 fail_timeout=10s;
-}
-
-proxy_http_version 1.1;
-
-server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name auth.<span class="userinput">your.domain</span>;
-
- ssl on;
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- index index.html;
-
- location / {
- proxy_pass http://sso;
- proxy_redirect off;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host $http_host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
-}
-</code></pre>
-</notextile>
-
-Finally, restart Nginx and your Arvados SSO server should be up and running. You can verify that by visiting the URL you configured your Nginx web server to listen on in the server section above (port 443). Read on if you want to configure your Arvados SSO server to use a different authentication backend.
-
-h2(#authentication_methods). Authentication methods
-
-Authentication methods are configured in @application.yml@. Currently three authentication methods are supported: local accounts, LDAP, and Google+. If neither Google+ nor LDAP are enabled, the SSO server defaults to local user accounts. Only one authentication mechanism should be in use at a time.
-
-h3(#local_accounts). Local account authentication
+h4(#local_accounts). Local account authentication
There are two configuration options for local accounts:
</code></pre>
</notextile>
-h3(#ldap). LDAP authentication
+h4(#ldap). LDAP authentication
The following options are available to configure LDAP authentication. Note that you must preserve the indentation of the fields listed under @use_ldap@.
|bind_dn|If required by server, username to log with in before performing directory lookup|
|password|If required by server, password to log with before performing directory lookup|
-h3(#google). Google+ authentication
+h4(#google). Google authentication
-In order to use Google+ authentication, you must use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
+First, visit "Setting up Google auth.":google-auth.html
-# Go to the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> and select or create a project; this will take you to the project page.
-# On the sidebar, click on *APIs & auth* then select *APIs*.
-## Search for *Contacts API* and click on *Enable API*.
-## Search for *Google+ API* and click on *Enable API*.
-# On the sidebar, click on *Credentials*; under *OAuth* click on *Create new Client ID* to bring up the *Create Client ID* dialog box.
-# Under *Application type* select *Web application*.
-# If the authorization origins are not displayed, clicking on *Create Client ID* will take you to *Consent screen* settings.
-## On consent screen settings, enter the appropriate details and click on *Save*.
-## This will return you to the *Create Client ID* dialog box.
-# You must set the authorization origins. Edit @auth.your.domain@ to the appropriate hostname that you will use to access the SSO service:
-## JavaScript origin should be @https://auth.your.domain/@
-## Redirect URI should be @https://auth.your.domain/users/auth/google_oauth2/callback@
-# Copy the values of *Client ID* and *Client secret* from the Google Developers Console into the Google section of @config/application.yml@, like this:
+Next, copy the values of *Client ID* and *Client secret* from the Google Developers Console into the Google section of @config/application.yml@, like this:
<notextile>
<pre><code> # Google API tokens required for OAuth2 login.
google_oauth2_client_id: <span class="userinput">"---YOUR---CLIENT---ID---HERE--"-</span>
google_oauth2_client_secret: <span class="userinput">"---YOUR---CLIENT---SECRET---HERE--"-</span></code></pre></notextile>
+
+h2(#update-nginx). Update nginx configuration
+
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-sso.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile>
+<pre><code>server {
+ listen <span class="userinput">auth.ClusterID.example.com</span>:443 ssl;
+ server_name <span class="userinput">auth.ClusterID.example.com</span>;
+
+ ssl on;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+
+ root /var/www/arvados-sso/current/public;
+ index index.html;
+
+ passenger_enabled on;
+
+ # <span class="userinput">If you are using RVM, uncomment the line below.</span>
+ # <span class="userinput">If you're using system ruby, leave it commented out.</span>
+ #passenger_ruby /usr/local/rvm/wrappers/default/ruby;
+}
+</code></pre>
+</notextile>
+
+h2(#install-packages). Install arvados-sso-server package
+
+h3. Centos 7
+
+<notextile>
+<pre><code># <span class="userinput">yum install arvados-sso-server</span>
+</code></pre>
+</notextile>
+
+h3. Debian and Ubuntu
+
+<notextile>
+<pre><code># <span class="userinput">apt-get --no-install-recommends arvados-sso-server</span>
+</code></pre>
+</notextile>
+
+h2(#client). Create arvados-server client record
+
+{% assign railshost = "" %}
+{% assign railsdir = "/var/www/arvados-sso/current" %}
+Use @rails console@ to create a @Client@ record that will be used by the Arvados API server. {% include 'install_rails_command' %}
+
+Enter the following commands at the console. The values that appear after you assign @app_id@ and @app_secret@ will be copied to @Login.ProviderAppID@ and @Login.ProviderAppSecret@ in @config.yml@.
+
+<notextile>
+<pre><code>:001 > <span class="userinput">c = Client.new</span>
+:002 > <span class="userinput">c.name = "joshid"</span>
+:003 > <span class="userinput">c.app_id = "arvados-server"</span>
+:004 > <span class="userinput">c.app_secret = "the value of Login.ProviderAppSecret"</span>
+:005 > <span class="userinput">c.save!</span>
+:006 > <span class="userinput">quit</span>
+</code></pre>
+</notextile>
+
+h2(#restart-api). Restart the API server and controller
+
+After adding the SSO server to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
+
+<notextile>
+<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
+</code></pre>
+</notextile>
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-h2. Install prerequisites
+# "Install dependencies":#dependencies
+# "Update config.yml":#update-config
+# "Update Nginx configuration":#update-nginx
+# "Trusted client flag":#trusted_client
+# "Install arvados-workbench":#install-packages
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
-The Arvados package repository includes a Workbench server package that can help automate much of the deployment.
+h2(#dependencies). Install dependencies
-h3(#install_ruby_and_bundler). Install Ruby and Bundler
+# "Install Ruby and Bundler":ruby.html
+# "Install nginx":nginx.html
+# "Install Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html
-{% include 'install_ruby_and_bundler' %}
+h2(#configure). Update config.yml
-h2(#install_workbench). Install Workbench and dependencies
-
-Workbench doesn't need its own database, so it does not need to have PostgreSQL installed.
-
-{% assign rh_version = "7" %}
-{% include 'note_python_sc' %}
-
-On a Debian-based system, install the following packages:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install bison build-essential graphviz git python-arvados-python-client arvados-workbench</span>
-</code></pre>
-</notextile>
-
-On a Red Hat-based system, install the following packages:
+Edit @config.yml@ to set the keys below. The full set of configuration options are in the "Workbench section of config.yml":{{site.baseurl}}/admin/config.html
<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install bison make automake gcc gcc-c++ graphviz git python-arvados-python-client arvados-workbench</span>
+<pre><code> Services:
+ Workbench1:
+ ExternalURL: <span class="userinput">"https://workbench.ClusterID.example.com"</span>
+ Workbench:
+ SecretKeyBase: <span class="userinput">aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa</span>
+ Users:
+ AutoAdminFirstUser: true
</code></pre>
</notextile>
-h2(#configure). Configure Workbench
-
-Edit @/etc/arvados/config.yml@ to set the keys below. Only the most important configuration options are listed here. The full set of configuration options are in the "Workbench section of config.yml":{{site.baseurl}}/admin/config.html
-
-h3. Workbench.SecretKeyBase
-
This application needs a secret token. Generate a new secret:
<notextile>
Then put that value in the @Workbench.SecretKeyBase@ field.
-<notextile>
-<pre><code>Cluster:
- zzzzz:
- Workbench:
- SecretKeyBase: <span class="userinput">aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa</span>
-</code></pre>
-</notextile>
+You probably want to enable @Users.AutoAdminFirstUser@ . The first user to log in when no other admin user exists will automatically be made an admin.
-h3. Services.Controller.ExternalURL
+h2(#update-nginx). Update nginx configuration
-Ensure that @Services.Controller.ExternalURL@ is configured for "Arvados Controller":install-controller.html . For example like this:
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-workbench.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
<notextile>
-<pre><code>Cluster:
- zzzzz:
- Services:
- Controller:
- ExternalURL: <span class="userinput">https://prefix_uuid.your.domain</span>
-</code></pre>
-</notextile>
-
-h3. Workbench.SiteName
-
-@Workbench.SiteName@ can be set to any arbitrary string. It is used to identify this Workbench to people visiting it.
-
-
-<notextile>
-<pre><code>Cluster:
- zzzzz:
- Workbench:
- SiteName: <span class="userinput">My Arvados</span>
-</code></pre>
-</notextile>
-
-h3. TLS.Insecure
-
-For testing only. Allows use of self-signed certificates. If true, workbench will not verify the TLS certificate of Arvados Controller.
-
-<notextile>
-<pre><code>Cluster:
- zzzzz:
- TLS:
- Insecure: <span class="userinput">false</span>
-</code></pre>
-</notextile>
-
-h2. Configure Piwik (optional)
-
-Piwik can be used to gather usage analytics. In @/var/www/arvados-workbench/current/config@, copy @piwik.yml.example@ to @piwik.yml@ and edit to suit.
-
-h2. Set up Web server
-
-For best performance, we recommend you use Nginx as your Web server front-end, with a Passenger backend to serve Workbench. To do that:
-
-<notextile>
-<ol>
-<li><a href="https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html">Install Nginx and Phusion Passenger</a>.</li>
-
-<li><p>Edit the http section of your Nginx configuration to run the Passenger server, and act as a front-end for it. You might add a block like the following, adding SSL and logging parameters to taste:</p>
-
<pre><code>server {
- listen 127.0.0.1:9000;
- server_name localhost-workbench;
-
- root /var/www/arvados-workbench/current/public;
- index index.html index.htm index.php;
-
- passenger_enabled on;
- # If you're using RVM, uncomment the line below.
- #passenger_ruby /usr/local/rvm/wrappers/default/ruby;
-
- # `client_max_body_size` should match the corresponding setting in
- # the API.MaxRequestSize and Controller's server's Nginx configuration.
- client_max_body_size 128m;
-}
-
-upstream workbench {
- server 127.0.0.1:9000 fail_timeout=10s;
+ listen 80;
+ server_name workbench.<span class="userinput">ClusterID.example.com</span>;
+ return 301 https://workbench.<span class="userinput">ClusterID.example.com</span>$request_uri;
}
-proxy_http_version 1.1;
-
server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name workbench.<span class="userinput">uuid-prefix.your.domain</span>;
+ listen *:443 ssl;
+ server_name workbench.<span class="userinput">ClusterID.example.com</span>;
ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
- index index.html index.htm index.php;
+ root /var/www/arvados-workbench/current/public;
+ index index.html;
+
+ passenger_enabled on;
+ # If you're using RVM, uncomment the line below.
+ #passenger_ruby /usr/local/rvm/wrappers/default/ruby;
+
# `client_max_body_size` should match the corresponding setting in
# the API.MaxRequestSize and Controller's server's Nginx configuration.
client_max_body_size 128m;
-
- location / {
- proxy_pass http://workbench;
- proxy_redirect off;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host $http_host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
}
</code></pre>
-</li>
-
-<li>Restart Nginx.</li>
-
-</ol>
-</notextile>
-
-h2. Prepare the Workbench deployment
-
-{% assign railspkg = "arvados-workbench" %}
-{% include 'install_rails_reconfigure' %}
-
-{% include 'notebox_begin' %}
-You can safely ignore the following error message you may see when Ruby Gems are installed:
-<notextile>
-<pre><code>themes_for_rails at /usr/local/rvm/gems/ruby-2.1.1/bundler/gems/themes_for_rails-1fd2d7897d75 did not have a valid gemspec.
-This prevents bundler from installing bins or native extensions, but that may not affect its functionality.
-The validation message from Rubygems was:
- duplicate dependency on rails (= 3.0.11, development), (>= 3.0.0) use:
- add_runtime_dependency 'rails', '= 3.0.11', '>= 3.0.0'
-Using themes_for_rails (0.5.1) from https://github.com/holtkampw/themes_for_rails (at 1fd2d78)
-</code></pre>
</notextile>
-{% include 'notebox_end' %}
-
-h2. Trusted client setting
-Log in to Workbench once to ensure that the Arvados API server has a record of the Workbench client. (It's OK if Workbench says your account hasn't been activated yet. We'll deal with that next.)
+h2(#trusted_client). Trusted client flag
In the <strong>API server</strong> project root, start the Rails console. {% include 'install_rails_command' %}
-At the console, enter the following commands to locate the ApiClient record for your Workbench installation (typically, while you're setting this up, the @last@ one in the database is the one you want), then set the @is_trusted@ flag for the appropriate client record:
+Create an ApiClient record for your Workbench installation with the @is_trusted@ flag set.
-<notextile><pre><code>irb(main):001:0> <span class="userinput">wb = ApiClient.all.last; [wb.url_prefix, wb.created_at]</span>
-=> ["https://workbench.example.com/", Sat, 19 Apr 2014 03:35:12 UTC +00:00]
-irb(main):002:0> <span class="userinput">include CurrentApiClient</span>
-=> true
-irb(main):003:0> <span class="userinput">act_as_system_user do wb.update_attributes!(is_trusted: true) end</span>
+<notextile><pre><code>irb(main):001:0> <span class="userinput">include CurrentApiClient</span>
=> true
+irb(main):002:0> <span class="userinput">act_as_system_user do ApiClient.create!(url_prefix: "https://workbench.ClusterID.example.com/", is_trusted: true) end</span>
+=> #<ApiClient id: 2, uuid: "...", owner_uuid: "...", modified_by_client_uuid: nil, modified_by_user_uuid: "...", modified_at: "2019-12-16 14:19:10", name: nil, url_prefix: "https://workbench.ClusterID.example.com/", created_at: "2019-12-16 14:19:10", updated_at: "2019-12-16 14:19:10", is_trusted: true>
</code></pre>
</notextile>
-h2(#admin-user). Add an admin user
+{% assign arvados_component = 'arvados-workbench' %}
-Next, we're going to use the Rails console on the <strong>API server</strong> to activate your account and give yourself admin privileges. {% include 'install_rails_command' %}
+{% include 'install_packages' %}
-Enter the following commands at the console:
+{% include 'restart_api' %}
-<notextile>
-<pre><code>irb(main):001:0> <span class="userinput">Thread.current[:user] = User.all.select(&:identity_url).last</span>
-irb(main):002:0> <span class="userinput">Thread.current[:user].update_attributes is_admin: true, is_active: true</span>
-irb(main):003:0> <span class="userinput">User.where(is_admin: true).collect &:email</span>
-=> ["root", "<b>your_address@example.com</b>"]
-</code></pre></notextile>
+h2(#confirm-working). Confirm working installation
-At this point, you should have a working Workbench login with administrator privileges. Revisit your Workbench URL in a browser and reload the page to access it.
+Visit @https://workbench.ClusterID.example.com@ in a browser. You should be able to log in using the login method you configured in the previous step. If @Users.AutoAdminFirstUser@ is true, you will be an admin user.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
+# "Update config.yml":#update-config
+# "Update Nginx configuration":#update-nginx
+# "Install arvados-workbench2":#install-packages
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
+# "Trusted client setting":#trusted_client
+
Workbench2 is the web-based user interface for Arvados.
{% include 'notebox_begin' %}
Workbench2 is the replacement for Arvados Workbench. Workbench2 is currently in <i>beta</i>, it is not yet feature complete.
{% include 'notebox_end' %}
-h2(#install_workbench). Install Workbench2 and dependencies
-
-Workbench2 does not require its own database. It is a set of html, javascript and css files that are served as static files from a web server like Nginx or Apache2.
+h2(#configure). Update config.yml
-On a Debian-based system, install the following package:
+Edit @config.yml@ to set the keys below. The full set of configuration options are in the "Workbench section of config.yml":{{site.baseurl}}/admin/config.html
<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-workbench2</span>
+<pre><code> Services:
+ Workbench2:
+ ExternalURL: <span class="userinput">"https://workbench2.ClusterID.example.com"</span>
</code></pre>
</notextile>
-On a Red Hat-based system, install the following package:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-workbench2</span>
-</code></pre>
-</notextile>
+h2(#update-nginx). Update Nginx configuration
-h2. Set up Web server
+Workbench2 does not require its own database. It is a set of html, javascript and css files that are served as static files from Nginx.
-For best performance, we recommend you use Nginx as your Web server to serve Workbench2. Workbench2 consists entirely of static files. To do that:
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-workbench2.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
<notextile>
-<ol>
-<li>Install Nginx</li>
-
-<li><p>Edit the http section of your Nginx configuration to serve Workbench2's files. You might add a block like the following, adding SSL and logging parameters to taste:</p>
-
<pre><code>server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name workbench2.<span class="userinput">uuid-prefix.your.domain</span>;
+ listen 80;
+ server_name workbench2.<span class="userinput">ClusterID.example.com</span>;
+ return 301 https://workbench2.<span class="userinput">ClusterID.example.com</span>$request_uri;
+}
+
+server {
+ listen *:443 ssl;
+ server_name workbench2.<span class="userinput">ClusterID.example.com</span>;
ssl on;
ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
index index.html;
- # Workbench2 uses a call to /config.json to bootstrap itself and talk to the desired API server
+ # <span class="userinput">Workbench2 uses a call to /config.json to bootstrap itself</span>
+ # <span class="userinput">and find out where to contact the API server.</span>
location /config.json {
- return 200 '{ "API_HOST": "<span class="userinput">uuid-prefix.your.domain</span>" }';
+ return 200 '{ "API_HOST": "<span class="userinput">ClusterID.example.com</span>" }';
}
location / {
}
}
</code></pre>
-</li>
+</notextile>
-<li>Restart Nginx.</li>
+h2. Vocabulary configuration (optional)
-</ol>
-</notextile>
+Workbench2 can load a vocabulary file which lists available metadata properties for groups and collections. To configure the property vocabulary definition, please visit the "Workbench2 Vocabulary Format":{{site.baseurl}}/admin/workbench2-vocabulary.html page in the Admin section.
+
+{% assign arvados_component = 'arvados-workbench2' %}
+
+{% include 'install_packages' %}
+
+{% include 'restart_api' %}
-h2. Trusted client setting
+h2(#confirm-working). Confirm working installation
-Log in to Workbench2 once to ensure that the Arvados API server has a record of the Workbench2 client.
+Visit @https://workbench2.ClusterID.example.com@ in a browser. You should be able to log in using the login method you configured in the previous step. If @Users.AutoAdminFirstUser@ is true, you will be an admin user.
+
+h2(#trusted_client). Trusted client flag
+
+Log in to Workbench once to ensure that the Arvados API server has a record of the Workbench client. (It's OK if Workbench says your account hasn't been activated yet. We'll deal with that next.)
In the <strong>API server</strong> project root, start the Rails console. {% include 'install_rails_command' %}
-At the console, enter the following commands to locate the ApiClient record for your Workbench2 installation (typically, while you're setting this up, the @last@ one in the database is the one you want), then set the @is_trusted@ flag for the appropriate client record:
+At the console, enter the following commands to locate the ApiClient record for your Workbench installation (typically, while you're setting this up, the @last@ one in the database is the one you want), then set the @is_trusted@ flag for the appropriate client record:
<notextile><pre><code>irb(main):001:0> <span class="userinput">wb = ApiClient.all.last; [wb.url_prefix, wb.created_at]</span>
-=> ["https://workbench2.<span class="userinput">uuid_prefix.your.domain</span>/", Sat, 20 Apr 2019 01:23:45 UTC +00:00]
+=> ["https://workbench.example.com/", Sat, 19 Apr 2014 03:35:12 UTC +00:00]
irb(main):002:0> <span class="userinput">include CurrentApiClient</span>
=> true
irb(main):003:0> <span class="userinput">act_as_system_user do wb.update_attributes!(is_trusted: true) end</span>
=> true
</code></pre>
</notextile>
-
-h2. Vocabulary configuration (optional)
-
-To configure the property vocabulary definition, please visit the "Workbench2 Vocabulary Format":{{site.baseurl}}/admin/workbench2-vocabulary.html page in the Admin section.
\ No newline at end of file
The arvados-ws server provides event notifications to websocket clients. It can be installed anywhere with access to Postgres database and the Arvados API server, typically behind a web proxy that provides SSL support. See the "godoc page":http://godoc.org/github.com/arvados/arvados/services/ws for additional information.
-By convention, we use the following hostname for the websocket service.
+# "Update config.yml":#update-config
+# "Update nginx configuration":#update-nginx
+# "Install arvados-ws package":#install-packages
+# "Start the service":#start-service
+# "Restart the API server and controller":#restart-api
+# "Confirm working installation":#confirm-working
-<notextile>
-<pre><code>ws.<span class="userinput">uuid_prefix.your.domain</span></code></pre>
-</notextile>
-
-The above hostname should resolve from anywhere on the internet.
-
-h2. Install arvados-ws
-
-Typically arvados-ws runs on the same host as the API server.
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install arvados-ws</span>
-</code></pre>
-</notextile>
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install arvados-ws</span>
-</code></pre>
-</notextile>
+h2(#configure). Update config.yml
-Verify that @arvados-ws@ is functional:
+Edit the cluster config at @config.yml@ and set @Services.Websocket.ExternalURL@ and @Services.Websocket.InternalURLs@. Replace @zzzzz@ with your cluster id.
<notextile>
-<pre><code>~$ <span class="userinput">arvados-ws -h</span>
-Usage of arvados-ws:
- -config path
- path to config file (default "/etc/arvados/config.yml")
- -dump-config
- show current configuration and exit
-</code></pre>
-</notextile>
-
-h3. Update cluster config
-
-Edit the cluster config at @/etc/arvados/config.yml@ and set @Services.Websocket.ExternalURL@ and @Services.Websocket.InternalURLs@. Replace @zzzzz@ with your cluster id.
-
-<notextile>
-<pre><code>Clusters:
- zzzzz:
- Services:
- <span class="userinput">Websocket:
- ExternalURL: wss://ws.uuid_prefix.your.domain/websocket
+<pre><code> Services:
+ Websocket:
InternalURLs:
- "http://localhost:9003": {}
+ "http://localhost:8005"</span>: {}
+ ExternalURL: <span class="userinput">wss://ws.ClusterID.example.com/websocket</span>
</span></code></pre>
</notextile>
-h3. Start the service (option 1: systemd)
-
-If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
-
-If your system uses systemd, the arvados-ws service should already be set up. Start it and check its status:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-ws</span>
-~$ <span class="userinput">sudo systemctl status arvados-ws</span>
-● arvados-ws.service - Arvados websocket server
- Loaded: loaded (/lib/systemd/system/arvados-ws.service; enabled)
- Active: active (running) since Tue 2016-12-06 11:20:48 EST; 10s ago
- Docs: https://doc.arvados.org/
- Main PID: 9421 (arvados-ws)
- CGroup: /system.slice/arvados-ws.service
- └─9421 /usr/bin/arvados-ws
-
-Dec 06 11:20:48 zzzzz arvados-ws[9421]: {"level":"info","msg":"started","time":"2016-12-06T11:20:48.207617188-05:00"}
-Dec 06 11:20:48 zzzzz arvados-ws[9421]: {"Listen":":9003","level":"info","msg":"listening","time":"2016-12-06T11:20:48.244956506-05:00"}
-Dec 06 11:20:48 zzzzz systemd[1]: Started Arvados websocket server.
-</code></pre>
-</notextile>
-
-If it is not running, use @journalctl@ to check logs for errors:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo journalctl -n10 -u arvados-ws</span>
-...
-Dec 06 11:12:48 zzzzz systemd[1]: Starting Arvados websocket server...
-Dec 06 11:12:48 zzzzz arvados-ws[8918]: {"level":"info","msg":"started","time":"2016-12-06T11:12:48.030496636-05:00"}
-Dec 06 11:12:48 zzzzz arvados-ws[8918]: {"error":"pq: password authentication failed for user \"arvados\"","level":"fatal","msg":"db.Ping failed","time":"2016-12-06T11:12:48.058206400-05:00"}
-</code></pre>
-</notextile>
-
-Skip ahead to "confirm the service is working":#confirm.
-
-h3(#runit). Start the service (option 2: runit)
-
-Install runit to supervise the arvados-ws daemon. {% include 'install_runit' %}
-
-Create a supervised service.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/arvados-ws</span>
-~$ <span class="userinput">cd /etc/service/arvados-ws</span>
-~$ <span class="userinput">sudo mkdir log log/main</span>
-~$ <span class="userinput">printf '#!/bin/sh\nexec arvados-ws 2>&1\n' | sudo tee run</span>
-~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
-~$ <span class="userinput">sudo chmod +x run log/run</span>
-~$ <span class="userinput">sudo sv exit .</span>
-~$ <span class="userinput">cd -</span>
-</code></pre>
-</notextile>
-
-Use @sv stat@ and check the log file to verify the service is running.
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/arvados-ws</span>
-run: /etc/service/arvados-ws: (pid 12520) 2s; run: log: (pid 12519) 2s
-~$ <span class="userinput">tail /etc/service/arvados-ws/log/main/current</span>
-{"level":"info","msg":"started","time":"2016-12-06T11:56:20.669171449-05:00"}
-{"Listen":":9003","level":"info","msg":"listening","time":"2016-12-06T11:56:20.708847627-05:00"}
-</code></pre>
-</notextile>
-
-h3(#confirm). Confirm the service is working
-
-Confirm the service is listening on its assigned port and responding to requests.
-
-<notextile>
-<pre><code>~$ <span class="userinput">curl http://0.0.0.0:<b>9003</b>/status.json</span>
-{"Clients":1}
-</code></pre>
-</notextile>
-
-h3. Set up a reverse proxy with SSL support
+h2(#update-nginx). Update Nginx configuration
The arvados-ws service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption.
-This is best achieved by putting a reverse proxy with SSL support in front of arvados-ws, running on port 443 and passing requests to arvados-ws on port 9003 (or whatever port you chose in your configuration file).
-
-For example, using Nginx:
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-ws.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
<notextile><pre>
upstream arvados-ws {
- server 127.0.0.1:<span class="userinput">9003</span>;
+ server 127.0.0.1:<span class="userinput">8005</span>;
}
server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name ws.<span class="userinput">uuid_prefix.your.domain</span>;
+ listen *:443 ssl;
+ server_name ws.<span class="userinput">ClusterID.example.com</span>;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
ssl on;
- ssl_certificate <span class="userinput"/>YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput"/>YOUR/PATH/TO/cert.key</span>;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
location / {
proxy_pass http://arvados-ws;
}
</pre></notextile>
-{% include 'notebox_begin' %}
-If you are upgrading a cluster where Nginx is configured to proxy @ws@ requests to puma, change the @server_name@ value in the old configuration block so it doesn't conflict. When the new configuration is working, delete the old Nginx configuration sections (i.e., the "upstream websockets" block, and the "server" block that references @http://websockets@), and disable/remove the runit or systemd files for the puma server.
-{% include 'notebox_end' %}
+{% assign arvados_component = 'arvados-ws' %}
+
+{% include 'install_packages' %}
+
+{% include 'start_service' %}
-h3. Update API server configuration
+{% include 'restart_api' %}
-Restart Nginx to reload the API server configuration.
+h2(#restart-api). Restart the API server and controller
+
+After adding the SSO server to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
<notextile>
-<pre><code>$ sudo nginx -s reload</span>
+<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
</code></pre>
</notextile>
-h3. Verify DNS and proxy setup
+h2(#confirm). Confirm working installation
-Use a host elsewhere on the Internet to confirm that your DNS, proxy, and SSL are configured correctly. For @Authorization: Bearer xxxx@ replace @xxxx@ with the value from @ManagementToken@ in @config.yml@.
+Confirm the service is listening on its assigned port and responding to requests.
<notextile>
-<pre><code>$ <span class="userinput">curl -H "Authorization: Bearer xxxx" https://ws.<b>uuid_prefix.your.domain</b>/_health/ping</span>
-{"health":"OK"}
+<pre><code>~$ <span class="userinput">curl https://<span class="userinput">ws.ClusterID.example.com</span>/status.json</span>
+{"Clients":1}
</code></pre>
</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Nginx
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+h3. Centos 7
+
+<notextile>
+<pre><code># <span class="userinput">yum install epel-release</span></code>
+<code># <span class="userinput">yum install nginx</span></code></pre>
+</notextile>
+
+h3. Debian and Ubuntu
+
+<notextile>
+<pre><code># <span class="userinput">apt-get --no-install-recommends install nginx</span></code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Arvados package repositories
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+On any host where you install Arvados software, you'll need to add the Arvados package repository. They're available for several popular distributions.
+
+* "Centos 7":#centos7
+* "Debian and Ubuntu":#debian
+
+h3(#centos7). CentOS
+
+Packages are available for CentOS 7. To install them with yum, save this configuration block in @/etc/yum.repos.d/arvados.repo@:
+
+<notextile>
+<pre><code>[arvados]
+name=Arvados
+baseurl=http://rpm.arvados.org/CentOS/$releasever/os/$basearch/
+gpgcheck=1
+gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-curoverse
+</code></pre>
+</notextile>
+
+{% include 'install_redhat_key' %}
+
+h3(#debian). Debian and Ubuntu
+
+Packages are available for recent versions of Debian and Ubuntu.
+
+First, register the Arvados signing key in apt's database:
+
+{% include 'install_debian_key' %}
+
+As root, add the Arvados package repository to your sources. This command depends on your OS vendor and version:
+
+table(table table-bordered table-condensed).
+|_. OS version|_. Command|
+|Debian 10 ("buster")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ buster main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+
+
+{% include 'notebox_begin' %}
+
+fn1. Arvados packages for Ubuntu may depend on third-party packages in Ubuntu's "universe" repository. If you're installing on Ubuntu, make sure you have the universe sources uncommented in @/etc/apt/sources.list@.
+
+{% include 'notebox_end' %}
+
+Retrieve the package list:
+
+<notextile>
+<pre><code># <span class="userinput">apt-get update</span>
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Ruby and Bundler
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+{% include 'install_ruby_and_bundler' %}
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Set up web based login
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Option 1: Google login through Arvados controller":#controller
+# "Option 2: Separate single-sign-on (SSO) server (Google, LDAP, local database)":#sso
+
+h2(#controller). Option 1: Google login through Arvados controller
+
+First, visit "Setting up Google auth.":google-auth.html
+
+Next, copy the values of *Client ID* and *Client secret* from the Google Developers Console into @Login.GoogleClientID@ and @Login.GoogleClientSecret@ of @config.yml@ :
+
+<pre>
+ Login:
+ GoogleClientID: ""
+ GoogleClientSecret: ""
+</pre>
+
+h2(#sso). Option 2: Separate single-sign-on (SSO) server (supports Google, LDAP, local database)
+
+See "Install the Single Sign On (SSO) server":install-sso.html
Arvados CLI tools are written in Ruby and Python. To use the @arv@ command, you can either install the @arvados-cli@ gem via RubyGems or build and install the package from source. The @arv@ command also relies on other Arvados tools. To get those, install the @arvados-python-client@ and @arvados-cwl-runner@ packages, either from PyPI or source.
-h3. Prerequisites: Ruby, Bundler, and curl libraries
+h2. Prerequisites
-{% include 'install_ruby_and_bundler' %}
+# "Install Ruby and Bundler":../../install/ruby.html
+# "Install the Python SDK":../python/sdk-python.html
-Install curl libraries with your system's package manager. For example, on Debian or Ubuntu:
-
-<notextile>
-<pre>
-~$ <code class="userinput">sudo apt-get install libcurl3 libcurl3-gnutls libcurl4-openssl-dev</code>
-</pre>
-</notextile>
-
-h3. Option 1: Install from RubyGems and PyPI
+h2. Install from RubyGems
<notextile>
<pre>
~$ <code class="userinput">sudo -i gem install arvados-cli</code>
</pre>
</notextile>
-
-<notextile>
-<pre>
-~$ <code class="userinput">pip install arvados-python-client arvados-cwl-runner</code>
-</pre>
-</notextile>
-
-h3. Option 2: Build and install from source
-
-<notextile>
-<pre>
-~$ <code class="userinput">git clone https://github.com/arvados/arvados.git</code>
-~$ <code class="userinput">cd arvados/sdk/cli</code>
-~/arvados/sdk/cli$ <code class="userinput">gem build arvados-cli.gemspec</code>
-~/arvados/sdk/cli$ <code class="userinput">sudo -i gem install arvados-cli-*.gem</code>
-~/arvados/sdk/cli$ <code class="userinput">cd ../python</code>
-~/arvados/sdk/python$ <code class="userinput">python setup.py install</code>
-~/arvados/sdk/python$ <code class="userinput">cd ../cwl</code>
-~/arvados/sdk/cwl$ <code class="userinput">python setup.py install</code>
-</pre>
-</notextile>
* "Command line SDK":{{site.baseurl}}/sdk/cli/install.html ("arv")
* "Go SDK":{{site.baseurl}}/sdk/go/index.html
* "R SDK":{{site.baseurl}}/sdk/R/index.html
-* "Perl SDK":{{site.baseurl}}/sdk/perl/index.html
* "Ruby SDK":{{site.baseurl}}/sdk/ruby/index.html
* "Java SDK v2":{{site.baseurl}}/sdk/java-v2/index.html
* "Java SDK v1":{{site.baseurl}}/sdk/java/index.html
+* "Perl SDK":{{site.baseurl}}/sdk/perl/index.html
Many Arvados Workbench pages, under the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
The Java SDK v1 provides a low level API to call Arvados from Java.
+This is a legacy SDK. It is no longer used or maintained regularly. The "Arvados Java SDK v2":../java-v2/index.html should be used.
+
h3. Introdution
* The Java SDK requires Java 6 or later
The Perl SDK provides a generic set of wrappers so you can make API calls easily.
-It should be treated as alpha/experimental. Currently, limitations include:
-* Verbose syntax.
-* No native Keep client.
-* No CPAN package.
+This is a legacy SDK. It is no longer used or maintained regularly.
h3. Installation
In these examples, the site prefix is @aaaaa@.
+See also the "cookbook":cookbook.html for more complex examples.
+
h2. Initialize SDK
{% codeblock as python %}
{% codeblock as python %}
result = api.users().current().execute()
{% endcodeblock %}
+
+h2. Get the User object for the current user
+
+{% codeblock as python %}
+current_user = arvados.api('v1').users().current().execute()
+{% endcodeblock %}
+
+h2. Get the UUID of an object that was retrieved using the SDK
+
+{% codeblock as python %}
+my_uuid = current_user['uuid']
+{% endcodeblock %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The Python SDK provides access from Python to the Arvados API and Keep. It also includes a number of command line tools for using and administering Arvados and Keep, and some conveniences for use in Crunch scripts; see "Crunch utility libraries":crunch-utility-libraries.html for details.
+The Python SDK provides access from Python to the Arvados API and Keep, along with a number of command line tools for using and administering Arvados and Keep.
h2. Installation
The Python SDK supports Python 2.7 and 3.4+
-h3. Option 1: Install with pip
+h2. Option 1: Install from a distribution package
-This installation method is recommended to make the SDK available for use in your own Python programs. It can coexist with the system-wide installation method from a distribution package (option 2, below).
+This installation method is recommended to make the CLI tools available system-wide. It can coexist with the installation method described in option 2, below.
+
+First, configure the "Arvados package repositories":../../install/packages.html
+
+{% assign arvados_component = 'python-arvados-python-client' %}
+
+{% include 'install_packages' %}
+
+h2. Option 2: Install with pip
+
+This installation method is recommended to use the SDK in your own Python programs. It can coexist with the system-wide installation method from a distribution package (option 2, below).
Run @pip install arvados-python-client@ in an appropriate installation environment, such as a @virtualenv@.
If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip install --pre arvados-python-client@.
-h3. Option 2: Install from a distribution package
-
-This installation method is recommended to make the CLI tools available system-wide. It can coexist with the installation method described in option 1, above.
-
-First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
-
-On Red Hat-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo yum install python-arvados-python-client</code>
-</code></pre>
-</notextile>
-
-On Debian-based systems:
-
-<notextile>
-<pre><code>~$ <span class="userinput">sudo apt-get install python-arvados-python-client</code>
-</code></pre>
-</notextile>
-
-h3. Test installation
+h2. Test installation
If the SDK is installed and your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), @import arvados@ should produce no errors.
</pre>
</notextile>
-h3. Examples
-
-Get the User object for the current user:
-
-<notextile>
-<pre><code class="userinput">current_user = arvados.api('v1').users().current().execute()
-</code></pre>
-</notextile>
-
-Get the UUID of an object that was retrieved using the SDK:
+h2. Usage
-<notextile>
-<pre><code class="userinput">my_uuid = current_user['uuid']
-</code></pre>
-</notextile>
-
-Retrieve an object by ID:
-
-<notextile>
-<pre><code class="userinput">some_user = arvados.api('v1').users().get(uuid=my_uuid).execute()
-</code></pre>
-</notextile>
-
-Create an object:
-
-<notextile>
-<pre><code class="userinput">test_link = arvados.api('v1').links().create(
- body={'link_class':'test','name':'test'}).execute()
-</code></pre>
-</notextile>
-
-Update an object:
-
-<notextile>
-<pre><code class="userinput">arvados.api('v1').links().update(
- uuid=test_link['uuid'],
- body={'properties':{'foo':'bar'}}).execute()
-</code></pre>
-</notextile>
-
-Get a list of objects:
-
-<notextile>
-<pre><code class="userinput">repos = arvados.api('v1').repositories().list().execute()
-len(repos['items'])</code>
-2
-<code class="userinput">repos['items'][0]['uuid']</code>
-u'qr1hi-s0uqq-kg8cawglrf74bmw'
-</code></pre>
-</notextile>
+Check out the "examples":example.html and "cookbook":cookbook.html
h3. Notes
Indicate that one or more input parameters are "secret". Must be applied at the top level Workflow. Secret parameters are not stored in keep, are hidden from logs and API responses, and are wiped from the database after the workflow completes.
+*Note: currently, workflows with secrets must be submitted on the command line using @arvados-cwl-runner@. Workflows with secrets submitted through Workbench will not properly obscure the secret inputs.*
+
table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|secrets|array<string>|Input parameters which are considered "secret". Must be strings.|
A URI reference to Keep uses the @keep:@ scheme followed by either the portable data hash or UUID of the collection and then the location of the file inside the collection. For example, @keep:2463fa9efeb75e099685528b3b9071e0+438/19.fasta.bwt@ or @keep:zzzzz-4zz18-zzzzzzzzzzzzzzz/19.fasta.bwt@.
-If you reference a file in "arv-mount":{{site.baseurl}}/user/tutorials/tutorial-keep-mount.html, such as @/home/example/keep/by_id/2463fa9efeb75e099685528b3b9071e0+438/19.fasta.bwt@, then @arvados-cwl-runner@ will automatically determine the appropriate Keep URI reference.
+If you reference a file in "arv-mount":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-gnu-linux.html, such as @/home/example/keep/by_id/2463fa9efeb75e099685528b3b9071e0+438/19.fasta.bwt@, then @arvados-cwl-runner@ will automatically determine the appropriate Keep URI reference.
If you reference a local file which is not in @arv-mount@, then @arvados-cwl-runner@ will upload the file to Keep and use the Keep URI reference from the upload.
For example, let's copy from the <a href="https://playground.arvados.org/">Arvados playground</a>, also known as *qr1hi*, to *dst_cluster*. The names *qr1hi* and *dst_cluster* are interchangable with any cluster name. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *qr1hi*-4zz18-tci4vn4fa95w0zx, the cluster name is qr1hi.
-In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon <span class="fa fa-lg fa-user"></span> <span class="caret"></span> in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files, one for each cluster. The names of the files must have the format of *uuid_prefix.conf*. In our example, let's make two files, one for *qr1hi* and one for *dst_cluster*. From your *Current token* page in *qr1hi* and *dst_cluster*, copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
+In order to communicate with both clusters, you must create custom configuration files for each cluster. In the Arvados Workbench, click on the dropdown menu icon <span class="fa fa-lg fa-user"></span> <span class="caret"></span> in the upper right corner of the top navigation menu to access the user settings menu, and click on the menu item *Current token*. Copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ in both of your clusters. Then, create two configuration files, one for each cluster. The names of the files must have the format of *ClusterID.conf*. In our example, let's make two files, one for *qr1hi* and one for *dst_cluster*. From your *Current token* page in *qr1hi* and *dst_cluster*, copy the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@.
!{display: block;margin-left: 25px;margin-right: auto;}{{ site.baseurl }}/images/api-token-host.png!
h3. Browsing Keep (read-only)
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.uuid_prefix.your.domain/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
h3. Accessing a specific collection in Keep (read-write)
-In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.uuid_prefix.your.domain/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
+In Finder, use "Connect to Server..." under the "Go" menu and enter @https://collections.ClusterID.example.com/@ in popup dialog. When prompted for credentials, put a valid Arvados token in the @Password@ field and anything in the Name field (it will be ignored by Arvados).
This collection is now accessible read/write.
h3. Browsing Keep (read-only)
-Use the 'Map network drive' functionality, and enter @https://collections.uuid_prefix.your.domain/@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid Arvados token in the @Password@ field.
+Use the 'Map network drive' functionality, and enter @https://collections.ClusterID.example.com/@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid Arvados token in the @Password@ field.
This mount is read-only. Write support for the @/users/@ directory is planned for a future release.
h3. Accessing a specific collection in Keep (read-write)
-Use the 'Map network drive' functionality, and enter @https://collections.uuid_prefix.your.domain/c=your-collection-uuid@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid token in the @Password@ field.
+Use the 'Map network drive' functionality, and enter @https://collections.ClusterID.example.com/c=your-collection-uuid@ in the Folder field. When prompted for credentials, you can fill in an arbitrary string for @Username@, it is ignored by Arvados. Windows will not accept an empty @Username@. Put a valid token in the @Password@ field.
This collection is now accessible read/write.
SAMPLE: true
Driver: s3
DriverParameters:
-
# for s3 driver -- see
# https://doc.arvados.org/install/configure-s3-object-storage.html
IAMRole: aaaaa
ConnectTimeout: 1m
ReadTimeout: 10m
RaceWindow: 24h
+
+ # For S3 driver, potentially unsafe tuning parameter,
+ # intentionally excluded from main documentation.
+ #
+ # Enable deletion (garbage collection) even when the
+ # configured BlobTrashLifetime is zero. WARNING: eventual
+ # consistency may result in race conditions that can cause
+ # data loss. Do not enable this unless you understand and
+ # accept the risk.
UnsafeDelete: false
# for azure driver -- see
# for local directory driver -- see
# https://doc.arvados.org/install/configure-fs-storage.html
Root: /var/lib/arvados/keep-data
+
+ # For local directory driver, potentially confusing tuning
+ # parameter, intentionally excluded from main documentation.
+ #
+ # When true, read and write operations (for whole 64MiB
+ # blocks) on an individual volume will queued and issued
+ # serially. When false, read and write operations will be
+ # issued concurrently.
+ #
+ # May possibly improve throughput if you have physical spinning disks
+ # and experience contention when there are multiple requests
+ # to the same volume.
+ #
+ # Otherwise, when using SSDs, RAID, or a shared network filesystem, you
+ # should leave this alone.
Serialize: false
Mail:
PATH
remote: .
specs:
- arvados-login-sync (1.4.1.20190930204434)
+ arvados-login-sync (1.3.1.pre20191210204053)
arvados (~> 1.3.0, >= 1.3.0)
faraday (< 0.16)
i18n (~> 0)
json (>= 1.7.7, < 3)
jwt (>= 0.1.5, < 2)
- arvados-google-api-client (0.8.7.2)
+ arvados-google-api-client (0.8.7.3)
activesupport (>= 3.2, < 5.1)
addressable (~> 2.3)
autoparse (~> 0.3)
minitest (5.11.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
- multi_json (1.13.1)
+ multi_json (1.14.1)
multipart-post (2.1.1)
os (1.0.1)
public_suffix (4.0.1)
rake
BUNDLED WITH
- 1.17.3
+ 2.0.2
projects / arvados.git / commitdiff
