do_install() {
skipit=false
- if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then
+ if [[ -z "${only_install}" || "${only_install}" == "${1}" || "${only_install}" == "${2}" ]]; then
retry do_install_once ${@}
else
skipit=true
- Run a workflow using Workbench:
- user/getting_started/workbench.html.textile.liquid
- user/tutorials/tutorial-workflow-workbench.html.textile.liquid
+ - user/composer/composer.html.textile.liquid
- Access an Arvados virtual machine:
- user/getting_started/vm-login-with-webshell.html.textile.liquid
- user/getting_started/ssh-access-unix.html.textile.liquid
- Core:
- install/install-postgresql.html.textile.liquid
- install/install-api-server.html.textile.liquid
+ - install/install-controller.html.textile.liquid
- Keep:
- install/install-keepstore.html.textile.liquid
- install/configure-fs-storage.html.textile.liquid
h2. API server
-Set @MangementToken@ in the appropriate section of @application.yml@
+Set @ManagementToken@ in the appropriate section of @application.yml@
<pre>
production:
h2. Node Manager
-Set @port@ (the listen port) and @MangementToken@ in the @Manage@ section of @node-manager.ini@ .
+Set @port@ (the listen port) and @ManagementToken@ in the @Manage@ section of @node-manager.ini@.
<pre>
[Manage]
h2. Other services
-The following services also support monitoring. Set @MangementToken@ in the respective yaml config file for each service.
+The following services also support monitoring. Set @ManagementToken@ in the respective yaml config file for each service.
* keepstore
* keep-web
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
+h3. 2018-07-31: "#13497":https://dev.arvados.org/issues/13497 "db5107dca":https://dev.arvados.org/projects/arvados/repository/revisions/db5107dca adds a new system service, arvados-controller
+* "Install the controller":../install/install-controller.html after upgrading your system.
+* Verify your setup by confirming that API calls appear in the controller's logs (_e.g._, @journalctl -fu arvados-controller@) while loading a workbench page.
+
h3. 2018-04-05: v1.1.4 regression in arvados-cwl-runner for workflows that rely on implicit discovery of secondaryFiles
h4. Secondary files missing from toplevel workflow inputs
This bug will be fixed in an upcoming release of Arvados.
-h3. 2017-12-08: #11908 commit:8f987a9271 now requires minimum of Postgres 9.4 (previously 9.3)
+h3. 2017-12-08: "#11908":https://dev.arvados.org/issues/11908 "8f987a9271":https://dev.arvados.org/projects/arvados/repository/revisions/8f987a9271 now requires minimum of Postgres 9.4 (previously 9.3)
* Debian 8 (pg 9.4) and Debian 9 (pg 9.6) do not require an upgrade
* Ubuntu 16.04 (pg 9.5) does not require an upgrade
* Ubuntu 14.04 (pg 9.3) requires upgrade to Postgres 9.4: https://www.postgresql.org/download/linux/ubuntu/
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3. 2017-09-25: #12032 commit:68bdf4cbb now requires minimum of Postgres 9.3 (previously 9.1)
+h3. 2017-09-25: "#12032":https://dev.arvados.org/issues/12032 "68bdf4cbb":https://dev.arvados.org/projects/arvados/repository/revisions/68bdf4cbb now requires minimum of Postgres 9.3 (previously 9.1)
* Debian 8 (pg 9.4) and Debian 9 (pg 9.6) do not require an upgrade
* Ubuntu 16.04 (pg 9.5) does not require an upgrade
* Ubuntu 14.04 (pg 9.3) is compatible, however upgrading to Postgres 9.4 is recommended: https://www.postgresql.org/download/linux/ubuntu/
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3. 2017-06-30: #11807 commit:55aafbb converts old "jobs" database records from YAML to JSON, making the upgrade process slower than usual.
-* The migration can take some time if your database contains a substantial number of YAML-serialized rows (i.e., you installed Arvados before March 3, 2017 commit:660a614 and used the jobs/pipelines APIs). Otherwise, the upgrade will be no slower than usual.
+h3. 2017-06-30: "#11807":https://dev.arvados.org/issues/11807 "55aafbb":https://dev.arvados.org/projects/arvados/repository/revisions/55aafbb converts old "jobs" database records from YAML to JSON, making the upgrade process slower than usual.
+* The migration can take some time if your database contains a substantial number of YAML-serialized rows (i.e., you installed Arvados before March 3, 2017 "660a614":https://dev.arvados.org/projects/arvados/repository/revisions/660a614 and used the jobs/pipelines APIs). Otherwise, the upgrade will be no slower than usual.
* The conversion runs as a database migration, i.e., during the deb/rpm package upgrade process, while your API server is unavailable.
* Expect it to take about 1 minute per 20K jobs that have ever been created/run.
-h3. 2017-06-05: #9005 commit:cb230b0 reduces service discovery overhead in keep-web requests.
+h3. 2017-06-05: "#9005":https://dev.arvados.org/issues/9005 "cb230b0":https://dev.arvados.org/projects/arvados/repository/revisions/cb230b0 reduces service discovery overhead in keep-web requests.
* When upgrading keep-web _or keepproxy_ to/past this version, make sure to update API server as well. Otherwise, a bad token in a request can cause keep-web to fail future requests until either keep-web restarts or API server gets upgraded.
-h3. 2017-04-12: #11349 commit:2c094e2 adds a "management" http server to nodemanager.
+h3. 2017-04-12: "#11349":https://dev.arvados.org/issues/11349 "2c094e2":https://dev.arvados.org/projects/arvados/repository/revisions/2c094e2 adds a "management" http server to nodemanager.
* To enable it, add to your configuration file: <pre>[Manage]
address = 127.0.0.1
port = 8989</pre> (see example configuration files in source:services/nodemanager/doc or https://doc.arvados.org/install/install-nodemanager.html for more info)
* The server responds to @http://{address}:{port}/status.json@ with a summary of how many nodes are in each state (booting, busy, shutdown, etc.)
-h3. 2017-03-23: #10766 commit:e8cc0d7 replaces puma with arvados-ws as the recommended websocket server.
+h3. 2017-03-23: "#10766":https://dev.arvados.org/issues/10766 "e8cc0d7":https://dev.arvados.org/projects/arvados/repository/revisions/e8cc0d7 replaces puma with arvados-ws as the recommended websocket server.
* See http://doc.arvados.org/install/install-ws.html for install/upgrade instructions.
* Remove the old puma server after the upgrade is complete. Example, with runit: <pre>
$ sudo sv down /etc/sv/puma
$ systemctl stop puma
</pre>
-h3. 2017-03-06: #11168 commit:660a614 uses JSON instead of YAML to encode hashes and arrays in the database.
+h3. 2017-03-06: "#11168":https://dev.arvados.org/issues/11168 "660a614":https://dev.arvados.org/projects/arvados/repository/revisions/660a614 uses JSON instead of YAML to encode hashes and arrays in the database.
* Aside from a slight performance improvement, this should have no externally visible effect.
* Downgrading past this version is not supported, and is likely to cause errors. If this happens, the solution is to upgrade past this version.
* After upgrading, make sure to restart puma and crunch-dispatch-* processes.
-h3. 2017-02-03: #10969 commit:74a9dec introduces a Docker image format compatibility check: the @arv keep docker@ command prevents users from inadvertently saving docker images that compute nodes won't be able to run.
+h3. 2017-02-03: "#10969":https://dev.arvados.org/issues/10969 "74a9dec":https://dev.arvados.org/projects/arvados/repository/revisions/74a9dec introduces a Docker image format compatibility check: the @arv keep docker@ command prevents users from inadvertently saving docker images that compute nodes won't be able to run.
* If your compute nodes run a version of *docker older than 1.10* you must override the default by adding to your API server configuration (@/etc/arvados/api/application.yml@): <pre><code class="yaml">docker_image_formats: ["v1"]</code></pre>
-* Refer to the comments above @docker_image_formats@ in @/var/www/arvados-api/current/config/application.default.yml@ or source:services/api/config/application.default.yml or issue #10969 for more detail.
+* Refer to the comments above @docker_image_formats@ in @/var/www/arvados-api/current/config/application.default.yml@ or source:services/api/config/application.default.yml or issue "#10969":https://dev.arvados.org/issues/10969 for more detail.
* *NOTE:* This does *not* include any support for migrating existing Docker images from v1 to v2 format. This will come later: for now, sites running Docker 1.9 or earlier should still *avoid upgrading Docker further than 1.9.*
-h3. 2016-09-27: several Debian and RPM packages -- keep-balance (commit:d9eec0b), keep-web (commit:3399e63), keepproxy (commit:6de67b6), and arvados-git-httpd (commit:9e27ddf) -- now enable their respective components using systemd. These components prefer YAML configuration files over command line flags (commit:3bbe1cd).
+h3. 2016-09-27: several Debian and RPM packages -- keep-balance ("d9eec0b":https://dev.arvados.org/projects/arvados/repository/revisions/d9eec0b), keep-web ("3399e63":https://dev.arvados.org/projects/arvados/repository/revisions/3399e63), keepproxy ("6de67b6":https://dev.arvados.org/projects/arvados/repository/revisions/6de67b6), and arvados-git-httpd ("9e27ddf":https://dev.arvados.org/projects/arvados/repository/revisions/9e27ddf) -- now enable their respective components using systemd. These components prefer YAML configuration files over command line flags ("3bbe1cd":https://dev.arvados.org/projects/arvados/repository/revisions/3bbe1cd).
* On Debian-based systems using systemd, services are enabled automatically when packages are installed.
* On RedHat-based systems using systemd, unit files are installed but services must be enabled explicitly: e.g., <code>"sudo systemctl enable keep-web; sudo systemctl start keep-web"</code>.
* The new systemd-supervised services will not start up successfully until configuration files are installed in /etc/arvados/: e.g., <code>"Sep 26 18:23:55 62751f5bb946 keep-web[74]: 2016/09/26 18:23:55 open /etc/arvados/keep-web/keep-web.yml: no such file or directory"</code>
** keepproxy - /etc/arvados/keepproxy/keepproxy.yml
** arvados-git-httpd - /etc/arvados/arv-git-httpd/arv-git-httpd.yml
-h3. 2016-05-31: commit:ae72b172c8 and commit:3aae316c25 install Python modules and scripts to different locations on the filesystem.
-* Previous packages installed these files to the distribution's preferred path under @/usr/local@ (or the equivalent location in a Software Collection). Now they get installed to a path under @/usr@. This improves compatibility with other Python packages provided by the distribution. See #9242 for more background.
+h3. 2016-05-31: "ae72b172c8":https://dev.arvados.org/projects/arvados/repository/revisions/ae72b172c8 and "3aae316c25":https://dev.arvados.org/projects/arvados/repository/revisions/3aae316c25 install Python modules and scripts to different locations on the filesystem.
+* Previous packages installed these files to the distribution's preferred path under @/usr/local@ (or the equivalent location in a Software Collection). Now they get installed to a path under @/usr@. This improves compatibility with other Python packages provided by the distribution. See "#9242":https://dev.arvados.org/issues/9242 for more background.
* If you simply import Python modules from scripts, or call Python tools relying on $PATH, you don't need to make any changes. If you have hardcoded full paths to some of these files (e.g., in symbolic links or configuration files), you will need to update those paths after this upgrade.
-h3. 2016-04-25: commit:eebcb5e requires the crunchrunner package to be installed on compute nodes and shell nodes in order to run CWL workflows.
+h3. 2016-04-25: "eebcb5e":https://dev.arvados.org/projects/arvados/repository/revisions/eebcb5e requires the crunchrunner package to be installed on compute nodes and shell nodes in order to run CWL workflows.
* On each Debian-based compute node and shell node, run: @sudo apt-get install crunchrunner@
* On each Red Hat-based compute node and shell node, run: @sudo yum install crunchrunner@
-h3. 2016-04-21: commit:3c88abd changes the Keep permission signature algorithm.
+h3. 2016-04-21: "3c88abd":https://dev.arvados.org/projects/arvados/repository/revisions/3c88abd changes the Keep permission signature algorithm.
* All software components that generate signatures must be upgraded together. These are: keepstore, API server, keep-block-check, and keep-rsync. For example, if keepstore < 0.1.20160421183420 but API server >= 0.1.20160421183420, clients will not be able to read or write data in Keep.
* Jobs and client operations that are in progress during the upgrade (including arv-put's "resume cache") will fail.
-h3. 2015-01-05: commit:e1276d6e disables Workbench's "Getting Started" popup by default.
+h3. 2015-01-05: "e1276d6e":https://dev.arvados.org/projects/arvados/repository/revisions/e1276d6e disables Workbench's "Getting Started" popup by default.
* If you want new users to continue seeing this popup, set @enable_getting_started_popup: true@ in Workbench's @application.yml@ configuration.
-h3. 2015-12-03: commit:5590c9ac makes a Keep-backed writable scratch directory available in crunch jobs (see #7751)
-* All compute nodes must be upgraded to arvados-fuse >= 0.1.2015112518060 because crunch-job uses some new arv-mount flags (--mount-tmp, --mount-by-pdh) introduced in merge commit:346a558
+h3. 2015-12-03: "5590c9ac":https://dev.arvados.org/projects/arvados/repository/revisions/5590c9ac makes a Keep-backed writable scratch directory available in crunch jobs (see "#7751":https://dev.arvados.org/issues/7751)
+* All compute nodes must be upgraded to arvados-fuse >= 0.1.2015112518060 because crunch-job uses some new arv-mount flags (--mount-tmp, --mount-by-pdh) introduced in merge "346a558":https://dev.arvados.org/projects/arvados/repository/revisions/346a558
* Jobs will fail if the API server (in particular crunch-job from the arvados-cli gem) is upgraded without upgrading arvados-fuse on compute nodes.
-h3. 2015-11-11: commit:1e2ace5 changes recommended config for keep-web (see #5824)
+h3. 2015-11-11: "1e2ace5":https://dev.arvados.org/projects/arvados/repository/revisions/1e2ace5 changes recommended config for keep-web (see "#5824":https://dev.arvados.org/issues/5824)
* proxy/dns/ssl config should be updated to route "https://download.uuid_prefix.arvadosapi.com/" requests to keep-web (alongside the existing "collections" routing)
* keep-web command line adds @-attachment-only-host download.uuid_prefix.arvadosapi.com@
* Workbench config adds @keep_web_download_url@
* More info on the (still beta/non-TOC-linked) "keep-web doc page":http://doc.arvados.org/install/install-keep-web.html
-h3. 2015-11-04: commit:1d1c6de removes stopped containers (see #7444)
+h3. 2015-11-04: "1d1c6de":https://dev.arvados.org/projects/arvados/repository/revisions/1d1c6de removes stopped containers (see "#7444":https://dev.arvados.org/issues/7444)
* arvados-docker-cleaner removes _all_ docker containers as soon as they exit, effectively making @docker run@ default to @--rm@. If you run arvados-docker-cleaner on a host that does anything other than run crunch-jobs, and you still want to be able to use @docker start@, read the "new doc page":http://doc.arvados.org/install/install-compute-node.html to learn how to turn this off before upgrading.
-h3. 2015-11-04: commit:21006cf adds a keep-web service (see #5824)
+h3. 2015-11-04: "21006cf":https://dev.arvados.org/projects/arvados/repository/revisions/21006cf adds a keep-web service (see "#5824":https://dev.arvados.org/issues/5824)
* Nothing relies on it yet, but early adopters can install it now by following http://doc.arvados.org/install/install-keep-web.html (it is not yet linked in the TOC).
img.full-width {
width: 100%
}
+
+img.screenshot {
+ max-width: calc(100% - 2em);
+ border: 3px;
+ border-style: solid;
+ margin-left: 2em;
+ margin-bottom: 2em;
+}
</code></pre>
</notextile>
+h3(#disable_api_methods). disable_api_methods
+
+Set the @disable_api_methods@ configuration option to disable the deprecated @jobs@ API. This will prevent users from accidentally submitting jobs that won't run. "All new installations should use the containers API.":crunch2-slurm/install-prerequisites.html
+
+<notextile>
+<pre><code>
+ disable_api_methods:
+ - jobs.create
+ - pipeline_instances.create
+ - pipeline_templates.create
+ - jobs.get
+ - pipeline_instances.get
+ - pipeline_templates.get
+ - jobs.list
+ - pipeline_instances.list
+ - pipeline_templates.list
+ - jobs.index
+ - pipeline_instances.index
+ - pipeline_templates.index
+ - jobs.update
+ - pipeline_instances.update
+ - pipeline_templates.update
+ - jobs.queue
+ - jobs.queue_size
+ - job_tasks.create
+ - job_tasks.get
+ - job_tasks.list
+ - job_tasks.index
+ - job_tasks.update
+ - jobs.show
+ - pipeline_instances.show
+ - pipeline_templates.show
+ - job_tasks.show
+</code></pre>
+</notextile>
+
h2(#set_up). Set up Nginx and Passenger
The Nginx server will serve API requests using Passenger. It will also be used to proxy SSL requests to other services which are covered later in this guide.
First, "Install Nginx and Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html.
-Edit the http section of your Nginx configuration to run the Passenger server, and serve SSL requests. Add a block like the following, adding SSL and logging parameters to taste:
+Edit the http section of your Nginx configuration to run the Passenger server. Add a block like the following, adding SSL and logging parameters to taste:
<notextile>
-<pre><code>server {
+<pre><code>
+server {
listen 127.0.0.1:8000;
server_name localhost-api;
default 1;
<span class="userinput">10.20.30.0/24</span> 0;
}
-
-server {
- listen <span class="userinput">[your public IP address]</span>:443 ssl;
- server_name <span class="userinput">uuid_prefix.your.domain</span>;
-
- ssl on;
- ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
- ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
- index index.html index.htm index.php;
-
- # Refer to the comment about this setting in the server section above.
- client_max_body_size 128m;
-
- location / {
- proxy_pass http://api;
- proxy_redirect off;
- proxy_connect_timeout 90s;
- proxy_read_timeout 300s;
-
- proxy_set_header X-Forwarded-Proto https;
- proxy_set_header Host $http_host;
- proxy_set_header X-External-Client $external_client;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- }
-}
</code></pre>
</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install the controller
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The arvados-controller service must be installed on your API server node.
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install arvados-controller</span>
+</code></pre>
+</notextile>
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install arvados-controller</span>
+</code></pre>
+</notextile>
+
+Verify the @arvados-controller@ program is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-controller -h</span>
+Usage:
+ -config file
+[...]
+</code></pre>
+</notextile>
+
+h3. Configure Nginx to route requests to the controller
+
+Add @upstream@ and @server@ definitions inside the @http@ section of your Nginx configuration using the following template.
+
+{% include 'notebox_begin' %}
+
+If you are adding arvados-controller to an existing system as part of the upgrade procedure, do not add a new "server" part here. Instead, add only the "upstream" part as shown here, and update your existing "server" section by changing its @proxy_pass@ directive from @http://api@ to @http://controller@.
+
+{% include 'notebox_end' %}
+
+<notextile>
+<pre><code>upstream controller {
+ server 127.0.0.1:9004 fail_timeout=10s;
+}
+
+server {
+ listen <span class="userinput">[your public IP address]</span>:443 ssl;
+ server_name <span class="userinput">uuid_prefix.your.domain</span>;
+
+ ssl on;
+ ssl_certificate <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+ ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+
+ # Refer to the comment about this setting in the passenger (arvados
+ # api server) section of your Nginx configuration.
+ client_max_body_size 128m;
+
+ location / {
+ proxy_pass http://controller;
+ proxy_redirect off;
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-External-Client $external_client;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+}
+</code></pre>
+</notextile>
+
+Restart Nginx to apply the new configuration.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo nginx -s reload</span>
+</code></pre>
+</notextile>
+
+h3. Configure arvados-controller
+
+Create the cluster configuration file @/etc/arvados/config.yml@ using the following template.
+
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ NodeProfiles:
+ apiserver:
+ arvados-controller:
+ Listen: ":<span class="userinput">9004</span>" # must match the "upstream controller" section of your Nginx config
+ arvados-api-server:
+ Listen: ":<span class="userinput">8000</span>" # must match the "upstream api" section of your Nginx config
+ PostgreSQL:
+ ConnectionPool: 128
+ Connection:
+ host: localhost
+ dbname: arvados_production
+ user: arvados
+ password: <span class="userinput">xxxxxxxx</span>
+ sslmode: require
+</code></pre>
+</notextile>
+
+Create the host configuration file @/etc/arvados/environment@.
+
+<notextile>
+<pre><code>ARVADOS_NODE_PROFILE=apiserver
+</code></pre>
+</notextile>
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the arvados-controller service should already be set up. Restart it to load the new configuration file, and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-controller</span>
+~$ <span class="userinput">sudo systemctl status arvados-controller</span>
+● arvados-controller.service - Arvados controller
+ Loaded: loaded (/lib/systemd/system/arvados-controller.service; enabled; vendor preset: enabled)
+ Active: active (running) since Tue 2018-07-31 13:17:44 UTC; 3s ago
+ Docs: https://doc.arvados.org/
+ Main PID: 25066 (arvados-control)
+ CGroup: /system.slice/arvados-controller.service
+ └─25066 /usr/bin/arvados-controller
+
+Jul 31 13:17:44 zzzzz systemd[1]: Starting Arvados controller...
+Jul 31 13:17:44 zzzzz arvados-controller[25191]: {"Listen":"[::]:9004","Service":"arvados-controller","level":"info","msg":"listening","time":"2018-07-31T13:17:44.521694195Z"}
+Jul 31 13:17:44 zzzzz systemd[1]: Started Arvados controller.
+</code></pre>
+</notextile>
+
+Skip ahead to "confirm the service is working":#confirm.
+
+h3(#runit). Start the service (option 2: runit)
+
+Install runit to supervise the arvados-controller daemon. {% include 'install_runit' %}
+
+Create a supervised service.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo mkdir /etc/service/arvados-controller</span>
+~$ <span class="userinput">cd /etc/service/arvados-controller</span>
+~$ <span class="userinput">sudo mkdir log log/main</span>
+~$ <span class="userinput">printf '#!/bin/sh\nset -a\n. /etc/arvados/environment\nexec arvados-controller 2>&1\n' | sudo tee run</span>
+~$ <span class="userinput">printf '#!/bin/sh\nexec svlogd main\n' | sudo tee log/run</span>
+~$ <span class="userinput">sudo chmod +x run log/run</span>
+~$ <span class="userinput">sudo sv exit .</span>
+~$ <span class="userinput">cd -</span>
+</code></pre>
+</notextile>
+
+Use @sv stat@ and check the log file to verify the service is running.
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo sv stat /etc/service/arvados-controller</span>
+run: /etc/service/arvados-controller: (pid 12520) 2s; run: log: (pid 12519) 2s
+~$ <span class="userinput">tail /etc/service/arvados-controller/log/main/current</span>
+{"Listen":"[::]:9004","Service":"arvados-controller","level":"info","msg":"listening","time":"2018-07-31T13:17:44.521694195Z"}
+</code></pre>
+</notextile>
+
+h3(#confirm). Confirm the service is working
+
+Confirm the service is listening on its assigned port and responding to requests.
+
+<notextile>
+<pre><code>~$ <span class="userinput">curl -X OPTIONS http://0.0.0.0:<b>9004</b>/login</span>
+{"errors":["Forbidden"],"error_token":"1533044555+684b532c"}
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Create a Workflow with Composer
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The Arvados Workflow Composer is a graphical interface for building Common Workflow Language (CWL) workflows to run on Arvados.
+
+This tutorial will demonstrate:
+
+# Creating a new git repository through Arvados to store the workflow
+# Creating CommandLineTools for "sort" and "uniq"
+# Creating a Workflow which uses "sort" and "uniq" to remove duplicate lines from a text file
+# Submitting the Workflow to run on Arvados
+
+h3. 1. Access from workbench
+
+!(screenshot)c1.png!
+
+h3. 2. Composer starting page
+
+!(screenshot)c2.png!
+
+h3. 3. Manage git repositories (opens Workbench in new tab)
+
+!(screenshot)c2b.png!
+
+h3. 4. Add a new repository
+
+!(screenshot)c4.png!
+
+!(screenshot)c3.png!
+
+h3. 5. Return to Composer. Use refresh button to discover new repository (may take a few moments to show up).
+
+!(screenshot)c2c.png!
+
+h3. 6. Create a new Command Line Tool
+
+!(screenshot)c5.png!
+
+!(screenshot)c20.png!
+
+h3. 7. Set Docker image, base command, and input port for "sort" tool
+
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:8@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+
+!(screenshot)c6.png!
+
+h3. 8. Redirect stdout to a file
+
+!(screenshot)c7.png!
+
+h3. 9. Capture output file
+
+!(screenshot)c8.png!
+
+h3. 10. Save Command Line Tool
+
+!(screenshot)c22.png!
+
+h3. 11. Repeat steps 6-10 for "uniq" tool
+
+Create a new tool with a "base command" of "uniq".
+
+h3. 12. Switch back to "Home" tab and create workflow
+
+!(screenshot)c24.png!
+
+!(screenshot)c9.png!
+
+!(screenshot)c10.png!
+
+h3. 13. Drag and drop tools into Workflow
+
+!(screenshot)c11.png!
+
+h3. 14. Drag from input port of "sort" to empty space to create workflow input
+
+!(screenshot)c21.png!
+
+h3. 15. Drag from output port of "sort" to input port of "uniq"
+
+!(screenshot)c13.png!
+
+h3. 16. Drag from output port of "uniq" to empty space to create workflow output
+
+!(screenshot)c14.png!
+
+h3. 17. Save Workflow
+
+!(screenshot)c23.png!
+
+h3. 18. Click on "Test" tab then click "Run"
+
+!(screenshot)c15.png!
+
+h3. 19. Choose input file
+
+You may need to "upload an input file":{{site.baseurl}}/user/tutorials/tutorial-keep.html
+
+!(screenshot)c16.png!
+
+h3. 20. Run the workflow
+
+!(screenshot)c17.png!
+
+h3. 21. Monitor progress (may take several minutes)
+
+!(screenshot)c18.png!
+
+h3. 22. Get workflow output
+
+!(screenshot)c19.png!
run: tool3.cwl
</pre>
-* When migrating from crunch v1 API (--api=jobs) to the crunch v2 API (--api=containers) there are a few differences in behavior:
+h2(#migrate). Migrating running CWL on jobs API to containers API
+
+* When migrating from jobs API (--api=jobs) (sometimes referred to as "crunch v1") to the containers API (--api=containers) ("crunch v2") there are a few differences in behavior:
** The tool is limited to accessing only collections which are explicitly listed in the input, and further limited to only the subdirectories of collections listed in input. For example, given an explicit file input @/dir/subdir/file1.txt@, a tool will not be able to implicitly access the file @/dir/file2.txt@. Use @secondaryFiles@ or a @Directory@ input to describe trees of files.
** Files listed in @InitialWorkDirRequirement@ appear in the output directory as normal files (not symlinks) but cannot be moved, renamed or deleted. These files will be added to the output collection but without any additional copies of the underlying data.
** Tools are disallowed network access by default. Tools which require network access must include @arv:APIRequirement: {}@ in their @requirements@ section.
h2. Upload your image
-Finally, we are ready to upload the new Docker image to Arvados. Use @arv keep docker@ with the image repository name to upload the image. Without arguments, @arv keep docker@ will print out the list of Docker images in Arvados that are available to you.
+Finally, we are ready to upload the new Docker image to Arvados. Use @arv-keepdocker@ with the image repository name to upload the image. Without arguments, @arv-keepdocker@ will print out the list of Docker images in Arvados that are available to you.
<notextile>
-<pre><code>$ <span class="userinput">arv keep docker arvados/jobs-with-r</span>
+<pre><code>$ <span class="userinput">arv-keepdocker arvados/jobs-with-r</span>
703M / 703M 100.0%
Collection saved as 'Docker image arvados/jobs-with-r:latest 2818853ff9f9'
qr1hi-4zz18-abcdefghijklmno
-$ <span class="userinput">arv keep docker</span>
+$ <span class="userinput">arv-keepdocker</span>
REPOSITORY TAG IMAGE ID COLLECTION CREATED
arvados/jobs-with-r latest 2818853ff9f9 qr1hi-4zz18-abcdefghijklmno Tue Jan 17 20:35:53 2017
</code></pre>
h2. Share Docker images
-Docker images are subject to normal Arvados permissions. If wish to share your Docker image with others (or wish to share a pipeline template that uses your Docker image) you will need to use @arv keep docker@ with the @--project-uuid@ option to upload the image to a shared project.
+Docker images are subject to normal Arvados permissions. If wish to share your Docker image with others (or wish to share a pipeline template that uses your Docker image) you will need to use @arv-keepdocker@ with the @--project-uuid@ option to upload the image to a shared project.
<notextile>
-<pre><code>$ <span class="userinput">arv keep docker --project-uuid qr1hi-j7d0g-xxxxxxxxxxxxxxx arvados/jobs-with-r</span>
+<pre><code>$ <span class="userinput">arv-keepdocker --project-uuid qr1hi-j7d0g-xxxxxxxxxxxxxxx arvados/jobs-with-r</span>
</code></pre>
</notextile>
h3. Steps
+notextile. <div class="spaced-out">
+
# Start from the *Workbench Dashboard*. You can access the Dashboard by clicking on *<i class="fa fa-lg fa-fw fa-dashboard"></i> Dashboard* in the upper left corner of any Workbench page.
# Click on the <span class="btn btn-sm btn-primary"><i class="fa fa-fw fa-gear"></i> Run a process...</span> button. This will open a dialog box titled *Choose a pipeline or workflow to run*.
# In the search box, type in *Tutorial bwa mem cwl*.
# After the process starts running, you can track the progress by watching log messages from the component(s). This page refreshes automatically. You will see a <span class="label label-success">complete</span> label when the process completes successfully.
# Click on the *Output* link to see the results of the process. This will load a new page listing the output files from this process. You'll see the output SAM file from the alignment tool under the *Files* tab.
# Click on the <span class="btn btn-sm btn-info"><i class="fa fa-download"></i></span> download button to the right of the SAM file to download your results.
+
+notextile. </div>
h2. Developing workflows
-For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":http://commonwl.org/v1.0/UserGuide.html and the "CWL Specification":http://commonwl.org/v1.0 .
+For an introduction and and detailed documentation about writing CWL, see the "CWL User Guide":https://www.commonwl.org/user_guide and the "CWL Specification":http://commonwl.org/v1.0 .
See "Best Practices for writing CWL":{{site.baseurl}}/user/cwl/cwl-style.html and "Arvados CWL Extensions":{{site.baseurl}}/user/cwl/cwl-extensions.html for additional information about using CWL on Arvados.
+h2. Using Composer
+
+You can create new workflows in the browser using "Arvados Composer":{{site.baseurl}}/user/composer/composer.html
+
h2. Registering a workflow to use in Workbench
Use @--create-workflow@ to register a CWL workflow with Arvados. This enables you to share workflows with other Arvados users, and run them by clicking the <span class="btn btn-sm btn-primary"><i class="fa fa-fw fa-gear"></i> Run a process...</span> button on the Workbench Dashboard and on the command line by UUID.
s.executables << "arv-crunch-job"
s.executables << "arv-tag"
s.required_ruby_version = '>= 2.1.0'
- s.add_runtime_dependency 'arvados', '~> 0.1', '>= 0.1.20150128223554'
+ s.add_runtime_dependency 'arvados', '~> 1.1.0', '>= 1.1.4'
# Our google-api-client dependency used to be < 0.9, but that could be
# satisfied by the buggy 0.9.pre*. https://dev.arvados.org/issues/9213
s.add_runtime_dependency 'google-api-client', '~> 0.6', '>= 0.6.3', '<0.8.9'
else:
raise Exception("Unsupported API '%s', expected one of %s" % (work_api, expected_api))
+ if self.work_api == "jobs":
+ logger.warn("""
+*******************************
+Using the deprecated 'jobs' API.
+
+To get rid of this warning:
+
+Users: read about migrating at
+http://doc.arvados.org/user/cwl/cwl-style.html#migrate
+and use the option --api=containers
+
+Admins: configure the cluster to disable the 'jobs' API as described at:
+http://doc.arvados.org/install/install-api-server.html#disable_api_methods
+*******************************""")
+
self.loadingContext = ArvLoadingContext(vars(arvargs))
self.loadingContext.fetcher_constructor = self.fetcher_constructor
self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
offset := 0
for {
params["offset"] = offset
+
+ // This list variable must be a new one declared
+ // inside the loop: otherwise, items in the API
+ // response would get deep-merged into the items
+ // loaded in previous iterations.
var list arvados.ContainerList
+
err := d.Arv.List("containers", params, &list)
if err != nil {
log.Printf("Error getting list of containers: %q", err)
// For the metrics to be accurate, the caller must ensure every
// request passed to the Handler also passes through
// LogRequests(logger, ...), and vice versa.
-func Instrument(logger *logrus.Logger, next http.Handler) Handler {
+//
+// If registry is nil, a new registry is created.
+//
+// If logger is nil, logrus.StandardLogger() is used.
+func Instrument(registry *prometheus.Registry, logger *logrus.Logger, next http.Handler) Handler {
if logger == nil {
logger = logrus.StandardLogger()
}
+ if registry == nil {
+ registry = prometheus.NewRegistry()
+ }
reqDuration := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "request_duration_seconds",
Help: "Summary of request duration.",
Name: "time_to_status_seconds",
Help: "Summary of request TTFB.",
}, []string{"code", "method"})
- registry := prometheus.NewRegistry()
registry.MustRegister(timeToStatus)
registry.MustRegister(reqDuration)
m := &metrics{
# SPDX-License-Identifier: Apache-2.0
require 'google/api_client'
+# Monkeypatch google-api-client gem to avoid sending newline characters
+# on headers to make ruby-2.3.7+ happy.
+# See: https://dev.arvados.org/issues/13920
+Google::APIClient::ENV::OS_VERSION.strip!
+
require 'json'
require 'tempfile'
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"github.com/hashicorp/golang-lru"
+ "github.com/prometheus/client_golang/prometheus"
)
+const metricsUpdateInterval = time.Second / 10
+
type cache struct {
TTL arvados.Duration
UUIDTTL arvados.Duration
MaxPermissionEntries int
MaxUUIDEntries int
+ registry *prometheus.Registry
stats cacheStats
+ metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
permissions *lru.TwoQueueCache
setupOnce sync.Once
}
+// cacheStats is EOL - add new metrics to cacheMetrics instead
type cacheStats struct {
Requests uint64 `json:"Cache.Requests"`
CollectionBytes uint64 `json:"Cache.CollectionBytes"`
APICalls uint64 `json:"Cache.APICalls"`
}
+type cacheMetrics struct {
+ requests prometheus.Counter
+ collectionBytes prometheus.Gauge
+ collectionEntries prometheus.Gauge
+ collectionHits prometheus.Counter
+ pdhHits prometheus.Counter
+ permissionHits prometheus.Counter
+ apiCalls prometheus.Counter
+}
+
+func (m *cacheMetrics) setup(reg *prometheus.Registry) {
+ m.requests = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "requests",
+ Help: "Number of targetID-to-manifest lookups handled.",
+ })
+ reg.MustRegister(m.requests)
+ m.collectionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "hits",
+ Help: "Number of pdh-to-manifest cache hits.",
+ })
+ reg.MustRegister(m.collectionHits)
+ m.pdhHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "pdh_hits",
+ Help: "Number of uuid-to-pdh cache hits.",
+ })
+ reg.MustRegister(m.pdhHits)
+ m.permissionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "permission_hits",
+ Help: "Number of targetID-to-permission cache hits.",
+ })
+ reg.MustRegister(m.permissionHits)
+ m.apiCalls = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "api_calls",
+ Help: "Number of outgoing API calls made by cache.",
+ })
+ reg.MustRegister(m.apiCalls)
+ m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifest_bytes",
+ Help: "Total size of all manifests in cache.",
+ })
+ reg.MustRegister(m.collectionBytes)
+ m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_collectioncache",
+ Name: "cached_manifests",
+ Help: "Number of manifests in cache.",
+ })
+ reg.MustRegister(m.collectionEntries)
+}
+
type cachedPDH struct {
expire time.Time
pdh string
if err != nil {
panic(err)
}
+
+ reg := c.registry
+ if reg == nil {
+ reg = prometheus.NewRegistry()
+ }
+ c.metrics.setup(reg)
+ go func() {
+ for range time.Tick(metricsUpdateInterval) {
+ c.updateGauges()
+ }
+ }()
+}
+
+func (c *cache) updateGauges() {
+ c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
+ c.metrics.collectionEntries.Set(float64(c.collections.Len()))
}
var selectPDH = map[string]interface{}{
c.setupOnce.Do(c.setup)
atomic.AddUint64(&c.stats.Requests, 1)
+ c.metrics.requests.Inc()
permOK := false
permKey := arv.ApiToken + "\000" + targetID
} else {
permOK = true
atomic.AddUint64(&c.stats.PermissionHits, 1)
+ c.metrics.permissionHits.Inc()
}
}
} else {
pdh = ent.pdh
atomic.AddUint64(&c.stats.PDHHits, 1)
+ c.metrics.pdhHits.Inc()
}
}
// _and_ the current token has permission, we can
// use our cached manifest.
atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
var current arvados.Collection
err := arv.Get("collections", targetID, selectPDH, ¤t)
if err != nil {
// Collection manifest is not cached.
atomic.AddUint64(&c.stats.APICalls, 1)
+ c.metrics.apiCalls.Inc()
err := arv.Get("collections", targetID, nil, &collection)
if err != nil {
return nil, err
}
func (c *cache) lookupCollection(key string) *arvados.Collection {
- if ent, cached := c.collections.Get(key); !cached {
+ e, cached := c.collections.Get(key)
+ if !cached {
+ return nil
+ }
+ ent := e.(*cachedCollection)
+ if ent.expire.Before(time.Now()) {
+ c.collections.Remove(key)
return nil
- } else {
- ent := ent.(*cachedCollection)
- if ent.expire.Before(time.Now()) {
- c.collections.Remove(key)
- return nil
- } else {
- atomic.AddUint64(&c.stats.CollectionHits, 1)
- return ent.collection
- }
}
+ atomic.AddUint64(&c.stats.CollectionHits, 1)
+ c.metrics.collectionHits.Inc()
+ return ent.collection
}
"net/http"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
+ "github.com/prometheus/client_golang/prometheus"
)
type server struct {
func (srv *server) Start() error {
h := &handler{Config: srv.Config}
- mh := httpserver.Instrument(nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
+ reg := prometheus.NewRegistry()
+ h.Config.Cache.registry = reg
+ mh := httpserver.Instrument(reg, nil, httpserver.AddRequestIDs(httpserver.LogRequests(nil, h)))
h.MetricsAPI = mh.ServeAPI(http.NotFoundHandler())
srv.Handler = mh
srv.Addr = srv.Config.Listen
resp, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
- req, _ = http.NewRequest("GET", origin+"/foo", nil)
- req.Host = arvadostest.FooCollection + ".example.com"
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
- resp, err = http.DefaultClient.Do(req)
- c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
- buf, _ := ioutil.ReadAll(resp.Body)
- c.Check(buf, check.DeepEquals, []byte("foo"))
+ for i := 0; i < 2; i++ {
+ req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req.Host = arvadostest.FooCollection + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, _ := ioutil.ReadAll(resp.Body)
+ c.Check(buf, check.DeepEquals, []byte("foo"))
+ resp.Body.Close()
+ }
+
+ s.testServer.Config.Cache.updateGauges()
req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
resp, err = http.DefaultClient.Do(req)
Value float64
}
}
+ type counter struct {
+ Value int64
+ }
+ type gauge struct {
+ Value float64
+ }
var ents []struct {
Name string
Help string
Name string
Value string
}
+ Counter counter
+ Gauge gauge
Summary summary
}
}
json.NewDecoder(resp.Body).Decode(&ents)
- flat := map[string]summary{}
+ summaries := map[string]summary{}
+ gauges := map[string]gauge{}
+ counters := map[string]counter{}
for _, e := range ents {
for _, m := range e.Metric {
labels := map[string]string{}
for _, lbl := range m.Label {
labels[lbl.Name] = lbl.Value
}
- flat[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+ gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
}
}
- c.Check(flat["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
- c.Check(flat["request_duration_seconds/get/200"].SampleCount, check.Equals, "2")
- c.Check(flat["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
- c.Check(flat["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+ c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+ c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+ c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+ // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
rtr.limiter = httpserver.NewRequestLimiter(theConfig.MaxRequests, rtr)
- stack := httpserver.Instrument(nil,
+ stack := httpserver.Instrument(nil, nil,
httpserver.AddRequestIDs(httpserver.LogRequests(nil, rtr.limiter)))
return stack.ServeAPI(stack)
}
s.files = ["bin/arvados-login-sync", "agpl-3.0.txt"]
s.executables << "arvados-login-sync"
s.required_ruby_version = '>= 2.1.0'
- s.add_runtime_dependency 'arvados', '~> 0.1', '>= 0.1.20150615153458'
+ s.add_runtime_dependency 'arvados', '~> 1.1.0', '>= 1.1.4'
s.homepage =
'https://arvados.org'
end
projects / arvados.git / commitdiff
